Compare commits

..
15 Commits
Author SHA1 Message Date
ZyyandCodex a709acb149 chore: bump version to 3.3.19
Co-authored-by: Codex <noreply@openai.com>
2026-09-03 17:56:10 +08:00
ZyyandCodex 466842acd8 fix(observability): search usage by API key
Resolve sk_ searches through client key references while only sending the visible key prefix from the frontend, and clarify the request detail search hint.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 17:38:29 +08:00
ZyyandCodex b91f28f95b fix(frontend): align configuration and usage copy
Generate the displayed Codex files and CCSwitch import payload from one source, carry the full settings through deep links, and clarify the quota percentage label.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 16:37:53 +08:00
ZyyandCodex 87164db3da chore: bump version to 3.3.18
Co-authored-by: Codex <noreply@openai.com>
2026-09-03 15:41:50 +08:00
ZyyandCodex ad16dec3db fix(openai): align official client routes
Remove unsupported review and model extension routes, preserve review semantics on standard Responses requests, and refine usage observability for review and empty performance data.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 15:37:14 +08:00
ZyyandCodex a0da0f4713 feat(openai): proxy standalone search
Add /v1/alpha/search as a transparent raw JSON endpoint, route requests through the OpenAI credential pipeline, and preserve account-scoped turn metadata and upstream response headers.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 14:27:15 +08:00
ZyyandCodex 65e5cde60d chore: bump version to 3.3.17
Co-authored-by: Codex <noreply@openai.com>
2026-09-03 12:44:42 +08:00
ZyyandCodex 0f0b464f4e fix(frontend): refine account usage presentation
Select quota summaries by the most recently used model, streamline dashboard account identities, and align diagnostic client metadata with successful records.

Co-authored-by: Codex <noreply@openai.com>
2026-09-03 12:21:15 +08:00
ZyyandCodex bd223f6540 chore: bump version to 3.3.16
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 22:45:09 +08:00
ZyyandCodex 168be0ca68 fix(openai): add continuation recovery observability
Persist stable continuation and WebSocket lifecycle facts, correlate successful client replays, and hide recovered failures from default business metrics.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 21:38:48 +08:00
ZyyandCodex f2e91bbdaa fix(openai): preserve websocket continuation routing
Limit session-sticky HTTP fallback to new chains, keep continuation requests on their safe transport, and classify rebuild-required failures separately from account health.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 18:21:30 +08:00
ZyyandCodex d0c9248478 fix(openai): refine smart scheduler failure scoring
Limit account feedback to an explicit upstream reason allowlist, aggregate same-account retries, and decay failure rates over time.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 15:08:31 +08:00
ZyyandCodex 22c697bbff fix(usage): refine record details and error filters
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 14:43:44 +08:00
ZyyandCodex 08d518324a fix(frontend): refine account quota layout
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 12:14:03 +08:00
ZyyandCodex dfacb20e43 fix(frontend): tighten usage attempt table layout
Co-authored-by: Codex <noreply@openai.com>
2026-09-01 19:46:49 +08:00
121 changed files with 4247 additions and 1155 deletions
+1 -4
View File
@@ -23,7 +23,7 @@
| 领域 | 当前能力 |
| --- | --- |
| 数据面 | Responses JSON、SSE、WebSocket、review 子代理、Images 生成/编辑与模型目录 |
| 数据面 | Responses JSON、SSE、WebSocket(含 Codex 子代理语义)、Images 生成/编辑与模型目录 |
| Provider | 固定 `openai`、`xai` 两个 Provider,各自拥有 OAuth、credential、额度、目录与 transport |
| 路由 | Client Key 限定账号分组;按模型能力编译 Provider 候选;支持全局精确模型映射、会话亲和和安全 fallback |
| 透明边界 | OpenAI 保留未知请求字段及字段顺序,并原样转发 SSE、WebSocket 与 Images 业务字节;xAI 在 Provider 内转换协议 |
@@ -136,12 +136,9 @@ curl http://127.0.0.1:8080/v1/responses \
| --- | --- |
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
| `GET /v1/responses` | Responses WebSocket 升级 |
| `POST /v1/responses/review` | review 子代理请求 |
| `POST /v1/images/generations` | OpenAI Images 生成 JSON 透明转发 |
| `POST /v1/images/edits` | OpenAI Images 编辑 JSON 透明转发 |
| `GET /v1/models` | 当前 Client Key 账号范围内的模型列表 |
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
所有 `/v1/*` 路由都需要 Client Key。Images 是 OpenAI Provider 自有端点,不读取模型字段,也不参与
@@ -135,13 +135,11 @@ pub struct OpsErrorFilter {
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub operation: Option<String>,
pub endpoint: Option<String>,
pub model: Option<String>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub failure_kind: Option<String>,
pub status_code: Option<u16>,
pub search: Option<String>,
}
@@ -793,6 +791,18 @@ pub struct OpsError {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: Option<bool>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
pub occurred_at: DateTime<Utc>,
pub stable_sort_id: String,
}
@@ -578,6 +578,18 @@ pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
component: error.component,
attempt_id: None,
account_label,
continuation_affinity_hash: error.continuation_affinity_hash,
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
continuation_unavailable_reason: error.continuation_unavailable_reason,
upstream_connection_id: error.upstream_connection_id,
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
upstream_connection_age_ms: error.upstream_connection_age_ms,
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
recovery_request_id: error.recovery_request_id,
recovered_at: error.recovered_at,
recovery_attempt_count: error.recovery_attempt_count,
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
recovery_total_latency_ms: error.recovery_total_latency_ms,
},
created_at: error.occurred_at,
created_at_display: china_datetime(&error.occurred_at),
@@ -107,14 +107,12 @@ pub struct OpsQuery {
pub provider: Option<String>,
pub request_id: Option<String>,
pub account_id: Option<String>,
pub route: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub client_status_code: Option<i64>,
pub upstream_status_code: Option<i64>,
pub transport: Option<String>,
pub attempt_index: Option<i64>,
pub failure_class: Option<String>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub search: Option<String>,
@@ -352,13 +350,11 @@ pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, Wir
provider_kind: non_empty(query.provider.clone()),
provider_account_ref: non_empty(query.account_id.clone()),
operation: non_empty(query.kind.clone()),
endpoint: non_empty(query.route.clone()),
model: non_empty(query.model.clone()),
transport: non_empty(query.transport.clone()),
attempt_index: parse_attempt_index(query.attempt_index)?,
response_id: non_empty(query.response_id.clone()),
upstream_request_id: non_empty(query.upstream_request_id.clone()),
failure_kind: non_empty(query.failure_class.clone()),
status_code,
search: non_empty(query.search.clone()),
},
@@ -805,4 +805,16 @@ pub struct OpsErrorMetadataView {
pub component: String,
pub attempt_id: Option<String>,
pub account_label: Option<String>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
}
@@ -3,22 +3,21 @@
use std::net::SocketAddr;
use axum::{
body::{Body, Bytes},
body::Bytes,
extract::{Extension, State, connect_info::ConnectInfo},
http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
response::{IntoResponse, Response},
http::HeaderMap,
response::Response,
};
use gateway_core::engine::execution::StartedExecution;
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
use gateway_core::operation::{ImageRequest, ImageRequestKind, Operation, RawJsonPayload};
use serde_json::{Map, Value};
use crate::ApiState;
use crate::openai::{
auth::{authenticate_client, client_access_error_response},
error::{engine_error_response, gateway_error_response, protocol_error_response},
responses::{PendingExecution, ProtocolError, ProtocolErrorBody, request_client_context},
error::gateway_error_response,
provider_endpoint::collect_raw_json_response,
responses::request_client_context,
};
const OPENAI_PROTOCOL: &str = "openai";
@@ -82,13 +81,13 @@ async fn handle_image_request(
Err(error) => return gateway_error_response(&error),
};
let started = match service
.start_image(client, operation, client_ip, user_agent, endpoint)
.start_provider_endpoint(client, operation, client_ip, user_agent, endpoint)
.await
{
Ok(started) => started,
Err(error) => return gateway_error_response(&error),
};
collect_image_response(started).await
collect_raw_json_response(started).await
}
fn image_operation(
@@ -118,116 +117,3 @@ fn image_operation(
kind, payload,
)))
}
async fn collect_image_response(started: StartedExecution) -> Response {
let mut execution = PendingExecution::new(started.session);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
let events = match session.collect_uncommitted().await {
Ok(events) => events,
Err(error) => {
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
};
let body = match raw_json_body(events) {
Some(body) => body,
None => {
let response = invalid_upstream_response();
let response = execution.record_response_status(response).await;
execution.cancel_and_finalize().await;
return response;
}
};
let status = session
.response_status_code()
.and_then(|status| StatusCode::from_u16(status).ok())
.filter(StatusCode::is_success)
.unwrap_or(StatusCode::OK);
let response_headers = session.response_headers().to_vec();
let response = json_body_response(body, status, &response_headers);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
execution.cancel_and_finalize().await;
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
if !session.is_finalized() {
execution.cancel_and_finalize().await;
return invalid_upstream_response();
}
execution.disarm();
response
}
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
let mut body = None;
for event in events {
let (_, wire) = event.into_parts();
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
continue;
};
let Some(raw) = wire.into_raw_json_body() else {
continue;
};
if body.replace(raw).is_some() {
return None;
}
}
body
}
fn json_body_response(
body: Bytes,
status: StatusCode,
response_headers: &[ProviderResponseHeader],
) -> Response {
let mut response = Response::new(Body::from(body));
*response.status_mut() = status;
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
apply_response_headers(response, response_headers)
}
fn apply_response_headers(
mut response: Response,
response_headers: &[ProviderResponseHeader],
) -> Response {
let connection_options =
crate::openai::responses::response_connection_options(response_headers);
for header in response_headers {
if !crate::openai::responses::response_header_is_forwardable(
header.name(),
&connection_options,
) {
continue;
}
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
continue;
};
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
continue;
};
response.headers_mut().append(name, value);
}
response
}
fn invalid_upstream_response() -> Response {
protocol_error_response(
StatusCode::BAD_GATEWAY,
ProtocolErrorBody {
error: ProtocolError {
kind: "server_error",
code: "invalid_upstream_response",
message: "The gateway could not forward the upstream image response.".to_owned(),
param: None,
},
},
)
.into_response()
}
@@ -4,6 +4,8 @@ pub mod auth;
pub mod error;
pub mod images;
pub mod models;
mod provider_endpoint;
pub mod responses;
pub mod router;
pub mod search;
pub(crate) mod service;
@@ -97,50 +97,6 @@ pub(crate) async fn model_detail(
.into_response()
}
/// `GET /v1/models/catalog`。
///
/// 新架构的目录由 core 按客户端 Provider 范围裁剪;这里不读取或重建任一
/// Provider 的 wire catalog,只恢复 Codex 客户端依赖的稳定展示合同。
pub(crate) async fn model_catalog(State(state): State<ApiState>, headers: HeaderMap) -> Response {
let service = state.openai();
let client = match authenticate_client(service, &headers) {
Ok(client) => client,
Err(error) => return client_access_error_response(error),
};
let models = service
.public_models(&client)
.into_iter()
.map(|model| catalog_model_json(model.as_str()))
.collect::<Vec<_>>();
(StatusCode::OK, Json(models)).into_response()
}
/// `GET /v1/models/{model_id}/info`。
pub(crate) async fn model_info(
State(state): State<ApiState>,
headers: HeaderMap,
Path(model_id): Path<String>,
) -> Response {
let service = state.openai();
let client = match authenticate_client(service, &headers) {
Ok(client) => client,
Err(error) => return client_access_error_response(error),
};
let Ok(public_model) = PublicModelId::new(model_id) else {
return model_not_found_response().into_response();
};
if !service.contains_public_model(&client, &public_model) {
return model_not_found_response().into_response();
}
(
StatusCode::OK,
Json(catalog_model_json(public_model.as_str())),
)
.into_response()
}
fn openai_model_json(id: &str) -> Value {
json!({
"id": id,
@@ -150,22 +106,6 @@ fn openai_model_json(id: &str) -> Value {
})
}
fn catalog_model_json(id: &str) -> Value {
json!({
"id": id,
"displayName": id,
"description": "",
"isDefault": false,
"supportedReasoningEfforts": [],
"defaultReasoningEffort": "",
"inputModalities": ["text"],
"outputModalities": ["text"],
"supportsPersonality": false,
"upgrade": Value::Null,
"source": "gateway",
})
}
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
let id = profile.model().as_str();
let presentation = profile.presentation();
@@ -0,0 +1,124 @@
//! OpenAI Provider 原生非流式 JSON 端点的公共交付边界。
use axum::{
body::{Body, Bytes},
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
response::{IntoResponse, Response},
};
use gateway_core::engine::execution::StartedExecution;
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
use super::{
error::{engine_error_response, protocol_error_response},
responses::{PendingExecution, ProtocolError, ProtocolErrorBody},
};
/// 收集一个 Provider 原生 JSON 响应,并在 Core commit 后按原始 bytes 交付。
pub(super) async fn collect_raw_json_response(started: StartedExecution) -> Response {
let mut execution = PendingExecution::new(started.session);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
let events = match session.collect_uncommitted().await {
Ok(events) => events,
Err(error) => {
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
};
let body = match raw_json_body(events) {
Some(body) => body,
None => {
let response = invalid_upstream_response();
let response = execution.record_response_status(response).await;
execution.cancel_and_finalize().await;
return response;
}
};
let status = session
.response_status_code()
.and_then(|status| StatusCode::from_u16(status).ok())
.filter(StatusCode::is_success)
.unwrap_or(StatusCode::OK);
let response_headers = session.response_headers().to_vec();
let response = json_body_response(body, status, &response_headers);
let Some(session) = execution.session_mut() else {
return invalid_upstream_response();
};
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
execution.cancel_and_finalize().await;
let response = engine_error_response(&error);
return execution.record_response_status(response).await;
}
if !session.is_finalized() {
execution.cancel_and_finalize().await;
return invalid_upstream_response();
}
execution.disarm();
response
}
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
let mut body = None;
for event in events {
let (_, wire) = event.into_parts();
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
continue;
};
let Some(raw) = wire.into_raw_json_body() else {
continue;
};
if body.replace(raw).is_some() {
return None;
}
}
body
}
fn json_body_response(
body: Bytes,
status: StatusCode,
response_headers: &[ProviderResponseHeader],
) -> Response {
let mut response = Response::new(Body::from(body));
*response.status_mut() = status;
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
apply_response_headers(response, response_headers)
}
fn apply_response_headers(
mut response: Response,
response_headers: &[ProviderResponseHeader],
) -> Response {
let connection_options = super::responses::response_connection_options(response_headers);
for header in response_headers {
if !super::responses::response_header_is_forwardable(header.name(), &connection_options) {
continue;
}
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
continue;
};
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
continue;
};
response.headers_mut().append(name, value);
}
response
}
fn invalid_upstream_response() -> Response {
protocol_error_response(
StatusCode::BAD_GATEWAY,
ProtocolErrorBody {
error: ProtocolError {
kind: "server_error",
code: "invalid_upstream_response",
message: "The gateway could not forward the upstream JSON response.".to_owned(),
param: None,
},
},
)
.into_response()
}
@@ -32,7 +32,7 @@ use crate::openai::{
use super::{
OpenAiResponsesEncoder, ProtocolErrorBody, ResponseEncodeError,
request::{decode_request_with_headers, decode_review_request_with_headers},
request::decode_request_with_headers,
};
/// `POST /v1/responses`。
@@ -41,37 +41,13 @@ pub(crate) async fn responses(
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
handle_responses(state, connect_info, headers, body, false).await
}
/// `POST /v1/responses/review`。
pub(crate) async fn review_responses(
State(state): State<ApiState>,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
handle_responses(state, connect_info, headers, body, true).await
}
async fn handle_responses(
state: ApiState,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
review: bool,
) -> Response {
let service = state.openai();
let client = match authenticate_client(service, &headers) {
Ok(client) => client,
Err(error) => return client_access_error_response(error),
};
let decoded = match if review {
decode_review_request_with_headers(&body, &headers)
} else {
decode_request_with_headers(&body, &headers)
} {
let decoded = match decode_request_with_headers(&body, &headers) {
Ok(decoded) => decoded,
Err(error) => {
return protocol_error_response(StatusCode::BAD_REQUEST, error.protocol_body());
@@ -100,11 +76,7 @@ async fn handle_responses(
} else {
ClientTransport::HttpJson
},
if review {
"/v1/responses/review"
} else {
"/v1/responses"
},
"/v1/responses",
)
.await
{
@@ -7,9 +7,9 @@ mod response;
pub mod websocket;
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
pub(crate) use http::responses;
pub(super) use http::{PendingExecution, request_client_context};
pub use http::{collect_execution_response, stream_execution_response};
pub(crate) use http::{responses, review_responses};
pub use request::{
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, ResponsesRequestMetadata,
decode_request_with_headers,
@@ -13,7 +13,6 @@ use serde_json::{Map, Value};
use super::error::RequestDecodeError;
const OPENAI_PROTOCOL: &str = "openai";
const REVIEW_SUBAGENT: &str = "review";
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
@@ -75,8 +74,8 @@ impl OpenAiRequestHeaders {
}
}
fn apply_subagent(&self, body: &mut Map<String, Value>, forced_subagent: Option<&str>) {
if let Some(subagent) = forced_subagent.or(self.subagent.as_deref()) {
fn apply_subagent(&self, body: &mut Map<String, Value>) {
if let Some(subagent) = self.subagent.as_deref() {
inject_subagent_metadata(body, subagent);
}
}
@@ -301,20 +300,11 @@ pub fn decode_request_with_headers(
body: &[u8],
headers: &HeaderMap,
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
decode_request_inner(body, false, &OpenAiRequestHeaders::from_headers(headers))
}
/// 解码 `POST /v1/responses/review` 请求并冻结 review subagent 语义。
pub(super) fn decode_review_request_with_headers(
body: &[u8],
headers: &HeaderMap,
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
decode_request_inner(body, true, &OpenAiRequestHeaders::from_headers(headers))
decode_request_inner(body, &OpenAiRequestHeaders::from_headers(headers))
}
pub(super) fn decode_request_inner(
body: &[u8],
review: bool,
request_headers: &OpenAiRequestHeaders,
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
let value =
@@ -322,13 +312,12 @@ pub(super) fn decode_request_inner(
let Value::Object(object) = value else {
return Err(RequestDecodeError::ExpectedObject);
};
decode_request_object(object, review, request_headers, RequestDecodeSource::Http)
decode_request_object(object, request_headers, RequestDecodeSource::Http)
}
/// 解码已解析的顶层 object;按下游传输来源恢复连接级协议上下文。
pub(super) fn decode_request_object(
mut object: Map<String, Value>,
review: bool,
request_headers: &OpenAiRequestHeaders,
source: RequestDecodeSource,
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
@@ -359,7 +348,7 @@ pub(super) fn decode_request_object(
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
.unwrap_or(ContinuationIntent::None);
request_headers.apply_subagent(&mut object, review.then_some(REVIEW_SUBAGENT));
request_headers.apply_subagent(&mut object);
let frame_turn_metadata = match source {
RequestDecodeSource::Http => None,
RequestDecodeSource::WebSocketFrame => frame_turn_metadata(&object),
@@ -43,7 +43,6 @@ fn decode_response_create_inner(
}
super::super::request::decode_request_object(
body,
false,
request_headers,
RequestDecodeSource::WebSocketFrame,
)
@@ -8,8 +8,9 @@ use axum::{
use super::{
images::{image_edits, image_generations},
models::{model_catalog, model_detail, model_info, models},
responses::{responses, responses_websocket, review_responses},
models::{model_detail, models},
responses::{responses, responses_websocket},
search::standalone_search,
};
use crate::ApiState;
@@ -19,11 +20,9 @@ pub(crate) fn router() -> Router<ApiState> {
Router::new()
.route("/v1/images/generations", post(image_generations))
.route("/v1/images/edits", post(image_edits))
.route("/v1/alpha/search", post(standalone_search))
.route("/v1/responses", get(responses_websocket).post(responses))
.route("/v1/responses/review", post(review_responses))
.route("/v1/models", get(models))
.route("/v1/models/catalog", get(model_catalog))
.route("/v1/models/{model_id}/info", get(model_info))
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
.route("/v1/models/{model_id}", get(model_detail))
// OpenAI 数据面正文属于客户端/上游协议;代理不能用私有大小上限提前拒绝
@@ -0,0 +1,78 @@
//! Codex standalone search 非流式 HTTP adapter。
use std::net::SocketAddr;
use axum::{
body::Bytes,
extract::{Extension, State, connect_info::ConnectInfo},
http::HeaderMap,
response::Response,
};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::operation::{Operation, RawJsonPayload, StandaloneSearchRequest};
use serde_json::{Map, Value};
use crate::ApiState;
use crate::openai::{
auth::{authenticate_client, client_access_error_response},
error::gateway_error_response,
provider_endpoint::collect_raw_json_response,
responses::request_client_context,
};
const OPENAI_PROTOCOL: &str = "openai";
const TURN_METADATA_CONTEXT_KEY: &str = "turn_metadata";
/// `POST /v1/alpha/search`。
pub(crate) async fn standalone_search(
State(state): State<ApiState>,
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
headers: HeaderMap,
body: Bytes,
) -> Response {
let service = state.openai();
let client = match authenticate_client(service, &headers) {
Ok(client) => client,
Err(error) => return client_access_error_response(error),
};
let (client_ip, user_agent) = request_client_context(
&headers,
connect_info.map(|Extension(ConnectInfo(address))| address),
);
let operation = match search_operation(body, &headers) {
Ok(operation) => operation,
Err(error) => return gateway_error_response(&error),
};
let started = match service
.start_provider_endpoint(client, operation, client_ip, user_agent, "/v1/alpha/search")
.await
{
Ok(started) => started,
Err(error) => return gateway_error_response(&error),
};
collect_raw_json_response(started).await
}
fn search_operation(body: Bytes, headers: &HeaderMap) -> Result<Operation, GatewayError> {
let mut context = Map::new();
if let Some(turn_metadata) = headers
.get("x-codex-turn-metadata")
.and_then(|value| value.to_str().ok())
{
context.insert(
TURN_METADATA_CONTEXT_KEY.to_owned(),
Value::String(turn_metadata.to_owned()),
);
}
let payload = RawJsonPayload::new(OPENAI_PROTOCOL, body)
.map_err(|_| {
GatewayError::new(
GatewayErrorKind::Internal,
"OpenAI protocol identifier is invalid",
)
})?
.with_context(context);
Ok(Operation::Search(StandaloneSearchRequest::from_raw_json(
payload,
)))
}
@@ -0,0 +1,5 @@
//! Codex standalone search HTTP adapter。
mod http;
pub(crate) use http::standalone_search;
@@ -106,7 +106,7 @@ impl OpenAiService {
.await
}
pub(crate) async fn start_image(
pub(crate) async fn start_provider_endpoint(
&self,
client: AuthenticatedClient,
operation: gateway_core::operation::Operation,
@@ -49,6 +49,12 @@ fn ops_query_should_reject_page_size_above_terminal_limit() {
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
}
#[test]
fn ops_query_should_reject_removed_filters() {
assert!(serde_json::from_value::<OpsQuery>(json!({"route": "/v1/responses"})).is_err());
assert!(serde_json::from_value::<OpsQuery>(json!({"failureClass": "rate_limited"})).is_err());
}
#[test]
fn diagnostics_query_should_keep_wire_dimension_name() {
let query: DiagnosticsQuery =
@@ -571,6 +571,18 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
request_kind: Some("root".to_owned()),
subagent_kind: None,
compact: Some(false),
continuation_affinity_hash: Some("affinity-hash".to_owned()),
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
upstream_connection_id: Some("connection-id".to_owned()),
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
upstream_connection_age_ms: Some(12_000),
upstream_connection_idle_ms: Some(4_000),
recovery_request_id: None,
recovered_at: None,
recovery_attempt_count: 0,
recovery_retry_delay_ms: None,
recovery_total_latency_ms: None,
occurred_at: Utc::now(),
stable_sort_id: "model_request:req_err".to_owned(),
});
@@ -607,6 +619,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
"rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"],
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
}),
serde_json::json!({
"provider": "openai",
@@ -623,6 +638,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"providerErrorCode": "upstream",
"upstreamSendState": "sent",
"rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
"continuationUnavailableReason": "reused_connection_lost",
"upstreamConnectionExitReason": "tcp_reset",
"upstreamConnectionAgeMs": 12000,
})
);
}
@@ -61,6 +61,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
"src/openai/images/mod.rs",
"src/openai/mod.rs",
"src/openai/models.rs",
"src/openai/provider_endpoint.rs",
"src/openai/responses/error.rs",
"src/openai/responses/http.rs",
"src/openai/responses/mod.rs",
@@ -71,6 +72,8 @@ fn source_tree_should_match_frozen_machine_manifest() {
"src/openai/responses/websocket/mod.rs",
"src/openai/responses/websocket/protocol.rs",
"src/openai/router.rs",
"src/openai/search/http.rs",
"src/openai/search/mod.rs",
"src/openai/service.rs",
];
expected.sort_unstable();
@@ -104,12 +107,14 @@ fn test_tree_should_match_frozen_rust_mirror() {
"tests/openai/images/mod.rs",
"tests/openai/mod.rs",
"tests/openai/models.rs",
"tests/openai/provider_endpoint.rs",
"tests/openai/responses/http.rs",
"tests/openai/responses/mod.rs",
"tests/openai/responses/websocket/connection.rs",
"tests/openai/responses/websocket/mod.rs",
"tests/openai/responses/websocket/protocol.rs",
"tests/openai/router.rs",
"tests/openai/search/mod.rs",
];
expected.sort_unstable();
@@ -398,3 +398,38 @@ async fn gateway_error_response_should_expose_only_structured_client_visible_ups
})
);
}
#[tokio::test]
async fn continuation_recovery_should_preserve_the_official_retry_signal() {
let error = GatewayError::from_provider(
&ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"Previous response was not found. Retrying the full request.",
Some("previous_response_not_found".to_owned()),
Some("invalid_request_error".to_owned()),
)),
);
let response = gateway_error_response(&error);
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read continuation recovery response");
let body: serde_json::Value =
serde_json::from_slice(&body).expect("continuation recovery JSON");
assert_eq!(status, StatusCode::BAD_REQUEST);
assert_eq!(
body,
serde_json::json!({
"error": {
"message": "Previous response was not found. Retrying the full request.",
"type": "invalid_request_error",
"code": "previous_response_not_found"
}
})
);
}
@@ -2,8 +2,10 @@ mod auth;
mod error;
mod images;
mod models;
mod provider_endpoint;
mod responses;
mod router;
mod search;
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
@@ -383,46 +383,6 @@ async fn model_detail_should_hide_unknown_models() {
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn model_catalog_should_keep_the_codex_catalog_contract() {
let response = api_router(ModelsExecution::new())
.await
.oneshot(authorized_request("/v1/models/catalog"))
.await
.expect("catalog response");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read catalog body");
let value: serde_json::Value = serde_json::from_slice(&body).expect("catalog JSON");
assert_eq!(value[0]["id"], "model-a");
assert_eq!(value[0]["displayName"], "model-a");
assert_eq!(value[0]["source"], "gateway");
}
#[tokio::test]
async fn model_info_should_return_a_single_visible_catalog_entry() {
let response = api_router(ModelsExecution::new())
.await
.oneshot(authorized_request("/v1/models/model-a/info"))
.await
.expect("model info response");
assert_eq!(response.status(), StatusCode::OK);
}
#[tokio::test]
async fn model_info_should_hide_unknown_models() {
let response = api_router(ModelsExecution::new())
.await
.oneshot(authorized_request("/v1/models/model-private/info"))
.await
.expect("missing model info response");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn models_should_authenticate_before_applying_the_version_gate() {
let response = api_router(ModelsExecution::with_cli_min())
@@ -0,0 +1,107 @@
use std::sync::{Arc, Mutex, atomic::AtomicBool, atomic::Ordering};
use bytes::Bytes;
use futures::future::BoxFuture;
use gateway_core::engine::execution::ExecutionSession;
use gateway_core::engine::{CoordinatedEvent, EngineError};
use gateway_core::error::ProviderError;
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
pub(super) struct BufferedJsonSession {
body: Option<Bytes>,
response_headers: Vec<ProviderResponseHeader>,
response_status: Option<u16>,
committed_statuses: Arc<Mutex<Vec<u16>>>,
finalized: AtomicBool,
failure: Option<ProviderError>,
}
impl BufferedJsonSession {
pub(super) fn success(
body: Bytes,
response_status: u16,
response_headers: Vec<ProviderResponseHeader>,
committed_statuses: Arc<Mutex<Vec<u16>>>,
) -> Self {
Self {
body: Some(body),
response_headers,
response_status: Some(response_status),
committed_statuses,
finalized: AtomicBool::new(false),
failure: None,
}
}
pub(super) fn failure(error: ProviderError) -> Self {
Self {
body: None,
response_headers: Vec::new(),
response_status: None,
committed_statuses: Arc::new(Mutex::new(Vec::new())),
finalized: AtomicBool::new(false),
failure: Some(error),
}
}
}
impl ExecutionSession for BufferedJsonSession {
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
Box::pin(async { unreachable!("buffered JSON delivery does not stream") })
}
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
Box::pin(async move {
if let Some(error) = self.failure.take() {
return Err(EngineError::Provider(error));
}
Ok(vec![ProviderEvent::wire(
ProtocolWireEvent::raw_json(
"openai",
self.body.take().expect("single buffered JSON response"),
)
.expect("OpenAI protocol"),
)])
})
}
fn response_headers(&self) -> &[ProviderResponseHeader] {
&self.response_headers
}
fn response_status_code(&self) -> Option<u16> {
self.response_status
}
fn commit_downstream(
&mut self,
client_status_code: Option<u16>,
) -> BoxFuture<'_, Result<(), EngineError>> {
Box::pin(async move {
self.committed_statuses
.lock()
.expect("committed status lock")
.push(client_status_code.expect("buffered JSON response status"));
self.finalized.store(true, Ordering::Release);
Ok(())
})
}
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
Box::pin(async { Ok(()) })
}
fn is_finalized(&self) -> bool {
self.finalized.load(Ordering::Acquire)
}
fn cancel(&self) {
self.finalized.store(true, Ordering::Release);
}
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
Box::pin(async move {
self.finalized.store(true, Ordering::Release);
})
}
}
@@ -36,7 +36,7 @@ use serde_json::{Value, json};
use gateway_api::openai::responses::{collect_execution_response, stream_execution_response};
use tower::ServiceExt;
use crate::openai::{api_router, authenticated_client, authenticated_client_for_provider};
use crate::openai::{api_router, authenticated_client_for_provider};
#[derive(Default)]
struct Trace {
@@ -689,92 +689,6 @@ async fn xai_private_headers_should_not_enter_openai_request_facts() {
assert!(captured.prompt_cache_key.is_none());
}
#[tokio::test]
async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
let observed = Arc::new(Mutex::new(None));
let execution = Arc::new(ContextCaptureExecution {
observed: Arc::clone(&observed),
client: authenticated_client("sk_context_test"),
});
let request = Request::post("/v1/responses/review")
.header(AUTHORIZATION, "Bearer sk_context_test")
.header("x-openai-subagent", "compact")
.body(Body::from(
json!({
"model": "smart-code",
"input": "review this",
"client_metadata": {"existing": "preserved"}
})
.to_string(),
))
.expect("review request");
let response = api_router(execution)
.await
.oneshot(request)
.await
.expect("review response");
let captured = observed
.lock()
.expect("review capture lock")
.clone()
.expect("captured review request");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(captured.endpoint, "/v1/responses/review");
assert_eq!(
captured
.client_metadata
.as_ref()
.and_then(|metadata| metadata.get("x-openai-subagent")),
Some(&json!("review"))
);
assert_eq!(
captured
.client_metadata
.as_ref()
.and_then(|metadata| metadata.get("existing")),
Some(&json!("preserved"))
);
}
#[tokio::test]
async fn review_route_should_not_replace_non_object_client_metadata() {
let observed = Arc::new(Mutex::new(None));
let execution = Arc::new(ContextCaptureExecution {
observed: Arc::clone(&observed),
client: authenticated_client("sk_context_test"),
});
let request = Request::post("/v1/responses/review")
.header(AUTHORIZATION, "Bearer sk_context_test")
.body(Body::from(
json!({
"model": "smart-code",
"input": "review this",
"client_metadata": ["opaque", "client", "value"]
})
.to_string(),
))
.expect("review request");
let response = api_router(execution)
.await
.oneshot(request)
.await
.expect("review response");
let captured = observed
.lock()
.expect("review capture lock")
.clone()
.expect("captured review request");
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
captured.client_metadata,
Some(json!(["opaque", "client", "value"]))
);
}
#[tokio::test]
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
let trace = Arc::new(Trace::default());
@@ -1,6 +1,6 @@
use axum::{
body::Body,
http::{Method, Request, StatusCode},
http::{Method, Request, StatusCode, header::AUTHORIZATION},
};
use tower::ServiceExt;
@@ -9,6 +9,54 @@ use super::models::ModelsExecution;
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
#[tokio::test]
async fn removed_responses_review_route_should_not_reach_the_responses_handler() {
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
.await
.oneshot(
Request::post("/v1/responses/review")
.header(AUTHORIZATION, "Bearer sk_models_test")
.body(Body::empty())
.expect("build removed review request"),
)
.await
.expect("route removed review request");
assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
}
#[tokio::test]
async fn removed_models_catalog_extension_should_use_standard_model_lookup() {
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
.await
.oneshot(
Request::get("/v1/models/catalog")
.header(AUTHORIZATION, "Bearer sk_models_test")
.body(Body::empty())
.expect("build removed model catalog request"),
)
.await
.expect("route removed model catalog request");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn removed_model_info_route_should_return_not_found() {
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
.await
.oneshot(
Request::get("/v1/models/model-a/info")
.header(AUTHORIZATION, "Bearer sk_models_test")
.body(Body::empty())
.expect("build removed model info request"),
)
.await
.expect("route removed model info request");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
@@ -0,0 +1,229 @@
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
use axum::{
body::{Body, to_bytes},
http::{Request, StatusCode, header::AUTHORIZATION},
};
use bytes::Bytes;
use futures::future::BoxFuture;
use gateway_core::engine::ModelRequestId;
use gateway_core::engine::execution::{
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionService,
StartExecution, StartProviderExecution, StartedExecution,
};
use gateway_core::error::{
ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
};
use gateway_core::event::ProviderResponseHeader;
use gateway_core::operation::Operation;
use gateway_core::routing::PublicModelId;
use gateway_core::upstream::UpstreamSendState;
use serde_json::{Value, json};
use tower::ServiceExt;
use super::provider_endpoint::BufferedJsonSession;
use super::{api_router, authenticated_client};
const SEARCH_RESPONSE: &[u8] = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
#[derive(Debug, Clone, PartialEq)]
struct CapturedSearchRequest {
provider: String,
endpoint: String,
transport: ClientTransport,
body: Bytes,
context: Value,
}
struct SearchExecution {
client: AuthenticatedClient,
captured: Mutex<Vec<CapturedSearchRequest>>,
committed_statuses: Arc<Mutex<Vec<u16>>>,
fail_with_upstream_response: bool,
}
impl SearchExecution {
fn new(fail_with_upstream_response: bool) -> Arc<Self> {
Arc::new(Self {
client: authenticated_client("sk_search_test"),
captured: Mutex::new(Vec::new()),
committed_statuses: Arc::new(Mutex::new(Vec::new())),
fail_with_upstream_response,
})
}
fn captured(&self) -> Vec<CapturedSearchRequest> {
self.captured.lock().expect("capture lock").clone()
}
}
impl ExecutionService for SearchExecution {
fn authenticate(
&self,
plaintext: &str,
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
(plaintext == "sk_search_test")
.then(|| self.client.clone())
.ok_or(ClientAuthenticationError::InvalidKey)
}
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
Vec::new()
}
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
false
}
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async {
Err(GatewayError::new(
GatewayErrorKind::Internal,
"search test must use provider endpoint execution",
))
})
}
fn start_provider_endpoint(
&self,
request: StartProviderExecution,
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
Box::pin(async move {
let Operation::Search(search) = &request.operation else {
return Err(GatewayError::new(
GatewayErrorKind::Internal,
"search test received a non-search operation",
));
};
let payload = search.payload();
self.captured
.lock()
.expect("capture lock")
.push(CapturedSearchRequest {
provider: request.provider.as_str().to_owned(),
endpoint: request.metadata.endpoint,
transport: request.metadata.transport,
body: payload.body().clone(),
context: Value::Object(payload.context().clone()),
});
let session = if self.fail_with_upstream_response {
BufferedJsonSession::failure(
ProviderError::new(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::Sent,
)
.with_status(StatusCode::UNPROCESSABLE_ENTITY.as_u16())
.with_client_visible_upstream_response(
ClientVisibleUpstreamResponse::new(
StatusCode::UNPROCESSABLE_ENTITY.as_u16(),
Some(b"application/problem+json".to_vec()),
Bytes::from_static(
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#,
),
)
.with_headers(vec![ProviderResponseHeader::new(
"x-future-search-error",
Bytes::from_static(b"preserved"),
)]),
),
)
} else {
BufferedJsonSession::success(
Bytes::from_static(SEARCH_RESPONSE),
StatusCode::CREATED.as_u16(),
vec![ProviderResponseHeader::new(
"x-search-rate-limit",
Bytes::from_static(b"42"),
)],
Arc::clone(&self.committed_statuses),
)
};
Ok(StartedExecution {
request_id: ModelRequestId::new("req_search_test").expect("request ID"),
created_at: SystemTime::now(),
stream: false,
session: Box::new(session),
})
})
}
}
#[tokio::test]
async fn search_route_should_preserve_request_and_success_response_bytes() {
let execution = SearchExecution::new(false);
let request_body = br#"{ "id":"search-session", "model":"gpt-future", "commands":{"search_query":[{"q":"private query"}]}, "future":1, "future":2 }"#;
let turn_metadata =
r#"{"session_id":"session","thread_id":"thread","turn_id":"turn","future":true}"#;
let response = api_router(execution.clone())
.await
.oneshot(
Request::post("/v1/alpha/search")
.header(AUTHORIZATION, "Bearer sk_search_test")
.header("content-type", "application/json")
.header("x-codex-turn-metadata", turn_metadata)
.body(Body::from(request_body.to_vec()))
.expect("search request"),
)
.await
.expect("search response");
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get("x-search-rate-limit"),
Some(&"42".parse().expect("header value"))
);
let response_body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read search response");
assert_eq!(response_body.as_ref(), SEARCH_RESPONSE);
let captured = execution.captured();
assert_eq!(captured.len(), 1);
assert_eq!(captured[0].provider, "openai");
assert_eq!(captured[0].endpoint, "/v1/alpha/search");
assert_eq!(captured[0].transport, ClientTransport::HttpJson);
assert_eq!(captured[0].body.as_ref(), request_body);
assert_eq!(captured[0].context, json!({"turn_metadata": turn_metadata}));
assert_eq!(
*execution
.committed_statuses
.lock()
.expect("committed status lock"),
vec![StatusCode::CREATED.as_u16()]
);
}
#[tokio::test]
async fn search_route_should_return_the_exact_upstream_error_response() {
let response = api_router(SearchExecution::new(true))
.await
.oneshot(
Request::post("/v1/alpha/search")
.header(AUTHORIZATION, "Bearer sk_search_test")
.header("content-type", "application/json")
.body(Body::from(
br#"{ "model":"gpt-future", "future_invalid":9007199254740993 }"#.to_vec(),
))
.expect("search request"),
)
.await
.expect("search response");
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
assert_eq!(
response.headers().get("content-type"),
Some(&"application/problem+json".parse().expect("content type"))
);
assert_eq!(
response.headers().get("x-future-search-error"),
Some(&"preserved".parse().expect("header value"))
);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read search response");
assert_eq!(
body.as_ref(),
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#
);
}
@@ -3,9 +3,9 @@
use std::cmp::Reverse;
use std::collections::{BTreeSet, HashMap};
use std::num::NonZeroU32;
use std::sync::RwLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime};
use std::sync::{Mutex, RwLock};
use std::time::{Duration, Instant, SystemTime};
use crate::routing::ProviderKind;
@@ -94,6 +94,7 @@ pub struct AccountRuntimeSignals {
}
const ACCOUNT_FEEDBACK_EWMA_ALPHA: f64 = 0.2;
const ACCOUNT_FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
const EMPTY_FEEDBACK_SAMPLE: u64 = f64::NAN.to_bits();
/// 一次真实上游 attempt 对账号级 Smart 调度产生的中立反馈。
@@ -111,37 +112,69 @@ struct AccountFeedbackKey {
#[derive(Debug)]
struct AccountFeedback {
failure_rate: AtomicU64,
failure_rate: Mutex<DecayingAccountFailureRate>,
first_output_ms: AtomicU64,
}
#[derive(Debug, Default)]
struct DecayingAccountFailureRate {
value: f64,
updated_at: Option<Instant>,
}
impl DecayingAccountFailureRate {
fn value_at(&self, now: Instant) -> f64 {
let Some(updated_at) = self.updated_at else {
return self.value;
};
let elapsed = now.checked_duration_since(updated_at).unwrap_or_default();
self.value
* 0.5_f64.powf(elapsed.as_secs_f64() / ACCOUNT_FAILURE_RATE_HALF_LIFE.as_secs_f64())
}
fn report_at(&mut self, sample: f64, now: Instant) {
let now = self
.updated_at
.map_or(now, |updated_at| updated_at.max(now));
let decayed = self.value_at(now);
self.value =
ACCOUNT_FEEDBACK_EWMA_ALPHA * sample + (1.0 - ACCOUNT_FEEDBACK_EWMA_ALPHA) * decayed;
self.updated_at = Some(now);
}
}
impl Default for AccountFeedback {
fn default() -> Self {
Self {
// 新账号从健康基线开始,首个失败样本按 EWMA 平滑,而不是直接判死。
failure_rate: AtomicU64::new(0.0_f64.to_bits()),
failure_rate: Mutex::new(DecayingAccountFailureRate::default()),
first_output_ms: AtomicU64::new(EMPTY_FEEDBACK_SAMPLE),
}
}
}
impl AccountFeedback {
fn report(&self, feedback: AccountAttemptFeedback) {
fn report_at(&self, feedback: AccountAttemptFeedback, now: Instant) {
let (failure, first_output_ms) = match feedback {
AccountAttemptFeedback::Succeeded { first_output_ms } => (0.0, first_output_ms),
AccountAttemptFeedback::Failed { first_output_ms } => (1.0, first_output_ms),
};
update_feedback_ewma(&self.failure_rate, failure);
self.failure_rate
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.report_at(failure, now);
if let Some(first_output_ms) = first_output_ms.filter(|value| *value > 0) {
update_feedback_ewma(&self.first_output_ms, first_output_ms as f64);
}
}
fn scheduling_signals(&self) -> (Option<u16>, Option<u64>) {
let failure_rate = load_feedback_ewma(&self.failure_rate).map(|value| {
let basis_points = (value.clamp(0.0, 1.0) * 10_000.0).round();
basis_points as u16
});
fn scheduling_signals_at(&self, now: Instant) -> (Option<u16>, Option<u64>) {
let failure_rate = self
.failure_rate
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.value_at(now);
let failure_rate = Some((failure_rate.clamp(0.0, 1.0) * 10_000.0).round() as u16);
let first_output_ms = load_feedback_ewma(&self.first_output_ms)
.filter(|value| value.is_finite() && *value > 0.0)
.map(|value| value.round() as u64);
@@ -162,6 +195,18 @@ impl AccountFeedbackStats {
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
) -> (Option<u16>, Option<u64>) {
self.scheduling_signals_at(provider_kind, account_id, Instant::now())
}
/// 读取账号在指定单调时刻的错误率与首个有效输出延迟 EWMA。
#[doc(hidden)]
#[must_use]
pub fn scheduling_signals_at(
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
now: Instant,
) -> (Option<u16>, Option<u64>) {
let key = AccountFeedbackKey {
provider_kind: provider_kind.clone(),
@@ -171,7 +216,7 @@ impl AccountFeedbackStats {
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
.map(AccountFeedback::scheduling_signals)
.map(|account| account.scheduling_signals_at(now))
.unwrap_or_default()
}
@@ -181,6 +226,18 @@ impl AccountFeedbackStats {
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
feedback: AccountAttemptFeedback,
) {
self.report_at(provider_kind, account_id, feedback, Instant::now());
}
/// 回灌一次在指定单调时刻真实发送的上游 attempt。
#[doc(hidden)]
pub fn report_at(
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
feedback: AccountAttemptFeedback,
observed_at: Instant,
) {
let key = AccountFeedbackKey {
provider_kind: provider_kind.clone(),
@@ -192,7 +249,7 @@ impl AccountFeedbackStats {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
{
account.report(feedback);
account.report_at(feedback, observed_at);
return;
}
self.accounts
@@ -200,7 +257,7 @@ impl AccountFeedbackStats {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.entry(key)
.or_default()
.report(feedback);
.report_at(feedback, observed_at);
}
}
@@ -13,9 +13,9 @@ use crate::engine::provider::{Provider, ProviderCallMetadata, ProviderRequest, P
use crate::engine::{
AccountAttemptContext, AttemptContext, AttemptRecord, AttemptTransport, AttemptTrigger,
CommitRequirement, ContinuationAttempt, CoordinatedEvent, EngineError, ExecutionOutcome,
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
RequestAttemptContext, UpstreamSendState,
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFailureObservation,
ModelRequestFinalization, ModelRequestId, ModelRequestTimings, NewModelRequest,
ProviderAccountStateOwner, ProviderAttemptOutcome, RequestAttemptContext, UpstreamSendState,
};
use crate::error::{
ContinuationRecoveryDisposition, GatewayError, GatewayErrorKind, ProviderError,
@@ -221,6 +221,7 @@ struct FailureFinalization {
provider_error_code: Option<String>,
raw_upstream_error: Option<String>,
retry_after_ms: Option<u64>,
observation: ModelRequestFailureObservation,
}
#[derive(Debug, Clone)]
@@ -638,6 +639,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::EmptyRoutingPlan);
@@ -795,6 +797,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::ProviderMetadataMismatch);
@@ -820,6 +823,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::AccountOutsideClientScope);
@@ -840,6 +844,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::RequiredAccountMismatch);
@@ -863,6 +868,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::ContinuationPinMismatch);
@@ -1368,6 +1374,7 @@ where
error: None,
provider_error_code: None,
raw_upstream_error: None,
failure_observation: ModelRequestFailureObservation::default(),
retry_after_ms: None,
usage: self.usage.clone(),
image_generation_succeeded: self.image_generation_succeeded(),
@@ -1410,6 +1417,12 @@ where
.raw_upstream_error()
.map(|raw| raw.as_str().to_owned()),
retry_after_ms: error.retry_after().map(duration_ms),
observation: ModelRequestFailureObservation {
continuation_unavailable_reason: error
.continuation_unavailable_reason()
.map(str::to_owned),
upstream_connection: error.connection_observation().cloned(),
},
})
.await
}
@@ -1464,6 +1477,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await
}
@@ -1532,6 +1546,7 @@ where
error: Some(finalization.error),
provider_error_code: finalization.provider_error_code,
raw_upstream_error: finalization.raw_upstream_error,
failure_observation: finalization.observation,
retry_after_ms: finalization.retry_after_ms,
usage: self.usage.clone(),
image_generation_succeeded: self.image_generation_succeeded(),
@@ -505,8 +505,11 @@ impl DefaultExecutionService {
.candidates()
.first()
.map_or_else(Default::default, |candidate| {
self.providers
.request_observation(candidate.provider(), &operation)
self.providers.request_observation(
candidate.provider(),
&operation,
client.policy.key_id(),
)
});
let new_request = NewModelRequest {
id: request_id.clone(),
@@ -526,6 +529,7 @@ impl DefaultExecutionService {
request_kind: observation.request_kind,
subagent_kind: observation.subagent_kind,
compact: observation.compact,
continuation: observation.continuation,
image_generation_requested: operation.image_generation_requested(),
admission_decision_ms: Some(admission_decision_ms),
started_at,
@@ -668,6 +672,7 @@ impl DefaultExecutionService {
request_kind: Some("account_connection_test".to_owned()),
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
admission_decision_ms: None,
started_at,
+11 -1
View File
@@ -22,7 +22,8 @@ use thiserror::Error;
use crate::account::{AccountSelectionPolicy, ProviderAccountId};
use crate::engine::continuation::{ContinuationBinding, NativeContinuationPin};
use crate::error::{
GatewayError, IdentifierError, ProviderError, ProviderErrorKind, StoreError, validate_text,
GatewayError, IdentifierError, ProviderConnectionObservation, ProviderError, ProviderErrorKind,
StoreError, validate_text,
};
use crate::event::ProviderEvent;
use crate::lifecycle::CancellationToken;
@@ -514,6 +515,7 @@ pub struct NewModelRequest {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: provider::ContinuationRequestObservation,
pub image_generation_requested: bool,
/// Client Key 准入判定的完整耗时;内部探测不经过该阶段。
pub admission_decision_ms: Option<u64>,
@@ -579,6 +581,13 @@ pub struct ModelRequestTimings {
pub latency_ms: Option<u64>,
}
/// 终态失败的 Provider 结构化连接与 continuation 观测。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ModelRequestFailureObservation {
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection: Option<ProviderConnectionObservation>,
}
/// 单行模型请求的终态写回。
#[derive(Debug)]
pub struct ModelRequestFinalization {
@@ -602,6 +611,7 @@ pub struct ModelRequestFinalization {
pub provider_error_code: Option<String>,
/// Provider 返回的原始错误正文或 WebSocket close/error frame。
pub raw_upstream_error: Option<String>,
pub failure_observation: ModelRequestFailureObservation,
pub retry_after_ms: Option<u64>,
pub usage: Usage,
pub image_generation_succeeded: Option<bool>,
@@ -15,9 +15,10 @@ use crate::account::{
AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, ProviderAccountId,
};
use crate::engine::AttemptContext;
use crate::error::{OpaqueUpstreamValue, ProviderError, ProviderErrorKind};
use crate::error::{OpaqueUpstreamValue, PreDeliveryRetry, ProviderError, ProviderErrorKind};
use crate::event::{EventSequenceValidator, ProviderEvent};
use crate::operation::Operation;
use crate::policy::ClientApiKeyId;
use crate::routing::{
ModelCapabilities, ModelPresentation, ProviderCandidate, ProviderKind, UpstreamModelId,
};
@@ -190,11 +191,16 @@ struct ProviderStreamAccountFeedback {
stats: Arc<AccountFeedbackStats>,
provider_kind: ProviderKind,
account_id: ProviderAccountId,
failure_filter: fn(&ProviderError) -> bool,
started_at: Option<Instant>,
first_output_ms: Option<u64>,
reported: bool,
}
fn score_all_confirmed_failures(_: &ProviderError) -> bool {
true
}
impl ProviderStreamAccountFeedback {
fn start(&mut self) {
self.started_at.get_or_insert_with(Instant::now);
@@ -243,6 +249,15 @@ impl ProviderStreamAccountFeedback {
error.kind(),
ProviderErrorKind::Cancelled | ProviderErrorKind::ProcessTerminated
)
|| matches!(
error.pre_delivery_retry(),
Some(
PreDeliveryRetry::SameAccountTransportRetry { .. }
| PreDeliveryRetry::SameAccountTransportFallback
)
)
|| error.retries_same_account()
|| !(self.failure_filter)(error)
{
return;
}
@@ -278,15 +293,35 @@ impl ProviderStream {
/// 让公共 stream 边界统一回灌账号成功率与首个有效输出延迟。
#[must_use]
pub fn with_account_feedback(mut self, stats: Arc<AccountFeedbackStats>) -> Self {
self.set_account_feedback(stats, score_all_confirmed_failures);
self
}
/// 使用 Provider 定义的闭集判断回灌账号成功率与首个有效输出延迟。
#[must_use]
pub fn with_filtered_account_feedback(
mut self,
stats: Arc<AccountFeedbackStats>,
failure_filter: fn(&ProviderError) -> bool,
) -> Self {
self.set_account_feedback(stats, failure_filter);
self
}
fn set_account_feedback(
&mut self,
stats: Arc<AccountFeedbackStats>,
failure_filter: fn(&ProviderError) -> bool,
) {
self.account_feedback = Some(ProviderStreamAccountFeedback {
stats,
provider_kind: self.metadata.provider().clone(),
account_id: self.metadata.provider_account_id().clone(),
failure_filter,
started_at: None,
first_output_ms: None,
reported: false,
});
self
}
/// 返回调用事实。
@@ -387,6 +422,15 @@ pub struct ProviderRequestObservation {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: ContinuationRequestObservation,
}
/// 仅用于恢复事件关联的客户端作用域不透明请求事实。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ContinuationRequestObservation {
pub affinity_hash: Option<String>,
pub previous_response_id_hash: Option<String>,
pub requested: bool,
}
/// Provider 实时目录编译后的单模型能力。
@@ -495,7 +539,11 @@ pub trait Provider: Send + Sync {
fn catalog_generation(&self) -> ProviderCatalogGeneration;
/// 解释 Provider 差异化观测字段;不参与路由和传输。
fn request_observation(&self, _operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
_operation: &Operation,
_client_api_key_id: &ClientApiKeyId,
) -> ProviderRequestObservation {
ProviderRequestObservation::default()
}
@@ -623,11 +671,12 @@ impl ProviderRegistry {
&self,
provider: &ProviderKind,
operation: &Operation,
client_api_key_id: &ClientApiKeyId,
) -> ProviderRequestObservation {
self.providers
.get(provider)
.map_or_else(ProviderRequestObservation::default, |registered| {
registered.request_observation(operation)
registered.request_observation(operation, client_api_key_id)
})
}
+108
View File
@@ -135,6 +135,8 @@ pub enum MeteringError {
pub enum ProviderErrorKind {
/// 请求语义错误。
InvalidRequest,
/// previous-response 续接状态已不可用,需要客户端重建请求链。
ContinuationRecoveryRequired,
/// Provider 不支持所需能力。
Unsupported,
/// Credential 认证失败。
@@ -171,6 +173,7 @@ impl ProviderErrorKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::InvalidRequest => "invalid_request",
Self::ContinuationRecoveryRequired => "continuation_recovery_required",
Self::Unsupported => "unsupported",
Self::Unauthorized => "unauthorized",
Self::PermissionDenied => "permission_denied",
@@ -357,6 +360,55 @@ impl fmt::Debug for RawUpstreamError {
}
}
/// Provider 上游物理连接结束时的低基数观测。
///
/// 连接标识必须是随机或单向派生值,不得携带凭据、原始会话 ID
/// 或请求正文。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderConnectionObservation {
connection_id: String,
exit_reason: String,
age_ms: u64,
idle_ms: u64,
}
impl ProviderConnectionObservation {
#[must_use]
pub fn new(
connection_id: impl Into<String>,
exit_reason: impl Into<String>,
age_ms: u64,
idle_ms: u64,
) -> Self {
Self {
connection_id: connection_id.into(),
exit_reason: exit_reason.into(),
age_ms,
idle_ms,
}
}
#[must_use]
pub fn connection_id(&self) -> &str {
&self.connection_id
}
#[must_use]
pub fn exit_reason(&self) -> &str {
&self.exit_reason
}
#[must_use]
pub const fn age_ms(&self) -> u64 {
self.age_ms
}
#[must_use]
pub const fn idle_ms(&self) -> u64 {
self.idle_ms
}
}
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
///
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
@@ -442,6 +494,7 @@ pub struct ProviderError {
retry_after: Option<Duration>,
continuation_failure: Option<ContinuationFailure>,
continuation_recovery_disposition: Option<ContinuationRecoveryDisposition>,
failure_observation: Option<Box<ProviderErrorFailureObservation>>,
replay_safe: bool,
pre_delivery_retry: Option<PreDeliveryRetry>,
credential_recovery_required: bool,
@@ -460,6 +513,12 @@ struct ProviderErrorUpstreamValues {
request_id: Option<OpaqueUpstreamValue>,
}
#[derive(Clone, Default)]
struct ProviderErrorFailureObservation {
continuation_unavailable_reason: Option<&'static str>,
connection: Option<ProviderConnectionObservation>,
}
#[derive(Clone)]
struct AtomicClientEvents(Vec<ProviderEvent>);
@@ -475,6 +534,7 @@ impl ProviderError {
retry_after: None,
continuation_failure: None,
continuation_recovery_disposition: None,
failure_observation: None,
replay_safe: false,
pre_delivery_retry: None,
credential_recovery_required: false,
@@ -540,6 +600,29 @@ impl ProviderError {
self
}
/// 附加 Provider 给出的低基数 continuation 不可用原因。
#[must_use]
pub fn with_continuation_unavailable_reason(mut self, reason: &'static str) -> Self {
self.failure_observation_mut()
.continuation_unavailable_reason = Some(reason);
self
}
/// 附加上游物理连接结束观测。
#[must_use]
pub fn with_connection_observation(
mut self,
observation: ProviderConnectionObservation,
) -> Self {
self.failure_observation_mut().connection = Some(observation);
self
}
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
self.failure_observation
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
}
/// 标记 Provider 已证明本次拒绝没有执行生成,可在下游提交前重放。
#[must_use]
pub const fn with_replay_safe(mut self) -> Self {
@@ -714,6 +797,21 @@ impl ProviderError {
self.continuation_recovery_disposition
}
#[must_use]
pub fn continuation_unavailable_reason(&self) -> Option<&'static str> {
match self.failure_observation.as_deref() {
Some(observation) => observation.continuation_unavailable_reason,
None => None,
}
}
#[must_use]
pub fn connection_observation(&self) -> Option<&ProviderConnectionObservation> {
self.failure_observation
.as_deref()
.and_then(|observation| observation.connection.as_ref())
}
/// 返回 Provider 是否已证明本次失败可安全重放。
#[must_use]
pub const fn replay_is_safe(&self) -> bool {
@@ -801,6 +899,7 @@ impl Clone for ProviderError {
retry_after: self.retry_after,
continuation_failure: self.continuation_failure,
continuation_recovery_disposition: self.continuation_recovery_disposition,
failure_observation: self.failure_observation.clone(),
replay_safe: self.replay_safe,
pre_delivery_retry: self.pre_delivery_retry,
credential_recovery_required: self.credential_recovery_required,
@@ -836,6 +935,11 @@ impl fmt::Debug for ProviderError {
"continuation_recovery_disposition",
&self.continuation_recovery_disposition,
)
.field(
"continuation_unavailable_reason",
&self.continuation_unavailable_reason(),
)
.field("connection_observation", &self.connection_observation())
.field("replay_safe", &self.replay_safe)
.field("pre_delivery_retry", &self.pre_delivery_retry)
.field(
@@ -968,6 +1072,10 @@ impl GatewayError {
ProviderErrorKind::InvalidRequest => {
Self::new(GatewayErrorKind::InvalidRequest, "invalid upstream request")
}
ProviderErrorKind::ContinuationRecoveryRequired => Self::new(
GatewayErrorKind::InvalidRequest,
"conversation continuation must be rebuilt",
),
ProviderErrorKind::Unsupported => Self::new(
GatewayErrorKind::Unsupported,
"requested capability is unsupported",
@@ -22,6 +22,8 @@ pub enum OperationKind {
Generate,
/// 图像生成。
GenerateImage,
/// Provider 原生 standalone search。
Search,
}
impl OperationKind {
@@ -31,6 +33,7 @@ impl OperationKind {
match self {
Self::Generate => "generate",
Self::GenerateImage => "generate_image",
Self::Search => "search",
}
}
}
@@ -501,6 +504,35 @@ impl fmt::Debug for ImageRequest {
}
}
/// Provider 原生 standalone search 请求。
#[derive(Clone, PartialEq, Eq)]
pub struct StandaloneSearchRequest {
payload: RawJsonPayload,
}
impl StandaloneSearchRequest {
/// 创建携带原始协议 JSON 正文的 standalone search 请求。
#[must_use]
pub const fn from_raw_json(payload: RawJsonPayload) -> Self {
Self { payload }
}
/// 返回未经重编码的协议 JSON 正文。
#[must_use]
pub const fn payload(&self) -> &RawJsonPayload {
&self.payload
}
}
impl fmt::Debug for StandaloneSearchRequest {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("StandaloneSearchRequest")
.field("payload", &"<not included in Debug>")
.finish()
}
}
/// 网关内部业务请求;不包含任何客户端 wire 或 Provider SDK 类型。
#[derive(Clone, PartialEq)]
#[non_exhaustive]
@@ -509,6 +541,8 @@ pub enum Operation {
Generate(GenerateRequest),
/// 图像生成。
GenerateImage(ImageRequest),
/// Provider 原生 standalone search。
Search(StandaloneSearchRequest),
}
impl Operation {
@@ -534,6 +568,7 @@ impl Operation {
match self {
Self::Generate(_) => OperationKind::Generate,
Self::GenerateImage(_) => OperationKind::GenerateImage,
Self::Search(_) => OperationKind::Search,
}
}
@@ -543,6 +578,7 @@ impl Operation {
match self {
Self::Generate(request) => request.requirements(),
Self::GenerateImage(_) => CapabilityRequirements::new(OperationKind::GenerateImage),
Self::Search(_) => CapabilityRequirements::new(OperationKind::Search),
}
}
@@ -552,6 +588,7 @@ impl Operation {
match self {
Self::Generate(request) => request.image_generation_requested(),
Self::GenerateImage(_) => true,
Self::Search(_) => false,
}
}
@@ -560,7 +597,7 @@ impl Operation {
pub fn provider_session_state(&self, provider: &str) -> Option<&ProviderSessionState> {
match self {
Self::Generate(request) => request.provider_session_state(provider),
Self::GenerateImage(_) => None,
Self::GenerateImage(_) | Self::Search(_) => None,
}
}
}
@@ -1,3 +1,5 @@
mod selection;
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
use std::sync::Arc;
@@ -0,0 +1,98 @@
use std::time::{Duration, Instant};
use gateway_core::account::{AccountAttemptFeedback, AccountFeedbackStats, ProviderAccountId};
use gateway_core::routing::ProviderKind;
const FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
fn feedback_subject() -> (AccountFeedbackStats, ProviderKind, ProviderAccountId) {
(
AccountFeedbackStats::default(),
ProviderKind::new("openai").expect("valid provider"),
ProviderAccountId::new("acct_decay").expect("valid account"),
)
}
fn report_failure(
feedback: &AccountFeedbackStats,
provider: &ProviderKind,
account: &ProviderAccountId,
observed_at: Instant,
) {
feedback.report_at(
provider,
account,
AccountAttemptFeedback::Failed {
first_output_ms: None,
},
observed_at,
);
}
#[test]
fn account_failure_rate_should_halve_after_one_half_life() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
.0;
assert_eq!(failure_rate, Some(1_000));
}
#[test]
fn account_failure_rate_should_quarter_after_two_half_lives() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
let failure_rate = feedback
.scheduling_signals_at(
&provider,
&account,
observed_at + FAILURE_RATE_HALF_LIFE * 2,
)
.0;
assert_eq!(failure_rate, Some(500));
}
#[test]
fn account_failure_rate_should_decay_before_applying_a_new_sample() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
feedback.report_at(
&provider,
&account,
AccountAttemptFeedback::Succeeded {
first_output_ms: None,
},
observed_at + FAILURE_RATE_HALF_LIFE,
);
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
.0;
assert_eq!(failure_rate, Some(800));
}
#[test]
fn concurrent_account_failures_should_not_lose_samples() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
std::thread::scope(|scope| {
for _ in 0..8 {
scope.spawn(|| report_failure(&feedback, &provider, &account, observed_at));
}
});
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at)
.0;
assert_eq!(failure_rate, Some(8_322));
}
@@ -501,10 +501,13 @@ fn atomic_history_unavailable(response_id: &str) -> ProviderError {
)
.expect("history unavailable wire"),
);
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
.with_status(400)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_atomic_client_events(vec![failed])
ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::Sent,
)
.with_status(400)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_atomic_client_events(vec![failed])
}
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
@@ -621,6 +624,7 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
request_kind: Some("responses".to_owned()),
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: operation.image_generation_requested(),
started_at: SystemTime::now(),
deadline_at: deadline,
@@ -1591,7 +1595,7 @@ fn native_continuation_client_replay_required_is_terminal() {
Script::Stream {
account_id: "acct_one",
items: vec![Err(ProviderError::new(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
@@ -62,6 +62,9 @@ fn only_provider_attributable_failures_should_affect_circuit() {
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::InvalidRequest
));
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::ContinuationRecoveryRequired
));
}
#[test]
@@ -1,6 +1,8 @@
use std::collections::BTreeSet;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use async_trait::async_trait;
use futures::{StreamExt, stream};
@@ -244,3 +246,139 @@ fn provider_stream_should_report_confirmed_sent_failure_but_ignore_unconfirmed_f
(None, None)
);
}
#[test]
fn provider_stream_should_ignore_repeated_failures_rejected_by_the_filter() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_client_config_error").expect("account");
for _ in 0..100 {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("valid transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::Sent,
))]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| false);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
assert_eq!(
feedback.scheduling_signals(&provider, &account),
(None, None)
);
}
#[test]
fn provider_stream_should_score_only_the_final_same_account_failure() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_same_account_failure").expect("account");
for retry_index in 1..=3 {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
.with_pre_delivery_transport_retry(
NonZeroU32::new(retry_index).expect("positive retry index"),
Duration::ZERO,
);
let events: EventStream = Box::pin(stream::iter([Err(error)]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
ProviderErrorKind::Transport,
UpstreamSendState::Sent,
))]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
assert_eq!(
feedback.scheduling_signals(&provider, &account).0,
Some(2_000)
);
}
#[test]
fn provider_stream_should_report_one_success_after_same_account_retry() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_same_account_success").expect("account");
let retry_metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let retry_error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
.with_pre_delivery_transport_fallback();
let retry_events: EventStream = Box::pin(stream::iter([Err(retry_error)]));
let mut retry_stream = ProviderStream::new(retry_metadata, retry_events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
retry_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
let success_metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("valid transport"),
);
let success_events: EventStream = Box::pin(stream::empty());
let mut success_stream = ProviderStream::new(success_metadata, success_events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(success_stream.next().await.is_none());
});
assert_eq!(feedback.scheduling_signals(&provider, &account).0, Some(0));
}
@@ -168,6 +168,30 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
assert!(!format!("{gateway:?}").contains(message));
}
#[test]
fn continuation_recovery_should_keep_its_internal_classification_and_client_contract() {
let error = ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"Previous response was not found. Retrying the full request.",
Some("previous_response_not_found".to_owned()),
Some("invalid_request_error".to_owned()),
));
let gateway = GatewayError::from_provider(&error);
assert_eq!(error.kind().as_str(), "continuation_recovery_required");
assert_eq!(
gateway.safe_message(),
"conversation continuation must be rebuilt"
);
assert_eq!(
gateway.client_error_code(),
Some("previous_response_not_found")
);
}
#[test]
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
let message = format!("\0{}\n", "m".repeat(9_000));
@@ -1,7 +1,7 @@
use bytes::Bytes;
use gateway_core::operation::{
Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
ProtocolPayload, ProviderSessionState, RawJsonPayload,
ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
};
use serde_json::{Map, Value, json};
@@ -121,10 +121,43 @@ fn operation_kind_should_remain_stable() {
ImageRequestKind::Generation,
json!({"model": "gpt-image-2", "prompt": "draw"}),
));
let search = Operation::Search(StandaloneSearchRequest::from_raw_json(
RawJsonPayload::new(
"openai",
Bytes::from_static(br#"{ "model":"gpt-test", "commands":{} }"#),
)
.expect("OpenAI search payload"),
));
assert_eq!(generate.kind(), OperationKind::Generate);
assert_eq!(image.kind(), OperationKind::GenerateImage);
assert_eq!(search.kind(), OperationKind::Search);
assert_eq!(OperationKind::Search.as_str(), "search");
assert!(image.image_generation_requested());
assert!(!search.image_generation_requested());
}
#[test]
fn standalone_search_should_keep_raw_body_and_context_opaque() {
let body = Bytes::from_static(
br#"{ "model":"gpt-future", "commands":{}, "future":9007199254740993, "future":2 }"#,
);
let secret = "private-turn-metadata";
let payload = RawJsonPayload::new("openai", body.clone())
.expect("OpenAI search payload")
.with_context(Map::from_iter([(
"turn_metadata".to_owned(),
Value::String(secret.to_owned()),
)]));
let request = StandaloneSearchRequest::from_raw_json(payload);
assert_eq!(request.payload().body(), &body);
assert_eq!(
request.payload().context().get("turn_metadata"),
Some(&Value::String(secret.to_owned()))
);
assert!(!format!("{request:?}").contains(secret));
assert!(!format!("{request:?}").contains("gpt-future"));
}
#[test]
@@ -11,7 +11,9 @@ use gateway_core::engine::{
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
};
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
use gateway_core::error::{
ProviderErrorKind, StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind,
};
use gateway_core::metering::{CostSource as CoreCostSource, Usage as CoreUsage};
use gateway_core::routing::{AccountRoutingScopeKind, AccountRoutingSnapshot};
use gateway_core::upstream::UpstreamSendState as CoreUpstreamSendState;
@@ -99,12 +101,20 @@ pub struct NewModelRequest {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: ContinuationRequestObservation,
pub image_generation_requested: bool,
pub admission_decision_ms: Option<u64>,
pub started_at: DateTime<Utc>,
pub deadline_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ContinuationRequestObservation {
pub affinity_hash: Option<String>,
pub previous_response_id_hash: Option<String>,
pub requested: bool,
}
impl NewModelRequest {
pub fn validate(&self) -> StoreResult<()> {
require_nonempty(ENTITY, "id", &self.id)?;
@@ -133,6 +143,31 @@ impl NewModelRequest {
{
return Err(invalid("live client key ID must equal its historical ref"));
}
self.continuation.validate()?;
Ok(())
}
}
impl ContinuationRequestObservation {
fn validate(&self) -> StoreResult<()> {
for value in [
self.affinity_hash.as_deref(),
self.previous_response_id_hash.as_deref(),
]
.into_iter()
.flatten()
{
if value.len() != 64
|| !value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err(invalid("continuation observation hash is invalid"));
}
}
if !self.requested && self.previous_response_id_hash.is_some() {
return Err(invalid("continuation request facts do not agree"));
}
Ok(())
}
}
@@ -252,6 +287,11 @@ pub struct ModelRequestFinalization {
pub provider_error_code: Option<String>,
pub error_message: Option<String>,
pub raw_upstream_error: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub retry_after_ms: Option<u64>,
pub usage: ModelRequestUsage,
pub image_generation_succeeded: Option<bool>,
@@ -313,6 +353,11 @@ impl ModelRequestFinalization {
{
return Err(invalid("provider observation must be a JSON object"));
}
validate_optional_stable_reason(
self.continuation_unavailable_reason.as_deref(),
"continuation unavailable reason",
)?;
validate_connection_observation(self)?;
self.timings.validate()
}
}
@@ -389,11 +434,13 @@ impl ModelRequestRepository for PgExecutionStore {
operation, endpoint, client_transport, requested_model_id,
client_ip, user_agent, reasoning_effort,
reasoning_preset, request_kind, subagent_kind, compact,
image_generation_requested, admission_decision_ms, started_at, deadline_at
image_generation_requested, admission_decision_ms, started_at, deadline_at,
continuation_affinity_hash, continuation_previous_response_id_hash,
continuation_requested
) values (
$1, $2, $3, $4, $5, $6, $7, $8,
$9, $10, $11, $12, $13::inet, $14, $15,
$16, $17, $18, $19, $20, $21, $22, $23
$16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26
)",
)
.bind(request.id)
@@ -422,6 +469,9 @@ impl ModelRequestRepository for PgExecutionStore {
)?)
.bind(request.started_at)
.bind(request.deadline_at)
.bind(request.continuation.affinity_hash)
.bind(request.continuation.previous_response_id_hash)
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request"))?;
@@ -452,13 +502,15 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_model_id, upstream_transport, http_version,
attempt_count, upstream_send_state, account_selection_wait_ms,
capacity_used_slots, capacity_total_slots
, continuation_affinity_hash, continuation_previous_response_id_hash,
continuation_requested
) select
$1, $2, $3, $4, $5, $6, $7, $8,
$9, $10, $11, $12, $13::inet, $14, $15,
$16, $17, $18, $19, $20, $21, $22, $23,
$24, $25, $26,
account.name, account.email, account.authentication_kind,
$27, $28, $29, 1, 'not_sent', $30, $31, $32
$27, $28, $29, 1, 'not_sent', $30, $31, $32, $33, $34, $35
from (values (true)) as seed(present)
left join provider_accounts account on account.id = $25",
)
@@ -506,6 +558,9 @@ impl ModelRequestRepository for PgExecutionStore {
attempt.capacity_total_slots,
"capacity_total_slots",
)?)
.bind(request.continuation.affinity_hash)
.bind(request.continuation.previous_response_id_hash)
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
@@ -644,8 +699,9 @@ impl ModelRequestRepository for PgExecutionStore {
finalization: ModelRequestFinalization,
) -> StoreResult<bool> {
finalization.validate()?;
let result = sqlx::query(
"update model_requests
let finalized = sqlx::query_scalar::<_, i64>(
"with finalized as (
update model_requests
set outcome = $2, upstream_send_state = $3, attempt_count = $4,
downstream_committed_at = $5,
client_status_code = coalesce(client_status_code, $6),
@@ -664,8 +720,64 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_transport = coalesce($37, upstream_transport),
http_version = coalesce($38, http_version), websocket_pool = $39,
service_tier = $40, provider_observation_json = $41,
raw_upstream_error = $42
where id = $1 and outcome = 'running'",
raw_upstream_error = $42,
continuation_unavailable_reason = $43,
upstream_connection_id = $44,
upstream_connection_exit_reason = $45,
upstream_connection_age_ms = $46,
upstream_connection_idle_ms = $47
where id = $1 and outcome = 'running'
returning id, client_api_key_ref, continuation_affinity_hash,
continuation_requested, outcome, started_at, completed_at
), recovery_target as (
select prior.id
from model_requests prior
cross join finalized current
where not current.continuation_requested
and current.continuation_affinity_hash is not null
and prior.client_api_key_ref = current.client_api_key_ref
and prior.continuation_affinity_hash = current.continuation_affinity_hash
and prior.continuation_requested
and prior.outcome = 'failed'
and prior.error_kind = 'continuation_recovery_required'
and prior.recovered_at is null
and current.started_at >= prior.completed_at
and current.started_at <= prior.completed_at + interval '30 seconds'
order by prior.completed_at desc, prior.id desc
limit 1
), recovery_update as (
update model_requests prior
set recovery_attempt_count = prior.recovery_attempt_count + 1,
recovery_request_id = case
when current.outcome = 'succeeded' then current.id
else prior.recovery_request_id
end,
recovered_at = case
when current.outcome = 'succeeded' then current.completed_at
else prior.recovered_at
end,
recovery_retry_delay_ms = case
when current.outcome = 'succeeded' then greatest(
0,
floor(extract(epoch from (current.started_at - prior.completed_at)) * 1000)
)::bigint
else prior.recovery_retry_delay_ms
end,
recovery_total_latency_ms = case
when current.outcome = 'succeeded' then greatest(
0,
floor(extract(epoch from (current.completed_at - prior.completed_at)) * 1000)
)::bigint
else prior.recovery_total_latency_ms
end
from finalized current
join recovery_target target on true
where prior.id = target.id
and prior.recovered_at is null
returning prior.id
)
select (select count(*) from finalized)::bigint
+ (select count(*) * 0 from recovery_update)::bigint",
)
.bind(&finalization.model_request_id)
.bind(finalization.outcome.as_str())
@@ -754,10 +866,21 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(finalization.service_tier)
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
.bind(finalization.raw_upstream_error)
.execute(&self.pool)
.bind(finalization.continuation_unavailable_reason)
.bind(finalization.upstream_connection_id)
.bind(finalization.upstream_connection_exit_reason)
.bind(optional_i64(
finalization.upstream_connection_age_ms,
"upstream_connection_age_ms",
)?)
.bind(optional_i64(
finalization.upstream_connection_idle_ms,
"upstream_connection_idle_ms",
)?)
.fetch_one(&self.pool)
.await
.map_err(|_| postgres_unavailable("finalize model request"))?;
Ok(result.rows_affected() == 1)
Ok(finalized == 1)
}
async fn recover_expired_model_requests(
@@ -973,6 +1096,22 @@ impl ExecutionStore for PgExecutionStore {
&self,
finalization: CoreModelRequestFinalization,
) -> Result<(), CoreStoreError> {
let continuation_unavailable_reason = finalization
.failure_observation
.continuation_unavailable_reason
.clone();
let connection_observation = finalization
.failure_observation
.upstream_connection
.as_ref();
let upstream_connection_id =
connection_observation.map(|observation| observation.connection_id().to_owned());
let upstream_connection_exit_reason =
connection_observation.map(|observation| observation.exit_reason().to_owned());
let upstream_connection_age_ms =
connection_observation.map(|observation| observation.age_ms());
let upstream_connection_idle_ms =
connection_observation.map(|observation| observation.idle_ms());
let provider_metadata_json = finalization
.provider_metadata_json
.as_deref()
@@ -999,10 +1138,18 @@ impl ExecutionStore for PgExecutionStore {
),
None => (CostSource::Unavailable, None, None),
};
let error_kind = finalization
.error
.as_ref()
.map(|error| error.kind().as_str().to_owned());
let error_kind = if continuation_unavailable_reason.is_some() {
Some(
ProviderErrorKind::ContinuationRecoveryRequired
.as_str()
.to_owned(),
)
} else {
finalization
.error
.as_ref()
.map(|error| error.kind().as_str().to_owned())
};
let error_message = finalization.error.as_ref().map(|error| {
error.diagnostic().map_or_else(
|| error.safe_message().to_owned(),
@@ -1033,6 +1180,11 @@ impl ExecutionStore for PgExecutionStore {
provider_error_code: finalization.provider_error_code,
error_message,
raw_upstream_error: finalization.raw_upstream_error,
continuation_unavailable_reason,
upstream_connection_id,
upstream_connection_exit_reason,
upstream_connection_age_ms,
upstream_connection_idle_ms,
retry_after_ms: finalization.retry_after_ms,
usage: usage_from_core(finalization.usage),
image_generation_succeeded: finalization.image_generation_succeeded,
@@ -1162,6 +1314,11 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
request_kind: request.request_kind,
subagent_kind: request.subagent_kind,
compact: request.compact,
continuation: ContinuationRequestObservation {
affinity_hash: request.continuation.affinity_hash,
previous_response_id_hash: request.continuation.previous_response_id_hash,
requested: request.continuation.requested,
},
image_generation_requested: request.image_generation_requested,
admission_decision_ms: request.admission_decision_ms,
started_at: DateTime::<Utc>::from(request.started_at),
@@ -1246,6 +1403,52 @@ fn validate_status_code(status: Option<u16>) -> StoreResult<()> {
Ok(())
}
fn validate_optional_stable_reason(value: Option<&str>, field: &str) -> StoreResult<()> {
if value.is_some_and(|value| {
value.is_empty()
|| value.len() > 64
|| !value.bytes().enumerate().all(|(index, byte)| {
byte.is_ascii_lowercase() || (index > 0 && (byte.is_ascii_digit() || byte == b'_'))
})
}) {
return Err(invalid(field));
}
Ok(())
}
fn validate_connection_observation(finalization: &ModelRequestFinalization) -> StoreResult<()> {
let fields_present = [
finalization.upstream_connection_id.is_some(),
finalization.upstream_connection_exit_reason.is_some(),
finalization.upstream_connection_age_ms.is_some(),
finalization.upstream_connection_idle_ms.is_some(),
];
if fields_present.iter().any(|present| *present)
&& !fields_present.iter().all(|present| *present)
{
return Err(invalid("upstream connection observation is incomplete"));
}
if let Some(connection_id) = finalization.upstream_connection_id.as_deref()
&& (connection_id.is_empty()
|| connection_id.len() > 128
|| connection_id.chars().any(char::is_control))
{
return Err(invalid("upstream connection ID is invalid"));
}
validate_optional_stable_reason(
finalization.upstream_connection_exit_reason.as_deref(),
"upstream connection exit reason is invalid",
)?;
if finalization
.upstream_connection_age_ms
.zip(finalization.upstream_connection_idle_ms)
.is_some_and(|(age, idle)| idle > age)
{
return Err(invalid("upstream connection idle time exceeds its age"));
}
Ok(())
}
fn to_i64(value: u64, field: &'static str) -> StoreResult<i64> {
i64::try_from(value).map_err(|_| invalid(field))
}
@@ -39,7 +39,10 @@ pub use provider_accounts::*;
pub use retention::*;
pub use runtime_settings::*;
pub use snapshot::*;
pub(crate) use usage_facts::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
pub(crate) use usage_facts::{
completed_usage_fact_predicate, push_completed_usage_fact_filter,
push_unrecovered_request_filter,
};
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
@@ -51,13 +51,11 @@ pub(crate) fn store_ops_error_filter(
provider_account_ref: filter.provider_account_ref,
provider_kind: filter.provider_kind,
operation: filter.operation,
endpoint: filter.endpoint,
model: filter.model,
transport: filter.transport,
attempt_index: filter.attempt_index,
response_id: filter.response_id,
upstream_request_id: filter.upstream_request_id,
failure_kind: filter.failure_kind,
status_code: filter.status_code,
search: filter.search,
}
@@ -731,6 +729,18 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
request_kind: error.request_kind,
subagent_kind: error.subagent_kind,
compact: error.compact,
continuation_affinity_hash: error.continuation_affinity_hash,
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
continuation_unavailable_reason: error.continuation_unavailable_reason,
upstream_connection_id: error.upstream_connection_id,
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
upstream_connection_age_ms: error.upstream_connection_age_ms,
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
recovery_request_id: error.recovery_request_id,
recovered_at: error.recovered_at,
recovery_attempt_count: error.recovery_attempt_count,
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
recovery_total_latency_ms: error.recovery_total_latency_ms,
occurred_at: error.occurred_at,
stable_sort_id: error.stable_sort_id,
}
@@ -910,6 +920,18 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
request_kind: get(row, "request_kind")?,
subagent_kind: get(row, "subagent_kind")?,
compact: get(row, "compact")?,
continuation_affinity_hash: get(row, "continuation_affinity_hash")?,
continuation_previous_response_id_hash: get(row, "continuation_previous_response_id_hash")?,
continuation_unavailable_reason: get(row, "continuation_unavailable_reason")?,
upstream_connection_id: get(row, "upstream_connection_id")?,
upstream_connection_exit_reason: get(row, "upstream_connection_exit_reason")?,
upstream_connection_age_ms: optional_unsigned(row, "upstream_connection_age_ms")?,
upstream_connection_idle_ms: optional_unsigned(row, "upstream_connection_idle_ms")?,
recovery_request_id: get(row, "recovery_request_id")?,
recovered_at: get(row, "recovered_at")?,
recovery_attempt_count: to_u32(get(row, "recovery_attempt_count")?)?,
recovery_retry_delay_ms: optional_unsigned(row, "recovery_retry_delay_ms")?,
recovery_total_latency_ms: optional_unsigned(row, "recovery_total_latency_ms")?,
occurred_at: get(row, "occurred_at")?,
stable_sort_id: get(row, "stable_sort_id")?,
})
@@ -18,7 +18,10 @@ use crate::{
require_nonempty,
};
use super::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
use super::{
completed_usage_fact_predicate, push_completed_usage_fact_filter,
push_unrecovered_request_filter,
};
mod admin_adapter;
mod mapping;
@@ -126,13 +126,11 @@ pub struct OpsErrorFilter {
pub provider_account_ref: Option<String>,
pub provider_kind: Option<String>,
pub operation: Option<String>,
pub endpoint: Option<String>,
pub model: Option<String>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub failure_kind: Option<String>,
pub status_code: Option<u16>,
pub search: Option<String>,
}
@@ -148,14 +146,12 @@ impl OpsErrorFilter {
),
(self.provider_kind.as_deref(), "provider filter"),
(self.operation.as_deref(), "operation filter"),
(self.endpoint.as_deref(), "endpoint filter"),
(self.model.as_deref(), "model filter"),
(self.transport.as_deref(), "transport filter"),
(
self.upstream_request_id.as_deref(),
"upstream request ID filter",
),
(self.failure_kind.as_deref(), "failure kind filter"),
] {
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
}
@@ -726,6 +722,18 @@ pub struct OpsErrorRecord {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: Option<bool>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
pub occurred_at: DateTime<Utc>,
pub stable_sort_id: String,
}
@@ -93,6 +93,7 @@ pub(crate) async fn request_metrics(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
let row = query
.build()
@@ -113,7 +114,8 @@ pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTota
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
sum(cost_amount) filter (where {fact} and cost_currency = 'USD')::text
as billing_usd
from model_requests mr"
from model_requests mr
where mr.recovered_at is null"
));
let row = query
.build()
@@ -247,6 +249,7 @@ async fn request_metric_series_inner(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by bucket_start order by bucket_start");
let rows = query
@@ -373,6 +376,7 @@ pub(crate) async fn attempt_metrics(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(format!(
"), request_aggregate as (
@@ -517,6 +521,7 @@ pub(crate) async fn provider_observations(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
let rows = query
@@ -24,6 +24,12 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
mr.continuation_unavailable_reason,
mr.upstream_connection_id, mr.upstream_connection_exit_reason,
mr.upstream_connection_age_ms, mr.upstream_connection_idle_ms,
mr.recovery_request_id, mr.recovered_at, mr.recovery_attempt_count,
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
mr.completed_at as occurred_at,
'model_request:' || mr.id as stable_sort_id
from model_requests mr
@@ -46,6 +52,15 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
null::text as continuation_unavailable_reason,
null::text as upstream_connection_id,
null::text as upstream_connection_exit_reason,
null::bigint as upstream_connection_age_ms,
null::bigint as upstream_connection_idle_ms,
mr.recovery_request_id, mr.recovered_at,
coalesce(mr.recovery_attempt_count, 0) as recovery_attempt_count,
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
oe.created_at as occurred_at,
'ops_event:' || oe.id as stable_sort_id
from ops_events oe
@@ -113,6 +128,7 @@ fn push_request_error_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and mr.recovered_at is null");
push_range(statement, "mr.completed_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -120,16 +136,11 @@ fn push_request_error_predicates(
("mr.provider_account_ref", &filter.provider_account_ref),
("mr.provider_kind", &filter.provider_kind),
("mr.operation", &filter.operation),
("mr.endpoint", &filter.endpoint),
("mr.upstream_transport", &filter.transport),
("mr.upstream_request_id", &filter.upstream_request_id),
] {
push_text_equality(statement, column, value);
}
if let Some(value) = &filter.failure_kind {
statement.push(" and coalesce(mr.error_kind, 'failed') = ");
statement.push_bind(value.clone());
}
push_response_id_filter(statement, "mr.client_response_id", filter);
push_text_equality(statement, "mr.upstream_model_id", &filter.model);
if let Some(index) = filter.attempt_index {
@@ -160,6 +171,7 @@ fn push_ops_event_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and (mr.id is null or mr.recovered_at is null)");
push_range(statement, "oe.created_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -167,9 +179,7 @@ fn push_ops_event_predicates(
("oe.provider_account_ref", &filter.provider_account_ref),
("oe.provider_kind", &filter.provider_kind),
("oe.operation", &filter.operation),
("mr.endpoint", &filter.endpoint),
("oe.upstream_request_id", &filter.upstream_request_id),
("oe.failure_kind", &filter.failure_kind),
] {
push_text_equality(statement, column, value);
}
@@ -88,7 +88,19 @@ pub(crate) fn push_usage_filter(
query.push(format!(" escape '\\' or {alias}.{column} like "));
query.push_bind(pattern.clone());
}
query.push(" escape '\\')");
if value.starts_with("sk_") {
query.push(format!(
" escape '\\' or exists (
select 1 from client_api_keys searched_client_key
where searched_client_key.id = {alias}.client_api_key_ref
and searched_client_key.key like "
));
query.push_bind(pattern);
query.push(" escape '\\')");
} else {
query.push(" escape '\\'");
}
query.push(")");
}
}
@@ -360,6 +372,7 @@ pub(crate) async fn usage_diagnostics(
statement.push_bind(range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(range.end);
push_unrecovered_request_filter(&mut statement, "mr");
push_diagnostic_dimension_filter(&mut statement, dimension);
push_usage_filter(&mut statement, filter, "mr");
statement.push(
@@ -18,3 +18,10 @@ pub(crate) fn completed_usage_fact_predicate(alias: &str) -> String {
pub(crate) fn push_completed_usage_fact_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
query.push(format!(" and {}", completed_usage_fact_predicate(alias)));
}
/// 排除已由后续成功请求恢复的会话续接中间失败。
///
/// 原始请求审计仍保留该行;默认业务指标只把最终成功链视为一次结果。
pub(crate) fn push_unrecovered_request_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
query.push(format!(" and {alias}.recovered_at is null"));
}
@@ -1,9 +1,10 @@
use chrono::{Duration, Utc};
use gateway_core::engine::{
ExecutionOutcome, ExecutionStore, ModelRequestFinalization as CoreModelRequestFinalization,
ModelRequestId, ModelRequestTimings as CoreModelRequestTimings,
ExecutionOutcome, ExecutionStore, ModelRequestFailureObservation,
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
ModelRequestTimings as CoreModelRequestTimings,
};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObservation};
use gateway_core::metering::{CalculatedCost, CostEstimate, Usage};
use gateway_core::upstream::UpstreamSendState;
use gateway_store::postgres::{
@@ -42,6 +43,7 @@ fn model_request_rejects_mismatched_client_key_live_id() {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -93,6 +95,7 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -210,6 +213,7 @@ async fn model_request_persists_group_routing_snapshot_without_live_group_foreig
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -477,6 +481,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
error: None,
provider_error_code: None,
raw_upstream_error: None,
failure_observation: Default::default(),
retry_after_ms: None,
usage: Usage::new(),
image_generation_succeeded: None,
@@ -600,6 +605,243 @@ async fn core_adapter_should_reject_non_object_provider_observation() {
database.close().await;
}
#[tokio::test]
async fn core_adapter_persists_continuation_connection_failure_observation() {
let Some(database) = TestDatabase::create("execution_continuation_observation").await else {
return;
};
seed_running_request(&database.pool, "req_continuation_observation")
.await
.expect("seed model request");
let store = PgExecutionStore::new(database.pool.clone());
let mut finalization = successful_core_finalization("req_continuation_observation");
finalization.outcome = ExecutionOutcome::Failed;
finalization.client_status_code = Some(400);
finalization.error = Some(GatewayError::new(
GatewayErrorKind::InvalidRequest,
"conversation continuation must be rebuilt",
));
finalization.failure_observation = ModelRequestFailureObservation {
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
upstream_connection: Some(ProviderConnectionObservation::new(
"connection-observed",
"tcp_reset",
12_000,
4_000,
)),
};
ExecutionStore::finalize_model_request(&store, finalization)
.await
.expect("persist continuation failure observation");
type PersistedContinuationObservation = (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
Option<i64>,
Option<i64>,
);
let persisted: PersistedContinuationObservation = sqlx::query_as(
"select error_kind, continuation_unavailable_reason,
upstream_connection_id, upstream_connection_exit_reason,
upstream_connection_age_ms, upstream_connection_idle_ms
from model_requests where id = 'req_continuation_observation'",
)
.fetch_one(&database.pool)
.await
.expect("load continuation failure observation");
assert_eq!(
persisted.0.as_deref(),
Some("continuation_recovery_required")
);
assert_eq!(persisted.1.as_deref(), Some("reused_connection_lost"));
assert_eq!(persisted.2.as_deref(), Some("connection-observed"));
assert_eq!(persisted.3.as_deref(), Some("tcp_reset"));
assert_eq!(persisted.4, Some(12_000));
assert_eq!(persisted.5, Some(4_000));
database.close().await;
}
#[tokio::test]
async fn successful_new_chain_should_mark_recent_continuation_failure_recovered() {
let Some(database) = TestDatabase::create("execution_continuation_recovered").await else {
return;
};
let affinity_hash = "a".repeat(64);
let failure_completed_at = Utc::now() - Duration::seconds(8);
seed_continuation_failure(
&database.pool,
"req_continuation_failed",
&affinity_hash,
failure_completed_at,
)
.await;
seed_recovery_request(
&database.pool,
"req_continuation_recovery",
&affinity_hash,
failure_completed_at + Duration::seconds(4),
)
.await;
let store = PgExecutionStore::new(database.pool.clone());
ExecutionStore::finalize_model_request(
&store,
successful_core_finalization("req_continuation_recovery"),
)
.await
.expect("finalize successful recovery request");
type PersistedRecoveryObservation = (
Option<String>,
Option<chrono::DateTime<Utc>>,
i32,
Option<i64>,
Option<i64>,
);
let recovered: PersistedRecoveryObservation = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count,
recovery_retry_delay_ms, recovery_total_latency_ms
from model_requests where id = 'req_continuation_failed'",
)
.fetch_one(&database.pool)
.await
.expect("load recovered continuation failure");
assert_eq!(recovered.0.as_deref(), Some("req_continuation_recovery"));
assert!(recovered.1.is_some());
assert_eq!(recovered.2, 1);
assert_eq!(recovered.3, Some(4_000));
assert!(recovered.4.is_some_and(|latency| latency >= 4_000));
database.close().await;
}
#[tokio::test]
async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() {
let Some(database) = TestDatabase::create("execution_continuation_unresolved").await else {
return;
};
let affinity_hash = "c".repeat(64);
let failure_completed_at = Utc::now() - Duration::seconds(60);
seed_continuation_failure(
&database.pool,
"req_continuation_unresolved",
&affinity_hash,
failure_completed_at,
)
.await;
seed_recovery_request(
&database.pool,
"req_continuation_retry_failed",
&affinity_hash,
failure_completed_at + Duration::seconds(3),
)
.await;
let store = PgExecutionStore::new(database.pool.clone());
let mut failed = successful_core_finalization("req_continuation_retry_failed");
failed.outcome = ExecutionOutcome::Failed;
failed.client_status_code = Some(502);
failed.upstream_status_code = Some(500);
failed.error = Some(GatewayError::new(
GatewayErrorKind::UpstreamUnavailable,
"recovery request failed",
));
ExecutionStore::finalize_model_request(&store, failed)
.await
.expect("finalize failed recovery request");
let after_failed: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count
from model_requests where id = 'req_continuation_unresolved'",
)
.fetch_one(&database.pool)
.await
.expect("load unresolved continuation failure");
assert_eq!(after_failed, (None, None, 1));
seed_recovery_request(
&database.pool,
"req_continuation_retry_late",
&affinity_hash,
failure_completed_at + Duration::seconds(31),
)
.await;
ExecutionStore::finalize_model_request(
&store,
successful_core_finalization("req_continuation_retry_late"),
)
.await
.expect("finalize late recovery request");
let after_late: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count
from model_requests where id = 'req_continuation_unresolved'",
)
.fetch_one(&database.pool)
.await
.expect("load late unresolved continuation failure");
assert_eq!(after_late, (None, None, 1));
database.close().await;
}
async fn seed_continuation_failure(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
completed_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed continuation failure request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
continuation_affinity_hash = $2,
continuation_previous_response_id_hash = $3,
continuation_requested = true,
outcome = 'failed', error_kind = 'continuation_recovery_required',
continuation_unavailable_reason = 'reused_connection_lost',
started_at = $4 - interval '1 second', deadline_at = $4,
completed_at = $4
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind("b".repeat(64))
.bind(completed_at)
.execute(pool)
.await
.expect("finalize seeded continuation failure");
}
async fn seed_recovery_request(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
started_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed continuation recovery request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
continuation_affinity_hash = $2, continuation_requested = false,
started_at = $3, deadline_at = $3 + interval '1 minute'
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind(started_at)
.execute(pool)
.await
.expect("prepare seeded recovery request");
}
async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx::Error> {
sqlx::query(
"insert into model_requests (
@@ -186,6 +186,48 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
database.close().await;
}
#[tokio::test]
async fn usage_search_should_match_client_api_key_prefix() {
let Some(database) = TestDatabase::create("usage_client_api_key_search").await else {
return;
};
let now = Utc::now();
seed_observability_facts(&database.pool, now)
.await
.expect("seed observability facts");
let plaintext_key = format!("sk_{}", "K".repeat(43));
sqlx::query(
"insert into client_api_keys (id, name, key, enabled, created_at, updated_at)
values ('key_observe', 'usage search', $1, true, $2, $2)",
)
.bind(&plaintext_key)
.bind(now)
.execute(&database.pool)
.await
.expect("seed searchable client API key");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
for search in [plaintext_key.as_str(), &plaintext_key[..10]] {
let page = observability_repository(&database.pool)
.list_usage_records(UsageRecordQuery {
range,
filter: UsageRecordFilter {
search: Some(search.to_owned()),
..UsageRecordFilter::default()
},
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("search usage by client API key");
assert_eq!(page.total, 1);
assert_eq!(page.items[0].id, "req_observe_success");
}
database.close().await;
}
#[tokio::test]
async fn ops_search_should_treat_sql_wildcards_as_literals() {
let Some(database) = TestDatabase::create("ops_literal_wildcard_search").await else {
@@ -215,6 +257,84 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
database.close().await;
}
#[tokio::test]
async fn recovered_continuation_failure_should_be_hidden_from_default_business_metrics() {
let Some(database) = TestDatabase::create("observability_recovered_continuation").await else {
return;
};
let now = Utc::now();
seed_observability_facts(&database.pool, now)
.await
.expect("seed observability facts");
sqlx::query(
"update model_requests
set error_kind = 'continuation_recovery_required',
continuation_affinity_hash = $2,
continuation_previous_response_id_hash = $3,
continuation_requested = true,
continuation_unavailable_reason = 'reused_connection_lost',
recovery_request_id = 'req_observe_success', recovered_at = $1,
recovery_attempt_count = 1, recovery_retry_delay_ms = 4000,
recovery_total_latency_ms = 8000
where id = 'req_observe_failed'",
)
.bind(now)
.bind("a".repeat(64))
.bind("b".repeat(64))
.execute(&database.pool)
.await
.expect("mark continuation failure recovered");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
let repository = observability_repository(&database.pool);
let overview = repository
.usage_summary(range, UsageRecordFilter::default())
.await
.expect("usage summary without recovered intermediates");
assert_eq!(overview.requests.request_count, 2);
assert_eq!(overview.requests.failure_count, 0);
assert_eq!(overview.attempts.attempt_count, 2);
assert_eq!(overview.attempts.failure_count, 0);
assert_eq!(overview.providers[0].request_count, 2);
assert_eq!(overview.providers[0].failure_count, 0);
let diagnostics = repository
.usage_diagnostics(
range,
UsageRecordFilter::default(),
DiagnosticDimension::Account,
)
.await
.expect("diagnostics without recovered intermediates");
assert_eq!(diagnostics[0].request_count, 2);
assert_eq!(diagnostics[0].failure_count, 0);
let errors = repository
.list_ops_errors(OpsErrorQuery {
range,
filter: OpsErrorFilter::default(),
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("ops errors without recovered intermediates");
assert_eq!(errors.total, 0);
let retained: (String, Option<String>) = sqlx::query_as(
"select outcome, recovery_request_id
from model_requests where id = 'req_observe_failed'",
)
.fetch_one(&database.pool)
.await
.expect("load retained recovery audit row");
assert_eq!(
retained,
("failed".to_owned(), Some("req_observe_success".to_owned()))
);
database.close().await;
}
#[tokio::test]
async fn dashboard_account_metrics_should_partition_account_statuses() {
let Some(database) = TestDatabase::create("dashboard_account_metrics").await else {
@@ -762,9 +882,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_observe".to_owned()),
operation: Some("responses".to_owned()),
endpoint: Some("/v1/responses".to_owned()),
model: Some("upstream-model".to_owned()),
failure_kind: Some("rate_limited".to_owned()),
status_code: Some(429),
search: Some("req_observe_failed".to_owned()),
..admin_observability::OpsErrorFilter::default()
@@ -875,6 +875,7 @@ fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + chrono::Duration::seconds(30),
@@ -32,6 +32,12 @@ impl CodexSessionAffinity {
&self.key_hash
}
/// 返回可持久化的客户端作用域不透明会话关联值。
#[must_use]
pub(crate) fn persistence_hash(&self) -> &str {
self.key.expose_to_store()
}
#[must_use]
pub(crate) const fn anchor_source(&self) -> &'static str {
self.anchor_source
@@ -58,6 +64,20 @@ impl CodexSessionAffinity {
}
}
/// 将原始 response ID 投影为客户端作用域的不可逆关联值。
#[must_use]
pub(crate) fn derive_previous_response_id_hash(
previous_response_id: &str,
client_api_key_id: &ClientApiKeyId,
) -> String {
let mut hasher = Sha256::new();
hasher.update(b"codex-previous-response-observation-v1\0");
hasher.update(client_api_key_id.as_str().as_bytes());
hasher.update(b"\0");
hasher.update(previous_response_id.as_bytes());
hex::encode(hasher.finalize())
}
pub(crate) fn derive_codex_session_affinity(
request: &CodexResponsesRequest,
client_api_key_id: &ClientApiKeyId,
@@ -17,7 +17,7 @@ mod types;
pub(crate) use affinity::{
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
derive_codex_session_affinity,
derive_codex_session_affinity, derive_previous_response_id_hash,
};
pub(crate) use oauth::oauth_owner_ref;
pub(crate) use types::parse_access_token_expiration;
+1 -1
View File
@@ -30,7 +30,7 @@ use crate::transport::{CodexWebSocketPool, build_reqwest_client};
pub use config::{CodexWireProfileConfig, OpenAiConfig, OpenAiConfigError};
pub use provider::{
CodexProvider, CodexProviderConfigError, CodexProviderTransport, OFFICIAL_CODEX_BASE_PATH,
OFFICIAL_CODEX_BASE_URL,
OFFICIAL_CODEX_BASE_URL, openai_failure_affects_account_score,
};
pub mod credential;
@@ -22,11 +22,66 @@ impl CodexProvider {
),
ImageRequestKind::Edit => (CODEX_IMAGE_EDITS_PATH, self.image_edits_url.clone()),
};
let image_turn_id = image
.payload()
.context()
.get("image_turn_id")
.and_then(Value::as_str)
.map(str::to_owned);
self.execute_raw_json_endpoint(
context,
RawJsonEndpointRequest {
response_origin,
endpoint_path,
body: image.payload().body().clone(),
image_turn_id,
turn_metadata: None,
},
)
.await
}
pub(super) async fn execute_search(
&self,
search: &StandaloneSearchRequest,
candidate: &ProviderCandidate,
context: AttemptContext,
) -> Result<ProviderStream, ProviderError> {
if search.payload().protocol() != PROVIDER_NAME || candidate.upstream_model().is_some() {
return Err(provider_error(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::NotSent,
));
}
let turn_metadata = search
.payload()
.context()
.get("turn_metadata")
.and_then(Value::as_str)
.map(str::to_owned);
self.execute_raw_json_endpoint(
context,
RawJsonEndpointRequest {
response_origin: self.search_url.clone(),
endpoint_path: CODEX_ALPHA_SEARCH_PATH,
body: search.payload().body().clone(),
image_turn_id: None,
turn_metadata,
},
)
.await
}
async fn execute_raw_json_endpoint(
&self,
context: AttemptContext,
request: RawJsonEndpointRequest,
) -> Result<ProviderStream, ProviderError> {
let selection_started_at = Instant::now();
let lease = self
.selector
.select_for_provider_endpoint(&SelectCodexProviderEndpointCredential {
request_url: &response_origin,
request_url: &request.response_origin,
attempt: &context,
})
.await
@@ -48,18 +103,18 @@ impl CodexProvider {
account_selection_wait_ms,
lease.capacity_snapshot(),
));
let image_turn_id = image
.payload()
.context()
.get("image_turn_id")
.and_then(Value::as_str)
.map(str::to_owned);
// Standalone Provider 端点没有可证明的账号 owner;Search metadata 必须按
// 跨账号输入收敛到当前 lease,不能沿用下游声明的账号或 installation identity。
let turn_metadata = request.turn_metadata.as_deref().and_then(|metadata| {
crate::transport::request::scope_turn_metadata(metadata, lease.installation_id(), true)
});
let events = cold_json_response_stream(ColdJsonResponse {
client: self.client.clone(),
response_origin,
endpoint_path,
body: image.payload().body().clone(),
image_turn_id,
response_origin: request.response_origin,
endpoint_path: request.endpoint_path,
body: request.body,
image_turn_id: request.image_turn_id,
turn_metadata,
context,
selector: Arc::clone(&self.selector),
quota: Arc::clone(&self.quota),
@@ -68,13 +123,24 @@ impl CodexProvider {
});
let stream = ProviderStream::new(metadata, events, lease);
Ok(if allows_account_state_mutation {
stream.with_account_feedback(Arc::clone(&self.account_feedback))
stream.with_filtered_account_feedback(
Arc::clone(&self.account_feedback),
openai_failure_affects_account_score,
)
} else {
stream
})
}
}
struct RawJsonEndpointRequest {
response_origin: Url,
endpoint_path: &'static str,
body: Bytes,
image_turn_id: Option<String>,
turn_metadata: Option<String>,
}
pub(super) struct ColdResponse {
pub(super) client: CodexBackendClient,
pub(super) response_origin: Url,
@@ -101,6 +167,7 @@ pub(super) struct ColdJsonResponse {
pub(super) endpoint_path: &'static str,
pub(super) body: Bytes,
pub(super) image_turn_id: Option<String>,
pub(super) turn_metadata: Option<String>,
pub(super) context: AttemptContext,
pub(super) selector: Arc<CodexCredentialSelector>,
pub(super) quota: Arc<CodexCredentialQuotaService>,
@@ -276,6 +343,7 @@ pub(super) async fn create_json_attempt(
Some(installation_id),
);
request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret);
request_context.turn_metadata = request.turn_metadata.as_deref();
request_context.account_selection = account_selection;
tokio::select! {
biased;
@@ -379,7 +447,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
tracing::warn!(
account_id = %active_account.id(),
error = %error,
"Failed to persist OpenAI image response cookies"
"Failed to persist OpenAI provider endpoint response cookies"
);
}
}
@@ -2,6 +2,40 @@
use super::*;
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。
///
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
"service_unavailable_error",
];
fn is_openai_account_score_failure_reason(value: &str) -> bool {
let value = value.trim();
OPENAI_ACCOUNT_SCORE_FAILURE_REASONS
.iter()
.any(|reason| value.eq_ignore_ascii_case(reason))
}
fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> {
if let Some(code) = error.upstream_code() {
return Some(code.as_str());
}
let upstream = error.client_visible_upstream_error()?;
upstream.code().or_else(|| upstream.error_type())
}
/// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。
#[doc(hidden)]
pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason)
}
pub(super) struct MappedProviderFailure {
pub(super) error: ProviderError,
pub(super) websocket_transport_retryable: bool,
@@ -454,7 +488,10 @@ pub(super) fn websocket_client_failure_policy(
Some(WebSocketFailurePolicy::Budgeted)
}
CodexClientError::WebSocket(error)
if !matches!(error, CodexWebSocketExchangeError::InvalidRequest(_)) =>
if !matches!(
error.classified(),
CodexWebSocketExchangeError::InvalidRequest(_)
) =>
{
Some(WebSocketFailurePolicy::Budgeted)
}
@@ -581,15 +618,16 @@ pub(super) fn websocket_retry_backoff(retry_index: NonZeroU32) -> Duration {
Duration::from_millis(base_ms.saturating_mul(jitter_per_mille) / 1_000)
}
pub(super) fn continuation_replay_required_error() -> ProviderError {
pub(super) fn continuation_replay_required_error(reason: &'static str) -> ProviderError {
ProviderError::new(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
)
.with_continuation_unavailable_reason(reason)
.with_upstream_code(OpaqueUpstreamValue::new(
PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned(),
))
@@ -620,7 +658,7 @@ pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientErro
| CodexClientError::HttpJson(_)
| CodexClientError::StreamIdleTimeout { .. } => true,
CodexClientError::WebSocket(error) => !matches!(
error,
error.classified(),
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::Upstream(_)
| CodexWebSocketExchangeError::ContinuationUnavailable { .. }
@@ -666,12 +704,32 @@ pub(super) fn map_client_error(
_ => None,
};
let continuation_failure = match &error {
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::ConnectionBusy,
}) => Some(ContinuationFailure::Busy),
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
..
}) => Some(ContinuationFailure::HistoryUnavailable),
CodexClientError::WebSocket(error)
if error.continuation_unavailable_reason()
== Some(PreviousResponseUnavailableReason::ConnectionBusy) =>
{
Some(ContinuationFailure::Busy)
}
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
Some(ContinuationFailure::HistoryUnavailable)
}
_ => None,
};
let continuation_unavailable_reason = match &error {
CodexClientError::WebSocket(error) => error
.continuation_unavailable_reason()
.map(PreviousResponseUnavailableReason::as_str),
_ => None,
};
let connection_observation = match &error {
CodexClientError::WebSocket(error) => error.connection_observation().map(|observation| {
ProviderConnectionObservation::new(
observation.connection_id().to_string(),
observation.exit_reason().to_owned(),
observation.age_ms(),
observation.idle_ms(),
)
}),
_ => None,
};
let observation = observe_transport
@@ -718,10 +776,10 @@ pub(super) fn map_client_error(
send_state,
))
}
CodexClientError::WebSocket(
error @ CodexWebSocketExchangeError::ContinuationUnavailable { .. },
) => {
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error());
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error(
continuation_unavailable_reason.unwrap_or("scope_unavailable"),
));
if let Some(client_visible_error) = websocket_client_visible_error(&error) {
failure.error = failure
.error
@@ -744,7 +802,10 @@ pub(super) fn map_client_error(
"websocket_close_{close_code}"
)));
}
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
if matches!(
error.classified(),
CodexWebSocketExchangeError::ConnectionLimitReached
) {
failure.error = failure.error.with_upstream_code(OpaqueUpstreamValue::new(
WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(),
));
@@ -771,6 +832,11 @@ pub(super) fn map_client_error(
if let Some(raw) = raw_upstream_error {
failure.error = failure.error.with_raw_upstream_error(raw);
}
if let Some(connection_observation) = connection_observation {
failure.error = failure
.error
.with_connection_observation(connection_observation);
}
failure.observation = observation;
failure
}
@@ -790,7 +856,7 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
return ProviderDiagnostic::new(message);
}
ProviderDiagnostic::new(match error {
ProviderDiagnostic::new(match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_) => {
"OpenAI WebSocket request could not be constructed".to_owned()
}
@@ -847,6 +913,9 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
CodexWebSocketExchangeError::ClosedBeforeTerminal(_) => {
unreachable!("close-before-terminal errors are handled before the variant match")
}
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
})
}
@@ -866,7 +935,10 @@ fn websocket_raw_error(error: &CodexWebSocketExchangeError) -> Option<RawUpstrea
pub(super) fn websocket_client_visible_error(
error: &CodexWebSocketExchangeError,
) -> Option<ClientVisibleUpstreamError> {
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
if matches!(
error.classified(),
CodexWebSocketExchangeError::ConnectionLimitReached
) {
return Some(ClientVisibleUpstreamError::new(
"websocket connection limit reached",
Some(WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned()),
@@ -900,7 +972,12 @@ pub(super) fn map_upstream_failure(
.filter(|code| is_history_failure_code(code))
.map(|_| ContinuationFailure::HistoryUnavailable);
let send_state = upstream_send_state(failure.send_phase);
let mut error = provider_error(provider_error_kind(category), send_state);
let error_kind = if continuation_failure.is_some() {
ProviderErrorKind::ContinuationRecoveryRequired
} else {
provider_error_kind(category)
};
let mut error = provider_error(error_kind, send_state);
error = error.with_raw_upstream_error(RawUpstreamError::new(failure.raw_body.clone()));
if let Some(message) = failure.client_message.as_ref() {
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
@@ -939,7 +1016,8 @@ pub(super) fn map_upstream_failure(
.with_continuation_failure(continuation_failure)
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
);
)
.with_continuation_unavailable_reason("upstream_rejected");
}
if let Some(retry_after) = failure.retry_after_seconds.map(Duration::from_secs) {
error = error.with_retry_after(retry_after);
@@ -1064,8 +1142,8 @@ pub(super) fn account_failure(
}
}
pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
match error {
pub(super) fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::Connect(_)
| CodexWebSocketExchangeError::ConnectTimeout { .. }
@@ -1085,11 +1163,14 @@ pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) ->
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. }
| CodexWebSocketExchangeError::InitialEventTimeout { .. } => UpstreamSendState::Ambiguous,
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
}
}
pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
match error {
pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::InvalidSse(_)
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => ProviderErrorKind::Protocol,
@@ -1113,6 +1194,9 @@ pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) ->
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. } => {
ProviderErrorKind::Transport
}
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
}
}
@@ -12,15 +12,15 @@ use futures::{StreamExt, future::BoxFuture};
use gateway_core::account::{AccountFeedbackStats, ProviderAccount};
use gateway_core::engine::continuation::{ContinuationBinding, NativeContinuationScope};
use gateway_core::engine::provider::{
EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration,
ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation,
ProviderSelectionObservation, ProviderStream,
ContinuationRequestObservation, EventStream, Provider, ProviderCallMetadata,
ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRequest,
ProviderRequestObservation, ProviderSelectionObservation, ProviderStream,
};
use gateway_core::engine::{AttemptContext, AttemptTransport, ContinuationAttempt};
use gateway_core::error::{
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderDiagnostic, ProviderError,
ProviderErrorKind, RawUpstreamError,
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderConnectionObservation,
ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError,
};
use gateway_core::event::{
FinishReason, GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader,
@@ -29,7 +29,8 @@ use gateway_core::event::{
};
use gateway_core::lifecycle::CancellationToken;
use gateway_core::operation::{
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind, ProviderSessionState,
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
ProviderSessionState, StandaloneSearchRequest,
};
use gateway_core::provider_ports::ProviderSessionAffinityKey;
use gateway_core::routing::{
@@ -58,6 +59,7 @@ use crate::credential::{
CodexQuotaRefreshPolicy, CodexSessionAffinity, CredentialSelectionError, RuntimeCodexCookie,
SelectCodexCredential, SelectCodexProviderEndpointCredential,
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
derive_previous_response_id_hash,
};
use crate::session_transport::CodexSessionTransportFallbacks;
use crate::transport::canonical::{
@@ -84,11 +86,11 @@ use crate::transport::session::CodexSessionIdentity;
use crate::transport::usage::normalize_service_tier;
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
use crate::transport::{
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
CodexAccountSelectionTelemetry, CodexBackendClient, CodexBackendJsonResponse,
CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError, CodexRateLimitUpdates,
CodexRequestContext, CodexResponseMetadata, CodexTransportMetrics, CodexUpstreamDiagnostics,
CodexWebSocketPool, endpoint_url,
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
CodexBackendJsonResponse, CodexBackendStreamingResponse, CodexBackendTransport,
CodexClientError, CodexRateLimitUpdates, CodexRequestContext, CodexResponseMetadata,
CodexTransportMetrics, CodexUpstreamDiagnostics, CodexWebSocketPool, endpoint_url,
};
mod execution;
@@ -97,6 +99,8 @@ mod observation;
mod workers;
use execution::*;
#[doc(hidden)]
pub use failure::openai_failure_affects_account_score;
use failure::*;
use observation::*;
pub(crate) use workers::worker_contributions;
@@ -139,6 +143,7 @@ pub struct CodexProvider {
responses_url: Url,
image_generations_url: Url,
image_edits_url: Url,
search_url: Url,
session_identity: Option<CodexSessionIdentity>,
session_transport_fallbacks: CodexSessionTransportFallbacks,
stream_max_retries: u32,
@@ -165,6 +170,8 @@ impl CodexProvider {
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
let image_edits_url = Url::parse(&endpoint_url(&base_url, CODEX_IMAGE_EDITS_PATH))
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
let search_url = Url::parse(&endpoint_url(&base_url, CODEX_ALPHA_SEARCH_PATH))
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
let client =
CodexBackendClient::new(http, base_url, profile).with_websocket_pool(websocket_pool);
Ok(Self {
@@ -176,6 +183,7 @@ impl CodexProvider {
responses_url,
image_generations_url,
image_edits_url,
search_url,
session_identity: None,
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
stream_max_retries,
@@ -208,23 +216,35 @@ impl Provider for CodexProvider {
self.catalog.catalog_generation()
}
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
operation: &Operation,
client_api_key_id: &gateway_core::policy::ClientApiKeyId,
) -> ProviderRequestObservation {
let Operation::Generate(request) = operation else {
return ProviderRequestObservation::default();
};
let (semantics, reasoning_effort) = encode_generate_request(request, "observability")
.map(|encoded| {
let semantics = encoded.semantics();
let reasoning_effort = semantics.reasoning_effort.clone();
(semantics, reasoning_effort)
})
.unwrap_or_default();
let Ok(encoded) = encode_generate_request(request, "observability") else {
return ProviderRequestObservation::default();
};
let semantics = encoded.semantics();
let reasoning_effort = semantics.reasoning_effort.clone();
let previous_response_id = encoded.previous_response_id();
let continuation = ContinuationRequestObservation {
affinity_hash: derive_codex_session_affinity(&encoded, client_api_key_id)
.map(|affinity| affinity.persistence_hash().to_owned()),
previous_response_id_hash: previous_response_id.map(|response_id| {
derive_previous_response_id_hash(response_id, client_api_key_id)
}),
requested: previous_response_id.is_some(),
};
ProviderRequestObservation {
reasoning_effort,
reasoning_preset: semantics.reasoning_preset.map(str::to_owned),
request_kind: semantics.request_kind,
subagent_kind: semantics.subagent_kind,
compact: semantics.compact,
continuation,
}
}
@@ -268,6 +288,9 @@ impl Provider for CodexProvider {
if let Operation::GenerateImage(image) = request.operation() {
return self.execute_image(image, candidate, context).await;
}
if let Operation::Search(search) = request.operation() {
return self.execute_search(search, candidate, context).await;
}
let Operation::Generate(generate) = request.operation() else {
return Err(provider_error(
ProviderErrorKind::Unsupported,
@@ -385,7 +408,7 @@ impl Provider for CodexProvider {
continuation_recovery_action = "stop_proxy_recovery",
"OpenAI connection-local continuation replay was rejected before send"
);
return Err(continuation_replay_required_error());
return Err(continuation_replay_required_error("scope_unavailable"));
}
let previous_response_scope = match context.continuation_attempt() {
ContinuationAttempt::Native => native_scope,
@@ -410,7 +433,7 @@ impl Provider for CodexProvider {
if upstream_request.previous_response_id().is_some()
&& !account_scope.can_reuse_account_state()
{
return Err(continuation_replay_required_error());
return Err(continuation_replay_required_error("scope_unavailable"));
}
scope_request_to_account(
&mut upstream_request,
@@ -421,7 +444,7 @@ impl Provider for CodexProvider {
let requested_transport = selected_transport(&upstream_request);
let sticky_http_fallback = context.transport() == AttemptTransport::Default
&& requested_transport == CodexProviderTransport::PreferWebSocket
&& !requirement.requires_websocket()
&& requirement.allows_sticky_http_fallback()
&& session_affinity.as_ref().is_some_and(|affinity| {
self.session_transport_fallbacks
.is_http_only(affinity.key())
@@ -502,7 +525,10 @@ impl Provider for CodexProvider {
});
let stream = ProviderStream::new(metadata, events, lease);
Ok(if allows_account_state_mutation {
stream.with_account_feedback(Arc::clone(&self.account_feedback))
stream.with_filtered_account_feedback(
Arc::clone(&self.account_feedback),
openai_failure_affects_account_score,
)
} else {
stream
})
@@ -391,7 +391,12 @@ pub(super) fn codex_error_observation(
)?
.with_status_code(status.as_u16());
}
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
CodexClientError::WebSocket(error)
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
{
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
unreachable!("websocket error was checked above")
};
observation = observation.with_status_code(upstream.status_code);
if let Some(request_id) = upstream.diagnostics.request_id.as_deref() {
observation =
@@ -313,7 +313,12 @@ impl CodexClientError {
rate_limit_headers,
*send_phase,
)),
Self::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
Self::WebSocket(error)
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
{
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
unreachable!("websocket error was checked above")
};
Some(CodexUpstreamFailure::from_response(
StatusCode::from_u16(upstream.status_code).unwrap_or(StatusCode::BAD_GATEWAY),
&upstream.body,
@@ -848,7 +853,7 @@ pub(super) fn websocket_success_decision(
pub(super) fn http_fallback_decision(
error: &CodexWebSocketExchangeError,
) -> CodexTransportDecision {
match error {
match error.classified() {
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
CodexTransportDecision::Http2WebSocketBudgetExhausted
}
@@ -39,6 +39,11 @@ impl CodexBackendClient {
context.installation_id,
)?;
insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id);
insert_optional_protocol_header(
&mut headers,
"x-codex-turn-metadata",
context.turn_metadata,
);
headers.insert(
HeaderName::from_static("x-client-request-id"),
HeaderValue::from_str(context.request_id)?,
@@ -58,6 +58,7 @@ const PERSISTABLE_UPSTREAM_CODES: &[&str] = &[
"rate_limit_exceeded",
"rate_limit_reached",
"refresh_token_invalidated",
"server_error",
"server_is_overloaded",
"slow_down",
"token_expired",
@@ -4,6 +4,8 @@ pub const CODEX_RESPONSES_PATH: &str = "/codex/responses";
pub const CODEX_IMAGE_GENERATIONS_PATH: &str = "/codex/images/generations";
/// `/codex/images/edits`
pub const CODEX_IMAGE_EDITS_PATH: &str = "/codex/images/edits";
/// `/codex/alpha/search`
pub const CODEX_ALPHA_SEARCH_PATH: &str = "/codex/alpha/search";
/// `/api/codex/usage`
pub const CODEX_USAGE_API_PATH: &str = "/api/codex/usage";
/// `/wham/usage`
@@ -36,8 +36,8 @@ pub use self::{
},
diagnostics::{CodexUpstreamDiagnostics, CodexUpstreamSendPhase},
endpoints::{
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
},
@@ -162,6 +162,11 @@ impl TransportRequirement {
matches!(self, Self::NewChain)
}
/// 会话级 sticky 状态是否可以把首选 WebSocket 改为 HTTP。
pub fn allows_sticky_http_fallback(self) -> bool {
matches!(self, Self::NewChain)
}
/// 用于审计与遥测的稳定名称。
pub fn as_str(self) -> &'static str {
match self {
@@ -193,8 +198,7 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
TransportRequirement::ExternalUnknown
}
},
None if request.use_websocket => TransportRequirement::NewChain,
None => TransportRequirement::HttpRequired,
None => TransportRequirement::NewChain,
}
}
@@ -645,10 +649,9 @@ impl CodexResponsesRequest {
/// 返回请求语义与 child transport 隔离所用的子代理区分值。
///
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
/// 传递给上游。它不拆分根会话的账号首选项;`thread_spawn` 仍用它派生独立
/// WebSocket/continuation transport identity。
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;兼容客户端也可通过
/// `client_metadata.x-openai-subagent` 声明同一语义。它不拆分根会话的账号首选项;
/// `thread_spawn` 仍用它派生独立 WebSocket/continuation transport identity。
pub fn subagent_kind(&self) -> Option<String> {
self.semantics()
.subagent_kind
@@ -427,7 +427,11 @@ fn metadata_string(request: &CodexResponsesRequest, key: &str) -> Option<String>
.map(ToOwned::to_owned)
}
fn scope_turn_metadata(raw: &str, installation_id: &str, cross_account: bool) -> Option<String> {
pub(crate) fn scope_turn_metadata(
raw: &str,
installation_id: &str,
cross_account: bool,
) -> Option<String> {
let Ok(Value::Object(mut metadata)) = serde_json::from_str::<Value>(raw) else {
return (!cross_account).then(|| raw.to_owned());
};
@@ -24,7 +24,7 @@ use super::{
WebSocketPoolConnectLease, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
WebSocketPoolLease,
},
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket},
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket, transport_metric_reason},
};
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
@@ -188,9 +188,11 @@ pub(crate) async fn prepare_response_create_request_with_pool(
}
}
}
WebSocketPoolAcquire::Bypass(reason) => {
Err(continuation_unavailable(bypass_unavailable_reason(reason)))
}
WebSocketPoolAcquire::ContinuationLost(observation) => Err(continuation_unavailable(
PreviousResponseUnavailableReason::ReusedConnectionLost,
)
.with_connection_observation(observation)),
WebSocketPoolAcquire::Bypass(reason) => Err(bypass_unavailable_error(reason)),
}
}
@@ -369,6 +371,7 @@ fn start_pooled_websocket_connect(
}
}
Err(error) => {
let connection_exit_reason = opening_failure_reason(&error);
let error_message = error.to_string();
// 先交付 opening 原始错误,避免连接池清理侵占前台 fast-path 预算。
let foreground_waiting = sender.send(Err(error)).is_ok();
@@ -378,6 +381,7 @@ fn start_pooled_websocket_connect(
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(started_at.elapsed()),
foreground_waiting,
connection_exit_reason,
error = %error_message,
ws_preconnect_outcome = "failed",
"WebSocket pool connect finished"
@@ -392,6 +396,21 @@ fn start_pooled_websocket_connect(
}
}
fn opening_failure_reason(error: &CodexWebSocketExchangeError) -> &'static str {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_) => "request_build_failure",
CodexWebSocketExchangeError::Connect(error)
| CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
CodexWebSocketExchangeError::ConnectTimeout { .. }
| CodexWebSocketExchangeError::FastPathTimeout { .. } => "connect_timeout",
CodexWebSocketExchangeError::Upstream(_) => "upgrade_rejected",
CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy => "connect_suppressed",
CodexWebSocketExchangeError::SharedConnectFailed => "shared_connect_failed",
_ => "connect_failure",
}
}
fn duration_millis_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_millis())
.unwrap_or(u64::MAX)
@@ -489,16 +508,16 @@ async fn wait_for_shared_connect(
}
}
fn bypass_unavailable_reason(
reason: WebSocketPoolBypassReason,
) -> PreviousResponseUnavailableReason {
fn bypass_unavailable_error(reason: WebSocketPoolBypassReason) -> CodexWebSocketExchangeError {
match reason {
WebSocketPoolBypassReason::Busy => PreviousResponseUnavailableReason::ConnectionBusy,
WebSocketPoolBypassReason::Busy => {
continuation_unavailable(PreviousResponseUnavailableReason::ConnectionBusy)
}
WebSocketPoolBypassReason::Disabled | WebSocketPoolBypassReason::Cap => {
PreviousResponseUnavailableReason::PoolUnavailable
continuation_unavailable(PreviousResponseUnavailableReason::PoolUnavailable)
}
WebSocketPoolBypassReason::ContinuationNotFound => {
PreviousResponseUnavailableReason::FreshConnectionRequired
continuation_unavailable(PreviousResponseUnavailableReason::FreshConnectionRequired)
}
}
}
@@ -527,8 +546,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
created_at,
} = connection;
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
discard_after_send(websocket, lease).await;
return Err(post_send_ambiguous(error));
let observation = websocket
.observation()
.with_exit_reason("outbound_transport_error");
discard_after_send(websocket, lease, observation.clone()).await;
return Err(post_send_ambiguous(
error.with_connection_observation(observation),
));
}
let connection_local_available = lease.is_some();
let pool_return = lease.map(|lease| WebSocketStreamPoolReturn {
@@ -548,9 +572,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
Ok(exchange)
}
async fn discard_after_send(websocket: PumpedWebSocket, lease: Option<WebSocketPoolLease>) {
async fn discard_after_send(
websocket: PumpedWebSocket,
lease: Option<WebSocketPoolLease>,
observation: super::pump::WebSocketConnectionObservation,
) {
if let Some(lease) = lease {
lease.discard().await;
lease.discard_with_observation(observation).await;
}
websocket.close().await;
}
@@ -11,6 +11,7 @@ use crate::transport::diagnostics::CodexUpstreamDiagnostics;
use crate::transport::diagnostics::CodexUpstreamSendPhase;
use super::PreviousResponseUnavailableReason;
use super::pump::WebSocketConnectionObservation;
/// Responses WebSocket 交互错误。
#[derive(Debug, Error)]
@@ -101,6 +102,13 @@ pub enum CodexWebSocketExchangeError {
/// 首个上游事件超时时长。
timeout: Duration,
},
/// 将一个已分类交互错误与物理连接生命周期快照绑定。
#[error("{source}")]
ConnectionObserved {
observation: WebSocketConnectionObservation,
#[source]
source: Box<CodexWebSocketExchangeError>,
},
}
/// 上游在 terminal 事件前发送的 WebSocket close 信息。
@@ -240,7 +248,9 @@ impl CodexWebSocketExchangeError {
)
}
pub(crate) fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
/// 返回错误链中的上游终态前 Close 帧(若存在)。
#[must_use]
pub fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
match self {
Self::ClosedBeforeTerminal(close) => Some(close),
Self::PostSendAmbiguous {
@@ -250,7 +260,54 @@ impl CodexWebSocketExchangeError {
| Self::ReusedConnectionDiedBeforeFirstEvent {
source: Some(source),
..
} => source.close_before_terminal(),
}
| Self::ConnectionObserved { source, .. } => source.close_before_terminal(),
_ => None,
}
}
pub(crate) fn with_connection_observation(
self,
observation: WebSocketConnectionObservation,
) -> Self {
match self {
Self::ConnectionObserved { .. } => self,
source => Self::ConnectionObserved {
observation,
source: Box::new(source),
},
}
}
/// 返回错误关联的物理 WebSocket 连接生命周期快照。
#[must_use]
pub fn connection_observation(&self) -> Option<&WebSocketConnectionObservation> {
match self {
Self::ConnectionObserved { observation, .. } => Some(observation),
Self::PostSendAmbiguous {
source: Some(source),
..
}
| Self::ReusedConnectionDiedBeforeFirstEvent {
source: Some(source),
..
} => source.connection_observation(),
_ => None,
}
}
/// 返回去掉观测 wrapper 后的原始分类错误。
pub(crate) fn classified(&self) -> &Self {
match self {
Self::ConnectionObserved { source, .. } => source.classified(),
_ => self,
}
}
#[must_use]
pub fn continuation_unavailable_reason(&self) -> Option<PreviousResponseUnavailableReason> {
match self.classified() {
Self::ContinuationUnavailable { reason } => Some(*reason),
_ => None,
}
}
@@ -277,6 +334,9 @@ impl CodexWebSocketExchangeError {
/// opening 阶段只有明确的 transport 可用性失败才能切到同账号 HTTP。
pub(in crate::transport) fn allows_pre_send_http_fallback(&self) -> bool {
if let Self::ConnectionObserved { source, .. } = self {
return source.allows_pre_send_http_fallback();
}
matches!(
self,
Self::Connect(_)
@@ -291,7 +351,7 @@ impl CodexWebSocketExchangeError {
/// 精确连接状态不可用属于本地池路由事实,不消耗上游 WS 重试预算。
pub(in crate::transport) fn requires_immediate_pool_fallback(&self) -> bool {
matches!(self, Self::ContinuationUnavailable { .. })
matches!(self.classified(), Self::ContinuationUnavailable { .. })
}
/// 首个业务事件交付前,普通 WebSocket 失败可切到同账号 HTTP。
@@ -302,6 +362,7 @@ impl CodexWebSocketExchangeError {
source: Some(source),
..
} => source.allows_pre_delivery_http_fallback(),
Self::ConnectionObserved { source, .. } => source.allows_pre_delivery_http_fallback(),
_ => true,
}
}
@@ -11,7 +11,9 @@ use super::super::{
CodexWebSocketConnectionMetadata, PooledWebSocketConnection, WebSocketContinuationState,
WebSocketPoolLease,
},
pump::{PumpExitReason, PumpedWebSocket},
pump::{
PumpExitReason, PumpedWebSocket, WebSocketConnectionObservation, transport_metric_reason,
},
};
use super::io::{next_websocket_message, receive_idle_timeout, reused_stream_receive_error};
use super::reducer::{ExchangeAction, WebSocketTerminalKind, reduce_websocket_event};
@@ -173,12 +175,22 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let message = match message {
Ok(message) => message,
Err(error) => {
let error = match error {
CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout }
if !saw_upstream_activity =>
{
CodexWebSocketExchangeError::InitialEventTimeout { timeout }
}
error => error,
};
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -197,6 +209,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
text.to_string()
}
tungstenite::Message::Binary(_) => {
let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent;
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
@@ -204,24 +218,26 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
)
.await;
let _ = tx
.send(Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent))
.send(Err(error.with_connection_observation(observation)))
.await;
return;
}
tungstenite::Message::Close(frame) => {
let websocket_connection_id = websocket.connection_id();
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
websocket_connection_id,
frame.as_ref().map(|frame| u16::from(frame.code)),
frame.map(|frame| frame.reason.to_string()),
last_event_type.clone(),
);
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -235,12 +251,19 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
Ok(reduced) => reduced,
Err(error) => {
let observation =
(!matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)))
.then(|| connection_observation(&websocket, &error));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
)
.await;
let error = match observation {
Some(observation) => error.with_connection_observation(observation),
None => error,
};
let _ = tx.send(Err(error)).await;
return;
}
@@ -301,12 +324,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
.exit_reason()
.and_then(|reason| reason.upstream_close().cloned())
.map(|close| close.with_connection_id(websocket_connection_id));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = match upstream_close {
Some(close) => CodexWebSocketExchangeError::ClosedBeforeTerminal(
close.with_last_event_type(last_event_type),
@@ -318,6 +335,14 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
last_event_type,
),
};
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -353,6 +378,12 @@ async fn discard_stream_websocket(
reason: StreamWebSocketDiscardReason,
) {
let websocket_connection_id = websocket.connection_id();
let observation = websocket.observation();
let tombstone_observation = if observation.exit_reason() == "running" {
observation.clone().with_exit_reason(reason.as_str())
} else {
observation.clone()
};
let pump_exit = websocket.exit_reason();
let pump_exit_reason = pump_exit
.as_ref()
@@ -367,11 +398,45 @@ async fn discard_stream_websocket(
reason = reason.as_str(),
pump_exit_reason,
pump_exit_detail,
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pooled = pool_return.is_some(),
"Discarding Responses WebSocket stream"
);
if let Some(pool_return) = pool_return {
pool_return.lease.discard().await;
pool_return
.lease
.discard_with_observation(tombstone_observation)
.await;
}
websocket.close().await;
}
fn connection_observation(
websocket: &PumpedWebSocket,
error: &CodexWebSocketExchangeError,
) -> WebSocketConnectionObservation {
let observation = websocket.observation();
if observation.exit_reason() != "running" {
return observation;
}
observation.with_exit_reason(exchange_exit_reason(error))
}
fn exchange_exit_reason(error: &CodexWebSocketExchangeError) -> &'static str {
match error.classified() {
CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
CodexWebSocketExchangeError::ClosedBeforeTerminal(close) => {
if close.code() == Some(1000) {
"normal_close"
} else {
"upstream_close"
}
}
CodexWebSocketExchangeError::ReceiveIdleTimeout { .. } => "receive_idle_timeout",
CodexWebSocketExchangeError::InitialEventTimeout { .. } => "initial_event_timeout",
CodexWebSocketExchangeError::UnexpectedBinaryEvent => "unexpected_binary_event",
_ => "exchange_failure",
}
}
@@ -33,6 +33,7 @@ pub use self::{
CodexWebSocketPool, CodexWebSocketPoolConfig, CodexWebSocketPoolKey,
WebSocketPoolBypassReason, WebSocketPoolDecision,
},
pump::WebSocketConnectionObservation,
};
pub(crate) use self::{
coordinator::{
@@ -70,16 +70,23 @@ pub enum PreviousResponseUnavailableReason {
UpstreamRejected,
}
impl fmt::Display for PreviousResponseUnavailableReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
impl PreviousResponseUnavailableReason {
#[must_use]
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::PoolUnavailable => "pool_unavailable",
Self::FreshConnectionRequired => "fresh_connection_required",
Self::ConnectionBusy => "connection_busy",
Self::LatestResponseMismatch => "latest_response_mismatch",
Self::ReusedConnectionLost => "reused_connection_lost",
Self::UpstreamRejected => "upstream_rejected",
})
}
}
}
impl fmt::Display for PreviousResponseUnavailableReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
@@ -14,6 +14,7 @@ use uuid::Uuid;
use super::state::{CodexWebSocketPoolKey, PooledWebSocketConnection, WebSocketPoolReservation};
use super::{CodexWebSocketPool, WebSocketPoolBypassReason};
use crate::transport::websocket::pump::WebSocketConnectionObservation;
pub(crate) enum WebSocketPoolAcquire {
Reused {
@@ -22,6 +23,7 @@ pub(crate) enum WebSocketPoolAcquire {
},
Connect(WebSocketPoolConnectLease),
Wait(WebSocketPoolConnectWaiter),
ContinuationLost(WebSocketConnectionObservation),
Bypass(WebSocketPoolBypassReason),
}
@@ -182,10 +184,12 @@ impl WebSocketPoolLease {
self.armed = false;
}
pub(crate) async fn discard(mut self) {
pub(crate) async fn discard_with_observation(
mut self,
observation: WebSocketConnectionObservation,
) {
self.pool
.discard_reserved(&self.key, self.reservation.id)
.await;
.discard_reserved_with_observation(&self.key, self.reservation.id, observation);
self.armed = false;
}
}
@@ -19,6 +19,7 @@ use self::state::{
close_pooled_connections,
};
use super::pump::PumpKeepalive;
use super::pump::WebSocketConnectionObservation;
pub use self::state::CodexWebSocketPoolKey;
pub(crate) use self::{
@@ -154,6 +155,7 @@ impl CodexWebSocketPool {
let mut connections_to_close = Vec::new();
let acquire = {
let mut state = self.lock_state();
let mut continuation_loss = None;
if !self.config.enabled || state.shutting_down {
return WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::Disabled);
}
@@ -163,9 +165,14 @@ impl CodexWebSocketPool {
&& slot.latest_response_id() == Some(response_id))
.then(|| candidate.clone())
}) else {
return WebSocketPoolAcquire::Bypass(
WebSocketPoolBypassReason::ContinuationNotFound,
);
return state
.continuation_loss(key, response_id, tokio::time::Instant::now())
.map_or(
WebSocketPoolAcquire::Bypass(
WebSocketPoolBypassReason::ContinuationNotFound,
),
WebSocketPoolAcquire::ContinuationLost,
);
};
key
} else {
@@ -188,9 +195,9 @@ impl CodexWebSocketPool {
};
// 零成本探活:后台 pump 已实时感知连接死亡(RST/Close/EOF/失活),
// 复用前只需读取 is_closed 标志,避免复用到静默死连接后卡到超时。
if connection.created_at.elapsed() < self.config.max_age
&& !connection.websocket.is_closed()
{
let expired = connection.created_at.elapsed() >= self.config.max_age;
let closed = connection.websocket.is_closed();
if !expired && !closed {
let lease = WebSocketPoolLease::reserve(
self.clone(),
key.clone(),
@@ -202,12 +209,31 @@ impl CodexWebSocketPool {
);
return WebSocketPoolAcquire::Reused { connection, lease };
}
let observation = if expired && !closed {
connection
.websocket
.observation()
.with_exit_reason("max_age_expired")
} else {
connection.websocket.observation()
};
state.remember_continuation_loss(
&key,
connection.continuation.latest_response_id(),
observation.clone(),
tokio::time::Instant::now(),
);
connections_to_close.push(*connection);
if required_response_id.is_some() {
continuation_loss = Some(observation);
}
}
None => {}
}
let acquire = if required_response_id.is_some() {
let acquire = if let Some(observation) = continuation_loss {
WebSocketPoolAcquire::ContinuationLost(observation)
} else if required_response_id.is_some() {
WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::ContinuationNotFound)
} else {
let connect_permit = self.connect_semaphore.clone().try_acquire_owned().ok();
@@ -259,6 +285,17 @@ impl CodexWebSocketPool {
if reservation.id == reservation_id
);
if owns_reservation && (expired || state.shutting_down || !self.config.enabled) {
if expired && let Some(connection) = connection.as_ref() {
state.remember_continuation_loss(
key,
connection.continuation.latest_response_id(),
connection
.websocket
.observation()
.with_exit_reason("max_age_expired"),
tokio::time::Instant::now(),
);
}
state.slots.remove(key);
} else if owns_reservation && let Some(connection) = connection.take() {
state.slots.insert(
@@ -274,10 +311,6 @@ impl CodexWebSocketPool {
}
}
async fn discard_reserved(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
self.discard_reserved_now(key, reservation_id);
}
fn discard_reserved_now(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
let mut state = self.lock_state();
if matches!(
@@ -288,6 +321,28 @@ impl CodexWebSocketPool {
}
}
fn discard_reserved_with_observation(
&self,
key: &CodexWebSocketPoolKey,
reservation_id: Uuid,
observation: WebSocketConnectionObservation,
) {
let mut state = self.lock_state();
let latest_response_id = match state.slots.get(key) {
Some(WebSocketPoolSlot::Busy(reservation)) if reservation.id == reservation_id => {
reservation.latest_response_id.clone()
}
_ => return,
};
state.remember_continuation_loss(
key,
latest_response_id.as_deref(),
observation,
tokio::time::Instant::now(),
);
state.slots.remove(key);
}
async fn finish_connect_reserved(
&self,
key: &CodexWebSocketPoolKey,
@@ -1,6 +1,9 @@
//! WebSocket 连接池状态和值对象。
use std::{collections::HashMap, time::Duration};
use std::{
collections::{HashMap, VecDeque},
time::Duration,
};
use sha2::{Digest, Sha256};
use tokio::{sync::watch, time::Instant};
@@ -10,7 +13,7 @@ use uuid::Uuid;
use super::super::super::{
diagnostics::CodexUpstreamDiagnostics, response_meta::CodexResponseMetadata,
};
use super::super::pump::PumpedWebSocket;
use super::super::pump::{PumpedWebSocket, WebSocketConnectionObservation};
use super::lease::WebSocketPoolConnectOutcome;
/// WebSocket 连接池 key。
@@ -67,12 +70,72 @@ impl CodexWebSocketPoolKey {
}
}
const CONTINUATION_TOMBSTONE_TTL: Duration = Duration::from_mins(30);
const MAX_CONTINUATION_TOMBSTONES: usize = 4_096;
#[derive(Default)]
pub(super) struct WebSocketPoolState {
pub(super) slots: HashMap<CodexWebSocketPoolKey, WebSocketPoolSlot>,
continuation_tombstones: VecDeque<WebSocketContinuationTombstone>,
pub(super) shutting_down: bool,
}
struct WebSocketContinuationTombstone {
key: CodexWebSocketPoolKey,
response_id: String,
observation: WebSocketConnectionObservation,
expires_at: Instant,
}
impl WebSocketPoolState {
pub(super) fn remember_continuation_loss(
&mut self,
key: &CodexWebSocketPoolKey,
response_id: Option<&str>,
observation: WebSocketConnectionObservation,
now: Instant,
) {
let Some(response_id) = response_id else {
return;
};
self.prune_continuation_tombstones(now);
self.continuation_tombstones.retain(|tombstone| {
!(tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id)
});
while self.continuation_tombstones.len() >= MAX_CONTINUATION_TOMBSTONES {
self.continuation_tombstones.pop_front();
}
self.continuation_tombstones
.push_back(WebSocketContinuationTombstone {
key: key.clone(),
response_id: response_id.to_owned(),
observation,
expires_at: now + CONTINUATION_TOMBSTONE_TTL,
});
}
pub(super) fn continuation_loss(
&mut self,
key: &CodexWebSocketPoolKey,
response_id: &str,
now: Instant,
) -> Option<WebSocketConnectionObservation> {
self.prune_continuation_tombstones(now);
self.continuation_tombstones
.iter()
.rev()
.find(|tombstone| {
tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id
})
.map(|tombstone| tombstone.observation.clone())
}
fn prune_continuation_tombstones(&mut self, now: Instant) {
self.continuation_tombstones
.retain(|tombstone| tombstone.expires_at > now);
}
}
#[derive(Clone)]
pub(crate) struct CodexWebSocketConnectionMetadata {
pub(crate) turn_state: Option<String>,
@@ -174,7 +174,23 @@ impl CodexWebSocketPool {
.collect::<Vec<_>>();
for key in keys {
match state.slots.remove(&key) {
Some(WebSocketPoolSlot::Idle { connection, .. }) => close.push(*connection),
Some(WebSocketPoolSlot::Idle { connection, .. }) => {
let observation = if connection.websocket.is_closed() {
connection.websocket.observation()
} else {
connection
.websocket
.observation()
.with_exit_reason("max_age_expired")
};
state.remember_continuation_loss(
&key,
connection.continuation.latest_response_id(),
observation,
now,
);
close.push(*connection);
}
Some(WebSocketPoolSlot::Busy(reservation)) => {
tracing::warn!(
account_id = key.account_id(),
@@ -89,6 +89,7 @@ pub(crate) enum PumpExitReason {
LocalClose,
OutboundTransportError {
message: String,
metric_reason: &'static str,
},
UpstreamCloseFrame {
close: Option<CodexWebSocketCloseError>,
@@ -96,6 +97,7 @@ pub(crate) enum PumpExitReason {
UpstreamEof,
InboundTransportError {
message: String,
metric_reason: &'static str,
},
PongTimeout {
timeout: Duration,
@@ -105,6 +107,7 @@ pub(crate) enum PumpExitReason {
},
KeepaliveTransportError {
message: String,
metric_reason: &'static str,
},
MessageReceiverClosed,
}
@@ -127,9 +130,9 @@ impl PumpExitReason {
pub(crate) fn detail(&self) -> Option<String> {
match self {
Self::OutboundTransportError { message }
| Self::InboundTransportError { message }
| Self::KeepaliveTransportError { message } => Some(message.clone()),
Self::OutboundTransportError { message, .. }
| Self::InboundTransportError { message, .. }
| Self::KeepaliveTransportError { message, .. } => Some(message.clone()),
Self::UpstreamCloseFrame { close } => close.as_ref().map(ToString::to_string),
Self::PongTimeout { timeout } | Self::LivenessTimeout { timeout } => {
Some(format!("{timeout:?}"))
@@ -141,6 +144,27 @@ impl PumpExitReason {
}
}
pub(crate) fn metric_reason(&self) -> &'static str {
match self {
Self::CommandChannelClosed => "command_channel_closed",
Self::LocalClose => "local_close",
Self::OutboundTransportError { metric_reason, .. }
| Self::InboundTransportError { metric_reason, .. }
| Self::KeepaliveTransportError { metric_reason, .. } => metric_reason,
Self::UpstreamCloseFrame { close } => {
if close.as_ref().and_then(CodexWebSocketCloseError::code) == Some(1000) {
"normal_close"
} else {
"upstream_close"
}
}
Self::UpstreamEof => "eof",
Self::PongTimeout { .. } => "pong_timeout",
Self::LivenessTimeout { .. } => "liveness_timeout",
Self::MessageReceiverClosed => "message_receiver_closed",
}
}
pub(crate) fn upstream_close(&self) -> Option<&CodexWebSocketCloseError> {
match self {
Self::UpstreamCloseFrame { close: Some(close) } => Some(close),
@@ -183,13 +207,71 @@ struct PendingPing {
deadline: Instant,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WebSocketConnectionObservation {
connection_id: Uuid,
exit_reason: String,
age_ms: u64,
idle_ms: u64,
}
impl WebSocketConnectionObservation {
pub(crate) fn with_exit_reason(mut self, exit_reason: &'static str) -> Self {
self.exit_reason = exit_reason.to_owned();
self
}
pub const fn connection_id(&self) -> Uuid {
self.connection_id
}
pub fn exit_reason(&self) -> &str {
&self.exit_reason
}
pub const fn age_ms(&self) -> u64 {
self.age_ms
}
pub const fn idle_ms(&self) -> u64 {
self.idle_ms
}
}
struct PumpLifecycleState {
opened_at: Instant,
last_activity: Instant,
closed_at: Option<Instant>,
exit_reason: Option<PumpExitReason>,
}
impl PumpLifecycleState {
fn observation(
&self,
connection_id: Uuid,
observed_at: Instant,
) -> WebSocketConnectionObservation {
let effective_end = self.closed_at.unwrap_or(observed_at);
WebSocketConnectionObservation {
connection_id,
exit_reason: self
.exit_reason
.as_ref()
.map_or("running", PumpExitReason::metric_reason)
.to_owned(),
age_ms: duration_millis(effective_end.duration_since(self.opened_at)),
idle_ms: duration_millis(effective_end.duration_since(self.last_activity)),
}
}
}
/// 由后台 pump 任务托管的 WebSocket 连接句柄。
pub(crate) struct PumpedWebSocket {
connection_id: Uuid,
tx_command: mpsc::Sender<PumpCommand>,
rx_message: mpsc::Receiver<Result<Message, tungstenite::Error>>,
closed: Arc<AtomicBool>,
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
lifecycle: Arc<Mutex<PumpLifecycleState>>,
pump: Option<JoinHandle<()>>,
}
@@ -212,10 +294,16 @@ impl PumpedWebSocket {
let (tx_command, rx_command) = mpsc::channel::<PumpCommand>(PUMP_COMMAND_BUFFER);
let (tx_message, rx_message) = mpsc::channel(PUMP_MESSAGE_BUFFER);
let connection_id = Uuid::new_v4();
let opened_at = Instant::now();
let closed = Arc::new(AtomicBool::new(false));
let closed_for_task = Arc::clone(&closed);
let exit_reason = Arc::new(Mutex::new(None));
let exit_reason_for_task = Arc::clone(&exit_reason);
let lifecycle = Arc::new(Mutex::new(PumpLifecycleState {
opened_at,
last_activity: opened_at,
closed_at: None,
exit_reason: None,
}));
let lifecycle_for_task = Arc::clone(&lifecycle);
let pump = tokio::spawn(async move {
pump_loop(
connection_id,
@@ -225,7 +313,7 @@ impl PumpedWebSocket {
keepalive,
context,
closed_for_task,
exit_reason_for_task,
lifecycle_for_task,
)
.await;
});
@@ -234,7 +322,7 @@ impl PumpedWebSocket {
tx_command,
rx_message,
closed,
exit_reason,
lifecycle,
pump: Some(pump),
}
}
@@ -272,12 +360,20 @@ impl PumpedWebSocket {
}
pub(crate) fn exit_reason(&self) -> Option<PumpExitReason> {
self.exit_reason
self.lifecycle
.lock()
.expect("WebSocket pump exit reason lock poisoned")
.expect("WebSocket pump lifecycle lock poisoned")
.exit_reason
.clone()
}
pub(crate) fn observation(&self) -> WebSocketConnectionObservation {
self.lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.observation(self.connection_id, Instant::now())
}
/// 主动关闭连接(best-effort 发送 Close 帧)。
pub(crate) async fn close(&self) {
if !self.is_closed() {
@@ -306,9 +402,12 @@ async fn pump_loop(
keepalive: PumpKeepalive,
context: PumpLogContext,
closed: Arc<AtomicBool>,
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
lifecycle: Arc<Mutex<PumpLifecycleState>>,
) {
let mut last_activity = Instant::now();
let mut last_activity = lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.last_activity;
let ping_interval = keepalive.ping_interval.filter(|d| !d.is_zero());
let ping_timeout = keepalive.ping_timeout.filter(|d| !d.is_zero());
let liveness_timeout = keepalive.liveness_timeout.filter(|d| !d.is_zero());
@@ -341,6 +440,7 @@ async fn pump_loop(
}
// 背压期间无法读取 socket,不能把这段本地下游阻塞误算成上游静默。
last_activity = Instant::now();
record_activity(&lifecycle, last_activity);
}
command = rx_command.recv() => {
let Some(command) = command else {
@@ -370,12 +470,14 @@ async fn pump_loop(
None => break 'pump PumpExitReason::UpstreamEof,
Some(Ok(message)) => {
last_activity = Instant::now();
record_activity(&lifecycle, last_activity);
observe_inbound(connection_id, &mut pending_ping, &message);
match message {
Message::Ping(payload) => {
if let Err(error) = inner.send(Message::Pong(payload)).await {
break 'pump PumpExitReason::KeepaliveTransportError {
message: error.to_string(),
metric_reason: transport_metric_reason(&error),
};
}
}
@@ -410,6 +512,7 @@ async fn pump_loop(
Some(Err(err)) => {
let terminal_reason = PumpExitReason::InboundTransportError {
message: err.to_string(),
metric_reason: transport_metric_reason(&err),
};
if let Some(reason) = enqueue_inbound(
&tx_message,
@@ -452,10 +555,22 @@ async fn pump_loop(
};
closed.store(true, Ordering::Release);
*exit_reason
.lock()
.expect("WebSocket pump exit reason lock poisoned") = Some(reason.clone());
log_pump_exit(connection_id, &context, &reason, backpressure_events);
let observation = {
let closed_at = Instant::now();
let mut lifecycle = lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned");
lifecycle.closed_at = Some(closed_at);
lifecycle.exit_reason = Some(reason.clone());
lifecycle.observation(connection_id, closed_at)
};
log_pump_exit(
connection_id,
&context,
&reason,
&observation,
backpressure_events,
);
if reason.should_send_close() {
let _ = inner.send(Message::Close(None)).await;
@@ -492,10 +607,16 @@ async fn handle_command(inner: &mut RawWsStream, command: PumpCommand) -> Option
let PumpCommand::Send { message, ack } = command;
let is_close = matches!(message, Message::Close(_));
let result = inner.send(message).await;
let transport_error = result.as_ref().err().map(ToString::to_string);
let transport_error = result
.as_ref()
.err()
.map(|error| (error.to_string(), transport_metric_reason(error)));
let _ = ack.send(result);
match transport_error {
Some(message) => Some(PumpExitReason::OutboundTransportError { message }),
Some((message, metric_reason)) => Some(PumpExitReason::OutboundTransportError {
message,
metric_reason,
}),
None if is_close => Some(PumpExitReason::LocalClose),
None => None,
}
@@ -517,6 +638,34 @@ fn observe_inbound(connection_id: Uuid, pending_ping: &mut Option<PendingPing>,
*pending_ping = None;
}
fn record_activity(lifecycle: &Mutex<PumpLifecycleState>, observed_at: Instant) {
lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.last_activity = observed_at;
}
pub(crate) fn transport_metric_reason(error: &tungstenite::Error) -> &'static str {
match error {
tungstenite::Error::Io(error) => match error.kind() {
std::io::ErrorKind::ConnectionReset => "tcp_reset",
std::io::ErrorKind::ConnectionAborted => "connection_aborted",
std::io::ErrorKind::BrokenPipe => "broken_pipe",
std::io::ErrorKind::UnexpectedEof => "unexpected_eof",
std::io::ErrorKind::TimedOut => "transport_timeout",
_ => "transport_error",
},
tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
) => "tcp_reset",
_ => "transport_error",
}
}
fn duration_millis(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
async fn send_keepalive_ping(
inner: &mut RawWsStream,
sequence: &mut u64,
@@ -529,6 +678,7 @@ async fn send_keepalive_ping(
.await
.map_err(|error| PumpExitReason::KeepaliveTransportError {
message: error.to_string(),
metric_reason: transport_metric_reason(&error),
})?;
Ok(ping_timeout.map(|timeout| PendingPing {
payload,
@@ -540,6 +690,7 @@ fn log_pump_exit(
connection_id: Uuid,
context: &PumpLogContext,
reason: &PumpExitReason,
observation: &WebSocketConnectionObservation,
backpressure_events: u64,
) {
let detail = reason.detail().unwrap_or_default();
@@ -551,6 +702,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
backpressure_events,
"Responses WebSocket pump stopped unexpectedly"
@@ -565,6 +719,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code),
upstream_error_raw,
@@ -578,6 +735,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
backpressure_events,
"Responses WebSocket pump stopped"
@@ -336,7 +336,10 @@ async fn openai_core_provider_projects_codex_request_observation_without_routing
)]));
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = bundle.core_provider().request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_admin_observation").expect("client key");
let observation = bundle
.core_provider()
.request_observation(&operation, &client_key_id);
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
assert_eq!(observation.subagent_kind.as_deref(), Some("review"));
@@ -28,7 +28,7 @@ use gateway_core::event::GatewayEvent;
use gateway_core::lifecycle::CancellationToken;
use gateway_core::operation::{
CapabilityRequirements, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload,
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::routing::{
@@ -1107,6 +1107,140 @@ async fn image_endpoint_returns_the_exact_upstream_error_response() {
server.verify().await;
}
#[tokio::test]
async fn standalone_search_preserves_wire_and_scopes_turn_metadata_to_the_selected_account() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
let server = MockServer::start().await;
let request_body = br#"{ "id":"search-session", "model":"gpt-future", "commands":{"search_query":[{"q":"private query","future":9007199254740993}]}, "future":1, "future":2 }"#;
let response_body = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
Mock::given(method("POST"))
.and(path("/codex/alpha/search"))
.and(header("originator", "codex_cli_rs"))
.and(body_bytes(request_body.to_vec()))
.respond_with(
ResponseTemplate::new(200)
.insert_header("content-type", "application/json")
.insert_header("x-future-search-header", "preserved")
.set_body_raw(response_body.to_vec(), "application/json"),
)
.expect(1)
.mount(&server)
.await;
let turn_metadata = r#"{"session_id":"session","thread_id":"thread","turn_id":"turn","installation_id":"client-installation","account_id":"client-account","future":true}"#;
let payload = RawJsonPayload::new("openai", Bytes::from_static(request_body))
.expect("search payload")
.with_context(Map::from_iter([(
"turn_metadata".to_owned(),
json!(turn_metadata),
)]));
let operation = Operation::Search(StandaloneSearchRequest::from_raw_json(payload));
let mut stream = provider_with_base_url(&store, server.uri())
.execute(
planned_provider_endpoint_request("openai", operation),
context("req_search_contract", CancellationToken::new()),
)
.await
.expect("prepare search provider stream");
let mut raw_response = None;
let mut completed = false;
let mut observed_http_json = false;
while let Some(event) = stream.next().await {
let event = event.expect("search provider event");
completed |= event
.canonical_facts()
.iter()
.any(|event| matches!(event, GatewayEvent::Completed(_)));
if let Some(raw) = event.wire_event().and_then(|wire| wire.raw_json_body()) {
assert!(raw_response.replace(raw.clone()).is_none());
}
if let Some(observation) = event.response_observation() {
observed_http_json |= observation.transport().as_str() == "http_json"
&& observation.status_code() == Some(200)
&& observation.client_headers().iter().any(|header| {
header.name() == "x-future-search-header"
&& header.value().as_ref() == b"preserved"
});
}
}
assert!(completed);
assert!(observed_http_json);
assert_eq!(raw_response.as_deref(), Some(response_body.as_slice()));
let requests = server.received_requests().await.expect("received requests");
let request = requests.first().expect("single search request");
assert_eq!(request.body.as_slice(), request_body);
let metadata_values = captured_header_values(request, "x-codex-turn-metadata");
assert_eq!(metadata_values.len(), 1);
let metadata: Value = serde_json::from_slice(&metadata_values[0]).expect("turn metadata JSON");
assert_eq!(metadata.get("session_id"), Some(&json!("session")));
assert_eq!(metadata.get("thread_id"), Some(&json!("thread")));
assert_eq!(metadata.get("turn_id"), Some(&json!("turn")));
assert_eq!(metadata.get("future"), Some(&json!(true)));
assert!(metadata.get("account_id").is_none());
assert_ne!(
metadata.get("installation_id"),
Some(&json!("client-installation"))
);
server.verify().await;
}
#[tokio::test]
async fn standalone_search_returns_the_exact_upstream_error_response() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
let server = MockServer::start().await;
let request_body =
br#"{ "id":"search-session", "model":"gpt-future", "commands":{"open":[]} }"#;
let response_body = br#"{ "error":{"message":"future search validation","type":"search_error","code":"future_code"}, "future":9007199254740993 }"#;
Mock::given(method("POST"))
.and(path("/codex/alpha/search"))
.and(body_bytes(request_body.to_vec()))
.respond_with(
ResponseTemplate::new(422)
.insert_header("content-type", "application/problem+json")
.insert_header("x-future-search-error", "preserved")
.set_body_bytes(response_body.to_vec()),
)
.expect(1)
.mount(&server)
.await;
let payload =
RawJsonPayload::new("openai", Bytes::from_static(request_body)).expect("search payload");
let operation = Operation::Search(StandaloneSearchRequest::from_raw_json(payload));
let mut stream = provider_with_base_url(&store, server.uri())
.execute(
planned_provider_endpoint_request("openai", operation),
context("req_search_error", CancellationToken::new()),
)
.await
.expect("prepare search provider stream");
let error = loop {
match stream.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("search rejection must surface an upstream response"),
}
};
assert_eq!(error.upstream_status(), Some(422));
let response = error
.client_visible_upstream_response()
.expect("raw upstream search error response");
assert_eq!(response.status(), 422);
assert_eq!(
response.content_type(),
Some(b"application/problem+json".as_slice())
);
assert_eq!(response.body().as_ref(), response_body);
assert!(response.headers().iter().any(|header| {
header.name() == "x-future-search-error" && header.value().as_ref() == b"preserved"
}));
server.verify().await;
}
#[tokio::test]
async fn capacity_selection_error_preserves_classification_and_retry_after() {
let store = Arc::new(MemoryAccountStore::default());
@@ -1295,7 +1429,7 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
}
#[tokio::test]
async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_session() {
async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_websocket_close").await;
let listener = TcpListener::bind("127.0.0.1:0")
@@ -1306,6 +1440,26 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
listener.local_addr().expect("listener address")
);
let server = tokio::spawn(async move {
let completed_response = |response_id: &str| {
Message::Text(
json!({
"type": "response.completed",
"response": {
"id": response_id,
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
)
};
let (stream, _) = listener
.accept()
.await
@@ -1371,6 +1525,31 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
.await
.expect("write sticky HTTP response");
let (stream, _) = listener
.accept()
.await
.expect("accept same-session continuation WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let request = websocket
.next()
.await
.expect("same-session continuation request")
.expect("valid same-session continuation request");
let payload: Value = serde_json::from_str(
request
.to_text()
.expect("same-session continuation request text"),
)
.expect("same-session continuation request JSON");
assert_eq!(
payload.get("previous_response_id"),
Some(&json!("external-previous-response"))
);
websocket
.send(completed_response("resp_external_continuation"))
.await
.expect("complete same-session continuation request");
let (stream, _) = listener
.accept()
.await
@@ -1382,24 +1561,7 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
.expect("other-session WebSocket request")
.expect("valid other-session WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.completed",
"response": {
"id": "resp_other_session",
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
))
.send(completed_response("resp_other_session"))
.await
.expect("complete other-session WebSocket request");
});
@@ -1472,6 +1634,11 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
raw_close.get("last_event_type"),
Some(&json!("response.output_item.added"))
);
let connection = error
.connection_observation()
.expect("normal close should retain its connection lifecycle observation");
assert_eq!(connection.exit_reason(), "normal_close");
assert!(connection.age_ms() >= connection.idle_ms());
let second_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
@@ -1491,6 +1658,27 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
}
drop(sticky_stream);
let continuation_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
Some("thread-continuation"),
None,
));
let mut continuation_stream = provider
.execute(
planned_request("openai", continuation_operation),
external_continuation_context("req_sticky_external_continuation"),
)
.await
.expect("sticky state must not force a continuation onto HTTP");
assert_eq!(
continuation_stream.metadata().transport().as_str(),
"websocket"
);
while let Some(event) = continuation_stream.next().await {
event.expect("same-session continuation WebSocket response");
}
drop(continuation_stream);
let other_operation = Operation::Generate(generate_with_session_context(
"other-websocket-session",
Some("thread-first"),
@@ -2044,7 +2232,7 @@ async fn cross_account_continuation_should_require_client_replay_without_an_upst
detail.error_type(),
),
(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
Some(ContinuationFailure::HistoryUnavailable),
Some("previous_response_not_found"),
@@ -2094,6 +2282,10 @@ async fn missing_affinity_continuation_should_require_client_replay_without_an_u
.await
.expect("captured upstream requests");
assert_eq!(
error.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
error
.client_visible_upstream_error()
@@ -3396,6 +3588,10 @@ async fn same_account_previous_response_not_found_should_remain_client_visible()
}
};
assert_eq!(
failure.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
failure.continuation_failure(),
Some(ContinuationFailure::HistoryUnavailable)
@@ -3625,6 +3821,10 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
None => panic!("competing exact continuation must fail while its socket is busy"),
}
};
assert_eq!(
exact_error.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
exact_error.continuation_failure(),
Some(ContinuationFailure::Busy)
@@ -3673,7 +3873,7 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
detail.message(),
),
(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
Some(ContinuationFailure::HistoryUnavailable),
Some(ContinuationRecoveryDisposition::ClientReplayRequired),
@@ -4085,7 +4285,8 @@ fn request_observation_reads_openai_metadata_without_changing_the_operation() {
let generation = GenerateRequest::from_protocol_payload(payload);
let operation = Operation::Generate(generation);
let observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let observation = provider(&store).request_observation(&operation, &client_key_id);
assert_eq!(observation.request_kind.as_deref(), Some("review"));
assert_eq!(observation.subagent_kind.as_deref(), Some("worker"));
@@ -4103,7 +4304,8 @@ fn request_observation_preserves_the_raw_reasoning_effort() {
.expect("protocol payload");
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let observation = provider(&store).request_observation(&operation, &client_key_id);
assert_eq!(
observation.reasoning_effort.as_deref(),
@@ -4133,7 +4335,8 @@ fn request_observation_ignores_future_session_state_without_rewriting_the_protoc
);
let operation = Operation::Generate(generation);
let _observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let _observation = provider(&store).request_observation(&operation, &client_key_id);
let Operation::Generate(generation) = &operation else {
panic!("operation should remain a generate request");
@@ -0,0 +1,200 @@
use std::sync::Arc;
use futures::{StreamExt, stream};
use gateway_core::account::{AccountFeedbackStats, ProviderAccountId};
use gateway_core::engine::provider::{EventStream, ProviderCallMetadata, ProviderStream};
use gateway_core::error::{
ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind,
};
use gateway_core::routing::{ProviderKind, UpstreamModelId};
use gateway_core::upstream::{UpstreamSendState, UpstreamTransport};
use provider_openai::openai_failure_affects_account_score;
fn sent_error(kind: ProviderErrorKind, code: Option<&str>) -> ProviderError {
let error = ProviderError::new(kind, UpstreamSendState::Sent);
match code {
Some(code) => error.with_upstream_code(OpaqueUpstreamValue::new(code)),
None => error,
}
}
fn deliver_error_with_openai_feedback(
feedback: &Arc<AccountFeedbackStats>,
provider: &ProviderKind,
account: &ProviderAccountId,
error: ProviderError,
) {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(error)]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(feedback), openai_failure_affects_account_score);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
#[test]
fn account_score_failure_filter_should_include_server_overload_as_a_regular_reason() {
let error = sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded"));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_accept_the_closed_reason_list() {
for code in [
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
"service_unavailable_error",
] {
let error = sent_error(ProviderErrorKind::Unavailable, Some(code));
assert!(
openai_failure_affects_account_score(&error),
"allowlisted code was rejected: {code}"
);
}
}
#[test]
fn account_score_failure_filter_should_normalize_case_and_whitespace() {
let error = sent_error(ProviderErrorKind::Unavailable, Some(" SERVER_ERROR "));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() {
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream unavailable",
None,
Some("SERVICE_UNAVAILABLE_ERROR".to_owned()),
));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_prefer_a_structured_code_over_type() {
let error = ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"bad request",
Some("invalid_request".to_owned()),
Some("server_error".to_owned()),
));
assert!(!openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_reject_client_and_unknown_failures() {
for error in [
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
sent_error(
ProviderErrorKind::ContinuationRecoveryRequired,
Some("previous_response_not_found"),
),
sent_error(ProviderErrorKind::Unsupported, Some("unsupported")),
sent_error(ProviderErrorKind::Unauthorized, Some("token_expired")),
sent_error(ProviderErrorKind::QuotaExhausted, Some("quota_exhausted")),
sent_error(ProviderErrorKind::RateLimited, Some("rate_limit_reached")),
sent_error(
ProviderErrorKind::Unavailable,
Some("internal_server_error"),
),
sent_error(ProviderErrorKind::Unavailable, Some("service_unavailable")),
sent_error(
ProviderErrorKind::Transport,
Some("upstream_transport_error"),
),
sent_error(ProviderErrorKind::Timeout, Some("first_output_timeout")),
sent_error(
ProviderErrorKind::Protocol,
Some("upstream_stream_truncated"),
),
sent_error(
ProviderErrorKind::Unavailable,
Some("upstream_empty_response"),
),
sent_error(ProviderErrorKind::Unavailable, Some("new_unknown_reason")),
sent_error(ProviderErrorKind::Transport, Some("new_unknown_reason")),
sent_error(ProviderErrorKind::Transport, Some("")),
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_status(503),
] {
assert!(
!openai_failure_affects_account_score(&error),
"non-allowlisted failure affected the account score"
);
}
}
#[test]
fn account_score_failure_filter_should_reject_internal_kinds_without_a_reason() {
for kind in [
ProviderErrorKind::Transport,
ProviderErrorKind::Timeout,
ProviderErrorKind::Protocol,
] {
let error = sent_error(kind, None);
assert!(
!openai_failure_affects_account_score(&error),
"unlisted internal failure affected the score: {}",
kind.as_str()
);
}
}
#[test]
fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_usable_account").expect("account");
for _ in 0..100 {
deliver_error_with_openai_feedback(
&feedback,
&provider,
&account,
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
);
}
assert_eq!(
feedback.scheduling_signals(&provider, &account),
(None, None)
);
}
#[test]
fn openai_feedback_should_score_server_overload_as_one_regular_failure() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_overloaded").expect("account");
deliver_error_with_openai_feedback(
&feedback,
&provider,
&account,
sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")),
);
assert_eq!(
feedback.scheduling_signals(&provider, &account).0,
Some(2_000)
);
}
@@ -1 +1,2 @@
mod contract;
mod failure;
@@ -837,6 +837,40 @@ async fn missing_exact_socket_should_fail_without_opening_a_connection() {
);
}
#[tokio::test]
async fn connection_local_continuation_without_a_pool_should_not_open_http_or_websocket() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
);
let mut request = new_chain_request("conversation-exact-no-pool");
request.set_previous_response_id(Some("resp_no_pool".to_owned()));
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let error = backend
.create_response(
&request,
request_context("req_exact_no_pool", Some("chatgpt-account")),
)
.await
.expect_err("connection-local continuation requires its owning pool");
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::PoolUnavailable
})
);
assert!(
timeout(Duration::from_millis(50), listener.accept())
.await
.is_err()
);
}
#[tokio::test]
async fn connection_local_continuation_should_fail_after_its_live_socket_disappears() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -852,7 +886,7 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
))
.await
.unwrap();
websocket.close(None).await.unwrap();
drop(websocket);
closed_tx.send(()).unwrap();
});
let backend = CodexBackendClient::new(
@@ -883,12 +917,18 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
)
.await
.expect_err("a disappeared live socket cannot be reconstructed from the database");
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::FreshConnectionRequired
})
let CodexClientError::WebSocket(error) = error else {
panic!("disappeared connection should remain a typed WebSocket error");
};
assert_eq!(
error.continuation_unavailable_reason(),
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
);
let observation = error
.connection_observation()
.expect("disappeared connection should retain its lifecycle observation");
assert_eq!(observation.exit_reason(), "tcp_reset");
assert!(observation.age_ms() >= observation.idle_ms());
}
#[tokio::test]
@@ -1114,9 +1154,9 @@ async fn websocket_failure_after_first_delivery_should_not_open_http_fallback()
else {
panic!("expected typed post-delivery WebSocket failure");
};
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = *source else {
panic!("expected upstream close frame source");
};
let close = source
.close_before_terminal()
.expect("expected upstream close frame source");
assert!(
close.connection_id().is_some(),
"close failures must retain their WebSocket connection correlation"
@@ -273,6 +273,16 @@ fn transport_should_prefer_websocket_when_requested_without_history() {
assert!(transport_requirement(&request).allows_pre_send_http_fallback());
}
#[test]
fn unspecified_transport_without_history_should_still_be_a_new_chain() {
let request = codex_request("gpt-test", "be brief", Vec::new());
assert_eq!(
transport_requirement(&request),
TransportRequirement::NewChain
);
}
#[test]
fn new_chain_should_allow_pre_delivery_http_fallback() {
let mut request = codex_request("gpt-test", "be brief", Vec::new());
@@ -284,6 +294,25 @@ fn new_chain_should_allow_pre_delivery_http_fallback() {
);
}
#[test]
fn sticky_http_fallback_should_only_apply_to_new_chains() {
assert!(TransportRequirement::NewChain.allows_sticky_http_fallback());
for requirement in [
TransportRequirement::HttpRequired,
TransportRequirement::ExplicitWebSocketWarmup,
TransportRequirement::ExactWebSocketContinuation,
TransportRequirement::PersistedContinuation,
TransportRequirement::ExternalUnknown,
] {
assert!(
!requirement.allows_sticky_http_fallback(),
"sticky HTTP must not override {}",
requirement.as_str()
);
}
}
#[test]
fn transport_should_mark_unknown_previous_response_as_external() {
let mut request = codex_request("gpt-test", "be brief", Vec::new());
@@ -647,7 +647,7 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
.send(Message::Binary(b"unexpected-binary".to_vec().into()))
.await
.unwrap();
websocket.close(None).await.unwrap();
let _client_close = websocket.next().await;
});
let mut request = codex_request("gpt-test", "be brief", Vec::new());
request.set_previous_response_id(Some("resp_binary_previous".to_owned()));
@@ -664,15 +664,30 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
.expect_err("binary websocket events should be rejected");
server.await.unwrap();
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
message,
source: Some(source),
})
if message.contains("unexpected binary websocket event")
&& matches!(*source, CodexWebSocketExchangeError::UnexpectedBinaryEvent)
let CodexClientError::WebSocket(error) = error else {
panic!("binary websocket event should remain a typed WebSocket error");
};
let CodexWebSocketExchangeError::PostSendAmbiguous {
message,
source: Some(source),
} = error
else {
panic!("binary websocket event should remain post-send ambiguous");
};
assert!(message.contains("unexpected binary websocket event"));
assert_eq!(
source
.connection_observation()
.map(|observation| observation.exit_reason()),
Some("unexpected_binary_event")
);
let CodexWebSocketExchangeError::ConnectionObserved { source, .. } = *source else {
panic!("binary websocket event should carry its connection observation");
};
assert!(matches!(
*source,
CodexWebSocketExchangeError::UnexpectedBinaryEvent
));
}
#[tokio::test]
@@ -1560,9 +1575,9 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
else {
panic!("close-before-terminal should preserve its typed source: {error:?}");
};
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = source.as_ref() else {
panic!("post-send ambiguity should wrap a close-before-terminal error: {source:?}");
};
let close = source
.close_before_terminal()
.unwrap_or_else(|| panic!("post-send ambiguity should retain its close: {source:?}"));
assert_eq!(close.last_event_type(), Some("response.output_text.delta"));
std::assert_matches!(
@@ -1,4 +1,5 @@
use super::*;
use provider_openai::transport::websocket::PreviousResponseUnavailableReason;
#[tokio::test]
async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and_conversation() {
@@ -1108,6 +1109,30 @@ async fn websocket_pool_should_replace_idle_connection_after_pong_deadline() {
.await
.expect("server should report the Pong-timeout close");
tokio::time::resume();
let mut continuation = request.clone();
continuation.set_previous_response_id(Some("resp_no_pong_first".to_owned()));
continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let error = backend
.create_response(
&continuation,
request_context("req_no_pong_continuation", Some("chatgpt-account")),
)
.await
.expect_err("Pong-timeout connection cannot satisfy an exact continuation");
let CodexClientError::WebSocket(error) = error else {
panic!("Pong-timeout continuation should remain a typed WebSocket error");
};
assert_eq!(
error.continuation_unavailable_reason(),
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
);
assert_eq!(
error
.connection_observation()
.expect("Pong-timeout tombstone should retain its lifecycle observation")
.exit_reason(),
"pong_timeout"
);
let second = backend
.create_response(
&request,
@@ -138,7 +138,11 @@ impl Provider for GrokBuildProvider {
self.catalog.catalog_generation()
}
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
operation: &Operation,
_client_api_key_id: &gateway_core::policy::ClientApiKeyId,
) -> ProviderRequestObservation {
let Operation::Generate(request) = operation else {
return ProviderRequestObservation::default();
};
@@ -153,6 +157,7 @@ impl Provider for GrokBuildProvider {
request_kind: semantics.request_kind,
subagent_kind: semantics.subagent_kind,
compact: semantics.compact,
continuation: Default::default(),
}
}
@@ -1063,8 +1063,12 @@ async fn model_alias_should_be_canonical_before_account_selection() {
async fn request_observation_preserves_raw_xai_reasoning_effort() {
let provider = provider(StubSelector::success(), StubInferenceTransport::success()).await;
let observation =
provider.request_observation(&operation_with_reasoning_effort("future-value"));
let client_key_id =
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
let observation = provider.request_observation(
&operation_with_reasoning_effort("future-value"),
&client_key_id,
);
assert_eq!(
observation.reasoning_effort.as_deref(),
@@ -1093,7 +1097,9 @@ async fn request_observation_preserves_codex_semantics_for_xai() {
)]));
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = provider.request_observation(&operation);
let client_key_id =
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
let observation = provider.request_observation(&operation, &client_key_id);
assert_eq!(observation.reasoning_effort.as_deref(), Some("xhigh"));
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
+1
View File
@@ -2,3 +2,4 @@
93219760c733616e7bba9a8933c207b26752b2ebf7e6edfe1ef42ba227c50a7d 0002_runtime_pressure_observability.sql
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
@@ -0,0 +1,80 @@
alter table model_requests
add column continuation_affinity_hash text,
add column continuation_previous_response_id_hash text,
add column continuation_requested boolean not null default false,
add column continuation_unavailable_reason text,
add column upstream_connection_id text,
add column upstream_connection_exit_reason text,
add column upstream_connection_age_ms bigint,
add column upstream_connection_idle_ms bigint,
add column recovery_request_id text,
add column recovered_at timestamptz,
add column recovery_attempt_count integer not null default 0,
add column recovery_retry_delay_ms bigint,
add column recovery_total_latency_ms bigint,
add constraint model_requests_continuation_hashes_ck check (
(continuation_affinity_hash is null
or continuation_affinity_hash ~ '^[0-9a-f]{64}$')
and (continuation_previous_response_id_hash is null
or continuation_previous_response_id_hash ~ '^[0-9a-f]{64}$')
and (continuation_requested
or continuation_previous_response_id_hash is null)
),
add constraint model_requests_continuation_reason_ck check (
continuation_unavailable_reason is null
or (
octet_length(continuation_unavailable_reason) between 1 and 64
and continuation_unavailable_reason ~ '^[a-z][a-z0-9_]*$'
)
),
add constraint model_requests_upstream_connection_ck check (
(
upstream_connection_id is null
and upstream_connection_exit_reason is null
and upstream_connection_age_ms is null
and upstream_connection_idle_ms is null
)
or (
octet_length(upstream_connection_id) between 1 and 128
and upstream_connection_id !~ '[[:cntrl:]]'
and octet_length(upstream_connection_exit_reason) between 1 and 64
and upstream_connection_exit_reason ~ '^[a-z][a-z0-9_]*$'
and upstream_connection_age_ms >= 0
and upstream_connection_idle_ms between 0 and upstream_connection_age_ms
)
),
add constraint model_requests_recovery_ck check (
recovery_attempt_count >= 0
and (
(
recovered_at is null
and recovery_request_id is null
and recovery_retry_delay_ms is null
and recovery_total_latency_ms is null
)
or (
recovered_at is not null
and recovered_at >= completed_at
and recovery_request_id is not null
and octet_length(recovery_request_id) between 1 and 128
and recovery_attempt_count > 0
and recovery_retry_delay_ms >= 0
and recovery_total_latency_ms >= recovery_retry_delay_ms
)
)
);
create index model_requests_pending_continuation_recovery_idx
on model_requests (
client_api_key_ref,
continuation_affinity_hash,
completed_at desc,
id desc
)
where error_kind = 'continuation_recovery_required'
and recovered_at is null
and continuation_affinity_hash is not null;
create index model_requests_recovery_request_idx
on model_requests (recovery_request_id)
where recovery_request_id is not null;
+11 -6
View File
@@ -108,20 +108,22 @@ mutation 请求不要求客户端提供全局配置版本。会改变路由快
## 3. OpenAI 数据面与模型目录
Responses 和 Images HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;协议可接受性由上游决定。
Responses、Images 和 standalone Search HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;
协议可接受性由上游决定。
| 方法 | 路由 | 说明 |
| --- | --- | --- |
| `POST` | `/v1/responses` | OpenAI Responses JSON;`stream=true` 返回 SSE,否则返回完整 JSON |
| `GET` | `/v1/responses` | 通过 HTTP Upgrade 建立 Responses WebSocket |
| `POST` | `/v1/responses/review` | 使用同一 Responses 合同发起 review 子代理请求 |
| `POST` | `/v1/alpha/search` | Codex standalone web search;JSON 请求与响应正文原样转发 |
| `POST` | `/v1/images/generations` | 通过 OpenAI Provider 发起图像生成;JSON 请求与响应正文原样转发 |
| `POST` | `/v1/images/edits` | 通过 OpenAI Provider 发起图像编辑;JSON 请求与响应正文原样转发 |
| `GET` | `/v1/models` | 返回当前 Client Key 账号范围内各 Provider 的可用公开模型并集;有两种响应形态,见下 |
| `GET` | `/v1/models/catalog` | 返回 Codex 客户端使用的模型目录 |
| `GET` | `/v1/models/{model_id}/info` | 返回 Codex 客户端使用的单模型信息 |
| `GET` | `/v1/models/{model_id}` | 返回 OpenAI 兼容的单模型详情 |
Codex 的 review 等子代理请求仍使用 `/v1/responses`,并通过 `x-openai-subagent` 请求头携带子代理类型;
网关不提供独立的子代理请求路径。
`GET /v1/models` 默认返回 OpenAI 兼容列表 `{"object": "list", "data": [...]}`;请求携带非空
`client_version` query 参数(Codex 客户端)时改为返回 Codex 专用目录合同 `{"models": [...]}`。
@@ -129,8 +131,11 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
映射除外),HTTP SSE 与 WebSocket 的上游业务事件字节原样转发,response ID 按 opaque 值处理而不
假设 UUID 或固定长度;OpenAI 上游错误 envelope 和允许下发的 opaque header 值也不由 canonical
观测结果重写。Images 请求不读取或重建 JSON,也不要求或映射模型字段;它固定使用 OpenAI Provider,
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。xAI 是
Grok wire 与 Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。
`/v1/alpha/search` 使用相同的 OpenAI Provider 原生端点边界:body(包括 `model`)不解析、不映射,
`x-codex-turn-metadata` 在移除客户端账号身份并按当前 lease 重写 installation ID 后转发;上游账号
Authorization、Cookie、account ID、originator 和 User-Agent 均由代理安全重建。xAI 是 Grok wire 与
Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
上游结构化错误的 message/code/type 会透传给客户端,其中内嵌的账号指纹 UUID 已脱敏。模型映射是
全局精确映射,未命中时模型名原样交给候选 Provider;分组只限定账号集合,不参与模型改名。
+7 -6
View File
@@ -8,7 +8,7 @@
Codex Proxy RS 是单进程、单副本运行的多 Provider AI 网关,同时提供:
- 面向客户端的 OpenAI Responses、Images 和模型目录协议;
- 面向客户端的 OpenAI Responses、Images、standalone Search 和模型目录协议;
- 面向管理员的 `/api/admin/*` 控制面和 Vue 管理端;
- OpenAI 与 xAI 两个编译期 Provider;
- PostgreSQL 持久化、Redis 协调状态以及 S3/R2 数据库备份。
@@ -61,7 +61,7 @@ flowchart LR
| `gateway-api` | HTTP/WS/SSE 解码与交付、Admin wire、静态管理端;不直接访问 Store 或具体 Provider |
| `gateway-store` | PostgreSQL、Redis、S3/R2、`pg_dump` 适配器;不拥有业务策略 |
| `gateway-host` | 配置加载、日志、HTTP 生命周期、Worker 监督和系统更新 |
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images transport |
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images/Search transport |
| `providers/xai` | xAI OAuth session、账号选择、目录、额度和 Grok/Responses 转换 |
| `frontend` | Vue 管理端,仅通过 Admin API 读写状态 |
@@ -151,16 +151,17 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别 Codex Desktop/
- 可恢复观测写入失败不能替换已经确定的客户端协议结果。
Responses 按模型目录编译候选;全局模型映射是精确映射,未命中时模型名原样交给候选 Provider。
Images 是 OpenAI Provider 自有端点:它不要求模型字段、不参与文本模型映射,只在 Client Key 的账号范围
确实包含 OpenAI 账号时生成单一 OpenAI 候选。
Images 与 standalone Search 是 OpenAI Provider 自有端点:两者都不参与文本模型映射,只在 Client Key
的账号范围确实包含 OpenAI 账号时生成单一 OpenAI 候选。Images 不要求模型字段;Search body 中的模型
及其他字段保持原始 bytes 并由上游解释。
## 5. Provider 与协议边界
Core 只理解 `Operation`、能力要求、Provider 候选、稳定错误和 canonical event,不读取 Provider SDK
类型。Provider 独占 credential schema、OAuth、账号选择、模型目录、额度投影和上游 transport。
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket 与 Images 的业务正文
按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket、Images 与 standalone
Search 的业务正文按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
- xAI 是翻译边界。Provider 把 Grok wire 转换为 Responses wire;上游结构化错误的 message/code/type
可以透出,但账号指纹会先脱敏。
- response ID 是不透明 UTF-8 bytes,不假设 UUID、固定长度或跨 Provider 可复用。
+13 -4
View File
@@ -204,6 +204,18 @@ export interface OpsErrorMetadata {
component: string
attemptId: string | null
accountLabel: string | null
continuationAffinityHash: string | null
continuationPreviousResponseIdHash: string | null
continuationUnavailableReason: string | null
upstreamConnectionId: string | null
upstreamConnectionExitReason: string | null
upstreamConnectionAgeMs: number | null
upstreamConnectionIdleMs: number | null
recoveryRequestId: string | null
recoveredAt: string | null
recoveryAttemptCount: number
recoveryRetryDelayMs: number | null
recoveryTotalLatencyMs: number | null
}
export interface OpsError {
@@ -420,10 +432,7 @@ interface PageQuery {
type UsagePageQuery = UsageRangeQuery & PageQuery
type OpsErrorPageQuery = UsagePageQuery & {
failureClass?: string
route?: string
}
type OpsErrorPageQuery = UsagePageQuery
interface UsageDetailQuery {
id: string
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { Key, Openai, Xai } from '@boxicons/vue'
import { Key, LinkAlt, Openai, Xai } from '@boxicons/vue'
import { computed } from 'vue'
import { formatProviderLabel } from '@/utils/providers'
@@ -38,6 +38,8 @@ const providerLabel = computed(() => formatProviderLabel(props.provider, '未知
const authenticationLabel = computed(() => {
if (normalizedAuthenticationKind.value === 'oauth')
return 'OAuth'
if (normalizedAuthenticationKind.value === 'api_key')
return 'API Key'
return props.authenticationKind?.trim() || '未知认证类型'
})
</script>
@@ -60,7 +62,8 @@ const authenticationLabel = computed(() => {
:class="iconContainerClass"
:title="authenticationLabel"
>
<Key v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
<LinkAlt v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
<Key v-else-if="normalizedAuthenticationKind === 'api_key'" :class="iconClass" />
<span v-else class="text-[10px] font-heavy text-cp-text-quaternary">?</span>
</span>
@@ -77,7 +77,7 @@ const profileOpen = shallowRef(false)
</div>
</div>
<div class="grid max-h-44 min-h-0 gap-3 overflow-y-auto pr-1">
<div class="grid min-h-0 gap-3">
<AccountQuotaPanelEntry
v-for="entry in quotaEntries"
:key="entry.key"
@@ -10,15 +10,19 @@ import AccountUsageWindow from '../AccountUsageWindow/index.vue'
import { resolveAccountUsageWindowPresentation } from '../AccountUsageWindow/presenter'
import AccountQuotaWindowGroup from './WindowGroup.vue'
const props = defineProps<{
const props = withDefaults(defineProps<{
label: string | null
windows: AccountQuotaWindow[]
}>()
showPercentage?: boolean
}>(), {
showPercentage: true,
})
const now = useUiClock()
const grouped = computed(() => props.windows.length > 1 && Boolean(props.label))
const detailHeading = computed(() => props.label)
const detailTitle = computed(() => detailHeading.value ?? props.windows[0]?.labelDisplay ?? '额度详情')
const summaryLabel = computed(() => props.label ?? props.windows[0]?.labelDisplay ?? '额度')
const hasQuotaWindow = computed(() => props.windows.some(window => typeof window.usedPercent === 'number'))
const detailItems = computed(() =>
props.windows.map((window) => {
const view = resolveAccountUsageWindowPresentation({
@@ -82,7 +86,12 @@ function quotaWindowCode(windowSeconds: number | null, role: AccountQuotaWindow[
:aria-expanded="open"
aria-haspopup="dialog"
>
<AccountQuotaWindowGroup v-if="grouped && label" :label="label" :windows="windows" />
<AccountQuotaWindowGroup
v-if="hasQuotaWindow"
:label="summaryLabel"
:windows="windows"
:show-percentage="showPercentage"
/>
<AccountUsageWindow
v-else
:window="windows[0]"
@@ -4,10 +4,13 @@ import type { AccountQuotaWindow } from '../../constants'
import { computed } from 'vue'
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
const props = defineProps<{
const props = withDefaults(defineProps<{
label: string
windows: AccountQuotaWindow[]
}>()
showPercentage?: boolean
}>(), {
showPercentage: true,
})
const windowItems = computed(() => props.windows.map(window => ({
key: window.key,
@@ -16,6 +19,17 @@ const windowItems = computed(() => props.windows.map(window => ({
usedPercentDisplay: window.usedPercentDisplay,
presentation: quotaWindowPresentation(window, '2px'),
})))
const highestUsageWindow = computed(() => props.windows.reduce((highest, window) => {
if (typeof window.usedPercent !== 'number')
return highest
if (!highest || typeof highest.usedPercent !== 'number' || window.usedPercent > highest.usedPercent)
return window
return highest
}, undefined as AccountQuotaWindow | undefined))
const highestUsageDisplay = computed(() => highestUsageWindow.value?.usedPercentDisplay ?? '—')
const highestUsageTextClass = computed(() => highestUsageWindow.value
? quotaWindowPresentation(highestUsageWindow.value, '2px').percentTextClass
: 'text-cp-text-quaternary')
const trackGridStyle = computed(() => ({
gridTemplateColumns: `repeat(${Math.max(windowItems.value.length, 1)}, minmax(0, 1fr))`,
@@ -23,16 +37,25 @@ const trackGridStyle = computed(() => ({
</script>
<template>
<div class="flex min-w-0 flex-col">
<div class="mb-1 min-w-0 truncate text-cp-xs leading-3.5 font-bold text-cp-text-quaternary">
{{ label }}
<div class="grid min-w-0 gap-1.5">
<div class="flex min-w-0 items-baseline justify-between gap-1 text-[10px] leading-3 font-bold">
<span class="min-w-0 truncate text-cp-text-quaternary" :title="label">
{{ label }}
</span>
<strong
v-if="showPercentage"
class="shrink-0 font-mono font-heavy tabular-nums"
:class="highestUsageTextClass"
>
{{ highestUsageDisplay }}
</strong>
</div>
<div class="grid min-w-0 gap-1" :style="trackGridStyle">
<div
v-for="item in windowItems"
:key="item.key"
class="h-0.75 w-full overflow-hidden rounded-full bg-cp-border-secondary"
class="h-1 w-full overflow-hidden rounded-full bg-cp-border-secondary"
role="progressbar"
:aria-label="item.labelTooltip"
aria-valuemin="0"
@@ -41,7 +64,7 @@ const trackGridStyle = computed(() => ({
:aria-valuetext="item.usedPercentDisplay"
>
<div
class="h-full rounded-full transition-[width,background-color] duration-200"
class="h-full rounded-full transition-[width,background-color] duration-200 motion-reduce:transition-none"
:class="item.presentation.barClass"
:style="item.presentation.barStyle"
/>
@@ -4,7 +4,9 @@ import type { AccountRow } from '../../constants'
import { computed } from 'vue'
import { groupedAccountQuotaWindows, visibleSummaryQuotaWindows } from '../../constants'
import AccountUsageWindow from '../AccountUsageWindow/index.vue'
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
import AccountQuotaSummaryEntry from './Entry.vue'
import { recentlyUsedQuotaEntry, representativeQuotaWindow } from './presenter'
const props = defineProps<{
account: AccountRow
@@ -13,24 +15,65 @@ const props = defineProps<{
const quotaWindows = computed(() => props.account.quota.windows)
const visibleQuotaWindows = computed(() => visibleSummaryQuotaWindows(quotaWindows.value))
const summaryEntries = computed(() => groupedAccountQuotaWindows(visibleQuotaWindows.value))
const summaryClass = computed(
() =>
`box-border grid w-full min-w-0 gap-1 py-1.5 ${
summaryEntries.value.length <= 1
? 'min-h-13 content-center'
: 'min-h-16.5 content-center'
}`,
)
const hasUsage = computed(() => (props.account.usage.requestCount ?? 0) > 0)
const recentUsageEntry = computed(() => recentlyUsedQuotaEntry(
summaryEntries.value,
props.account.usage.models,
))
const currentUsageWindow = computed(() => representativeQuotaWindow(recentUsageEntry.value))
const currentUsageDisplay = computed(() => currentUsageWindow.value?.usedPercentDisplay ?? '—')
const currentUsageTextClass = computed(() => currentUsageWindow.value
? quotaWindowPresentation(currentUsageWindow.value, '2px').percentTextClass
: 'text-cp-text-quaternary')
const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length - 1, 0))
</script>
<template>
<div :class="summaryClass">
<AccountQuotaSummaryEntry
v-for="entry in summaryEntries"
:key="entry.key"
:label="entry.label"
:windows="entry.windows"
/>
<AccountUsageWindow v-if="summaryEntries.length === 0" variant="compact" />
<div class="box-border grid min-h-16.5 w-full min-w-0 content-center gap-1.5 py-1.5">
<template v-if="summaryEntries.length > 0">
<div
v-if="hasUsage"
class="flex min-w-0 items-baseline justify-between gap-2 leading-none"
>
<span
class="flex min-w-0 items-baseline gap-1 font-mono tabular-nums"
title="当前额度窗口总 Token"
>
<strong class="truncate text-cp-xs font-heavy text-cp-text">
{{ account.usage.totalTokensDisplay }}
</strong>
<span class="shrink-0 text-[9px] font-emphasis tracking-[0.02em] text-cp-text-quaternary">
Tokens
</span>
</span>
<span
class="flex shrink-0 items-baseline gap-1 text-[9px] font-emphasis text-cp-text-quaternary"
title="最近使用额度的当前已用比例"
>
<span>使用率</span>
<strong class="font-mono font-heavy tabular-nums" :class="currentUsageTextClass">
{{ currentUsageDisplay }}
</strong>
</span>
</div>
<div v-if="recentUsageEntry" class="flex min-w-0 items-end gap-2">
<div class="min-w-0 flex-1">
<AccountQuotaSummaryEntry
:label="recentUsageEntry.label"
:windows="recentUsageEntry.windows"
:show-percentage="false"
/>
</div>
<span
v-if="additionalEntryCount > 0"
class="grid h-5 min-w-5 shrink-0 place-items-center rounded-cp bg-cp-fill-quaternary px-1.5 font-mono text-[9px] font-heavy tabular-nums text-cp-text-tertiary"
:title="`另有 ${additionalEntryCount} 个额度组,可展开账号查看`"
>
+{{ additionalEntryCount }}
</span>
</div>
</template>
<AccountUsageWindow v-else variant="compact" />
</div>
</template>
@@ -0,0 +1,74 @@
import type { AccountQuotaWindow, AccountQuotaWindowEntry, AccountRow } from '../../constants'
type AccountModelUsage = AccountRow['usage']['models'][number]
/**
* 根据账号最近一次模型请求选择对应额度组。
* 没有专属额度的模型统一回退到 Codex 通用额度。
*/
export function recentlyUsedQuotaEntry(
entries: readonly AccountQuotaWindowEntry[],
models: readonly AccountModelUsage[],
) {
const recentModel = latestModelUsage(models)
if (recentModel) {
const modelKey = quotaIdentity(recentModel.model)
const modelEntry = entries.find(entry => quotaEntryIdentities(entry).has(modelKey))
if (modelEntry)
return modelEntry
}
return entries.find(entry => entry.windows.some(window => window.limitId === 'codex'))
?? entries[0]
}
/** 一个额度组可能包含多个滚动窗口,摘要使用其中当前占用最高的窗口。 */
export function representativeQuotaWindow(entry: AccountQuotaWindowEntry | undefined) {
return entry?.windows.reduce<AccountQuotaWindow | undefined>((representative, window) => {
if (typeof window.usedPercent !== 'number')
return representative
if (
!representative
|| typeof representative.usedPercent !== 'number'
|| window.usedPercent > representative.usedPercent
) {
return window
}
return representative
}, undefined)
}
function latestModelUsage(models: readonly AccountModelUsage[]) {
let latest: AccountModelUsage | undefined
let latestTimestamp = Number.NEGATIVE_INFINITY
for (const model of models) {
const timestamp = Date.parse(model.lastUsedAt)
if (!Number.isFinite(timestamp) || timestamp <= latestTimestamp)
continue
latest = model
latestTimestamp = timestamp
}
return latest
}
function quotaEntryIdentities(entry: AccountQuotaWindowEntry) {
const identities = new Set<string>()
addQuotaIdentity(identities, entry.label)
for (const window of entry.windows) {
addQuotaIdentity(identities, window.limitId)
addQuotaIdentity(identities, window.limitName)
}
return identities
}
function addQuotaIdentity(identities: Set<string>, value: string | null) {
const identity = quotaIdentity(value)
if (identity)
identities.add(identity)
}
function quotaIdentity(value: string | null) {
return value?.trim().toLowerCase().replace(/[^a-z0-9]+/g, '') ?? ''
}
@@ -31,15 +31,15 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</script>
<template>
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:grid-cols-[0.52fr_1.48fr]">
<div>
<div class="mb-3 flex items-baseline justify-between gap-3">
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:min-h-0 xl:grid-cols-[0.52fr_1.48fr]">
<div class="xl:flex xl:min-h-0 xl:flex-col">
<div class="mb-3 flex shrink-0 items-baseline justify-between gap-3">
<h3 class="m-0 text-cp-lg font-heavy text-cp-text">
Token 结构
</h3>
<span class="text-cp-xs font-emphasis text-cp-text-quaternary">当前额度窗口</span>
</div>
<div class="grid gap-2">
<div class="grid gap-2 xl:min-h-0 xl:flex-1 xl:grid-rows-5">
<div class="flex items-center justify-between rounded-lg bg-cp-green-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入 Tokens</span>
<strong class="font-mono text-cp text-cp-text">
@@ -73,8 +73,8 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</div>
</div>
<div class="min-w-0 pt-4 xl:pt-0 xl:pl-4">
<div class="mb-3 flex flex-wrap items-center gap-x-4 gap-y-2">
<div class="min-w-0 pt-4 xl:flex xl:min-h-0 xl:flex-col xl:pt-0 xl:pl-4">
<div class="mb-3 flex shrink-0 flex-wrap items-center gap-x-4 gap-y-2">
<h3 class="m-0 shrink-0 text-cp-lg font-heavy text-cp-text">
模型使用排行
</h3>
@@ -100,7 +100,7 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</div>
</div>
<div class="h-52 min-w-0">
<div class="h-56 min-w-0 xl:h-auto xl:min-h-0 xl:flex-1 xl:basis-0">
<BaseTable
:columns="modelUsageColumns"
:rows="account.usage.models"
+1 -1
View File
@@ -287,7 +287,7 @@ const {
</template>
<template #expanded="{ row }">
<div class="grid gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr]">
<div class="grid items-stretch gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr] xl:min-h-[19.25rem]">
<AccountQuotaPanel
:account="row"
:refreshing="refreshingQuotaAccountIds.has(row.id)"

Some files were not shown because too many files have changed in this diff Show More