mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-12 16:06:16 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87164db3da | ||
|
|
ad16dec3db | ||
|
|
a0da0f4713 | ||
|
|
65e5cde60d | ||
|
|
0f0b464f4e | ||
|
|
bd223f6540 | ||
|
|
168be0ca68 | ||
|
|
f2e91bbdaa | ||
|
|
d0c9248478 | ||
|
|
22c697bbff | ||
|
|
08d518324a | ||
|
|
dfacb20e43 |
@@ -23,7 +23,7 @@
|
||||
|
||||
| 领域 | 当前能力 |
|
||||
| --- | --- |
|
||||
| 数据面 | Responses JSON、SSE、WebSocket、review 子代理、Images 生成/编辑与模型目录 |
|
||||
| 数据面 | Responses JSON、SSE、WebSocket(含 Codex 子代理语义)、Images 生成/编辑与模型目录 |
|
||||
| Provider | 固定 `openai`、`xai` 两个 Provider,各自拥有 OAuth、credential、额度、目录与 transport |
|
||||
| 路由 | Client Key 限定账号分组;按模型能力编译 Provider 候选;支持全局精确模型映射、会话亲和和安全 fallback |
|
||||
| 透明边界 | OpenAI 保留未知请求字段及字段顺序,并原样转发 SSE、WebSocket 与 Images 业务字节;xAI 在 Provider 内转换协议 |
|
||||
@@ -136,12 +136,9 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
| --- | --- |
|
||||
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
|
||||
| `GET /v1/responses` | Responses WebSocket 升级 |
|
||||
| `POST /v1/responses/review` | review 子代理请求 |
|
||||
| `POST /v1/images/generations` | OpenAI Images 生成 JSON 透明转发 |
|
||||
| `POST /v1/images/edits` | OpenAI Images 编辑 JSON 透明转发 |
|
||||
| `GET /v1/models` | 当前 Client Key 账号范围内的模型列表 |
|
||||
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
|
||||
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
|
||||
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
|
||||
|
||||
所有 `/v1/*` 路由都需要 Client Key。Images 是 OpenAI Provider 自有端点,不读取模型字段,也不参与
|
||||
|
||||
@@ -135,13 +135,11 @@ pub struct OpsErrorFilter {
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub failure_kind: Option<String>,
|
||||
pub status_code: Option<u16>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
@@ -793,6 +791,18 @@ pub struct OpsError {
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub continuation_affinity_hash: Option<String>,
|
||||
pub continuation_previous_response_id_hash: Option<String>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_id: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub upstream_connection_age_ms: Option<u64>,
|
||||
pub upstream_connection_idle_ms: Option<u64>,
|
||||
pub recovery_request_id: Option<String>,
|
||||
pub recovered_at: Option<DateTime<Utc>>,
|
||||
pub recovery_attempt_count: u32,
|
||||
pub recovery_retry_delay_ms: Option<u64>,
|
||||
pub recovery_total_latency_ms: Option<u64>,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub stable_sort_id: String,
|
||||
}
|
||||
|
||||
@@ -578,6 +578,18 @@ pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
component: error.component,
|
||||
attempt_id: None,
|
||||
account_label,
|
||||
continuation_affinity_hash: error.continuation_affinity_hash,
|
||||
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
|
||||
continuation_unavailable_reason: error.continuation_unavailable_reason,
|
||||
upstream_connection_id: error.upstream_connection_id,
|
||||
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms: error.upstream_connection_age_ms,
|
||||
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
|
||||
recovery_request_id: error.recovery_request_id,
|
||||
recovered_at: error.recovered_at,
|
||||
recovery_attempt_count: error.recovery_attempt_count,
|
||||
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
|
||||
recovery_total_latency_ms: error.recovery_total_latency_ms,
|
||||
},
|
||||
created_at: error.occurred_at,
|
||||
created_at_display: china_datetime(&error.occurred_at),
|
||||
|
||||
@@ -107,14 +107,12 @@ pub struct OpsQuery {
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub failure_class: Option<String>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
@@ -352,13 +350,11 @@ pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, Wir
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.kind.clone()),
|
||||
endpoint: non_empty(query.route.clone()),
|
||||
model: non_empty(query.model.clone()),
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
failure_kind: non_empty(query.failure_class.clone()),
|
||||
status_code,
|
||||
search: non_empty(query.search.clone()),
|
||||
},
|
||||
|
||||
@@ -805,4 +805,16 @@ pub struct OpsErrorMetadataView {
|
||||
pub component: String,
|
||||
pub attempt_id: Option<String>,
|
||||
pub account_label: Option<String>,
|
||||
pub continuation_affinity_hash: Option<String>,
|
||||
pub continuation_previous_response_id_hash: Option<String>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_id: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub upstream_connection_age_ms: Option<u64>,
|
||||
pub upstream_connection_idle_ms: Option<u64>,
|
||||
pub recovery_request_id: Option<String>,
|
||||
pub recovered_at: Option<DateTime<Utc>>,
|
||||
pub recovery_attempt_count: u32,
|
||||
pub recovery_retry_delay_ms: Option<u64>,
|
||||
pub recovery_total_latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
@@ -3,22 +3,21 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
body::Bytes,
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
http::HeaderMap,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::operation::{ImageRequest, ImageRequestKind, Operation, RawJsonPayload};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
error::{engine_error_response, gateway_error_response, protocol_error_response},
|
||||
responses::{PendingExecution, ProtocolError, ProtocolErrorBody, request_client_context},
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::request_client_context,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
@@ -82,13 +81,13 @@ async fn handle_image_request(
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
let started = match service
|
||||
.start_image(client, operation, client_ip, user_agent, endpoint)
|
||||
.start_provider_endpoint(client, operation, client_ip, user_agent, endpoint)
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
collect_image_response(started).await
|
||||
collect_raw_json_response(started).await
|
||||
}
|
||||
|
||||
fn image_operation(
|
||||
@@ -118,116 +117,3 @@ fn image_operation(
|
||||
kind, payload,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn collect_image_response(started: StartedExecution) -> Response {
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
let body = match raw_json_body(events) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
let response = invalid_upstream_response();
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
};
|
||||
let status = session
|
||||
.response_status_code()
|
||||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.filter(StatusCode::is_success)
|
||||
.unwrap_or(StatusCode::OK);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = json_body_response(body, status, &response_headers);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
execution.cancel_and_finalize().await;
|
||||
return invalid_upstream_response();
|
||||
}
|
||||
execution.disarm();
|
||||
response
|
||||
}
|
||||
|
||||
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
continue;
|
||||
};
|
||||
if body.replace(raw).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn json_body_response(
|
||||
body: Bytes,
|
||||
status: StatusCode,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options =
|
||||
crate::openai::responses::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
if !crate::openai::responses::response_header_is_forwardable(
|
||||
header.name(),
|
||||
&connection_options,
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn invalid_upstream_response() -> Response {
|
||||
protocol_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream image response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ pub mod auth;
|
||||
pub mod error;
|
||||
pub mod images;
|
||||
pub mod models;
|
||||
mod provider_endpoint;
|
||||
pub mod responses;
|
||||
pub mod router;
|
||||
pub mod search;
|
||||
pub(crate) mod service;
|
||||
|
||||
@@ -97,50 +97,6 @@ pub(crate) async fn model_detail(
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `GET /v1/models/catalog`。
|
||||
///
|
||||
/// 新架构的目录由 core 按客户端 Provider 范围裁剪;这里不读取或重建任一
|
||||
/// Provider 的 wire catalog,只恢复 Codex 客户端依赖的稳定展示合同。
|
||||
pub(crate) async fn model_catalog(State(state): State<ApiState>, headers: HeaderMap) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let models = service
|
||||
.public_models(&client)
|
||||
.into_iter()
|
||||
.map(|model| catalog_model_json(model.as_str()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
(StatusCode::OK, Json(models)).into_response()
|
||||
}
|
||||
|
||||
/// `GET /v1/models/{model_id}/info`。
|
||||
pub(crate) async fn model_info(
|
||||
State(state): State<ApiState>,
|
||||
headers: HeaderMap,
|
||||
Path(model_id): Path<String>,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let Ok(public_model) = PublicModelId::new(model_id) else {
|
||||
return model_not_found_response().into_response();
|
||||
};
|
||||
if !service.contains_public_model(&client, &public_model) {
|
||||
return model_not_found_response().into_response();
|
||||
}
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(catalog_model_json(public_model.as_str())),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn openai_model_json(id: &str) -> Value {
|
||||
json!({
|
||||
"id": id,
|
||||
@@ -150,22 +106,6 @@ fn openai_model_json(id: &str) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
fn catalog_model_json(id: &str) -> Value {
|
||||
json!({
|
||||
"id": id,
|
||||
"displayName": id,
|
||||
"description": "",
|
||||
"isDefault": false,
|
||||
"supportedReasoningEfforts": [],
|
||||
"defaultReasoningEffort": "",
|
||||
"inputModalities": ["text"],
|
||||
"outputModalities": ["text"],
|
||||
"supportsPersonality": false,
|
||||
"upgrade": Value::Null,
|
||||
"source": "gateway",
|
||||
})
|
||||
}
|
||||
|
||||
fn codex_model_json(profile: &PublicModelProfile, index: usize) -> Value {
|
||||
let id = profile.model().as_str();
|
||||
let presentation = profile.presentation();
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! OpenAI Provider 原生非流式 JSON 端点的公共交付边界。
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
use super::{
|
||||
error::{engine_error_response, protocol_error_response},
|
||||
responses::{PendingExecution, ProtocolError, ProtocolErrorBody},
|
||||
};
|
||||
|
||||
/// 收集一个 Provider 原生 JSON 响应,并在 Core commit 后按原始 bytes 交付。
|
||||
pub(super) async fn collect_raw_json_response(started: StartedExecution) -> Response {
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
let body = match raw_json_body(events) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
let response = invalid_upstream_response();
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
};
|
||||
let status = session
|
||||
.response_status_code()
|
||||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.filter(StatusCode::is_success)
|
||||
.unwrap_or(StatusCode::OK);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = json_body_response(body, status, &response_headers);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
execution.cancel_and_finalize().await;
|
||||
return invalid_upstream_response();
|
||||
}
|
||||
execution.disarm();
|
||||
response
|
||||
}
|
||||
|
||||
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
continue;
|
||||
};
|
||||
if body.replace(raw).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn json_body_response(
|
||||
body: Bytes,
|
||||
status: StatusCode,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options = super::responses::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
if !super::responses::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn invalid_upstream_response() -> Response {
|
||||
protocol_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream JSON response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -32,7 +32,7 @@ use crate::openai::{
|
||||
|
||||
use super::{
|
||||
OpenAiResponsesEncoder, ProtocolErrorBody, ResponseEncodeError,
|
||||
request::{decode_request_with_headers, decode_review_request_with_headers},
|
||||
request::decode_request_with_headers,
|
||||
};
|
||||
|
||||
/// `POST /v1/responses`。
|
||||
@@ -41,37 +41,13 @@ pub(crate) async fn responses(
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_responses(state, connect_info, headers, body, false).await
|
||||
}
|
||||
|
||||
/// `POST /v1/responses/review`。
|
||||
pub(crate) async fn review_responses(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_responses(state, connect_info, headers, body, true).await
|
||||
}
|
||||
|
||||
async fn handle_responses(
|
||||
state: ApiState,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
review: bool,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let decoded = match if review {
|
||||
decode_review_request_with_headers(&body, &headers)
|
||||
} else {
|
||||
decode_request_with_headers(&body, &headers)
|
||||
} {
|
||||
let decoded = match decode_request_with_headers(&body, &headers) {
|
||||
Ok(decoded) => decoded,
|
||||
Err(error) => {
|
||||
return protocol_error_response(StatusCode::BAD_REQUEST, error.protocol_body());
|
||||
@@ -100,11 +76,7 @@ async fn handle_responses(
|
||||
} else {
|
||||
ClientTransport::HttpJson
|
||||
},
|
||||
if review {
|
||||
"/v1/responses/review"
|
||||
} else {
|
||||
"/v1/responses"
|
||||
},
|
||||
"/v1/responses",
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -7,9 +7,9 @@ mod response;
|
||||
pub mod websocket;
|
||||
|
||||
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub(crate) use http::responses;
|
||||
pub(super) use http::{PendingExecution, request_client_context};
|
||||
pub use http::{collect_execution_response, stream_execution_response};
|
||||
pub(crate) use http::{responses, review_responses};
|
||||
pub use request::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, ResponsesRequestMetadata,
|
||||
decode_request_with_headers,
|
||||
|
||||
@@ -13,7 +13,6 @@ use serde_json::{Map, Value};
|
||||
use super::error::RequestDecodeError;
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
@@ -75,8 +74,8 @@ impl OpenAiRequestHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_subagent(&self, body: &mut Map<String, Value>, forced_subagent: Option<&str>) {
|
||||
if let Some(subagent) = forced_subagent.or(self.subagent.as_deref()) {
|
||||
fn apply_subagent(&self, body: &mut Map<String, Value>) {
|
||||
if let Some(subagent) = self.subagent.as_deref() {
|
||||
inject_subagent_metadata(body, subagent);
|
||||
}
|
||||
}
|
||||
@@ -301,20 +300,11 @@ pub fn decode_request_with_headers(
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
decode_request_inner(body, false, &OpenAiRequestHeaders::from_headers(headers))
|
||||
}
|
||||
|
||||
/// 解码 `POST /v1/responses/review` 请求并冻结 review subagent 语义。
|
||||
pub(super) fn decode_review_request_with_headers(
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
decode_request_inner(body, true, &OpenAiRequestHeaders::from_headers(headers))
|
||||
decode_request_inner(body, &OpenAiRequestHeaders::from_headers(headers))
|
||||
}
|
||||
|
||||
pub(super) fn decode_request_inner(
|
||||
body: &[u8],
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
let value =
|
||||
@@ -322,13 +312,12 @@ pub(super) fn decode_request_inner(
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
decode_request_object(object, review, request_headers, RequestDecodeSource::Http)
|
||||
decode_request_object(object, request_headers, RequestDecodeSource::Http)
|
||||
}
|
||||
|
||||
/// 解码已解析的顶层 object;按下游传输来源恢复连接级协议上下文。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
source: RequestDecodeSource,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
@@ -359,7 +348,7 @@ pub(super) fn decode_request_object(
|
||||
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
request_headers.apply_subagent(&mut object, review.then_some(REVIEW_SUBAGENT));
|
||||
request_headers.apply_subagent(&mut object);
|
||||
let frame_turn_metadata = match source {
|
||||
RequestDecodeSource::Http => None,
|
||||
RequestDecodeSource::WebSocketFrame => frame_turn_metadata(&object),
|
||||
|
||||
@@ -43,7 +43,6 @@ fn decode_response_create_inner(
|
||||
}
|
||||
super::super::request::decode_request_object(
|
||||
body,
|
||||
false,
|
||||
request_headers,
|
||||
RequestDecodeSource::WebSocketFrame,
|
||||
)
|
||||
|
||||
@@ -8,8 +8,9 @@ use axum::{
|
||||
|
||||
use super::{
|
||||
images::{image_edits, image_generations},
|
||||
models::{model_catalog, model_detail, model_info, models},
|
||||
responses::{responses, responses_websocket, review_responses},
|
||||
models::{model_detail, models},
|
||||
responses::{responses, responses_websocket},
|
||||
search::standalone_search,
|
||||
};
|
||||
|
||||
use crate::ApiState;
|
||||
@@ -19,11 +20,9 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
.route("/v1/images/generations", post(image_generations))
|
||||
.route("/v1/images/edits", post(image_edits))
|
||||
.route("/v1/alpha/search", post(standalone_search))
|
||||
.route("/v1/responses", get(responses_websocket).post(responses))
|
||||
.route("/v1/responses/review", post(review_responses))
|
||||
.route("/v1/models", get(models))
|
||||
.route("/v1/models/catalog", get(model_catalog))
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
// OpenAI 数据面正文属于客户端/上游协议;代理不能用私有大小上限提前拒绝
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
//! Codex standalone search 非流式 HTTP adapter。
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::{
|
||||
body::Bytes,
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::HeaderMap,
|
||||
response::Response,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::operation::{Operation, RawJsonPayload, StandaloneSearchRequest};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::request_client_context,
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const TURN_METADATA_CONTEXT_KEY: &str = "turn_metadata";
|
||||
|
||||
/// `POST /v1/alpha/search`。
|
||||
pub(crate) async fn standalone_search(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return client_access_error_response(error),
|
||||
};
|
||||
let (client_ip, user_agent) = request_client_context(
|
||||
&headers,
|
||||
connect_info.map(|Extension(ConnectInfo(address))| address),
|
||||
);
|
||||
let operation = match search_operation(body, &headers) {
|
||||
Ok(operation) => operation,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
let started = match service
|
||||
.start_provider_endpoint(client, operation, client_ip, user_agent, "/v1/alpha/search")
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
collect_raw_json_response(started).await
|
||||
}
|
||||
|
||||
fn search_operation(body: Bytes, headers: &HeaderMap) -> Result<Operation, GatewayError> {
|
||||
let mut context = Map::new();
|
||||
if let Some(turn_metadata) = headers
|
||||
.get("x-codex-turn-metadata")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
context.insert(
|
||||
TURN_METADATA_CONTEXT_KEY.to_owned(),
|
||||
Value::String(turn_metadata.to_owned()),
|
||||
);
|
||||
}
|
||||
let payload = RawJsonPayload::new(OPENAI_PROTOCOL, body)
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI protocol identifier is invalid",
|
||||
)
|
||||
})?
|
||||
.with_context(context);
|
||||
Ok(Operation::Search(StandaloneSearchRequest::from_raw_json(
|
||||
payload,
|
||||
)))
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Codex standalone search HTTP adapter。
|
||||
|
||||
mod http;
|
||||
|
||||
pub(crate) use http::standalone_search;
|
||||
@@ -106,7 +106,7 @@ impl OpenAiService {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_image(
|
||||
pub(crate) async fn start_provider_endpoint(
|
||||
&self,
|
||||
client: AuthenticatedClient,
|
||||
operation: gateway_core::operation::Operation,
|
||||
|
||||
@@ -49,6 +49,12 @@ fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_removed_filters() {
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"route": "/v1/responses"})).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"failureClass": "rate_limited"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
|
||||
@@ -571,6 +571,18 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
request_kind: Some("root".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: Some(false),
|
||||
continuation_affinity_hash: Some("affinity-hash".to_owned()),
|
||||
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
|
||||
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
|
||||
upstream_connection_id: Some("connection-id".to_owned()),
|
||||
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
|
||||
upstream_connection_age_ms: Some(12_000),
|
||||
upstream_connection_idle_ms: Some(4_000),
|
||||
recovery_request_id: None,
|
||||
recovered_at: None,
|
||||
recovery_attempt_count: 0,
|
||||
recovery_retry_delay_ms: None,
|
||||
recovery_total_latency_ms: None,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
@@ -607,6 +619,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
|
||||
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
|
||||
"rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"],
|
||||
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
|
||||
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
|
||||
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"provider": "openai",
|
||||
@@ -623,6 +638,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"providerErrorCode": "upstream",
|
||||
"upstreamSendState": "sent",
|
||||
"rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
|
||||
"continuationUnavailableReason": "reused_connection_lost",
|
||||
"upstreamConnectionExitReason": "tcp_reset",
|
||||
"upstreamConnectionAgeMs": 12000,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/openai/images/mod.rs",
|
||||
"src/openai/mod.rs",
|
||||
"src/openai/models.rs",
|
||||
"src/openai/provider_endpoint.rs",
|
||||
"src/openai/responses/error.rs",
|
||||
"src/openai/responses/http.rs",
|
||||
"src/openai/responses/mod.rs",
|
||||
@@ -71,6 +72,8 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/openai/responses/websocket/mod.rs",
|
||||
"src/openai/responses/websocket/protocol.rs",
|
||||
"src/openai/router.rs",
|
||||
"src/openai/search/http.rs",
|
||||
"src/openai/search/mod.rs",
|
||||
"src/openai/service.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
@@ -104,12 +107,14 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/openai/images/mod.rs",
|
||||
"tests/openai/mod.rs",
|
||||
"tests/openai/models.rs",
|
||||
"tests/openai/provider_endpoint.rs",
|
||||
"tests/openai/responses/http.rs",
|
||||
"tests/openai/responses/mod.rs",
|
||||
"tests/openai/responses/websocket/connection.rs",
|
||||
"tests/openai/responses/websocket/mod.rs",
|
||||
"tests/openai/responses/websocket/protocol.rs",
|
||||
"tests/openai/router.rs",
|
||||
"tests/openai/search/mod.rs",
|
||||
];
|
||||
expected.sort_unstable();
|
||||
|
||||
|
||||
@@ -398,3 +398,38 @@ async fn gateway_error_response_should_expose_only_structured_client_visible_ups
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuation_recovery_should_preserve_the_official_retry_signal() {
|
||||
let error = GatewayError::from_provider(
|
||||
&ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Previous response was not found. Retrying the full request.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
)),
|
||||
);
|
||||
|
||||
let response = gateway_error_response(&error);
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read continuation recovery response");
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("continuation recovery JSON");
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(
|
||||
body,
|
||||
serde_json::json!({
|
||||
"error": {
|
||||
"message": "Previous response was not found. Retrying the full request.",
|
||||
"type": "invalid_request_error",
|
||||
"code": "previous_response_not_found"
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ mod auth;
|
||||
mod error;
|
||||
mod images;
|
||||
mod models;
|
||||
mod provider_endpoint;
|
||||
mod responses;
|
||||
mod router;
|
||||
mod search;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
@@ -383,46 +383,6 @@ async fn model_detail_should_hide_unknown_models() {
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_catalog_should_keep_the_codex_catalog_contract() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/catalog"))
|
||||
.await
|
||||
.expect("catalog response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read catalog body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("catalog JSON");
|
||||
assert_eq!(value[0]["id"], "model-a");
|
||||
assert_eq!(value[0]["displayName"], "model-a");
|
||||
assert_eq!(value[0]["source"], "gateway");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_info_should_return_a_single_visible_catalog_entry() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/model-a/info"))
|
||||
.await
|
||||
.expect("model info response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_info_should_hide_unknown_models() {
|
||||
let response = api_router(ModelsExecution::new())
|
||||
.await
|
||||
.oneshot(authorized_request("/v1/models/model-private/info"))
|
||||
.await
|
||||
.expect("missing model info response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn models_should_authenticate_before_applying_the_version_gate() {
|
||||
let response = api_router(ModelsExecution::with_cli_min())
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
use std::sync::{Arc, Mutex, atomic::AtomicBool, atomic::Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::execution::ExecutionSession;
|
||||
use gateway_core::engine::{CoordinatedEvent, EngineError};
|
||||
use gateway_core::error::ProviderError;
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
pub(super) struct BufferedJsonSession {
|
||||
body: Option<Bytes>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
response_status: Option<u16>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
finalized: AtomicBool,
|
||||
failure: Option<ProviderError>,
|
||||
}
|
||||
|
||||
impl BufferedJsonSession {
|
||||
pub(super) fn success(
|
||||
body: Bytes,
|
||||
response_status: u16,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
body: Some(body),
|
||||
response_headers,
|
||||
response_status: Some(response_status),
|
||||
committed_statuses,
|
||||
finalized: AtomicBool::new(false),
|
||||
failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn failure(error: ProviderError) -> Self {
|
||||
Self {
|
||||
body: None,
|
||||
response_headers: Vec::new(),
|
||||
response_status: None,
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
finalized: AtomicBool::new(false),
|
||||
failure: Some(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionSession for BufferedJsonSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async { unreachable!("buffered JSON delivery does not stream") })
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
if let Some(error) = self.failure.take() {
|
||||
return Err(EngineError::Provider(error));
|
||||
}
|
||||
Ok(vec![ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_json(
|
||||
"openai",
|
||||
self.body.take().expect("single buffered JSON response"),
|
||||
)
|
||||
.expect("OpenAI protocol"),
|
||||
)])
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
self.response_status
|
||||
}
|
||||
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock")
|
||||
.push(client_status_code.expect("buffered JSON response status"));
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ use serde_json::{Value, json};
|
||||
use gateway_api::openai::responses::{collect_execution_response, stream_execution_response};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::openai::{api_router, authenticated_client, authenticated_client_for_provider};
|
||||
use crate::openai::{api_router, authenticated_client_for_provider};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Trace {
|
||||
@@ -689,92 +689,6 @@ async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
assert!(captured.prompt_cache_key.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.header("x-openai-subagent", "compact")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": {"existing": "preserved"}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(captured.endpoint, "/v1/responses/review");
|
||||
assert_eq!(
|
||||
captured
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent")),
|
||||
Some(&json!("review"))
|
||||
);
|
||||
assert_eq!(
|
||||
captured
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("existing")),
|
||||
Some(&json!("preserved"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_not_replace_non_object_client_metadata() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": ["opaque", "client", "value"]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(
|
||||
captured.client_metadata,
|
||||
Some(json!(["opaque", "client", "value"]))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode},
|
||||
http::{Method, Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
@@ -9,6 +9,54 @@ use super::models::ModelsExecution;
|
||||
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_responses_review_route_should_not_reach_the_responses_handler() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed review request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_models_catalog_extension_should_use_standard_model_lookup() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/catalog")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model catalog request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model catalog request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_model_info_route_should_return_not_found() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/model-a/info")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model info request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model info request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::ModelRequestId;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionService,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
use gateway_core::operation::Operation;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::provider_endpoint::BufferedJsonSession;
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
const SEARCH_RESPONSE: &[u8] = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedSearchRequest {
|
||||
provider: String,
|
||||
endpoint: String,
|
||||
transport: ClientTransport,
|
||||
body: Bytes,
|
||||
context: Value,
|
||||
}
|
||||
|
||||
struct SearchExecution {
|
||||
client: AuthenticatedClient,
|
||||
captured: Mutex<Vec<CapturedSearchRequest>>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
fail_with_upstream_response: bool,
|
||||
}
|
||||
|
||||
impl SearchExecution {
|
||||
fn new(fail_with_upstream_response: bool) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_search_test"),
|
||||
captured: Mutex::new(Vec::new()),
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
fail_with_upstream_response,
|
||||
})
|
||||
}
|
||||
|
||||
fn captured(&self) -> Vec<CapturedSearchRequest> {
|
||||
self.captured.lock().expect("capture lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionService for SearchExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_search_test")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"search test must use provider endpoint execution",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let Operation::Search(search) = &request.operation else {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"search test received a non-search operation",
|
||||
));
|
||||
};
|
||||
let payload = search.payload();
|
||||
self.captured
|
||||
.lock()
|
||||
.expect("capture lock")
|
||||
.push(CapturedSearchRequest {
|
||||
provider: request.provider.as_str().to_owned(),
|
||||
endpoint: request.metadata.endpoint,
|
||||
transport: request.metadata.transport,
|
||||
body: payload.body().clone(),
|
||||
context: Value::Object(payload.context().clone()),
|
||||
});
|
||||
let session = if self.fail_with_upstream_response {
|
||||
BufferedJsonSession::failure(
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(StatusCode::UNPROCESSABLE_ENTITY.as_u16())
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
StatusCode::UNPROCESSABLE_ENTITY.as_u16(),
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
Bytes::from_static(
|
||||
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#,
|
||||
),
|
||||
)
|
||||
.with_headers(vec![ProviderResponseHeader::new(
|
||||
"x-future-search-error",
|
||||
Bytes::from_static(b"preserved"),
|
||||
)]),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
BufferedJsonSession::success(
|
||||
Bytes::from_static(SEARCH_RESPONSE),
|
||||
StatusCode::CREATED.as_u16(),
|
||||
vec![ProviderResponseHeader::new(
|
||||
"x-search-rate-limit",
|
||||
Bytes::from_static(b"42"),
|
||||
)],
|
||||
Arc::clone(&self.committed_statuses),
|
||||
)
|
||||
};
|
||||
Ok(StartedExecution {
|
||||
request_id: ModelRequestId::new("req_search_test").expect("request ID"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: false,
|
||||
session: Box::new(session),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_route_should_preserve_request_and_success_response_bytes() {
|
||||
let execution = SearchExecution::new(false);
|
||||
let request_body = br#"{ "id":"search-session", "model":"gpt-future", "commands":{"search_query":[{"q":"private query"}]}, "future":1, "future":2 }"#;
|
||||
let turn_metadata =
|
||||
r#"{"session_id":"session","thread_id":"thread","turn_id":"turn","future":true}"#;
|
||||
let response = api_router(execution.clone())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/alpha/search")
|
||||
.header(AUTHORIZATION, "Bearer sk_search_test")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-codex-turn-metadata", turn_metadata)
|
||||
.body(Body::from(request_body.to_vec()))
|
||||
.expect("search request"),
|
||||
)
|
||||
.await
|
||||
.expect("search response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
assert_eq!(
|
||||
response.headers().get("x-search-rate-limit"),
|
||||
Some(&"42".parse().expect("header value"))
|
||||
);
|
||||
let response_body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read search response");
|
||||
assert_eq!(response_body.as_ref(), SEARCH_RESPONSE);
|
||||
|
||||
let captured = execution.captured();
|
||||
assert_eq!(captured.len(), 1);
|
||||
assert_eq!(captured[0].provider, "openai");
|
||||
assert_eq!(captured[0].endpoint, "/v1/alpha/search");
|
||||
assert_eq!(captured[0].transport, ClientTransport::HttpJson);
|
||||
assert_eq!(captured[0].body.as_ref(), request_body);
|
||||
assert_eq!(captured[0].context, json!({"turn_metadata": turn_metadata}));
|
||||
assert_eq!(
|
||||
*execution
|
||||
.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock"),
|
||||
vec![StatusCode::CREATED.as_u16()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_route_should_return_the_exact_upstream_error_response() {
|
||||
let response = api_router(SearchExecution::new(true))
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/alpha/search")
|
||||
.header(AUTHORIZATION, "Bearer sk_search_test")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
br#"{ "model":"gpt-future", "future_invalid":9007199254740993 }"#.to_vec(),
|
||||
))
|
||||
.expect("search request"),
|
||||
)
|
||||
.await
|
||||
.expect("search response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(
|
||||
response.headers().get("content-type"),
|
||||
Some(&"application/problem+json".parse().expect("content type"))
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers().get("x-future-search-error"),
|
||||
Some(&"preserved".parse().expect("header value"))
|
||||
);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read search response");
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
br#"{ "error":{"message":"future search validation"}, "future":9007199254740993 }"#
|
||||
);
|
||||
}
|
||||
@@ -3,9 +3,9 @@
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::sync::{Mutex, RwLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use crate::routing::ProviderKind;
|
||||
|
||||
@@ -94,6 +94,7 @@ pub struct AccountRuntimeSignals {
|
||||
}
|
||||
|
||||
const ACCOUNT_FEEDBACK_EWMA_ALPHA: f64 = 0.2;
|
||||
const ACCOUNT_FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
|
||||
const EMPTY_FEEDBACK_SAMPLE: u64 = f64::NAN.to_bits();
|
||||
|
||||
/// 一次真实上游 attempt 对账号级 Smart 调度产生的中立反馈。
|
||||
@@ -111,37 +112,69 @@ struct AccountFeedbackKey {
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AccountFeedback {
|
||||
failure_rate: AtomicU64,
|
||||
failure_rate: Mutex<DecayingAccountFailureRate>,
|
||||
first_output_ms: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct DecayingAccountFailureRate {
|
||||
value: f64,
|
||||
updated_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl DecayingAccountFailureRate {
|
||||
fn value_at(&self, now: Instant) -> f64 {
|
||||
let Some(updated_at) = self.updated_at else {
|
||||
return self.value;
|
||||
};
|
||||
let elapsed = now.checked_duration_since(updated_at).unwrap_or_default();
|
||||
self.value
|
||||
* 0.5_f64.powf(elapsed.as_secs_f64() / ACCOUNT_FAILURE_RATE_HALF_LIFE.as_secs_f64())
|
||||
}
|
||||
|
||||
fn report_at(&mut self, sample: f64, now: Instant) {
|
||||
let now = self
|
||||
.updated_at
|
||||
.map_or(now, |updated_at| updated_at.max(now));
|
||||
let decayed = self.value_at(now);
|
||||
self.value =
|
||||
ACCOUNT_FEEDBACK_EWMA_ALPHA * sample + (1.0 - ACCOUNT_FEEDBACK_EWMA_ALPHA) * decayed;
|
||||
self.updated_at = Some(now);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AccountFeedback {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// 新账号从健康基线开始,首个失败样本按 EWMA 平滑,而不是直接判死。
|
||||
failure_rate: AtomicU64::new(0.0_f64.to_bits()),
|
||||
failure_rate: Mutex::new(DecayingAccountFailureRate::default()),
|
||||
first_output_ms: AtomicU64::new(EMPTY_FEEDBACK_SAMPLE),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AccountFeedback {
|
||||
fn report(&self, feedback: AccountAttemptFeedback) {
|
||||
fn report_at(&self, feedback: AccountAttemptFeedback, now: Instant) {
|
||||
let (failure, first_output_ms) = match feedback {
|
||||
AccountAttemptFeedback::Succeeded { first_output_ms } => (0.0, first_output_ms),
|
||||
AccountAttemptFeedback::Failed { first_output_ms } => (1.0, first_output_ms),
|
||||
};
|
||||
update_feedback_ewma(&self.failure_rate, failure);
|
||||
self.failure_rate
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.report_at(failure, now);
|
||||
if let Some(first_output_ms) = first_output_ms.filter(|value| *value > 0) {
|
||||
update_feedback_ewma(&self.first_output_ms, first_output_ms as f64);
|
||||
}
|
||||
}
|
||||
|
||||
fn scheduling_signals(&self) -> (Option<u16>, Option<u64>) {
|
||||
let failure_rate = load_feedback_ewma(&self.failure_rate).map(|value| {
|
||||
let basis_points = (value.clamp(0.0, 1.0) * 10_000.0).round();
|
||||
basis_points as u16
|
||||
});
|
||||
fn scheduling_signals_at(&self, now: Instant) -> (Option<u16>, Option<u64>) {
|
||||
let failure_rate = self
|
||||
.failure_rate
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.value_at(now);
|
||||
let failure_rate = Some((failure_rate.clamp(0.0, 1.0) * 10_000.0).round() as u16);
|
||||
let first_output_ms = load_feedback_ewma(&self.first_output_ms)
|
||||
.filter(|value| value.is_finite() && *value > 0.0)
|
||||
.map(|value| value.round() as u64);
|
||||
@@ -162,6 +195,18 @@ impl AccountFeedbackStats {
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> (Option<u16>, Option<u64>) {
|
||||
self.scheduling_signals_at(provider_kind, account_id, Instant::now())
|
||||
}
|
||||
|
||||
/// 读取账号在指定单调时刻的错误率与首个有效输出延迟 EWMA。
|
||||
#[doc(hidden)]
|
||||
#[must_use]
|
||||
pub fn scheduling_signals_at(
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
now: Instant,
|
||||
) -> (Option<u16>, Option<u64>) {
|
||||
let key = AccountFeedbackKey {
|
||||
provider_kind: provider_kind.clone(),
|
||||
@@ -171,7 +216,7 @@ impl AccountFeedbackStats {
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&key)
|
||||
.map(AccountFeedback::scheduling_signals)
|
||||
.map(|account| account.scheduling_signals_at(now))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
@@ -181,6 +226,18 @@ impl AccountFeedbackStats {
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
feedback: AccountAttemptFeedback,
|
||||
) {
|
||||
self.report_at(provider_kind, account_id, feedback, Instant::now());
|
||||
}
|
||||
|
||||
/// 回灌一次在指定单调时刻真实发送的上游 attempt。
|
||||
#[doc(hidden)]
|
||||
pub fn report_at(
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
account_id: &ProviderAccountId,
|
||||
feedback: AccountAttemptFeedback,
|
||||
observed_at: Instant,
|
||||
) {
|
||||
let key = AccountFeedbackKey {
|
||||
provider_kind: provider_kind.clone(),
|
||||
@@ -192,7 +249,7 @@ impl AccountFeedbackStats {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&key)
|
||||
{
|
||||
account.report(feedback);
|
||||
account.report_at(feedback, observed_at);
|
||||
return;
|
||||
}
|
||||
self.accounts
|
||||
@@ -200,7 +257,7 @@ impl AccountFeedbackStats {
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.entry(key)
|
||||
.or_default()
|
||||
.report(feedback);
|
||||
.report_at(feedback, observed_at);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,9 +13,9 @@ use crate::engine::provider::{Provider, ProviderCallMetadata, ProviderRequest, P
|
||||
use crate::engine::{
|
||||
AccountAttemptContext, AttemptContext, AttemptRecord, AttemptTransport, AttemptTrigger,
|
||||
CommitRequirement, ContinuationAttempt, CoordinatedEvent, EngineError, ExecutionOutcome,
|
||||
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
|
||||
RequestAttemptContext, UpstreamSendState,
|
||||
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFailureObservation,
|
||||
ModelRequestFinalization, ModelRequestId, ModelRequestTimings, NewModelRequest,
|
||||
ProviderAccountStateOwner, ProviderAttemptOutcome, RequestAttemptContext, UpstreamSendState,
|
||||
};
|
||||
use crate::error::{
|
||||
ContinuationRecoveryDisposition, GatewayError, GatewayErrorKind, ProviderError,
|
||||
@@ -221,6 +221,7 @@ struct FailureFinalization {
|
||||
provider_error_code: Option<String>,
|
||||
raw_upstream_error: Option<String>,
|
||||
retry_after_ms: Option<u64>,
|
||||
observation: ModelRequestFailureObservation,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -638,6 +639,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
@@ -795,6 +797,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::ProviderMetadataMismatch);
|
||||
@@ -820,6 +823,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::AccountOutsideClientScope);
|
||||
@@ -840,6 +844,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::RequiredAccountMismatch);
|
||||
@@ -863,6 +868,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::ContinuationPinMismatch);
|
||||
@@ -1368,6 +1374,7 @@ where
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
failure_observation: ModelRequestFailureObservation::default(),
|
||||
retry_after_ms: None,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
@@ -1410,6 +1417,12 @@ where
|
||||
.raw_upstream_error()
|
||||
.map(|raw| raw.as_str().to_owned()),
|
||||
retry_after_ms: error.retry_after().map(duration_ms),
|
||||
observation: ModelRequestFailureObservation {
|
||||
continuation_unavailable_reason: error
|
||||
.continuation_unavailable_reason()
|
||||
.map(str::to_owned),
|
||||
upstream_connection: error.connection_observation().cloned(),
|
||||
},
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -1464,6 +1477,7 @@ where
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
retry_after_ms: None,
|
||||
observation: ModelRequestFailureObservation::default(),
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -1532,6 +1546,7 @@ where
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
raw_upstream_error: finalization.raw_upstream_error,
|
||||
failure_observation: finalization.observation,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
|
||||
@@ -505,8 +505,11 @@ impl DefaultExecutionService {
|
||||
.candidates()
|
||||
.first()
|
||||
.map_or_else(Default::default, |candidate| {
|
||||
self.providers
|
||||
.request_observation(candidate.provider(), &operation)
|
||||
self.providers.request_observation(
|
||||
candidate.provider(),
|
||||
&operation,
|
||||
client.policy.key_id(),
|
||||
)
|
||||
});
|
||||
let new_request = NewModelRequest {
|
||||
id: request_id.clone(),
|
||||
@@ -526,6 +529,7 @@ impl DefaultExecutionService {
|
||||
request_kind: observation.request_kind,
|
||||
subagent_kind: observation.subagent_kind,
|
||||
compact: observation.compact,
|
||||
continuation: observation.continuation,
|
||||
image_generation_requested: operation.image_generation_requested(),
|
||||
admission_decision_ms: Some(admission_decision_ms),
|
||||
started_at,
|
||||
@@ -668,6 +672,7 @@ impl DefaultExecutionService {
|
||||
request_kind: Some("account_connection_test".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
admission_decision_ms: None,
|
||||
started_at,
|
||||
|
||||
@@ -22,7 +22,8 @@ use thiserror::Error;
|
||||
use crate::account::{AccountSelectionPolicy, ProviderAccountId};
|
||||
use crate::engine::continuation::{ContinuationBinding, NativeContinuationPin};
|
||||
use crate::error::{
|
||||
GatewayError, IdentifierError, ProviderError, ProviderErrorKind, StoreError, validate_text,
|
||||
GatewayError, IdentifierError, ProviderConnectionObservation, ProviderError, ProviderErrorKind,
|
||||
StoreError, validate_text,
|
||||
};
|
||||
use crate::event::ProviderEvent;
|
||||
use crate::lifecycle::CancellationToken;
|
||||
@@ -514,6 +515,7 @@ pub struct NewModelRequest {
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: bool,
|
||||
pub continuation: provider::ContinuationRequestObservation,
|
||||
pub image_generation_requested: bool,
|
||||
/// Client Key 准入判定的完整耗时;内部探测不经过该阶段。
|
||||
pub admission_decision_ms: Option<u64>,
|
||||
@@ -579,6 +581,13 @@ pub struct ModelRequestTimings {
|
||||
pub latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// 终态失败的 Provider 结构化连接与 continuation 观测。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelRequestFailureObservation {
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection: Option<ProviderConnectionObservation>,
|
||||
}
|
||||
|
||||
/// 单行模型请求的终态写回。
|
||||
#[derive(Debug)]
|
||||
pub struct ModelRequestFinalization {
|
||||
@@ -602,6 +611,7 @@ pub struct ModelRequestFinalization {
|
||||
pub provider_error_code: Option<String>,
|
||||
/// Provider 返回的原始错误正文或 WebSocket close/error frame。
|
||||
pub raw_upstream_error: Option<String>,
|
||||
pub failure_observation: ModelRequestFailureObservation,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub usage: Usage,
|
||||
pub image_generation_succeeded: Option<bool>,
|
||||
|
||||
@@ -15,9 +15,10 @@ use crate::account::{
|
||||
AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, ProviderAccountId,
|
||||
};
|
||||
use crate::engine::AttemptContext;
|
||||
use crate::error::{OpaqueUpstreamValue, ProviderError, ProviderErrorKind};
|
||||
use crate::error::{OpaqueUpstreamValue, PreDeliveryRetry, ProviderError, ProviderErrorKind};
|
||||
use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::operation::Operation;
|
||||
use crate::policy::ClientApiKeyId;
|
||||
use crate::routing::{
|
||||
ModelCapabilities, ModelPresentation, ProviderCandidate, ProviderKind, UpstreamModelId,
|
||||
};
|
||||
@@ -190,11 +191,16 @@ struct ProviderStreamAccountFeedback {
|
||||
stats: Arc<AccountFeedbackStats>,
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
failure_filter: fn(&ProviderError) -> bool,
|
||||
started_at: Option<Instant>,
|
||||
first_output_ms: Option<u64>,
|
||||
reported: bool,
|
||||
}
|
||||
|
||||
fn score_all_confirmed_failures(_: &ProviderError) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl ProviderStreamAccountFeedback {
|
||||
fn start(&mut self) {
|
||||
self.started_at.get_or_insert_with(Instant::now);
|
||||
@@ -243,6 +249,15 @@ impl ProviderStreamAccountFeedback {
|
||||
error.kind(),
|
||||
ProviderErrorKind::Cancelled | ProviderErrorKind::ProcessTerminated
|
||||
)
|
||||
|| matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(
|
||||
PreDeliveryRetry::SameAccountTransportRetry { .. }
|
||||
| PreDeliveryRetry::SameAccountTransportFallback
|
||||
)
|
||||
)
|
||||
|| error.retries_same_account()
|
||||
|| !(self.failure_filter)(error)
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -278,15 +293,35 @@ impl ProviderStream {
|
||||
/// 让公共 stream 边界统一回灌账号成功率与首个有效输出延迟。
|
||||
#[must_use]
|
||||
pub fn with_account_feedback(mut self, stats: Arc<AccountFeedbackStats>) -> Self {
|
||||
self.set_account_feedback(stats, score_all_confirmed_failures);
|
||||
self
|
||||
}
|
||||
|
||||
/// 使用 Provider 定义的闭集判断回灌账号成功率与首个有效输出延迟。
|
||||
#[must_use]
|
||||
pub fn with_filtered_account_feedback(
|
||||
mut self,
|
||||
stats: Arc<AccountFeedbackStats>,
|
||||
failure_filter: fn(&ProviderError) -> bool,
|
||||
) -> Self {
|
||||
self.set_account_feedback(stats, failure_filter);
|
||||
self
|
||||
}
|
||||
|
||||
fn set_account_feedback(
|
||||
&mut self,
|
||||
stats: Arc<AccountFeedbackStats>,
|
||||
failure_filter: fn(&ProviderError) -> bool,
|
||||
) {
|
||||
self.account_feedback = Some(ProviderStreamAccountFeedback {
|
||||
stats,
|
||||
provider_kind: self.metadata.provider().clone(),
|
||||
account_id: self.metadata.provider_account_id().clone(),
|
||||
failure_filter,
|
||||
started_at: None,
|
||||
first_output_ms: None,
|
||||
reported: false,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// 返回调用事实。
|
||||
@@ -387,6 +422,15 @@ pub struct ProviderRequestObservation {
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: bool,
|
||||
pub continuation: ContinuationRequestObservation,
|
||||
}
|
||||
|
||||
/// 仅用于恢复事件关联的客户端作用域不透明请求事实。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ContinuationRequestObservation {
|
||||
pub affinity_hash: Option<String>,
|
||||
pub previous_response_id_hash: Option<String>,
|
||||
pub requested: bool,
|
||||
}
|
||||
|
||||
/// Provider 实时目录编译后的单模型能力。
|
||||
@@ -495,7 +539,11 @@ pub trait Provider: Send + Sync {
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration;
|
||||
|
||||
/// 解释 Provider 差异化观测字段;不参与路由和传输。
|
||||
fn request_observation(&self, _operation: &Operation) -> ProviderRequestObservation {
|
||||
fn request_observation(
|
||||
&self,
|
||||
_operation: &Operation,
|
||||
_client_api_key_id: &ClientApiKeyId,
|
||||
) -> ProviderRequestObservation {
|
||||
ProviderRequestObservation::default()
|
||||
}
|
||||
|
||||
@@ -623,11 +671,12 @@ impl ProviderRegistry {
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
operation: &Operation,
|
||||
client_api_key_id: &ClientApiKeyId,
|
||||
) -> ProviderRequestObservation {
|
||||
self.providers
|
||||
.get(provider)
|
||||
.map_or_else(ProviderRequestObservation::default, |registered| {
|
||||
registered.request_observation(operation)
|
||||
registered.request_observation(operation, client_api_key_id)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -135,6 +135,8 @@ pub enum MeteringError {
|
||||
pub enum ProviderErrorKind {
|
||||
/// 请求语义错误。
|
||||
InvalidRequest,
|
||||
/// previous-response 续接状态已不可用,需要客户端重建请求链。
|
||||
ContinuationRecoveryRequired,
|
||||
/// Provider 不支持所需能力。
|
||||
Unsupported,
|
||||
/// Credential 认证失败。
|
||||
@@ -171,6 +173,7 @@ impl ProviderErrorKind {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::InvalidRequest => "invalid_request",
|
||||
Self::ContinuationRecoveryRequired => "continuation_recovery_required",
|
||||
Self::Unsupported => "unsupported",
|
||||
Self::Unauthorized => "unauthorized",
|
||||
Self::PermissionDenied => "permission_denied",
|
||||
@@ -357,6 +360,55 @@ impl fmt::Debug for RawUpstreamError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 上游物理连接结束时的低基数观测。
|
||||
///
|
||||
/// 连接标识必须是随机或单向派生值,不得携带凭据、原始会话 ID
|
||||
/// 或请求正文。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderConnectionObservation {
|
||||
connection_id: String,
|
||||
exit_reason: String,
|
||||
age_ms: u64,
|
||||
idle_ms: u64,
|
||||
}
|
||||
|
||||
impl ProviderConnectionObservation {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
connection_id: impl Into<String>,
|
||||
exit_reason: impl Into<String>,
|
||||
age_ms: u64,
|
||||
idle_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
connection_id: connection_id.into(),
|
||||
exit_reason: exit_reason.into(),
|
||||
age_ms,
|
||||
idle_ms,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn connection_id(&self) -> &str {
|
||||
&self.connection_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn exit_reason(&self) -> &str {
|
||||
&self.exit_reason
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn age_ms(&self) -> u64 {
|
||||
self.age_ms
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn idle_ms(&self) -> u64 {
|
||||
self.idle_ms
|
||||
}
|
||||
}
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
@@ -442,6 +494,7 @@ pub struct ProviderError {
|
||||
retry_after: Option<Duration>,
|
||||
continuation_failure: Option<ContinuationFailure>,
|
||||
continuation_recovery_disposition: Option<ContinuationRecoveryDisposition>,
|
||||
failure_observation: Option<Box<ProviderErrorFailureObservation>>,
|
||||
replay_safe: bool,
|
||||
pre_delivery_retry: Option<PreDeliveryRetry>,
|
||||
credential_recovery_required: bool,
|
||||
@@ -460,6 +513,12 @@ struct ProviderErrorUpstreamValues {
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorFailureObservation {
|
||||
continuation_unavailable_reason: Option<&'static str>,
|
||||
connection: Option<ProviderConnectionObservation>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AtomicClientEvents(Vec<ProviderEvent>);
|
||||
|
||||
@@ -475,6 +534,7 @@ impl ProviderError {
|
||||
retry_after: None,
|
||||
continuation_failure: None,
|
||||
continuation_recovery_disposition: None,
|
||||
failure_observation: None,
|
||||
replay_safe: false,
|
||||
pre_delivery_retry: None,
|
||||
credential_recovery_required: false,
|
||||
@@ -540,6 +600,29 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 Provider 给出的低基数 continuation 不可用原因。
|
||||
#[must_use]
|
||||
pub fn with_continuation_unavailable_reason(mut self, reason: &'static str) -> Self {
|
||||
self.failure_observation_mut()
|
||||
.continuation_unavailable_reason = Some(reason);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加上游物理连接结束观测。
|
||||
#[must_use]
|
||||
pub fn with_connection_observation(
|
||||
mut self,
|
||||
observation: ProviderConnectionObservation,
|
||||
) -> Self {
|
||||
self.failure_observation_mut().connection = Some(observation);
|
||||
self
|
||||
}
|
||||
|
||||
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
|
||||
self.failure_observation
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
|
||||
}
|
||||
|
||||
/// 标记 Provider 已证明本次拒绝没有执行生成,可在下游提交前重放。
|
||||
#[must_use]
|
||||
pub const fn with_replay_safe(mut self) -> Self {
|
||||
@@ -714,6 +797,21 @@ impl ProviderError {
|
||||
self.continuation_recovery_disposition
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn continuation_unavailable_reason(&self) -> Option<&'static str> {
|
||||
match self.failure_observation.as_deref() {
|
||||
Some(observation) => observation.continuation_unavailable_reason,
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn connection_observation(&self) -> Option<&ProviderConnectionObservation> {
|
||||
self.failure_observation
|
||||
.as_deref()
|
||||
.and_then(|observation| observation.connection.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否已证明本次失败可安全重放。
|
||||
#[must_use]
|
||||
pub const fn replay_is_safe(&self) -> bool {
|
||||
@@ -801,6 +899,7 @@ impl Clone for ProviderError {
|
||||
retry_after: self.retry_after,
|
||||
continuation_failure: self.continuation_failure,
|
||||
continuation_recovery_disposition: self.continuation_recovery_disposition,
|
||||
failure_observation: self.failure_observation.clone(),
|
||||
replay_safe: self.replay_safe,
|
||||
pre_delivery_retry: self.pre_delivery_retry,
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
@@ -836,6 +935,11 @@ impl fmt::Debug for ProviderError {
|
||||
"continuation_recovery_disposition",
|
||||
&self.continuation_recovery_disposition,
|
||||
)
|
||||
.field(
|
||||
"continuation_unavailable_reason",
|
||||
&self.continuation_unavailable_reason(),
|
||||
)
|
||||
.field("connection_observation", &self.connection_observation())
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
@@ -968,6 +1072,10 @@ impl GatewayError {
|
||||
ProviderErrorKind::InvalidRequest => {
|
||||
Self::new(GatewayErrorKind::InvalidRequest, "invalid upstream request")
|
||||
}
|
||||
ProviderErrorKind::ContinuationRecoveryRequired => Self::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"conversation continuation must be rebuilt",
|
||||
),
|
||||
ProviderErrorKind::Unsupported => Self::new(
|
||||
GatewayErrorKind::Unsupported,
|
||||
"requested capability is unsupported",
|
||||
|
||||
@@ -22,6 +22,8 @@ pub enum OperationKind {
|
||||
Generate,
|
||||
/// 图像生成。
|
||||
GenerateImage,
|
||||
/// Provider 原生 standalone search。
|
||||
Search,
|
||||
}
|
||||
|
||||
impl OperationKind {
|
||||
@@ -31,6 +33,7 @@ impl OperationKind {
|
||||
match self {
|
||||
Self::Generate => "generate",
|
||||
Self::GenerateImage => "generate_image",
|
||||
Self::Search => "search",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -501,6 +504,35 @@ impl fmt::Debug for ImageRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 原生 standalone search 请求。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct StandaloneSearchRequest {
|
||||
payload: RawJsonPayload,
|
||||
}
|
||||
|
||||
impl StandaloneSearchRequest {
|
||||
/// 创建携带原始协议 JSON 正文的 standalone search 请求。
|
||||
#[must_use]
|
||||
pub const fn from_raw_json(payload: RawJsonPayload) -> Self {
|
||||
Self { payload }
|
||||
}
|
||||
|
||||
/// 返回未经重编码的协议 JSON 正文。
|
||||
#[must_use]
|
||||
pub const fn payload(&self) -> &RawJsonPayload {
|
||||
&self.payload
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for StandaloneSearchRequest {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("StandaloneSearchRequest")
|
||||
.field("payload", &"<not included in Debug>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 网关内部业务请求;不包含任何客户端 wire 或 Provider SDK 类型。
|
||||
#[derive(Clone, PartialEq)]
|
||||
#[non_exhaustive]
|
||||
@@ -509,6 +541,8 @@ pub enum Operation {
|
||||
Generate(GenerateRequest),
|
||||
/// 图像生成。
|
||||
GenerateImage(ImageRequest),
|
||||
/// Provider 原生 standalone search。
|
||||
Search(StandaloneSearchRequest),
|
||||
}
|
||||
|
||||
impl Operation {
|
||||
@@ -534,6 +568,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(_) => OperationKind::Generate,
|
||||
Self::GenerateImage(_) => OperationKind::GenerateImage,
|
||||
Self::Search(_) => OperationKind::Search,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -543,6 +578,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(request) => request.requirements(),
|
||||
Self::GenerateImage(_) => CapabilityRequirements::new(OperationKind::GenerateImage),
|
||||
Self::Search(_) => CapabilityRequirements::new(OperationKind::Search),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,6 +588,7 @@ impl Operation {
|
||||
match self {
|
||||
Self::Generate(request) => request.image_generation_requested(),
|
||||
Self::GenerateImage(_) => true,
|
||||
Self::Search(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -560,7 +597,7 @@ impl Operation {
|
||||
pub fn provider_session_state(&self, provider: &str) -> Option<&ProviderSessionState> {
|
||||
match self {
|
||||
Self::Generate(request) => request.provider_session_state(provider),
|
||||
Self::GenerateImage(_) => None,
|
||||
Self::GenerateImage(_) | Self::Search(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
mod selection;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use gateway_core::account::{AccountAttemptFeedback, AccountFeedbackStats, ProviderAccountId};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
const FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
fn feedback_subject() -> (AccountFeedbackStats, ProviderKind, ProviderAccountId) {
|
||||
(
|
||||
AccountFeedbackStats::default(),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
ProviderAccountId::new("acct_decay").expect("valid account"),
|
||||
)
|
||||
}
|
||||
|
||||
fn report_failure(
|
||||
feedback: &AccountFeedbackStats,
|
||||
provider: &ProviderKind,
|
||||
account: &ProviderAccountId,
|
||||
observed_at: Instant,
|
||||
) {
|
||||
feedback.report_at(
|
||||
provider,
|
||||
account,
|
||||
AccountAttemptFeedback::Failed {
|
||||
first_output_ms: None,
|
||||
},
|
||||
observed_at,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_failure_rate_should_halve_after_one_half_life() {
|
||||
let (feedback, provider, account) = feedback_subject();
|
||||
let observed_at = Instant::now();
|
||||
report_failure(&feedback, &provider, &account, observed_at);
|
||||
|
||||
let failure_rate = feedback
|
||||
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
|
||||
.0;
|
||||
|
||||
assert_eq!(failure_rate, Some(1_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_failure_rate_should_quarter_after_two_half_lives() {
|
||||
let (feedback, provider, account) = feedback_subject();
|
||||
let observed_at = Instant::now();
|
||||
report_failure(&feedback, &provider, &account, observed_at);
|
||||
|
||||
let failure_rate = feedback
|
||||
.scheduling_signals_at(
|
||||
&provider,
|
||||
&account,
|
||||
observed_at + FAILURE_RATE_HALF_LIFE * 2,
|
||||
)
|
||||
.0;
|
||||
|
||||
assert_eq!(failure_rate, Some(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_failure_rate_should_decay_before_applying_a_new_sample() {
|
||||
let (feedback, provider, account) = feedback_subject();
|
||||
let observed_at = Instant::now();
|
||||
report_failure(&feedback, &provider, &account, observed_at);
|
||||
feedback.report_at(
|
||||
&provider,
|
||||
&account,
|
||||
AccountAttemptFeedback::Succeeded {
|
||||
first_output_ms: None,
|
||||
},
|
||||
observed_at + FAILURE_RATE_HALF_LIFE,
|
||||
);
|
||||
|
||||
let failure_rate = feedback
|
||||
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
|
||||
.0;
|
||||
|
||||
assert_eq!(failure_rate, Some(800));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_account_failures_should_not_lose_samples() {
|
||||
let (feedback, provider, account) = feedback_subject();
|
||||
let observed_at = Instant::now();
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..8 {
|
||||
scope.spawn(|| report_failure(&feedback, &provider, &account, observed_at));
|
||||
}
|
||||
});
|
||||
|
||||
let failure_rate = feedback
|
||||
.scheduling_signals_at(&provider, &account, observed_at)
|
||||
.0;
|
||||
|
||||
assert_eq!(failure_rate, Some(8_322));
|
||||
}
|
||||
@@ -501,10 +501,13 @@ fn atomic_history_unavailable(response_id: &str) -> ProviderError {
|
||||
)
|
||||
.expect("history unavailable wire"),
|
||||
);
|
||||
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
|
||||
.with_status(400)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_atomic_client_events(vec![failed])
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(400)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_atomic_client_events(vec![failed])
|
||||
}
|
||||
|
||||
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
|
||||
@@ -621,6 +624,7 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
|
||||
request_kind: Some("responses".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: operation.image_generation_requested(),
|
||||
started_at: SystemTime::now(),
|
||||
deadline_at: deadline,
|
||||
@@ -1591,7 +1595,7 @@ fn native_continuation_client_replay_required_is_terminal() {
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
|
||||
@@ -62,6 +62,9 @@ fn only_provider_attributable_failures_should_affect_circuit() {
|
||||
assert!(!provider_failure_affects_circuit(
|
||||
ProviderErrorKind::InvalidRequest
|
||||
));
|
||||
assert!(!provider_failure_affects_circuit(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::{StreamExt, stream};
|
||||
@@ -244,3 +246,139 @@ fn provider_stream_should_report_confirmed_sent_failure_but_ignore_unconfirmed_f
|
||||
(None, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_stream_should_ignore_repeated_failures_rejected_by_the_filter() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("valid provider");
|
||||
let account = ProviderAccountId::new("acct_client_config_error").expect("account");
|
||||
|
||||
for _ in 0..100 {
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("valid model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("http_sse").expect("valid transport"),
|
||||
);
|
||||
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::Sent,
|
||||
))]));
|
||||
let mut provider_stream = ProviderStream::new(metadata, events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(&feedback), |_| false);
|
||||
|
||||
futures::executor::block_on(async {
|
||||
assert!(
|
||||
provider_stream
|
||||
.next()
|
||||
.await
|
||||
.is_some_and(|event| event.is_err())
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
feedback.scheduling_signals(&provider, &account),
|
||||
(None, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_stream_should_score_only_the_final_same_account_failure() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("valid provider");
|
||||
let account = ProviderAccountId::new("acct_same_account_failure").expect("account");
|
||||
|
||||
for retry_index in 1..=3 {
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("valid model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("websocket").expect("valid transport"),
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
|
||||
.with_pre_delivery_transport_retry(
|
||||
NonZeroU32::new(retry_index).expect("positive retry index"),
|
||||
Duration::ZERO,
|
||||
);
|
||||
let events: EventStream = Box::pin(stream::iter([Err(error)]));
|
||||
let mut provider_stream = ProviderStream::new(metadata, events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
|
||||
|
||||
futures::executor::block_on(async {
|
||||
assert!(
|
||||
provider_stream
|
||||
.next()
|
||||
.await
|
||||
.is_some_and(|event| event.is_err())
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("valid model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("websocket").expect("valid transport"),
|
||||
);
|
||||
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
))]));
|
||||
let mut provider_stream = ProviderStream::new(metadata, events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
|
||||
futures::executor::block_on(async {
|
||||
assert!(
|
||||
provider_stream
|
||||
.next()
|
||||
.await
|
||||
.is_some_and(|event| event.is_err())
|
||||
);
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
feedback.scheduling_signals(&provider, &account).0,
|
||||
Some(2_000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_stream_should_report_one_success_after_same_account_retry() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("valid provider");
|
||||
let account = ProviderAccountId::new("acct_same_account_success").expect("account");
|
||||
let retry_metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("valid model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("websocket").expect("valid transport"),
|
||||
);
|
||||
let retry_error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
|
||||
.with_pre_delivery_transport_fallback();
|
||||
let retry_events: EventStream = Box::pin(stream::iter([Err(retry_error)]));
|
||||
let mut retry_stream = ProviderStream::new(retry_metadata, retry_events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
|
||||
futures::executor::block_on(async {
|
||||
assert!(
|
||||
retry_stream
|
||||
.next()
|
||||
.await
|
||||
.is_some_and(|event| event.is_err())
|
||||
);
|
||||
});
|
||||
|
||||
let success_metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("valid model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("http_sse").expect("valid transport"),
|
||||
);
|
||||
let success_events: EventStream = Box::pin(stream::empty());
|
||||
let mut success_stream = ProviderStream::new(success_metadata, success_events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
|
||||
futures::executor::block_on(async {
|
||||
assert!(success_stream.next().await.is_none());
|
||||
});
|
||||
|
||||
assert_eq!(feedback.scheduling_signals(&provider, &account).0, Some(0));
|
||||
}
|
||||
|
||||
@@ -168,6 +168,30 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
|
||||
assert!(!format!("{gateway:?}").contains(message));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuation_recovery_should_keep_its_internal_classification_and_client_contract() {
|
||||
let error = ProviderError::new(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Previous response was not found. Retrying the full request.",
|
||||
Some("previous_response_not_found".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
));
|
||||
let gateway = GatewayError::from_provider(&error);
|
||||
|
||||
assert_eq!(error.kind().as_str(), "continuation_recovery_required");
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"conversation continuation must be rebuilt"
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.client_error_code(),
|
||||
Some("previous_response_not_found")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
|
||||
let message = format!("\0{}\n", "m".repeat(9_000));
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::operation::{
|
||||
Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
|
||||
ProtocolPayload, ProviderSessionState, RawJsonPayload,
|
||||
ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
@@ -121,10 +121,43 @@ fn operation_kind_should_remain_stable() {
|
||||
ImageRequestKind::Generation,
|
||||
json!({"model": "gpt-image-2", "prompt": "draw"}),
|
||||
));
|
||||
let search = Operation::Search(StandaloneSearchRequest::from_raw_json(
|
||||
RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from_static(br#"{ "model":"gpt-test", "commands":{} }"#),
|
||||
)
|
||||
.expect("OpenAI search payload"),
|
||||
));
|
||||
|
||||
assert_eq!(generate.kind(), OperationKind::Generate);
|
||||
assert_eq!(image.kind(), OperationKind::GenerateImage);
|
||||
assert_eq!(search.kind(), OperationKind::Search);
|
||||
assert_eq!(OperationKind::Search.as_str(), "search");
|
||||
assert!(image.image_generation_requested());
|
||||
assert!(!search.image_generation_requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standalone_search_should_keep_raw_body_and_context_opaque() {
|
||||
let body = Bytes::from_static(
|
||||
br#"{ "model":"gpt-future", "commands":{}, "future":9007199254740993, "future":2 }"#,
|
||||
);
|
||||
let secret = "private-turn-metadata";
|
||||
let payload = RawJsonPayload::new("openai", body.clone())
|
||||
.expect("OpenAI search payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
Value::String(secret.to_owned()),
|
||||
)]));
|
||||
let request = StandaloneSearchRequest::from_raw_json(payload);
|
||||
|
||||
assert_eq!(request.payload().body(), &body);
|
||||
assert_eq!(
|
||||
request.payload().context().get("turn_metadata"),
|
||||
Some(&Value::String(secret.to_owned()))
|
||||
);
|
||||
assert!(!format!("{request:?}").contains(secret));
|
||||
assert!(!format!("{request:?}").contains("gpt-future"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -11,7 +11,9 @@ use gateway_core::engine::{
|
||||
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
use gateway_core::error::{
|
||||
ProviderErrorKind, StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind,
|
||||
};
|
||||
use gateway_core::metering::{CostSource as CoreCostSource, Usage as CoreUsage};
|
||||
use gateway_core::routing::{AccountRoutingScopeKind, AccountRoutingSnapshot};
|
||||
use gateway_core::upstream::UpstreamSendState as CoreUpstreamSendState;
|
||||
@@ -99,12 +101,20 @@ pub struct NewModelRequest {
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: bool,
|
||||
pub continuation: ContinuationRequestObservation,
|
||||
pub image_generation_requested: bool,
|
||||
pub admission_decision_ms: Option<u64>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub deadline_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ContinuationRequestObservation {
|
||||
pub affinity_hash: Option<String>,
|
||||
pub previous_response_id_hash: Option<String>,
|
||||
pub requested: bool,
|
||||
}
|
||||
|
||||
impl NewModelRequest {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "id", &self.id)?;
|
||||
@@ -133,6 +143,31 @@ impl NewModelRequest {
|
||||
{
|
||||
return Err(invalid("live client key ID must equal its historical ref"));
|
||||
}
|
||||
self.continuation.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ContinuationRequestObservation {
|
||||
fn validate(&self) -> StoreResult<()> {
|
||||
for value in [
|
||||
self.affinity_hash.as_deref(),
|
||||
self.previous_response_id_hash.as_deref(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if value.len() != 64
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err(invalid("continuation observation hash is invalid"));
|
||||
}
|
||||
}
|
||||
if !self.requested && self.previous_response_id_hash.is_some() {
|
||||
return Err(invalid("continuation request facts do not agree"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -252,6 +287,11 @@ pub struct ModelRequestFinalization {
|
||||
pub provider_error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub raw_upstream_error: Option<String>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_id: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub upstream_connection_age_ms: Option<u64>,
|
||||
pub upstream_connection_idle_ms: Option<u64>,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub usage: ModelRequestUsage,
|
||||
pub image_generation_succeeded: Option<bool>,
|
||||
@@ -313,6 +353,11 @@ impl ModelRequestFinalization {
|
||||
{
|
||||
return Err(invalid("provider observation must be a JSON object"));
|
||||
}
|
||||
validate_optional_stable_reason(
|
||||
self.continuation_unavailable_reason.as_deref(),
|
||||
"continuation unavailable reason",
|
||||
)?;
|
||||
validate_connection_observation(self)?;
|
||||
self.timings.validate()
|
||||
}
|
||||
}
|
||||
@@ -389,11 +434,13 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
operation, endpoint, client_transport, requested_model_id,
|
||||
client_ip, user_agent, reasoning_effort,
|
||||
reasoning_preset, request_kind, subagent_kind, compact,
|
||||
image_generation_requested, admission_decision_ms, started_at, deadline_at
|
||||
image_generation_requested, admission_decision_ms, started_at, deadline_at,
|
||||
continuation_affinity_hash, continuation_previous_response_id_hash,
|
||||
continuation_requested
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8,
|
||||
$9, $10, $11, $12, $13::inet, $14, $15,
|
||||
$16, $17, $18, $19, $20, $21, $22, $23
|
||||
$16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26
|
||||
)",
|
||||
)
|
||||
.bind(request.id)
|
||||
@@ -422,6 +469,9 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
)?)
|
||||
.bind(request.started_at)
|
||||
.bind(request.deadline_at)
|
||||
.bind(request.continuation.affinity_hash)
|
||||
.bind(request.continuation.previous_response_id_hash)
|
||||
.bind(request.continuation.requested)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("insert model request"))?;
|
||||
@@ -452,13 +502,15 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
upstream_model_id, upstream_transport, http_version,
|
||||
attempt_count, upstream_send_state, account_selection_wait_ms,
|
||||
capacity_used_slots, capacity_total_slots
|
||||
, continuation_affinity_hash, continuation_previous_response_id_hash,
|
||||
continuation_requested
|
||||
) select
|
||||
$1, $2, $3, $4, $5, $6, $7, $8,
|
||||
$9, $10, $11, $12, $13::inet, $14, $15,
|
||||
$16, $17, $18, $19, $20, $21, $22, $23,
|
||||
$24, $25, $26,
|
||||
account.name, account.email, account.authentication_kind,
|
||||
$27, $28, $29, 1, 'not_sent', $30, $31, $32
|
||||
$27, $28, $29, 1, 'not_sent', $30, $31, $32, $33, $34, $35
|
||||
from (values (true)) as seed(present)
|
||||
left join provider_accounts account on account.id = $25",
|
||||
)
|
||||
@@ -506,6 +558,9 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
attempt.capacity_total_slots,
|
||||
"capacity_total_slots",
|
||||
)?)
|
||||
.bind(request.continuation.affinity_hash)
|
||||
.bind(request.continuation.previous_response_id_hash)
|
||||
.bind(request.continuation.requested)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
|
||||
@@ -644,8 +699,9 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
finalization: ModelRequestFinalization,
|
||||
) -> StoreResult<bool> {
|
||||
finalization.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update model_requests
|
||||
let finalized = sqlx::query_scalar::<_, i64>(
|
||||
"with finalized as (
|
||||
update model_requests
|
||||
set outcome = $2, upstream_send_state = $3, attempt_count = $4,
|
||||
downstream_committed_at = $5,
|
||||
client_status_code = coalesce(client_status_code, $6),
|
||||
@@ -664,8 +720,64 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
upstream_transport = coalesce($37, upstream_transport),
|
||||
http_version = coalesce($38, http_version), websocket_pool = $39,
|
||||
service_tier = $40, provider_observation_json = $41,
|
||||
raw_upstream_error = $42
|
||||
where id = $1 and outcome = 'running'",
|
||||
raw_upstream_error = $42,
|
||||
continuation_unavailable_reason = $43,
|
||||
upstream_connection_id = $44,
|
||||
upstream_connection_exit_reason = $45,
|
||||
upstream_connection_age_ms = $46,
|
||||
upstream_connection_idle_ms = $47
|
||||
where id = $1 and outcome = 'running'
|
||||
returning id, client_api_key_ref, continuation_affinity_hash,
|
||||
continuation_requested, outcome, started_at, completed_at
|
||||
), recovery_target as (
|
||||
select prior.id
|
||||
from model_requests prior
|
||||
cross join finalized current
|
||||
where not current.continuation_requested
|
||||
and current.continuation_affinity_hash is not null
|
||||
and prior.client_api_key_ref = current.client_api_key_ref
|
||||
and prior.continuation_affinity_hash = current.continuation_affinity_hash
|
||||
and prior.continuation_requested
|
||||
and prior.outcome = 'failed'
|
||||
and prior.error_kind = 'continuation_recovery_required'
|
||||
and prior.recovered_at is null
|
||||
and current.started_at >= prior.completed_at
|
||||
and current.started_at <= prior.completed_at + interval '30 seconds'
|
||||
order by prior.completed_at desc, prior.id desc
|
||||
limit 1
|
||||
), recovery_update as (
|
||||
update model_requests prior
|
||||
set recovery_attempt_count = prior.recovery_attempt_count + 1,
|
||||
recovery_request_id = case
|
||||
when current.outcome = 'succeeded' then current.id
|
||||
else prior.recovery_request_id
|
||||
end,
|
||||
recovered_at = case
|
||||
when current.outcome = 'succeeded' then current.completed_at
|
||||
else prior.recovered_at
|
||||
end,
|
||||
recovery_retry_delay_ms = case
|
||||
when current.outcome = 'succeeded' then greatest(
|
||||
0,
|
||||
floor(extract(epoch from (current.started_at - prior.completed_at)) * 1000)
|
||||
)::bigint
|
||||
else prior.recovery_retry_delay_ms
|
||||
end,
|
||||
recovery_total_latency_ms = case
|
||||
when current.outcome = 'succeeded' then greatest(
|
||||
0,
|
||||
floor(extract(epoch from (current.completed_at - prior.completed_at)) * 1000)
|
||||
)::bigint
|
||||
else prior.recovery_total_latency_ms
|
||||
end
|
||||
from finalized current
|
||||
join recovery_target target on true
|
||||
where prior.id = target.id
|
||||
and prior.recovered_at is null
|
||||
returning prior.id
|
||||
)
|
||||
select (select count(*) from finalized)::bigint
|
||||
+ (select count(*) * 0 from recovery_update)::bigint",
|
||||
)
|
||||
.bind(&finalization.model_request_id)
|
||||
.bind(finalization.outcome.as_str())
|
||||
@@ -754,10 +866,21 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
.bind(finalization.service_tier)
|
||||
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
|
||||
.bind(finalization.raw_upstream_error)
|
||||
.execute(&self.pool)
|
||||
.bind(finalization.continuation_unavailable_reason)
|
||||
.bind(finalization.upstream_connection_id)
|
||||
.bind(finalization.upstream_connection_exit_reason)
|
||||
.bind(optional_i64(
|
||||
finalization.upstream_connection_age_ms,
|
||||
"upstream_connection_age_ms",
|
||||
)?)
|
||||
.bind(optional_i64(
|
||||
finalization.upstream_connection_idle_ms,
|
||||
"upstream_connection_idle_ms",
|
||||
)?)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("finalize model request"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
Ok(finalized == 1)
|
||||
}
|
||||
|
||||
async fn recover_expired_model_requests(
|
||||
@@ -973,6 +1096,22 @@ impl ExecutionStore for PgExecutionStore {
|
||||
&self,
|
||||
finalization: CoreModelRequestFinalization,
|
||||
) -> Result<(), CoreStoreError> {
|
||||
let continuation_unavailable_reason = finalization
|
||||
.failure_observation
|
||||
.continuation_unavailable_reason
|
||||
.clone();
|
||||
let connection_observation = finalization
|
||||
.failure_observation
|
||||
.upstream_connection
|
||||
.as_ref();
|
||||
let upstream_connection_id =
|
||||
connection_observation.map(|observation| observation.connection_id().to_owned());
|
||||
let upstream_connection_exit_reason =
|
||||
connection_observation.map(|observation| observation.exit_reason().to_owned());
|
||||
let upstream_connection_age_ms =
|
||||
connection_observation.map(|observation| observation.age_ms());
|
||||
let upstream_connection_idle_ms =
|
||||
connection_observation.map(|observation| observation.idle_ms());
|
||||
let provider_metadata_json = finalization
|
||||
.provider_metadata_json
|
||||
.as_deref()
|
||||
@@ -999,10 +1138,18 @@ impl ExecutionStore for PgExecutionStore {
|
||||
),
|
||||
None => (CostSource::Unavailable, None, None),
|
||||
};
|
||||
let error_kind = finalization
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.kind().as_str().to_owned());
|
||||
let error_kind = if continuation_unavailable_reason.is_some() {
|
||||
Some(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
.as_str()
|
||||
.to_owned(),
|
||||
)
|
||||
} else {
|
||||
finalization
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.kind().as_str().to_owned())
|
||||
};
|
||||
let error_message = finalization.error.as_ref().map(|error| {
|
||||
error.diagnostic().map_or_else(
|
||||
|| error.safe_message().to_owned(),
|
||||
@@ -1033,6 +1180,11 @@ impl ExecutionStore for PgExecutionStore {
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
error_message,
|
||||
raw_upstream_error: finalization.raw_upstream_error,
|
||||
continuation_unavailable_reason,
|
||||
upstream_connection_id,
|
||||
upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms,
|
||||
upstream_connection_idle_ms,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: usage_from_core(finalization.usage),
|
||||
image_generation_succeeded: finalization.image_generation_succeeded,
|
||||
@@ -1162,6 +1314,11 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
|
||||
request_kind: request.request_kind,
|
||||
subagent_kind: request.subagent_kind,
|
||||
compact: request.compact,
|
||||
continuation: ContinuationRequestObservation {
|
||||
affinity_hash: request.continuation.affinity_hash,
|
||||
previous_response_id_hash: request.continuation.previous_response_id_hash,
|
||||
requested: request.continuation.requested,
|
||||
},
|
||||
image_generation_requested: request.image_generation_requested,
|
||||
admission_decision_ms: request.admission_decision_ms,
|
||||
started_at: DateTime::<Utc>::from(request.started_at),
|
||||
@@ -1246,6 +1403,52 @@ fn validate_status_code(status: Option<u16>) -> StoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_optional_stable_reason(value: Option<&str>, field: &str) -> StoreResult<()> {
|
||||
if value.is_some_and(|value| {
|
||||
value.is_empty()
|
||||
|| value.len() > 64
|
||||
|| !value.bytes().enumerate().all(|(index, byte)| {
|
||||
byte.is_ascii_lowercase() || (index > 0 && (byte.is_ascii_digit() || byte == b'_'))
|
||||
})
|
||||
}) {
|
||||
return Err(invalid(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_connection_observation(finalization: &ModelRequestFinalization) -> StoreResult<()> {
|
||||
let fields_present = [
|
||||
finalization.upstream_connection_id.is_some(),
|
||||
finalization.upstream_connection_exit_reason.is_some(),
|
||||
finalization.upstream_connection_age_ms.is_some(),
|
||||
finalization.upstream_connection_idle_ms.is_some(),
|
||||
];
|
||||
if fields_present.iter().any(|present| *present)
|
||||
&& !fields_present.iter().all(|present| *present)
|
||||
{
|
||||
return Err(invalid("upstream connection observation is incomplete"));
|
||||
}
|
||||
if let Some(connection_id) = finalization.upstream_connection_id.as_deref()
|
||||
&& (connection_id.is_empty()
|
||||
|| connection_id.len() > 128
|
||||
|| connection_id.chars().any(char::is_control))
|
||||
{
|
||||
return Err(invalid("upstream connection ID is invalid"));
|
||||
}
|
||||
validate_optional_stable_reason(
|
||||
finalization.upstream_connection_exit_reason.as_deref(),
|
||||
"upstream connection exit reason is invalid",
|
||||
)?;
|
||||
if finalization
|
||||
.upstream_connection_age_ms
|
||||
.zip(finalization.upstream_connection_idle_ms)
|
||||
.is_some_and(|(age, idle)| idle > age)
|
||||
{
|
||||
return Err(invalid("upstream connection idle time exceeds its age"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn to_i64(value: u64, field: &'static str) -> StoreResult<i64> {
|
||||
i64::try_from(value).map_err(|_| invalid(field))
|
||||
}
|
||||
|
||||
@@ -39,7 +39,10 @@ pub use provider_accounts::*;
|
||||
pub use retention::*;
|
||||
pub use runtime_settings::*;
|
||||
pub use snapshot::*;
|
||||
pub(crate) use usage_facts::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
|
||||
pub(crate) use usage_facts::{
|
||||
completed_usage_fact_predicate, push_completed_usage_fact_filter,
|
||||
push_unrecovered_request_filter,
|
||||
};
|
||||
|
||||
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
|
||||
|
||||
|
||||
@@ -51,13 +51,11 @@ pub(crate) fn store_ops_error_filter(
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
provider_kind: filter.provider_kind,
|
||||
operation: filter.operation,
|
||||
endpoint: filter.endpoint,
|
||||
model: filter.model,
|
||||
transport: filter.transport,
|
||||
attempt_index: filter.attempt_index,
|
||||
response_id: filter.response_id,
|
||||
upstream_request_id: filter.upstream_request_id,
|
||||
failure_kind: filter.failure_kind,
|
||||
status_code: filter.status_code,
|
||||
search: filter.search,
|
||||
}
|
||||
@@ -731,6 +729,18 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
|
||||
request_kind: error.request_kind,
|
||||
subagent_kind: error.subagent_kind,
|
||||
compact: error.compact,
|
||||
continuation_affinity_hash: error.continuation_affinity_hash,
|
||||
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
|
||||
continuation_unavailable_reason: error.continuation_unavailable_reason,
|
||||
upstream_connection_id: error.upstream_connection_id,
|
||||
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms: error.upstream_connection_age_ms,
|
||||
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
|
||||
recovery_request_id: error.recovery_request_id,
|
||||
recovered_at: error.recovered_at,
|
||||
recovery_attempt_count: error.recovery_attempt_count,
|
||||
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
|
||||
recovery_total_latency_ms: error.recovery_total_latency_ms,
|
||||
occurred_at: error.occurred_at,
|
||||
stable_sort_id: error.stable_sort_id,
|
||||
}
|
||||
@@ -910,6 +920,18 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
|
||||
request_kind: get(row, "request_kind")?,
|
||||
subagent_kind: get(row, "subagent_kind")?,
|
||||
compact: get(row, "compact")?,
|
||||
continuation_affinity_hash: get(row, "continuation_affinity_hash")?,
|
||||
continuation_previous_response_id_hash: get(row, "continuation_previous_response_id_hash")?,
|
||||
continuation_unavailable_reason: get(row, "continuation_unavailable_reason")?,
|
||||
upstream_connection_id: get(row, "upstream_connection_id")?,
|
||||
upstream_connection_exit_reason: get(row, "upstream_connection_exit_reason")?,
|
||||
upstream_connection_age_ms: optional_unsigned(row, "upstream_connection_age_ms")?,
|
||||
upstream_connection_idle_ms: optional_unsigned(row, "upstream_connection_idle_ms")?,
|
||||
recovery_request_id: get(row, "recovery_request_id")?,
|
||||
recovered_at: get(row, "recovered_at")?,
|
||||
recovery_attempt_count: to_u32(get(row, "recovery_attempt_count")?)?,
|
||||
recovery_retry_delay_ms: optional_unsigned(row, "recovery_retry_delay_ms")?,
|
||||
recovery_total_latency_ms: optional_unsigned(row, "recovery_total_latency_ms")?,
|
||||
occurred_at: get(row, "occurred_at")?,
|
||||
stable_sort_id: get(row, "stable_sort_id")?,
|
||||
})
|
||||
|
||||
@@ -18,7 +18,10 @@ use crate::{
|
||||
require_nonempty,
|
||||
};
|
||||
|
||||
use super::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
|
||||
use super::{
|
||||
completed_usage_fact_predicate, push_completed_usage_fact_filter,
|
||||
push_unrecovered_request_filter,
|
||||
};
|
||||
|
||||
mod admin_adapter;
|
||||
mod mapping;
|
||||
|
||||
@@ -126,13 +126,11 @@ pub struct OpsErrorFilter {
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub failure_kind: Option<String>,
|
||||
pub status_code: Option<u16>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
@@ -148,14 +146,12 @@ impl OpsErrorFilter {
|
||||
),
|
||||
(self.provider_kind.as_deref(), "provider filter"),
|
||||
(self.operation.as_deref(), "operation filter"),
|
||||
(self.endpoint.as_deref(), "endpoint filter"),
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(
|
||||
self.upstream_request_id.as_deref(),
|
||||
"upstream request ID filter",
|
||||
),
|
||||
(self.failure_kind.as_deref(), "failure kind filter"),
|
||||
] {
|
||||
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
|
||||
}
|
||||
@@ -726,6 +722,18 @@ pub struct OpsErrorRecord {
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub continuation_affinity_hash: Option<String>,
|
||||
pub continuation_previous_response_id_hash: Option<String>,
|
||||
pub continuation_unavailable_reason: Option<String>,
|
||||
pub upstream_connection_id: Option<String>,
|
||||
pub upstream_connection_exit_reason: Option<String>,
|
||||
pub upstream_connection_age_ms: Option<u64>,
|
||||
pub upstream_connection_idle_ms: Option<u64>,
|
||||
pub recovery_request_id: Option<String>,
|
||||
pub recovered_at: Option<DateTime<Utc>>,
|
||||
pub recovery_attempt_count: u32,
|
||||
pub recovery_retry_delay_ms: Option<u64>,
|
||||
pub recovery_total_latency_ms: Option<u64>,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub stable_sort_id: String,
|
||||
}
|
||||
|
||||
@@ -93,6 +93,7 @@ pub(crate) async fn request_metrics(
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
let row = query
|
||||
.build()
|
||||
@@ -113,7 +114,8 @@ pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTota
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
sum(cost_amount) filter (where {fact} and cost_currency = 'USD')::text
|
||||
as billing_usd
|
||||
from model_requests mr"
|
||||
from model_requests mr
|
||||
where mr.recovered_at is null"
|
||||
));
|
||||
let row = query
|
||||
.build()
|
||||
@@ -247,6 +249,7 @@ async fn request_metric_series_inner(
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start order by bucket_start");
|
||||
let rows = query
|
||||
@@ -373,6 +376,7 @@ pub(crate) async fn attempt_metrics(
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(format!(
|
||||
"), request_aggregate as (
|
||||
@@ -517,6 +521,7 @@ pub(crate) async fn provider_observations(
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
|
||||
let rows = query
|
||||
|
||||
@@ -24,6 +24,12 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact,
|
||||
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
|
||||
mr.continuation_unavailable_reason,
|
||||
mr.upstream_connection_id, mr.upstream_connection_exit_reason,
|
||||
mr.upstream_connection_age_ms, mr.upstream_connection_idle_ms,
|
||||
mr.recovery_request_id, mr.recovered_at, mr.recovery_attempt_count,
|
||||
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
|
||||
mr.completed_at as occurred_at,
|
||||
'model_request:' || mr.id as stable_sort_id
|
||||
from model_requests mr
|
||||
@@ -46,6 +52,15 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact,
|
||||
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
|
||||
null::text as continuation_unavailable_reason,
|
||||
null::text as upstream_connection_id,
|
||||
null::text as upstream_connection_exit_reason,
|
||||
null::bigint as upstream_connection_age_ms,
|
||||
null::bigint as upstream_connection_idle_ms,
|
||||
mr.recovery_request_id, mr.recovered_at,
|
||||
coalesce(mr.recovery_attempt_count, 0) as recovery_attempt_count,
|
||||
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
|
||||
oe.created_at as occurred_at,
|
||||
'ops_event:' || oe.id as stable_sort_id
|
||||
from ops_events oe
|
||||
@@ -113,6 +128,7 @@ fn push_request_error_predicates(
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
statement.push(" and mr.recovered_at is null");
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
@@ -120,16 +136,11 @@ fn push_request_error_predicates(
|
||||
("mr.provider_account_ref", &filter.provider_account_ref),
|
||||
("mr.provider_kind", &filter.provider_kind),
|
||||
("mr.operation", &filter.operation),
|
||||
("mr.endpoint", &filter.endpoint),
|
||||
("mr.upstream_transport", &filter.transport),
|
||||
("mr.upstream_request_id", &filter.upstream_request_id),
|
||||
] {
|
||||
push_text_equality(statement, column, value);
|
||||
}
|
||||
if let Some(value) = &filter.failure_kind {
|
||||
statement.push(" and coalesce(mr.error_kind, 'failed') = ");
|
||||
statement.push_bind(value.clone());
|
||||
}
|
||||
push_response_id_filter(statement, "mr.client_response_id", filter);
|
||||
push_text_equality(statement, "mr.upstream_model_id", &filter.model);
|
||||
if let Some(index) = filter.attempt_index {
|
||||
@@ -160,6 +171,7 @@ fn push_ops_event_predicates(
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
statement.push(" and (mr.id is null or mr.recovered_at is null)");
|
||||
push_range(statement, "oe.created_at", range);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
@@ -167,9 +179,7 @@ fn push_ops_event_predicates(
|
||||
("oe.provider_account_ref", &filter.provider_account_ref),
|
||||
("oe.provider_kind", &filter.provider_kind),
|
||||
("oe.operation", &filter.operation),
|
||||
("mr.endpoint", &filter.endpoint),
|
||||
("oe.upstream_request_id", &filter.upstream_request_id),
|
||||
("oe.failure_kind", &filter.failure_kind),
|
||||
] {
|
||||
push_text_equality(statement, column, value);
|
||||
}
|
||||
|
||||
@@ -360,6 +360,7 @@ pub(crate) async fn usage_diagnostics(
|
||||
statement.push_bind(range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(range.end);
|
||||
push_unrecovered_request_filter(&mut statement, "mr");
|
||||
push_diagnostic_dimension_filter(&mut statement, dimension);
|
||||
push_usage_filter(&mut statement, filter, "mr");
|
||||
statement.push(
|
||||
|
||||
@@ -18,3 +18,10 @@ pub(crate) fn completed_usage_fact_predicate(alias: &str) -> String {
|
||||
pub(crate) fn push_completed_usage_fact_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
|
||||
query.push(format!(" and {}", completed_usage_fact_predicate(alias)));
|
||||
}
|
||||
|
||||
/// 排除已由后续成功请求恢复的会话续接中间失败。
|
||||
///
|
||||
/// 原始请求审计仍保留该行;默认业务指标只把最终成功链视为一次结果。
|
||||
pub(crate) fn push_unrecovered_request_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
|
||||
query.push(format!(" and {alias}.recovered_at is null"));
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_core::engine::{
|
||||
ExecutionOutcome, ExecutionStore, ModelRequestFinalization as CoreModelRequestFinalization,
|
||||
ModelRequestId, ModelRequestTimings as CoreModelRequestTimings,
|
||||
ExecutionOutcome, ExecutionStore, ModelRequestFailureObservation,
|
||||
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
|
||||
ModelRequestTimings as CoreModelRequestTimings,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObservation};
|
||||
use gateway_core::metering::{CalculatedCost, CostEstimate, Usage};
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use gateway_store::postgres::{
|
||||
@@ -42,6 +43,7 @@ fn model_request_rejects_mismatched_client_key_live_id() {
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at,
|
||||
deadline_at: started_at + Duration::seconds(30),
|
||||
@@ -93,6 +95,7 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() {
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at,
|
||||
deadline_at: started_at + Duration::seconds(30),
|
||||
@@ -210,6 +213,7 @@ async fn model_request_persists_group_routing_snapshot_without_live_group_foreig
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at,
|
||||
deadline_at: started_at + Duration::seconds(30),
|
||||
@@ -477,6 +481,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
raw_upstream_error: None,
|
||||
failure_observation: Default::default(),
|
||||
retry_after_ms: None,
|
||||
usage: Usage::new(),
|
||||
image_generation_succeeded: None,
|
||||
@@ -600,6 +605,243 @@ async fn core_adapter_should_reject_non_object_provider_observation() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn core_adapter_persists_continuation_connection_failure_observation() {
|
||||
let Some(database) = TestDatabase::create("execution_continuation_observation").await else {
|
||||
return;
|
||||
};
|
||||
seed_running_request(&database.pool, "req_continuation_observation")
|
||||
.await
|
||||
.expect("seed model request");
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let mut finalization = successful_core_finalization("req_continuation_observation");
|
||||
finalization.outcome = ExecutionOutcome::Failed;
|
||||
finalization.client_status_code = Some(400);
|
||||
finalization.error = Some(GatewayError::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"conversation continuation must be rebuilt",
|
||||
));
|
||||
finalization.failure_observation = ModelRequestFailureObservation {
|
||||
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
|
||||
upstream_connection: Some(ProviderConnectionObservation::new(
|
||||
"connection-observed",
|
||||
"tcp_reset",
|
||||
12_000,
|
||||
4_000,
|
||||
)),
|
||||
};
|
||||
|
||||
ExecutionStore::finalize_model_request(&store, finalization)
|
||||
.await
|
||||
.expect("persist continuation failure observation");
|
||||
|
||||
type PersistedContinuationObservation = (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<i64>,
|
||||
Option<i64>,
|
||||
);
|
||||
let persisted: PersistedContinuationObservation = sqlx::query_as(
|
||||
"select error_kind, continuation_unavailable_reason,
|
||||
upstream_connection_id, upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms, upstream_connection_idle_ms
|
||||
from model_requests where id = 'req_continuation_observation'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load continuation failure observation");
|
||||
assert_eq!(
|
||||
persisted.0.as_deref(),
|
||||
Some("continuation_recovery_required")
|
||||
);
|
||||
assert_eq!(persisted.1.as_deref(), Some("reused_connection_lost"));
|
||||
assert_eq!(persisted.2.as_deref(), Some("connection-observed"));
|
||||
assert_eq!(persisted.3.as_deref(), Some("tcp_reset"));
|
||||
assert_eq!(persisted.4, Some(12_000));
|
||||
assert_eq!(persisted.5, Some(4_000));
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_new_chain_should_mark_recent_continuation_failure_recovered() {
|
||||
let Some(database) = TestDatabase::create("execution_continuation_recovered").await else {
|
||||
return;
|
||||
};
|
||||
let affinity_hash = "a".repeat(64);
|
||||
let failure_completed_at = Utc::now() - Duration::seconds(8);
|
||||
seed_continuation_failure(
|
||||
&database.pool,
|
||||
"req_continuation_failed",
|
||||
&affinity_hash,
|
||||
failure_completed_at,
|
||||
)
|
||||
.await;
|
||||
seed_recovery_request(
|
||||
&database.pool,
|
||||
"req_continuation_recovery",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(4),
|
||||
)
|
||||
.await;
|
||||
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
ExecutionStore::finalize_model_request(
|
||||
&store,
|
||||
successful_core_finalization("req_continuation_recovery"),
|
||||
)
|
||||
.await
|
||||
.expect("finalize successful recovery request");
|
||||
|
||||
type PersistedRecoveryObservation = (
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
i32,
|
||||
Option<i64>,
|
||||
Option<i64>,
|
||||
);
|
||||
let recovered: PersistedRecoveryObservation = sqlx::query_as(
|
||||
"select recovery_request_id, recovered_at, recovery_attempt_count,
|
||||
recovery_retry_delay_ms, recovery_total_latency_ms
|
||||
from model_requests where id = 'req_continuation_failed'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load recovered continuation failure");
|
||||
|
||||
assert_eq!(recovered.0.as_deref(), Some("req_continuation_recovery"));
|
||||
assert!(recovered.1.is_some());
|
||||
assert_eq!(recovered.2, 1);
|
||||
assert_eq!(recovered.3, Some(4_000));
|
||||
assert!(recovered.4.is_some_and(|latency| latency >= 4_000));
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() {
|
||||
let Some(database) = TestDatabase::create("execution_continuation_unresolved").await else {
|
||||
return;
|
||||
};
|
||||
let affinity_hash = "c".repeat(64);
|
||||
let failure_completed_at = Utc::now() - Duration::seconds(60);
|
||||
seed_continuation_failure(
|
||||
&database.pool,
|
||||
"req_continuation_unresolved",
|
||||
&affinity_hash,
|
||||
failure_completed_at,
|
||||
)
|
||||
.await;
|
||||
seed_recovery_request(
|
||||
&database.pool,
|
||||
"req_continuation_retry_failed",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(3),
|
||||
)
|
||||
.await;
|
||||
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let mut failed = successful_core_finalization("req_continuation_retry_failed");
|
||||
failed.outcome = ExecutionOutcome::Failed;
|
||||
failed.client_status_code = Some(502);
|
||||
failed.upstream_status_code = Some(500);
|
||||
failed.error = Some(GatewayError::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"recovery request failed",
|
||||
));
|
||||
ExecutionStore::finalize_model_request(&store, failed)
|
||||
.await
|
||||
.expect("finalize failed recovery request");
|
||||
|
||||
let after_failed: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
|
||||
"select recovery_request_id, recovered_at, recovery_attempt_count
|
||||
from model_requests where id = 'req_continuation_unresolved'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load unresolved continuation failure");
|
||||
assert_eq!(after_failed, (None, None, 1));
|
||||
|
||||
seed_recovery_request(
|
||||
&database.pool,
|
||||
"req_continuation_retry_late",
|
||||
&affinity_hash,
|
||||
failure_completed_at + Duration::seconds(31),
|
||||
)
|
||||
.await;
|
||||
ExecutionStore::finalize_model_request(
|
||||
&store,
|
||||
successful_core_finalization("req_continuation_retry_late"),
|
||||
)
|
||||
.await
|
||||
.expect("finalize late recovery request");
|
||||
|
||||
let after_late: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
|
||||
"select recovery_request_id, recovered_at, recovery_attempt_count
|
||||
from model_requests where id = 'req_continuation_unresolved'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load late unresolved continuation failure");
|
||||
assert_eq!(after_late, (None, None, 1));
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn seed_continuation_failure(
|
||||
pool: &sqlx::PgPool,
|
||||
id: &str,
|
||||
affinity_hash: &str,
|
||||
completed_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
seed_running_request(pool, id)
|
||||
.await
|
||||
.expect("seed continuation failure request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
|
||||
continuation_affinity_hash = $2,
|
||||
continuation_previous_response_id_hash = $3,
|
||||
continuation_requested = true,
|
||||
outcome = 'failed', error_kind = 'continuation_recovery_required',
|
||||
continuation_unavailable_reason = 'reused_connection_lost',
|
||||
started_at = $4 - interval '1 second', deadline_at = $4,
|
||||
completed_at = $4
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(affinity_hash)
|
||||
.bind("b".repeat(64))
|
||||
.bind(completed_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("finalize seeded continuation failure");
|
||||
}
|
||||
|
||||
async fn seed_recovery_request(
|
||||
pool: &sqlx::PgPool,
|
||||
id: &str,
|
||||
affinity_hash: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
seed_running_request(pool, id)
|
||||
.await
|
||||
.expect("seed continuation recovery request");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
|
||||
continuation_affinity_hash = $2, continuation_requested = false,
|
||||
started_at = $3, deadline_at = $3 + interval '1 minute'
|
||||
where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(affinity_hash)
|
||||
.bind(started_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("prepare seeded recovery request");
|
||||
}
|
||||
|
||||
async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
|
||||
@@ -215,6 +215,84 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_continuation_failure_should_be_hidden_from_default_business_metrics() {
|
||||
let Some(database) = TestDatabase::create("observability_recovered_continuation").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now)
|
||||
.await
|
||||
.expect("seed observability facts");
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set error_kind = 'continuation_recovery_required',
|
||||
continuation_affinity_hash = $2,
|
||||
continuation_previous_response_id_hash = $3,
|
||||
continuation_requested = true,
|
||||
continuation_unavailable_reason = 'reused_connection_lost',
|
||||
recovery_request_id = 'req_observe_success', recovered_at = $1,
|
||||
recovery_attempt_count = 1, recovery_retry_delay_ms = 4000,
|
||||
recovery_total_latency_ms = 8000
|
||||
where id = 'req_observe_failed'",
|
||||
)
|
||||
.bind(now)
|
||||
.bind("a".repeat(64))
|
||||
.bind("b".repeat(64))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("mark continuation failure recovered");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = observability_repository(&database.pool);
|
||||
|
||||
let overview = repository
|
||||
.usage_summary(range, UsageRecordFilter::default())
|
||||
.await
|
||||
.expect("usage summary without recovered intermediates");
|
||||
assert_eq!(overview.requests.request_count, 2);
|
||||
assert_eq!(overview.requests.failure_count, 0);
|
||||
assert_eq!(overview.attempts.attempt_count, 2);
|
||||
assert_eq!(overview.attempts.failure_count, 0);
|
||||
assert_eq!(overview.providers[0].request_count, 2);
|
||||
assert_eq!(overview.providers[0].failure_count, 0);
|
||||
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range,
|
||||
UsageRecordFilter::default(),
|
||||
DiagnosticDimension::Account,
|
||||
)
|
||||
.await
|
||||
.expect("diagnostics without recovered intermediates");
|
||||
assert_eq!(diagnostics[0].request_count, 2);
|
||||
assert_eq!(diagnostics[0].failure_count, 0);
|
||||
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range,
|
||||
filter: OpsErrorFilter::default(),
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("ops errors without recovered intermediates");
|
||||
assert_eq!(errors.total, 0);
|
||||
|
||||
let retained: (String, Option<String>) = sqlx::query_as(
|
||||
"select outcome, recovery_request_id
|
||||
from model_requests where id = 'req_observe_failed'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load retained recovery audit row");
|
||||
assert_eq!(
|
||||
retained,
|
||||
("failed".to_owned(), Some("req_observe_success".to_owned()))
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics").await else {
|
||||
@@ -762,9 +840,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_observe".to_owned()),
|
||||
operation: Some("responses".to_owned()),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
model: Some("upstream-model".to_owned()),
|
||||
failure_kind: Some("rate_limited".to_owned()),
|
||||
status_code: Some(429),
|
||||
search: Some("req_observe_failed".to_owned()),
|
||||
..admin_observability::OpsErrorFilter::default()
|
||||
|
||||
@@ -875,6 +875,7 @@ fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at,
|
||||
deadline_at: started_at + chrono::Duration::seconds(30),
|
||||
|
||||
@@ -32,6 +32,12 @@ impl CodexSessionAffinity {
|
||||
&self.key_hash
|
||||
}
|
||||
|
||||
/// 返回可持久化的客户端作用域不透明会话关联值。
|
||||
#[must_use]
|
||||
pub(crate) fn persistence_hash(&self) -> &str {
|
||||
self.key.expose_to_store()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) const fn anchor_source(&self) -> &'static str {
|
||||
self.anchor_source
|
||||
@@ -58,6 +64,20 @@ impl CodexSessionAffinity {
|
||||
}
|
||||
}
|
||||
|
||||
/// 将原始 response ID 投影为客户端作用域的不可逆关联值。
|
||||
#[must_use]
|
||||
pub(crate) fn derive_previous_response_id_hash(
|
||||
previous_response_id: &str,
|
||||
client_api_key_id: &ClientApiKeyId,
|
||||
) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"codex-previous-response-observation-v1\0");
|
||||
hasher.update(client_api_key_id.as_str().as_bytes());
|
||||
hasher.update(b"\0");
|
||||
hasher.update(previous_response_id.as_bytes());
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
pub(crate) fn derive_codex_session_affinity(
|
||||
request: &CodexResponsesRequest,
|
||||
client_api_key_id: &ClientApiKeyId,
|
||||
|
||||
@@ -17,7 +17,7 @@ mod types;
|
||||
|
||||
pub(crate) use affinity::{
|
||||
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
|
||||
derive_codex_session_affinity,
|
||||
derive_codex_session_affinity, derive_previous_response_id_hash,
|
||||
};
|
||||
pub(crate) use oauth::oauth_owner_ref;
|
||||
pub(crate) use types::parse_access_token_expiration;
|
||||
|
||||
@@ -30,7 +30,7 @@ use crate::transport::{CodexWebSocketPool, build_reqwest_client};
|
||||
pub use config::{CodexWireProfileConfig, OpenAiConfig, OpenAiConfigError};
|
||||
pub use provider::{
|
||||
CodexProvider, CodexProviderConfigError, CodexProviderTransport, OFFICIAL_CODEX_BASE_PATH,
|
||||
OFFICIAL_CODEX_BASE_URL,
|
||||
OFFICIAL_CODEX_BASE_URL, openai_failure_affects_account_score,
|
||||
};
|
||||
|
||||
pub mod credential;
|
||||
|
||||
@@ -22,11 +22,66 @@ impl CodexProvider {
|
||||
),
|
||||
ImageRequestKind::Edit => (CODEX_IMAGE_EDITS_PATH, self.image_edits_url.clone()),
|
||||
};
|
||||
let image_turn_id = image
|
||||
.payload()
|
||||
.context()
|
||||
.get("image_turn_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
self.execute_raw_json_endpoint(
|
||||
context,
|
||||
RawJsonEndpointRequest {
|
||||
response_origin,
|
||||
endpoint_path,
|
||||
body: image.payload().body().clone(),
|
||||
image_turn_id,
|
||||
turn_metadata: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn execute_search(
|
||||
&self,
|
||||
search: &StandaloneSearchRequest,
|
||||
candidate: &ProviderCandidate,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
if search.payload().protocol() != PROVIDER_NAME || candidate.upstream_model().is_some() {
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
}
|
||||
let turn_metadata = search
|
||||
.payload()
|
||||
.context()
|
||||
.get("turn_metadata")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
self.execute_raw_json_endpoint(
|
||||
context,
|
||||
RawJsonEndpointRequest {
|
||||
response_origin: self.search_url.clone(),
|
||||
endpoint_path: CODEX_ALPHA_SEARCH_PATH,
|
||||
body: search.payload().body().clone(),
|
||||
image_turn_id: None,
|
||||
turn_metadata,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn execute_raw_json_endpoint(
|
||||
&self,
|
||||
context: AttemptContext,
|
||||
request: RawJsonEndpointRequest,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let selection_started_at = Instant::now();
|
||||
let lease = self
|
||||
.selector
|
||||
.select_for_provider_endpoint(&SelectCodexProviderEndpointCredential {
|
||||
request_url: &response_origin,
|
||||
request_url: &request.response_origin,
|
||||
attempt: &context,
|
||||
})
|
||||
.await
|
||||
@@ -48,18 +103,18 @@ impl CodexProvider {
|
||||
account_selection_wait_ms,
|
||||
lease.capacity_snapshot(),
|
||||
));
|
||||
let image_turn_id = image
|
||||
.payload()
|
||||
.context()
|
||||
.get("image_turn_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
// Standalone Provider 端点没有可证明的账号 owner;Search metadata 必须按
|
||||
// 跨账号输入收敛到当前 lease,不能沿用下游声明的账号或 installation identity。
|
||||
let turn_metadata = request.turn_metadata.as_deref().and_then(|metadata| {
|
||||
crate::transport::request::scope_turn_metadata(metadata, lease.installation_id(), true)
|
||||
});
|
||||
let events = cold_json_response_stream(ColdJsonResponse {
|
||||
client: self.client.clone(),
|
||||
response_origin,
|
||||
endpoint_path,
|
||||
body: image.payload().body().clone(),
|
||||
image_turn_id,
|
||||
response_origin: request.response_origin,
|
||||
endpoint_path: request.endpoint_path,
|
||||
body: request.body,
|
||||
image_turn_id: request.image_turn_id,
|
||||
turn_metadata,
|
||||
context,
|
||||
selector: Arc::clone(&self.selector),
|
||||
quota: Arc::clone(&self.quota),
|
||||
@@ -68,13 +123,24 @@ impl CodexProvider {
|
||||
});
|
||||
let stream = ProviderStream::new(metadata, events, lease);
|
||||
Ok(if allows_account_state_mutation {
|
||||
stream.with_account_feedback(Arc::clone(&self.account_feedback))
|
||||
stream.with_filtered_account_feedback(
|
||||
Arc::clone(&self.account_feedback),
|
||||
openai_failure_affects_account_score,
|
||||
)
|
||||
} else {
|
||||
stream
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RawJsonEndpointRequest {
|
||||
response_origin: Url,
|
||||
endpoint_path: &'static str,
|
||||
body: Bytes,
|
||||
image_turn_id: Option<String>,
|
||||
turn_metadata: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct ColdResponse {
|
||||
pub(super) client: CodexBackendClient,
|
||||
pub(super) response_origin: Url,
|
||||
@@ -101,6 +167,7 @@ pub(super) struct ColdJsonResponse {
|
||||
pub(super) endpoint_path: &'static str,
|
||||
pub(super) body: Bytes,
|
||||
pub(super) image_turn_id: Option<String>,
|
||||
pub(super) turn_metadata: Option<String>,
|
||||
pub(super) context: AttemptContext,
|
||||
pub(super) selector: Arc<CodexCredentialSelector>,
|
||||
pub(super) quota: Arc<CodexCredentialQuotaService>,
|
||||
@@ -276,6 +343,7 @@ pub(super) async fn create_json_attempt(
|
||||
Some(installation_id),
|
||||
);
|
||||
request_context.cookie_header = cookie_header.map(ExposeSecret::expose_secret);
|
||||
request_context.turn_metadata = request.turn_metadata.as_deref();
|
||||
request_context.account_selection = account_selection;
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -379,7 +447,7 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
tracing::warn!(
|
||||
account_id = %active_account.id(),
|
||||
error = %error,
|
||||
"Failed to persist OpenAI image response cookies"
|
||||
"Failed to persist OpenAI provider endpoint response cookies"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,40 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。
|
||||
///
|
||||
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。
|
||||
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。
|
||||
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
"rate_limit_exceeded",
|
||||
"rate_limit_error",
|
||||
"server_error",
|
||||
"service_unavailable_error",
|
||||
];
|
||||
|
||||
fn is_openai_account_score_failure_reason(value: &str) -> bool {
|
||||
let value = value.trim();
|
||||
OPENAI_ACCOUNT_SCORE_FAILURE_REASONS
|
||||
.iter()
|
||||
.any(|reason| value.eq_ignore_ascii_case(reason))
|
||||
}
|
||||
|
||||
fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> {
|
||||
if let Some(code) = error.upstream_code() {
|
||||
return Some(code.as_str());
|
||||
}
|
||||
let upstream = error.client_visible_upstream_error()?;
|
||||
upstream.code().or_else(|| upstream.error_type())
|
||||
}
|
||||
|
||||
/// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。
|
||||
#[doc(hidden)]
|
||||
pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
|
||||
openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason)
|
||||
}
|
||||
|
||||
pub(super) struct MappedProviderFailure {
|
||||
pub(super) error: ProviderError,
|
||||
pub(super) websocket_transport_retryable: bool,
|
||||
@@ -454,7 +488,10 @@ pub(super) fn websocket_client_failure_policy(
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
CodexClientError::WebSocket(error)
|
||||
if !matches!(error, CodexWebSocketExchangeError::InvalidRequest(_)) =>
|
||||
if !matches!(
|
||||
error.classified(),
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
) =>
|
||||
{
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
@@ -581,15 +618,16 @@ pub(super) fn websocket_retry_backoff(retry_index: NonZeroU32) -> Duration {
|
||||
Duration::from_millis(base_ms.saturating_mul(jitter_per_mille) / 1_000)
|
||||
}
|
||||
|
||||
pub(super) fn continuation_replay_required_error() -> ProviderError {
|
||||
pub(super) fn continuation_replay_required_error(reason: &'static str) -> ProviderError {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
)
|
||||
.with_continuation_unavailable_reason(reason)
|
||||
.with_upstream_code(OpaqueUpstreamValue::new(
|
||||
PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned(),
|
||||
))
|
||||
@@ -620,7 +658,7 @@ pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientErro
|
||||
| CodexClientError::HttpJson(_)
|
||||
| CodexClientError::StreamIdleTimeout { .. } => true,
|
||||
CodexClientError::WebSocket(error) => !matches!(
|
||||
error,
|
||||
error.classified(),
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::Upstream(_)
|
||||
| CodexWebSocketExchangeError::ContinuationUnavailable { .. }
|
||||
@@ -666,12 +704,32 @@ pub(super) fn map_client_error(
|
||||
_ => None,
|
||||
};
|
||||
let continuation_failure = match &error {
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
reason: PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
}) => Some(ContinuationFailure::Busy),
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
..
|
||||
}) => Some(ContinuationFailure::HistoryUnavailable),
|
||||
CodexClientError::WebSocket(error)
|
||||
if error.continuation_unavailable_reason()
|
||||
== Some(PreviousResponseUnavailableReason::ConnectionBusy) =>
|
||||
{
|
||||
Some(ContinuationFailure::Busy)
|
||||
}
|
||||
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
|
||||
Some(ContinuationFailure::HistoryUnavailable)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let continuation_unavailable_reason = match &error {
|
||||
CodexClientError::WebSocket(error) => error
|
||||
.continuation_unavailable_reason()
|
||||
.map(PreviousResponseUnavailableReason::as_str),
|
||||
_ => None,
|
||||
};
|
||||
let connection_observation = match &error {
|
||||
CodexClientError::WebSocket(error) => error.connection_observation().map(|observation| {
|
||||
ProviderConnectionObservation::new(
|
||||
observation.connection_id().to_string(),
|
||||
observation.exit_reason().to_owned(),
|
||||
observation.age_ms(),
|
||||
observation.idle_ms(),
|
||||
)
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
let observation = observe_transport
|
||||
@@ -718,10 +776,10 @@ pub(super) fn map_client_error(
|
||||
send_state,
|
||||
))
|
||||
}
|
||||
CodexClientError::WebSocket(
|
||||
error @ CodexWebSocketExchangeError::ContinuationUnavailable { .. },
|
||||
) => {
|
||||
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error());
|
||||
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
|
||||
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error(
|
||||
continuation_unavailable_reason.unwrap_or("scope_unavailable"),
|
||||
));
|
||||
if let Some(client_visible_error) = websocket_client_visible_error(&error) {
|
||||
failure.error = failure
|
||||
.error
|
||||
@@ -744,7 +802,10 @@ pub(super) fn map_client_error(
|
||||
"websocket_close_{close_code}"
|
||||
)));
|
||||
}
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
if matches!(
|
||||
error.classified(),
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached
|
||||
) {
|
||||
failure.error = failure.error.with_upstream_code(OpaqueUpstreamValue::new(
|
||||
WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(),
|
||||
));
|
||||
@@ -771,6 +832,11 @@ pub(super) fn map_client_error(
|
||||
if let Some(raw) = raw_upstream_error {
|
||||
failure.error = failure.error.with_raw_upstream_error(raw);
|
||||
}
|
||||
if let Some(connection_observation) = connection_observation {
|
||||
failure.error = failure
|
||||
.error
|
||||
.with_connection_observation(connection_observation);
|
||||
}
|
||||
failure.observation = observation;
|
||||
failure
|
||||
}
|
||||
@@ -790,7 +856,7 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
|
||||
return ProviderDiagnostic::new(message);
|
||||
}
|
||||
|
||||
ProviderDiagnostic::new(match error {
|
||||
ProviderDiagnostic::new(match error.classified() {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_) => {
|
||||
"OpenAI WebSocket request could not be constructed".to_owned()
|
||||
}
|
||||
@@ -847,6 +913,9 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal(_) => {
|
||||
unreachable!("close-before-terminal errors are handled before the variant match")
|
||||
}
|
||||
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
|
||||
unreachable!("classified websocket errors never retain observation wrappers")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -866,7 +935,10 @@ fn websocket_raw_error(error: &CodexWebSocketExchangeError) -> Option<RawUpstrea
|
||||
pub(super) fn websocket_client_visible_error(
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> Option<ClientVisibleUpstreamError> {
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
if matches!(
|
||||
error.classified(),
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached
|
||||
) {
|
||||
return Some(ClientVisibleUpstreamError::new(
|
||||
"websocket connection limit reached",
|
||||
Some(WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned()),
|
||||
@@ -900,7 +972,12 @@ pub(super) fn map_upstream_failure(
|
||||
.filter(|code| is_history_failure_code(code))
|
||||
.map(|_| ContinuationFailure::HistoryUnavailable);
|
||||
let send_state = upstream_send_state(failure.send_phase);
|
||||
let mut error = provider_error(provider_error_kind(category), send_state);
|
||||
let error_kind = if continuation_failure.is_some() {
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
} else {
|
||||
provider_error_kind(category)
|
||||
};
|
||||
let mut error = provider_error(error_kind, send_state);
|
||||
error = error.with_raw_upstream_error(RawUpstreamError::new(failure.raw_body.clone()));
|
||||
if let Some(message) = failure.client_message.as_ref() {
|
||||
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
@@ -939,7 +1016,8 @@ pub(super) fn map_upstream_failure(
|
||||
.with_continuation_failure(continuation_failure)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
);
|
||||
)
|
||||
.with_continuation_unavailable_reason("upstream_rejected");
|
||||
}
|
||||
if let Some(retry_after) = failure.retry_after_seconds.map(Duration::from_secs) {
|
||||
error = error.with_retry_after(retry_after);
|
||||
@@ -1064,8 +1142,8 @@ pub(super) fn account_failure(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
|
||||
match error {
|
||||
pub(super) fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::Connect(_)
|
||||
| CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
@@ -1085,11 +1163,14 @@ pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) ->
|
||||
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. }
|
||||
| CodexWebSocketExchangeError::InitialEventTimeout { .. } => UpstreamSendState::Ambiguous,
|
||||
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
|
||||
unreachable!("classified websocket errors never retain observation wrappers")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
|
||||
match error {
|
||||
pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::InvalidSse(_)
|
||||
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => ProviderErrorKind::Protocol,
|
||||
@@ -1113,6 +1194,9 @@ pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) ->
|
||||
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. } => {
|
||||
ProviderErrorKind::Transport
|
||||
}
|
||||
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
|
||||
unreachable!("classified websocket errors never retain observation wrappers")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,15 +12,15 @@ use futures::{StreamExt, future::BoxFuture};
|
||||
use gateway_core::account::{AccountFeedbackStats, ProviderAccount};
|
||||
use gateway_core::engine::continuation::{ContinuationBinding, NativeContinuationScope};
|
||||
use gateway_core::engine::provider::{
|
||||
EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration,
|
||||
ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation,
|
||||
ProviderSelectionObservation, ProviderStream,
|
||||
ContinuationRequestObservation, EventStream, Provider, ProviderCallMetadata,
|
||||
ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRequest,
|
||||
ProviderRequestObservation, ProviderSelectionObservation, ProviderStream,
|
||||
};
|
||||
use gateway_core::engine::{AttemptContext, AttemptTransport, ContinuationAttempt};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
|
||||
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderDiagnostic, ProviderError,
|
||||
ProviderErrorKind, RawUpstreamError,
|
||||
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderConnectionObservation,
|
||||
ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
FinishReason, GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader,
|
||||
@@ -29,7 +29,8 @@ use gateway_core::event::{
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::operation::{
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind, ProviderSessionState,
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind,
|
||||
ProviderSessionState, StandaloneSearchRequest,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
use gateway_core::routing::{
|
||||
@@ -58,6 +59,7 @@ use crate::credential::{
|
||||
CodexQuotaRefreshPolicy, CodexSessionAffinity, CredentialSelectionError, RuntimeCodexCookie,
|
||||
SelectCodexCredential, SelectCodexProviderEndpointCredential,
|
||||
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
derive_previous_response_id_hash,
|
||||
};
|
||||
use crate::session_transport::CodexSessionTransportFallbacks;
|
||||
use crate::transport::canonical::{
|
||||
@@ -84,11 +86,11 @@ use crate::transport::session::CodexSessionIdentity;
|
||||
use crate::transport::usage::normalize_service_tier;
|
||||
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
|
||||
use crate::transport::{
|
||||
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
|
||||
CodexAccountSelectionTelemetry, CodexBackendClient, CodexBackendJsonResponse,
|
||||
CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError, CodexRateLimitUpdates,
|
||||
CodexRequestContext, CodexResponseMetadata, CodexTransportMetrics, CodexUpstreamDiagnostics,
|
||||
CodexWebSocketPool, endpoint_url,
|
||||
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
|
||||
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
|
||||
CodexBackendJsonResponse, CodexBackendStreamingResponse, CodexBackendTransport,
|
||||
CodexClientError, CodexRateLimitUpdates, CodexRequestContext, CodexResponseMetadata,
|
||||
CodexTransportMetrics, CodexUpstreamDiagnostics, CodexWebSocketPool, endpoint_url,
|
||||
};
|
||||
|
||||
mod execution;
|
||||
@@ -97,6 +99,8 @@ mod observation;
|
||||
mod workers;
|
||||
|
||||
use execution::*;
|
||||
#[doc(hidden)]
|
||||
pub use failure::openai_failure_affects_account_score;
|
||||
use failure::*;
|
||||
use observation::*;
|
||||
pub(crate) use workers::worker_contributions;
|
||||
@@ -139,6 +143,7 @@ pub struct CodexProvider {
|
||||
responses_url: Url,
|
||||
image_generations_url: Url,
|
||||
image_edits_url: Url,
|
||||
search_url: Url,
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
stream_max_retries: u32,
|
||||
@@ -165,6 +170,8 @@ impl CodexProvider {
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let image_edits_url = Url::parse(&endpoint_url(&base_url, CODEX_IMAGE_EDITS_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let search_url = Url::parse(&endpoint_url(&base_url, CODEX_ALPHA_SEARCH_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
let client =
|
||||
CodexBackendClient::new(http, base_url, profile).with_websocket_pool(websocket_pool);
|
||||
Ok(Self {
|
||||
@@ -176,6 +183,7 @@ impl CodexProvider {
|
||||
responses_url,
|
||||
image_generations_url,
|
||||
image_edits_url,
|
||||
search_url,
|
||||
session_identity: None,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
|
||||
stream_max_retries,
|
||||
@@ -208,23 +216,35 @@ impl Provider for CodexProvider {
|
||||
self.catalog.catalog_generation()
|
||||
}
|
||||
|
||||
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
|
||||
fn request_observation(
|
||||
&self,
|
||||
operation: &Operation,
|
||||
client_api_key_id: &gateway_core::policy::ClientApiKeyId,
|
||||
) -> ProviderRequestObservation {
|
||||
let Operation::Generate(request) = operation else {
|
||||
return ProviderRequestObservation::default();
|
||||
};
|
||||
let (semantics, reasoning_effort) = encode_generate_request(request, "observability")
|
||||
.map(|encoded| {
|
||||
let semantics = encoded.semantics();
|
||||
let reasoning_effort = semantics.reasoning_effort.clone();
|
||||
(semantics, reasoning_effort)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let Ok(encoded) = encode_generate_request(request, "observability") else {
|
||||
return ProviderRequestObservation::default();
|
||||
};
|
||||
let semantics = encoded.semantics();
|
||||
let reasoning_effort = semantics.reasoning_effort.clone();
|
||||
let previous_response_id = encoded.previous_response_id();
|
||||
let continuation = ContinuationRequestObservation {
|
||||
affinity_hash: derive_codex_session_affinity(&encoded, client_api_key_id)
|
||||
.map(|affinity| affinity.persistence_hash().to_owned()),
|
||||
previous_response_id_hash: previous_response_id.map(|response_id| {
|
||||
derive_previous_response_id_hash(response_id, client_api_key_id)
|
||||
}),
|
||||
requested: previous_response_id.is_some(),
|
||||
};
|
||||
ProviderRequestObservation {
|
||||
reasoning_effort,
|
||||
reasoning_preset: semantics.reasoning_preset.map(str::to_owned),
|
||||
request_kind: semantics.request_kind,
|
||||
subagent_kind: semantics.subagent_kind,
|
||||
compact: semantics.compact,
|
||||
continuation,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,6 +288,9 @@ impl Provider for CodexProvider {
|
||||
if let Operation::GenerateImage(image) = request.operation() {
|
||||
return self.execute_image(image, candidate, context).await;
|
||||
}
|
||||
if let Operation::Search(search) = request.operation() {
|
||||
return self.execute_search(search, candidate, context).await;
|
||||
}
|
||||
let Operation::Generate(generate) = request.operation() else {
|
||||
return Err(provider_error(
|
||||
ProviderErrorKind::Unsupported,
|
||||
@@ -385,7 +408,7 @@ impl Provider for CodexProvider {
|
||||
continuation_recovery_action = "stop_proxy_recovery",
|
||||
"OpenAI connection-local continuation replay was rejected before send"
|
||||
);
|
||||
return Err(continuation_replay_required_error());
|
||||
return Err(continuation_replay_required_error("scope_unavailable"));
|
||||
}
|
||||
let previous_response_scope = match context.continuation_attempt() {
|
||||
ContinuationAttempt::Native => native_scope,
|
||||
@@ -410,7 +433,7 @@ impl Provider for CodexProvider {
|
||||
if upstream_request.previous_response_id().is_some()
|
||||
&& !account_scope.can_reuse_account_state()
|
||||
{
|
||||
return Err(continuation_replay_required_error());
|
||||
return Err(continuation_replay_required_error("scope_unavailable"));
|
||||
}
|
||||
scope_request_to_account(
|
||||
&mut upstream_request,
|
||||
@@ -421,7 +444,7 @@ impl Provider for CodexProvider {
|
||||
let requested_transport = selected_transport(&upstream_request);
|
||||
let sticky_http_fallback = context.transport() == AttemptTransport::Default
|
||||
&& requested_transport == CodexProviderTransport::PreferWebSocket
|
||||
&& !requirement.requires_websocket()
|
||||
&& requirement.allows_sticky_http_fallback()
|
||||
&& session_affinity.as_ref().is_some_and(|affinity| {
|
||||
self.session_transport_fallbacks
|
||||
.is_http_only(affinity.key())
|
||||
@@ -502,7 +525,10 @@ impl Provider for CodexProvider {
|
||||
});
|
||||
let stream = ProviderStream::new(metadata, events, lease);
|
||||
Ok(if allows_account_state_mutation {
|
||||
stream.with_account_feedback(Arc::clone(&self.account_feedback))
|
||||
stream.with_filtered_account_feedback(
|
||||
Arc::clone(&self.account_feedback),
|
||||
openai_failure_affects_account_score,
|
||||
)
|
||||
} else {
|
||||
stream
|
||||
})
|
||||
|
||||
@@ -391,7 +391,12 @@ pub(super) fn codex_error_observation(
|
||||
)?
|
||||
.with_status_code(status.as_u16());
|
||||
}
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
|
||||
CodexClientError::WebSocket(error)
|
||||
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
|
||||
{
|
||||
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
|
||||
unreachable!("websocket error was checked above")
|
||||
};
|
||||
observation = observation.with_status_code(upstream.status_code);
|
||||
if let Some(request_id) = upstream.diagnostics.request_id.as_deref() {
|
||||
observation =
|
||||
|
||||
@@ -313,7 +313,12 @@ impl CodexClientError {
|
||||
rate_limit_headers,
|
||||
*send_phase,
|
||||
)),
|
||||
Self::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
|
||||
Self::WebSocket(error)
|
||||
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
|
||||
{
|
||||
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
|
||||
unreachable!("websocket error was checked above")
|
||||
};
|
||||
Some(CodexUpstreamFailure::from_response(
|
||||
StatusCode::from_u16(upstream.status_code).unwrap_or(StatusCode::BAD_GATEWAY),
|
||||
&upstream.body,
|
||||
@@ -848,7 +853,7 @@ pub(super) fn websocket_success_decision(
|
||||
pub(super) fn http_fallback_decision(
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> CodexTransportDecision {
|
||||
match error {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
|
||||
CodexTransportDecision::Http2WebSocketBudgetExhausted
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ impl CodexBackendClient {
|
||||
context.installation_id,
|
||||
)?;
|
||||
insert_optional_protocol_header(&mut headers, "x-codex-image-turn-id", image_turn_id);
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-client-request-id"),
|
||||
HeaderValue::from_str(context.request_id)?,
|
||||
|
||||
@@ -58,6 +58,7 @@ const PERSISTABLE_UPSTREAM_CODES: &[&str] = &[
|
||||
"rate_limit_exceeded",
|
||||
"rate_limit_reached",
|
||||
"refresh_token_invalidated",
|
||||
"server_error",
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
"token_expired",
|
||||
|
||||
@@ -4,6 +4,8 @@ pub const CODEX_RESPONSES_PATH: &str = "/codex/responses";
|
||||
pub const CODEX_IMAGE_GENERATIONS_PATH: &str = "/codex/images/generations";
|
||||
/// `/codex/images/edits`
|
||||
pub const CODEX_IMAGE_EDITS_PATH: &str = "/codex/images/edits";
|
||||
/// `/codex/alpha/search`
|
||||
pub const CODEX_ALPHA_SEARCH_PATH: &str = "/codex/alpha/search";
|
||||
/// `/api/codex/usage`
|
||||
pub const CODEX_USAGE_API_PATH: &str = "/api/codex/usage";
|
||||
/// `/wham/usage`
|
||||
|
||||
@@ -36,8 +36,8 @@ pub use self::{
|
||||
},
|
||||
diagnostics::{CodexUpstreamDiagnostics, CodexUpstreamSendPhase},
|
||||
endpoints::{
|
||||
CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH, CODEX_RESPONSES_PATH,
|
||||
CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
|
||||
CODEX_ALPHA_SEARCH_PATH, CODEX_IMAGE_EDITS_PATH, CODEX_IMAGE_GENERATIONS_PATH,
|
||||
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, WHAM_PROFILE_STATISTICS_PATH,
|
||||
WHAM_RATE_LIMIT_RESET_CREDITS_CONSUME_PATH, WHAM_RATE_LIMIT_RESET_CREDITS_PATH,
|
||||
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
|
||||
},
|
||||
|
||||
@@ -162,6 +162,11 @@ impl TransportRequirement {
|
||||
matches!(self, Self::NewChain)
|
||||
}
|
||||
|
||||
/// 会话级 sticky 状态是否可以把首选 WebSocket 改为 HTTP。
|
||||
pub fn allows_sticky_http_fallback(self) -> bool {
|
||||
matches!(self, Self::NewChain)
|
||||
}
|
||||
|
||||
/// 用于审计与遥测的稳定名称。
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
@@ -193,8 +198,7 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
|
||||
TransportRequirement::ExternalUnknown
|
||||
}
|
||||
},
|
||||
None if request.use_websocket => TransportRequirement::NewChain,
|
||||
None => TransportRequirement::HttpRequired,
|
||||
None => TransportRequirement::NewChain,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -645,10 +649,9 @@ impl CodexResponsesRequest {
|
||||
|
||||
/// 返回请求语义与 child transport 隔离所用的子代理区分值。
|
||||
///
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
|
||||
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
|
||||
/// 传递给上游。它不拆分根会话的账号首选项;`thread_spawn` 仍用它派生独立
|
||||
/// WebSocket/continuation transport identity。
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;兼容客户端也可通过
|
||||
/// `client_metadata.x-openai-subagent` 声明同一语义。它不拆分根会话的账号首选项;
|
||||
/// `thread_spawn` 仍用它派生独立 WebSocket/continuation transport identity。
|
||||
pub fn subagent_kind(&self) -> Option<String> {
|
||||
self.semantics()
|
||||
.subagent_kind
|
||||
|
||||
@@ -427,7 +427,11 @@ fn metadata_string(request: &CodexResponsesRequest, key: &str) -> Option<String>
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn scope_turn_metadata(raw: &str, installation_id: &str, cross_account: bool) -> Option<String> {
|
||||
pub(crate) fn scope_turn_metadata(
|
||||
raw: &str,
|
||||
installation_id: &str,
|
||||
cross_account: bool,
|
||||
) -> Option<String> {
|
||||
let Ok(Value::Object(mut metadata)) = serde_json::from_str::<Value>(raw) else {
|
||||
return (!cross_account).then(|| raw.to_owned());
|
||||
};
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::{
|
||||
WebSocketPoolConnectLease, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
|
||||
WebSocketPoolLease,
|
||||
},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket},
|
||||
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket, transport_metric_reason},
|
||||
};
|
||||
|
||||
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
|
||||
@@ -188,9 +188,11 @@ pub(crate) async fn prepare_response_create_request_with_pool(
|
||||
}
|
||||
}
|
||||
}
|
||||
WebSocketPoolAcquire::Bypass(reason) => {
|
||||
Err(continuation_unavailable(bypass_unavailable_reason(reason)))
|
||||
}
|
||||
WebSocketPoolAcquire::ContinuationLost(observation) => Err(continuation_unavailable(
|
||||
PreviousResponseUnavailableReason::ReusedConnectionLost,
|
||||
)
|
||||
.with_connection_observation(observation)),
|
||||
WebSocketPoolAcquire::Bypass(reason) => Err(bypass_unavailable_error(reason)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,6 +371,7 @@ fn start_pooled_websocket_connect(
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let connection_exit_reason = opening_failure_reason(&error);
|
||||
let error_message = error.to_string();
|
||||
// 先交付 opening 原始错误,避免连接池清理侵占前台 fast-path 预算。
|
||||
let foreground_waiting = sender.send(Err(error)).is_ok();
|
||||
@@ -378,6 +381,7 @@ fn start_pooled_websocket_connect(
|
||||
conversation_id_hash = task_key.conversation_id_hash(),
|
||||
ws_preconnect_duration_ms = duration_millis_u64(started_at.elapsed()),
|
||||
foreground_waiting,
|
||||
connection_exit_reason,
|
||||
error = %error_message,
|
||||
ws_preconnect_outcome = "failed",
|
||||
"WebSocket pool connect finished"
|
||||
@@ -392,6 +396,21 @@ fn start_pooled_websocket_connect(
|
||||
}
|
||||
}
|
||||
|
||||
fn opening_failure_reason(error: &CodexWebSocketExchangeError) -> &'static str {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_) => "request_build_failure",
|
||||
CodexWebSocketExchangeError::Connect(error)
|
||||
| CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. }
|
||||
| CodexWebSocketExchangeError::FastPathTimeout { .. } => "connect_timeout",
|
||||
CodexWebSocketExchangeError::Upstream(_) => "upgrade_rejected",
|
||||
CodexWebSocketExchangeError::OriginCircuitOpen
|
||||
| CodexWebSocketExchangeError::OriginHalfOpenBusy => "connect_suppressed",
|
||||
CodexWebSocketExchangeError::SharedConnectFailed => "shared_connect_failed",
|
||||
_ => "connect_failure",
|
||||
}
|
||||
}
|
||||
|
||||
fn duration_millis_u64(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis())
|
||||
.unwrap_or(u64::MAX)
|
||||
@@ -489,16 +508,16 @@ async fn wait_for_shared_connect(
|
||||
}
|
||||
}
|
||||
|
||||
fn bypass_unavailable_reason(
|
||||
reason: WebSocketPoolBypassReason,
|
||||
) -> PreviousResponseUnavailableReason {
|
||||
fn bypass_unavailable_error(reason: WebSocketPoolBypassReason) -> CodexWebSocketExchangeError {
|
||||
match reason {
|
||||
WebSocketPoolBypassReason::Busy => PreviousResponseUnavailableReason::ConnectionBusy,
|
||||
WebSocketPoolBypassReason::Busy => {
|
||||
continuation_unavailable(PreviousResponseUnavailableReason::ConnectionBusy)
|
||||
}
|
||||
WebSocketPoolBypassReason::Disabled | WebSocketPoolBypassReason::Cap => {
|
||||
PreviousResponseUnavailableReason::PoolUnavailable
|
||||
continuation_unavailable(PreviousResponseUnavailableReason::PoolUnavailable)
|
||||
}
|
||||
WebSocketPoolBypassReason::ContinuationNotFound => {
|
||||
PreviousResponseUnavailableReason::FreshConnectionRequired
|
||||
continuation_unavailable(PreviousResponseUnavailableReason::FreshConnectionRequired)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -527,8 +546,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
created_at,
|
||||
} = connection;
|
||||
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
|
||||
discard_after_send(websocket, lease).await;
|
||||
return Err(post_send_ambiguous(error));
|
||||
let observation = websocket
|
||||
.observation()
|
||||
.with_exit_reason("outbound_transport_error");
|
||||
discard_after_send(websocket, lease, observation.clone()).await;
|
||||
return Err(post_send_ambiguous(
|
||||
error.with_connection_observation(observation),
|
||||
));
|
||||
}
|
||||
let connection_local_available = lease.is_some();
|
||||
let pool_return = lease.map(|lease| WebSocketStreamPoolReturn {
|
||||
@@ -548,9 +572,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
Ok(exchange)
|
||||
}
|
||||
|
||||
async fn discard_after_send(websocket: PumpedWebSocket, lease: Option<WebSocketPoolLease>) {
|
||||
async fn discard_after_send(
|
||||
websocket: PumpedWebSocket,
|
||||
lease: Option<WebSocketPoolLease>,
|
||||
observation: super::pump::WebSocketConnectionObservation,
|
||||
) {
|
||||
if let Some(lease) = lease {
|
||||
lease.discard().await;
|
||||
lease.discard_with_observation(observation).await;
|
||||
}
|
||||
websocket.close().await;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::transport::diagnostics::CodexUpstreamDiagnostics;
|
||||
use crate::transport::diagnostics::CodexUpstreamSendPhase;
|
||||
|
||||
use super::PreviousResponseUnavailableReason;
|
||||
use super::pump::WebSocketConnectionObservation;
|
||||
|
||||
/// Responses WebSocket 交互错误。
|
||||
#[derive(Debug, Error)]
|
||||
@@ -101,6 +102,13 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// 首个上游事件超时时长。
|
||||
timeout: Duration,
|
||||
},
|
||||
/// 将一个已分类交互错误与物理连接生命周期快照绑定。
|
||||
#[error("{source}")]
|
||||
ConnectionObserved {
|
||||
observation: WebSocketConnectionObservation,
|
||||
#[source]
|
||||
source: Box<CodexWebSocketExchangeError>,
|
||||
},
|
||||
}
|
||||
|
||||
/// 上游在 terminal 事件前发送的 WebSocket close 信息。
|
||||
@@ -240,7 +248,9 @@ impl CodexWebSocketExchangeError {
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
|
||||
/// 返回错误链中的上游终态前 Close 帧(若存在)。
|
||||
#[must_use]
|
||||
pub fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
|
||||
match self {
|
||||
Self::ClosedBeforeTerminal(close) => Some(close),
|
||||
Self::PostSendAmbiguous {
|
||||
@@ -250,7 +260,54 @@ impl CodexWebSocketExchangeError {
|
||||
| Self::ReusedConnectionDiedBeforeFirstEvent {
|
||||
source: Some(source),
|
||||
..
|
||||
} => source.close_before_terminal(),
|
||||
}
|
||||
| Self::ConnectionObserved { source, .. } => source.close_before_terminal(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_connection_observation(
|
||||
self,
|
||||
observation: WebSocketConnectionObservation,
|
||||
) -> Self {
|
||||
match self {
|
||||
Self::ConnectionObserved { .. } => self,
|
||||
source => Self::ConnectionObserved {
|
||||
observation,
|
||||
source: Box::new(source),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回错误关联的物理 WebSocket 连接生命周期快照。
|
||||
#[must_use]
|
||||
pub fn connection_observation(&self) -> Option<&WebSocketConnectionObservation> {
|
||||
match self {
|
||||
Self::ConnectionObserved { observation, .. } => Some(observation),
|
||||
Self::PostSendAmbiguous {
|
||||
source: Some(source),
|
||||
..
|
||||
}
|
||||
| Self::ReusedConnectionDiedBeforeFirstEvent {
|
||||
source: Some(source),
|
||||
..
|
||||
} => source.connection_observation(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回去掉观测 wrapper 后的原始分类错误。
|
||||
pub(crate) fn classified(&self) -> &Self {
|
||||
match self {
|
||||
Self::ConnectionObserved { source, .. } => source.classified(),
|
||||
_ => self,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn continuation_unavailable_reason(&self) -> Option<PreviousResponseUnavailableReason> {
|
||||
match self.classified() {
|
||||
Self::ContinuationUnavailable { reason } => Some(*reason),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -277,6 +334,9 @@ impl CodexWebSocketExchangeError {
|
||||
|
||||
/// opening 阶段只有明确的 transport 可用性失败才能切到同账号 HTTP。
|
||||
pub(in crate::transport) fn allows_pre_send_http_fallback(&self) -> bool {
|
||||
if let Self::ConnectionObserved { source, .. } = self {
|
||||
return source.allows_pre_send_http_fallback();
|
||||
}
|
||||
matches!(
|
||||
self,
|
||||
Self::Connect(_)
|
||||
@@ -291,7 +351,7 @@ impl CodexWebSocketExchangeError {
|
||||
|
||||
/// 精确连接状态不可用属于本地池路由事实,不消耗上游 WS 重试预算。
|
||||
pub(in crate::transport) fn requires_immediate_pool_fallback(&self) -> bool {
|
||||
matches!(self, Self::ContinuationUnavailable { .. })
|
||||
matches!(self.classified(), Self::ContinuationUnavailable { .. })
|
||||
}
|
||||
|
||||
/// 首个业务事件交付前,普通 WebSocket 失败可切到同账号 HTTP。
|
||||
@@ -302,6 +362,7 @@ impl CodexWebSocketExchangeError {
|
||||
source: Some(source),
|
||||
..
|
||||
} => source.allows_pre_delivery_http_fallback(),
|
||||
Self::ConnectionObserved { source, .. } => source.allows_pre_delivery_http_fallback(),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,9 @@ use super::super::{
|
||||
CodexWebSocketConnectionMetadata, PooledWebSocketConnection, WebSocketContinuationState,
|
||||
WebSocketPoolLease,
|
||||
},
|
||||
pump::{PumpExitReason, PumpedWebSocket},
|
||||
pump::{
|
||||
PumpExitReason, PumpedWebSocket, WebSocketConnectionObservation, transport_metric_reason,
|
||||
},
|
||||
};
|
||||
use super::io::{next_websocket_message, receive_idle_timeout, reused_stream_receive_error};
|
||||
use super::reducer::{ExchangeAction, WebSocketTerminalKind, reduce_websocket_event};
|
||||
@@ -173,12 +175,22 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let message = match message {
|
||||
Ok(message) => message,
|
||||
Err(error) => {
|
||||
let error = match error {
|
||||
CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout }
|
||||
if !saw_upstream_activity =>
|
||||
{
|
||||
CodexWebSocketExchangeError::InitialEventTimeout { timeout }
|
||||
}
|
||||
error => error,
|
||||
};
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
|
||||
)
|
||||
.await;
|
||||
let error = error.with_connection_observation(observation);
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
} else {
|
||||
@@ -197,6 +209,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
text.to_string()
|
||||
}
|
||||
tungstenite::Message::Binary(_) => {
|
||||
let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent;
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
@@ -204,24 +218,26 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
)
|
||||
.await;
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent))
|
||||
.send(Err(error.with_connection_observation(observation)))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
tungstenite::Message::Close(frame) => {
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
frame.as_ref().map(|frame| u16::from(frame.code)),
|
||||
frame.map(|frame| frame.reason.to_string()),
|
||||
last_event_type.clone(),
|
||||
);
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = error.with_connection_observation(observation);
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
} else {
|
||||
@@ -235,12 +251,19 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
|
||||
Ok(reduced) => reduced,
|
||||
Err(error) => {
|
||||
let observation =
|
||||
(!matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)))
|
||||
.then(|| connection_observation(&websocket, &error));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
|
||||
)
|
||||
.await;
|
||||
let error = match observation {
|
||||
Some(observation) => error.with_connection_observation(observation),
|
||||
None => error,
|
||||
};
|
||||
let _ = tx.send(Err(error)).await;
|
||||
return;
|
||||
}
|
||||
@@ -301,12 +324,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
.exit_reason()
|
||||
.and_then(|reason| reason.upstream_close().cloned())
|
||||
.map(|close| close.with_connection_id(websocket_connection_id));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = match upstream_close {
|
||||
Some(close) => CodexWebSocketExchangeError::ClosedBeforeTerminal(
|
||||
close.with_last_event_type(last_event_type),
|
||||
@@ -318,6 +335,14 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
last_event_type,
|
||||
),
|
||||
};
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = error.with_connection_observation(observation);
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
} else {
|
||||
@@ -353,6 +378,12 @@ async fn discard_stream_websocket(
|
||||
reason: StreamWebSocketDiscardReason,
|
||||
) {
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
let observation = websocket.observation();
|
||||
let tombstone_observation = if observation.exit_reason() == "running" {
|
||||
observation.clone().with_exit_reason(reason.as_str())
|
||||
} else {
|
||||
observation.clone()
|
||||
};
|
||||
let pump_exit = websocket.exit_reason();
|
||||
let pump_exit_reason = pump_exit
|
||||
.as_ref()
|
||||
@@ -367,11 +398,45 @@ async fn discard_stream_websocket(
|
||||
reason = reason.as_str(),
|
||||
pump_exit_reason,
|
||||
pump_exit_detail,
|
||||
connection_exit_reason = observation.exit_reason(),
|
||||
connection_age_ms = observation.age_ms(),
|
||||
connection_idle_ms = observation.idle_ms(),
|
||||
pooled = pool_return.is_some(),
|
||||
"Discarding Responses WebSocket stream"
|
||||
);
|
||||
if let Some(pool_return) = pool_return {
|
||||
pool_return.lease.discard().await;
|
||||
pool_return
|
||||
.lease
|
||||
.discard_with_observation(tombstone_observation)
|
||||
.await;
|
||||
}
|
||||
websocket.close().await;
|
||||
}
|
||||
|
||||
fn connection_observation(
|
||||
websocket: &PumpedWebSocket,
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> WebSocketConnectionObservation {
|
||||
let observation = websocket.observation();
|
||||
if observation.exit_reason() != "running" {
|
||||
return observation;
|
||||
}
|
||||
observation.with_exit_reason(exchange_exit_reason(error))
|
||||
}
|
||||
|
||||
fn exchange_exit_reason(error: &CodexWebSocketExchangeError) -> &'static str {
|
||||
match error.classified() {
|
||||
CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal(close) => {
|
||||
if close.code() == Some(1000) {
|
||||
"normal_close"
|
||||
} else {
|
||||
"upstream_close"
|
||||
}
|
||||
}
|
||||
CodexWebSocketExchangeError::ReceiveIdleTimeout { .. } => "receive_idle_timeout",
|
||||
CodexWebSocketExchangeError::InitialEventTimeout { .. } => "initial_event_timeout",
|
||||
CodexWebSocketExchangeError::UnexpectedBinaryEvent => "unexpected_binary_event",
|
||||
_ => "exchange_failure",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ pub use self::{
|
||||
CodexWebSocketPool, CodexWebSocketPoolConfig, CodexWebSocketPoolKey,
|
||||
WebSocketPoolBypassReason, WebSocketPoolDecision,
|
||||
},
|
||||
pump::WebSocketConnectionObservation,
|
||||
};
|
||||
pub(crate) use self::{
|
||||
coordinator::{
|
||||
|
||||
@@ -70,16 +70,23 @@ pub enum PreviousResponseUnavailableReason {
|
||||
UpstreamRejected,
|
||||
}
|
||||
|
||||
impl fmt::Display for PreviousResponseUnavailableReason {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
impl PreviousResponseUnavailableReason {
|
||||
#[must_use]
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::PoolUnavailable => "pool_unavailable",
|
||||
Self::FreshConnectionRequired => "fresh_connection_required",
|
||||
Self::ConnectionBusy => "connection_busy",
|
||||
Self::LatestResponseMismatch => "latest_response_mismatch",
|
||||
Self::ReusedConnectionLost => "reused_connection_lost",
|
||||
Self::UpstreamRejected => "upstream_rejected",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PreviousResponseUnavailableReason {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ use uuid::Uuid;
|
||||
|
||||
use super::state::{CodexWebSocketPoolKey, PooledWebSocketConnection, WebSocketPoolReservation};
|
||||
use super::{CodexWebSocketPool, WebSocketPoolBypassReason};
|
||||
use crate::transport::websocket::pump::WebSocketConnectionObservation;
|
||||
|
||||
pub(crate) enum WebSocketPoolAcquire {
|
||||
Reused {
|
||||
@@ -22,6 +23,7 @@ pub(crate) enum WebSocketPoolAcquire {
|
||||
},
|
||||
Connect(WebSocketPoolConnectLease),
|
||||
Wait(WebSocketPoolConnectWaiter),
|
||||
ContinuationLost(WebSocketConnectionObservation),
|
||||
Bypass(WebSocketPoolBypassReason),
|
||||
}
|
||||
|
||||
@@ -182,10 +184,12 @@ impl WebSocketPoolLease {
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
pub(crate) async fn discard(mut self) {
|
||||
pub(crate) async fn discard_with_observation(
|
||||
mut self,
|
||||
observation: WebSocketConnectionObservation,
|
||||
) {
|
||||
self.pool
|
||||
.discard_reserved(&self.key, self.reservation.id)
|
||||
.await;
|
||||
.discard_reserved_with_observation(&self.key, self.reservation.id, observation);
|
||||
self.armed = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ use self::state::{
|
||||
close_pooled_connections,
|
||||
};
|
||||
use super::pump::PumpKeepalive;
|
||||
use super::pump::WebSocketConnectionObservation;
|
||||
|
||||
pub use self::state::CodexWebSocketPoolKey;
|
||||
pub(crate) use self::{
|
||||
@@ -154,6 +155,7 @@ impl CodexWebSocketPool {
|
||||
let mut connections_to_close = Vec::new();
|
||||
let acquire = {
|
||||
let mut state = self.lock_state();
|
||||
let mut continuation_loss = None;
|
||||
if !self.config.enabled || state.shutting_down {
|
||||
return WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::Disabled);
|
||||
}
|
||||
@@ -163,9 +165,14 @@ impl CodexWebSocketPool {
|
||||
&& slot.latest_response_id() == Some(response_id))
|
||||
.then(|| candidate.clone())
|
||||
}) else {
|
||||
return WebSocketPoolAcquire::Bypass(
|
||||
WebSocketPoolBypassReason::ContinuationNotFound,
|
||||
);
|
||||
return state
|
||||
.continuation_loss(key, response_id, tokio::time::Instant::now())
|
||||
.map_or(
|
||||
WebSocketPoolAcquire::Bypass(
|
||||
WebSocketPoolBypassReason::ContinuationNotFound,
|
||||
),
|
||||
WebSocketPoolAcquire::ContinuationLost,
|
||||
);
|
||||
};
|
||||
key
|
||||
} else {
|
||||
@@ -188,9 +195,9 @@ impl CodexWebSocketPool {
|
||||
};
|
||||
// 零成本探活:后台 pump 已实时感知连接死亡(RST/Close/EOF/失活),
|
||||
// 复用前只需读取 is_closed 标志,避免复用到静默死连接后卡到超时。
|
||||
if connection.created_at.elapsed() < self.config.max_age
|
||||
&& !connection.websocket.is_closed()
|
||||
{
|
||||
let expired = connection.created_at.elapsed() >= self.config.max_age;
|
||||
let closed = connection.websocket.is_closed();
|
||||
if !expired && !closed {
|
||||
let lease = WebSocketPoolLease::reserve(
|
||||
self.clone(),
|
||||
key.clone(),
|
||||
@@ -202,12 +209,31 @@ impl CodexWebSocketPool {
|
||||
);
|
||||
return WebSocketPoolAcquire::Reused { connection, lease };
|
||||
}
|
||||
let observation = if expired && !closed {
|
||||
connection
|
||||
.websocket
|
||||
.observation()
|
||||
.with_exit_reason("max_age_expired")
|
||||
} else {
|
||||
connection.websocket.observation()
|
||||
};
|
||||
state.remember_continuation_loss(
|
||||
&key,
|
||||
connection.continuation.latest_response_id(),
|
||||
observation.clone(),
|
||||
tokio::time::Instant::now(),
|
||||
);
|
||||
connections_to_close.push(*connection);
|
||||
if required_response_id.is_some() {
|
||||
continuation_loss = Some(observation);
|
||||
}
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
let acquire = if required_response_id.is_some() {
|
||||
let acquire = if let Some(observation) = continuation_loss {
|
||||
WebSocketPoolAcquire::ContinuationLost(observation)
|
||||
} else if required_response_id.is_some() {
|
||||
WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::ContinuationNotFound)
|
||||
} else {
|
||||
let connect_permit = self.connect_semaphore.clone().try_acquire_owned().ok();
|
||||
@@ -259,6 +285,17 @@ impl CodexWebSocketPool {
|
||||
if reservation.id == reservation_id
|
||||
);
|
||||
if owns_reservation && (expired || state.shutting_down || !self.config.enabled) {
|
||||
if expired && let Some(connection) = connection.as_ref() {
|
||||
state.remember_continuation_loss(
|
||||
key,
|
||||
connection.continuation.latest_response_id(),
|
||||
connection
|
||||
.websocket
|
||||
.observation()
|
||||
.with_exit_reason("max_age_expired"),
|
||||
tokio::time::Instant::now(),
|
||||
);
|
||||
}
|
||||
state.slots.remove(key);
|
||||
} else if owns_reservation && let Some(connection) = connection.take() {
|
||||
state.slots.insert(
|
||||
@@ -274,10 +311,6 @@ impl CodexWebSocketPool {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discard_reserved(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
|
||||
self.discard_reserved_now(key, reservation_id);
|
||||
}
|
||||
|
||||
fn discard_reserved_now(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
|
||||
let mut state = self.lock_state();
|
||||
if matches!(
|
||||
@@ -288,6 +321,28 @@ impl CodexWebSocketPool {
|
||||
}
|
||||
}
|
||||
|
||||
fn discard_reserved_with_observation(
|
||||
&self,
|
||||
key: &CodexWebSocketPoolKey,
|
||||
reservation_id: Uuid,
|
||||
observation: WebSocketConnectionObservation,
|
||||
) {
|
||||
let mut state = self.lock_state();
|
||||
let latest_response_id = match state.slots.get(key) {
|
||||
Some(WebSocketPoolSlot::Busy(reservation)) if reservation.id == reservation_id => {
|
||||
reservation.latest_response_id.clone()
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
state.remember_continuation_loss(
|
||||
key,
|
||||
latest_response_id.as_deref(),
|
||||
observation,
|
||||
tokio::time::Instant::now(),
|
||||
);
|
||||
state.slots.remove(key);
|
||||
}
|
||||
|
||||
async fn finish_connect_reserved(
|
||||
&self,
|
||||
key: &CodexWebSocketPoolKey,
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
//! WebSocket 连接池状态和值对象。
|
||||
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::{sync::watch, time::Instant};
|
||||
@@ -10,7 +13,7 @@ use uuid::Uuid;
|
||||
use super::super::super::{
|
||||
diagnostics::CodexUpstreamDiagnostics, response_meta::CodexResponseMetadata,
|
||||
};
|
||||
use super::super::pump::PumpedWebSocket;
|
||||
use super::super::pump::{PumpedWebSocket, WebSocketConnectionObservation};
|
||||
use super::lease::WebSocketPoolConnectOutcome;
|
||||
|
||||
/// WebSocket 连接池 key。
|
||||
@@ -67,12 +70,72 @@ impl CodexWebSocketPoolKey {
|
||||
}
|
||||
}
|
||||
|
||||
const CONTINUATION_TOMBSTONE_TTL: Duration = Duration::from_mins(30);
|
||||
const MAX_CONTINUATION_TOMBSTONES: usize = 4_096;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct WebSocketPoolState {
|
||||
pub(super) slots: HashMap<CodexWebSocketPoolKey, WebSocketPoolSlot>,
|
||||
continuation_tombstones: VecDeque<WebSocketContinuationTombstone>,
|
||||
pub(super) shutting_down: bool,
|
||||
}
|
||||
|
||||
struct WebSocketContinuationTombstone {
|
||||
key: CodexWebSocketPoolKey,
|
||||
response_id: String,
|
||||
observation: WebSocketConnectionObservation,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
impl WebSocketPoolState {
|
||||
pub(super) fn remember_continuation_loss(
|
||||
&mut self,
|
||||
key: &CodexWebSocketPoolKey,
|
||||
response_id: Option<&str>,
|
||||
observation: WebSocketConnectionObservation,
|
||||
now: Instant,
|
||||
) {
|
||||
let Some(response_id) = response_id else {
|
||||
return;
|
||||
};
|
||||
self.prune_continuation_tombstones(now);
|
||||
self.continuation_tombstones.retain(|tombstone| {
|
||||
!(tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id)
|
||||
});
|
||||
while self.continuation_tombstones.len() >= MAX_CONTINUATION_TOMBSTONES {
|
||||
self.continuation_tombstones.pop_front();
|
||||
}
|
||||
self.continuation_tombstones
|
||||
.push_back(WebSocketContinuationTombstone {
|
||||
key: key.clone(),
|
||||
response_id: response_id.to_owned(),
|
||||
observation,
|
||||
expires_at: now + CONTINUATION_TOMBSTONE_TTL,
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn continuation_loss(
|
||||
&mut self,
|
||||
key: &CodexWebSocketPoolKey,
|
||||
response_id: &str,
|
||||
now: Instant,
|
||||
) -> Option<WebSocketConnectionObservation> {
|
||||
self.prune_continuation_tombstones(now);
|
||||
self.continuation_tombstones
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|tombstone| {
|
||||
tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id
|
||||
})
|
||||
.map(|tombstone| tombstone.observation.clone())
|
||||
}
|
||||
|
||||
fn prune_continuation_tombstones(&mut self, now: Instant) {
|
||||
self.continuation_tombstones
|
||||
.retain(|tombstone| tombstone.expires_at > now);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CodexWebSocketConnectionMetadata {
|
||||
pub(crate) turn_state: Option<String>,
|
||||
|
||||
@@ -174,7 +174,23 @@ impl CodexWebSocketPool {
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
match state.slots.remove(&key) {
|
||||
Some(WebSocketPoolSlot::Idle { connection, .. }) => close.push(*connection),
|
||||
Some(WebSocketPoolSlot::Idle { connection, .. }) => {
|
||||
let observation = if connection.websocket.is_closed() {
|
||||
connection.websocket.observation()
|
||||
} else {
|
||||
connection
|
||||
.websocket
|
||||
.observation()
|
||||
.with_exit_reason("max_age_expired")
|
||||
};
|
||||
state.remember_continuation_loss(
|
||||
&key,
|
||||
connection.continuation.latest_response_id(),
|
||||
observation,
|
||||
now,
|
||||
);
|
||||
close.push(*connection);
|
||||
}
|
||||
Some(WebSocketPoolSlot::Busy(reservation)) => {
|
||||
tracing::warn!(
|
||||
account_id = key.account_id(),
|
||||
|
||||
@@ -89,6 +89,7 @@ pub(crate) enum PumpExitReason {
|
||||
LocalClose,
|
||||
OutboundTransportError {
|
||||
message: String,
|
||||
metric_reason: &'static str,
|
||||
},
|
||||
UpstreamCloseFrame {
|
||||
close: Option<CodexWebSocketCloseError>,
|
||||
@@ -96,6 +97,7 @@ pub(crate) enum PumpExitReason {
|
||||
UpstreamEof,
|
||||
InboundTransportError {
|
||||
message: String,
|
||||
metric_reason: &'static str,
|
||||
},
|
||||
PongTimeout {
|
||||
timeout: Duration,
|
||||
@@ -105,6 +107,7 @@ pub(crate) enum PumpExitReason {
|
||||
},
|
||||
KeepaliveTransportError {
|
||||
message: String,
|
||||
metric_reason: &'static str,
|
||||
},
|
||||
MessageReceiverClosed,
|
||||
}
|
||||
@@ -127,9 +130,9 @@ impl PumpExitReason {
|
||||
|
||||
pub(crate) fn detail(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::OutboundTransportError { message }
|
||||
| Self::InboundTransportError { message }
|
||||
| Self::KeepaliveTransportError { message } => Some(message.clone()),
|
||||
Self::OutboundTransportError { message, .. }
|
||||
| Self::InboundTransportError { message, .. }
|
||||
| Self::KeepaliveTransportError { message, .. } => Some(message.clone()),
|
||||
Self::UpstreamCloseFrame { close } => close.as_ref().map(ToString::to_string),
|
||||
Self::PongTimeout { timeout } | Self::LivenessTimeout { timeout } => {
|
||||
Some(format!("{timeout:?}"))
|
||||
@@ -141,6 +144,27 @@ impl PumpExitReason {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metric_reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::CommandChannelClosed => "command_channel_closed",
|
||||
Self::LocalClose => "local_close",
|
||||
Self::OutboundTransportError { metric_reason, .. }
|
||||
| Self::InboundTransportError { metric_reason, .. }
|
||||
| Self::KeepaliveTransportError { metric_reason, .. } => metric_reason,
|
||||
Self::UpstreamCloseFrame { close } => {
|
||||
if close.as_ref().and_then(CodexWebSocketCloseError::code) == Some(1000) {
|
||||
"normal_close"
|
||||
} else {
|
||||
"upstream_close"
|
||||
}
|
||||
}
|
||||
Self::UpstreamEof => "eof",
|
||||
Self::PongTimeout { .. } => "pong_timeout",
|
||||
Self::LivenessTimeout { .. } => "liveness_timeout",
|
||||
Self::MessageReceiverClosed => "message_receiver_closed",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_close(&self) -> Option<&CodexWebSocketCloseError> {
|
||||
match self {
|
||||
Self::UpstreamCloseFrame { close: Some(close) } => Some(close),
|
||||
@@ -183,13 +207,71 @@ struct PendingPing {
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct WebSocketConnectionObservation {
|
||||
connection_id: Uuid,
|
||||
exit_reason: String,
|
||||
age_ms: u64,
|
||||
idle_ms: u64,
|
||||
}
|
||||
|
||||
impl WebSocketConnectionObservation {
|
||||
pub(crate) fn with_exit_reason(mut self, exit_reason: &'static str) -> Self {
|
||||
self.exit_reason = exit_reason.to_owned();
|
||||
self
|
||||
}
|
||||
|
||||
pub const fn connection_id(&self) -> Uuid {
|
||||
self.connection_id
|
||||
}
|
||||
|
||||
pub fn exit_reason(&self) -> &str {
|
||||
&self.exit_reason
|
||||
}
|
||||
|
||||
pub const fn age_ms(&self) -> u64 {
|
||||
self.age_ms
|
||||
}
|
||||
|
||||
pub const fn idle_ms(&self) -> u64 {
|
||||
self.idle_ms
|
||||
}
|
||||
}
|
||||
|
||||
struct PumpLifecycleState {
|
||||
opened_at: Instant,
|
||||
last_activity: Instant,
|
||||
closed_at: Option<Instant>,
|
||||
exit_reason: Option<PumpExitReason>,
|
||||
}
|
||||
|
||||
impl PumpLifecycleState {
|
||||
fn observation(
|
||||
&self,
|
||||
connection_id: Uuid,
|
||||
observed_at: Instant,
|
||||
) -> WebSocketConnectionObservation {
|
||||
let effective_end = self.closed_at.unwrap_or(observed_at);
|
||||
WebSocketConnectionObservation {
|
||||
connection_id,
|
||||
exit_reason: self
|
||||
.exit_reason
|
||||
.as_ref()
|
||||
.map_or("running", PumpExitReason::metric_reason)
|
||||
.to_owned(),
|
||||
age_ms: duration_millis(effective_end.duration_since(self.opened_at)),
|
||||
idle_ms: duration_millis(effective_end.duration_since(self.last_activity)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由后台 pump 任务托管的 WebSocket 连接句柄。
|
||||
pub(crate) struct PumpedWebSocket {
|
||||
connection_id: Uuid,
|
||||
tx_command: mpsc::Sender<PumpCommand>,
|
||||
rx_message: mpsc::Receiver<Result<Message, tungstenite::Error>>,
|
||||
closed: Arc<AtomicBool>,
|
||||
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
|
||||
lifecycle: Arc<Mutex<PumpLifecycleState>>,
|
||||
pump: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
@@ -212,10 +294,16 @@ impl PumpedWebSocket {
|
||||
let (tx_command, rx_command) = mpsc::channel::<PumpCommand>(PUMP_COMMAND_BUFFER);
|
||||
let (tx_message, rx_message) = mpsc::channel(PUMP_MESSAGE_BUFFER);
|
||||
let connection_id = Uuid::new_v4();
|
||||
let opened_at = Instant::now();
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let closed_for_task = Arc::clone(&closed);
|
||||
let exit_reason = Arc::new(Mutex::new(None));
|
||||
let exit_reason_for_task = Arc::clone(&exit_reason);
|
||||
let lifecycle = Arc::new(Mutex::new(PumpLifecycleState {
|
||||
opened_at,
|
||||
last_activity: opened_at,
|
||||
closed_at: None,
|
||||
exit_reason: None,
|
||||
}));
|
||||
let lifecycle_for_task = Arc::clone(&lifecycle);
|
||||
let pump = tokio::spawn(async move {
|
||||
pump_loop(
|
||||
connection_id,
|
||||
@@ -225,7 +313,7 @@ impl PumpedWebSocket {
|
||||
keepalive,
|
||||
context,
|
||||
closed_for_task,
|
||||
exit_reason_for_task,
|
||||
lifecycle_for_task,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -234,7 +322,7 @@ impl PumpedWebSocket {
|
||||
tx_command,
|
||||
rx_message,
|
||||
closed,
|
||||
exit_reason,
|
||||
lifecycle,
|
||||
pump: Some(pump),
|
||||
}
|
||||
}
|
||||
@@ -272,12 +360,20 @@ impl PumpedWebSocket {
|
||||
}
|
||||
|
||||
pub(crate) fn exit_reason(&self) -> Option<PumpExitReason> {
|
||||
self.exit_reason
|
||||
self.lifecycle
|
||||
.lock()
|
||||
.expect("WebSocket pump exit reason lock poisoned")
|
||||
.expect("WebSocket pump lifecycle lock poisoned")
|
||||
.exit_reason
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn observation(&self) -> WebSocketConnectionObservation {
|
||||
self.lifecycle
|
||||
.lock()
|
||||
.expect("WebSocket pump lifecycle lock poisoned")
|
||||
.observation(self.connection_id, Instant::now())
|
||||
}
|
||||
|
||||
/// 主动关闭连接(best-effort 发送 Close 帧)。
|
||||
pub(crate) async fn close(&self) {
|
||||
if !self.is_closed() {
|
||||
@@ -306,9 +402,12 @@ async fn pump_loop(
|
||||
keepalive: PumpKeepalive,
|
||||
context: PumpLogContext,
|
||||
closed: Arc<AtomicBool>,
|
||||
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
|
||||
lifecycle: Arc<Mutex<PumpLifecycleState>>,
|
||||
) {
|
||||
let mut last_activity = Instant::now();
|
||||
let mut last_activity = lifecycle
|
||||
.lock()
|
||||
.expect("WebSocket pump lifecycle lock poisoned")
|
||||
.last_activity;
|
||||
let ping_interval = keepalive.ping_interval.filter(|d| !d.is_zero());
|
||||
let ping_timeout = keepalive.ping_timeout.filter(|d| !d.is_zero());
|
||||
let liveness_timeout = keepalive.liveness_timeout.filter(|d| !d.is_zero());
|
||||
@@ -341,6 +440,7 @@ async fn pump_loop(
|
||||
}
|
||||
// 背压期间无法读取 socket,不能把这段本地下游阻塞误算成上游静默。
|
||||
last_activity = Instant::now();
|
||||
record_activity(&lifecycle, last_activity);
|
||||
}
|
||||
command = rx_command.recv() => {
|
||||
let Some(command) = command else {
|
||||
@@ -370,12 +470,14 @@ async fn pump_loop(
|
||||
None => break 'pump PumpExitReason::UpstreamEof,
|
||||
Some(Ok(message)) => {
|
||||
last_activity = Instant::now();
|
||||
record_activity(&lifecycle, last_activity);
|
||||
observe_inbound(connection_id, &mut pending_ping, &message);
|
||||
match message {
|
||||
Message::Ping(payload) => {
|
||||
if let Err(error) = inner.send(Message::Pong(payload)).await {
|
||||
break 'pump PumpExitReason::KeepaliveTransportError {
|
||||
message: error.to_string(),
|
||||
metric_reason: transport_metric_reason(&error),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -410,6 +512,7 @@ async fn pump_loop(
|
||||
Some(Err(err)) => {
|
||||
let terminal_reason = PumpExitReason::InboundTransportError {
|
||||
message: err.to_string(),
|
||||
metric_reason: transport_metric_reason(&err),
|
||||
};
|
||||
if let Some(reason) = enqueue_inbound(
|
||||
&tx_message,
|
||||
@@ -452,10 +555,22 @@ async fn pump_loop(
|
||||
};
|
||||
|
||||
closed.store(true, Ordering::Release);
|
||||
*exit_reason
|
||||
.lock()
|
||||
.expect("WebSocket pump exit reason lock poisoned") = Some(reason.clone());
|
||||
log_pump_exit(connection_id, &context, &reason, backpressure_events);
|
||||
let observation = {
|
||||
let closed_at = Instant::now();
|
||||
let mut lifecycle = lifecycle
|
||||
.lock()
|
||||
.expect("WebSocket pump lifecycle lock poisoned");
|
||||
lifecycle.closed_at = Some(closed_at);
|
||||
lifecycle.exit_reason = Some(reason.clone());
|
||||
lifecycle.observation(connection_id, closed_at)
|
||||
};
|
||||
log_pump_exit(
|
||||
connection_id,
|
||||
&context,
|
||||
&reason,
|
||||
&observation,
|
||||
backpressure_events,
|
||||
);
|
||||
|
||||
if reason.should_send_close() {
|
||||
let _ = inner.send(Message::Close(None)).await;
|
||||
@@ -492,10 +607,16 @@ async fn handle_command(inner: &mut RawWsStream, command: PumpCommand) -> Option
|
||||
let PumpCommand::Send { message, ack } = command;
|
||||
let is_close = matches!(message, Message::Close(_));
|
||||
let result = inner.send(message).await;
|
||||
let transport_error = result.as_ref().err().map(ToString::to_string);
|
||||
let transport_error = result
|
||||
.as_ref()
|
||||
.err()
|
||||
.map(|error| (error.to_string(), transport_metric_reason(error)));
|
||||
let _ = ack.send(result);
|
||||
match transport_error {
|
||||
Some(message) => Some(PumpExitReason::OutboundTransportError { message }),
|
||||
Some((message, metric_reason)) => Some(PumpExitReason::OutboundTransportError {
|
||||
message,
|
||||
metric_reason,
|
||||
}),
|
||||
None if is_close => Some(PumpExitReason::LocalClose),
|
||||
None => None,
|
||||
}
|
||||
@@ -517,6 +638,34 @@ fn observe_inbound(connection_id: Uuid, pending_ping: &mut Option<PendingPing>,
|
||||
*pending_ping = None;
|
||||
}
|
||||
|
||||
fn record_activity(lifecycle: &Mutex<PumpLifecycleState>, observed_at: Instant) {
|
||||
lifecycle
|
||||
.lock()
|
||||
.expect("WebSocket pump lifecycle lock poisoned")
|
||||
.last_activity = observed_at;
|
||||
}
|
||||
|
||||
pub(crate) fn transport_metric_reason(error: &tungstenite::Error) -> &'static str {
|
||||
match error {
|
||||
tungstenite::Error::Io(error) => match error.kind() {
|
||||
std::io::ErrorKind::ConnectionReset => "tcp_reset",
|
||||
std::io::ErrorKind::ConnectionAborted => "connection_aborted",
|
||||
std::io::ErrorKind::BrokenPipe => "broken_pipe",
|
||||
std::io::ErrorKind::UnexpectedEof => "unexpected_eof",
|
||||
std::io::ErrorKind::TimedOut => "transport_timeout",
|
||||
_ => "transport_error",
|
||||
},
|
||||
tungstenite::Error::Protocol(
|
||||
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
|
||||
) => "tcp_reset",
|
||||
_ => "transport_error",
|
||||
}
|
||||
}
|
||||
|
||||
fn duration_millis(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
async fn send_keepalive_ping(
|
||||
inner: &mut RawWsStream,
|
||||
sequence: &mut u64,
|
||||
@@ -529,6 +678,7 @@ async fn send_keepalive_ping(
|
||||
.await
|
||||
.map_err(|error| PumpExitReason::KeepaliveTransportError {
|
||||
message: error.to_string(),
|
||||
metric_reason: transport_metric_reason(&error),
|
||||
})?;
|
||||
Ok(ping_timeout.map(|timeout| PendingPing {
|
||||
payload,
|
||||
@@ -540,6 +690,7 @@ fn log_pump_exit(
|
||||
connection_id: Uuid,
|
||||
context: &PumpLogContext,
|
||||
reason: &PumpExitReason,
|
||||
observation: &WebSocketConnectionObservation,
|
||||
backpressure_events: u64,
|
||||
) {
|
||||
let detail = reason.detail().unwrap_or_default();
|
||||
@@ -551,6 +702,9 @@ fn log_pump_exit(
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
connection_exit_reason = observation.exit_reason(),
|
||||
connection_age_ms = observation.age_ms(),
|
||||
connection_idle_ms = observation.idle_ms(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
"Responses WebSocket pump stopped unexpectedly"
|
||||
@@ -565,6 +719,9 @@ fn log_pump_exit(
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
connection_exit_reason = observation.exit_reason(),
|
||||
connection_age_ms = observation.age_ms(),
|
||||
connection_idle_ms = observation.idle_ms(),
|
||||
pump_exit_detail = detail,
|
||||
upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code),
|
||||
upstream_error_raw,
|
||||
@@ -578,6 +735,9 @@ fn log_pump_exit(
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
connection_exit_reason = observation.exit_reason(),
|
||||
connection_age_ms = observation.age_ms(),
|
||||
connection_idle_ms = observation.idle_ms(),
|
||||
pump_exit_detail = detail,
|
||||
backpressure_events,
|
||||
"Responses WebSocket pump stopped"
|
||||
|
||||
@@ -336,7 +336,10 @@ async fn openai_core_provider_projects_codex_request_observation_without_routing
|
||||
)]));
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
|
||||
|
||||
let observation = bundle.core_provider().request_observation(&operation);
|
||||
let client_key_id = ClientApiKeyId::new("key_openai_admin_observation").expect("client key");
|
||||
let observation = bundle
|
||||
.core_provider()
|
||||
.request_observation(&operation, &client_key_id);
|
||||
|
||||
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
|
||||
assert_eq!(observation.subagent_kind.as_deref(), Some("review"));
|
||||
|
||||
@@ -28,7 +28,7 @@ use gateway_core::event::GatewayEvent;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::operation::{
|
||||
CapabilityRequirements, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
|
||||
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload,
|
||||
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::{
|
||||
@@ -1107,6 +1107,140 @@ async fn image_endpoint_returns_the_exact_upstream_error_response() {
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_search_preserves_wire_and_scopes_turn_metadata_to_the_selected_account() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let server = MockServer::start().await;
|
||||
let request_body = br#"{ "id":"search-session", "model":"gpt-future", "commands":{"search_query":[{"q":"private query","future":9007199254740993}]}, "future":1, "future":2 }"#;
|
||||
let response_body = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/alpha/search"))
|
||||
.and(header("originator", "codex_cli_rs"))
|
||||
.and(body_bytes(request_body.to_vec()))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "application/json")
|
||||
.insert_header("x-future-search-header", "preserved")
|
||||
.set_body_raw(response_body.to_vec(), "application/json"),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let turn_metadata = r#"{"session_id":"session","thread_id":"thread","turn_id":"turn","installation_id":"client-installation","account_id":"client-account","future":true}"#;
|
||||
let payload = RawJsonPayload::new("openai", Bytes::from_static(request_body))
|
||||
.expect("search payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
json!(turn_metadata),
|
||||
)]));
|
||||
let operation = Operation::Search(StandaloneSearchRequest::from_raw_json(payload));
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_provider_endpoint_request("openai", operation),
|
||||
context("req_search_contract", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare search provider stream");
|
||||
let mut raw_response = None;
|
||||
let mut completed = false;
|
||||
let mut observed_http_json = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("search provider event");
|
||||
completed |= event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
if let Some(raw) = event.wire_event().and_then(|wire| wire.raw_json_body()) {
|
||||
assert!(raw_response.replace(raw.clone()).is_none());
|
||||
}
|
||||
if let Some(observation) = event.response_observation() {
|
||||
observed_http_json |= observation.transport().as_str() == "http_json"
|
||||
&& observation.status_code() == Some(200)
|
||||
&& observation.client_headers().iter().any(|header| {
|
||||
header.name() == "x-future-search-header"
|
||||
&& header.value().as_ref() == b"preserved"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
assert!(completed);
|
||||
assert!(observed_http_json);
|
||||
assert_eq!(raw_response.as_deref(), Some(response_body.as_slice()));
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
let request = requests.first().expect("single search request");
|
||||
assert_eq!(request.body.as_slice(), request_body);
|
||||
let metadata_values = captured_header_values(request, "x-codex-turn-metadata");
|
||||
assert_eq!(metadata_values.len(), 1);
|
||||
let metadata: Value = serde_json::from_slice(&metadata_values[0]).expect("turn metadata JSON");
|
||||
assert_eq!(metadata.get("session_id"), Some(&json!("session")));
|
||||
assert_eq!(metadata.get("thread_id"), Some(&json!("thread")));
|
||||
assert_eq!(metadata.get("turn_id"), Some(&json!("turn")));
|
||||
assert_eq!(metadata.get("future"), Some(&json!(true)));
|
||||
assert!(metadata.get("account_id").is_none());
|
||||
assert_ne!(
|
||||
metadata.get("installation_id"),
|
||||
Some(&json!("client-installation"))
|
||||
);
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_search_returns_the_exact_upstream_error_response() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let server = MockServer::start().await;
|
||||
let request_body =
|
||||
br#"{ "id":"search-session", "model":"gpt-future", "commands":{"open":[]} }"#;
|
||||
let response_body = br#"{ "error":{"message":"future search validation","type":"search_error","code":"future_code"}, "future":9007199254740993 }"#;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/alpha/search"))
|
||||
.and(body_bytes(request_body.to_vec()))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(422)
|
||||
.insert_header("content-type", "application/problem+json")
|
||||
.insert_header("x-future-search-error", "preserved")
|
||||
.set_body_bytes(response_body.to_vec()),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let payload =
|
||||
RawJsonPayload::new("openai", Bytes::from_static(request_body)).expect("search payload");
|
||||
let operation = Operation::Search(StandaloneSearchRequest::from_raw_json(payload));
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_provider_endpoint_request("openai", operation),
|
||||
context("req_search_error", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare search provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("search rejection must surface an upstream response"),
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(error.upstream_status(), Some(422));
|
||||
let response = error
|
||||
.client_visible_upstream_response()
|
||||
.expect("raw upstream search error response");
|
||||
assert_eq!(response.status(), 422);
|
||||
assert_eq!(
|
||||
response.content_type(),
|
||||
Some(b"application/problem+json".as_slice())
|
||||
);
|
||||
assert_eq!(response.body().as_ref(), response_body);
|
||||
assert!(response.headers().iter().any(|header| {
|
||||
header.name() == "x-future-search-error" && header.value().as_ref() == b"preserved"
|
||||
}));
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capacity_selection_error_preserves_classification_and_retry_after() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1295,7 +1429,7 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_session() {
|
||||
async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
@@ -1306,6 +1440,26 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
let completed_response = |response_id: &str| {
|
||||
Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
)
|
||||
};
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
@@ -1371,6 +1525,31 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
|
||||
.await
|
||||
.expect("write sticky HTTP response");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept same-session continuation WebSocket connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("same-session continuation request")
|
||||
.expect("valid same-session continuation request");
|
||||
let payload: Value = serde_json::from_str(
|
||||
request
|
||||
.to_text()
|
||||
.expect("same-session continuation request text"),
|
||||
)
|
||||
.expect("same-session continuation request JSON");
|
||||
assert_eq!(
|
||||
payload.get("previous_response_id"),
|
||||
Some(&json!("external-previous-response"))
|
||||
);
|
||||
websocket
|
||||
.send(completed_response("resp_external_continuation"))
|
||||
.await
|
||||
.expect("complete same-session continuation request");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
@@ -1382,24 +1561,7 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
|
||||
.expect("other-session WebSocket request")
|
||||
.expect("valid other-session WebSocket request");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_other_session",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.send(completed_response("resp_other_session"))
|
||||
.await
|
||||
.expect("complete other-session WebSocket request");
|
||||
});
|
||||
@@ -1472,6 +1634,11 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
|
||||
raw_close.get("last_event_type"),
|
||||
Some(&json!("response.output_item.added"))
|
||||
);
|
||||
let connection = error
|
||||
.connection_observation()
|
||||
.expect("normal close should retain its connection lifecycle observation");
|
||||
assert_eq!(connection.exit_reason(), "normal_close");
|
||||
assert!(connection.age_ms() >= connection.idle_ms());
|
||||
|
||||
let second_operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
@@ -1491,6 +1658,27 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
|
||||
}
|
||||
drop(sticky_stream);
|
||||
|
||||
let continuation_operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-continuation"),
|
||||
None,
|
||||
));
|
||||
let mut continuation_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", continuation_operation),
|
||||
external_continuation_context("req_sticky_external_continuation"),
|
||||
)
|
||||
.await
|
||||
.expect("sticky state must not force a continuation onto HTTP");
|
||||
assert_eq!(
|
||||
continuation_stream.metadata().transport().as_str(),
|
||||
"websocket"
|
||||
);
|
||||
while let Some(event) = continuation_stream.next().await {
|
||||
event.expect("same-session continuation WebSocket response");
|
||||
}
|
||||
drop(continuation_stream);
|
||||
|
||||
let other_operation = Operation::Generate(generate_with_session_context(
|
||||
"other-websocket-session",
|
||||
Some("thread-first"),
|
||||
@@ -2044,7 +2232,7 @@ async fn cross_account_continuation_should_require_client_replay_without_an_upst
|
||||
detail.error_type(),
|
||||
),
|
||||
(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
Some(ContinuationFailure::HistoryUnavailable),
|
||||
Some("previous_response_not_found"),
|
||||
@@ -2094,6 +2282,10 @@ async fn missing_affinity_continuation_should_require_client_replay_without_an_u
|
||||
.await
|
||||
.expect("captured upstream requests");
|
||||
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
);
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_error()
|
||||
@@ -3396,6 +3588,10 @@ async fn same_account_previous_response_not_found_should_remain_client_visible()
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
failure.kind(),
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
);
|
||||
assert_eq!(
|
||||
failure.continuation_failure(),
|
||||
Some(ContinuationFailure::HistoryUnavailable)
|
||||
@@ -3625,6 +3821,10 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
|
||||
None => panic!("competing exact continuation must fail while its socket is busy"),
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
exact_error.kind(),
|
||||
ProviderErrorKind::ContinuationRecoveryRequired
|
||||
);
|
||||
assert_eq!(
|
||||
exact_error.continuation_failure(),
|
||||
Some(ContinuationFailure::Busy)
|
||||
@@ -3673,7 +3873,7 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
|
||||
detail.message(),
|
||||
),
|
||||
(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
UpstreamSendState::NotSent,
|
||||
Some(ContinuationFailure::HistoryUnavailable),
|
||||
Some(ContinuationRecoveryDisposition::ClientReplayRequired),
|
||||
@@ -4085,7 +4285,8 @@ fn request_observation_reads_openai_metadata_without_changing_the_operation() {
|
||||
let generation = GenerateRequest::from_protocol_payload(payload);
|
||||
let operation = Operation::Generate(generation);
|
||||
|
||||
let observation = provider(&store).request_observation(&operation);
|
||||
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
|
||||
let observation = provider(&store).request_observation(&operation, &client_key_id);
|
||||
|
||||
assert_eq!(observation.request_kind.as_deref(), Some("review"));
|
||||
assert_eq!(observation.subagent_kind.as_deref(), Some("worker"));
|
||||
@@ -4103,7 +4304,8 @@ fn request_observation_preserves_the_raw_reasoning_effort() {
|
||||
.expect("protocol payload");
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
|
||||
|
||||
let observation = provider(&store).request_observation(&operation);
|
||||
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
|
||||
let observation = provider(&store).request_observation(&operation, &client_key_id);
|
||||
|
||||
assert_eq!(
|
||||
observation.reasoning_effort.as_deref(),
|
||||
@@ -4133,7 +4335,8 @@ fn request_observation_ignores_future_session_state_without_rewriting_the_protoc
|
||||
);
|
||||
let operation = Operation::Generate(generation);
|
||||
|
||||
let _observation = provider(&store).request_observation(&operation);
|
||||
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
|
||||
let _observation = provider(&store).request_observation(&operation, &client_key_id);
|
||||
|
||||
let Operation::Generate(generation) = &operation else {
|
||||
panic!("operation should remain a generate request");
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt, stream};
|
||||
use gateway_core::account::{AccountFeedbackStats, ProviderAccountId};
|
||||
use gateway_core::engine::provider::{EventStream, ProviderCallMetadata, ProviderStream};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
use gateway_core::upstream::{UpstreamSendState, UpstreamTransport};
|
||||
use provider_openai::openai_failure_affects_account_score;
|
||||
|
||||
fn sent_error(kind: ProviderErrorKind, code: Option<&str>) -> ProviderError {
|
||||
let error = ProviderError::new(kind, UpstreamSendState::Sent);
|
||||
match code {
|
||||
Some(code) => error.with_upstream_code(OpaqueUpstreamValue::new(code)),
|
||||
None => error,
|
||||
}
|
||||
}
|
||||
|
||||
fn deliver_error_with_openai_feedback(
|
||||
feedback: &Arc<AccountFeedbackStats>,
|
||||
provider: &ProviderKind,
|
||||
account: &ProviderAccountId,
|
||||
error: ProviderError,
|
||||
) {
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-5").expect("model"),
|
||||
account.clone(),
|
||||
UpstreamTransport::new("http_sse").expect("transport"),
|
||||
);
|
||||
let events: EventStream = Box::pin(stream::iter([Err(error)]));
|
||||
let mut provider_stream = ProviderStream::new(metadata, events, ())
|
||||
.with_filtered_account_feedback(Arc::clone(feedback), openai_failure_affects_account_score);
|
||||
|
||||
futures::executor::block_on(async {
|
||||
assert!(
|
||||
provider_stream
|
||||
.next()
|
||||
.await
|
||||
.is_some_and(|event| event.is_err())
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_include_server_overload_as_a_regular_reason() {
|
||||
let error = sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded"));
|
||||
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_accept_the_closed_reason_list() {
|
||||
for code in [
|
||||
"server_is_overloaded",
|
||||
"slow_down",
|
||||
"rate_limit_exceeded",
|
||||
"rate_limit_error",
|
||||
"server_error",
|
||||
"service_unavailable_error",
|
||||
] {
|
||||
let error = sent_error(ProviderErrorKind::Unavailable, Some(code));
|
||||
assert!(
|
||||
openai_failure_affects_account_score(&error),
|
||||
"allowlisted code was rejected: {code}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_normalize_case_and_whitespace() {
|
||||
let error = sent_error(ProviderErrorKind::Unavailable, Some(" SERVER_ERROR "));
|
||||
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() {
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"upstream unavailable",
|
||||
None,
|
||||
Some("SERVICE_UNAVAILABLE_ERROR".to_owned()),
|
||||
));
|
||||
|
||||
assert!(openai_failure_affects_account_score(&error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_prefer_a_structured_code_over_type() {
|
||||
let error = ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"bad request",
|
||||
Some("invalid_request".to_owned()),
|
||||
Some("server_error".to_owned()),
|
||||
));
|
||||
|
||||
assert!(!openai_failure_affects_account_score(&error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_reject_client_and_unknown_failures() {
|
||||
for error in [
|
||||
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
|
||||
sent_error(
|
||||
ProviderErrorKind::ContinuationRecoveryRequired,
|
||||
Some("previous_response_not_found"),
|
||||
),
|
||||
sent_error(ProviderErrorKind::Unsupported, Some("unsupported")),
|
||||
sent_error(ProviderErrorKind::Unauthorized, Some("token_expired")),
|
||||
sent_error(ProviderErrorKind::QuotaExhausted, Some("quota_exhausted")),
|
||||
sent_error(ProviderErrorKind::RateLimited, Some("rate_limit_reached")),
|
||||
sent_error(
|
||||
ProviderErrorKind::Unavailable,
|
||||
Some("internal_server_error"),
|
||||
),
|
||||
sent_error(ProviderErrorKind::Unavailable, Some("service_unavailable")),
|
||||
sent_error(
|
||||
ProviderErrorKind::Transport,
|
||||
Some("upstream_transport_error"),
|
||||
),
|
||||
sent_error(ProviderErrorKind::Timeout, Some("first_output_timeout")),
|
||||
sent_error(
|
||||
ProviderErrorKind::Protocol,
|
||||
Some("upstream_stream_truncated"),
|
||||
),
|
||||
sent_error(
|
||||
ProviderErrorKind::Unavailable,
|
||||
Some("upstream_empty_response"),
|
||||
),
|
||||
sent_error(ProviderErrorKind::Unavailable, Some("new_unknown_reason")),
|
||||
sent_error(ProviderErrorKind::Transport, Some("new_unknown_reason")),
|
||||
sent_error(ProviderErrorKind::Transport, Some("")),
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_status(503),
|
||||
] {
|
||||
assert!(
|
||||
!openai_failure_affects_account_score(&error),
|
||||
"non-allowlisted failure affected the account score"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_score_failure_filter_should_reject_internal_kinds_without_a_reason() {
|
||||
for kind in [
|
||||
ProviderErrorKind::Transport,
|
||||
ProviderErrorKind::Timeout,
|
||||
ProviderErrorKind::Protocol,
|
||||
] {
|
||||
let error = sent_error(kind, None);
|
||||
assert!(
|
||||
!openai_failure_affects_account_score(&error),
|
||||
"unlisted internal failure affected the score: {}",
|
||||
kind.as_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let account = ProviderAccountId::new("acct_usable_account").expect("account");
|
||||
|
||||
for _ in 0..100 {
|
||||
deliver_error_with_openai_feedback(
|
||||
&feedback,
|
||||
&provider,
|
||||
&account,
|
||||
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
feedback.scheduling_signals(&provider, &account),
|
||||
(None, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_feedback_should_score_server_overload_as_one_regular_failure() {
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let account = ProviderAccountId::new("acct_overloaded").expect("account");
|
||||
|
||||
deliver_error_with_openai_feedback(
|
||||
&feedback,
|
||||
&provider,
|
||||
&account,
|
||||
sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
feedback.scheduling_signals(&provider, &account).0,
|
||||
Some(2_000)
|
||||
);
|
||||
}
|
||||
@@ -1 +1,2 @@
|
||||
mod contract;
|
||||
mod failure;
|
||||
|
||||
@@ -837,6 +837,40 @@ async fn missing_exact_socket_should_fail_without_opening_a_connection() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_local_continuation_without_a_pool_should_not_open_http_or_websocket() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
);
|
||||
let mut request = new_chain_request("conversation-exact-no-pool");
|
||||
request.set_previous_response_id(Some("resp_no_pool".to_owned()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
|
||||
let error = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_exact_no_pool", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect_err("connection-local continuation requires its owning pool");
|
||||
|
||||
std::assert_matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
reason: PreviousResponseUnavailableReason::PoolUnavailable
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
timeout(Duration::from_millis(50), listener.accept())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_local_continuation_should_fail_after_its_live_socket_disappears() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -852,7 +886,7 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
drop(websocket);
|
||||
closed_tx.send(()).unwrap();
|
||||
});
|
||||
let backend = CodexBackendClient::new(
|
||||
@@ -883,12 +917,18 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
|
||||
)
|
||||
.await
|
||||
.expect_err("a disappeared live socket cannot be reconstructed from the database");
|
||||
std::assert_matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
|
||||
reason: PreviousResponseUnavailableReason::FreshConnectionRequired
|
||||
})
|
||||
let CodexClientError::WebSocket(error) = error else {
|
||||
panic!("disappeared connection should remain a typed WebSocket error");
|
||||
};
|
||||
assert_eq!(
|
||||
error.continuation_unavailable_reason(),
|
||||
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
|
||||
);
|
||||
let observation = error
|
||||
.connection_observation()
|
||||
.expect("disappeared connection should retain its lifecycle observation");
|
||||
assert_eq!(observation.exit_reason(), "tcp_reset");
|
||||
assert!(observation.age_ms() >= observation.idle_ms());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1114,9 +1154,9 @@ async fn websocket_failure_after_first_delivery_should_not_open_http_fallback()
|
||||
else {
|
||||
panic!("expected typed post-delivery WebSocket failure");
|
||||
};
|
||||
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = *source else {
|
||||
panic!("expected upstream close frame source");
|
||||
};
|
||||
let close = source
|
||||
.close_before_terminal()
|
||||
.expect("expected upstream close frame source");
|
||||
assert!(
|
||||
close.connection_id().is_some(),
|
||||
"close failures must retain their WebSocket connection correlation"
|
||||
|
||||
@@ -273,6 +273,16 @@ fn transport_should_prefer_websocket_when_requested_without_history() {
|
||||
assert!(transport_requirement(&request).allows_pre_send_http_fallback());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unspecified_transport_without_history_should_still_be_a_new_chain() {
|
||||
let request = codex_request("gpt-test", "be brief", Vec::new());
|
||||
|
||||
assert_eq!(
|
||||
transport_requirement(&request),
|
||||
TransportRequirement::NewChain
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_chain_should_allow_pre_delivery_http_fallback() {
|
||||
let mut request = codex_request("gpt-test", "be brief", Vec::new());
|
||||
@@ -284,6 +294,25 @@ fn new_chain_should_allow_pre_delivery_http_fallback() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sticky_http_fallback_should_only_apply_to_new_chains() {
|
||||
assert!(TransportRequirement::NewChain.allows_sticky_http_fallback());
|
||||
|
||||
for requirement in [
|
||||
TransportRequirement::HttpRequired,
|
||||
TransportRequirement::ExplicitWebSocketWarmup,
|
||||
TransportRequirement::ExactWebSocketContinuation,
|
||||
TransportRequirement::PersistedContinuation,
|
||||
TransportRequirement::ExternalUnknown,
|
||||
] {
|
||||
assert!(
|
||||
!requirement.allows_sticky_http_fallback(),
|
||||
"sticky HTTP must not override {}",
|
||||
requirement.as_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_should_mark_unknown_previous_response_as_external() {
|
||||
let mut request = codex_request("gpt-test", "be brief", Vec::new());
|
||||
|
||||
@@ -647,7 +647,7 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
|
||||
.send(Message::Binary(b"unexpected-binary".to_vec().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
let _client_close = websocket.next().await;
|
||||
});
|
||||
let mut request = codex_request("gpt-test", "be brief", Vec::new());
|
||||
request.set_previous_response_id(Some("resp_binary_previous".to_owned()));
|
||||
@@ -664,15 +664,30 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
|
||||
.expect_err("binary websocket events should be rejected");
|
||||
server.await.unwrap();
|
||||
|
||||
std::assert_matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
message,
|
||||
source: Some(source),
|
||||
})
|
||||
if message.contains("unexpected binary websocket event")
|
||||
&& matches!(*source, CodexWebSocketExchangeError::UnexpectedBinaryEvent)
|
||||
let CodexClientError::WebSocket(error) = error else {
|
||||
panic!("binary websocket event should remain a typed WebSocket error");
|
||||
};
|
||||
let CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
message,
|
||||
source: Some(source),
|
||||
} = error
|
||||
else {
|
||||
panic!("binary websocket event should remain post-send ambiguous");
|
||||
};
|
||||
assert!(message.contains("unexpected binary websocket event"));
|
||||
assert_eq!(
|
||||
source
|
||||
.connection_observation()
|
||||
.map(|observation| observation.exit_reason()),
|
||||
Some("unexpected_binary_event")
|
||||
);
|
||||
let CodexWebSocketExchangeError::ConnectionObserved { source, .. } = *source else {
|
||||
panic!("binary websocket event should carry its connection observation");
|
||||
};
|
||||
assert!(matches!(
|
||||
*source,
|
||||
CodexWebSocketExchangeError::UnexpectedBinaryEvent
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1560,9 +1575,9 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
else {
|
||||
panic!("close-before-terminal should preserve its typed source: {error:?}");
|
||||
};
|
||||
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = source.as_ref() else {
|
||||
panic!("post-send ambiguity should wrap a close-before-terminal error: {source:?}");
|
||||
};
|
||||
let close = source
|
||||
.close_before_terminal()
|
||||
.unwrap_or_else(|| panic!("post-send ambiguity should retain its close: {source:?}"));
|
||||
assert_eq!(close.last_event_type(), Some("response.output_text.delta"));
|
||||
|
||||
std::assert_matches!(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::*;
|
||||
use provider_openai::transport::websocket::PreviousResponseUnavailableReason;
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and_conversation() {
|
||||
@@ -1108,6 +1109,30 @@ async fn websocket_pool_should_replace_idle_connection_after_pong_deadline() {
|
||||
.await
|
||||
.expect("server should report the Pong-timeout close");
|
||||
tokio::time::resume();
|
||||
let mut continuation = request.clone();
|
||||
continuation.set_previous_response_id(Some("resp_no_pong_first".to_owned()));
|
||||
continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let error = backend
|
||||
.create_response(
|
||||
&continuation,
|
||||
request_context("req_no_pong_continuation", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect_err("Pong-timeout connection cannot satisfy an exact continuation");
|
||||
let CodexClientError::WebSocket(error) = error else {
|
||||
panic!("Pong-timeout continuation should remain a typed WebSocket error");
|
||||
};
|
||||
assert_eq!(
|
||||
error.continuation_unavailable_reason(),
|
||||
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
|
||||
);
|
||||
assert_eq!(
|
||||
error
|
||||
.connection_observation()
|
||||
.expect("Pong-timeout tombstone should retain its lifecycle observation")
|
||||
.exit_reason(),
|
||||
"pong_timeout"
|
||||
);
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
|
||||
@@ -138,7 +138,11 @@ impl Provider for GrokBuildProvider {
|
||||
self.catalog.catalog_generation()
|
||||
}
|
||||
|
||||
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
|
||||
fn request_observation(
|
||||
&self,
|
||||
operation: &Operation,
|
||||
_client_api_key_id: &gateway_core::policy::ClientApiKeyId,
|
||||
) -> ProviderRequestObservation {
|
||||
let Operation::Generate(request) = operation else {
|
||||
return ProviderRequestObservation::default();
|
||||
};
|
||||
@@ -153,6 +157,7 @@ impl Provider for GrokBuildProvider {
|
||||
request_kind: semantics.request_kind,
|
||||
subagent_kind: semantics.subagent_kind,
|
||||
compact: semantics.compact,
|
||||
continuation: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1063,8 +1063,12 @@ async fn model_alias_should_be_canonical_before_account_selection() {
|
||||
async fn request_observation_preserves_raw_xai_reasoning_effort() {
|
||||
let provider = provider(StubSelector::success(), StubInferenceTransport::success()).await;
|
||||
|
||||
let observation =
|
||||
provider.request_observation(&operation_with_reasoning_effort("future-value"));
|
||||
let client_key_id =
|
||||
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
|
||||
let observation = provider.request_observation(
|
||||
&operation_with_reasoning_effort("future-value"),
|
||||
&client_key_id,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
observation.reasoning_effort.as_deref(),
|
||||
@@ -1093,7 +1097,9 @@ async fn request_observation_preserves_codex_semantics_for_xai() {
|
||||
)]));
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
|
||||
|
||||
let observation = provider.request_observation(&operation);
|
||||
let client_key_id =
|
||||
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
|
||||
let observation = provider.request_observation(&operation, &client_key_id);
|
||||
|
||||
assert_eq!(observation.reasoning_effort.as_deref(), Some("xhigh"));
|
||||
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
|
||||
|
||||
@@ -2,3 +2,4 @@
|
||||
93219760c733616e7bba9a8933c207b26752b2ebf7e6edfe1ef42ba227c50a7d 0002_runtime_pressure_observability.sql
|
||||
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
|
||||
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
|
||||
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
alter table model_requests
|
||||
add column continuation_affinity_hash text,
|
||||
add column continuation_previous_response_id_hash text,
|
||||
add column continuation_requested boolean not null default false,
|
||||
add column continuation_unavailable_reason text,
|
||||
add column upstream_connection_id text,
|
||||
add column upstream_connection_exit_reason text,
|
||||
add column upstream_connection_age_ms bigint,
|
||||
add column upstream_connection_idle_ms bigint,
|
||||
add column recovery_request_id text,
|
||||
add column recovered_at timestamptz,
|
||||
add column recovery_attempt_count integer not null default 0,
|
||||
add column recovery_retry_delay_ms bigint,
|
||||
add column recovery_total_latency_ms bigint,
|
||||
add constraint model_requests_continuation_hashes_ck check (
|
||||
(continuation_affinity_hash is null
|
||||
or continuation_affinity_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_previous_response_id_hash is null
|
||||
or continuation_previous_response_id_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_requested
|
||||
or continuation_previous_response_id_hash is null)
|
||||
),
|
||||
add constraint model_requests_continuation_reason_ck check (
|
||||
continuation_unavailable_reason is null
|
||||
or (
|
||||
octet_length(continuation_unavailable_reason) between 1 and 64
|
||||
and continuation_unavailable_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
)
|
||||
),
|
||||
add constraint model_requests_upstream_connection_ck check (
|
||||
(
|
||||
upstream_connection_id is null
|
||||
and upstream_connection_exit_reason is null
|
||||
and upstream_connection_age_ms is null
|
||||
and upstream_connection_idle_ms is null
|
||||
)
|
||||
or (
|
||||
octet_length(upstream_connection_id) between 1 and 128
|
||||
and upstream_connection_id !~ '[[:cntrl:]]'
|
||||
and octet_length(upstream_connection_exit_reason) between 1 and 64
|
||||
and upstream_connection_exit_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
and upstream_connection_age_ms >= 0
|
||||
and upstream_connection_idle_ms between 0 and upstream_connection_age_ms
|
||||
)
|
||||
),
|
||||
add constraint model_requests_recovery_ck check (
|
||||
recovery_attempt_count >= 0
|
||||
and (
|
||||
(
|
||||
recovered_at is null
|
||||
and recovery_request_id is null
|
||||
and recovery_retry_delay_ms is null
|
||||
and recovery_total_latency_ms is null
|
||||
)
|
||||
or (
|
||||
recovered_at is not null
|
||||
and recovered_at >= completed_at
|
||||
and recovery_request_id is not null
|
||||
and octet_length(recovery_request_id) between 1 and 128
|
||||
and recovery_attempt_count > 0
|
||||
and recovery_retry_delay_ms >= 0
|
||||
and recovery_total_latency_ms >= recovery_retry_delay_ms
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
create index model_requests_pending_continuation_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where error_kind = 'continuation_recovery_required'
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
create index model_requests_recovery_request_idx
|
||||
on model_requests (recovery_request_id)
|
||||
where recovery_request_id is not null;
|
||||
+11
-6
@@ -108,20 +108,22 @@ mutation 请求不要求客户端提供全局配置版本。会改变路由快
|
||||
|
||||
## 3. OpenAI 数据面与模型目录
|
||||
|
||||
Responses 和 Images HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;协议可接受性由上游决定。
|
||||
Responses、Images 和 standalone Search HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;
|
||||
协议可接受性由上游决定。
|
||||
|
||||
| 方法 | 路由 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `POST` | `/v1/responses` | OpenAI Responses JSON;`stream=true` 返回 SSE,否则返回完整 JSON |
|
||||
| `GET` | `/v1/responses` | 通过 HTTP Upgrade 建立 Responses WebSocket |
|
||||
| `POST` | `/v1/responses/review` | 使用同一 Responses 合同发起 review 子代理请求 |
|
||||
| `POST` | `/v1/alpha/search` | Codex standalone web search;JSON 请求与响应正文原样转发 |
|
||||
| `POST` | `/v1/images/generations` | 通过 OpenAI Provider 发起图像生成;JSON 请求与响应正文原样转发 |
|
||||
| `POST` | `/v1/images/edits` | 通过 OpenAI Provider 发起图像编辑;JSON 请求与响应正文原样转发 |
|
||||
| `GET` | `/v1/models` | 返回当前 Client Key 账号范围内各 Provider 的可用公开模型并集;有两种响应形态,见下 |
|
||||
| `GET` | `/v1/models/catalog` | 返回 Codex 客户端使用的模型目录 |
|
||||
| `GET` | `/v1/models/{model_id}/info` | 返回 Codex 客户端使用的单模型信息 |
|
||||
| `GET` | `/v1/models/{model_id}` | 返回 OpenAI 兼容的单模型详情 |
|
||||
|
||||
Codex 的 review 等子代理请求仍使用 `/v1/responses`,并通过 `x-openai-subagent` 请求头携带子代理类型;
|
||||
网关不提供独立的子代理请求路径。
|
||||
|
||||
`GET /v1/models` 默认返回 OpenAI 兼容列表 `{"object": "list", "data": [...]}`;请求携带非空
|
||||
`client_version` query 参数(Codex 客户端)时改为返回 Codex 专用目录合同 `{"models": [...]}`。
|
||||
|
||||
@@ -129,8 +131,11 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
|
||||
映射除外),HTTP SSE 与 WebSocket 的上游业务事件字节原样转发,response ID 按 opaque 值处理而不
|
||||
假设 UUID 或固定长度;OpenAI 上游错误 envelope 和允许下发的 opaque header 值也不由 canonical
|
||||
观测结果重写。Images 请求不读取或重建 JSON,也不要求或映射模型字段;它固定使用 OpenAI Provider,
|
||||
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。xAI 是
|
||||
Grok wire 与 Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
只在原始字节之外完成账号选择、鉴权头替换和端点路由,成功与失败响应正文同样保持原始字节。
|
||||
`/v1/alpha/search` 使用相同的 OpenAI Provider 原生端点边界:body(包括 `model`)不解析、不映射,
|
||||
`x-codex-turn-metadata` 在移除客户端账号身份并按当前 lease 重写 installation ID 后转发;上游账号
|
||||
Authorization、Cookie、account ID、originator 和 User-Agent 均由代理安全重建。xAI 是 Grok wire 与
|
||||
Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
上游结构化错误的 message/code/type 会透传给客户端,其中内嵌的账号指纹 UUID 已脱敏。模型映射是
|
||||
全局精确映射,未命中时模型名原样交给候选 Provider;分组只限定账号集合,不参与模型改名。
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
Codex Proxy RS 是单进程、单副本运行的多 Provider AI 网关,同时提供:
|
||||
|
||||
- 面向客户端的 OpenAI Responses、Images 和模型目录协议;
|
||||
- 面向客户端的 OpenAI Responses、Images、standalone Search 和模型目录协议;
|
||||
- 面向管理员的 `/api/admin/*` 控制面和 Vue 管理端;
|
||||
- OpenAI 与 xAI 两个编译期 Provider;
|
||||
- PostgreSQL 持久化、Redis 协调状态以及 S3/R2 数据库备份。
|
||||
@@ -61,7 +61,7 @@ flowchart LR
|
||||
| `gateway-api` | HTTP/WS/SSE 解码与交付、Admin wire、静态管理端;不直接访问 Store 或具体 Provider |
|
||||
| `gateway-store` | PostgreSQL、Redis、S3/R2、`pg_dump` 适配器;不拥有业务策略 |
|
||||
| `gateway-host` | 配置加载、日志、HTTP 生命周期、Worker 监督和系统更新 |
|
||||
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images transport |
|
||||
| `providers/openai` | OpenAI OAuth、账号选择、目录、额度、Responses/Images/Search transport |
|
||||
| `providers/xai` | xAI OAuth session、账号选择、目录、额度和 Grok/Responses 转换 |
|
||||
| `frontend` | Vue 管理端,仅通过 Admin API 读写状态 |
|
||||
|
||||
@@ -151,16 +151,17 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别 Codex Desktop/
|
||||
- 可恢复观测写入失败不能替换已经确定的客户端协议结果。
|
||||
|
||||
Responses 按模型目录编译候选;全局模型映射是精确映射,未命中时模型名原样交给候选 Provider。
|
||||
Images 是 OpenAI Provider 自有端点:它不要求模型字段、不参与文本模型映射,只在 Client Key 的账号范围
|
||||
确实包含 OpenAI 账号时生成单一 OpenAI 候选。
|
||||
Images 与 standalone Search 是 OpenAI Provider 自有端点:两者都不参与文本模型映射,只在 Client Key
|
||||
的账号范围确实包含 OpenAI 账号时生成单一 OpenAI 候选。Images 不要求模型字段;Search body 中的模型
|
||||
及其他字段保持原始 bytes 并由上游解释。
|
||||
|
||||
## 5. Provider 与协议边界
|
||||
|
||||
Core 只理解 `Operation`、能力要求、Provider 候选、稳定错误和 canonical event,不读取 Provider SDK
|
||||
类型。Provider 独占 credential schema、OAuth、账号选择、模型目录、额度投影和上游 transport。
|
||||
|
||||
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket 与 Images 的业务正文
|
||||
按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
|
||||
- OpenAI 是透明边界。Responses 请求保留未知字段和字段顺序;SSE、WebSocket、Images 与 standalone
|
||||
Search 的业务正文按原始字节转发。canonical facts 从同一数据旁路提取,只用于路由、观测和计费。
|
||||
- xAI 是翻译边界。Provider 把 Grok wire 转换为 Responses wire;上游结构化错误的 message/code/type
|
||||
可以透出,但账号指纹会先脱敏。
|
||||
- response ID 是不透明 UTF-8 bytes,不假设 UUID、固定长度或跨 Provider 可复用。
|
||||
|
||||
@@ -204,6 +204,18 @@ export interface OpsErrorMetadata {
|
||||
component: string
|
||||
attemptId: string | null
|
||||
accountLabel: string | null
|
||||
continuationAffinityHash: string | null
|
||||
continuationPreviousResponseIdHash: string | null
|
||||
continuationUnavailableReason: string | null
|
||||
upstreamConnectionId: string | null
|
||||
upstreamConnectionExitReason: string | null
|
||||
upstreamConnectionAgeMs: number | null
|
||||
upstreamConnectionIdleMs: number | null
|
||||
recoveryRequestId: string | null
|
||||
recoveredAt: string | null
|
||||
recoveryAttemptCount: number
|
||||
recoveryRetryDelayMs: number | null
|
||||
recoveryTotalLatencyMs: number | null
|
||||
}
|
||||
|
||||
export interface OpsError {
|
||||
@@ -420,10 +432,7 @@ interface PageQuery {
|
||||
|
||||
type UsagePageQuery = UsageRangeQuery & PageQuery
|
||||
|
||||
type OpsErrorPageQuery = UsagePageQuery & {
|
||||
failureClass?: string
|
||||
route?: string
|
||||
}
|
||||
type OpsErrorPageQuery = UsagePageQuery
|
||||
|
||||
interface UsageDetailQuery {
|
||||
id: string
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { Key, Openai, Xai } from '@boxicons/vue'
|
||||
import { Key, LinkAlt, Openai, Xai } from '@boxicons/vue'
|
||||
import { computed } from 'vue'
|
||||
import { formatProviderLabel } from '@/utils/providers'
|
||||
|
||||
@@ -38,6 +38,8 @@ const providerLabel = computed(() => formatProviderLabel(props.provider, '未知
|
||||
const authenticationLabel = computed(() => {
|
||||
if (normalizedAuthenticationKind.value === 'oauth')
|
||||
return 'OAuth'
|
||||
if (normalizedAuthenticationKind.value === 'api_key')
|
||||
return 'API Key'
|
||||
return props.authenticationKind?.trim() || '未知认证类型'
|
||||
})
|
||||
</script>
|
||||
@@ -60,7 +62,8 @@ const authenticationLabel = computed(() => {
|
||||
:class="iconContainerClass"
|
||||
:title="authenticationLabel"
|
||||
>
|
||||
<Key v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
|
||||
<LinkAlt v-if="normalizedAuthenticationKind === 'oauth'" :class="iconClass" />
|
||||
<Key v-else-if="normalizedAuthenticationKind === 'api_key'" :class="iconClass" />
|
||||
<span v-else class="text-[10px] font-heavy text-cp-text-quaternary">?</span>
|
||||
</span>
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ const profileOpen = shallowRef(false)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid max-h-44 min-h-0 gap-3 overflow-y-auto pr-1">
|
||||
<div class="grid min-h-0 gap-3">
|
||||
<AccountQuotaPanelEntry
|
||||
v-for="entry in quotaEntries"
|
||||
:key="entry.key"
|
||||
|
||||
@@ -10,15 +10,19 @@ import AccountUsageWindow from '../AccountUsageWindow/index.vue'
|
||||
import { resolveAccountUsageWindowPresentation } from '../AccountUsageWindow/presenter'
|
||||
import AccountQuotaWindowGroup from './WindowGroup.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
const props = withDefaults(defineProps<{
|
||||
label: string | null
|
||||
windows: AccountQuotaWindow[]
|
||||
}>()
|
||||
showPercentage?: boolean
|
||||
}>(), {
|
||||
showPercentage: true,
|
||||
})
|
||||
|
||||
const now = useUiClock()
|
||||
const grouped = computed(() => props.windows.length > 1 && Boolean(props.label))
|
||||
const detailHeading = computed(() => props.label)
|
||||
const detailTitle = computed(() => detailHeading.value ?? props.windows[0]?.labelDisplay ?? '额度详情')
|
||||
const summaryLabel = computed(() => props.label ?? props.windows[0]?.labelDisplay ?? '额度')
|
||||
const hasQuotaWindow = computed(() => props.windows.some(window => typeof window.usedPercent === 'number'))
|
||||
const detailItems = computed(() =>
|
||||
props.windows.map((window) => {
|
||||
const view = resolveAccountUsageWindowPresentation({
|
||||
@@ -82,7 +86,12 @@ function quotaWindowCode(windowSeconds: number | null, role: AccountQuotaWindow[
|
||||
:aria-expanded="open"
|
||||
aria-haspopup="dialog"
|
||||
>
|
||||
<AccountQuotaWindowGroup v-if="grouped && label" :label="label" :windows="windows" />
|
||||
<AccountQuotaWindowGroup
|
||||
v-if="hasQuotaWindow"
|
||||
:label="summaryLabel"
|
||||
:windows="windows"
|
||||
:show-percentage="showPercentage"
|
||||
/>
|
||||
<AccountUsageWindow
|
||||
v-else
|
||||
:window="windows[0]"
|
||||
|
||||
@@ -4,10 +4,13 @@ import type { AccountQuotaWindow } from '../../constants'
|
||||
import { computed } from 'vue'
|
||||
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
|
||||
|
||||
const props = defineProps<{
|
||||
const props = withDefaults(defineProps<{
|
||||
label: string
|
||||
windows: AccountQuotaWindow[]
|
||||
}>()
|
||||
showPercentage?: boolean
|
||||
}>(), {
|
||||
showPercentage: true,
|
||||
})
|
||||
|
||||
const windowItems = computed(() => props.windows.map(window => ({
|
||||
key: window.key,
|
||||
@@ -16,6 +19,17 @@ const windowItems = computed(() => props.windows.map(window => ({
|
||||
usedPercentDisplay: window.usedPercentDisplay,
|
||||
presentation: quotaWindowPresentation(window, '2px'),
|
||||
})))
|
||||
const highestUsageWindow = computed(() => props.windows.reduce((highest, window) => {
|
||||
if (typeof window.usedPercent !== 'number')
|
||||
return highest
|
||||
if (!highest || typeof highest.usedPercent !== 'number' || window.usedPercent > highest.usedPercent)
|
||||
return window
|
||||
return highest
|
||||
}, undefined as AccountQuotaWindow | undefined))
|
||||
const highestUsageDisplay = computed(() => highestUsageWindow.value?.usedPercentDisplay ?? '—')
|
||||
const highestUsageTextClass = computed(() => highestUsageWindow.value
|
||||
? quotaWindowPresentation(highestUsageWindow.value, '2px').percentTextClass
|
||||
: 'text-cp-text-quaternary')
|
||||
|
||||
const trackGridStyle = computed(() => ({
|
||||
gridTemplateColumns: `repeat(${Math.max(windowItems.value.length, 1)}, minmax(0, 1fr))`,
|
||||
@@ -23,16 +37,25 @@ const trackGridStyle = computed(() => ({
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex min-w-0 flex-col">
|
||||
<div class="mb-1 min-w-0 truncate text-cp-xs leading-3.5 font-bold text-cp-text-quaternary">
|
||||
{{ label }}
|
||||
<div class="grid min-w-0 gap-1.5">
|
||||
<div class="flex min-w-0 items-baseline justify-between gap-1 text-[10px] leading-3 font-bold">
|
||||
<span class="min-w-0 truncate text-cp-text-quaternary" :title="label">
|
||||
{{ label }}
|
||||
</span>
|
||||
<strong
|
||||
v-if="showPercentage"
|
||||
class="shrink-0 font-mono font-heavy tabular-nums"
|
||||
:class="highestUsageTextClass"
|
||||
>
|
||||
{{ highestUsageDisplay }}
|
||||
</strong>
|
||||
</div>
|
||||
|
||||
<div class="grid min-w-0 gap-1" :style="trackGridStyle">
|
||||
<div
|
||||
v-for="item in windowItems"
|
||||
:key="item.key"
|
||||
class="h-0.75 w-full overflow-hidden rounded-full bg-cp-border-secondary"
|
||||
class="h-1 w-full overflow-hidden rounded-full bg-cp-border-secondary"
|
||||
role="progressbar"
|
||||
:aria-label="item.labelTooltip"
|
||||
aria-valuemin="0"
|
||||
@@ -41,7 +64,7 @@ const trackGridStyle = computed(() => ({
|
||||
:aria-valuetext="item.usedPercentDisplay"
|
||||
>
|
||||
<div
|
||||
class="h-full rounded-full transition-[width,background-color] duration-200"
|
||||
class="h-full rounded-full transition-[width,background-color] duration-200 motion-reduce:transition-none"
|
||||
:class="item.presentation.barClass"
|
||||
:style="item.presentation.barStyle"
|
||||
/>
|
||||
|
||||
@@ -4,7 +4,9 @@ import type { AccountRow } from '../../constants'
|
||||
import { computed } from 'vue'
|
||||
import { groupedAccountQuotaWindows, visibleSummaryQuotaWindows } from '../../constants'
|
||||
import AccountUsageWindow from '../AccountUsageWindow/index.vue'
|
||||
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
|
||||
import AccountQuotaSummaryEntry from './Entry.vue'
|
||||
import { recentlyUsedQuotaEntry, representativeQuotaWindow } from './presenter'
|
||||
|
||||
const props = defineProps<{
|
||||
account: AccountRow
|
||||
@@ -13,24 +15,65 @@ const props = defineProps<{
|
||||
const quotaWindows = computed(() => props.account.quota.windows)
|
||||
const visibleQuotaWindows = computed(() => visibleSummaryQuotaWindows(quotaWindows.value))
|
||||
const summaryEntries = computed(() => groupedAccountQuotaWindows(visibleQuotaWindows.value))
|
||||
const summaryClass = computed(
|
||||
() =>
|
||||
`box-border grid w-full min-w-0 gap-1 py-1.5 ${
|
||||
summaryEntries.value.length <= 1
|
||||
? 'min-h-13 content-center'
|
||||
: 'min-h-16.5 content-center'
|
||||
}`,
|
||||
)
|
||||
const hasUsage = computed(() => (props.account.usage.requestCount ?? 0) > 0)
|
||||
const recentUsageEntry = computed(() => recentlyUsedQuotaEntry(
|
||||
summaryEntries.value,
|
||||
props.account.usage.models,
|
||||
))
|
||||
const currentUsageWindow = computed(() => representativeQuotaWindow(recentUsageEntry.value))
|
||||
const currentUsageDisplay = computed(() => currentUsageWindow.value?.usedPercentDisplay ?? '—')
|
||||
const currentUsageTextClass = computed(() => currentUsageWindow.value
|
||||
? quotaWindowPresentation(currentUsageWindow.value, '2px').percentTextClass
|
||||
: 'text-cp-text-quaternary')
|
||||
const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length - 1, 0))
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div :class="summaryClass">
|
||||
<AccountQuotaSummaryEntry
|
||||
v-for="entry in summaryEntries"
|
||||
:key="entry.key"
|
||||
:label="entry.label"
|
||||
:windows="entry.windows"
|
||||
/>
|
||||
<AccountUsageWindow v-if="summaryEntries.length === 0" variant="compact" />
|
||||
<div class="box-border grid min-h-16.5 w-full min-w-0 content-center gap-1.5 py-1.5">
|
||||
<template v-if="summaryEntries.length > 0">
|
||||
<div
|
||||
v-if="hasUsage"
|
||||
class="flex min-w-0 items-baseline justify-between gap-2 leading-none"
|
||||
>
|
||||
<span
|
||||
class="flex min-w-0 items-baseline gap-1 font-mono tabular-nums"
|
||||
title="当前额度窗口总 Token"
|
||||
>
|
||||
<strong class="truncate text-cp-xs font-heavy text-cp-text">
|
||||
{{ account.usage.totalTokensDisplay }}
|
||||
</strong>
|
||||
<span class="shrink-0 text-[9px] font-emphasis tracking-[0.02em] text-cp-text-quaternary">
|
||||
Tokens
|
||||
</span>
|
||||
</span>
|
||||
<span
|
||||
class="flex shrink-0 items-baseline gap-1 text-[9px] font-emphasis text-cp-text-quaternary"
|
||||
title="最近使用额度的当前已用比例"
|
||||
>
|
||||
<span>消耗比例</span>
|
||||
<strong class="font-mono font-heavy tabular-nums" :class="currentUsageTextClass">
|
||||
{{ currentUsageDisplay }}
|
||||
</strong>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div v-if="recentUsageEntry" class="flex min-w-0 items-end gap-2">
|
||||
<div class="min-w-0 flex-1">
|
||||
<AccountQuotaSummaryEntry
|
||||
:label="recentUsageEntry.label"
|
||||
:windows="recentUsageEntry.windows"
|
||||
:show-percentage="false"
|
||||
/>
|
||||
</div>
|
||||
<span
|
||||
v-if="additionalEntryCount > 0"
|
||||
class="grid h-5 min-w-5 shrink-0 place-items-center rounded-cp bg-cp-fill-quaternary px-1.5 font-mono text-[9px] font-heavy tabular-nums text-cp-text-tertiary"
|
||||
:title="`另有 ${additionalEntryCount} 个额度组,可展开账号查看`"
|
||||
>
|
||||
+{{ additionalEntryCount }}
|
||||
</span>
|
||||
</div>
|
||||
</template>
|
||||
<AccountUsageWindow v-else variant="compact" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { AccountQuotaWindow, AccountQuotaWindowEntry, AccountRow } from '../../constants'
|
||||
|
||||
type AccountModelUsage = AccountRow['usage']['models'][number]
|
||||
|
||||
/**
|
||||
* 根据账号最近一次模型请求选择对应额度组。
|
||||
* 没有专属额度的模型统一回退到 Codex 通用额度。
|
||||
*/
|
||||
export function recentlyUsedQuotaEntry(
|
||||
entries: readonly AccountQuotaWindowEntry[],
|
||||
models: readonly AccountModelUsage[],
|
||||
) {
|
||||
const recentModel = latestModelUsage(models)
|
||||
if (recentModel) {
|
||||
const modelKey = quotaIdentity(recentModel.model)
|
||||
const modelEntry = entries.find(entry => quotaEntryIdentities(entry).has(modelKey))
|
||||
if (modelEntry)
|
||||
return modelEntry
|
||||
}
|
||||
|
||||
return entries.find(entry => entry.windows.some(window => window.limitId === 'codex'))
|
||||
?? entries[0]
|
||||
}
|
||||
|
||||
/** 一个额度组可能包含多个滚动窗口,摘要使用其中当前占用最高的窗口。 */
|
||||
export function representativeQuotaWindow(entry: AccountQuotaWindowEntry | undefined) {
|
||||
return entry?.windows.reduce<AccountQuotaWindow | undefined>((representative, window) => {
|
||||
if (typeof window.usedPercent !== 'number')
|
||||
return representative
|
||||
if (
|
||||
!representative
|
||||
|| typeof representative.usedPercent !== 'number'
|
||||
|| window.usedPercent > representative.usedPercent
|
||||
) {
|
||||
return window
|
||||
}
|
||||
return representative
|
||||
}, undefined)
|
||||
}
|
||||
|
||||
function latestModelUsage(models: readonly AccountModelUsage[]) {
|
||||
let latest: AccountModelUsage | undefined
|
||||
let latestTimestamp = Number.NEGATIVE_INFINITY
|
||||
|
||||
for (const model of models) {
|
||||
const timestamp = Date.parse(model.lastUsedAt)
|
||||
if (!Number.isFinite(timestamp) || timestamp <= latestTimestamp)
|
||||
continue
|
||||
latest = model
|
||||
latestTimestamp = timestamp
|
||||
}
|
||||
|
||||
return latest
|
||||
}
|
||||
|
||||
function quotaEntryIdentities(entry: AccountQuotaWindowEntry) {
|
||||
const identities = new Set<string>()
|
||||
addQuotaIdentity(identities, entry.label)
|
||||
for (const window of entry.windows) {
|
||||
addQuotaIdentity(identities, window.limitId)
|
||||
addQuotaIdentity(identities, window.limitName)
|
||||
}
|
||||
return identities
|
||||
}
|
||||
|
||||
function addQuotaIdentity(identities: Set<string>, value: string | null) {
|
||||
const identity = quotaIdentity(value)
|
||||
if (identity)
|
||||
identities.add(identity)
|
||||
}
|
||||
|
||||
function quotaIdentity(value: string | null) {
|
||||
return value?.trim().toLowerCase().replace(/[^a-z0-9]+/g, '') ?? ''
|
||||
}
|
||||
@@ -31,15 +31,15 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:grid-cols-[0.52fr_1.48fr]">
|
||||
<div>
|
||||
<div class="mb-3 flex items-baseline justify-between gap-3">
|
||||
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:min-h-0 xl:grid-cols-[0.52fr_1.48fr]">
|
||||
<div class="xl:flex xl:min-h-0 xl:flex-col">
|
||||
<div class="mb-3 flex shrink-0 items-baseline justify-between gap-3">
|
||||
<h3 class="m-0 text-cp-lg font-heavy text-cp-text">
|
||||
Token 结构
|
||||
</h3>
|
||||
<span class="text-cp-xs font-emphasis text-cp-text-quaternary">当前额度窗口</span>
|
||||
</div>
|
||||
<div class="grid gap-2">
|
||||
<div class="grid gap-2 xl:min-h-0 xl:flex-1 xl:grid-rows-5">
|
||||
<div class="flex items-center justify-between rounded-lg bg-cp-green-bg px-3 py-2">
|
||||
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入 Tokens</span>
|
||||
<strong class="font-mono text-cp text-cp-text">
|
||||
@@ -73,8 +73,8 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="min-w-0 pt-4 xl:pt-0 xl:pl-4">
|
||||
<div class="mb-3 flex flex-wrap items-center gap-x-4 gap-y-2">
|
||||
<div class="min-w-0 pt-4 xl:flex xl:min-h-0 xl:flex-col xl:pt-0 xl:pl-4">
|
||||
<div class="mb-3 flex shrink-0 flex-wrap items-center gap-x-4 gap-y-2">
|
||||
<h3 class="m-0 shrink-0 text-cp-lg font-heavy text-cp-text">
|
||||
模型使用排行
|
||||
</h3>
|
||||
@@ -100,7 +100,7 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="h-52 min-w-0">
|
||||
<div class="h-56 min-w-0 xl:h-auto xl:min-h-0 xl:flex-1 xl:basis-0">
|
||||
<BaseTable
|
||||
:columns="modelUsageColumns"
|
||||
:rows="account.usage.models"
|
||||
|
||||
@@ -287,7 +287,7 @@ const {
|
||||
</template>
|
||||
|
||||
<template #expanded="{ row }">
|
||||
<div class="grid gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr]">
|
||||
<div class="grid items-stretch gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr] xl:min-h-[19.25rem]">
|
||||
<AccountQuotaPanel
|
||||
:account="row"
|
||||
:refreshing="refreshingQuotaAccountIds.has(row.id)"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user