Compare commits

..
7 Commits
Author SHA1 Message Date
ZyyandCodex bd223f6540 chore: bump version to 3.3.16
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 22:45:09 +08:00
ZyyandCodex 168be0ca68 fix(openai): add continuation recovery observability
Persist stable continuation and WebSocket lifecycle facts, correlate successful client replays, and hide recovered failures from default business metrics.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 21:38:48 +08:00
ZyyandCodex f2e91bbdaa fix(openai): preserve websocket continuation routing
Limit session-sticky HTTP fallback to new chains, keep continuation requests on their safe transport, and classify rebuild-required failures separately from account health.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 18:21:30 +08:00
ZyyandCodex d0c9248478 fix(openai): refine smart scheduler failure scoring
Limit account feedback to an explicit upstream reason allowlist, aggregate same-account retries, and decay failure rates over time.

Co-authored-by: Codex <noreply@openai.com>
2026-09-02 15:08:31 +08:00
ZyyandCodex 22c697bbff fix(usage): refine record details and error filters
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 14:43:44 +08:00
ZyyandCodex 08d518324a fix(frontend): refine account quota layout
Co-authored-by: Codex <noreply@openai.com>
2026-09-02 12:14:03 +08:00
ZyyandCodex dfacb20e43 fix(frontend): tighten usage attempt table layout
Co-authored-by: Codex <noreply@openai.com>
2026-09-01 19:46:49 +08:00
78 changed files with 2957 additions and 683 deletions
@@ -135,13 +135,11 @@ pub struct OpsErrorFilter {
pub provider_kind: Option<String>,
pub provider_account_ref: Option<String>,
pub operation: Option<String>,
pub endpoint: Option<String>,
pub model: Option<String>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub failure_kind: Option<String>,
pub status_code: Option<u16>,
pub search: Option<String>,
}
@@ -793,6 +791,18 @@ pub struct OpsError {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: Option<bool>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
pub occurred_at: DateTime<Utc>,
pub stable_sort_id: String,
}
@@ -578,6 +578,18 @@ pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
component: error.component,
attempt_id: None,
account_label,
continuation_affinity_hash: error.continuation_affinity_hash,
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
continuation_unavailable_reason: error.continuation_unavailable_reason,
upstream_connection_id: error.upstream_connection_id,
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
upstream_connection_age_ms: error.upstream_connection_age_ms,
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
recovery_request_id: error.recovery_request_id,
recovered_at: error.recovered_at,
recovery_attempt_count: error.recovery_attempt_count,
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
recovery_total_latency_ms: error.recovery_total_latency_ms,
},
created_at: error.occurred_at,
created_at_display: china_datetime(&error.occurred_at),
@@ -107,14 +107,12 @@ pub struct OpsQuery {
pub provider: Option<String>,
pub request_id: Option<String>,
pub account_id: Option<String>,
pub route: Option<String>,
pub model: Option<String>,
pub status_code: Option<i64>,
pub client_status_code: Option<i64>,
pub upstream_status_code: Option<i64>,
pub transport: Option<String>,
pub attempt_index: Option<i64>,
pub failure_class: Option<String>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub search: Option<String>,
@@ -352,13 +350,11 @@ pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, Wir
provider_kind: non_empty(query.provider.clone()),
provider_account_ref: non_empty(query.account_id.clone()),
operation: non_empty(query.kind.clone()),
endpoint: non_empty(query.route.clone()),
model: non_empty(query.model.clone()),
transport: non_empty(query.transport.clone()),
attempt_index: parse_attempt_index(query.attempt_index)?,
response_id: non_empty(query.response_id.clone()),
upstream_request_id: non_empty(query.upstream_request_id.clone()),
failure_kind: non_empty(query.failure_class.clone()),
status_code,
search: non_empty(query.search.clone()),
},
@@ -805,4 +805,16 @@ pub struct OpsErrorMetadataView {
pub component: String,
pub attempt_id: Option<String>,
pub account_label: Option<String>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
}
@@ -49,6 +49,12 @@ fn ops_query_should_reject_page_size_above_terminal_limit() {
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
}
#[test]
fn ops_query_should_reject_removed_filters() {
assert!(serde_json::from_value::<OpsQuery>(json!({"route": "/v1/responses"})).is_err());
assert!(serde_json::from_value::<OpsQuery>(json!({"failureClass": "rate_limited"})).is_err());
}
#[test]
fn diagnostics_query_should_keep_wire_dimension_name() {
let query: DiagnosticsQuery =
@@ -571,6 +571,18 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
request_kind: Some("root".to_owned()),
subagent_kind: None,
compact: Some(false),
continuation_affinity_hash: Some("affinity-hash".to_owned()),
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
upstream_connection_id: Some("connection-id".to_owned()),
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
upstream_connection_age_ms: Some(12_000),
upstream_connection_idle_ms: Some(4_000),
recovery_request_id: None,
recovered_at: None,
recovery_attempt_count: 0,
recovery_retry_delay_ms: None,
recovery_total_latency_ms: None,
occurred_at: Utc::now(),
stable_sort_id: "model_request:req_err".to_owned(),
});
@@ -607,6 +619,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
"rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"],
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
}),
serde_json::json!({
"provider": "openai",
@@ -623,6 +638,9 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"providerErrorCode": "upstream",
"upstreamSendState": "sent",
"rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
"continuationUnavailableReason": "reused_connection_lost",
"upstreamConnectionExitReason": "tcp_reset",
"upstreamConnectionAgeMs": 12000,
})
);
}
@@ -398,3 +398,38 @@ async fn gateway_error_response_should_expose_only_structured_client_visible_ups
})
);
}
#[tokio::test]
async fn continuation_recovery_should_preserve_the_official_retry_signal() {
let error = GatewayError::from_provider(
&ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"Previous response was not found. Retrying the full request.",
Some("previous_response_not_found".to_owned()),
Some("invalid_request_error".to_owned()),
)),
);
let response = gateway_error_response(&error);
let status = response.status();
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("read continuation recovery response");
let body: serde_json::Value =
serde_json::from_slice(&body).expect("continuation recovery JSON");
assert_eq!(status, StatusCode::BAD_REQUEST);
assert_eq!(
body,
serde_json::json!({
"error": {
"message": "Previous response was not found. Retrying the full request.",
"type": "invalid_request_error",
"code": "previous_response_not_found"
}
})
);
}
@@ -3,9 +3,9 @@
use std::cmp::Reverse;
use std::collections::{BTreeSet, HashMap};
use std::num::NonZeroU32;
use std::sync::RwLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime};
use std::sync::{Mutex, RwLock};
use std::time::{Duration, Instant, SystemTime};
use crate::routing::ProviderKind;
@@ -94,6 +94,7 @@ pub struct AccountRuntimeSignals {
}
const ACCOUNT_FEEDBACK_EWMA_ALPHA: f64 = 0.2;
const ACCOUNT_FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
const EMPTY_FEEDBACK_SAMPLE: u64 = f64::NAN.to_bits();
/// 一次真实上游 attempt 对账号级 Smart 调度产生的中立反馈。
@@ -111,37 +112,69 @@ struct AccountFeedbackKey {
#[derive(Debug)]
struct AccountFeedback {
failure_rate: AtomicU64,
failure_rate: Mutex<DecayingAccountFailureRate>,
first_output_ms: AtomicU64,
}
#[derive(Debug, Default)]
struct DecayingAccountFailureRate {
value: f64,
updated_at: Option<Instant>,
}
impl DecayingAccountFailureRate {
fn value_at(&self, now: Instant) -> f64 {
let Some(updated_at) = self.updated_at else {
return self.value;
};
let elapsed = now.checked_duration_since(updated_at).unwrap_or_default();
self.value
* 0.5_f64.powf(elapsed.as_secs_f64() / ACCOUNT_FAILURE_RATE_HALF_LIFE.as_secs_f64())
}
fn report_at(&mut self, sample: f64, now: Instant) {
let now = self
.updated_at
.map_or(now, |updated_at| updated_at.max(now));
let decayed = self.value_at(now);
self.value =
ACCOUNT_FEEDBACK_EWMA_ALPHA * sample + (1.0 - ACCOUNT_FEEDBACK_EWMA_ALPHA) * decayed;
self.updated_at = Some(now);
}
}
impl Default for AccountFeedback {
fn default() -> Self {
Self {
// 新账号从健康基线开始,首个失败样本按 EWMA 平滑,而不是直接判死。
failure_rate: AtomicU64::new(0.0_f64.to_bits()),
failure_rate: Mutex::new(DecayingAccountFailureRate::default()),
first_output_ms: AtomicU64::new(EMPTY_FEEDBACK_SAMPLE),
}
}
}
impl AccountFeedback {
fn report(&self, feedback: AccountAttemptFeedback) {
fn report_at(&self, feedback: AccountAttemptFeedback, now: Instant) {
let (failure, first_output_ms) = match feedback {
AccountAttemptFeedback::Succeeded { first_output_ms } => (0.0, first_output_ms),
AccountAttemptFeedback::Failed { first_output_ms } => (1.0, first_output_ms),
};
update_feedback_ewma(&self.failure_rate, failure);
self.failure_rate
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.report_at(failure, now);
if let Some(first_output_ms) = first_output_ms.filter(|value| *value > 0) {
update_feedback_ewma(&self.first_output_ms, first_output_ms as f64);
}
}
fn scheduling_signals(&self) -> (Option<u16>, Option<u64>) {
let failure_rate = load_feedback_ewma(&self.failure_rate).map(|value| {
let basis_points = (value.clamp(0.0, 1.0) * 10_000.0).round();
basis_points as u16
});
fn scheduling_signals_at(&self, now: Instant) -> (Option<u16>, Option<u64>) {
let failure_rate = self
.failure_rate
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.value_at(now);
let failure_rate = Some((failure_rate.clamp(0.0, 1.0) * 10_000.0).round() as u16);
let first_output_ms = load_feedback_ewma(&self.first_output_ms)
.filter(|value| value.is_finite() && *value > 0.0)
.map(|value| value.round() as u64);
@@ -162,6 +195,18 @@ impl AccountFeedbackStats {
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
) -> (Option<u16>, Option<u64>) {
self.scheduling_signals_at(provider_kind, account_id, Instant::now())
}
/// 读取账号在指定单调时刻的错误率与首个有效输出延迟 EWMA。
#[doc(hidden)]
#[must_use]
pub fn scheduling_signals_at(
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
now: Instant,
) -> (Option<u16>, Option<u64>) {
let key = AccountFeedbackKey {
provider_kind: provider_kind.clone(),
@@ -171,7 +216,7 @@ impl AccountFeedbackStats {
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
.map(AccountFeedback::scheduling_signals)
.map(|account| account.scheduling_signals_at(now))
.unwrap_or_default()
}
@@ -181,6 +226,18 @@ impl AccountFeedbackStats {
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
feedback: AccountAttemptFeedback,
) {
self.report_at(provider_kind, account_id, feedback, Instant::now());
}
/// 回灌一次在指定单调时刻真实发送的上游 attempt。
#[doc(hidden)]
pub fn report_at(
&self,
provider_kind: &ProviderKind,
account_id: &ProviderAccountId,
feedback: AccountAttemptFeedback,
observed_at: Instant,
) {
let key = AccountFeedbackKey {
provider_kind: provider_kind.clone(),
@@ -192,7 +249,7 @@ impl AccountFeedbackStats {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
{
account.report(feedback);
account.report_at(feedback, observed_at);
return;
}
self.accounts
@@ -200,7 +257,7 @@ impl AccountFeedbackStats {
.unwrap_or_else(std::sync::PoisonError::into_inner)
.entry(key)
.or_default()
.report(feedback);
.report_at(feedback, observed_at);
}
}
@@ -13,9 +13,9 @@ use crate::engine::provider::{Provider, ProviderCallMetadata, ProviderRequest, P
use crate::engine::{
AccountAttemptContext, AttemptContext, AttemptRecord, AttemptTransport, AttemptTrigger,
CommitRequirement, ContinuationAttempt, CoordinatedEvent, EngineError, ExecutionOutcome,
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
RequestAttemptContext, UpstreamSendState,
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFailureObservation,
ModelRequestFinalization, ModelRequestId, ModelRequestTimings, NewModelRequest,
ProviderAccountStateOwner, ProviderAttemptOutcome, RequestAttemptContext, UpstreamSendState,
};
use crate::error::{
ContinuationRecoveryDisposition, GatewayError, GatewayErrorKind, ProviderError,
@@ -221,6 +221,7 @@ struct FailureFinalization {
provider_error_code: Option<String>,
raw_upstream_error: Option<String>,
retry_after_ms: Option<u64>,
observation: ModelRequestFailureObservation,
}
#[derive(Debug, Clone)]
@@ -638,6 +639,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::EmptyRoutingPlan);
@@ -795,6 +797,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::ProviderMetadataMismatch);
@@ -820,6 +823,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::AccountOutsideClientScope);
@@ -840,6 +844,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::RequiredAccountMismatch);
@@ -863,6 +868,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await?;
return Err(EngineError::ContinuationPinMismatch);
@@ -1368,6 +1374,7 @@ where
error: None,
provider_error_code: None,
raw_upstream_error: None,
failure_observation: ModelRequestFailureObservation::default(),
retry_after_ms: None,
usage: self.usage.clone(),
image_generation_succeeded: self.image_generation_succeeded(),
@@ -1410,6 +1417,12 @@ where
.raw_upstream_error()
.map(|raw| raw.as_str().to_owned()),
retry_after_ms: error.retry_after().map(duration_ms),
observation: ModelRequestFailureObservation {
continuation_unavailable_reason: error
.continuation_unavailable_reason()
.map(str::to_owned),
upstream_connection: error.connection_observation().cloned(),
},
})
.await
}
@@ -1464,6 +1477,7 @@ where
provider_error_code: None,
raw_upstream_error: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
.await
}
@@ -1532,6 +1546,7 @@ where
error: Some(finalization.error),
provider_error_code: finalization.provider_error_code,
raw_upstream_error: finalization.raw_upstream_error,
failure_observation: finalization.observation,
retry_after_ms: finalization.retry_after_ms,
usage: self.usage.clone(),
image_generation_succeeded: self.image_generation_succeeded(),
@@ -505,8 +505,11 @@ impl DefaultExecutionService {
.candidates()
.first()
.map_or_else(Default::default, |candidate| {
self.providers
.request_observation(candidate.provider(), &operation)
self.providers.request_observation(
candidate.provider(),
&operation,
client.policy.key_id(),
)
});
let new_request = NewModelRequest {
id: request_id.clone(),
@@ -526,6 +529,7 @@ impl DefaultExecutionService {
request_kind: observation.request_kind,
subagent_kind: observation.subagent_kind,
compact: observation.compact,
continuation: observation.continuation,
image_generation_requested: operation.image_generation_requested(),
admission_decision_ms: Some(admission_decision_ms),
started_at,
@@ -668,6 +672,7 @@ impl DefaultExecutionService {
request_kind: Some("account_connection_test".to_owned()),
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
admission_decision_ms: None,
started_at,
+11 -1
View File
@@ -22,7 +22,8 @@ use thiserror::Error;
use crate::account::{AccountSelectionPolicy, ProviderAccountId};
use crate::engine::continuation::{ContinuationBinding, NativeContinuationPin};
use crate::error::{
GatewayError, IdentifierError, ProviderError, ProviderErrorKind, StoreError, validate_text,
GatewayError, IdentifierError, ProviderConnectionObservation, ProviderError, ProviderErrorKind,
StoreError, validate_text,
};
use crate::event::ProviderEvent;
use crate::lifecycle::CancellationToken;
@@ -514,6 +515,7 @@ pub struct NewModelRequest {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: provider::ContinuationRequestObservation,
pub image_generation_requested: bool,
/// Client Key 准入判定的完整耗时;内部探测不经过该阶段。
pub admission_decision_ms: Option<u64>,
@@ -579,6 +581,13 @@ pub struct ModelRequestTimings {
pub latency_ms: Option<u64>,
}
/// 终态失败的 Provider 结构化连接与 continuation 观测。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ModelRequestFailureObservation {
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection: Option<ProviderConnectionObservation>,
}
/// 单行模型请求的终态写回。
#[derive(Debug)]
pub struct ModelRequestFinalization {
@@ -602,6 +611,7 @@ pub struct ModelRequestFinalization {
pub provider_error_code: Option<String>,
/// Provider 返回的原始错误正文或 WebSocket close/error frame。
pub raw_upstream_error: Option<String>,
pub failure_observation: ModelRequestFailureObservation,
pub retry_after_ms: Option<u64>,
pub usage: Usage,
pub image_generation_succeeded: Option<bool>,
@@ -15,9 +15,10 @@ use crate::account::{
AccountAttemptFeedback, AccountCapacitySnapshot, AccountFeedbackStats, ProviderAccountId,
};
use crate::engine::AttemptContext;
use crate::error::{OpaqueUpstreamValue, ProviderError, ProviderErrorKind};
use crate::error::{OpaqueUpstreamValue, PreDeliveryRetry, ProviderError, ProviderErrorKind};
use crate::event::{EventSequenceValidator, ProviderEvent};
use crate::operation::Operation;
use crate::policy::ClientApiKeyId;
use crate::routing::{
ModelCapabilities, ModelPresentation, ProviderCandidate, ProviderKind, UpstreamModelId,
};
@@ -190,11 +191,16 @@ struct ProviderStreamAccountFeedback {
stats: Arc<AccountFeedbackStats>,
provider_kind: ProviderKind,
account_id: ProviderAccountId,
failure_filter: fn(&ProviderError) -> bool,
started_at: Option<Instant>,
first_output_ms: Option<u64>,
reported: bool,
}
fn score_all_confirmed_failures(_: &ProviderError) -> bool {
true
}
impl ProviderStreamAccountFeedback {
fn start(&mut self) {
self.started_at.get_or_insert_with(Instant::now);
@@ -243,6 +249,15 @@ impl ProviderStreamAccountFeedback {
error.kind(),
ProviderErrorKind::Cancelled | ProviderErrorKind::ProcessTerminated
)
|| matches!(
error.pre_delivery_retry(),
Some(
PreDeliveryRetry::SameAccountTransportRetry { .. }
| PreDeliveryRetry::SameAccountTransportFallback
)
)
|| error.retries_same_account()
|| !(self.failure_filter)(error)
{
return;
}
@@ -278,15 +293,35 @@ impl ProviderStream {
/// 让公共 stream 边界统一回灌账号成功率与首个有效输出延迟。
#[must_use]
pub fn with_account_feedback(mut self, stats: Arc<AccountFeedbackStats>) -> Self {
self.set_account_feedback(stats, score_all_confirmed_failures);
self
}
/// 使用 Provider 定义的闭集判断回灌账号成功率与首个有效输出延迟。
#[must_use]
pub fn with_filtered_account_feedback(
mut self,
stats: Arc<AccountFeedbackStats>,
failure_filter: fn(&ProviderError) -> bool,
) -> Self {
self.set_account_feedback(stats, failure_filter);
self
}
fn set_account_feedback(
&mut self,
stats: Arc<AccountFeedbackStats>,
failure_filter: fn(&ProviderError) -> bool,
) {
self.account_feedback = Some(ProviderStreamAccountFeedback {
stats,
provider_kind: self.metadata.provider().clone(),
account_id: self.metadata.provider_account_id().clone(),
failure_filter,
started_at: None,
first_output_ms: None,
reported: false,
});
self
}
/// 返回调用事实。
@@ -387,6 +422,15 @@ pub struct ProviderRequestObservation {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: ContinuationRequestObservation,
}
/// 仅用于恢复事件关联的客户端作用域不透明请求事实。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ContinuationRequestObservation {
pub affinity_hash: Option<String>,
pub previous_response_id_hash: Option<String>,
pub requested: bool,
}
/// Provider 实时目录编译后的单模型能力。
@@ -495,7 +539,11 @@ pub trait Provider: Send + Sync {
fn catalog_generation(&self) -> ProviderCatalogGeneration;
/// 解释 Provider 差异化观测字段;不参与路由和传输。
fn request_observation(&self, _operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
_operation: &Operation,
_client_api_key_id: &ClientApiKeyId,
) -> ProviderRequestObservation {
ProviderRequestObservation::default()
}
@@ -623,11 +671,12 @@ impl ProviderRegistry {
&self,
provider: &ProviderKind,
operation: &Operation,
client_api_key_id: &ClientApiKeyId,
) -> ProviderRequestObservation {
self.providers
.get(provider)
.map_or_else(ProviderRequestObservation::default, |registered| {
registered.request_observation(operation)
registered.request_observation(operation, client_api_key_id)
})
}
+108
View File
@@ -135,6 +135,8 @@ pub enum MeteringError {
pub enum ProviderErrorKind {
/// 请求语义错误。
InvalidRequest,
/// previous-response 续接状态已不可用,需要客户端重建请求链。
ContinuationRecoveryRequired,
/// Provider 不支持所需能力。
Unsupported,
/// Credential 认证失败。
@@ -171,6 +173,7 @@ impl ProviderErrorKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::InvalidRequest => "invalid_request",
Self::ContinuationRecoveryRequired => "continuation_recovery_required",
Self::Unsupported => "unsupported",
Self::Unauthorized => "unauthorized",
Self::PermissionDenied => "permission_denied",
@@ -357,6 +360,55 @@ impl fmt::Debug for RawUpstreamError {
}
}
/// Provider 上游物理连接结束时的低基数观测。
///
/// 连接标识必须是随机或单向派生值,不得携带凭据、原始会话 ID
/// 或请求正文。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderConnectionObservation {
connection_id: String,
exit_reason: String,
age_ms: u64,
idle_ms: u64,
}
impl ProviderConnectionObservation {
#[must_use]
pub fn new(
connection_id: impl Into<String>,
exit_reason: impl Into<String>,
age_ms: u64,
idle_ms: u64,
) -> Self {
Self {
connection_id: connection_id.into(),
exit_reason: exit_reason.into(),
age_ms,
idle_ms,
}
}
#[must_use]
pub fn connection_id(&self) -> &str {
&self.connection_id
}
#[must_use]
pub fn exit_reason(&self) -> &str {
&self.exit_reason
}
#[must_use]
pub const fn age_ms(&self) -> u64 {
self.age_ms
}
#[must_use]
pub const fn idle_ms(&self) -> u64 {
self.idle_ms
}
}
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
///
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
@@ -442,6 +494,7 @@ pub struct ProviderError {
retry_after: Option<Duration>,
continuation_failure: Option<ContinuationFailure>,
continuation_recovery_disposition: Option<ContinuationRecoveryDisposition>,
failure_observation: Option<Box<ProviderErrorFailureObservation>>,
replay_safe: bool,
pre_delivery_retry: Option<PreDeliveryRetry>,
credential_recovery_required: bool,
@@ -460,6 +513,12 @@ struct ProviderErrorUpstreamValues {
request_id: Option<OpaqueUpstreamValue>,
}
#[derive(Clone, Default)]
struct ProviderErrorFailureObservation {
continuation_unavailable_reason: Option<&'static str>,
connection: Option<ProviderConnectionObservation>,
}
#[derive(Clone)]
struct AtomicClientEvents(Vec<ProviderEvent>);
@@ -475,6 +534,7 @@ impl ProviderError {
retry_after: None,
continuation_failure: None,
continuation_recovery_disposition: None,
failure_observation: None,
replay_safe: false,
pre_delivery_retry: None,
credential_recovery_required: false,
@@ -540,6 +600,29 @@ impl ProviderError {
self
}
/// 附加 Provider 给出的低基数 continuation 不可用原因。
#[must_use]
pub fn with_continuation_unavailable_reason(mut self, reason: &'static str) -> Self {
self.failure_observation_mut()
.continuation_unavailable_reason = Some(reason);
self
}
/// 附加上游物理连接结束观测。
#[must_use]
pub fn with_connection_observation(
mut self,
observation: ProviderConnectionObservation,
) -> Self {
self.failure_observation_mut().connection = Some(observation);
self
}
fn failure_observation_mut(&mut self) -> &mut ProviderErrorFailureObservation {
self.failure_observation
.get_or_insert_with(|| Box::new(ProviderErrorFailureObservation::default()))
}
/// 标记 Provider 已证明本次拒绝没有执行生成,可在下游提交前重放。
#[must_use]
pub const fn with_replay_safe(mut self) -> Self {
@@ -714,6 +797,21 @@ impl ProviderError {
self.continuation_recovery_disposition
}
#[must_use]
pub fn continuation_unavailable_reason(&self) -> Option<&'static str> {
match self.failure_observation.as_deref() {
Some(observation) => observation.continuation_unavailable_reason,
None => None,
}
}
#[must_use]
pub fn connection_observation(&self) -> Option<&ProviderConnectionObservation> {
self.failure_observation
.as_deref()
.and_then(|observation| observation.connection.as_ref())
}
/// 返回 Provider 是否已证明本次失败可安全重放。
#[must_use]
pub const fn replay_is_safe(&self) -> bool {
@@ -801,6 +899,7 @@ impl Clone for ProviderError {
retry_after: self.retry_after,
continuation_failure: self.continuation_failure,
continuation_recovery_disposition: self.continuation_recovery_disposition,
failure_observation: self.failure_observation.clone(),
replay_safe: self.replay_safe,
pre_delivery_retry: self.pre_delivery_retry,
credential_recovery_required: self.credential_recovery_required,
@@ -836,6 +935,11 @@ impl fmt::Debug for ProviderError {
"continuation_recovery_disposition",
&self.continuation_recovery_disposition,
)
.field(
"continuation_unavailable_reason",
&self.continuation_unavailable_reason(),
)
.field("connection_observation", &self.connection_observation())
.field("replay_safe", &self.replay_safe)
.field("pre_delivery_retry", &self.pre_delivery_retry)
.field(
@@ -968,6 +1072,10 @@ impl GatewayError {
ProviderErrorKind::InvalidRequest => {
Self::new(GatewayErrorKind::InvalidRequest, "invalid upstream request")
}
ProviderErrorKind::ContinuationRecoveryRequired => Self::new(
GatewayErrorKind::InvalidRequest,
"conversation continuation must be rebuilt",
),
ProviderErrorKind::Unsupported => Self::new(
GatewayErrorKind::Unsupported,
"requested capability is unsupported",
@@ -1,3 +1,5 @@
mod selection;
use std::collections::{BTreeMap, BTreeSet};
use std::num::NonZeroU32;
use std::sync::Arc;
@@ -0,0 +1,98 @@
use std::time::{Duration, Instant};
use gateway_core::account::{AccountAttemptFeedback, AccountFeedbackStats, ProviderAccountId};
use gateway_core::routing::ProviderKind;
const FAILURE_RATE_HALF_LIFE: Duration = Duration::from_secs(15 * 60);
fn feedback_subject() -> (AccountFeedbackStats, ProviderKind, ProviderAccountId) {
(
AccountFeedbackStats::default(),
ProviderKind::new("openai").expect("valid provider"),
ProviderAccountId::new("acct_decay").expect("valid account"),
)
}
fn report_failure(
feedback: &AccountFeedbackStats,
provider: &ProviderKind,
account: &ProviderAccountId,
observed_at: Instant,
) {
feedback.report_at(
provider,
account,
AccountAttemptFeedback::Failed {
first_output_ms: None,
},
observed_at,
);
}
#[test]
fn account_failure_rate_should_halve_after_one_half_life() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
.0;
assert_eq!(failure_rate, Some(1_000));
}
#[test]
fn account_failure_rate_should_quarter_after_two_half_lives() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
let failure_rate = feedback
.scheduling_signals_at(
&provider,
&account,
observed_at + FAILURE_RATE_HALF_LIFE * 2,
)
.0;
assert_eq!(failure_rate, Some(500));
}
#[test]
fn account_failure_rate_should_decay_before_applying_a_new_sample() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
report_failure(&feedback, &provider, &account, observed_at);
feedback.report_at(
&provider,
&account,
AccountAttemptFeedback::Succeeded {
first_output_ms: None,
},
observed_at + FAILURE_RATE_HALF_LIFE,
);
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at + FAILURE_RATE_HALF_LIFE)
.0;
assert_eq!(failure_rate, Some(800));
}
#[test]
fn concurrent_account_failures_should_not_lose_samples() {
let (feedback, provider, account) = feedback_subject();
let observed_at = Instant::now();
std::thread::scope(|scope| {
for _ in 0..8 {
scope.spawn(|| report_failure(&feedback, &provider, &account, observed_at));
}
});
let failure_rate = feedback
.scheduling_signals_at(&provider, &account, observed_at)
.0;
assert_eq!(failure_rate, Some(8_322));
}
@@ -501,10 +501,13 @@ fn atomic_history_unavailable(response_id: &str) -> ProviderError {
)
.expect("history unavailable wire"),
);
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
.with_status(400)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_atomic_client_events(vec![failed])
ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::Sent,
)
.with_status(400)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_atomic_client_events(vec![failed])
}
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
@@ -621,6 +624,7 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
request_kind: Some("responses".to_owned()),
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: operation.image_generation_requested(),
started_at: SystemTime::now(),
deadline_at: deadline,
@@ -1591,7 +1595,7 @@ fn native_continuation_client_replay_required_is_terminal() {
Script::Stream {
account_id: "acct_one",
items: vec![Err(ProviderError::new(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
@@ -62,6 +62,9 @@ fn only_provider_attributable_failures_should_affect_circuit() {
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::InvalidRequest
));
assert!(!provider_failure_affects_circuit(
ProviderErrorKind::ContinuationRecoveryRequired
));
}
#[test]
@@ -1,6 +1,8 @@
use std::collections::BTreeSet;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use async_trait::async_trait;
use futures::{StreamExt, stream};
@@ -244,3 +246,139 @@ fn provider_stream_should_report_confirmed_sent_failure_but_ignore_unconfirmed_f
(None, None)
);
}
#[test]
fn provider_stream_should_ignore_repeated_failures_rejected_by_the_filter() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_client_config_error").expect("account");
for _ in 0..100 {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("valid transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::Sent,
))]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| false);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
assert_eq!(
feedback.scheduling_signals(&provider, &account),
(None, None)
);
}
#[test]
fn provider_stream_should_score_only_the_final_same_account_failure() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_same_account_failure").expect("account");
for retry_index in 1..=3 {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
.with_pre_delivery_transport_retry(
NonZeroU32::new(retry_index).expect("positive retry index"),
Duration::ZERO,
);
let events: EventStream = Box::pin(stream::iter([Err(error)]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(ProviderError::new(
ProviderErrorKind::Transport,
UpstreamSendState::Sent,
))]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
assert_eq!(
feedback.scheduling_signals(&provider, &account).0,
Some(2_000)
);
}
#[test]
fn provider_stream_should_report_one_success_after_same_account_retry() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("valid provider");
let account = ProviderAccountId::new("acct_same_account_success").expect("account");
let retry_metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("websocket").expect("valid transport"),
);
let retry_error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Sent)
.with_pre_delivery_transport_fallback();
let retry_events: EventStream = Box::pin(stream::iter([Err(retry_error)]));
let mut retry_stream = ProviderStream::new(retry_metadata, retry_events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(
retry_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
let success_metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("valid model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("valid transport"),
);
let success_events: EventStream = Box::pin(stream::empty());
let mut success_stream = ProviderStream::new(success_metadata, success_events, ())
.with_filtered_account_feedback(Arc::clone(&feedback), |_| true);
futures::executor::block_on(async {
assert!(success_stream.next().await.is_none());
});
assert_eq!(feedback.scheduling_signals(&provider, &account).0, Some(0));
}
@@ -168,6 +168,30 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
assert!(!format!("{gateway:?}").contains(message));
}
#[test]
fn continuation_recovery_should_keep_its_internal_classification_and_client_contract() {
let error = ProviderError::new(
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"Previous response was not found. Retrying the full request.",
Some("previous_response_not_found".to_owned()),
Some("invalid_request_error".to_owned()),
));
let gateway = GatewayError::from_provider(&error);
assert_eq!(error.kind().as_str(), "continuation_recovery_required");
assert_eq!(
gateway.safe_message(),
"conversation continuation must be rebuilt"
);
assert_eq!(
gateway.client_error_code(),
Some("previous_response_not_found")
);
}
#[test]
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
let message = format!("\0{}\n", "m".repeat(9_000));
@@ -11,7 +11,9 @@ use gateway_core::engine::{
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
};
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
use gateway_core::error::{
ProviderErrorKind, StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind,
};
use gateway_core::metering::{CostSource as CoreCostSource, Usage as CoreUsage};
use gateway_core::routing::{AccountRoutingScopeKind, AccountRoutingSnapshot};
use gateway_core::upstream::UpstreamSendState as CoreUpstreamSendState;
@@ -99,12 +101,20 @@ pub struct NewModelRequest {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: bool,
pub continuation: ContinuationRequestObservation,
pub image_generation_requested: bool,
pub admission_decision_ms: Option<u64>,
pub started_at: DateTime<Utc>,
pub deadline_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ContinuationRequestObservation {
pub affinity_hash: Option<String>,
pub previous_response_id_hash: Option<String>,
pub requested: bool,
}
impl NewModelRequest {
pub fn validate(&self) -> StoreResult<()> {
require_nonempty(ENTITY, "id", &self.id)?;
@@ -133,6 +143,31 @@ impl NewModelRequest {
{
return Err(invalid("live client key ID must equal its historical ref"));
}
self.continuation.validate()?;
Ok(())
}
}
impl ContinuationRequestObservation {
fn validate(&self) -> StoreResult<()> {
for value in [
self.affinity_hash.as_deref(),
self.previous_response_id_hash.as_deref(),
]
.into_iter()
.flatten()
{
if value.len() != 64
|| !value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err(invalid("continuation observation hash is invalid"));
}
}
if !self.requested && self.previous_response_id_hash.is_some() {
return Err(invalid("continuation request facts do not agree"));
}
Ok(())
}
}
@@ -252,6 +287,11 @@ pub struct ModelRequestFinalization {
pub provider_error_code: Option<String>,
pub error_message: Option<String>,
pub raw_upstream_error: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub retry_after_ms: Option<u64>,
pub usage: ModelRequestUsage,
pub image_generation_succeeded: Option<bool>,
@@ -313,6 +353,11 @@ impl ModelRequestFinalization {
{
return Err(invalid("provider observation must be a JSON object"));
}
validate_optional_stable_reason(
self.continuation_unavailable_reason.as_deref(),
"continuation unavailable reason",
)?;
validate_connection_observation(self)?;
self.timings.validate()
}
}
@@ -389,11 +434,13 @@ impl ModelRequestRepository for PgExecutionStore {
operation, endpoint, client_transport, requested_model_id,
client_ip, user_agent, reasoning_effort,
reasoning_preset, request_kind, subagent_kind, compact,
image_generation_requested, admission_decision_ms, started_at, deadline_at
image_generation_requested, admission_decision_ms, started_at, deadline_at,
continuation_affinity_hash, continuation_previous_response_id_hash,
continuation_requested
) values (
$1, $2, $3, $4, $5, $6, $7, $8,
$9, $10, $11, $12, $13::inet, $14, $15,
$16, $17, $18, $19, $20, $21, $22, $23
$16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26
)",
)
.bind(request.id)
@@ -422,6 +469,9 @@ impl ModelRequestRepository for PgExecutionStore {
)?)
.bind(request.started_at)
.bind(request.deadline_at)
.bind(request.continuation.affinity_hash)
.bind(request.continuation.previous_response_id_hash)
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request"))?;
@@ -452,13 +502,15 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_model_id, upstream_transport, http_version,
attempt_count, upstream_send_state, account_selection_wait_ms,
capacity_used_slots, capacity_total_slots
, continuation_affinity_hash, continuation_previous_response_id_hash,
continuation_requested
) select
$1, $2, $3, $4, $5, $6, $7, $8,
$9, $10, $11, $12, $13::inet, $14, $15,
$16, $17, $18, $19, $20, $21, $22, $23,
$24, $25, $26,
account.name, account.email, account.authentication_kind,
$27, $28, $29, 1, 'not_sent', $30, $31, $32
$27, $28, $29, 1, 'not_sent', $30, $31, $32, $33, $34, $35
from (values (true)) as seed(present)
left join provider_accounts account on account.id = $25",
)
@@ -506,6 +558,9 @@ impl ModelRequestRepository for PgExecutionStore {
attempt.capacity_total_slots,
"capacity_total_slots",
)?)
.bind(request.continuation.affinity_hash)
.bind(request.continuation.previous_response_id_hash)
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
@@ -644,8 +699,9 @@ impl ModelRequestRepository for PgExecutionStore {
finalization: ModelRequestFinalization,
) -> StoreResult<bool> {
finalization.validate()?;
let result = sqlx::query(
"update model_requests
let finalized = sqlx::query_scalar::<_, i64>(
"with finalized as (
update model_requests
set outcome = $2, upstream_send_state = $3, attempt_count = $4,
downstream_committed_at = $5,
client_status_code = coalesce(client_status_code, $6),
@@ -664,8 +720,64 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_transport = coalesce($37, upstream_transport),
http_version = coalesce($38, http_version), websocket_pool = $39,
service_tier = $40, provider_observation_json = $41,
raw_upstream_error = $42
where id = $1 and outcome = 'running'",
raw_upstream_error = $42,
continuation_unavailable_reason = $43,
upstream_connection_id = $44,
upstream_connection_exit_reason = $45,
upstream_connection_age_ms = $46,
upstream_connection_idle_ms = $47
where id = $1 and outcome = 'running'
returning id, client_api_key_ref, continuation_affinity_hash,
continuation_requested, outcome, started_at, completed_at
), recovery_target as (
select prior.id
from model_requests prior
cross join finalized current
where not current.continuation_requested
and current.continuation_affinity_hash is not null
and prior.client_api_key_ref = current.client_api_key_ref
and prior.continuation_affinity_hash = current.continuation_affinity_hash
and prior.continuation_requested
and prior.outcome = 'failed'
and prior.error_kind = 'continuation_recovery_required'
and prior.recovered_at is null
and current.started_at >= prior.completed_at
and current.started_at <= prior.completed_at + interval '30 seconds'
order by prior.completed_at desc, prior.id desc
limit 1
), recovery_update as (
update model_requests prior
set recovery_attempt_count = prior.recovery_attempt_count + 1,
recovery_request_id = case
when current.outcome = 'succeeded' then current.id
else prior.recovery_request_id
end,
recovered_at = case
when current.outcome = 'succeeded' then current.completed_at
else prior.recovered_at
end,
recovery_retry_delay_ms = case
when current.outcome = 'succeeded' then greatest(
0,
floor(extract(epoch from (current.started_at - prior.completed_at)) * 1000)
)::bigint
else prior.recovery_retry_delay_ms
end,
recovery_total_latency_ms = case
when current.outcome = 'succeeded' then greatest(
0,
floor(extract(epoch from (current.completed_at - prior.completed_at)) * 1000)
)::bigint
else prior.recovery_total_latency_ms
end
from finalized current
join recovery_target target on true
where prior.id = target.id
and prior.recovered_at is null
returning prior.id
)
select (select count(*) from finalized)::bigint
+ (select count(*) * 0 from recovery_update)::bigint",
)
.bind(&finalization.model_request_id)
.bind(finalization.outcome.as_str())
@@ -754,10 +866,21 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(finalization.service_tier)
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
.bind(finalization.raw_upstream_error)
.execute(&self.pool)
.bind(finalization.continuation_unavailable_reason)
.bind(finalization.upstream_connection_id)
.bind(finalization.upstream_connection_exit_reason)
.bind(optional_i64(
finalization.upstream_connection_age_ms,
"upstream_connection_age_ms",
)?)
.bind(optional_i64(
finalization.upstream_connection_idle_ms,
"upstream_connection_idle_ms",
)?)
.fetch_one(&self.pool)
.await
.map_err(|_| postgres_unavailable("finalize model request"))?;
Ok(result.rows_affected() == 1)
Ok(finalized == 1)
}
async fn recover_expired_model_requests(
@@ -973,6 +1096,22 @@ impl ExecutionStore for PgExecutionStore {
&self,
finalization: CoreModelRequestFinalization,
) -> Result<(), CoreStoreError> {
let continuation_unavailable_reason = finalization
.failure_observation
.continuation_unavailable_reason
.clone();
let connection_observation = finalization
.failure_observation
.upstream_connection
.as_ref();
let upstream_connection_id =
connection_observation.map(|observation| observation.connection_id().to_owned());
let upstream_connection_exit_reason =
connection_observation.map(|observation| observation.exit_reason().to_owned());
let upstream_connection_age_ms =
connection_observation.map(|observation| observation.age_ms());
let upstream_connection_idle_ms =
connection_observation.map(|observation| observation.idle_ms());
let provider_metadata_json = finalization
.provider_metadata_json
.as_deref()
@@ -999,10 +1138,18 @@ impl ExecutionStore for PgExecutionStore {
),
None => (CostSource::Unavailable, None, None),
};
let error_kind = finalization
.error
.as_ref()
.map(|error| error.kind().as_str().to_owned());
let error_kind = if continuation_unavailable_reason.is_some() {
Some(
ProviderErrorKind::ContinuationRecoveryRequired
.as_str()
.to_owned(),
)
} else {
finalization
.error
.as_ref()
.map(|error| error.kind().as_str().to_owned())
};
let error_message = finalization.error.as_ref().map(|error| {
error.diagnostic().map_or_else(
|| error.safe_message().to_owned(),
@@ -1033,6 +1180,11 @@ impl ExecutionStore for PgExecutionStore {
provider_error_code: finalization.provider_error_code,
error_message,
raw_upstream_error: finalization.raw_upstream_error,
continuation_unavailable_reason,
upstream_connection_id,
upstream_connection_exit_reason,
upstream_connection_age_ms,
upstream_connection_idle_ms,
retry_after_ms: finalization.retry_after_ms,
usage: usage_from_core(finalization.usage),
image_generation_succeeded: finalization.image_generation_succeeded,
@@ -1162,6 +1314,11 @@ fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
request_kind: request.request_kind,
subagent_kind: request.subagent_kind,
compact: request.compact,
continuation: ContinuationRequestObservation {
affinity_hash: request.continuation.affinity_hash,
previous_response_id_hash: request.continuation.previous_response_id_hash,
requested: request.continuation.requested,
},
image_generation_requested: request.image_generation_requested,
admission_decision_ms: request.admission_decision_ms,
started_at: DateTime::<Utc>::from(request.started_at),
@@ -1246,6 +1403,52 @@ fn validate_status_code(status: Option<u16>) -> StoreResult<()> {
Ok(())
}
fn validate_optional_stable_reason(value: Option<&str>, field: &str) -> StoreResult<()> {
if value.is_some_and(|value| {
value.is_empty()
|| value.len() > 64
|| !value.bytes().enumerate().all(|(index, byte)| {
byte.is_ascii_lowercase() || (index > 0 && (byte.is_ascii_digit() || byte == b'_'))
})
}) {
return Err(invalid(field));
}
Ok(())
}
fn validate_connection_observation(finalization: &ModelRequestFinalization) -> StoreResult<()> {
let fields_present = [
finalization.upstream_connection_id.is_some(),
finalization.upstream_connection_exit_reason.is_some(),
finalization.upstream_connection_age_ms.is_some(),
finalization.upstream_connection_idle_ms.is_some(),
];
if fields_present.iter().any(|present| *present)
&& !fields_present.iter().all(|present| *present)
{
return Err(invalid("upstream connection observation is incomplete"));
}
if let Some(connection_id) = finalization.upstream_connection_id.as_deref()
&& (connection_id.is_empty()
|| connection_id.len() > 128
|| connection_id.chars().any(char::is_control))
{
return Err(invalid("upstream connection ID is invalid"));
}
validate_optional_stable_reason(
finalization.upstream_connection_exit_reason.as_deref(),
"upstream connection exit reason is invalid",
)?;
if finalization
.upstream_connection_age_ms
.zip(finalization.upstream_connection_idle_ms)
.is_some_and(|(age, idle)| idle > age)
{
return Err(invalid("upstream connection idle time exceeds its age"));
}
Ok(())
}
fn to_i64(value: u64, field: &'static str) -> StoreResult<i64> {
i64::try_from(value).map_err(|_| invalid(field))
}
@@ -39,7 +39,10 @@ pub use provider_accounts::*;
pub use retention::*;
pub use runtime_settings::*;
pub use snapshot::*;
pub(crate) use usage_facts::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
pub(crate) use usage_facts::{
completed_usage_fact_predicate, push_completed_usage_fact_filter,
push_unrecovered_request_filter,
};
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
@@ -51,13 +51,11 @@ pub(crate) fn store_ops_error_filter(
provider_account_ref: filter.provider_account_ref,
provider_kind: filter.provider_kind,
operation: filter.operation,
endpoint: filter.endpoint,
model: filter.model,
transport: filter.transport,
attempt_index: filter.attempt_index,
response_id: filter.response_id,
upstream_request_id: filter.upstream_request_id,
failure_kind: filter.failure_kind,
status_code: filter.status_code,
search: filter.search,
}
@@ -731,6 +729,18 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
request_kind: error.request_kind,
subagent_kind: error.subagent_kind,
compact: error.compact,
continuation_affinity_hash: error.continuation_affinity_hash,
continuation_previous_response_id_hash: error.continuation_previous_response_id_hash,
continuation_unavailable_reason: error.continuation_unavailable_reason,
upstream_connection_id: error.upstream_connection_id,
upstream_connection_exit_reason: error.upstream_connection_exit_reason,
upstream_connection_age_ms: error.upstream_connection_age_ms,
upstream_connection_idle_ms: error.upstream_connection_idle_ms,
recovery_request_id: error.recovery_request_id,
recovered_at: error.recovered_at,
recovery_attempt_count: error.recovery_attempt_count,
recovery_retry_delay_ms: error.recovery_retry_delay_ms,
recovery_total_latency_ms: error.recovery_total_latency_ms,
occurred_at: error.occurred_at,
stable_sort_id: error.stable_sort_id,
}
@@ -910,6 +920,18 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
request_kind: get(row, "request_kind")?,
subagent_kind: get(row, "subagent_kind")?,
compact: get(row, "compact")?,
continuation_affinity_hash: get(row, "continuation_affinity_hash")?,
continuation_previous_response_id_hash: get(row, "continuation_previous_response_id_hash")?,
continuation_unavailable_reason: get(row, "continuation_unavailable_reason")?,
upstream_connection_id: get(row, "upstream_connection_id")?,
upstream_connection_exit_reason: get(row, "upstream_connection_exit_reason")?,
upstream_connection_age_ms: optional_unsigned(row, "upstream_connection_age_ms")?,
upstream_connection_idle_ms: optional_unsigned(row, "upstream_connection_idle_ms")?,
recovery_request_id: get(row, "recovery_request_id")?,
recovered_at: get(row, "recovered_at")?,
recovery_attempt_count: to_u32(get(row, "recovery_attempt_count")?)?,
recovery_retry_delay_ms: optional_unsigned(row, "recovery_retry_delay_ms")?,
recovery_total_latency_ms: optional_unsigned(row, "recovery_total_latency_ms")?,
occurred_at: get(row, "occurred_at")?,
stable_sort_id: get(row, "stable_sort_id")?,
})
@@ -18,7 +18,10 @@ use crate::{
require_nonempty,
};
use super::{completed_usage_fact_predicate, push_completed_usage_fact_filter};
use super::{
completed_usage_fact_predicate, push_completed_usage_fact_filter,
push_unrecovered_request_filter,
};
mod admin_adapter;
mod mapping;
@@ -126,13 +126,11 @@ pub struct OpsErrorFilter {
pub provider_account_ref: Option<String>,
pub provider_kind: Option<String>,
pub operation: Option<String>,
pub endpoint: Option<String>,
pub model: Option<String>,
pub transport: Option<String>,
pub attempt_index: Option<u32>,
pub response_id: Option<String>,
pub upstream_request_id: Option<String>,
pub failure_kind: Option<String>,
pub status_code: Option<u16>,
pub search: Option<String>,
}
@@ -148,14 +146,12 @@ impl OpsErrorFilter {
),
(self.provider_kind.as_deref(), "provider filter"),
(self.operation.as_deref(), "operation filter"),
(self.endpoint.as_deref(), "endpoint filter"),
(self.model.as_deref(), "model filter"),
(self.transport.as_deref(), "transport filter"),
(
self.upstream_request_id.as_deref(),
"upstream request ID filter",
),
(self.failure_kind.as_deref(), "failure kind filter"),
] {
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
}
@@ -726,6 +722,18 @@ pub struct OpsErrorRecord {
pub request_kind: Option<String>,
pub subagent_kind: Option<String>,
pub compact: Option<bool>,
pub continuation_affinity_hash: Option<String>,
pub continuation_previous_response_id_hash: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
pub upstream_connection_age_ms: Option<u64>,
pub upstream_connection_idle_ms: Option<u64>,
pub recovery_request_id: Option<String>,
pub recovered_at: Option<DateTime<Utc>>,
pub recovery_attempt_count: u32,
pub recovery_retry_delay_ms: Option<u64>,
pub recovery_total_latency_ms: Option<u64>,
pub occurred_at: DateTime<Utc>,
pub stable_sort_id: String,
}
@@ -93,6 +93,7 @@ pub(crate) async fn request_metrics(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
let row = query
.build()
@@ -113,7 +114,8 @@ pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTota
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
sum(cost_amount) filter (where {fact} and cost_currency = 'USD')::text
as billing_usd
from model_requests mr"
from model_requests mr
where mr.recovered_at is null"
));
let row = query
.build()
@@ -247,6 +249,7 @@ async fn request_metric_series_inner(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by bucket_start order by bucket_start");
let rows = query
@@ -373,6 +376,7 @@ pub(crate) async fn attempt_metrics(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(format!(
"), request_aggregate as (
@@ -517,6 +521,7 @@ pub(crate) async fn provider_observations(
query.push_bind(range.start);
query.push(" and mr.started_at < ");
query.push_bind(range.end);
push_unrecovered_request_filter(&mut query, "mr");
push_usage_filter(&mut query, filter, "mr");
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
let rows = query
@@ -24,6 +24,12 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
mr.continuation_unavailable_reason,
mr.upstream_connection_id, mr.upstream_connection_exit_reason,
mr.upstream_connection_age_ms, mr.upstream_connection_idle_ms,
mr.recovery_request_id, mr.recovered_at, mr.recovery_attempt_count,
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
mr.completed_at as occurred_at,
'model_request:' || mr.id as stable_sort_id
from model_requests mr
@@ -46,6 +52,15 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
mr.continuation_affinity_hash, mr.continuation_previous_response_id_hash,
null::text as continuation_unavailable_reason,
null::text as upstream_connection_id,
null::text as upstream_connection_exit_reason,
null::bigint as upstream_connection_age_ms,
null::bigint as upstream_connection_idle_ms,
mr.recovery_request_id, mr.recovered_at,
coalesce(mr.recovery_attempt_count, 0) as recovery_attempt_count,
mr.recovery_retry_delay_ms, mr.recovery_total_latency_ms,
oe.created_at as occurred_at,
'ops_event:' || oe.id as stable_sort_id
from ops_events oe
@@ -113,6 +128,7 @@ fn push_request_error_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and mr.recovered_at is null");
push_range(statement, "mr.completed_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -120,16 +136,11 @@ fn push_request_error_predicates(
("mr.provider_account_ref", &filter.provider_account_ref),
("mr.provider_kind", &filter.provider_kind),
("mr.operation", &filter.operation),
("mr.endpoint", &filter.endpoint),
("mr.upstream_transport", &filter.transport),
("mr.upstream_request_id", &filter.upstream_request_id),
] {
push_text_equality(statement, column, value);
}
if let Some(value) = &filter.failure_kind {
statement.push(" and coalesce(mr.error_kind, 'failed') = ");
statement.push_bind(value.clone());
}
push_response_id_filter(statement, "mr.client_response_id", filter);
push_text_equality(statement, "mr.upstream_model_id", &filter.model);
if let Some(index) = filter.attempt_index {
@@ -160,6 +171,7 @@ fn push_ops_event_predicates(
range: ObservabilityRange,
filter: &OpsErrorFilter,
) {
statement.push(" and (mr.id is null or mr.recovered_at is null)");
push_range(statement, "oe.created_at", range);
for (column, value) in [
("mr.client_api_key_ref", &filter.client_api_key_ref),
@@ -167,9 +179,7 @@ fn push_ops_event_predicates(
("oe.provider_account_ref", &filter.provider_account_ref),
("oe.provider_kind", &filter.provider_kind),
("oe.operation", &filter.operation),
("mr.endpoint", &filter.endpoint),
("oe.upstream_request_id", &filter.upstream_request_id),
("oe.failure_kind", &filter.failure_kind),
] {
push_text_equality(statement, column, value);
}
@@ -360,6 +360,7 @@ pub(crate) async fn usage_diagnostics(
statement.push_bind(range.start);
statement.push(" and mr.started_at < ");
statement.push_bind(range.end);
push_unrecovered_request_filter(&mut statement, "mr");
push_diagnostic_dimension_filter(&mut statement, dimension);
push_usage_filter(&mut statement, filter, "mr");
statement.push(
@@ -18,3 +18,10 @@ pub(crate) fn completed_usage_fact_predicate(alias: &str) -> String {
pub(crate) fn push_completed_usage_fact_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
query.push(format!(" and {}", completed_usage_fact_predicate(alias)));
}
/// 排除已由后续成功请求恢复的会话续接中间失败。
///
/// 原始请求审计仍保留该行;默认业务指标只把最终成功链视为一次结果。
pub(crate) fn push_unrecovered_request_filter(query: &mut QueryBuilder<Postgres>, alias: &str) {
query.push(format!(" and {alias}.recovered_at is null"));
}
@@ -1,9 +1,10 @@
use chrono::{Duration, Utc};
use gateway_core::engine::{
ExecutionOutcome, ExecutionStore, ModelRequestFinalization as CoreModelRequestFinalization,
ModelRequestId, ModelRequestTimings as CoreModelRequestTimings,
ExecutionOutcome, ExecutionStore, ModelRequestFailureObservation,
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
ModelRequestTimings as CoreModelRequestTimings,
};
use gateway_core::error::{GatewayError, GatewayErrorKind};
use gateway_core::error::{GatewayError, GatewayErrorKind, ProviderConnectionObservation};
use gateway_core::metering::{CalculatedCost, CostEstimate, Usage};
use gateway_core::upstream::UpstreamSendState;
use gateway_store::postgres::{
@@ -42,6 +43,7 @@ fn model_request_rejects_mismatched_client_key_live_id() {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -93,6 +95,7 @@ async fn merged_model_less_first_attempt_should_match_sequential_semantics() {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -210,6 +213,7 @@ async fn model_request_persists_group_routing_snapshot_without_live_group_foreig
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + Duration::seconds(30),
@@ -477,6 +481,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
error: None,
provider_error_code: None,
raw_upstream_error: None,
failure_observation: Default::default(),
retry_after_ms: None,
usage: Usage::new(),
image_generation_succeeded: None,
@@ -600,6 +605,243 @@ async fn core_adapter_should_reject_non_object_provider_observation() {
database.close().await;
}
#[tokio::test]
async fn core_adapter_persists_continuation_connection_failure_observation() {
let Some(database) = TestDatabase::create("execution_continuation_observation").await else {
return;
};
seed_running_request(&database.pool, "req_continuation_observation")
.await
.expect("seed model request");
let store = PgExecutionStore::new(database.pool.clone());
let mut finalization = successful_core_finalization("req_continuation_observation");
finalization.outcome = ExecutionOutcome::Failed;
finalization.client_status_code = Some(400);
finalization.error = Some(GatewayError::new(
GatewayErrorKind::InvalidRequest,
"conversation continuation must be rebuilt",
));
finalization.failure_observation = ModelRequestFailureObservation {
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
upstream_connection: Some(ProviderConnectionObservation::new(
"connection-observed",
"tcp_reset",
12_000,
4_000,
)),
};
ExecutionStore::finalize_model_request(&store, finalization)
.await
.expect("persist continuation failure observation");
type PersistedContinuationObservation = (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
Option<i64>,
Option<i64>,
);
let persisted: PersistedContinuationObservation = sqlx::query_as(
"select error_kind, continuation_unavailable_reason,
upstream_connection_id, upstream_connection_exit_reason,
upstream_connection_age_ms, upstream_connection_idle_ms
from model_requests where id = 'req_continuation_observation'",
)
.fetch_one(&database.pool)
.await
.expect("load continuation failure observation");
assert_eq!(
persisted.0.as_deref(),
Some("continuation_recovery_required")
);
assert_eq!(persisted.1.as_deref(), Some("reused_connection_lost"));
assert_eq!(persisted.2.as_deref(), Some("connection-observed"));
assert_eq!(persisted.3.as_deref(), Some("tcp_reset"));
assert_eq!(persisted.4, Some(12_000));
assert_eq!(persisted.5, Some(4_000));
database.close().await;
}
#[tokio::test]
async fn successful_new_chain_should_mark_recent_continuation_failure_recovered() {
let Some(database) = TestDatabase::create("execution_continuation_recovered").await else {
return;
};
let affinity_hash = "a".repeat(64);
let failure_completed_at = Utc::now() - Duration::seconds(8);
seed_continuation_failure(
&database.pool,
"req_continuation_failed",
&affinity_hash,
failure_completed_at,
)
.await;
seed_recovery_request(
&database.pool,
"req_continuation_recovery",
&affinity_hash,
failure_completed_at + Duration::seconds(4),
)
.await;
let store = PgExecutionStore::new(database.pool.clone());
ExecutionStore::finalize_model_request(
&store,
successful_core_finalization("req_continuation_recovery"),
)
.await
.expect("finalize successful recovery request");
type PersistedRecoveryObservation = (
Option<String>,
Option<chrono::DateTime<Utc>>,
i32,
Option<i64>,
Option<i64>,
);
let recovered: PersistedRecoveryObservation = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count,
recovery_retry_delay_ms, recovery_total_latency_ms
from model_requests where id = 'req_continuation_failed'",
)
.fetch_one(&database.pool)
.await
.expect("load recovered continuation failure");
assert_eq!(recovered.0.as_deref(), Some("req_continuation_recovery"));
assert!(recovered.1.is_some());
assert_eq!(recovered.2, 1);
assert_eq!(recovered.3, Some(4_000));
assert!(recovered.4.is_some_and(|latency| latency >= 4_000));
database.close().await;
}
#[tokio::test]
async fn failed_or_late_new_chain_should_leave_continuation_failure_unresolved() {
let Some(database) = TestDatabase::create("execution_continuation_unresolved").await else {
return;
};
let affinity_hash = "c".repeat(64);
let failure_completed_at = Utc::now() - Duration::seconds(60);
seed_continuation_failure(
&database.pool,
"req_continuation_unresolved",
&affinity_hash,
failure_completed_at,
)
.await;
seed_recovery_request(
&database.pool,
"req_continuation_retry_failed",
&affinity_hash,
failure_completed_at + Duration::seconds(3),
)
.await;
let store = PgExecutionStore::new(database.pool.clone());
let mut failed = successful_core_finalization("req_continuation_retry_failed");
failed.outcome = ExecutionOutcome::Failed;
failed.client_status_code = Some(502);
failed.upstream_status_code = Some(500);
failed.error = Some(GatewayError::new(
GatewayErrorKind::UpstreamUnavailable,
"recovery request failed",
));
ExecutionStore::finalize_model_request(&store, failed)
.await
.expect("finalize failed recovery request");
let after_failed: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count
from model_requests where id = 'req_continuation_unresolved'",
)
.fetch_one(&database.pool)
.await
.expect("load unresolved continuation failure");
assert_eq!(after_failed, (None, None, 1));
seed_recovery_request(
&database.pool,
"req_continuation_retry_late",
&affinity_hash,
failure_completed_at + Duration::seconds(31),
)
.await;
ExecutionStore::finalize_model_request(
&store,
successful_core_finalization("req_continuation_retry_late"),
)
.await
.expect("finalize late recovery request");
let after_late: (Option<String>, Option<chrono::DateTime<Utc>>, i32) = sqlx::query_as(
"select recovery_request_id, recovered_at, recovery_attempt_count
from model_requests where id = 'req_continuation_unresolved'",
)
.fetch_one(&database.pool)
.await
.expect("load late unresolved continuation failure");
assert_eq!(after_late, (None, None, 1));
database.close().await;
}
async fn seed_continuation_failure(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
completed_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed continuation failure request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
continuation_affinity_hash = $2,
continuation_previous_response_id_hash = $3,
continuation_requested = true,
outcome = 'failed', error_kind = 'continuation_recovery_required',
continuation_unavailable_reason = 'reused_connection_lost',
started_at = $4 - interval '1 second', deadline_at = $4,
completed_at = $4
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind("b".repeat(64))
.bind(completed_at)
.execute(pool)
.await
.expect("finalize seeded continuation failure");
}
async fn seed_recovery_request(
pool: &sqlx::PgPool,
id: &str,
affinity_hash: &str,
started_at: chrono::DateTime<Utc>,
) {
seed_running_request(pool, id)
.await
.expect("seed continuation recovery request");
sqlx::query(
"update model_requests
set client_api_key_ref = 'key_continuation', client_transport = 'websocket',
continuation_affinity_hash = $2, continuation_requested = false,
started_at = $3, deadline_at = $3 + interval '1 minute'
where id = $1",
)
.bind(id)
.bind(affinity_hash)
.bind(started_at)
.execute(pool)
.await
.expect("prepare seeded recovery request");
}
async fn seed_running_request(pool: &sqlx::PgPool, id: &str) -> Result<(), sqlx::Error> {
sqlx::query(
"insert into model_requests (
@@ -215,6 +215,84 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
database.close().await;
}
#[tokio::test]
async fn recovered_continuation_failure_should_be_hidden_from_default_business_metrics() {
let Some(database) = TestDatabase::create("observability_recovered_continuation").await else {
return;
};
let now = Utc::now();
seed_observability_facts(&database.pool, now)
.await
.expect("seed observability facts");
sqlx::query(
"update model_requests
set error_kind = 'continuation_recovery_required',
continuation_affinity_hash = $2,
continuation_previous_response_id_hash = $3,
continuation_requested = true,
continuation_unavailable_reason = 'reused_connection_lost',
recovery_request_id = 'req_observe_success', recovered_at = $1,
recovery_attempt_count = 1, recovery_retry_delay_ms = 4000,
recovery_total_latency_ms = 8000
where id = 'req_observe_failed'",
)
.bind(now)
.bind("a".repeat(64))
.bind("b".repeat(64))
.execute(&database.pool)
.await
.expect("mark continuation failure recovered");
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
.expect("observability range");
let repository = observability_repository(&database.pool);
let overview = repository
.usage_summary(range, UsageRecordFilter::default())
.await
.expect("usage summary without recovered intermediates");
assert_eq!(overview.requests.request_count, 2);
assert_eq!(overview.requests.failure_count, 0);
assert_eq!(overview.attempts.attempt_count, 2);
assert_eq!(overview.attempts.failure_count, 0);
assert_eq!(overview.providers[0].request_count, 2);
assert_eq!(overview.providers[0].failure_count, 0);
let diagnostics = repository
.usage_diagnostics(
range,
UsageRecordFilter::default(),
DiagnosticDimension::Account,
)
.await
.expect("diagnostics without recovered intermediates");
assert_eq!(diagnostics[0].request_count, 2);
assert_eq!(diagnostics[0].failure_count, 0);
let errors = repository
.list_ops_errors(OpsErrorQuery {
range,
filter: OpsErrorFilter::default(),
current_page: 1,
page_size: ObservabilityPageSize::new(10).expect("page size"),
})
.await
.expect("ops errors without recovered intermediates");
assert_eq!(errors.total, 0);
let retained: (String, Option<String>) = sqlx::query_as(
"select outcome, recovery_request_id
from model_requests where id = 'req_observe_failed'",
)
.fetch_one(&database.pool)
.await
.expect("load retained recovery audit row");
assert_eq!(
retained,
("failed".to_owned(), Some("req_observe_success".to_owned()))
);
database.close().await;
}
#[tokio::test]
async fn dashboard_account_metrics_should_partition_account_statuses() {
let Some(database) = TestDatabase::create("dashboard_account_metrics").await else {
@@ -762,9 +840,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
provider_kind: Some("openai".to_owned()),
provider_account_ref: Some("acct_observe".to_owned()),
operation: Some("responses".to_owned()),
endpoint: Some("/v1/responses".to_owned()),
model: Some("upstream-model".to_owned()),
failure_kind: Some("rate_limited".to_owned()),
status_code: Some(429),
search: Some("req_observe_failed".to_owned()),
..admin_observability::OpsErrorFilter::default()
@@ -875,6 +875,7 @@ fn new_request(id: &str, started_at: DateTime<Utc>) -> NewModelRequest {
request_kind: None,
subagent_kind: None,
compact: false,
continuation: Default::default(),
image_generation_requested: false,
started_at,
deadline_at: started_at + chrono::Duration::seconds(30),
@@ -32,6 +32,12 @@ impl CodexSessionAffinity {
&self.key_hash
}
/// 返回可持久化的客户端作用域不透明会话关联值。
#[must_use]
pub(crate) fn persistence_hash(&self) -> &str {
self.key.expose_to_store()
}
#[must_use]
pub(crate) const fn anchor_source(&self) -> &'static str {
self.anchor_source
@@ -58,6 +64,20 @@ impl CodexSessionAffinity {
}
}
/// 将原始 response ID 投影为客户端作用域的不可逆关联值。
#[must_use]
pub(crate) fn derive_previous_response_id_hash(
previous_response_id: &str,
client_api_key_id: &ClientApiKeyId,
) -> String {
let mut hasher = Sha256::new();
hasher.update(b"codex-previous-response-observation-v1\0");
hasher.update(client_api_key_id.as_str().as_bytes());
hasher.update(b"\0");
hasher.update(previous_response_id.as_bytes());
hex::encode(hasher.finalize())
}
pub(crate) fn derive_codex_session_affinity(
request: &CodexResponsesRequest,
client_api_key_id: &ClientApiKeyId,
@@ -17,7 +17,7 @@ mod types;
pub(crate) use affinity::{
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
derive_codex_session_affinity,
derive_codex_session_affinity, derive_previous_response_id_hash,
};
pub(crate) use oauth::oauth_owner_ref;
pub(crate) use types::parse_access_token_expiration;
+1 -1
View File
@@ -30,7 +30,7 @@ use crate::transport::{CodexWebSocketPool, build_reqwest_client};
pub use config::{CodexWireProfileConfig, OpenAiConfig, OpenAiConfigError};
pub use provider::{
CodexProvider, CodexProviderConfigError, CodexProviderTransport, OFFICIAL_CODEX_BASE_PATH,
OFFICIAL_CODEX_BASE_URL,
OFFICIAL_CODEX_BASE_URL, openai_failure_affects_account_score,
};
pub mod credential;
@@ -68,7 +68,10 @@ impl CodexProvider {
});
let stream = ProviderStream::new(metadata, events, lease);
Ok(if allows_account_state_mutation {
stream.with_account_feedback(Arc::clone(&self.account_feedback))
stream.with_filtered_account_feedback(
Arc::clone(&self.account_feedback),
openai_failure_affects_account_score,
)
} else {
stream
})
@@ -2,6 +2,40 @@
use super::*;
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集。
///
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type。
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集。
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
"service_unavailable_error",
];
fn is_openai_account_score_failure_reason(value: &str) -> bool {
let value = value.trim();
OPENAI_ACCOUNT_SCORE_FAILURE_REASONS
.iter()
.any(|reason| value.eq_ignore_ascii_case(reason))
}
fn openai_account_score_failure_reason(error: &ProviderError) -> Option<&str> {
if let Some(code) = error.upstream_code() {
return Some(code.as_str());
}
let upstream = error.client_visible_upstream_error()?;
upstream.code().or_else(|| upstream.error_type())
}
/// 返回该 OpenAI 失败是否属于 Smart 账号计分闭集。
#[doc(hidden)]
pub fn openai_failure_affects_account_score(error: &ProviderError) -> bool {
openai_account_score_failure_reason(error).is_some_and(is_openai_account_score_failure_reason)
}
pub(super) struct MappedProviderFailure {
pub(super) error: ProviderError,
pub(super) websocket_transport_retryable: bool,
@@ -454,7 +488,10 @@ pub(super) fn websocket_client_failure_policy(
Some(WebSocketFailurePolicy::Budgeted)
}
CodexClientError::WebSocket(error)
if !matches!(error, CodexWebSocketExchangeError::InvalidRequest(_)) =>
if !matches!(
error.classified(),
CodexWebSocketExchangeError::InvalidRequest(_)
) =>
{
Some(WebSocketFailurePolicy::Budgeted)
}
@@ -581,15 +618,16 @@ pub(super) fn websocket_retry_backoff(retry_index: NonZeroU32) -> Duration {
Duration::from_millis(base_ms.saturating_mul(jitter_per_mille) / 1_000)
}
pub(super) fn continuation_replay_required_error() -> ProviderError {
pub(super) fn continuation_replay_required_error(reason: &'static str) -> ProviderError {
ProviderError::new(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
)
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
)
.with_continuation_unavailable_reason(reason)
.with_upstream_code(OpaqueUpstreamValue::new(
PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned(),
))
@@ -620,7 +658,7 @@ pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientErro
| CodexClientError::HttpJson(_)
| CodexClientError::StreamIdleTimeout { .. } => true,
CodexClientError::WebSocket(error) => !matches!(
error,
error.classified(),
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::Upstream(_)
| CodexWebSocketExchangeError::ContinuationUnavailable { .. }
@@ -666,12 +704,32 @@ pub(super) fn map_client_error(
_ => None,
};
let continuation_failure = match &error {
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::ConnectionBusy,
}) => Some(ContinuationFailure::Busy),
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
..
}) => Some(ContinuationFailure::HistoryUnavailable),
CodexClientError::WebSocket(error)
if error.continuation_unavailable_reason()
== Some(PreviousResponseUnavailableReason::ConnectionBusy) =>
{
Some(ContinuationFailure::Busy)
}
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
Some(ContinuationFailure::HistoryUnavailable)
}
_ => None,
};
let continuation_unavailable_reason = match &error {
CodexClientError::WebSocket(error) => error
.continuation_unavailable_reason()
.map(PreviousResponseUnavailableReason::as_str),
_ => None,
};
let connection_observation = match &error {
CodexClientError::WebSocket(error) => error.connection_observation().map(|observation| {
ProviderConnectionObservation::new(
observation.connection_id().to_string(),
observation.exit_reason().to_owned(),
observation.age_ms(),
observation.idle_ms(),
)
}),
_ => None,
};
let observation = observe_transport
@@ -718,10 +776,10 @@ pub(super) fn map_client_error(
send_state,
))
}
CodexClientError::WebSocket(
error @ CodexWebSocketExchangeError::ContinuationUnavailable { .. },
) => {
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error());
CodexClientError::WebSocket(error) if error.continuation_unavailable_reason().is_some() => {
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error(
continuation_unavailable_reason.unwrap_or("scope_unavailable"),
));
if let Some(client_visible_error) = websocket_client_visible_error(&error) {
failure.error = failure
.error
@@ -744,7 +802,10 @@ pub(super) fn map_client_error(
"websocket_close_{close_code}"
)));
}
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
if matches!(
error.classified(),
CodexWebSocketExchangeError::ConnectionLimitReached
) {
failure.error = failure.error.with_upstream_code(OpaqueUpstreamValue::new(
WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(),
));
@@ -771,6 +832,11 @@ pub(super) fn map_client_error(
if let Some(raw) = raw_upstream_error {
failure.error = failure.error.with_raw_upstream_error(raw);
}
if let Some(connection_observation) = connection_observation {
failure.error = failure
.error
.with_connection_observation(connection_observation);
}
failure.observation = observation;
failure
}
@@ -790,7 +856,7 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
return ProviderDiagnostic::new(message);
}
ProviderDiagnostic::new(match error {
ProviderDiagnostic::new(match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_) => {
"OpenAI WebSocket request could not be constructed".to_owned()
}
@@ -847,6 +913,9 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
CodexWebSocketExchangeError::ClosedBeforeTerminal(_) => {
unreachable!("close-before-terminal errors are handled before the variant match")
}
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
})
}
@@ -866,7 +935,10 @@ fn websocket_raw_error(error: &CodexWebSocketExchangeError) -> Option<RawUpstrea
pub(super) fn websocket_client_visible_error(
error: &CodexWebSocketExchangeError,
) -> Option<ClientVisibleUpstreamError> {
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
if matches!(
error.classified(),
CodexWebSocketExchangeError::ConnectionLimitReached
) {
return Some(ClientVisibleUpstreamError::new(
"websocket connection limit reached",
Some(WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned()),
@@ -900,7 +972,12 @@ pub(super) fn map_upstream_failure(
.filter(|code| is_history_failure_code(code))
.map(|_| ContinuationFailure::HistoryUnavailable);
let send_state = upstream_send_state(failure.send_phase);
let mut error = provider_error(provider_error_kind(category), send_state);
let error_kind = if continuation_failure.is_some() {
ProviderErrorKind::ContinuationRecoveryRequired
} else {
provider_error_kind(category)
};
let mut error = provider_error(error_kind, send_state);
error = error.with_raw_upstream_error(RawUpstreamError::new(failure.raw_body.clone()));
if let Some(message) = failure.client_message.as_ref() {
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
@@ -939,7 +1016,8 @@ pub(super) fn map_upstream_failure(
.with_continuation_failure(continuation_failure)
.with_continuation_recovery_disposition(
ContinuationRecoveryDisposition::ClientReplayRequired,
);
)
.with_continuation_unavailable_reason("upstream_rejected");
}
if let Some(retry_after) = failure.retry_after_seconds.map(Duration::from_secs) {
error = error.with_retry_after(retry_after);
@@ -1064,8 +1142,8 @@ pub(super) fn account_failure(
}
}
pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
match error {
pub(super) fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSendState {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::Connect(_)
| CodexWebSocketExchangeError::ConnectTimeout { .. }
@@ -1085,11 +1163,14 @@ pub(super) const fn websocket_send_state(error: &CodexWebSocketExchangeError) ->
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. }
| CodexWebSocketExchangeError::InitialEventTimeout { .. } => UpstreamSendState::Ambiguous,
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
}
}
pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
match error {
pub(super) fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderErrorKind {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_)
| CodexWebSocketExchangeError::InvalidSse(_)
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => ProviderErrorKind::Protocol,
@@ -1113,6 +1194,9 @@ pub(super) const fn websocket_error_kind(error: &CodexWebSocketExchangeError) ->
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. } => {
ProviderErrorKind::Transport
}
CodexWebSocketExchangeError::ConnectionObserved { .. } => {
unreachable!("classified websocket errors never retain observation wrappers")
}
}
}
@@ -12,15 +12,15 @@ use futures::{StreamExt, future::BoxFuture};
use gateway_core::account::{AccountFeedbackStats, ProviderAccount};
use gateway_core::engine::continuation::{ContinuationBinding, NativeContinuationScope};
use gateway_core::engine::provider::{
EventStream, Provider, ProviderCallMetadata, ProviderCatalogGeneration,
ProviderModelCapabilities, ProviderRequest, ProviderRequestObservation,
ProviderSelectionObservation, ProviderStream,
ContinuationRequestObservation, EventStream, Provider, ProviderCallMetadata,
ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRequest,
ProviderRequestObservation, ProviderSelectionObservation, ProviderStream,
};
use gateway_core::engine::{AttemptContext, AttemptTransport, ContinuationAttempt};
use gateway_core::error::{
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderDiagnostic, ProviderError,
ProviderErrorKind, RawUpstreamError,
ContinuationRecoveryDisposition, OpaqueUpstreamValue, ProviderConnectionObservation,
ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError,
};
use gateway_core::event::{
FinishReason, GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader,
@@ -58,6 +58,7 @@ use crate::credential::{
CodexQuotaRefreshPolicy, CodexSessionAffinity, CredentialSelectionError, RuntimeCodexCookie,
SelectCodexCredential, SelectCodexProviderEndpointCredential,
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
derive_previous_response_id_hash,
};
use crate::session_transport::CodexSessionTransportFallbacks;
use crate::transport::canonical::{
@@ -97,6 +98,8 @@ mod observation;
mod workers;
use execution::*;
#[doc(hidden)]
pub use failure::openai_failure_affects_account_score;
use failure::*;
use observation::*;
pub(crate) use workers::worker_contributions;
@@ -208,23 +211,35 @@ impl Provider for CodexProvider {
self.catalog.catalog_generation()
}
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
operation: &Operation,
client_api_key_id: &gateway_core::policy::ClientApiKeyId,
) -> ProviderRequestObservation {
let Operation::Generate(request) = operation else {
return ProviderRequestObservation::default();
};
let (semantics, reasoning_effort) = encode_generate_request(request, "observability")
.map(|encoded| {
let semantics = encoded.semantics();
let reasoning_effort = semantics.reasoning_effort.clone();
(semantics, reasoning_effort)
})
.unwrap_or_default();
let Ok(encoded) = encode_generate_request(request, "observability") else {
return ProviderRequestObservation::default();
};
let semantics = encoded.semantics();
let reasoning_effort = semantics.reasoning_effort.clone();
let previous_response_id = encoded.previous_response_id();
let continuation = ContinuationRequestObservation {
affinity_hash: derive_codex_session_affinity(&encoded, client_api_key_id)
.map(|affinity| affinity.persistence_hash().to_owned()),
previous_response_id_hash: previous_response_id.map(|response_id| {
derive_previous_response_id_hash(response_id, client_api_key_id)
}),
requested: previous_response_id.is_some(),
};
ProviderRequestObservation {
reasoning_effort,
reasoning_preset: semantics.reasoning_preset.map(str::to_owned),
request_kind: semantics.request_kind,
subagent_kind: semantics.subagent_kind,
compact: semantics.compact,
continuation,
}
}
@@ -385,7 +400,7 @@ impl Provider for CodexProvider {
continuation_recovery_action = "stop_proxy_recovery",
"OpenAI connection-local continuation replay was rejected before send"
);
return Err(continuation_replay_required_error());
return Err(continuation_replay_required_error("scope_unavailable"));
}
let previous_response_scope = match context.continuation_attempt() {
ContinuationAttempt::Native => native_scope,
@@ -410,7 +425,7 @@ impl Provider for CodexProvider {
if upstream_request.previous_response_id().is_some()
&& !account_scope.can_reuse_account_state()
{
return Err(continuation_replay_required_error());
return Err(continuation_replay_required_error("scope_unavailable"));
}
scope_request_to_account(
&mut upstream_request,
@@ -421,7 +436,7 @@ impl Provider for CodexProvider {
let requested_transport = selected_transport(&upstream_request);
let sticky_http_fallback = context.transport() == AttemptTransport::Default
&& requested_transport == CodexProviderTransport::PreferWebSocket
&& !requirement.requires_websocket()
&& requirement.allows_sticky_http_fallback()
&& session_affinity.as_ref().is_some_and(|affinity| {
self.session_transport_fallbacks
.is_http_only(affinity.key())
@@ -502,7 +517,10 @@ impl Provider for CodexProvider {
});
let stream = ProviderStream::new(metadata, events, lease);
Ok(if allows_account_state_mutation {
stream.with_account_feedback(Arc::clone(&self.account_feedback))
stream.with_filtered_account_feedback(
Arc::clone(&self.account_feedback),
openai_failure_affects_account_score,
)
} else {
stream
})
@@ -391,7 +391,12 @@ pub(super) fn codex_error_observation(
)?
.with_status_code(status.as_u16());
}
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
CodexClientError::WebSocket(error)
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
{
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
unreachable!("websocket error was checked above")
};
observation = observation.with_status_code(upstream.status_code);
if let Some(request_id) = upstream.diagnostics.request_id.as_deref() {
observation =
@@ -313,7 +313,12 @@ impl CodexClientError {
rate_limit_headers,
*send_phase,
)),
Self::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
Self::WebSocket(error)
if matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)) =>
{
let CodexWebSocketExchangeError::Upstream(upstream) = error.classified() else {
unreachable!("websocket error was checked above")
};
Some(CodexUpstreamFailure::from_response(
StatusCode::from_u16(upstream.status_code).unwrap_or(StatusCode::BAD_GATEWAY),
&upstream.body,
@@ -848,7 +853,7 @@ pub(super) fn websocket_success_decision(
pub(super) fn http_fallback_decision(
error: &CodexWebSocketExchangeError,
) -> CodexTransportDecision {
match error {
match error.classified() {
CodexWebSocketExchangeError::FastPathTimeout { .. } => {
CodexTransportDecision::Http2WebSocketBudgetExhausted
}
@@ -58,6 +58,7 @@ const PERSISTABLE_UPSTREAM_CODES: &[&str] = &[
"rate_limit_exceeded",
"rate_limit_reached",
"refresh_token_invalidated",
"server_error",
"server_is_overloaded",
"slow_down",
"token_expired",
@@ -162,6 +162,11 @@ impl TransportRequirement {
matches!(self, Self::NewChain)
}
/// 会话级 sticky 状态是否可以把首选 WebSocket 改为 HTTP。
pub fn allows_sticky_http_fallback(self) -> bool {
matches!(self, Self::NewChain)
}
/// 用于审计与遥测的稳定名称。
pub fn as_str(self) -> &'static str {
match self {
@@ -193,8 +198,7 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
TransportRequirement::ExternalUnknown
}
},
None if request.use_websocket => TransportRequirement::NewChain,
None => TransportRequirement::HttpRequired,
None => TransportRequirement::NewChain,
}
}
@@ -24,7 +24,7 @@ use super::{
WebSocketPoolConnectLease, WebSocketPoolConnectOutcome, WebSocketPoolDecision,
WebSocketPoolLease,
},
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket},
pump::{PumpKeepalive, PumpLogContext, PumpedWebSocket, transport_metric_reason},
};
pub(crate) const WEBSOCKET_FAST_PATH_BUDGET: Duration = Duration::from_millis(800);
@@ -188,9 +188,11 @@ pub(crate) async fn prepare_response_create_request_with_pool(
}
}
}
WebSocketPoolAcquire::Bypass(reason) => {
Err(continuation_unavailable(bypass_unavailable_reason(reason)))
}
WebSocketPoolAcquire::ContinuationLost(observation) => Err(continuation_unavailable(
PreviousResponseUnavailableReason::ReusedConnectionLost,
)
.with_connection_observation(observation)),
WebSocketPoolAcquire::Bypass(reason) => Err(bypass_unavailable_error(reason)),
}
}
@@ -369,6 +371,7 @@ fn start_pooled_websocket_connect(
}
}
Err(error) => {
let connection_exit_reason = opening_failure_reason(&error);
let error_message = error.to_string();
// 先交付 opening 原始错误,避免连接池清理侵占前台 fast-path 预算。
let foreground_waiting = sender.send(Err(error)).is_ok();
@@ -378,6 +381,7 @@ fn start_pooled_websocket_connect(
conversation_id_hash = task_key.conversation_id_hash(),
ws_preconnect_duration_ms = duration_millis_u64(started_at.elapsed()),
foreground_waiting,
connection_exit_reason,
error = %error_message,
ws_preconnect_outcome = "failed",
"WebSocket pool connect finished"
@@ -392,6 +396,21 @@ fn start_pooled_websocket_connect(
}
}
fn opening_failure_reason(error: &CodexWebSocketExchangeError) -> &'static str {
match error.classified() {
CodexWebSocketExchangeError::InvalidRequest(_) => "request_build_failure",
CodexWebSocketExchangeError::Connect(error)
| CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
CodexWebSocketExchangeError::ConnectTimeout { .. }
| CodexWebSocketExchangeError::FastPathTimeout { .. } => "connect_timeout",
CodexWebSocketExchangeError::Upstream(_) => "upgrade_rejected",
CodexWebSocketExchangeError::OriginCircuitOpen
| CodexWebSocketExchangeError::OriginHalfOpenBusy => "connect_suppressed",
CodexWebSocketExchangeError::SharedConnectFailed => "shared_connect_failed",
_ => "connect_failure",
}
}
fn duration_millis_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_millis())
.unwrap_or(u64::MAX)
@@ -489,16 +508,16 @@ async fn wait_for_shared_connect(
}
}
fn bypass_unavailable_reason(
reason: WebSocketPoolBypassReason,
) -> PreviousResponseUnavailableReason {
fn bypass_unavailable_error(reason: WebSocketPoolBypassReason) -> CodexWebSocketExchangeError {
match reason {
WebSocketPoolBypassReason::Busy => PreviousResponseUnavailableReason::ConnectionBusy,
WebSocketPoolBypassReason::Busy => {
continuation_unavailable(PreviousResponseUnavailableReason::ConnectionBusy)
}
WebSocketPoolBypassReason::Disabled | WebSocketPoolBypassReason::Cap => {
PreviousResponseUnavailableReason::PoolUnavailable
continuation_unavailable(PreviousResponseUnavailableReason::PoolUnavailable)
}
WebSocketPoolBypassReason::ContinuationNotFound => {
PreviousResponseUnavailableReason::FreshConnectionRequired
continuation_unavailable(PreviousResponseUnavailableReason::FreshConnectionRequired)
}
}
}
@@ -527,8 +546,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
created_at,
} = connection;
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
discard_after_send(websocket, lease).await;
return Err(post_send_ambiguous(error));
let observation = websocket
.observation()
.with_exit_reason("outbound_transport_error");
discard_after_send(websocket, lease, observation.clone()).await;
return Err(post_send_ambiguous(
error.with_connection_observation(observation),
));
}
let connection_local_available = lease.is_some();
let pool_return = lease.map(|lease| WebSocketStreamPoolReturn {
@@ -548,9 +572,13 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
Ok(exchange)
}
async fn discard_after_send(websocket: PumpedWebSocket, lease: Option<WebSocketPoolLease>) {
async fn discard_after_send(
websocket: PumpedWebSocket,
lease: Option<WebSocketPoolLease>,
observation: super::pump::WebSocketConnectionObservation,
) {
if let Some(lease) = lease {
lease.discard().await;
lease.discard_with_observation(observation).await;
}
websocket.close().await;
}
@@ -11,6 +11,7 @@ use crate::transport::diagnostics::CodexUpstreamDiagnostics;
use crate::transport::diagnostics::CodexUpstreamSendPhase;
use super::PreviousResponseUnavailableReason;
use super::pump::WebSocketConnectionObservation;
/// Responses WebSocket 交互错误。
#[derive(Debug, Error)]
@@ -101,6 +102,13 @@ pub enum CodexWebSocketExchangeError {
/// 首个上游事件超时时长。
timeout: Duration,
},
/// 将一个已分类交互错误与物理连接生命周期快照绑定。
#[error("{source}")]
ConnectionObserved {
observation: WebSocketConnectionObservation,
#[source]
source: Box<CodexWebSocketExchangeError>,
},
}
/// 上游在 terminal 事件前发送的 WebSocket close 信息。
@@ -240,7 +248,9 @@ impl CodexWebSocketExchangeError {
)
}
pub(crate) fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
/// 返回错误链中的上游终态前 Close 帧(若存在)。
#[must_use]
pub fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
match self {
Self::ClosedBeforeTerminal(close) => Some(close),
Self::PostSendAmbiguous {
@@ -250,7 +260,54 @@ impl CodexWebSocketExchangeError {
| Self::ReusedConnectionDiedBeforeFirstEvent {
source: Some(source),
..
} => source.close_before_terminal(),
}
| Self::ConnectionObserved { source, .. } => source.close_before_terminal(),
_ => None,
}
}
pub(crate) fn with_connection_observation(
self,
observation: WebSocketConnectionObservation,
) -> Self {
match self {
Self::ConnectionObserved { .. } => self,
source => Self::ConnectionObserved {
observation,
source: Box::new(source),
},
}
}
/// 返回错误关联的物理 WebSocket 连接生命周期快照。
#[must_use]
pub fn connection_observation(&self) -> Option<&WebSocketConnectionObservation> {
match self {
Self::ConnectionObserved { observation, .. } => Some(observation),
Self::PostSendAmbiguous {
source: Some(source),
..
}
| Self::ReusedConnectionDiedBeforeFirstEvent {
source: Some(source),
..
} => source.connection_observation(),
_ => None,
}
}
/// 返回去掉观测 wrapper 后的原始分类错误。
pub(crate) fn classified(&self) -> &Self {
match self {
Self::ConnectionObserved { source, .. } => source.classified(),
_ => self,
}
}
#[must_use]
pub fn continuation_unavailable_reason(&self) -> Option<PreviousResponseUnavailableReason> {
match self.classified() {
Self::ContinuationUnavailable { reason } => Some(*reason),
_ => None,
}
}
@@ -277,6 +334,9 @@ impl CodexWebSocketExchangeError {
/// opening 阶段只有明确的 transport 可用性失败才能切到同账号 HTTP。
pub(in crate::transport) fn allows_pre_send_http_fallback(&self) -> bool {
if let Self::ConnectionObserved { source, .. } = self {
return source.allows_pre_send_http_fallback();
}
matches!(
self,
Self::Connect(_)
@@ -291,7 +351,7 @@ impl CodexWebSocketExchangeError {
/// 精确连接状态不可用属于本地池路由事实,不消耗上游 WS 重试预算。
pub(in crate::transport) fn requires_immediate_pool_fallback(&self) -> bool {
matches!(self, Self::ContinuationUnavailable { .. })
matches!(self.classified(), Self::ContinuationUnavailable { .. })
}
/// 首个业务事件交付前,普通 WebSocket 失败可切到同账号 HTTP。
@@ -302,6 +362,7 @@ impl CodexWebSocketExchangeError {
source: Some(source),
..
} => source.allows_pre_delivery_http_fallback(),
Self::ConnectionObserved { source, .. } => source.allows_pre_delivery_http_fallback(),
_ => true,
}
}
@@ -11,7 +11,9 @@ use super::super::{
CodexWebSocketConnectionMetadata, PooledWebSocketConnection, WebSocketContinuationState,
WebSocketPoolLease,
},
pump::{PumpExitReason, PumpedWebSocket},
pump::{
PumpExitReason, PumpedWebSocket, WebSocketConnectionObservation, transport_metric_reason,
},
};
use super::io::{next_websocket_message, receive_idle_timeout, reused_stream_receive_error};
use super::reducer::{ExchangeAction, WebSocketTerminalKind, reduce_websocket_event};
@@ -173,12 +175,22 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let message = match message {
Ok(message) => message,
Err(error) => {
let error = match error {
CodexWebSocketExchangeError::ReceiveIdleTimeout { timeout }
if !saw_upstream_activity =>
{
CodexWebSocketExchangeError::InitialEventTimeout { timeout }
}
error => error,
};
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -197,6 +209,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
text.to_string()
}
tungstenite::Message::Binary(_) => {
let error = CodexWebSocketExchangeError::UnexpectedBinaryEvent;
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
@@ -204,24 +218,26 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
)
.await;
let _ = tx
.send(Err(CodexWebSocketExchangeError::UnexpectedBinaryEvent))
.send(Err(error.with_connection_observation(observation)))
.await;
return;
}
tungstenite::Message::Close(frame) => {
let websocket_connection_id = websocket.connection_id();
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
websocket_connection_id,
frame.as_ref().map(|frame| u16::from(frame.code)),
frame.map(|frame| frame.reason.to_string()),
last_event_type.clone(),
);
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -235,12 +251,19 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
Ok(reduced) => reduced,
Err(error) => {
let observation =
(!matches!(error.classified(), CodexWebSocketExchangeError::Upstream(_)))
.then(|| connection_observation(&websocket, &error));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamReceiveFailed,
)
.await;
let error = match observation {
Some(observation) => error.with_connection_observation(observation),
None => error,
};
let _ = tx.send(Err(error)).await;
return;
}
@@ -301,12 +324,6 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
.exit_reason()
.and_then(|reason| reason.upstream_close().cloned())
.map(|close| close.with_connection_id(websocket_connection_id));
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = match upstream_close {
Some(close) => CodexWebSocketExchangeError::ClosedBeforeTerminal(
close.with_last_event_type(last_event_type),
@@ -318,6 +335,14 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
last_event_type,
),
};
let observation = connection_observation(&websocket, &error);
discard_stream_websocket(
websocket,
pool_return,
StreamWebSocketDiscardReason::UpstreamClosed,
)
.await;
let error = error.with_connection_observation(observation);
let error = if reused_connection {
reused_stream_receive_error(error)
} else {
@@ -353,6 +378,12 @@ async fn discard_stream_websocket(
reason: StreamWebSocketDiscardReason,
) {
let websocket_connection_id = websocket.connection_id();
let observation = websocket.observation();
let tombstone_observation = if observation.exit_reason() == "running" {
observation.clone().with_exit_reason(reason.as_str())
} else {
observation.clone()
};
let pump_exit = websocket.exit_reason();
let pump_exit_reason = pump_exit
.as_ref()
@@ -367,11 +398,45 @@ async fn discard_stream_websocket(
reason = reason.as_str(),
pump_exit_reason,
pump_exit_detail,
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pooled = pool_return.is_some(),
"Discarding Responses WebSocket stream"
);
if let Some(pool_return) = pool_return {
pool_return.lease.discard().await;
pool_return
.lease
.discard_with_observation(tombstone_observation)
.await;
}
websocket.close().await;
}
fn connection_observation(
websocket: &PumpedWebSocket,
error: &CodexWebSocketExchangeError,
) -> WebSocketConnectionObservation {
let observation = websocket.observation();
if observation.exit_reason() != "running" {
return observation;
}
observation.with_exit_reason(exchange_exit_reason(error))
}
fn exchange_exit_reason(error: &CodexWebSocketExchangeError) -> &'static str {
match error.classified() {
CodexWebSocketExchangeError::Transport(error) => transport_metric_reason(error),
CodexWebSocketExchangeError::ClosedBeforeTerminal(close) => {
if close.code() == Some(1000) {
"normal_close"
} else {
"upstream_close"
}
}
CodexWebSocketExchangeError::ReceiveIdleTimeout { .. } => "receive_idle_timeout",
CodexWebSocketExchangeError::InitialEventTimeout { .. } => "initial_event_timeout",
CodexWebSocketExchangeError::UnexpectedBinaryEvent => "unexpected_binary_event",
_ => "exchange_failure",
}
}
@@ -33,6 +33,7 @@ pub use self::{
CodexWebSocketPool, CodexWebSocketPoolConfig, CodexWebSocketPoolKey,
WebSocketPoolBypassReason, WebSocketPoolDecision,
},
pump::WebSocketConnectionObservation,
};
pub(crate) use self::{
coordinator::{
@@ -70,16 +70,23 @@ pub enum PreviousResponseUnavailableReason {
UpstreamRejected,
}
impl fmt::Display for PreviousResponseUnavailableReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
impl PreviousResponseUnavailableReason {
#[must_use]
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::PoolUnavailable => "pool_unavailable",
Self::FreshConnectionRequired => "fresh_connection_required",
Self::ConnectionBusy => "connection_busy",
Self::LatestResponseMismatch => "latest_response_mismatch",
Self::ReusedConnectionLost => "reused_connection_lost",
Self::UpstreamRejected => "upstream_rejected",
})
}
}
}
impl fmt::Display for PreviousResponseUnavailableReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
@@ -14,6 +14,7 @@ use uuid::Uuid;
use super::state::{CodexWebSocketPoolKey, PooledWebSocketConnection, WebSocketPoolReservation};
use super::{CodexWebSocketPool, WebSocketPoolBypassReason};
use crate::transport::websocket::pump::WebSocketConnectionObservation;
pub(crate) enum WebSocketPoolAcquire {
Reused {
@@ -22,6 +23,7 @@ pub(crate) enum WebSocketPoolAcquire {
},
Connect(WebSocketPoolConnectLease),
Wait(WebSocketPoolConnectWaiter),
ContinuationLost(WebSocketConnectionObservation),
Bypass(WebSocketPoolBypassReason),
}
@@ -182,10 +184,12 @@ impl WebSocketPoolLease {
self.armed = false;
}
pub(crate) async fn discard(mut self) {
pub(crate) async fn discard_with_observation(
mut self,
observation: WebSocketConnectionObservation,
) {
self.pool
.discard_reserved(&self.key, self.reservation.id)
.await;
.discard_reserved_with_observation(&self.key, self.reservation.id, observation);
self.armed = false;
}
}
@@ -19,6 +19,7 @@ use self::state::{
close_pooled_connections,
};
use super::pump::PumpKeepalive;
use super::pump::WebSocketConnectionObservation;
pub use self::state::CodexWebSocketPoolKey;
pub(crate) use self::{
@@ -154,6 +155,7 @@ impl CodexWebSocketPool {
let mut connections_to_close = Vec::new();
let acquire = {
let mut state = self.lock_state();
let mut continuation_loss = None;
if !self.config.enabled || state.shutting_down {
return WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::Disabled);
}
@@ -163,9 +165,14 @@ impl CodexWebSocketPool {
&& slot.latest_response_id() == Some(response_id))
.then(|| candidate.clone())
}) else {
return WebSocketPoolAcquire::Bypass(
WebSocketPoolBypassReason::ContinuationNotFound,
);
return state
.continuation_loss(key, response_id, tokio::time::Instant::now())
.map_or(
WebSocketPoolAcquire::Bypass(
WebSocketPoolBypassReason::ContinuationNotFound,
),
WebSocketPoolAcquire::ContinuationLost,
);
};
key
} else {
@@ -188,9 +195,9 @@ impl CodexWebSocketPool {
};
// 零成本探活:后台 pump 已实时感知连接死亡(RST/Close/EOF/失活),
// 复用前只需读取 is_closed 标志,避免复用到静默死连接后卡到超时。
if connection.created_at.elapsed() < self.config.max_age
&& !connection.websocket.is_closed()
{
let expired = connection.created_at.elapsed() >= self.config.max_age;
let closed = connection.websocket.is_closed();
if !expired && !closed {
let lease = WebSocketPoolLease::reserve(
self.clone(),
key.clone(),
@@ -202,12 +209,31 @@ impl CodexWebSocketPool {
);
return WebSocketPoolAcquire::Reused { connection, lease };
}
let observation = if expired && !closed {
connection
.websocket
.observation()
.with_exit_reason("max_age_expired")
} else {
connection.websocket.observation()
};
state.remember_continuation_loss(
&key,
connection.continuation.latest_response_id(),
observation.clone(),
tokio::time::Instant::now(),
);
connections_to_close.push(*connection);
if required_response_id.is_some() {
continuation_loss = Some(observation);
}
}
None => {}
}
let acquire = if required_response_id.is_some() {
let acquire = if let Some(observation) = continuation_loss {
WebSocketPoolAcquire::ContinuationLost(observation)
} else if required_response_id.is_some() {
WebSocketPoolAcquire::Bypass(WebSocketPoolBypassReason::ContinuationNotFound)
} else {
let connect_permit = self.connect_semaphore.clone().try_acquire_owned().ok();
@@ -259,6 +285,17 @@ impl CodexWebSocketPool {
if reservation.id == reservation_id
);
if owns_reservation && (expired || state.shutting_down || !self.config.enabled) {
if expired && let Some(connection) = connection.as_ref() {
state.remember_continuation_loss(
key,
connection.continuation.latest_response_id(),
connection
.websocket
.observation()
.with_exit_reason("max_age_expired"),
tokio::time::Instant::now(),
);
}
state.slots.remove(key);
} else if owns_reservation && let Some(connection) = connection.take() {
state.slots.insert(
@@ -274,10 +311,6 @@ impl CodexWebSocketPool {
}
}
async fn discard_reserved(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
self.discard_reserved_now(key, reservation_id);
}
fn discard_reserved_now(&self, key: &CodexWebSocketPoolKey, reservation_id: Uuid) {
let mut state = self.lock_state();
if matches!(
@@ -288,6 +321,28 @@ impl CodexWebSocketPool {
}
}
fn discard_reserved_with_observation(
&self,
key: &CodexWebSocketPoolKey,
reservation_id: Uuid,
observation: WebSocketConnectionObservation,
) {
let mut state = self.lock_state();
let latest_response_id = match state.slots.get(key) {
Some(WebSocketPoolSlot::Busy(reservation)) if reservation.id == reservation_id => {
reservation.latest_response_id.clone()
}
_ => return,
};
state.remember_continuation_loss(
key,
latest_response_id.as_deref(),
observation,
tokio::time::Instant::now(),
);
state.slots.remove(key);
}
async fn finish_connect_reserved(
&self,
key: &CodexWebSocketPoolKey,
@@ -1,6 +1,9 @@
//! WebSocket 连接池状态和值对象。
use std::{collections::HashMap, time::Duration};
use std::{
collections::{HashMap, VecDeque},
time::Duration,
};
use sha2::{Digest, Sha256};
use tokio::{sync::watch, time::Instant};
@@ -10,7 +13,7 @@ use uuid::Uuid;
use super::super::super::{
diagnostics::CodexUpstreamDiagnostics, response_meta::CodexResponseMetadata,
};
use super::super::pump::PumpedWebSocket;
use super::super::pump::{PumpedWebSocket, WebSocketConnectionObservation};
use super::lease::WebSocketPoolConnectOutcome;
/// WebSocket 连接池 key。
@@ -67,12 +70,72 @@ impl CodexWebSocketPoolKey {
}
}
const CONTINUATION_TOMBSTONE_TTL: Duration = Duration::from_mins(30);
const MAX_CONTINUATION_TOMBSTONES: usize = 4_096;
#[derive(Default)]
pub(super) struct WebSocketPoolState {
pub(super) slots: HashMap<CodexWebSocketPoolKey, WebSocketPoolSlot>,
continuation_tombstones: VecDeque<WebSocketContinuationTombstone>,
pub(super) shutting_down: bool,
}
struct WebSocketContinuationTombstone {
key: CodexWebSocketPoolKey,
response_id: String,
observation: WebSocketConnectionObservation,
expires_at: Instant,
}
impl WebSocketPoolState {
pub(super) fn remember_continuation_loss(
&mut self,
key: &CodexWebSocketPoolKey,
response_id: Option<&str>,
observation: WebSocketConnectionObservation,
now: Instant,
) {
let Some(response_id) = response_id else {
return;
};
self.prune_continuation_tombstones(now);
self.continuation_tombstones.retain(|tombstone| {
!(tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id)
});
while self.continuation_tombstones.len() >= MAX_CONTINUATION_TOMBSTONES {
self.continuation_tombstones.pop_front();
}
self.continuation_tombstones
.push_back(WebSocketContinuationTombstone {
key: key.clone(),
response_id: response_id.to_owned(),
observation,
expires_at: now + CONTINUATION_TOMBSTONE_TTL,
});
}
pub(super) fn continuation_loss(
&mut self,
key: &CodexWebSocketPoolKey,
response_id: &str,
now: Instant,
) -> Option<WebSocketConnectionObservation> {
self.prune_continuation_tombstones(now);
self.continuation_tombstones
.iter()
.rev()
.find(|tombstone| {
tombstone.key.same_logical_connection(key) && tombstone.response_id == response_id
})
.map(|tombstone| tombstone.observation.clone())
}
fn prune_continuation_tombstones(&mut self, now: Instant) {
self.continuation_tombstones
.retain(|tombstone| tombstone.expires_at > now);
}
}
#[derive(Clone)]
pub(crate) struct CodexWebSocketConnectionMetadata {
pub(crate) turn_state: Option<String>,
@@ -174,7 +174,23 @@ impl CodexWebSocketPool {
.collect::<Vec<_>>();
for key in keys {
match state.slots.remove(&key) {
Some(WebSocketPoolSlot::Idle { connection, .. }) => close.push(*connection),
Some(WebSocketPoolSlot::Idle { connection, .. }) => {
let observation = if connection.websocket.is_closed() {
connection.websocket.observation()
} else {
connection
.websocket
.observation()
.with_exit_reason("max_age_expired")
};
state.remember_continuation_loss(
&key,
connection.continuation.latest_response_id(),
observation,
now,
);
close.push(*connection);
}
Some(WebSocketPoolSlot::Busy(reservation)) => {
tracing::warn!(
account_id = key.account_id(),
@@ -89,6 +89,7 @@ pub(crate) enum PumpExitReason {
LocalClose,
OutboundTransportError {
message: String,
metric_reason: &'static str,
},
UpstreamCloseFrame {
close: Option<CodexWebSocketCloseError>,
@@ -96,6 +97,7 @@ pub(crate) enum PumpExitReason {
UpstreamEof,
InboundTransportError {
message: String,
metric_reason: &'static str,
},
PongTimeout {
timeout: Duration,
@@ -105,6 +107,7 @@ pub(crate) enum PumpExitReason {
},
KeepaliveTransportError {
message: String,
metric_reason: &'static str,
},
MessageReceiverClosed,
}
@@ -127,9 +130,9 @@ impl PumpExitReason {
pub(crate) fn detail(&self) -> Option<String> {
match self {
Self::OutboundTransportError { message }
| Self::InboundTransportError { message }
| Self::KeepaliveTransportError { message } => Some(message.clone()),
Self::OutboundTransportError { message, .. }
| Self::InboundTransportError { message, .. }
| Self::KeepaliveTransportError { message, .. } => Some(message.clone()),
Self::UpstreamCloseFrame { close } => close.as_ref().map(ToString::to_string),
Self::PongTimeout { timeout } | Self::LivenessTimeout { timeout } => {
Some(format!("{timeout:?}"))
@@ -141,6 +144,27 @@ impl PumpExitReason {
}
}
pub(crate) fn metric_reason(&self) -> &'static str {
match self {
Self::CommandChannelClosed => "command_channel_closed",
Self::LocalClose => "local_close",
Self::OutboundTransportError { metric_reason, .. }
| Self::InboundTransportError { metric_reason, .. }
| Self::KeepaliveTransportError { metric_reason, .. } => metric_reason,
Self::UpstreamCloseFrame { close } => {
if close.as_ref().and_then(CodexWebSocketCloseError::code) == Some(1000) {
"normal_close"
} else {
"upstream_close"
}
}
Self::UpstreamEof => "eof",
Self::PongTimeout { .. } => "pong_timeout",
Self::LivenessTimeout { .. } => "liveness_timeout",
Self::MessageReceiverClosed => "message_receiver_closed",
}
}
pub(crate) fn upstream_close(&self) -> Option<&CodexWebSocketCloseError> {
match self {
Self::UpstreamCloseFrame { close: Some(close) } => Some(close),
@@ -183,13 +207,71 @@ struct PendingPing {
deadline: Instant,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WebSocketConnectionObservation {
connection_id: Uuid,
exit_reason: String,
age_ms: u64,
idle_ms: u64,
}
impl WebSocketConnectionObservation {
pub(crate) fn with_exit_reason(mut self, exit_reason: &'static str) -> Self {
self.exit_reason = exit_reason.to_owned();
self
}
pub const fn connection_id(&self) -> Uuid {
self.connection_id
}
pub fn exit_reason(&self) -> &str {
&self.exit_reason
}
pub const fn age_ms(&self) -> u64 {
self.age_ms
}
pub const fn idle_ms(&self) -> u64 {
self.idle_ms
}
}
struct PumpLifecycleState {
opened_at: Instant,
last_activity: Instant,
closed_at: Option<Instant>,
exit_reason: Option<PumpExitReason>,
}
impl PumpLifecycleState {
fn observation(
&self,
connection_id: Uuid,
observed_at: Instant,
) -> WebSocketConnectionObservation {
let effective_end = self.closed_at.unwrap_or(observed_at);
WebSocketConnectionObservation {
connection_id,
exit_reason: self
.exit_reason
.as_ref()
.map_or("running", PumpExitReason::metric_reason)
.to_owned(),
age_ms: duration_millis(effective_end.duration_since(self.opened_at)),
idle_ms: duration_millis(effective_end.duration_since(self.last_activity)),
}
}
}
/// 由后台 pump 任务托管的 WebSocket 连接句柄。
pub(crate) struct PumpedWebSocket {
connection_id: Uuid,
tx_command: mpsc::Sender<PumpCommand>,
rx_message: mpsc::Receiver<Result<Message, tungstenite::Error>>,
closed: Arc<AtomicBool>,
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
lifecycle: Arc<Mutex<PumpLifecycleState>>,
pump: Option<JoinHandle<()>>,
}
@@ -212,10 +294,16 @@ impl PumpedWebSocket {
let (tx_command, rx_command) = mpsc::channel::<PumpCommand>(PUMP_COMMAND_BUFFER);
let (tx_message, rx_message) = mpsc::channel(PUMP_MESSAGE_BUFFER);
let connection_id = Uuid::new_v4();
let opened_at = Instant::now();
let closed = Arc::new(AtomicBool::new(false));
let closed_for_task = Arc::clone(&closed);
let exit_reason = Arc::new(Mutex::new(None));
let exit_reason_for_task = Arc::clone(&exit_reason);
let lifecycle = Arc::new(Mutex::new(PumpLifecycleState {
opened_at,
last_activity: opened_at,
closed_at: None,
exit_reason: None,
}));
let lifecycle_for_task = Arc::clone(&lifecycle);
let pump = tokio::spawn(async move {
pump_loop(
connection_id,
@@ -225,7 +313,7 @@ impl PumpedWebSocket {
keepalive,
context,
closed_for_task,
exit_reason_for_task,
lifecycle_for_task,
)
.await;
});
@@ -234,7 +322,7 @@ impl PumpedWebSocket {
tx_command,
rx_message,
closed,
exit_reason,
lifecycle,
pump: Some(pump),
}
}
@@ -272,12 +360,20 @@ impl PumpedWebSocket {
}
pub(crate) fn exit_reason(&self) -> Option<PumpExitReason> {
self.exit_reason
self.lifecycle
.lock()
.expect("WebSocket pump exit reason lock poisoned")
.expect("WebSocket pump lifecycle lock poisoned")
.exit_reason
.clone()
}
pub(crate) fn observation(&self) -> WebSocketConnectionObservation {
self.lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.observation(self.connection_id, Instant::now())
}
/// 主动关闭连接(best-effort 发送 Close 帧)。
pub(crate) async fn close(&self) {
if !self.is_closed() {
@@ -306,9 +402,12 @@ async fn pump_loop(
keepalive: PumpKeepalive,
context: PumpLogContext,
closed: Arc<AtomicBool>,
exit_reason: Arc<Mutex<Option<PumpExitReason>>>,
lifecycle: Arc<Mutex<PumpLifecycleState>>,
) {
let mut last_activity = Instant::now();
let mut last_activity = lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.last_activity;
let ping_interval = keepalive.ping_interval.filter(|d| !d.is_zero());
let ping_timeout = keepalive.ping_timeout.filter(|d| !d.is_zero());
let liveness_timeout = keepalive.liveness_timeout.filter(|d| !d.is_zero());
@@ -341,6 +440,7 @@ async fn pump_loop(
}
// 背压期间无法读取 socket,不能把这段本地下游阻塞误算成上游静默。
last_activity = Instant::now();
record_activity(&lifecycle, last_activity);
}
command = rx_command.recv() => {
let Some(command) = command else {
@@ -370,12 +470,14 @@ async fn pump_loop(
None => break 'pump PumpExitReason::UpstreamEof,
Some(Ok(message)) => {
last_activity = Instant::now();
record_activity(&lifecycle, last_activity);
observe_inbound(connection_id, &mut pending_ping, &message);
match message {
Message::Ping(payload) => {
if let Err(error) = inner.send(Message::Pong(payload)).await {
break 'pump PumpExitReason::KeepaliveTransportError {
message: error.to_string(),
metric_reason: transport_metric_reason(&error),
};
}
}
@@ -410,6 +512,7 @@ async fn pump_loop(
Some(Err(err)) => {
let terminal_reason = PumpExitReason::InboundTransportError {
message: err.to_string(),
metric_reason: transport_metric_reason(&err),
};
if let Some(reason) = enqueue_inbound(
&tx_message,
@@ -452,10 +555,22 @@ async fn pump_loop(
};
closed.store(true, Ordering::Release);
*exit_reason
.lock()
.expect("WebSocket pump exit reason lock poisoned") = Some(reason.clone());
log_pump_exit(connection_id, &context, &reason, backpressure_events);
let observation = {
let closed_at = Instant::now();
let mut lifecycle = lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned");
lifecycle.closed_at = Some(closed_at);
lifecycle.exit_reason = Some(reason.clone());
lifecycle.observation(connection_id, closed_at)
};
log_pump_exit(
connection_id,
&context,
&reason,
&observation,
backpressure_events,
);
if reason.should_send_close() {
let _ = inner.send(Message::Close(None)).await;
@@ -492,10 +607,16 @@ async fn handle_command(inner: &mut RawWsStream, command: PumpCommand) -> Option
let PumpCommand::Send { message, ack } = command;
let is_close = matches!(message, Message::Close(_));
let result = inner.send(message).await;
let transport_error = result.as_ref().err().map(ToString::to_string);
let transport_error = result
.as_ref()
.err()
.map(|error| (error.to_string(), transport_metric_reason(error)));
let _ = ack.send(result);
match transport_error {
Some(message) => Some(PumpExitReason::OutboundTransportError { message }),
Some((message, metric_reason)) => Some(PumpExitReason::OutboundTransportError {
message,
metric_reason,
}),
None if is_close => Some(PumpExitReason::LocalClose),
None => None,
}
@@ -517,6 +638,34 @@ fn observe_inbound(connection_id: Uuid, pending_ping: &mut Option<PendingPing>,
*pending_ping = None;
}
fn record_activity(lifecycle: &Mutex<PumpLifecycleState>, observed_at: Instant) {
lifecycle
.lock()
.expect("WebSocket pump lifecycle lock poisoned")
.last_activity = observed_at;
}
pub(crate) fn transport_metric_reason(error: &tungstenite::Error) -> &'static str {
match error {
tungstenite::Error::Io(error) => match error.kind() {
std::io::ErrorKind::ConnectionReset => "tcp_reset",
std::io::ErrorKind::ConnectionAborted => "connection_aborted",
std::io::ErrorKind::BrokenPipe => "broken_pipe",
std::io::ErrorKind::UnexpectedEof => "unexpected_eof",
std::io::ErrorKind::TimedOut => "transport_timeout",
_ => "transport_error",
},
tungstenite::Error::Protocol(
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
) => "tcp_reset",
_ => "transport_error",
}
}
fn duration_millis(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
async fn send_keepalive_ping(
inner: &mut RawWsStream,
sequence: &mut u64,
@@ -529,6 +678,7 @@ async fn send_keepalive_ping(
.await
.map_err(|error| PumpExitReason::KeepaliveTransportError {
message: error.to_string(),
metric_reason: transport_metric_reason(&error),
})?;
Ok(ping_timeout.map(|timeout| PendingPing {
payload,
@@ -540,6 +690,7 @@ fn log_pump_exit(
connection_id: Uuid,
context: &PumpLogContext,
reason: &PumpExitReason,
observation: &WebSocketConnectionObservation,
backpressure_events: u64,
) {
let detail = reason.detail().unwrap_or_default();
@@ -551,6 +702,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
backpressure_events,
"Responses WebSocket pump stopped unexpectedly"
@@ -565,6 +719,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code),
upstream_error_raw,
@@ -578,6 +735,9 @@ fn log_pump_exit(
account_id = %account_id,
conversation_id_hash = %conversation_id_hash,
pump_exit_reason = reason.as_str(),
connection_exit_reason = observation.exit_reason(),
connection_age_ms = observation.age_ms(),
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
backpressure_events,
"Responses WebSocket pump stopped"
@@ -336,7 +336,10 @@ async fn openai_core_provider_projects_codex_request_observation_without_routing
)]));
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = bundle.core_provider().request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_admin_observation").expect("client key");
let observation = bundle
.core_provider()
.request_observation(&operation, &client_key_id);
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
assert_eq!(observation.subagent_kind.as_deref(), Some("review"));
@@ -1295,7 +1295,7 @@ async fn websocket_close_after_delivery_preserves_details_without_enabling_stick
}
#[tokio::test]
async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_session() {
async fn ambiguous_websocket_close_only_sticks_new_chains_in_the_current_session() {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_websocket_close").await;
let listener = TcpListener::bind("127.0.0.1:0")
@@ -1306,6 +1306,26 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
listener.local_addr().expect("listener address")
);
let server = tokio::spawn(async move {
let completed_response = |response_id: &str| {
Message::Text(
json!({
"type": "response.completed",
"response": {
"id": response_id,
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
)
};
let (stream, _) = listener
.accept()
.await
@@ -1371,6 +1391,31 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
.await
.expect("write sticky HTTP response");
let (stream, _) = listener
.accept()
.await
.expect("accept same-session continuation WebSocket connection");
let mut websocket = accept_codex_test_websocket(stream).await;
let request = websocket
.next()
.await
.expect("same-session continuation request")
.expect("valid same-session continuation request");
let payload: Value = serde_json::from_str(
request
.to_text()
.expect("same-session continuation request text"),
)
.expect("same-session continuation request JSON");
assert_eq!(
payload.get("previous_response_id"),
Some(&json!("external-previous-response"))
);
websocket
.send(completed_response("resp_external_continuation"))
.await
.expect("complete same-session continuation request");
let (stream, _) = listener
.accept()
.await
@@ -1382,24 +1427,7 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
.expect("other-session WebSocket request")
.expect("valid other-session WebSocket request");
websocket
.send(Message::Text(
json!({
"type": "response.completed",
"response": {
"id": "resp_other_session",
"model": "gpt-5.4",
"status": "completed",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2
}
}
})
.to_string()
.into(),
))
.send(completed_response("resp_other_session"))
.await
.expect("complete other-session WebSocket request");
});
@@ -1472,6 +1500,11 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
raw_close.get("last_event_type"),
Some(&json!("response.output_item.added"))
);
let connection = error
.connection_observation()
.expect("normal close should retain its connection lifecycle observation");
assert_eq!(connection.exit_reason(), "normal_close");
assert!(connection.age_ms() >= connection.idle_ms());
let second_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
@@ -1491,6 +1524,27 @@ async fn ambiguous_websocket_close_is_terminal_and_sticky_only_for_the_current_s
}
drop(sticky_stream);
let continuation_operation = Operation::Generate(generate_with_session_context(
"sticky-websocket-session",
Some("thread-continuation"),
None,
));
let mut continuation_stream = provider
.execute(
planned_request("openai", continuation_operation),
external_continuation_context("req_sticky_external_continuation"),
)
.await
.expect("sticky state must not force a continuation onto HTTP");
assert_eq!(
continuation_stream.metadata().transport().as_str(),
"websocket"
);
while let Some(event) = continuation_stream.next().await {
event.expect("same-session continuation WebSocket response");
}
drop(continuation_stream);
let other_operation = Operation::Generate(generate_with_session_context(
"other-websocket-session",
Some("thread-first"),
@@ -2044,7 +2098,7 @@ async fn cross_account_continuation_should_require_client_replay_without_an_upst
detail.error_type(),
),
(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
Some(ContinuationFailure::HistoryUnavailable),
Some("previous_response_not_found"),
@@ -2094,6 +2148,10 @@ async fn missing_affinity_continuation_should_require_client_replay_without_an_u
.await
.expect("captured upstream requests");
assert_eq!(
error.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
error
.client_visible_upstream_error()
@@ -3396,6 +3454,10 @@ async fn same_account_previous_response_not_found_should_remain_client_visible()
}
};
assert_eq!(
failure.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
failure.continuation_failure(),
Some(ContinuationFailure::HistoryUnavailable)
@@ -3625,6 +3687,10 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
None => panic!("competing exact continuation must fail while its socket is busy"),
}
};
assert_eq!(
exact_error.kind(),
ProviderErrorKind::ContinuationRecoveryRequired
);
assert_eq!(
exact_error.continuation_failure(),
Some(ContinuationFailure::Busy)
@@ -3673,7 +3739,7 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
detail.message(),
),
(
ProviderErrorKind::InvalidRequest,
ProviderErrorKind::ContinuationRecoveryRequired,
UpstreamSendState::NotSent,
Some(ContinuationFailure::HistoryUnavailable),
Some(ContinuationRecoveryDisposition::ClientReplayRequired),
@@ -4085,7 +4151,8 @@ fn request_observation_reads_openai_metadata_without_changing_the_operation() {
let generation = GenerateRequest::from_protocol_payload(payload);
let operation = Operation::Generate(generation);
let observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let observation = provider(&store).request_observation(&operation, &client_key_id);
assert_eq!(observation.request_kind.as_deref(), Some("review"));
assert_eq!(observation.subagent_kind.as_deref(), Some("worker"));
@@ -4103,7 +4170,8 @@ fn request_observation_preserves_the_raw_reasoning_effort() {
.expect("protocol payload");
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let observation = provider(&store).request_observation(&operation, &client_key_id);
assert_eq!(
observation.reasoning_effort.as_deref(),
@@ -4133,7 +4201,8 @@ fn request_observation_ignores_future_session_state_without_rewriting_the_protoc
);
let operation = Operation::Generate(generation);
let _observation = provider(&store).request_observation(&operation);
let client_key_id = ClientApiKeyId::new("key_openai_observation").expect("client key");
let _observation = provider(&store).request_observation(&operation, &client_key_id);
let Operation::Generate(generation) = &operation else {
panic!("operation should remain a generate request");
@@ -0,0 +1,200 @@
use std::sync::Arc;
use futures::{StreamExt, stream};
use gateway_core::account::{AccountFeedbackStats, ProviderAccountId};
use gateway_core::engine::provider::{EventStream, ProviderCallMetadata, ProviderStream};
use gateway_core::error::{
ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind,
};
use gateway_core::routing::{ProviderKind, UpstreamModelId};
use gateway_core::upstream::{UpstreamSendState, UpstreamTransport};
use provider_openai::openai_failure_affects_account_score;
fn sent_error(kind: ProviderErrorKind, code: Option<&str>) -> ProviderError {
let error = ProviderError::new(kind, UpstreamSendState::Sent);
match code {
Some(code) => error.with_upstream_code(OpaqueUpstreamValue::new(code)),
None => error,
}
}
fn deliver_error_with_openai_feedback(
feedback: &Arc<AccountFeedbackStats>,
provider: &ProviderKind,
account: &ProviderAccountId,
error: ProviderError,
) {
let metadata = ProviderCallMetadata::new(
provider.clone(),
UpstreamModelId::new("gpt-5").expect("model"),
account.clone(),
UpstreamTransport::new("http_sse").expect("transport"),
);
let events: EventStream = Box::pin(stream::iter([Err(error)]));
let mut provider_stream = ProviderStream::new(metadata, events, ())
.with_filtered_account_feedback(Arc::clone(feedback), openai_failure_affects_account_score);
futures::executor::block_on(async {
assert!(
provider_stream
.next()
.await
.is_some_and(|event| event.is_err())
);
});
}
#[test]
fn account_score_failure_filter_should_include_server_overload_as_a_regular_reason() {
let error = sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded"));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_accept_the_closed_reason_list() {
for code in [
"server_is_overloaded",
"slow_down",
"rate_limit_exceeded",
"rate_limit_error",
"server_error",
"service_unavailable_error",
] {
let error = sent_error(ProviderErrorKind::Unavailable, Some(code));
assert!(
openai_failure_affects_account_score(&error),
"allowlisted code was rejected: {code}"
);
}
}
#[test]
fn account_score_failure_filter_should_normalize_case_and_whitespace() {
let error = sent_error(ProviderErrorKind::Unavailable, Some(" SERVER_ERROR "));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_use_a_structured_type_when_code_is_absent() {
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"upstream unavailable",
None,
Some("SERVICE_UNAVAILABLE_ERROR".to_owned()),
));
assert!(openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_prefer_a_structured_code_over_type() {
let error = ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
"bad request",
Some("invalid_request".to_owned()),
Some("server_error".to_owned()),
));
assert!(!openai_failure_affects_account_score(&error));
}
#[test]
fn account_score_failure_filter_should_reject_client_and_unknown_failures() {
for error in [
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
sent_error(
ProviderErrorKind::ContinuationRecoveryRequired,
Some("previous_response_not_found"),
),
sent_error(ProviderErrorKind::Unsupported, Some("unsupported")),
sent_error(ProviderErrorKind::Unauthorized, Some("token_expired")),
sent_error(ProviderErrorKind::QuotaExhausted, Some("quota_exhausted")),
sent_error(ProviderErrorKind::RateLimited, Some("rate_limit_reached")),
sent_error(
ProviderErrorKind::Unavailable,
Some("internal_server_error"),
),
sent_error(ProviderErrorKind::Unavailable, Some("service_unavailable")),
sent_error(
ProviderErrorKind::Transport,
Some("upstream_transport_error"),
),
sent_error(ProviderErrorKind::Timeout, Some("first_output_timeout")),
sent_error(
ProviderErrorKind::Protocol,
Some("upstream_stream_truncated"),
),
sent_error(
ProviderErrorKind::Unavailable,
Some("upstream_empty_response"),
),
sent_error(ProviderErrorKind::Unavailable, Some("new_unknown_reason")),
sent_error(ProviderErrorKind::Transport, Some("new_unknown_reason")),
sent_error(ProviderErrorKind::Transport, Some("")),
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_status(503),
] {
assert!(
!openai_failure_affects_account_score(&error),
"non-allowlisted failure affected the account score"
);
}
}
#[test]
fn account_score_failure_filter_should_reject_internal_kinds_without_a_reason() {
for kind in [
ProviderErrorKind::Transport,
ProviderErrorKind::Timeout,
ProviderErrorKind::Protocol,
] {
let error = sent_error(kind, None);
assert!(
!openai_failure_affects_account_score(&error),
"unlisted internal failure affected the score: {}",
kind.as_str()
);
}
}
#[test]
fn openai_feedback_should_not_amplify_repeated_client_configuration_errors() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_usable_account").expect("account");
for _ in 0..100 {
deliver_error_with_openai_feedback(
&feedback,
&provider,
&account,
sent_error(ProviderErrorKind::InvalidRequest, Some("invalid_request")),
);
}
assert_eq!(
feedback.scheduling_signals(&provider, &account),
(None, None)
);
}
#[test]
fn openai_feedback_should_score_server_overload_as_one_regular_failure() {
let feedback = Arc::new(AccountFeedbackStats::default());
let provider = ProviderKind::new("openai").expect("provider");
let account = ProviderAccountId::new("acct_overloaded").expect("account");
deliver_error_with_openai_feedback(
&feedback,
&provider,
&account,
sent_error(ProviderErrorKind::Unavailable, Some("server_is_overloaded")),
);
assert_eq!(
feedback.scheduling_signals(&provider, &account).0,
Some(2_000)
);
}
@@ -1 +1,2 @@
mod contract;
mod failure;
@@ -837,6 +837,40 @@ async fn missing_exact_socket_should_fail_without_opening_a_connection() {
);
}
#[tokio::test]
async fn connection_local_continuation_without_a_pool_should_not_open_http_or_websocket() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let backend = CodexBackendClient::new(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{addr}"),
test_wire_profile(),
);
let mut request = new_chain_request("conversation-exact-no-pool");
request.set_previous_response_id(Some("resp_no_pool".to_owned()));
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let error = backend
.create_response(
&request,
request_context("req_exact_no_pool", Some("chatgpt-account")),
)
.await
.expect_err("connection-local continuation requires its owning pool");
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::PoolUnavailable
})
);
assert!(
timeout(Duration::from_millis(50), listener.accept())
.await
.is_err()
);
}
#[tokio::test]
async fn connection_local_continuation_should_fail_after_its_live_socket_disappears() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -852,7 +886,7 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
))
.await
.unwrap();
websocket.close(None).await.unwrap();
drop(websocket);
closed_tx.send(()).unwrap();
});
let backend = CodexBackendClient::new(
@@ -883,12 +917,18 @@ async fn connection_local_continuation_should_fail_after_its_live_socket_disappe
)
.await
.expect_err("a disappeared live socket cannot be reconstructed from the database");
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::ContinuationUnavailable {
reason: PreviousResponseUnavailableReason::FreshConnectionRequired
})
let CodexClientError::WebSocket(error) = error else {
panic!("disappeared connection should remain a typed WebSocket error");
};
assert_eq!(
error.continuation_unavailable_reason(),
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
);
let observation = error
.connection_observation()
.expect("disappeared connection should retain its lifecycle observation");
assert_eq!(observation.exit_reason(), "tcp_reset");
assert!(observation.age_ms() >= observation.idle_ms());
}
#[tokio::test]
@@ -1114,9 +1154,9 @@ async fn websocket_failure_after_first_delivery_should_not_open_http_fallback()
else {
panic!("expected typed post-delivery WebSocket failure");
};
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = *source else {
panic!("expected upstream close frame source");
};
let close = source
.close_before_terminal()
.expect("expected upstream close frame source");
assert!(
close.connection_id().is_some(),
"close failures must retain their WebSocket connection correlation"
@@ -273,6 +273,16 @@ fn transport_should_prefer_websocket_when_requested_without_history() {
assert!(transport_requirement(&request).allows_pre_send_http_fallback());
}
#[test]
fn unspecified_transport_without_history_should_still_be_a_new_chain() {
let request = codex_request("gpt-test", "be brief", Vec::new());
assert_eq!(
transport_requirement(&request),
TransportRequirement::NewChain
);
}
#[test]
fn new_chain_should_allow_pre_delivery_http_fallback() {
let mut request = codex_request("gpt-test", "be brief", Vec::new());
@@ -284,6 +294,25 @@ fn new_chain_should_allow_pre_delivery_http_fallback() {
);
}
#[test]
fn sticky_http_fallback_should_only_apply_to_new_chains() {
assert!(TransportRequirement::NewChain.allows_sticky_http_fallback());
for requirement in [
TransportRequirement::HttpRequired,
TransportRequirement::ExplicitWebSocketWarmup,
TransportRequirement::ExactWebSocketContinuation,
TransportRequirement::PersistedContinuation,
TransportRequirement::ExternalUnknown,
] {
assert!(
!requirement.allows_sticky_http_fallback(),
"sticky HTTP must not override {}",
requirement.as_str()
);
}
}
#[test]
fn transport_should_mark_unknown_previous_response_as_external() {
let mut request = codex_request("gpt-test", "be brief", Vec::new());
@@ -647,7 +647,7 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
.send(Message::Binary(b"unexpected-binary".to_vec().into()))
.await
.unwrap();
websocket.close(None).await.unwrap();
let _client_close = websocket.next().await;
});
let mut request = codex_request("gpt-test", "be brief", Vec::new());
request.set_previous_response_id(Some("resp_binary_previous".to_owned()));
@@ -664,15 +664,30 @@ async fn websocket_execute_response_create_request_should_reject_binary_event()
.expect_err("binary websocket events should be rejected");
server.await.unwrap();
std::assert_matches!(
error,
CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
message,
source: Some(source),
})
if message.contains("unexpected binary websocket event")
&& matches!(*source, CodexWebSocketExchangeError::UnexpectedBinaryEvent)
let CodexClientError::WebSocket(error) = error else {
panic!("binary websocket event should remain a typed WebSocket error");
};
let CodexWebSocketExchangeError::PostSendAmbiguous {
message,
source: Some(source),
} = error
else {
panic!("binary websocket event should remain post-send ambiguous");
};
assert!(message.contains("unexpected binary websocket event"));
assert_eq!(
source
.connection_observation()
.map(|observation| observation.exit_reason()),
Some("unexpected_binary_event")
);
let CodexWebSocketExchangeError::ConnectionObserved { source, .. } = *source else {
panic!("binary websocket event should carry its connection observation");
};
assert!(matches!(
*source,
CodexWebSocketExchangeError::UnexpectedBinaryEvent
));
}
#[tokio::test]
@@ -1560,9 +1575,9 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
else {
panic!("close-before-terminal should preserve its typed source: {error:?}");
};
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = source.as_ref() else {
panic!("post-send ambiguity should wrap a close-before-terminal error: {source:?}");
};
let close = source
.close_before_terminal()
.unwrap_or_else(|| panic!("post-send ambiguity should retain its close: {source:?}"));
assert_eq!(close.last_event_type(), Some("response.output_text.delta"));
std::assert_matches!(
@@ -1,4 +1,5 @@
use super::*;
use provider_openai::transport::websocket::PreviousResponseUnavailableReason;
#[tokio::test]
async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and_conversation() {
@@ -1108,6 +1109,30 @@ async fn websocket_pool_should_replace_idle_connection_after_pong_deadline() {
.await
.expect("server should report the Pong-timeout close");
tokio::time::resume();
let mut continuation = request.clone();
continuation.set_previous_response_id(Some("resp_no_pong_first".to_owned()));
continuation.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
let error = backend
.create_response(
&continuation,
request_context("req_no_pong_continuation", Some("chatgpt-account")),
)
.await
.expect_err("Pong-timeout connection cannot satisfy an exact continuation");
let CodexClientError::WebSocket(error) = error else {
panic!("Pong-timeout continuation should remain a typed WebSocket error");
};
assert_eq!(
error.continuation_unavailable_reason(),
Some(PreviousResponseUnavailableReason::ReusedConnectionLost)
);
assert_eq!(
error
.connection_observation()
.expect("Pong-timeout tombstone should retain its lifecycle observation")
.exit_reason(),
"pong_timeout"
);
let second = backend
.create_response(
&request,
@@ -138,7 +138,11 @@ impl Provider for GrokBuildProvider {
self.catalog.catalog_generation()
}
fn request_observation(&self, operation: &Operation) -> ProviderRequestObservation {
fn request_observation(
&self,
operation: &Operation,
_client_api_key_id: &gateway_core::policy::ClientApiKeyId,
) -> ProviderRequestObservation {
let Operation::Generate(request) = operation else {
return ProviderRequestObservation::default();
};
@@ -153,6 +157,7 @@ impl Provider for GrokBuildProvider {
request_kind: semantics.request_kind,
subagent_kind: semantics.subagent_kind,
compact: semantics.compact,
continuation: Default::default(),
}
}
@@ -1063,8 +1063,12 @@ async fn model_alias_should_be_canonical_before_account_selection() {
async fn request_observation_preserves_raw_xai_reasoning_effort() {
let provider = provider(StubSelector::success(), StubInferenceTransport::success()).await;
let observation =
provider.request_observation(&operation_with_reasoning_effort("future-value"));
let client_key_id =
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
let observation = provider.request_observation(
&operation_with_reasoning_effort("future-value"),
&client_key_id,
);
assert_eq!(
observation.reasoning_effort.as_deref(),
@@ -1093,7 +1097,9 @@ async fn request_observation_preserves_codex_semantics_for_xai() {
)]));
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
let observation = provider.request_observation(&operation);
let client_key_id =
gateway_core::policy::ClientApiKeyId::new("key_xai_observation").expect("client key");
let observation = provider.request_observation(&operation, &client_key_id);
assert_eq!(observation.reasoning_effort.as_deref(), Some("xhigh"));
assert_eq!(observation.request_kind.as_deref(), Some("compaction"));
+1
View File
@@ -2,3 +2,4 @@
93219760c733616e7bba9a8933c207b26752b2ebf7e6edfe1ef42ba227c50a7d 0002_runtime_pressure_observability.sql
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
@@ -0,0 +1,80 @@
alter table model_requests
add column continuation_affinity_hash text,
add column continuation_previous_response_id_hash text,
add column continuation_requested boolean not null default false,
add column continuation_unavailable_reason text,
add column upstream_connection_id text,
add column upstream_connection_exit_reason text,
add column upstream_connection_age_ms bigint,
add column upstream_connection_idle_ms bigint,
add column recovery_request_id text,
add column recovered_at timestamptz,
add column recovery_attempt_count integer not null default 0,
add column recovery_retry_delay_ms bigint,
add column recovery_total_latency_ms bigint,
add constraint model_requests_continuation_hashes_ck check (
(continuation_affinity_hash is null
or continuation_affinity_hash ~ '^[0-9a-f]{64}$')
and (continuation_previous_response_id_hash is null
or continuation_previous_response_id_hash ~ '^[0-9a-f]{64}$')
and (continuation_requested
or continuation_previous_response_id_hash is null)
),
add constraint model_requests_continuation_reason_ck check (
continuation_unavailable_reason is null
or (
octet_length(continuation_unavailable_reason) between 1 and 64
and continuation_unavailable_reason ~ '^[a-z][a-z0-9_]*$'
)
),
add constraint model_requests_upstream_connection_ck check (
(
upstream_connection_id is null
and upstream_connection_exit_reason is null
and upstream_connection_age_ms is null
and upstream_connection_idle_ms is null
)
or (
octet_length(upstream_connection_id) between 1 and 128
and upstream_connection_id !~ '[[:cntrl:]]'
and octet_length(upstream_connection_exit_reason) between 1 and 64
and upstream_connection_exit_reason ~ '^[a-z][a-z0-9_]*$'
and upstream_connection_age_ms >= 0
and upstream_connection_idle_ms between 0 and upstream_connection_age_ms
)
),
add constraint model_requests_recovery_ck check (
recovery_attempt_count >= 0
and (
(
recovered_at is null
and recovery_request_id is null
and recovery_retry_delay_ms is null
and recovery_total_latency_ms is null
)
or (
recovered_at is not null
and recovered_at >= completed_at
and recovery_request_id is not null
and octet_length(recovery_request_id) between 1 and 128
and recovery_attempt_count > 0
and recovery_retry_delay_ms >= 0
and recovery_total_latency_ms >= recovery_retry_delay_ms
)
)
);
create index model_requests_pending_continuation_recovery_idx
on model_requests (
client_api_key_ref,
continuation_affinity_hash,
completed_at desc,
id desc
)
where error_kind = 'continuation_recovery_required'
and recovered_at is null
and continuation_affinity_hash is not null;
create index model_requests_recovery_request_idx
on model_requests (recovery_request_id)
where recovery_request_id is not null;
+13 -4
View File
@@ -204,6 +204,18 @@ export interface OpsErrorMetadata {
component: string
attemptId: string | null
accountLabel: string | null
continuationAffinityHash: string | null
continuationPreviousResponseIdHash: string | null
continuationUnavailableReason: string | null
upstreamConnectionId: string | null
upstreamConnectionExitReason: string | null
upstreamConnectionAgeMs: number | null
upstreamConnectionIdleMs: number | null
recoveryRequestId: string | null
recoveredAt: string | null
recoveryAttemptCount: number
recoveryRetryDelayMs: number | null
recoveryTotalLatencyMs: number | null
}
export interface OpsError {
@@ -420,10 +432,7 @@ interface PageQuery {
type UsagePageQuery = UsageRangeQuery & PageQuery
type OpsErrorPageQuery = UsagePageQuery & {
failureClass?: string
route?: string
}
type OpsErrorPageQuery = UsagePageQuery
interface UsageDetailQuery {
id: string
@@ -77,7 +77,7 @@ const profileOpen = shallowRef(false)
</div>
</div>
<div class="grid max-h-44 min-h-0 gap-3 overflow-y-auto pr-1">
<div class="grid min-h-0 gap-3">
<AccountQuotaPanelEntry
v-for="entry in quotaEntries"
:key="entry.key"
@@ -10,15 +10,19 @@ import AccountUsageWindow from '../AccountUsageWindow/index.vue'
import { resolveAccountUsageWindowPresentation } from '../AccountUsageWindow/presenter'
import AccountQuotaWindowGroup from './WindowGroup.vue'
const props = defineProps<{
const props = withDefaults(defineProps<{
label: string | null
windows: AccountQuotaWindow[]
}>()
showPercentage?: boolean
}>(), {
showPercentage: true,
})
const now = useUiClock()
const grouped = computed(() => props.windows.length > 1 && Boolean(props.label))
const detailHeading = computed(() => props.label)
const detailTitle = computed(() => detailHeading.value ?? props.windows[0]?.labelDisplay ?? '额度详情')
const summaryLabel = computed(() => props.label ?? props.windows[0]?.labelDisplay ?? '额度')
const hasQuotaWindow = computed(() => props.windows.some(window => typeof window.usedPercent === 'number'))
const detailItems = computed(() =>
props.windows.map((window) => {
const view = resolveAccountUsageWindowPresentation({
@@ -82,7 +86,12 @@ function quotaWindowCode(windowSeconds: number | null, role: AccountQuotaWindow[
:aria-expanded="open"
aria-haspopup="dialog"
>
<AccountQuotaWindowGroup v-if="grouped && label" :label="label" :windows="windows" />
<AccountQuotaWindowGroup
v-if="hasQuotaWindow"
:label="summaryLabel"
:windows="windows"
:show-percentage="showPercentage"
/>
<AccountUsageWindow
v-else
:window="windows[0]"
@@ -4,10 +4,13 @@ import type { AccountQuotaWindow } from '../../constants'
import { computed } from 'vue'
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
const props = defineProps<{
const props = withDefaults(defineProps<{
label: string
windows: AccountQuotaWindow[]
}>()
showPercentage?: boolean
}>(), {
showPercentage: true,
})
const windowItems = computed(() => props.windows.map(window => ({
key: window.key,
@@ -16,6 +19,17 @@ const windowItems = computed(() => props.windows.map(window => ({
usedPercentDisplay: window.usedPercentDisplay,
presentation: quotaWindowPresentation(window, '2px'),
})))
const highestUsageWindow = computed(() => props.windows.reduce((highest, window) => {
if (typeof window.usedPercent !== 'number')
return highest
if (!highest || typeof highest.usedPercent !== 'number' || window.usedPercent > highest.usedPercent)
return window
return highest
}, undefined as AccountQuotaWindow | undefined))
const highestUsageDisplay = computed(() => highestUsageWindow.value?.usedPercentDisplay ?? '—')
const highestUsageTextClass = computed(() => highestUsageWindow.value
? quotaWindowPresentation(highestUsageWindow.value, '2px').percentTextClass
: 'text-cp-text-quaternary')
const trackGridStyle = computed(() => ({
gridTemplateColumns: `repeat(${Math.max(windowItems.value.length, 1)}, minmax(0, 1fr))`,
@@ -23,16 +37,25 @@ const trackGridStyle = computed(() => ({
</script>
<template>
<div class="flex min-w-0 flex-col">
<div class="mb-1 min-w-0 truncate text-cp-xs leading-3.5 font-bold text-cp-text-quaternary">
{{ label }}
<div class="grid min-w-0 gap-1.5">
<div class="flex min-w-0 items-baseline justify-between gap-1 text-[10px] leading-3 font-bold">
<span class="min-w-0 truncate text-cp-text-quaternary" :title="label">
{{ label }}
</span>
<strong
v-if="showPercentage"
class="shrink-0 font-mono font-heavy tabular-nums"
:class="highestUsageTextClass"
>
{{ highestUsageDisplay }}
</strong>
</div>
<div class="grid min-w-0 gap-1" :style="trackGridStyle">
<div
v-for="item in windowItems"
:key="item.key"
class="h-0.75 w-full overflow-hidden rounded-full bg-cp-border-secondary"
class="h-1 w-full overflow-hidden rounded-full bg-cp-border-secondary"
role="progressbar"
:aria-label="item.labelTooltip"
aria-valuemin="0"
@@ -41,7 +64,7 @@ const trackGridStyle = computed(() => ({
:aria-valuetext="item.usedPercentDisplay"
>
<div
class="h-full rounded-full transition-[width,background-color] duration-200"
class="h-full rounded-full transition-[width,background-color] duration-200 motion-reduce:transition-none"
:class="item.presentation.barClass"
:style="item.presentation.barStyle"
/>
@@ -4,6 +4,7 @@ import type { AccountRow } from '../../constants'
import { computed } from 'vue'
import { groupedAccountQuotaWindows, visibleSummaryQuotaWindows } from '../../constants'
import AccountUsageWindow from '../AccountUsageWindow/index.vue'
import { quotaWindowPresentation } from '../AccountUsageWindow/presenter'
import AccountQuotaSummaryEntry from './Entry.vue'
const props = defineProps<{
@@ -13,24 +14,76 @@ const props = defineProps<{
const quotaWindows = computed(() => props.account.quota.windows)
const visibleQuotaWindows = computed(() => visibleSummaryQuotaWindows(quotaWindows.value))
const summaryEntries = computed(() => groupedAccountQuotaWindows(visibleQuotaWindows.value))
const summaryClass = computed(
() =>
`box-border grid w-full min-w-0 gap-1 py-1.5 ${
summaryEntries.value.length <= 1
? 'min-h-13 content-center'
: 'min-h-16.5 content-center'
}`,
)
const hasUsage = computed(() => (props.account.usage.requestCount ?? 0) > 0)
const highestUsageWindow = computed(() => visibleQuotaWindows.value.reduce((highest, window) => {
if (typeof window.usedPercent !== 'number')
return highest
if (!highest || typeof highest.usedPercent !== 'number' || window.usedPercent > highest.usedPercent)
return window
return highest
}, undefined as (typeof visibleQuotaWindows.value)[number] | undefined))
const highestUsageDisplay = computed(() => highestUsageWindow.value?.usedPercentDisplay ?? '—')
const highestUsageTextClass = computed(() => highestUsageWindow.value
? quotaWindowPresentation(highestUsageWindow.value, '2px').percentTextClass
: 'text-cp-text-quaternary')
const highestUsageEntry = computed(() => {
const highestWindow = highestUsageWindow.value
if (!highestWindow)
return summaryEntries.value[0]
return summaryEntries.value.find(entry =>
entry.windows.some(window => window.key === highestWindow.key),
) ?? summaryEntries.value[0]
})
const additionalEntryCount = computed(() => Math.max(summaryEntries.value.length - 1, 0))
</script>
<template>
<div :class="summaryClass">
<AccountQuotaSummaryEntry
v-for="entry in summaryEntries"
:key="entry.key"
:label="entry.label"
:windows="entry.windows"
/>
<AccountUsageWindow v-if="summaryEntries.length === 0" variant="compact" />
<div class="box-border grid min-h-16.5 w-full min-w-0 content-center gap-1.5 py-1.5">
<template v-if="summaryEntries.length > 0">
<div
v-if="hasUsage"
class="flex min-w-0 items-baseline justify-between gap-2 leading-none"
>
<span
class="flex min-w-0 items-baseline gap-1 font-mono tabular-nums"
title="当前额度窗口总 Token"
>
<strong class="truncate text-cp-xs font-heavy text-cp-text">
{{ account.usage.totalTokensDisplay }}
</strong>
<span class="shrink-0 text-[9px] font-emphasis tracking-[0.02em] text-cp-text-quaternary">
Tokens
</span>
</span>
<span
class="flex shrink-0 items-baseline gap-1 text-[9px] font-emphasis text-cp-text-quaternary"
title="所有额度窗口中的最高已用比例"
>
<span>最高占用</span>
<strong class="font-mono font-heavy tabular-nums" :class="highestUsageTextClass">
{{ highestUsageDisplay }}
</strong>
</span>
</div>
<div v-if="highestUsageEntry" class="flex min-w-0 items-end gap-2">
<div class="min-w-0 flex-1">
<AccountQuotaSummaryEntry
:label="highestUsageEntry.label"
:windows="highestUsageEntry.windows"
:show-percentage="false"
/>
</div>
<span
v-if="additionalEntryCount > 0"
class="grid h-5 min-w-5 shrink-0 place-items-center rounded-cp bg-cp-fill-quaternary px-1.5 font-mono text-[9px] font-heavy tabular-nums text-cp-text-tertiary"
:title="`另有 ${additionalEntryCount} 个额度组,可展开账号查看`"
>
+{{ additionalEntryCount }}
</span>
</div>
</template>
<AccountUsageWindow v-else variant="compact" />
</div>
</template>
@@ -31,15 +31,15 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</script>
<template>
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:grid-cols-[0.52fr_1.48fr]">
<div>
<div class="mb-3 flex items-baseline justify-between gap-3">
<section class="grid gap-4 rounded-lg bg-cp-bg-container p-4 shadow-cp-tertiary xl:min-h-0 xl:grid-cols-[0.52fr_1.48fr]">
<div class="xl:flex xl:min-h-0 xl:flex-col">
<div class="mb-3 flex shrink-0 items-baseline justify-between gap-3">
<h3 class="m-0 text-cp-lg font-heavy text-cp-text">
Token 结构
</h3>
<span class="text-cp-xs font-emphasis text-cp-text-quaternary">当前额度窗口</span>
</div>
<div class="grid gap-2">
<div class="grid gap-2 xl:min-h-0 xl:flex-1 xl:grid-rows-5">
<div class="flex items-center justify-between rounded-lg bg-cp-green-bg px-3 py-2">
<span class="text-cp-sm font-bold text-cp-green-text-on-bg">输入 Tokens</span>
<strong class="font-mono text-cp text-cp-text">
@@ -73,8 +73,8 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</div>
</div>
<div class="min-w-0 pt-4 xl:pt-0 xl:pl-4">
<div class="mb-3 flex flex-wrap items-center gap-x-4 gap-y-2">
<div class="min-w-0 pt-4 xl:flex xl:min-h-0 xl:flex-col xl:pt-0 xl:pl-4">
<div class="mb-3 flex shrink-0 flex-wrap items-center gap-x-4 gap-y-2">
<h3 class="m-0 shrink-0 text-cp-lg font-heavy text-cp-text">
模型使用排行
</h3>
@@ -100,7 +100,7 @@ const modelUsageColumns = defineTableColumns<AccountModelUsage>([
</div>
</div>
<div class="h-52 min-w-0">
<div class="h-56 min-w-0 xl:h-auto xl:min-h-0 xl:flex-1 xl:basis-0">
<BaseTable
:columns="modelUsageColumns"
:rows="account.usage.models"
+1 -1
View File
@@ -287,7 +287,7 @@ const {
</template>
<template #expanded="{ row }">
<div class="grid gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr]">
<div class="grid items-stretch gap-3 p-4 lg:grid-cols-[1.05fr_2.45fr] xl:min-h-[19.25rem]">
<AccountQuotaPanel
:account="row"
:refreshing="refreshingQuotaAccountIds.has(row.id)"
@@ -2,12 +2,19 @@
import type { OpsError } from '@/api'
import { computed } from 'vue'
import BaseButton from '@/components/base/BaseButton.vue'
import BaseModal from '@/components/base/BaseModal/index.vue'
import { displayValue } from '../utils/detail'
import { presentOpsError } from '../utils/opsErrorPresentation'
import { failureClassText } from '../utils/opsErrorPresentation'
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
interface DetailField {
label: string
value: unknown
mono?: boolean
wrap?: boolean
fullWidth?: boolean
}
const props = defineProps<{
record: OpsError | null
@@ -15,8 +22,6 @@ const props = defineProps<{
const open = defineModel<boolean>({ default: false })
const presentation = computed(() => props.record ? presentOpsError(props.record) : null)
const panelClass = 'min-w-0 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5'
const panelTitleClass = 'm-0 text-cp-sm leading-none font-heavy text-cp-text-secondary'
@@ -25,72 +30,90 @@ const accountLabel = computed(() => props.record?.accountEmail
|| props.record?.metadata.accountLabel
|| props.record?.accountId)
const requestFields = computed(() => [
{ label: '客户端 IP', value: props.record?.clientIp, mono: true },
{ label: 'User-Agent', value: props.record?.userAgent, mono: true, wrap: true },
{ label: '协议', value: props.record?.protocol, mono: true },
{ label: '客户端传输', value: props.record?.clientTransport, mono: true },
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true },
{ label: '请求 ID', value: props.record?.requestId, mono: true },
{ label: '响应 ID', value: props.record?.responseId, mono: true },
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, mono: true },
])
const errorFields = computed(() => visibleFields([
{ label: 'Provider 错误码', value: props.record?.providerErrorCode, mono: true },
{ label: '失败分类', value: failureClassText(props.record?.failureClass) },
{ label: '分类代码', value: props.record?.failureClass, mono: true },
{ label: '发送状态', value: props.record?.upstreamSendState, mono: true },
{ label: '客户端状态', value: props.record?.clientStatusCode, mono: true },
{ label: '上游状态', value: props.record?.upstreamStatusCode, mono: true },
{ label: '耗时(ms)', value: props.record?.latencyMs, mono: true },
]))
const routeFields = computed(() => [
const requestFields = computed(() => visibleFields([
{ label: '账号', value: accountLabel.value, mono: true },
{ label: '时间', value: props.record?.createdAtDisplay, mono: true },
{ label: '请求 ID', value: props.record?.requestId, mono: true },
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true },
{ label: '客户端 IP', value: props.record?.clientIp, mono: true },
{ label: '协议', value: props.record?.protocol, mono: true },
{ label: 'User-Agent', value: props.record?.userAgent, mono: true, wrap: true, fullWidth: true },
]))
const routeFields = computed(() => visibleFields([
{ label: '端点', value: props.record?.route, mono: true },
{ label: 'Provider', value: props.record?.provider, mono: true },
{ label: '认证类型', value: props.record?.authenticationKind, mono: true },
{ label: '账号 ID', value: props.record?.accountId, mono: true },
{ label: '请求模型', value: props.record?.requestedModel, mono: true },
{ label: '上游模型', value: props.record?.upstreamModel, mono: true },
{ label: '记录模型', value: props.record?.model, mono: true },
{ label: '推理强度', value: props.record?.reasoningEffort, mono: true },
{ label: '推理预设', value: props.record?.reasoningPreset, mono: true },
{ label: '客户端传输', value: props.record?.clientTransport, mono: true },
{ label: '上游传输', value: props.record?.transport, mono: true },
{ label: '服务档位', value: props.record?.serviceTier, mono: true },
{ label: '请求类型', value: props.record?.requestKind, mono: true },
{ label: '子代理类型', value: props.record?.subagentKind, mono: true },
{ label: '压缩请求', value: booleanDisplay(props.record?.compact) },
{ label: '服务档位', value: props.record?.serviceTier, mono: true },
{ label: '上游传输', value: props.record?.transport, mono: true },
{ label: '压缩请求', value: props.record?.compact, mono: true },
]))
const eventFields = computed(() => visibleFields([
{ label: '记录来源', value: props.record?.metadata.source, mono: true },
{ label: '组件', value: props.record?.metadata.component || props.record?.kind, mono: true },
{ label: '事件类型', value: props.record?.kind, mono: true },
{ label: '操作', value: props.record?.operation, mono: true },
{ label: '尝试序号', value: props.record?.attemptIndex },
])
const diagnosticFields = computed(() => [
{ label: '账号', value: accountLabel.value, mono: true },
{ label: '时间', value: props.record?.createdAtDisplay, mono: true },
{ label: '记录来源', value: presentation.value?.sourceLabel },
{ label: '组件', value: presentation.value?.componentLabel },
{ label: '操作/触发', value: props.record?.operation, mono: true },
{ label: '失败分类', value: presentation.value?.failureClassLabel },
{ label: '发送状态', value: props.record?.upstreamSendState, mono: true },
{ label: '平台/类型', value: providerKindLabel(props.record) },
{ label: '账号 ID', value: props.record?.accountId, mono: true },
{ label: 'Provider 错误码', value: props.record?.providerErrorCode, mono: true },
{ label: '聚合次数', value: props.record?.occurrenceCount },
{ label: '耗时', value: latencyDisplay(props.record?.latencyMs), mono: true },
])
{ label: '响应 ID', value: props.record?.responseId, mono: true },
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, mono: true },
]))
const metadataText = computed(() => {
const metadata = props.record?.metadata
if (!metadata || (typeof metadata === 'object' && Object.keys(metadata).length === 0))
return ''
return JSON.stringify(metadata, null, 2)
})
const continuationFields = computed(() => visibleFields([
{
label: '会话关联 Hash',
value: props.record?.metadata.continuationAffinityHash,
mono: true,
wrap: true,
fullWidth: true,
},
{
label: 'Previous Response Hash',
value: props.record?.metadata.continuationPreviousResponseIdHash,
mono: true,
wrap: true,
fullWidth: true,
},
{ label: '续接不可用原因', value: props.record?.metadata.continuationUnavailableReason, mono: true },
{ label: '上游连接 ID', value: props.record?.metadata.upstreamConnectionId, mono: true },
{ label: '连接退出原因', value: props.record?.metadata.upstreamConnectionExitReason, mono: true },
{ label: '连接存活(ms)', value: props.record?.metadata.upstreamConnectionAgeMs, mono: true },
{ label: '最后空闲(ms)', value: props.record?.metadata.upstreamConnectionIdleMs, mono: true },
{ label: '恢复请求 ID', value: props.record?.metadata.recoveryRequestId, mono: true },
{ label: '恢复时间', value: props.record?.metadata.recoveredAt, mono: true },
{
label: '恢复尝试次数',
value: props.record?.metadata.recoveryAttemptCount
? props.record.metadata.recoveryAttemptCount
: null,
mono: true,
},
{ label: '客户端重连(ms)', value: props.record?.metadata.recoveryRetryDelayMs, mono: true },
{ label: '恢复总延迟(ms)', value: props.record?.metadata.recoveryTotalLatencyMs, mono: true },
]))
function latencyDisplay(value: unknown) {
return typeof value === 'number' ? `${value} ms` : '—'
}
function booleanDisplay(value: boolean | null | undefined) {
if (value === true)
return '是'
if (value === false)
return '否'
return '—'
}
function providerKindLabel(record: OpsError | null) {
if (!record?.provider)
return '—'
return record.authenticationKind
? `${record.provider} · ${record.authenticationKind}`
: record.provider
function visibleFields(items: DetailField[]) {
return items.filter(({ value }) => value !== null && value !== undefined && value !== '')
}
</script>
@@ -98,68 +121,64 @@ function providerKindLabel(record: OpsError | null) {
<BaseModal
v-model="open"
title="错误明细"
description="请求来源、链路上下文与上游错误原文"
description="上游错误与请求上下文"
tone="danger"
size="xl"
>
<template v-if="record">
<section class="rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5">
<div class="flex flex-wrap items-center gap-3">
<UsageStatusCodeBadge :status-code="record.clientStatusCode" />
<span class="text-cp-sm font-bold text-cp-text-secondary">
客户端 {{ displayValue(record.clientStatusCode) }}
</span>
<span class="text-cp-sm font-bold text-cp-text-secondary">
上游 {{ displayValue(record.upstreamStatusCode) }}
</span>
<section :class="panelClass">
<h3 :class="panelTitleClass">
错误
</h3>
<UsageDetailFieldGrid :items="errorFields" />
<div v-if="record.message" class="mt-3">
<UsageDetailCodePanel title="错误信息" max-height="220px" :content="record.message" />
</div>
<p class="mt-3 mb-0 text-cp leading-relaxed font-bold text-cp-text">
{{ displayValue(presentation?.summary) }}
</p>
</section>
<section v-if="record.rawUpstreamError" class="mt-3" :class="panelClass">
<UsageDetailCodePanel
title="上游返回原文"
max-height="360px"
:content="record.rawUpstreamError"
/>
</section>
<div class="mt-3 grid min-w-0 gap-3 lg:grid-cols-2">
<section :class="panelClass">
<section v-if="requestFields.length" :class="panelClass">
<h3 :class="panelTitleClass">
请求追踪
请求与客户端
</h3>
<UsageDetailFieldGrid :items="requestFields" />
</section>
<section :class="panelClass">
<section v-if="routeFields.length" :class="panelClass">
<h3 :class="panelTitleClass">
模型与链路
路由与模型
</h3>
<UsageDetailFieldGrid :items="routeFields" />
</section>
</div>
<section class="mt-3" :class="panelClass">
<section v-if="eventFields.length" class="mt-3" :class="panelClass">
<h3 :class="panelTitleClass">
错误事件
事件记录
</h3>
<UsageDetailFieldGrid :items="diagnosticFields" />
<UsageDetailFieldGrid :items="eventFields" />
</section>
<section class="mt-3" :class="panelClass">
<UsageDetailCodePanel title="诊断摘要" max-height="260px" :content="record.message" />
<section v-if="continuationFields.length" class="mt-3" :class="panelClass">
<h3 :class="panelTitleClass">
会话续接与物理连接
</h3>
<UsageDetailFieldGrid :items="continuationFields" />
</section>
</template>
<section
v-if="record.rawUpstreamError"
class="mt-3"
:class="panelClass"
>
<UsageDetailCodePanel title="上游返回原文" max-height="360px" :content="record.rawUpstreamError" />
</section>
<section
v-if="metadataText"
class="mt-3"
:class="panelClass"
>
<UsageDetailCodePanel title="Metadata" max-height="260px" :content="metadataText" />
</section>
<template #footer>
<BaseButton variant="primary" @click="open = false">
关闭
</BaseButton>
</template>
</BaseModal>
</template>
@@ -11,10 +11,8 @@ import BaseTable from '@/components/base/BaseTable/index.vue'
import ProviderIconGroup from '@/components/ProviderIconGroup.vue'
import { useOpsErrorsTable } from '../composables/useOpsErrorsTable'
import { opsErrorColumns } from '../constants'
import { presentOpsError } from '../utils/opsErrorPresentation'
import { opsErrorSummary } from '../utils/opsErrorPresentation'
import OpsErrorDetailModal from './OpsErrorDetailModal.vue'
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
import UsageTransportBadge from './UsageTransportBadge.vue'
const props = defineProps<{
timeRangeParams: UsageTimeRangeParams
@@ -25,8 +23,6 @@ const {
refreshing,
records,
searchQuery,
failureClass,
route,
pagination,
handlePageChange,
handlePageSizeChange,
@@ -36,29 +32,27 @@ const {
const selectedRecord = shallowRef<OpsError | null>(null)
const detailOpen = shallowRef(false)
const upstreamSendStateLabels: Record<string, string> = {
sent: '已发送',
not_sent: '未发送',
ambiguous: '状态不明',
}
function showDetail(record: OpsError) {
selectedRecord.value = record
detailOpen.value = true
}
function failureClassText(record: OpsError) {
return presentOpsError(record).failureClassLabel
}
function errorSummary(record: OpsError) {
return presentOpsError(record).summary
}
function accountText(record: OpsError) {
return record.accountEmail
|| record.accountName
|| record.metadata.accountLabel
|| record.accountId
|| '—'
|| '未记录'
}
function modelText(record: OpsError) {
return record.requestedModel || record.model || record.upstreamModel || '—'
return record.requestedModel || record.model || record.upstreamModel || '未记录'
}
function modelTitle(record: OpsError) {
@@ -69,8 +63,10 @@ function modelTitle(record: OpsError) {
: modelText(record)
}
function reasoningText(record: OpsError) {
return record.reasoningPreset || record.reasoningEffort || '—'
function upstreamSendStateText(value: string | null | undefined) {
if (!value)
return '未记录'
return upstreamSendStateLabels[value] ?? value
}
</script>
@@ -79,22 +75,18 @@ function reasoningText(record: OpsError) {
<div
class="flex w-full flex-col gap-3 lg:flex-row lg:flex-wrap lg:items-center"
role="group"
aria-label="错误明细筛选与操作"
aria-label="错误筛选与操作"
>
<div class="min-w-0 flex-1">
<div class="grid min-w-0 grid-cols-1 gap-2 sm:grid-cols-2 lg:flex lg:items-center lg:gap-3">
<BaseInput
v-model="searchQuery"
placeholder="事件、请求、Key 或账号 ID 前缀"
class="min-w-0 sm:col-span-2 lg:min-w-64 lg:flex-1 lg:max-w-96"
>
<template #prefix>
<Search class="size-4.5 text-cp-text-tertiary" />
</template>
</BaseInput>
<BaseInput v-model="failureClass" placeholder="失败分类(精确)" class="min-w-0" />
<BaseInput v-model="route" placeholder="端点(精确)" class="min-w-0" />
</div>
<BaseInput
v-model="searchQuery"
placeholder="请求 ID、Key 或账号"
class="min-w-0 w-full lg:max-w-96"
>
<template #prefix>
<Search class="size-4.5 text-cp-text-tertiary" />
</template>
</BaseInput>
</div>
<div class="flex shrink-0 self-end items-center justify-end gap-2 lg:ml-auto">
@@ -120,23 +112,8 @@ function reasoningText(record: OpsError) {
:columns="opsErrorColumns"
:rows="records"
:loading="loading"
empty-text="暂无错误明细"
empty-text="当前时段没有错误"
>
<template #upstreamStatusCode="{ row }">
<UsageStatusCodeBadge
:status-code="typeof row.upstreamStatusCode === 'number' ? row.upstreamStatusCode : null"
/>
</template>
<template #clientStatusCode="{ row }">
<UsageStatusCodeBadge
:status-code="typeof row.clientStatusCode === 'number' ? row.clientStatusCode : null"
/>
</template>
<template #failureClass="{ row }">
<span class="font-mono text-cp-sm font-bold text-cp-error-text">
{{ failureClassText(row) }}
</span>
</template>
<template #provider="{ row }">
<ProviderIconGroup
:provider="String(row.provider || '')"
@@ -144,8 +121,24 @@ function reasoningText(record: OpsError) {
/>
</template>
<template #message="{ row }">
<span class="block max-w-full truncate text-cp-sm font-emphasis text-cp-text" :title="errorSummary(row)">
{{ errorSummary(row) }}
<div class="min-w-0 py-0.5" :title="row.message || opsErrorSummary(row)">
<code class="block max-w-full truncate font-mono text-cp-sm font-bold text-cp-error-text">
{{ opsErrorSummary(row) }}
</code>
<p
v-if="row.message"
class="mt-1 mb-0 line-clamp-1 text-cp-xs leading-[1.45] font-emphasis text-cp-text-secondary"
>
{{ row.message }}
</p>
</div>
</template>
<template #upstreamSendState="{ row }">
<span
class="inline-flex h-6 max-w-full items-center rounded-full bg-cp-fill-quaternary px-2.5 font-mono text-cp-sm leading-none font-bold text-cp-text-secondary"
:title="row.upstreamSendState || '未记录'"
>
<span class="min-w-0 truncate">{{ upstreamSendStateText(row.upstreamSendState) }}</span>
</span>
</template>
<template #accountId="{ row }">
@@ -164,38 +157,14 @@ function reasoningText(record: OpsError) {
{{ modelText(row) }}
</span>
</template>
<template #reasoningEffort="{ row }">
<span class="text-cp-sm font-bold text-cp-text-secondary">
{{ reasoningText(row) }}
</span>
</template>
<template #transport="{ row }">
<UsageTransportBadge :transport="row.transport" />
</template>
<template #clientTransport="{ row }">
<UsageTransportBadge :transport="row.clientTransport" />
</template>
<template #clientIp="{ row }">
<span
class="inline-flex h-6 max-w-full items-center rounded-full bg-cp-blue-bg px-2.5 font-mono text-cp-sm leading-none font-bold text-cp-blue-text-on-bg"
:title="row.clientIp || '—'"
>
<span class="min-w-0 truncate">{{ row.clientIp || '—' }}</span>
</span>
</template>
<template #userAgent="{ row }">
<span class="block max-w-full wrap-break-word whitespace-normal font-mono text-cp-sm leading-[1.4] font-emphasis text-cp-text-secondary">
{{ row.userAgent || '—' }}
</span>
</template>
<template #actions="{ row }">
<BaseIconButton
variant="ghost"
size="sm"
size="md"
label="查看错误详情"
@click="showDetail(row)"
>
<Eye class="size-3.5" />
<Eye class="size-4.5" />
</BaseIconButton>
</template>
</BaseTable>
@@ -3,7 +3,7 @@ import { displayValue, fieldLabelClass, fieldValueClass } from '../utils/detail'
// 详情弹窗的两列字段网格:label + 值(可选等宽字体),空值以“—”占位。
defineProps<{
items: Array<{ label: string, value: unknown, mono?: boolean, wrap?: boolean }>
items: Array<{ label: string, value: unknown, mono?: boolean, wrap?: boolean, fullWidth?: boolean }>
}>()
function valueClass(item: { mono?: boolean, wrap?: boolean }) {
@@ -19,7 +19,12 @@ function valueClass(item: { mono?: boolean, wrap?: boolean }) {
<template>
<dl class="mt-3 grid min-w-0 grid-cols-1 gap-x-4 gap-y-3 sm:grid-cols-2">
<div v-for="item in items" :key="item.label" class="min-w-0">
<div
v-for="item in items"
:key="item.label"
class="min-w-0"
:class="item.fullWidth ? 'sm:col-span-2' : undefined"
>
<dt :class="fieldLabelClass">
{{ item.label }}
</dt>
@@ -57,58 +57,51 @@ const finalAttemptIndex = computed(() => {
return last?.attemptIndex ?? props.record?.attemptCount
})
const overviewItems = computed(() => [
{ label: '时间', value: props.record?.createdAtDisplay, mono: true },
{ label: '客户端传输', value: usageTransportType(props.record?.clientTransport) },
{ label: '上游传输', value: usageTransportType(props.record?.upstreamTransport) },
{ label: '耗时', value: props.record?.latencyMsDisplay, mono: true },
{ label: '端点', value: props.record?.route, mono: true },
{ label: '客户端传输', value: usageTransportType(props.record?.clientTransport), mono: true },
{ label: '上游传输', value: usageTransportType(props.record?.upstreamTransport), mono: true },
{ label: '总耗时', value: props.record?.latencyMsDisplay, mono: true },
{
label: latencyDetails.value?.firstOutputLabel ?? '首字',
value: latencyDetails.value?.firstOutputDisplay ?? '—',
mono: true,
},
{ label: '总 Token', value: tokenDetails.value?.totalTokensDisplay, mono: true },
{ label: '请求 ID', value: props.record?.requestId, mono: true, wide: true },
{ label: '消息', value: props.record?.message, wide: true },
])
const detailGroups = computed(() => [
const modelRouteItems = computed(() => [
{ label: '端点', value: props.record?.route, mono: true },
{ label: '推理强度', value: props.record ? usageReasoningEffort(props.record) : '—' },
{ label: '请求模型', value: modelDisplay.value.primary, mono: true },
{
title: '模型与链路',
items: [
{ label: '端点', value: props.record?.route, mono: true },
{ label: '请求模型', value: modelDisplay.value.primary, mono: true },
{
label: '上游模型',
value: modelDisplay.value.secondary || props.record?.upstreamModel,
mono: true,
},
{ label: '存储模型', value: props.record?.model, mono: true },
{ label: '推理强度', value: props.record ? usageReasoningEffort(props.record) : '—' },
{ label: '账号 ID', value: props.record?.accountId, mono: true },
{ label: '请求 ID', value: props.record?.requestId, mono: true },
{ label: '响应 ID', value: props.record?.responseId, mono: true },
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, mono: true },
],
label: '上游模型',
value: modelDisplay.value.secondary || props.record?.upstreamModel,
mono: true,
},
{ label: '存储模型', value: props.record?.model, mono: true },
])
const clientUpstreamItems = computed(() => [
{ label: '客户端 IP', value: props.record ? usageClientIp(props.record) : '—', mono: true },
{ label: '服务档位', value: props.record?.serviceTier, mono: true },
{ label: '事件类型', value: props.record?.kind, mono: true },
{ label: '尝试序号', value: finalAttemptIndex.value },
{
title: '客户端与上游',
items: [
{ label: '客户端 IP', value: props.record ? usageClientIp(props.record) : '—', mono: true },
{
label: 'User-Agent',
value: props.record ? usageUserAgent(props.record) : '',
mono: true,
wrap: true,
},
{ label: '事件类型', value: props.record?.kind, mono: true },
{ label: '尝试序号', value: finalAttemptIndex.value },
{ label: '服务档位', value: props.record?.serviceTier, mono: true },
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true },
],
label: 'User-Agent',
value: props.record ? usageUserAgent(props.record) : '',
mono: true,
wrap: true,
fullWidth: true,
},
])
const modelRouteGroup = computed(() => detailGroups.value[0])
const clientUpstreamGroup = computed(() => detailGroups.value[1])
const identifierItems = computed(() => [
{ label: '请求 ID', value: props.record?.requestId, mono: true, wrap: true, fullWidth: true },
{ label: '响应 ID', value: props.record?.responseId, mono: true, wrap: true },
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, mono: true, wrap: true },
{ label: '账号 ID', value: props.record?.accountId, mono: true, wrap: true },
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true, wrap: true },
])
interface AttemptRow {
id: string
@@ -130,7 +123,7 @@ const attemptColumns = defineTableColumns<AttemptRow>([
{ key: 'model', label: '模型', kind: 'mono', size: 'lg' },
{ key: 'transport', label: '上游传输', kind: 'status', size: 'md' },
{ key: 'statusCode', label: '状态', kind: 'status', size: 'sm' },
{ key: 'accountLabel', label: '账号', kind: 'mono', size: '3xl' },
{ key: 'accountLabel', label: '账号', kind: 'mono', size: '2xl' },
{ key: 'latencyMs', label: '耗时', kind: 'numeric', size: 'sm' },
])
@@ -286,34 +279,53 @@ const tokenDonutOption = computed<EChartsOption>(() => {
>
<div v-if="record" class="grid min-w-0 gap-3">
<section :class="panelClass">
<div class="min-w-0">
<span :class="fieldLabelClass">账号</span>
<p
class="mt-1.5 mb-0 truncate font-mono text-cp leading-none font-heavy text-cp-text"
:title="displayValue(accountDisplay)"
>
{{ displayValue(accountDisplay) }}
</p>
</div>
<dl
class="mt-4 grid grid-cols-2 gap-x-5 gap-y-3 lg:grid-cols-[120px_120px_repeat(5,minmax(0,1fr))]"
class="grid min-w-0 grid-cols-2 gap-x-6 gap-y-4 sm:grid-cols-4 lg:grid-cols-[minmax(0,2fr)_100px_minmax(0,1fr)_180px]"
>
<div class="col-span-2 min-w-0 lg:col-span-1">
<dt :class="fieldLabelClass">
账号
</dt>
<dd
class="mt-1.5 mb-0 min-w-0 break-all font-mono text-cp-sm leading-snug font-heavy text-cp-text"
:title="displayValue(accountDisplay)"
>
{{ displayValue(accountDisplay) }}
</dd>
</div>
<div class="min-w-0">
<dt :class="fieldLabelClass">
状态码
</dt>
<dd class="mt-1.5 mb-0">
<UsageStatusCodeBadge :status-code="record.statusCode" />
<dd :class="fieldValueClass(true)" :title="displayValue(record.statusCode)">
{{ displayValue(record.statusCode) }}
</dd>
</div>
<div
v-for="item in overviewItems"
:key="item.label"
class="min-w-0"
:class="item.wide ? 'col-span-2 lg:col-span-2' : undefined"
>
<div class="min-w-0">
<dt :class="fieldLabelClass">
消息
</dt>
<dd :class="fieldValueClass()" :title="displayValue(record.message)">
{{ displayValue(record.message) }}
</dd>
</div>
<div class="col-span-2 min-w-0 lg:col-span-1">
<dt :class="fieldLabelClass">
时间
</dt>
<dd :class="fieldValueClass(true)" :title="displayValue(record.createdAtDisplay)">
{{ displayValue(record.createdAtDisplay) }}
</dd>
</div>
</dl>
<dl
class="mt-4 grid min-w-0 grid-cols-2 gap-x-6 gap-y-4 sm:grid-cols-3 lg:grid-cols-[minmax(160px,1.4fr)_repeat(5,minmax(0,1fr))]"
>
<div v-for="item in overviewItems" :key="item.label" class="min-w-0">
<dt :class="fieldLabelClass">
{{ item.label }}
</dt>
@@ -324,90 +336,87 @@ const tokenDonutOption = computed<EChartsOption>(() => {
</dl>
</section>
<section class="grid min-w-0 gap-3 lg:grid-cols-[minmax(0,0.95fr)_minmax(0,1.05fr)]">
<div class="flex min-h-0 min-w-0 flex-col gap-3">
<section :class="panelClass">
<h3 :class="panelTitleClass">
{{ modelRouteGroup.title }}
</h3>
<UsageDetailFieldGrid :items="modelRouteGroup.items" />
</section>
<section class="grid min-w-0 gap-3 lg:grid-cols-2">
<section :class="panelClass">
<h3 :class="panelTitleClass">
模型与路由
</h3>
<UsageDetailFieldGrid :items="modelRouteItems" />
</section>
<section class="flex min-h-0 flex-1 flex-col" :class="[panelClass]">
<h3 :class="panelTitleClass">
Token
</h3>
<div
class="mt-3 grid min-h-38 min-w-0 flex-1 grid-cols-1 content-center items-center gap-3 sm:grid-cols-[150px_minmax(0,1fr)]"
>
<div class="relative mx-auto w-38 sm:mx-0">
<BaseChart :option="tokenDonutOption" :height="152" />
<div class="pointer-events-none absolute inset-0 flex items-center justify-center">
<div class="grid text-center">
<span class="text-cp-xs leading-none font-bold text-cp-text-quaternary">
总计
</span>
<strong
class="mt-1 font-mono text-[16px] leading-none font-extrabold tabular-nums text-cp-text"
>
{{ tokenDetails?.totalTokensDisplay ?? '—' }}
</strong>
</div>
<section :class="panelClass">
<h3 :class="panelTitleClass">
客户端与上游
</h3>
<UsageDetailFieldGrid :items="clientUpstreamItems" />
</section>
</section>
<section :class="panelClass">
<h3 :class="panelTitleClass">
请求标识
</h3>
<UsageDetailFieldGrid :items="identifierItems" />
</section>
<section class="grid min-w-0 gap-3 lg:grid-cols-2">
<section class="flex min-h-0 flex-col" :class="panelClass">
<h3 :class="panelTitleClass">
Token
</h3>
<div
class="mt-3 grid min-h-38 min-w-0 flex-1 grid-cols-1 content-center items-center gap-3 sm:grid-cols-[150px_minmax(0,1fr)]"
>
<div class="relative mx-auto w-38 sm:mx-0">
<BaseChart :option="tokenDonutOption" :height="152" />
<div class="pointer-events-none absolute inset-0 flex items-center justify-center">
<div class="grid text-center">
<span class="text-cp-xs leading-none font-bold text-cp-text-quaternary">
总计
</span>
<strong
class="mt-1 font-mono text-[16px] leading-none font-extrabold tabular-nums text-cp-text"
>
{{ tokenDetails?.totalTokensDisplay ?? '—' }}
</strong>
</div>
</div>
<dl class="grid min-w-0 grid-cols-2 gap-x-4 gap-y-3">
<div v-for="item in tokenChartItems" :key="item.label" class="min-w-0">
<dt class="flex min-w-0 items-center gap-1.5" :class="fieldLabelClass">
<i
class="size-1.75 shrink-0 rounded-full"
:style="{ backgroundColor: item.color }"
/>
<span class="truncate">{{ item.label }}</span>
</dt>
<dd
class="font-mono tabular-nums" :class="[fieldValueBaseClass]"
:title="displayValue(item.display)"
>
{{ displayValue(item.display) }}
</dd>
</div>
</dl>
</div>
</section>
</div>
<div class="flex min-h-0 min-w-0 flex-col gap-3">
<section class="flex-1" :class="[panelClass]">
<h3 :class="panelTitleClass">
{{ clientUpstreamGroup.title }}
</h3>
<UsageDetailFieldGrid :items="clientUpstreamGroup.items" />
</section>
<dl class="grid min-w-0 grid-cols-2 gap-x-4 gap-y-3">
<div v-for="item in tokenChartItems" :key="item.label" class="min-w-0">
<dt class="flex min-w-0 items-center gap-1.5" :class="fieldLabelClass">
<i
class="size-1.75 shrink-0 rounded-full"
:style="{ backgroundColor: item.color }"
/>
<span class="truncate">{{ item.label }}</span>
</dt>
<dd
class="font-mono tabular-nums" :class="[fieldValueBaseClass]"
:title="displayValue(item.display)"
>
{{ displayValue(item.display) }}
</dd>
</div>
</dl>
</div>
</section>
<section :class="panelClass">
<h3 :class="panelTitleClass">
费用
</h3>
<UsageDetailFieldGrid :items="billingItems" />
</section>
</div>
<section :class="panelClass">
<h3 :class="panelTitleClass">
费用
</h3>
<UsageDetailFieldGrid :items="billingItems" />
</section>
</section>
<section v-if="attemptRows.length" :class="panelClass">
<div class="flex items-center gap-2">
<h3 :class="panelTitleClass">
尝试链路
</h3>
<span
class="text-cp-xs leading-none font-emphasis text-cp-text-quaternary"
title="中间尝试可能有缺口,仅展示可观测到的记录"
>
尽力观测
</span>
</div>
<h3 :class="panelTitleClass">
尝试链路
</h3>
<BaseTable
class="attempt-table mt-2.5 min-w-0 font-mono tabular-nums"
class="attempt-table mt-2.5 h-auto! min-w-0 font-mono tabular-nums"
:columns="attemptColumns"
:rows="attemptRows"
density="compact"
@@ -11,16 +11,12 @@ import { errorMessage, withMinimumDuration } from '@/utils/async'
export function useOpsErrorsTable(timeRangeParams: Readonly<Ref<UsageTimeRangeParams>>) {
const refreshing = shallowRef(false)
const searchQuery = shallowRef('')
const failureClass = shallowRef('')
const route = shallowRef('')
const query = useStablePagedQuery({
initialPageSize: 10,
load: ({ currentPage, pageSize }) => getOpsErrors({
currentPage,
pageSize,
search: searchQuery.value.trim() || undefined,
failureClass: failureClass.value.trim() || undefined,
route: route.value.trim() || undefined,
...timeRangeParams.value,
}),
onError: error => toast.error(errorMessage(error, '加载错误明细失败')),
@@ -56,7 +52,7 @@ export function useOpsErrorsTable(timeRangeParams: Readonly<Ref<UsageTimeRangePa
}
watchDebounced(
[searchQuery, failureClass, route],
searchQuery,
() => {
void query.reloadFromStart()
},
@@ -74,8 +70,6 @@ export function useOpsErrorsTable(timeRangeParams: Readonly<Ref<UsageTimeRangePa
refreshing,
records: query.items,
searchQuery,
failureClass,
route,
pagination,
handlePageChange,
handlePageSizeChange,
+8 -14
View File
@@ -34,22 +34,16 @@ export const usageRecordColumns = defineTableColumns<UsageDisplayRecord>([
])
export const opsErrorColumns = defineTableColumns<OpsErrorRow>([
{ key: 'accountId', label: '账号', kind: 'mono', size: '3xl' },
{ key: 'accountId', label: '账号', kind: 'identity', size: '3xl', emptyText: '未知账号' },
{ key: 'provider', label: '平台/类型', kind: 'custom', size: 'sm' },
{ key: 'message', label: '错误摘要', kind: 'custom', size: '3xl' },
{ key: 'failureClass', label: '失败分类', kind: 'mono' },
{ key: 'upstreamStatusCode', label: '上游状态', kind: 'status', size: 'sm' },
{ key: 'clientStatusCode', label: '客户端状态', kind: 'status', size: 'md' },
{ key: 'model', label: '模型', kind: 'custom', size: 'xl' },
{ key: 'reasoningEffort', label: '推理强度', kind: 'custom' },
{ key: 'route', label: '端点', kind: 'mono' },
{ key: 'transport', label: '上游', kind: 'custom', size: 'md' },
{ key: 'clientTransport', label: '接入', kind: 'custom', size: 'md' },
{ key: 'message', label: '错误', kind: 'custom', size: '4xl' },
{ key: 'upstreamSendState', label: '发送状态', kind: 'custom', size: 'xl' },
{ key: 'model', label: '模型', kind: 'custom', size: 'xl', emptyText: '未记录模型' },
{ key: 'route', label: '端点', kind: 'mono', size: 'xl', emptyText: '未记录' },
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl' },
{ key: 'kind', label: '事件', kind: 'mono' },
{ key: 'clientIp', label: 'IP', kind: 'custom', size: '3xl' },
{ key: 'userAgent', label: 'User-Agent', kind: 'custom', size: '4xl' },
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl', emptyText: '未记录' },
{ key: 'clientIp', label: 'IP', kind: 'mono', size: '3xl', emptyText: '未记录' },
{ key: 'userAgent', label: 'User-Agent', kind: 'mono', size: '4xl', emptyText: '未记录' },
{ key: 'actions', label: '操作', kind: 'actions', size: 'sm' },
])
@@ -1,83 +1,16 @@
import type { OpsError } from '@/api/modules/usage'
import type { OpsError } from '@/api'
const FAILURE_CLASS_LABELS: Record<string, string> = {
invalid_request: '请求不合法',
unsupported: '请求能力不受支持',
unauthorized: '认证失败',
auth_failed: '认证失败',
permission_denied: '上游权限不足',
policy_denied: '策略拒绝',
model_not_found: '模型不存在',
no_available_provider: '没有可用 Provider',
no_eligible_account: '没有符合条件的账号',
account_capacity_unavailable: '账号容量不足',
provider_infrastructure_unavailable: 'Provider 基础设施不可用',
rate_limited: '上游限流',
quota_exhausted: '账号额度已耗尽',
upstream_unavailable: '上游服务不可用',
upstream_error: '上游响应错误',
transport: '上游传输失败',
protocol: '上游协议错误',
unavailable: 'Provider 暂不可用',
timeout: '请求超时',
cancelled: '请求已取消',
process_terminated: '进程终止',
internal_error: '内部错误',
failed: '请求失败',
const failureClassLabels: Readonly<Record<string, string>> = {
continuation_recovery_required: '会话续接需要重建',
}
const COMPONENT_LABELS: Record<string, string> = {
model_request: '模型请求',
routing: '路由调度',
account_probe: '账号连接测试',
export function failureClassText(value: string | null | undefined) {
if (!value)
return '未记录'
return failureClassLabels[value] ?? value
}
const SOURCE_LABELS: Record<string, string> = {
model_request: '模型请求记录',
ops_event: '运维事件',
}
export interface OpsErrorPresentation {
summary: string
failureClassLabel: string
componentLabel: string
sourceLabel: string
}
/** 将运维错误的稳定机器字段转换为中文摘要;原始诊断仍由调用方单独展示。 */
export function presentOpsError(record: OpsError): OpsErrorPresentation {
const failureClassLabel = mappedOrOriginal(FAILURE_CLASS_LABELS, record.failureClass, '未分类错误')
const componentLabel = mappedOrOriginal(
COMPONENT_LABELS,
record.metadata.component || record.kind,
'未知组件',
)
const sourceLabel = mappedOrOriginal(SOURCE_LABELS, record.metadata.source, '未知来源')
const statuses = [
statusText('客户端', record.clientStatusCode),
statusText('上游', record.upstreamStatusCode),
].filter(Boolean)
const statusSuffix = statuses.length > 0 ? `(${statuses.join(',')})` : ''
const providerCode = record.providerErrorCode?.trim()
const codeSuffix = providerCode ? ` [${providerCode}]` : ''
return {
summary: `${componentLabel}:${failureClassLabel}${codeSuffix}${statusSuffix}`,
failureClassLabel,
componentLabel,
sourceLabel,
}
}
function mappedOrOriginal(
labels: Record<string, string>,
value: string | null | undefined,
fallback: string,
) {
const normalized = value?.trim()
return normalized ? labels[normalized] || normalized : fallback
}
function statusText(owner: string, status: number | null) {
return typeof status === 'number' ? `${owner} HTTP ${status}` : ''
export function opsErrorSummary(record: OpsError) {
const failureLabel = failureClassLabels[record.failureClass]
return failureLabel ?? record.providerErrorCode ?? record.failureClass
}
+1 -1
View File
@@ -1 +1 @@
version: "3.3.15"
version: "3.3.16"