diff --git a/backend/crates/gateway-admin/src/model/observability.rs b/backend/crates/gateway-admin/src/model/observability.rs index f3107e2ff..0012f8d47 100644 --- a/backend/crates/gateway-admin/src/model/observability.rs +++ b/backend/crates/gateway-admin/src/model/observability.rs @@ -851,7 +851,7 @@ pub struct OpsError { pub upstream_request_id: Option, pub latency_ms: Option, pub message: String, - pub raw_upstream_error: Option, + pub error_details: Option, pub client_ip: Option, pub user_agent: Option, pub reasoning_effort: Option, diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 0c713979b..00b0b9203 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -52,12 +52,13 @@ pub enum AdminStoreErrorKind { } /// 隐藏数据库实现细节的持久化错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("{resource} store operation failed: {message}")] pub struct AdminStoreError { kind: AdminStoreErrorKind, resource: &'static str, message: String, + source: Option, } impl AdminStoreError { @@ -71,9 +72,17 @@ impl AdminStoreError { kind, resource, message: message.into(), + source: None, } } + /// 来源留在内部诊断链，公开消息不展开基础设施错误 + #[must_use] + pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { + self.source = Some(gateway_core::error::ErrorSource::new(source)); + self + } + #[must_use] pub const fn kind(&self) -> AdminStoreErrorKind { self.kind diff --git a/backend/crates/gateway-api/src/admin/observability/presenter.rs b/backend/crates/gateway-api/src/admin/observability/presenter.rs index cd11d5ffb..ccc8c9fb0 100644 --- a/backend/crates/gateway-api/src/admin/observability/presenter.rs +++ b/backend/crates/gateway-api/src/admin/observability/presenter.rs @@ -617,7 +617,7 @@ pub(crate) fn ops_error_view( subagent_kind: error.subagent_kind, compact: error.compact, message: error.message, - raw_upstream_error: error.raw_upstream_error, + error_details: error.error_details, metadata: OpsErrorMetadataView { source: error.source, component: error.component, diff --git a/backend/crates/gateway-api/src/admin/observability/wire.rs b/backend/crates/gateway-api/src/admin/observability/wire.rs index 15a68e80d..ab95a494b 100644 --- a/backend/crates/gateway-api/src/admin/observability/wire.rs +++ b/backend/crates/gateway-api/src/admin/observability/wire.rs @@ -814,8 +814,8 @@ pub struct OpsErrorView { pub subagent_kind: Option, pub compact: Option, pub message: String, - /// 上游错误正文或 WebSocket close/error frame 原文；不做脱敏 - pub raw_upstream_error: Option, + /// 受控错误详情，分别承载本地原因链与上游错误正文 + pub error_details: Option, pub metadata: OpsErrorMetadataView, pub created_at: DateTime, pub created_at_display: String, diff --git a/backend/crates/gateway-api/tests/admin/observability/response.rs b/backend/crates/gateway-api/tests/admin/observability/response.rs index d79013cf6..789d01e61 100644 --- a/backend/crates/gateway-api/tests/admin/observability/response.rs +++ b/backend/crates/gateway-api/tests/admin/observability/response.rs @@ -466,7 +466,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() { upstream_request_id: None, latency_ms: Some(90), message: "snapshot error".to_owned(), - raw_upstream_error: Some( + error_details: Some( r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(), ), client_ip: Some("203.0.113.8".to_owned()), @@ -526,7 +526,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() { "userAgent": value["data"]["items"][0]["userAgent"], "providerErrorCode": value["data"]["items"][0]["providerErrorCode"], "upstreamSendState": value["data"]["items"][0]["upstreamSendState"], - "rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"], + "errorDetails": value["data"]["items"][0]["errorDetails"], "continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"], "upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"], "upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"], @@ -547,7 +547,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() { "userAgent": "codex-cli/0.144.0", "providerErrorCode": "upstream", "upstreamSendState": "sent", - "rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}", + "errorDetails": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}", "continuationUnavailableReason": "reused_connection_lost", "upstreamConnectionExitReason": "tcp_reset", "upstreamConnectionAgeMs": 12000, diff --git a/backend/crates/gateway-api/tests/key_usage/fixtures.rs b/backend/crates/gateway-api/tests/key_usage/fixtures.rs index 9d89e4bf9..4d3a046d4 100644 --- a/backend/crates/gateway-api/tests/key_usage/fixtures.rs +++ b/backend/crates/gateway-api/tests/key_usage/fixtures.rs @@ -183,7 +183,7 @@ fn error_record() -> OpsError { upstream_request_id: Some("private-sentinel".to_owned()), latency_ms: Some(420), message: "private-sentinel".to_owned(), - raw_upstream_error: Some("private-sentinel".to_owned()), + error_details: Some("private-sentinel".to_owned()), client_ip: Some("192.0.2.42".to_owned()), user_agent: Some("key-usage-test/1.0".to_owned()), reasoning_effort: Some("xhigh".to_owned()), diff --git a/backend/crates/gateway-core/src/diagnostics/trace.rs b/backend/crates/gateway-core/src/diagnostics/trace.rs index 17328989d..8874c5166 100644 --- a/backend/crates/gateway-core/src/diagnostics/trace.rs +++ b/backend/crates/gateway-core/src/diagnostics/trace.rs @@ -10,6 +10,8 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use serde::Serialize; use serde_json::{Value, json}; +use crate::error::ProviderError; + use super::capture::{bounded, diagnostic_event_json, diagnostic_event_type}; use super::{body_fingerprint, diagnostic_headers}; @@ -123,6 +125,22 @@ impl TraceContext { self.push(stage, data, false); } + /// 诊断正文与导出分类来自同一失败，导出路径不接收任意 JSON 或上游文案 + pub fn record_provider_failure(&self, error: &ProviderError) { + if !self.is_enabled() { + return; + } + let data = json!({ + "kind": error.kind().as_str(), "sendState": error.send_state().as_str(), + "diagnostic": error.diagnostic().map(|diagnostic| json!({ + "stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(), + })), + "upstreamStatus": error.upstream_status(), + "upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()), + }); + self.push("attempt.failed", data, false); + } + /// 同一个头部边界只采集一次：默认脱敏，显式开启 dump 时另存完整字节 /// facts 必须为调用方构造的安全对象；仅受控诊断头可明文保留，完整头部另存 dump pub fn headers<'a>( @@ -243,7 +261,7 @@ impl TraceContext { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); Some(json!({ - "schemaVersion": 1, "requestId": state.request_id, + "schemaVersion": 2, "requestId": state.request_id, "startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence, "droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS, "captureMode": "sanitized", "events": state.events, @@ -258,10 +276,35 @@ impl TraceContext { let Some(state) = &self.state else { return }; let mut encoded = serde_json::to_vec(&data).unwrap_or_default(); if encoded.len() > MAX_DATA_BYTES { - data = json!({"truncated": true, "summary": body_fingerprint(&encoded), + let mut summary = json!({"truncated": true, "summary": body_fingerprint(&encoded), "eventType": data.get("eventType"), "body": data.get("body"), "jsonValid": data.get("jsonValid"), }); + if stage == "attempt.failed" { + // 过长诊断的指纹不能挤掉已经由 Provider 分类的失败原因 + // 保持有界，完整诊断仍由终态错误记录保存 + summary = json!({"truncated": true, "summary": body_fingerprint(&encoded)}); + for key in ["kind", "sendState"] { + if let Some(value) = data.get(key).and_then(Value::as_str) { + summary[key] = json!(bounded(value, 64)); + } + } + summary["upstreamStatus"] = + json!(data.get("upstreamStatus").and_then(Value::as_u64)); + if let Some(diagnostic) = data.get("diagnostic").and_then(Value::as_object) { + let mut kept = json!({}); + for (key, limit) in [("stage", 96), ("code", 96), ("message", 256)] { + if let Some(value) = diagnostic.get(key).and_then(Value::as_str) { + kept[key] = json!(bounded(value, limit)); + if value.chars().count() > limit { + kept["truncated"] = json!(true); + } + } + } + summary["diagnostic"] = kept; + } + } + data = summary; encoded = serde_json::to_vec(&data).unwrap_or_default(); } let mut state = state diff --git a/backend/crates/gateway-core/src/engine/coordinator.rs b/backend/crates/gateway-core/src/engine/coordinator.rs index 1710009a7..aeb199a30 100644 --- a/backend/crates/gateway-core/src/engine/coordinator.rs +++ b/backend/crates/gateway-core/src/engine/coordinator.rs @@ -398,7 +398,7 @@ struct FailureFinalization { upstream_status_code: Option, upstream_request_id: Option, provider_error_code: Option, - raw_upstream_error: Option, + error_details: Option, retry_after_ms: Option, observation: ModelRequestFailureObservation, } @@ -523,6 +523,10 @@ where self.trace.clone() } + pub(super) fn request_id(&self) -> &ModelRequestId { + &self.request_id + } + /// 读取下一条 canonical event；首条未提交事件会携带 commit 要求 /// /// # Errors @@ -956,7 +960,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -1026,6 +1030,9 @@ where } Ok(None) => {} Err(error) => { + self.trace + .attempt(self.attempts) + .record_provider_failure(&error); self.finish_provider_error(&error).await?; return Err(provider_engine_error(error)); } @@ -1101,7 +1108,7 @@ where ProviderBoundary::Result(result) => match *result { Ok(stream) => stream, Err(error) => { - record_trace_error(&attempt_trace, &error); + attempt_trace.record_provider_failure(&error); let continuation_retry = !error.retry_is_prohibited() && self.prepare_unavailable_native_continuation_replay(&error); let candidate_retry = !error.retry_is_prohibited() && !continuation_retry @@ -1200,7 +1207,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -1233,7 +1240,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -1256,7 +1263,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -1282,7 +1289,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -1438,7 +1445,9 @@ where ) -> Result { // 原始 wire 只活在 request-local 决策状态；clone、attempt 记录与持久化终态 // 均只接触已剥离的稳定错误字段 - record_trace_error(&self.trace.attempt(self.attempts), &error); + self.trace + .attempt(self.attempts) + .record_provider_failure(&error); let mut atomic_client_events = error.take_atomic_client_events(); let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?; if self.request_observation.is_some() { @@ -1961,7 +1970,7 @@ where diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()), error: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, failure_observation: ModelRequestFailureObservation::default(), retry_after_ms: None, usage: self.observation.usage.clone(), @@ -2002,9 +2011,7 @@ where upstream_status_code: error.upstream_status(), upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()), provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()), - raw_upstream_error: error - .raw_upstream_error() - .map(|raw| raw.as_str().to_owned()), + error_details: error.error_details(), retry_after_ms: error.retry_after().map(duration_ms), observation: ModelRequestFailureObservation { continuation_unavailable_reason: error @@ -2065,7 +2072,7 @@ where upstream_status_code: None, upstream_request_id: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, retry_after_ms: None, observation: ModelRequestFailureObservation::default(), }) @@ -2128,7 +2135,7 @@ where diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()), error: Some(finalization.error), provider_error_code: finalization.provider_error_code, - raw_upstream_error: finalization.raw_upstream_error, + error_details: finalization.error_details, failure_observation: finalization.observation, retry_after_ms: finalization.retry_after_ms, usage: self.observation.usage.clone(), @@ -2489,19 +2496,3 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst fn duration_ms(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } - -fn record_trace_error(trace: &TraceContext, error: &ProviderError) { - trace.record("attempt.failed", json!({ - "kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()), - "diagnostic": error.diagnostic().map(|diagnostic| json!({ - "stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(), - })), - "upstreamStatus": error.upstream_status(), - "upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()), - "upstreamCode": error.upstream_code().map(|code| code.as_str()), - "rawError": error.raw_upstream_error().map(|raw| { - let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str())); - crate::diagnostics::diagnostic_json(&value) - }), - })); -} diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index ce64bc0ab..7610d53d5 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -1310,8 +1310,12 @@ impl DefaultExecutionService { let timeout = Delay::new(COORDINATION_TIMEOUT).fuse(); pin_mut!(write, timeout); select_biased! { - result = write => { if result.is_err() { tracing::warn!("入口拒绝观测写入失败"); } }, - _ = timeout => tracing::warn!("入口拒绝观测写入超时"), + result = write => { + if let Err(error) = result { + tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", error_kind = ?error.kind(), "入口拒绝观测写入失败"); + } + }, + _ = timeout => tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", "入口拒绝观测写入超时"), } } if let (Some(observation), Err(error)) = (&request_observation, &result) { @@ -2091,7 +2095,8 @@ impl DefaultExecutionService { failure_kind = provider_error.kind().as_str(), send_state = ?provider_error.send_state(), upstream_status = ?provider_error.upstream_status(), - provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()), + diagnostic_stage = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.stage()), + diagnostic_code = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.code()), latency_ms, "账号连接测试失败" ); @@ -2477,8 +2482,9 @@ struct AdmissionLease { } async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) { + let request_id = charge.request_id.clone(); if let Err(error) = port.settle(charge).await { - tracing::error!(%error, "Client budget settlement failed; storage will retry on the next request"); + tracing::error!(request_id = request_id.as_str(), operation = "settle_client_budget", %error, "Client budget settlement failed; storage will retry on the next request"); } } @@ -2490,7 +2496,7 @@ impl AdmissionLease { .release(&self.client_api_key_id, &self.model_request_id) .await { - tracing::warn!(%error, "Client admission 释放失败，依赖租约 TTL 收敛"); + tracing::warn!(request_id = self.model_request_id.as_str(), key_id = self.client_api_key_id.as_str(), operation = "release_client_admission", %error, "Client admission 释放失败，依赖租约 TTL 收敛"); } self.armed = false; } @@ -2621,7 +2627,7 @@ impl DefaultExecutionSession { async fn finalize_detached(&mut self) { if let Err(error) = self.core.cancel_and_finalize().await { - tracing::warn!(%error, "Detached execution 终态收敛失败"); + tracing::warn!(request_id = self.core.request_id().as_str(), operation = "finalize_detached_execution", %error, "Detached execution 终态收敛失败"); } self.settle_if_finalized().await; } diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index b43c115b4..6fd4b841b 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -934,7 +934,7 @@ pub struct ModelRequestFinalization { pub error: Option, pub provider_error_code: Option, /// Provider 返回的原始错误正文或 WebSocket close/error frame - pub raw_upstream_error: Option, + pub error_details: Option, pub failure_observation: ModelRequestFailureObservation, pub retry_after_ms: Option, pub usage: Usage, diff --git a/backend/crates/gateway-core/src/error.rs b/backend/crates/gateway-core/src/error/mod.rs similarity index 93% rename from backend/crates/gateway-core/src/error.rs rename to backend/crates/gateway-core/src/error/mod.rs index ce3c344f9..d860eb504 100644 --- a/backend/crates/gateway-core/src/error.rs +++ b/backend/crates/gateway-core/src/error/mod.rs @@ -1,5 +1,9 @@ //! 网关核心使用的稳定错误分类 +mod source; + +pub use source::ErrorSource; + use std::fmt; use std::num::NonZeroU32; use std::time::Duration; @@ -231,6 +235,8 @@ impl ProviderDiagnostic { while let Some(error) = source { if let Some(error) = error.downcast_ref::() { let code = match error.kind() { + std::io::ErrorKind::NotFound => Some("io_not_found"), + std::io::ErrorKind::PermissionDenied => Some("io_permission_denied"), std::io::ErrorKind::ConnectionRefused => Some("connection_refused"), std::io::ErrorKind::ConnectionReset => Some("tcp_reset"), std::io::ErrorKind::ConnectionAborted => Some("connection_aborted"), @@ -244,6 +250,15 @@ impl ProviderDiagnostic { if let Some(code) = code { self.message.push_str("; I/O cause: "); self.message.push_str(code); + if let Some(os_code) = error.raw_os_error() { + // 只重建操作系统错误，不能格式化可能包含 URL、路径或正文的自定义 I/O 错误 + use std::fmt::Write as _; + let _ = write!( + self.message, + "; OS cause: {}", + std::io::Error::from_raw_os_error(os_code) + ); + } self.code = Some(code); break; } @@ -277,7 +292,7 @@ impl fmt::Debug for ProviderDiagnostic { /// 运维错误详情中按原样展示的上游错误返回 /// -/// 该值只接收响应方向的错误正文或 WebSocket close/error frame；不得放入请求 +/// 该值只接收响应方向的错误正文、WebSocket close/error frame 或上游适配插件 fault；不得放入请求 /// Authorization、Cookie 或客户端输入 /// 它会被持久化并由管理端显式返回，但普通 /// `Debug`/`Display` 不输出正文，避免在非运维日志中重复扩散 @@ -427,8 +442,9 @@ impl fmt::Debug for ClientVisibleUpstreamResponse { /// /// 稳定诊断字段不接收原始响应正文，也不会在 `Debug` 或 `Display` 中打印上游 /// code/request ID/response ID -/// Adapter 若捕获到可能含 secret 的上下文，只能调用 -/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记 +/// 原始错误来源与上游响应正文仅由 [`ProviderError::error_details`] 提取到受控运维详情 +/// 请求方向的凭据等敏感上下文必须先移除，并通过 +/// [`ProviderError::redact_sensitive_context`] 留下脱敏标记 /// /// 唯一例外是 [`ProviderError::with_atomic_client_events`]：它只承载尚未交付客户端、 /// 必须与本错误一起完成重试判断的协议事件 @@ -450,6 +466,7 @@ pub struct ProviderError { retry_same_account: bool, sensitive_context_redacted: bool, diagnostic: Option>, + source: Option, raw_upstream_error: Option>, client_visible_upstream_error: Option>, client_visible_upstream_response: Option>, @@ -491,6 +508,7 @@ impl ProviderError { retry_same_account: false, sensitive_context_redacted: false, diagnostic: None, + source: None, raw_upstream_error: None, client_visible_upstream_error: None, client_visible_upstream_response: None, @@ -507,7 +525,7 @@ impl ProviderError { self } - /// 附加 adapter 已分类为安全的上游错误 code + /// 保留原始上游错误 code，不参与普通格式化或替代稳定分类 #[must_use] pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self { self.upstream_values_mut().code = Some(code); @@ -845,6 +863,29 @@ impl ProviderError { self.diagnostic.as_deref() } + /// 包装底层失败；来源只供显式诊断访问，不改变分类和发送事实 + #[must_use] + pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { + self.source = Some(ErrorSource::new(source)); + self + } + + /// 运维详情的唯一投影入口，来源链与上游正文分别保留，不进入公共响应或诊断包 + #[must_use] + pub fn error_details(&self) -> Option { + if self.source.is_none() && self.raw_upstream_error.is_none() { + return None; + } + Some( + serde_json::json!({ + "causes": self.source.as_ref().map(ErrorSource::snapshot), + "upstream": self.raw_upstream_error().map(RawUpstreamError::as_str), + "redacted": self.sensitive_context_was_redacted(), + }) + .to_string(), + ) + } + /// 返回将由运维错误详情原样展示的上游错误返回 #[must_use] pub fn raw_upstream_error(&self) -> Option<&RawUpstreamError> { @@ -882,7 +923,7 @@ impl ProviderError { /// 复制可安全进入诊断和持久化边界的稳定错误事实 /// - /// 请求局部的原始上游响应与尚未提交的客户端事件不会进入快照 + /// 保留原因链和上游错误详情；供协议透传的完整响应与尚未提交的客户端事件不进入快照 #[must_use] pub fn stable_snapshot(&self) -> Self { Self { @@ -901,6 +942,7 @@ impl ProviderError { retry_same_account: self.retry_same_account, sensitive_context_redacted: self.sensitive_context_redacted, diagnostic: self.diagnostic.clone(), + source: self.source.clone(), raw_upstream_error: self.raw_upstream_error.clone(), client_visible_upstream_error: self.client_visible_upstream_error.clone(), client_visible_upstream_response: None, @@ -916,10 +958,7 @@ impl fmt::Debug for ProviderError { .field("kind", &self.kind) .field("send_state", &self.send_state) .field("upstream_status", &self.upstream_status) - .field( - "upstream_code", - &self.upstream_code().map(|_| ""), - ) + .field("upstream_code", &self.upstream_code().map(|_| "")) .field( "upstream_request_id", &self.upstream_request_id().map(|_| ""), @@ -986,7 +1025,11 @@ impl fmt::Display for ProviderError { } } -impl std::error::Error for ProviderError {} +impl std::error::Error for ProviderError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_deref().map(|source| source as _) + } +} /// 对客户端协议稳定的网关错误分类 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -1277,18 +1320,31 @@ pub enum StoreErrorKind { InvalidData, } -/// Core port 返回的脱敏存储错误 -#[derive(Debug, Clone, PartialEq, Eq, Error)] +/// Core port 返回的存储错误，普通格式化不展开原始来源 +#[derive(Debug, Clone, Error)] #[error("execution store failed: {kind:?}")] pub struct StoreError { kind: StoreErrorKind, + source: Option, } impl StoreError { - /// 创建不携带数据库正文的 store 错误 + /// 创建只有分类的本地存储错误；包装实际异常使用 caused_by #[must_use] pub const fn new(kind: StoreErrorKind) -> Self { - Self { kind } + Self { kind, source: None } + } + + /// 包装已有存储错误并保留来源 + #[must_use] + pub fn caused_by( + kind: StoreErrorKind, + source: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self { + kind, + source: Some(ErrorSource::new(source)), + } } /// 返回稳定错误分类 diff --git a/backend/crates/gateway-core/src/error/source.rs b/backend/crates/gateway-core/src/error/source.rs new file mode 100644 index 000000000..54a848c67 --- /dev/null +++ b/backend/crates/gateway-core/src/error/source.rs @@ -0,0 +1,71 @@ +//! 跨领域边界共享原始错误来源，不在普通格式化中展开详情 + +use std::{error::Error, fmt, ops::Deref, sync::Arc}; + +/// 保留底层错误的类型和来源链，避免领域端口依赖具体基础设施 +#[derive(Clone)] +pub struct ErrorSource(Arc); + +impl ErrorSource { + #[must_use] + pub fn new(error: impl Error + Send + Sync + 'static) -> Self { + Self(Arc::new(error)) + } + + /// 仅供受控错误详情落盘，限制链深度和总文本大小并显式标记缺口 + pub(super) fn snapshot(&self) -> serde_json::Value { + let mut messages = Vec::new(); + let mut source: Option<&(dyn Error + 'static)> = Some(self.0.as_ref()); + let mut remaining = 64 * 1024; + let mut truncated = false; + for _ in 0..32 { + let Some(error) = source else { break }; + let mut message = BoundedMessage { + text: String::new(), + remaining, + }; + if fmt::write(&mut message, format_args!("{error}")).is_err() { + truncated = true; + } + remaining = message.remaining; + messages.push(message.text); + source = error.source(); + if truncated || remaining == 0 { + break; + } + } + serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()}) + } +} + +struct BoundedMessage { + text: String, + remaining: usize, +} + +impl fmt::Write for BoundedMessage { + fn write_str(&mut self, value: &str) -> fmt::Result { + let end = value.floor_char_boundary(self.remaining.min(value.len())); + self.text.push_str(&value[..end]); + self.remaining -= end; + if end == value.len() { + Ok(()) + } else { + Err(fmt::Error) + } + } +} + +impl Deref for ErrorSource { + type Target = dyn Error + Send + Sync; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl fmt::Debug for ErrorSource { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ErrorSource()") + } +} diff --git a/backend/crates/gateway-core/src/provider_ports.rs b/backend/crates/gateway-core/src/provider_ports.rs index 95ac0ae1f..6034824b5 100644 --- a/backend/crates/gateway-core/src/provider_ports.rs +++ b/backend/crates/gateway-core/src/provider_ports.rs @@ -28,17 +28,36 @@ pub enum ProviderStoreErrorKind { } /// Provider 存储端口的脱敏错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("provider store {operation} failed: {kind:?}")] pub struct ProviderStoreError { kind: ProviderStoreErrorKind, operation: &'static str, + source: Option, } impl ProviderStoreError { #[must_use] pub const fn new(kind: ProviderStoreErrorKind, operation: &'static str) -> Self { - Self { kind, operation } + Self { + kind, + operation, + source: None, + } + } + + /// 包装基础设施失败，分类与原始来源分别保留 + #[must_use] + pub fn caused_by( + kind: ProviderStoreErrorKind, + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self { + kind, + operation, + source: Some(crate::error::ErrorSource::new(source)), + } } #[must_use] diff --git a/backend/crates/gateway-core/tests/diagnostics/trace.rs b/backend/crates/gateway-core/tests/diagnostics/trace.rs index e323df27e..f08fb6745 100644 --- a/backend/crates/gateway-core/tests/diagnostics/trace.rs +++ b/backend/crates/gateway-core/tests/diagnostics/trace.rs @@ -6,6 +6,8 @@ use std::{ }; use gateway_core::diagnostics::{TraceContext, body_fingerprint}; +use gateway_core::error::{ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError}; +use gateway_core::upstream::UpstreamSendState; use serde_json::json; use tracing::{ Event, Metadata, Subscriber, @@ -47,6 +49,55 @@ fn bounded_history_preserves_start_failure_and_final_result() { ); } +#[test] +fn provider_failure_trace_keeps_classification_when_diagnostic_is_truncated() { + let trace = TraceContext::new("req_export"); + let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) + .with_diagnostic( + ProviderDiagnostic::new("Safe diagnostic".repeat(500)) + .with_classification("prepare", "account_proxy_invalid"), + ) + .with_raw_upstream_error(RawUpstreamError::new("PRIVATE_RAW")); + trace.attempt(3).record_provider_failure(&error); + let snapshot = trace.snapshot().unwrap(); + let data = &snapshot["events"][0]["data"]; + assert_eq!(data["kind"], "unavailable"); + assert_eq!(data["sendState"], "not_sent"); + assert_eq!(data["diagnostic"]["stage"], "prepare"); + assert_eq!(data["diagnostic"]["code"], "account_proxy_invalid"); + assert_eq!(data["diagnostic"]["truncated"], true); + assert!(!snapshot.to_string().contains("PRIVATE_RAW")); +} + +#[test] +fn oversized_failure_keeps_classification_and_marks_truncated_message() { + let trace = TraceContext::new("req_oversized_failure"); + trace.attempt(2).record("attempt.failed", json!({ + "kind": "transport", "sendState": "not_sent", "upstreamStatus": 502, + "diagnostic": {"stage": "connect", "code": "connection_refused", "message": "连接被拒绝".repeat(2000)}, + "rawError": {"summary": "x".repeat(10_000)}, + })); + let snapshot = trace.snapshot().unwrap(); + let event = &snapshot["events"][0]; + assert_eq!(event["attemptIndex"], 2); + let data = &event["data"]; + assert_eq!(data["truncated"], true); + assert_eq!(data["kind"], "transport"); + assert_eq!(data["sendState"], "not_sent"); + assert_eq!(data["upstreamStatus"], 502); + assert_eq!(data["diagnostic"]["stage"], "connect"); + assert_eq!(data["diagnostic"]["code"], "connection_refused"); + assert_eq!(data["diagnostic"]["truncated"], true); + assert!( + data["diagnostic"]["message"] + .as_str() + .unwrap() + .starts_with("连接被拒绝") + ); + assert!(serde_json::to_vec(data).unwrap().len() <= 4096); + assert!(data.get("rawError").is_none()); +} + #[test] fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() { let trace = TraceContext::new("req_trace"); @@ -314,6 +365,18 @@ fn ordinary_trace_logs_and_serialized_snapshots_share_the_sanitized_capture_boun ); exchange.capture_named_event("upstream.event", b"{}", Some("PRIVATE_SSE_EVENT")); exchange.wire_event("openai", Some("PRIVATE_WIRE_EVENT")); + exchange.record_provider_failure( + &ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent) + .with_upstream_code(gateway_core::error::OpaqueUpstreamValue::new( + "PRIVATE_CODE", + )) + .with_raw_upstream_error(RawUpstreamError::new("PRIVATE_UPSTREAM_BODY")) + .with_source(std::io::Error::other("PRIVATE_NATIVE_CAUSE")) + .with_diagnostic( + ProviderDiagnostic::new("upstream rejected request") + .with_classification("upstream", "upstream_failure"), + ), + ); exchange.headers( "upstream.response.headers", json!({"status": 429}), diff --git a/backend/crates/gateway-core/tests/engine/coordinator.rs b/backend/crates/gateway-core/tests/engine/coordinator.rs index 52f3a68c7..6050fca28 100644 --- a/backend/crates/gateway-core/tests/engine/coordinator.rs +++ b/backend/crates/gateway-core/tests/engine/coordinator.rs @@ -65,6 +65,7 @@ struct FinalState { image_output_tokens: Option, image_generation_succeeded: Option, provider_error_code: Option, + error_details: Option, retry_after_ms: Option, latency_ms: Option, client_response_id: Option, @@ -232,6 +233,7 @@ impl ExecutionStore for FakeStore { image_output_tokens: finalization.usage.image_output_tokens, image_generation_succeeded: finalization.image_generation_succeeded, provider_error_code: finalization.provider_error_code, + error_details: finalization.error_details, retry_after_ms: finalization.retry_after_ms, latency_ms: finalization.timings.latency_ms, client_response_id: finalization.client_response_id, @@ -297,6 +299,7 @@ struct ScriptedProvider { profile_generation: AtomicUsize, default_profile_calls: AtomicUsize, default_profile: Mutex>, + default_profile_error: Mutex>, scripts: Mutex>, contexts: Mutex>, operations: Mutex>, @@ -323,6 +326,7 @@ impl ScriptedProvider { profile_generation: AtomicUsize::new(1), default_profile_calls: AtomicUsize::new(0), default_profile: Mutex::new(None), + default_profile_error: Mutex::new(None), scripts: Mutex::new(scripts.into()), contexts: Mutex::new(Vec::new()), operations: Mutex::new(Vec::new()), @@ -349,6 +353,9 @@ impl Provider for ScriptedProvider { &self, ) -> Result, ProviderError> { self.default_profile_calls.fetch_add(1, Ordering::SeqCst); + if let Some(error) = self.default_profile_error.lock().unwrap().take() { + return Err(error); + } Ok(self.default_profile.lock().unwrap().clone()) } @@ -2849,6 +2856,7 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() { ProviderErrorKind::RateLimited, UpstreamSendState::Sent, ) + .with_source(std::io::Error::other("ORIGINAL_RATE_LIMIT_CAUSE")) .with_status(429) .with_upstream_code(OpaqueUpstreamValue::new("rate_limit_exceeded")) .with_retry_after(Duration::from_secs(30)) @@ -2894,6 +2902,12 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() { state.finalizations[0].provider_error_code.as_deref(), Some("rate_limit_exceeded") ); + let details: Value = + serde_json::from_str(state.finalizations[0].error_details.as_deref().unwrap()).unwrap(); + assert_eq!( + details["causes"]["messages"], + json!(["ORIGINAL_RATE_LIMIT_CAUSE"]) + ); } #[test] @@ -4905,6 +4919,59 @@ fn global_location_and_group_fast_policy_reach_every_account_retry() { } } +#[test] +fn profile_preparation_failure_is_traced_and_finalized_without_starting_an_attempt() { + let operation = generate_operation(); + let route_plan = plan(&operation); + let (coordinator, store, provider) = coordinator(vec![]); + *provider.default_profile_error.lock().unwrap() = Some( + ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) + .with_diagnostic( + gateway_core::error::ProviderDiagnostic::new("Profile release unavailable") + .with_classification("prepare", "request_profile_release_unavailable"), + ), + ); + let mut session = block_on(coordinator.start( + model_request(&operation, SystemTime::now() + Duration::from_secs(30)), + operation, + route_plan, + None, + None, + CancellationToken::new(), + )) + .unwrap(); + assert!(block_on(session.collect_uncommitted()).is_err()); + assert!(provider.contexts.lock().unwrap().is_empty()); + let state = store.state.lock().unwrap(); + assert_eq!(state.finalizations.len(), 1); + let trace: Value = serde_json::from_str( + state.finalizations[0] + .diagnostic_trace_json + .as_deref() + .unwrap(), + ) + .unwrap(); + let events = trace["events"].as_array().unwrap(); + let failure = events + .iter() + .find(|event| event["stage"] == "attempt.failed") + .unwrap(); + assert_eq!(failure["attemptIndex"], 0); + assert_eq!( + failure["data"]["diagnostic"]["code"], + "request_profile_release_unavailable" + ); + assert_eq!( + failure["data"]["diagnostic"]["message"], + "Profile release unavailable" + ); + assert!( + events + .iter() + .all(|event| event["stage"] != "attempt.started") + ); +} + #[test] fn first_resolved_profile_is_frozen_across_account_retries() { use gateway_core::account::OpaqueProviderData; diff --git a/backend/crates/gateway-core/tests/error.rs b/backend/crates/gateway-core/tests/error/mod.rs similarity index 66% rename from backend/crates/gateway-core/tests/error.rs rename to backend/crates/gateway-core/tests/error/mod.rs index 2e7aa6d64..d95d24486 100644 --- a/backend/crates/gateway-core/tests/error.rs +++ b/backend/crates/gateway-core/tests/error/mod.rs @@ -9,6 +9,91 @@ use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHead use gateway_core::upstream::UpstreamSendState; use serde_json::json; +#[test] +fn source_chain_survives_domain_conversion_and_snapshot_without_entering_formatting() { + use gateway_core::error::{StoreError, StoreErrorKind}; + use gateway_core::provider_ports::{ProviderStoreError, ProviderStoreErrorKind}; + use std::error::Error as _; + + let store = StoreError::caused_by( + StoreErrorKind::Unavailable, + std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "PRIVATE_DB_CAUSE"), + ); + let port = + ProviderStoreError::caused_by(ProviderStoreErrorKind::Unavailable, "select account", store); + let error = ProviderError::new( + ProviderErrorKind::ProviderInfrastructureUnavailable, + UpstreamSendState::NotSent, + ) + .with_source(port); + let snapshot = error.stable_snapshot(); + drop(error); + + let port = snapshot + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + let store = port.source().unwrap().downcast_ref::().unwrap(); + let original = store + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert_eq!(original.kind(), std::io::ErrorKind::ConnectionRefused); + assert_eq!(original.to_string(), "PRIVATE_DB_CAUSE"); + assert!( + !format!("{snapshot:?} {snapshot} {port:?} {port} {store:?} {store}") + .contains("PRIVATE_DB_CAUSE") + ); + assert_eq!(snapshot.send_state(), UpstreamSendState::NotSent); +} + +#[test] +fn restricted_details_preserve_causes_and_verbatim_upstream_body_separately() { + let body = "{ \"error\": {\"code\":\"Vendor.Unknown\",\"message\":\"PRIVATE_BODY\"} }"; + let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent) + .with_source(std::io::Error::other("PRIVATE_CAUSE")) + .with_raw_upstream_error(RawUpstreamError::new(body)); + let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap(); + assert_eq!(details["upstream"], body); + assert_eq!(details["causes"]["messages"], json!(["PRIVATE_CAUSE"])); + assert_eq!(details["causes"]["truncated"], false); + assert!(!format!("{error:?} {error}").contains("PRIVATE_")); + assert!( + ProviderError::new(ProviderErrorKind::Cancelled, UpstreamSendState::NotSent) + .error_details() + .is_none() + ); +} + +#[test] +fn restricted_cause_snapshot_bounds_unicode_and_cycles_with_an_explicit_marker() { + let error = ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) + .with_source(std::io::Error::other("原始原因".repeat(20_000))); + let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap(); + assert_eq!(details["causes"]["truncated"], true); + assert!(details["causes"]["messages"][0].as_str().unwrap().len() <= 64 * 1024); + + #[derive(Debug)] + struct CyclicError; + impl std::fmt::Display for CyclicError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("cyclic source") + } + } + impl std::error::Error for CyclicError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self) + } + } + let error = ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) + .with_source(CyclicError); + let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap(); + assert_eq!(details["causes"]["truncated"], true); + assert_eq!(details["causes"]["messages"].as_array().unwrap().len(), 32); +} + #[test] fn provider_error_debug_should_not_expose_sensitive_context() { let secret = "sk-do-not-log-this"; @@ -55,6 +140,28 @@ fn classified_provider_diagnostic_survives_stable_snapshot_without_entering_debu assert!(!format!("{gateway:?}").contains(message)); } +#[test] +fn io_diagnostic_keeps_os_cause_without_copying_custom_error_text() { + let os_error = std::fs::File::open( + std::env::temp_dir().join(format!("cpr-missing-ca-{}/missing.pem", std::process::id())), + ) + .unwrap_err(); + let diagnostic = ProviderDiagnostic::new("CA read failed") + .with_classification("prepare", "custom_ca_read_failed") + .with_io_cause(&os_error); + assert!(diagnostic.as_str().contains(&os_error.to_string())); + assert_eq!(diagnostic.stage(), Some("prepare")); + assert_eq!(diagnostic.code(), Some("io_not_found")); + + let private = std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + "PRIVATE_URL_AND_CREDENTIAL", + ); + let diagnostic = ProviderDiagnostic::new("Connect failed").with_io_cause(&private); + assert_eq!(diagnostic.code(), Some("connection_refused")); + assert!(!diagnostic.as_str().contains("PRIVATE_")); +} + #[test] fn raw_upstream_error_survives_stable_snapshot_without_entering_debug() { let raw = r#"{"error":{"message":"verbatim upstream marker"}}"#; diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs index a19860c96..c14efd417 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/event.rs @@ -1,24 +1,22 @@ -//! 校验并解码插件上游事件，转换为宿主执行事件与错误 +//! 校验插件流式事件，生成领域事件并保持序列约束 -use std::time::Duration; +use super::failure::{failure_error, invalid}; use bytes::Bytes; use gateway_core::{ engine::upstream_adapter::UpstreamAccountConnection, - error::{ClientVisibleUpstreamError, ProviderError, ProviderErrorKind}, + error::ProviderError, event::{ ContentItem, ContentKind, EventSequenceValidator, FinishReason, GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseObservation, ReasoningDelta, ResponseMeta, TextDelta, ToolCallDelta, }, metering::Usage, - upstream::{OpaqueUpstreamValue, UpstreamSendState, UpstreamTransport}, + upstream::{UpstreamSendState, UpstreamTransport}, }; use gateway_plugin_sdk::call::{ model::{CanonicalEvent, WireEvent, WirePayload}, - upstream_adapter::{ - UpstreamAdapterEvent, UpstreamContinuation, UpstreamFailure, UpstreamFailureKind, - }, + upstream_adapter::{UpstreamAdapterEvent, UpstreamContinuation}, }; use gateway_protocol::openai::sse::SseEventDecoder; @@ -57,7 +55,8 @@ impl EventDecoder { account: &dyn UpstreamAccountConnection, sent: UpstreamSendState, ) -> Result { - let message = UpstreamAdapterEvent::decode(payload).map_err(|_| invalid(sent))?; + let message = UpstreamAdapterEvent::decode(payload) + .map_err(|source| invalid(sent).with_source(source))?; if self.completed || message.event.facts.len() > 64 { return Err(invalid(sent).into()); } @@ -185,7 +184,9 @@ impl EventDecoder { account.calculate_cost(self.service_tier.as_deref(), &self.usage) { let cost = GatewayEvent::CalculatedCost(cost); - self.sequence.observe(&cost).map_err(|_| invalid(sent))?; + self.sequence + .observe(&cost) + .map_err(|source| invalid(sent).with_source(source))?; facts.push(cost); } self.completed = true; @@ -204,7 +205,9 @@ impl EventDecoder { })) } }; - self.sequence.observe(&fact).map_err(|_| invalid(sent))?; + self.sequence + .observe(&fact) + .map_err(|source| invalid(sent).with_source(source))?; facts.push(fact); } if message.continuation.is_some() && !self.completed { @@ -219,10 +222,11 @@ impl EventDecoder { }; if let Some(tier) = &self.service_tier { let observation = ProviderResponseObservation::new( - UpstreamTransport::new(transport).map_err(|_| invalid(sent))?, + UpstreamTransport::new(transport) + .map_err(|source| invalid(sent).with_source(source))?, ) .try_with_service_tier(tier.clone()) - .map_err(|_| invalid(sent))?; + .map_err(|source| invalid(sent).with_source(source))?; event.attach_observation(observation); } Ok(DecodedEvent { @@ -232,7 +236,9 @@ impl EventDecoder { } pub(super) fn finish(&self, sent: UpstreamSendState) -> Result<(), ProviderError> { - self.sequence.finish().map_err(|_| invalid(sent)) + self.sequence + .finish() + .map_err(|source| invalid(sent).with_source(source)) } } @@ -307,14 +313,15 @@ fn decode_wire( ProtocolWireEvent::raw_sse(protocol, Bytes::from(frame)) } WirePayload::RawJson { body } => { - serde_json::from_slice::(&body).map_err(|_| invalid(sent))?; + serde_json::from_slice::(&body) + .map_err(|source| invalid(sent).with_source(source))?; ProtocolWireEvent::raw_json(protocol, Bytes::from(body)) } WirePayload::RawBody { body } => { ProtocolWireEvent::raw_http_body(protocol, Bytes::from(body)) } }; - result.map_err(|_| invalid(sent)) + result.map_err(|source| invalid(sent).with_source(source)) } fn single_sse( @@ -327,51 +334,3 @@ fn single_sse( } frames.pop().ok_or_else(|| invalid(sent)) } - -fn failure_error( - failure: UpstreamFailure, - sent: UpstreamSendState, -) -> Result { - if failure.message.len() > 64 * 1024 - || failure.code.as_ref().is_some_and(|code| code.len() > 256) - || failure - .status - .is_some_and(|status| !(400..=599).contains(&status)) - || failure - .retry_after_ms - .is_some_and(|delay| delay > 24 * 60 * 60 * 1000) - { - return Err(invalid(sent)); - } - let kind = match failure.kind { - UpstreamFailureKind::InvalidRequest => ProviderErrorKind::InvalidRequest, - UpstreamFailureKind::Unsupported => ProviderErrorKind::Unsupported, - UpstreamFailureKind::Unauthorized => ProviderErrorKind::Unauthorized, - UpstreamFailureKind::PermissionDenied => ProviderErrorKind::PermissionDenied, - UpstreamFailureKind::RateLimited => ProviderErrorKind::RateLimited, - UpstreamFailureKind::QuotaExhausted => ProviderErrorKind::QuotaExhausted, - UpstreamFailureKind::Timeout => ProviderErrorKind::Timeout, - UpstreamFailureKind::Unavailable => ProviderErrorKind::Unavailable, - UpstreamFailureKind::Protocol => ProviderErrorKind::Protocol, - }; - let mut error = ProviderError::new(kind, sent); - if let Some(code) = &failure.code { - error = error.with_upstream_code(OpaqueUpstreamValue::new(code.clone())); - } - error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new( - failure.message, - failure.code, - None, - )); - if let Some(status) = failure.status { - error = error.with_status(status); - } - if let Some(delay) = failure.retry_after_ms { - error = error.with_retry_after(Duration::from_millis(delay)); - } - Ok(error) -} - -pub(super) fn invalid(sent: UpstreamSendState) -> ProviderError { - ProviderError::new(ProviderErrorKind::Protocol, sent) -} diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs index 1d61dbc91..cb2d99cf3 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/execution.rs @@ -10,7 +10,7 @@ use gateway_core::{ upstream::UpstreamSendState, }; use gateway_plugin_sdk::{ - CallContext, ErrorCode, PluginFault, Stage, + CallContext, Stage, call::upstream_adapter::{ ContinuationScope, EXECUTE_METHOD, UpstreamAdapterRequest, UpstreamContinuation, UpstreamTransport, @@ -20,10 +20,11 @@ use serde::{Deserialize, Serialize}; use super::{ PluginUpstreamAdapter, - event::{DecodedEvent, EventDecodeError, EventDecoder, invalid}, + event::{DecodedEvent, EventDecodeError, EventDecoder}, + failure::{fault_error, invalid, rpc_error}, }; use crate::{ - RpcError, RpcStream, + RpcStream, callback::{ CallbackScope, upstream::{ConnectionOwner, ManagedUpstream}, @@ -240,10 +241,10 @@ impl Execution { invocation .operation .middleware_body() - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .to_vec(), ) - .map_err(|_| invalid())?; + .map_err(|source| invalid().with_source(source))?; invocation.context.trace().record("plugin.upstream", serde_json::json!({ "instanceId": self.adapter.instance_id, "adapterId": declaration.id, "generation": context.generation, "transport": declaration.transport.as_str() })); let stream = tokio::select! { biased; @@ -252,7 +253,7 @@ impl Execution { }; if stream.initial.result != serde_json::json!({}) || !stream.initial.payload.is_empty() { self.adapter.session.invalid_response(Stage::Upstream); - return Err(super::event::invalid(managed.send_state.get())); + return Err(super::failure::invalid(managed.send_state.get())); } Ok(ActiveCall { context, @@ -292,7 +293,7 @@ fn previous_continuation( } let stored: StoredContinuation = serde_json::from_value(serde_json::Value::Object(state.payload().clone())) - .map_err(|_| invalid())?; + .map_err(|source| invalid().with_source(source))?; if stored.client_key_id != owner.client_key_id || stored.account_id != owner.account_id || stored.credential_revision != owner.credential_revision @@ -325,7 +326,7 @@ async fn attach_continuation( if continuation.upstream_response_id.is_empty() || continuation.upstream_response_id.len() > 512 || serde_json::to_vec(&continuation) - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .len() > 32 * 1024 { @@ -350,12 +351,13 @@ async fn attach_continuation( account_id: invocation.account.account_id().as_str().to_owned(), credential_revision: invocation.account.credential_revision().get(), }; - let serde_json::Value::Object(payload) = serde_json::to_value(state).map_err(|_| invalid())? + let serde_json::Value::Object(payload) = + serde_json::to_value(state).map_err(|source| invalid().with_source(source))? else { return Err(invalid()); }; let state = ProviderSessionState::new(adapter.declaration.provider.as_str(), payload) - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .with_extension_owner(extension_owner(adapter, &active.context, local)); decoded.event.attach_session_update(state); Ok(()) @@ -375,25 +377,3 @@ fn extension_owner( connection_local, } } - -fn rpc_error(error: RpcError, sent: UpstreamSendState) -> ProviderError { - match error { - RpcError::Remote(fault) => fault_error(fault, sent), - RpcError::Timeout => ProviderError::new(ProviderErrorKind::Timeout, sent), - RpcError::Cancelled => ProviderError::new(ProviderErrorKind::Cancelled, sent), - _ => ProviderError::new(ProviderErrorKind::Unavailable, sent), - } -} - -fn fault_error(fault: PluginFault, sent: UpstreamSendState) -> ProviderError { - let kind = match fault.code { - ErrorCode::Timeout => ProviderErrorKind::Timeout, - ErrorCode::Cancelled => ProviderErrorKind::Cancelled, - ErrorCode::Unsupported => ProviderErrorKind::Unsupported, - ErrorCode::InvalidInput | ErrorCode::Rejected | ErrorCode::PermissionDenied => { - ProviderErrorKind::InvalidRequest - } - _ => ProviderErrorKind::Unavailable, - }; - ProviderError::new(kind, sent) -} diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/failure.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/failure.rs new file mode 100644 index 000000000..694e3a42a --- /dev/null +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/failure.rs @@ -0,0 +1,115 @@ +//! 插件上游适配的唯一错误投影边界，保留来源并生成安全分类 + +use std::time::Duration; + +use crate::RpcError; +use gateway_core::{ + error::{ + ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic, ProviderError, + ProviderErrorKind, RawUpstreamError, + }, + upstream::UpstreamSendState, +}; +use gateway_plugin_sdk::{ + ErrorCode, PluginFault, + call::upstream_adapter::{UpstreamFailure, UpstreamFailureKind}, +}; + +pub(super) fn rpc_error(error: RpcError, sent: UpstreamSendState) -> ProviderError { + let (code, message) = error.diagnostic(); + let kind = match error { + RpcError::Remote(fault) => return fault_error(fault, sent), + RpcError::Timeout => ProviderErrorKind::Timeout, + RpcError::Cancelled => ProviderErrorKind::Cancelled, + _ => ProviderErrorKind::Unavailable, + }; + ProviderError::new(kind, sent) + .with_diagnostic(ProviderDiagnostic::new(message).with_classification("plugin_rpc", code)) + .with_source(error) +} + +pub(super) fn fault_error(fault: PluginFault, sent: UpstreamSendState) -> ProviderError { + let kind = match fault.code { + ErrorCode::Timeout => ProviderErrorKind::Timeout, + ErrorCode::Cancelled => ProviderErrorKind::Cancelled, + ErrorCode::Unsupported => ProviderErrorKind::Unsupported, + ErrorCode::InvalidInput | ErrorCode::Rejected | ErrorCode::PermissionDenied => { + ProviderErrorKind::InvalidRequest + } + _ => ProviderErrorKind::Unavailable, + }; + // 仅保存响应方向的插件 fault，普通日志与公共错误继续使用宿主分类 + let raw = serde_json::json!({"source": "plugin_upstream_adapter", "fault": fault}).to_string(); + ProviderError::new(kind, sent) + .with_diagnostic( + ProviderDiagnostic::new(format!( + "Plugin upstream adapter returned a fault: {:?}", + fault.code, + )) + .with_classification("plugin_rpc", "remote"), + ) + .with_raw_upstream_error(RawUpstreamError::new(raw)) +} + +pub(super) fn failure_error( + failure: UpstreamFailure, + sent: UpstreamSendState, +) -> Result { + if failure.message.len() > 64 * 1024 + || failure.code.as_ref().is_some_and(|code| code.len() > 256) + || failure + .status + .is_some_and(|status| !(400..=599).contains(&status)) + || failure + .retry_after_ms + .is_some_and(|delay| delay > 24 * 60 * 60 * 1000) + { + return Err(invalid(sent)); + } + let kind = match failure.kind { + UpstreamFailureKind::InvalidRequest => ProviderErrorKind::InvalidRequest, + UpstreamFailureKind::Unsupported => ProviderErrorKind::Unsupported, + UpstreamFailureKind::Unauthorized => ProviderErrorKind::Unauthorized, + UpstreamFailureKind::PermissionDenied => ProviderErrorKind::PermissionDenied, + UpstreamFailureKind::RateLimited => ProviderErrorKind::RateLimited, + UpstreamFailureKind::QuotaExhausted => ProviderErrorKind::QuotaExhausted, + UpstreamFailureKind::Timeout => ProviderErrorKind::Timeout, + UpstreamFailureKind::Unavailable => ProviderErrorKind::Unavailable, + UpstreamFailureKind::Protocol => ProviderErrorKind::Protocol, + }; + // 插件已经解析的上游错误与客户端投影分别承载，消费 message 前保存原始返回 + let raw = + serde_json::to_string(&failure).map_err(|source| invalid(sent).with_source(source))?; + let mut error = ProviderError::new(kind, sent) + .with_raw_upstream_error(RawUpstreamError::new(raw)) + .with_diagnostic( + ProviderDiagnostic::new(format!( + "Plugin upstream failure: kind={}, status={:?}", + kind.as_str(), + failure.status, + )) + .with_classification("upstream", "plugin_upstream_failure"), + ); + if let Some(code) = &failure.code { + error = error.with_upstream_code(OpaqueUpstreamValue::new(code.clone())); + } + error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new( + failure.message, + failure.code, + None, + )); + if let Some(status) = failure.status { + error = error.with_status(status); + } + if let Some(delay) = failure.retry_after_ms { + error = error.with_retry_after(Duration::from_millis(delay)); + } + Ok(error) +} + +pub(super) fn invalid(sent: UpstreamSendState) -> ProviderError { + ProviderError::new(ProviderErrorKind::Protocol, sent).with_diagnostic( + ProviderDiagnostic::new("Plugin upstream adapter returned an invalid event") + .with_classification("plugin_protocol", "invalid_event"), + ) +} diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/mod.rs b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/mod.rs index f225c0abf..d472985e8 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/mod.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/upstream_adapter/mod.rs @@ -2,6 +2,7 @@ mod event; mod execution; +mod failure; mod registration; pub(crate) mod target; diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs index 46883a3b2..2bd0af2c9 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs @@ -69,6 +69,44 @@ pub enum RpcError { Remote(PluginFault), } +impl RpcError { + /// 会话状态与请求错误共用安全分类，不展开插件返回正文 + pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) { + match self { + RpcError::Start(_) => ("process_start", "插件进程启动失败"), + RpcError::Handshake => ("handshake", "插件握手失败"), + RpcError::Protocol => ("protocol", "插件协议错误"), + RpcError::InvalidResponse(stage) => ( + "invalid_response", + match stage { + Stage::Registration => "注册阶段：插件返回了无效响应", + Stage::Configuration => "配置阶段：插件返回了无效响应", + Stage::Authentication => "认证阶段：插件返回了无效响应", + Stage::Routing => "路由阶段：插件返回了无效响应", + Stage::Scheduling => "调度阶段：插件返回了无效响应", + Stage::Retry => "重试阶段：插件返回了无效响应", + Stage::Http => "HTTP 中间件：插件返回了无效响应", + Stage::Service => "服务中间件：插件返回了无效响应", + Stage::WebSocket => "WebSocket 中间件：插件返回了无效响应", + Stage::Request => "请求中间件：插件返回了无效响应", + Stage::Attempt => "尝试中间件：插件返回了无效响应", + Stage::Upstream => "上游适配阶段：插件返回了无效响应", + Stage::Observation => "观察阶段：插件返回了无效响应", + Stage::Management | Stage::PublicManagement => "管理阶段：插件返回了无效响应", + Stage::CommandLine => "命令行阶段：插件返回了无效响应", + Stage::Maintenance => "维护阶段：插件返回了无效响应", + }, + ), + RpcError::Closed => ("closed", "插件进程或传输已停止"), + RpcError::Timeout => ("timeout", "插件调用超时"), + RpcError::Cancelled => ("cancelled", "插件会话已取消"), + RpcError::Capacity => ("capacity", "插件调用容量已耗尽"), + RpcError::Context => ("context", "插件调用上下文无效"), + RpcError::Remote(_) => ("remote", "插件返回错误"), + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum RpcSessionDiagnostic { Ready, @@ -657,40 +695,7 @@ impl RpcSession { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(error) = &state.failure { - let (code, message) = match error { - RpcError::Start(_) => ("process_start", "插件进程启动失败"), - RpcError::Handshake => ("handshake", "插件握手失败"), - RpcError::Protocol => ("protocol", "插件协议错误"), - RpcError::InvalidResponse(stage) => ( - "invalid_response", - match stage { - Stage::Registration => "注册阶段：插件返回了无效响应", - Stage::Configuration => "配置阶段：插件返回了无效响应", - Stage::Authentication => "认证阶段：插件返回了无效响应", - Stage::Routing => "路由阶段：插件返回了无效响应", - Stage::Scheduling => "调度阶段：插件返回了无效响应", - Stage::Retry => "重试阶段：插件返回了无效响应", - Stage::Http => "HTTP 中间件：插件返回了无效响应", - Stage::Service => "服务中间件：插件返回了无效响应", - Stage::WebSocket => "WebSocket 中间件：插件返回了无效响应", - Stage::Request => "请求中间件：插件返回了无效响应", - Stage::Attempt => "尝试中间件：插件返回了无效响应", - Stage::Upstream => "上游适配阶段：插件返回了无效响应", - Stage::Observation => "观察阶段：插件返回了无效响应", - Stage::Management | Stage::PublicManagement => { - "管理阶段：插件返回了无效响应" - } - Stage::CommandLine => "命令行阶段：插件返回了无效响应", - Stage::Maintenance => "维护阶段：插件返回了无效响应", - }, - ), - RpcError::Closed => ("closed", "插件进程或传输已停止"), - RpcError::Timeout => ("timeout", "插件调用超时"), - RpcError::Cancelled => ("cancelled", "插件会话已取消"), - RpcError::Capacity => ("capacity", "插件调用容量已耗尽"), - RpcError::Context => ("context", "插件调用上下文无效"), - RpcError::Remote(_) => ("remote", "插件返回错误"), - }; + let (code, message) = error.diagnostic(); return RpcSessionDiagnostic::Failed { code, message }; } if self.slots.is_closed() { diff --git a/backend/crates/gateway-plugin/runtime/tests/generation/upstream_adapter.rs b/backend/crates/gateway-plugin/runtime/tests/generation/upstream_adapter.rs index 275c8e131..d287a56aa 100644 --- a/backend/crates/gateway-plugin/runtime/tests/generation/upstream_adapter.rs +++ b/backend/crates/gateway-plugin/runtime/tests/generation/upstream_adapter.rs @@ -378,6 +378,15 @@ async fn adapter_failure_preserves_upstream_diagnostics_and_native_feedback() { "rate_limit_exceeded" ); assert_eq!(error.retry_after(), Some(Duration::from_millis(1500))); + let snapshot = error.stable_snapshot(); + let raw: Value = serde_json::from_str(snapshot.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(raw["message"], "fixture rate limit"); + assert_eq!(raw["code"], "rate_limit_exceeded"); + assert_eq!(raw["status"], 429); + let diagnostic = snapshot.diagnostic().unwrap(); + assert_eq!(diagnostic.code(), Some("plugin_upstream_failure")); + assert!(!diagnostic.as_str().contains("fixture rate limit")); + assert!(!format!("{snapshot:?}").contains("fixture rate limit")); assert!(generation.is_ready(), "上游 429 不能停止插件"); assert!(generation.can_serve()); assert_eq!(account.failures.load(Ordering::SeqCst), 1); @@ -464,6 +473,18 @@ async fn adapter_does_not_publish_completed_when_rpc_fails_after_terminal_frame( assert_eq!(error.kind(), ProviderErrorKind::Unavailable); assert_eq!(error.send_state(), UpstreamSendState::Sent); let mut error = error; + let raw: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(raw["source"], "plugin_upstream_adapter"); + assert_eq!(raw["fault"]["message"], "fixture terminal fault"); + assert_eq!(error.diagnostic().unwrap().stage(), Some("plugin_rpc")); + assert!( + !error + .diagnostic() + .unwrap() + .as_str() + .contains("fixture terminal fault") + ); + assert!(!format!("{error:?}").contains("fixture terminal fault")); let facts = error .take_atomic_client_events() .into_iter() @@ -960,38 +981,56 @@ async fn upstream_adapter_can_dispatch_http_after_its_instance_entered_the_scope #[tokio::test] async fn malformed_adapter_events_stop_only_the_plugin_session() { use gateway_admin::ports::plugins::PluginRuntimeDiagnostics as _; - let server = MockServer::start().await; - Mock::given(method("POST")) - .respond_with(ResponseTemplate::new(200).set_body_string("{}")) - .mount(&server) - .await; - let mut config = configuration(&server.uri(), false); - config["upstream_events"] = json!([{ - "event":{"facts":[{"type":"completed","id":"never-started","model":"native-model","reason":"stop"}]} - }]); - let (cache, store, runtime) = setup(config).await; - let generation = ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap()) - .await - .unwrap(); - let mut stream = execute(&generation, Account::new(), "key-one", None); - assert_eq!( - stream.next().await.unwrap().unwrap_err().kind(), - ProviderErrorKind::Protocol - ); - assert!(!generation.is_ready()); - assert!(generation.can_serve()); - let snapshot = store.snapshot.lock().unwrap().clone(); - assert!(snapshot.instances[0].enabled); - let diagnostics = runtime - .runtime_diagnostics(&snapshot, Some(1), Some(&generation)) - .await - .unwrap(); - assert_eq!( - diagnostics["instance-one"].failure.as_ref().unwrap().code, - "invalid_response" - ); - drop(stream); - drop(generation); - runtime.shutdown().await; - super::wait_until_empty(cache.path()).await; + use std::error::Error as _; + for (fact, has_sequence_error) in [ + ( + json!({"type":"completed","id":"never-started","model":"native-model","reason":"stop"}), + false, + ), + ( + json!({"type":"text_delta","index":0,"text":"delta before start"}), + true, + ), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .mount(&server) + .await; + let mut config = configuration(&server.uri(), false); + config["upstream_events"] = json!([{"event":{"facts":[fact]}}]); + let (cache, store, runtime) = setup(config).await; + let generation = + ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap()) + .await + .unwrap(); + let mut stream = execute(&generation, Account::new(), "key-one", None); + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.kind(), ProviderErrorKind::Protocol); + if has_sequence_error { + assert!( + error + .source() + .unwrap() + .is::() + ); + } + assert_eq!(error.diagnostic().unwrap().code(), Some("invalid_event")); + assert!(!generation.is_ready()); + assert!(generation.can_serve()); + let snapshot = store.snapshot.lock().unwrap().clone(); + assert!(snapshot.instances[0].enabled); + let diagnostics = runtime + .runtime_diagnostics(&snapshot, Some(1), Some(&generation)) + .await + .unwrap(); + assert_eq!( + diagnostics["instance-one"].failure.as_ref().unwrap().code, + "invalid_response" + ); + drop(stream); + drop(generation); + runtime.shutdown().await; + super::wait_until_empty(cache.path()).await; + } } diff --git a/backend/crates/gateway-store/src/backup/staging.rs b/backend/crates/gateway-store/src/backup/staging.rs index 53161f0ac..b20f83611 100644 --- a/backend/crates/gateway-store/src/backup/staging.rs +++ b/backend/crates/gateway-store/src/backup/staging.rs @@ -24,15 +24,16 @@ impl StagingArea { /// /// 目录创建或权限设置失败时返回 [`StoreError`] pub fn open(base_dir: PathBuf, max_archive_bytes: u64) -> StoreResult { - std::fs::create_dir_all(&base_dir).map_err(|_| unavailable("create staging directory"))?; + std::fs::create_dir_all(&base_dir) + .map_err(|source| unavailable("create staging directory", source))?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(&base_dir, std::fs::Permissions::from_mode(0o700)) - .map_err(|_| unavailable("set staging directory permissions"))?; + .map_err(|source| unavailable("set staging directory permissions", source))?; } let metadata = std::fs::metadata(&base_dir) - .map_err(|_| unavailable("read staging directory metadata"))?; + .map_err(|source| unavailable("read staging directory metadata", source))?; if !metadata.is_dir() { return Err(invalid("staging path is not a directory")); } @@ -70,7 +71,7 @@ impl StagingArea { /// 剩余空间不足或读取失败时返回 [`crate::StoreError`] pub fn ensure_capacity(&self) -> StoreResult<()> { let free = fs2::available_space(&self.base_dir) - .map_err(|_| unavailable("read staging free space"))?; + .map_err(|source| unavailable("read staging free space", source))?; if free < MIN_STAGING_FREE_BYTES { return Err(StoreError::InvalidData { entity: "backup staging", @@ -93,8 +94,11 @@ impl StagingArea { } } -fn unavailable(operation: &'static str) -> StoreError { - crate::postgres_unavailable(operation) +fn unavailable( + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> StoreError { + crate::postgres_unavailable(operation, source) } fn invalid(message: &str) -> StoreError { diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs index 3fb23b57f..4a6406a21 100644 --- a/backend/crates/gateway-store/src/bundle.rs +++ b/backend/crates/gateway-store/src/bundle.rs @@ -113,11 +113,11 @@ async fn connect( config.pool.acquire_timeout(), )?; let redis_client = ::redis::Client::open(config.redis_url()?) - .map_err(|_| redis_unavailable("create Redis client"))?; + .map_err(|source| redis_unavailable("create Redis client", source))?; let redis_connection = redis_client .get_connection_manager() .await - .map_err(|_| redis_unavailable("connect Redis manager"))?; + .map_err(|source| redis_unavailable("connect Redis manager", source))?; let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone())); let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new( diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index 7bd1c7d9c..fcae8fe90 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -90,10 +90,9 @@ impl PgAccountGroupRepository { where F: for<'a> FnOnce(&'a mut Transaction<'_, Postgres>) -> BoxFuture<'a, StoreResult<()>>, { - let mut transaction = - self.pool.begin().await.map_err(|_| { - admin_store_error(ENTITY, unavailable("begin account group mutation")) - })?; + let mut transaction = self.pool.begin().await.map_err(|source| { + admin_store_error(ENTITY, unavailable("begin account group mutation", source)) + })?; let result = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; mutation(&mut transaction).await?; @@ -103,14 +102,17 @@ impl PgAccountGroupRepository { .await; match result { Ok(revision) => { - transaction.commit().await.map_err(|_| { - admin_store_error(ENTITY, unavailable("commit account group mutation")) + transaction.commit().await.map_err(|source| { + admin_store_error(ENTITY, unavailable("commit account group mutation", source)) })?; admin_revision(revision) } Err(error) => { - transaction.rollback().await.map_err(|_| { - admin_store_error(ENTITY, unavailable("rollback account group mutation")) + transaction.rollback().await.map_err(|source| { + admin_store_error( + ENTITY, + unavailable("rollback account group mutation", source), + ) })?; Err(admin_store_error(ENTITY, error)) } @@ -139,7 +141,9 @@ impl AccountGroupStore for PgAccountGroupRepository { .build() .fetch_all(&self.pool) .await - .map_err(|_| admin_store_error(ENTITY, unavailable("list account groups")))?; + .map_err(|source| { + admin_store_error(ENTITY, unavailable("list account groups", source)) + })?; let mut items = rows .iter() .map(group_record) @@ -201,7 +205,7 @@ impl AccountGroupStore for PgAccountGroupRepository { .bind(group_ids) .fetch_all(&self.pool) .await - .map_err(|_| admin_store_error(ENTITY, unavailable("load account group members")))?; + .map_err(|source| admin_store_error(ENTITY, unavailable("load account group members", source)))?; rows.into_iter() .map(|row| { let group_id = AccountGroupId::new( @@ -336,7 +340,7 @@ impl AccountGroupStore for PgAccountGroupRepository { .bind(command.enabled) .execute(&mut **transaction) .await - .map_err(|_| unavailable("set account group state"))?; + .map_err(|source| unavailable("set account group state", source))?; require_one(result.rows_affected(), command.id.as_str()) }) }) @@ -373,7 +377,7 @@ impl AccountGroupStore for PgAccountGroupRepository { .bind(command.id.as_str()) .execute(&mut **transaction) .await - .map_err(|_| unavailable("delete account group"))?; + .map_err(|source| unavailable("delete account group", source))?; if result.rows_affected() == 1 { return Ok(()); } @@ -383,7 +387,7 @@ impl AccountGroupStore for PgAccountGroupRepository { .bind(command.id.as_str()) .fetch_one(&mut **transaction) .await - .map_err(|_| unavailable("check account group delete conflict"))?; + .map_err(|source| unavailable("check account group delete conflict", source))?; if exists { Err(conflict(command.id.as_str())) } else { @@ -454,7 +458,7 @@ async fn count_groups(pool: &PgPool, query: &AccountGroupListQuery) -> StoreResu .build_query_scalar::() .fetch_one(pool) .await - .map_err(|_| unavailable("count account groups"))?; + .map_err(|source| unavailable("count account groups", source))?; u64::try_from(count).map_err(|_| invalid("negative account group count")) } @@ -466,7 +470,7 @@ async fn load_record(pool: &PgPool, id: &str) -> StoreResult StoreError { { conflict(id) } else { - unavailable("write account group") + unavailable("write account group", error) } } @@ -678,8 +682,11 @@ fn invalid(message: &str) -> StoreError { } } -fn unavailable(message: &'static str) -> StoreError { - postgres_unavailable(message) +fn unavailable( + message: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> StoreError { + postgres_unavailable(message, source) } /// 原生管理与插件自有分组共用字段校验和写入规则 diff --git a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs index 79244379c..ad8d5f2eb 100644 --- a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs +++ b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs @@ -120,7 +120,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { .bind(admin_user_id) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("read admin password hash")) + .map_err(|source| postgres_unavailable("read admin password hash", source)) } async fn change_password( @@ -136,7 +136,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin password change"))?; + .map_err(|source| postgres_unavailable("begin password change", source))?; let changed = sqlx::query( "update admin_users set password_hash = $3, updated_at = now() where id = $1 and password_hash = $2", @@ -146,7 +146,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { .bind(password_hash) .execute(&mut *transaction) .await - .map_err(|_| postgres_unavailable("change admin password"))? + .map_err(|source| postgres_unavailable("change admin password", source))? .rows_affected() == 1; if !changed { @@ -156,7 +156,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { transaction .commit() .await - .map_err(|_| postgres_unavailable("commit password change"))?; + .map_err(|source| postgres_unavailable("commit password change", source))?; Ok(true) } @@ -176,7 +176,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { .bind(password_hash) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("create admin password hash"))?; + .map_err(|source| postgres_unavailable("create admin password hash", source))?; Ok(result.rows_affected() == 1) } @@ -201,7 +201,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository { .bind(event.created_at) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("append admin audit event"))?; + .map_err(|source| postgres_unavailable("append admin audit event", source))?; Ok(()) } } @@ -238,7 +238,7 @@ pub(crate) async fn append_admin_audit_event_in_transaction( .bind(event.created_at) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("append admin audit event in transaction"))?; + .map_err(|source| postgres_unavailable("append admin audit event in transaction", source))?; Ok(()) } diff --git a/backend/crates/gateway-store/src/postgres/admission_recovery.rs b/backend/crates/gateway-store/src/postgres/admission_recovery.rs index 5db74e3d8..217bf4c96 100644 --- a/backend/crates/gateway-store/src/postgres/admission_recovery.rs +++ b/backend/crates/gateway-store/src/postgres/admission_recovery.rs @@ -72,7 +72,7 @@ impl ClientAdmissionRecoveryRepository for PgClientAdmissionRecoveryRepository { .bind(window_started_at) .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("load client admission recovery"))?; + .map_err(|source| postgres_unavailable("load client admission recovery", source))?; let mut recoveries = BTreeMap::::new(); for (client_api_key_ref, model_request_id, started_at, deadline_at, outcome) in rows { let recovery = recoveries diff --git a/backend/crates/gateway-store/src/postgres/backup.rs b/backend/crates/gateway-store/src/postgres/backup.rs index 31c0041ae..243cf4b3d 100644 --- a/backend/crates/gateway-store/src/postgres/backup.rs +++ b/backend/crates/gateway-store/src/postgres/backup.rs @@ -841,7 +841,7 @@ async fn append_audit_in_transaction( .bind(event.created_at) .execute(&mut **transaction) .await - .map_err(|_| crate::postgres_unavailable("append backup audit event"))?; + .map_err(|source| crate::postgres_unavailable("append backup audit event", source))?; Ok(()) } diff --git a/backend/crates/gateway-store/src/postgres/client_budgets.rs b/backend/crates/gateway-store/src/postgres/client_budgets.rs index 91c76df52..d1e7dad32 100644 --- a/backend/crates/gateway-store/src/postgres/client_budgets.rs +++ b/backend/crates/gateway-store/src/postgres/client_budgets.rs @@ -30,10 +30,10 @@ pub(super) async fn reset_client_key_budget( context: &MutationContext, ) -> AdminStoreResult<()> { let mut tx = match &origin { - ClientKeyBudgetMutationOrigin::Admin => pool.begin().await.map_err(|_| { + ClientKeyBudgetMutationOrigin::Admin => pool.begin().await.map_err(|source| { crate::admin_store_error( "client API key budget", - postgres_unavailable("begin budget reset"), + postgres_unavailable("begin budget reset", source), ) })?, ClientKeyBudgetMutationOrigin::Plugin(owner) => { @@ -43,10 +43,10 @@ pub(super) async fn reset_client_key_budget( reset_client_key_budget_in_transaction(&mut tx, &command, context) .await .map_err(|error| crate::admin_store_error("client API key budget", error))?; - tx.commit().await.map_err(|_| { + tx.commit().await.map_err(|source| { crate::admin_store_error( "client API key budget", - postgres_unavailable("commit budget reset"), + postgres_unavailable("commit budget reset", source), ) }) } @@ -62,7 +62,7 @@ async fn reset_client_key_budget_in_transaction( .bind(command.id.as_str()) .fetch_optional(&mut **tx) .await - .map_err(|_| postgres_unavailable("lock budget reset key"))?; + .map_err(|source| postgres_unavailable("lock budget reset key", source))?; if exists.is_none() { return Err(StoreError::NotFound { entity: "client API key", @@ -95,7 +95,7 @@ async fn reset_client_key_budget_in_transaction( .bind(reset_at) .execute(&mut **tx) .await - .map_err(|_| postgres_unavailable("reset client budget"))?; + .map_err(|source| postgres_unavailable("reset client budget", source))?; let mut fields = Vec::new(); if daily { fields.extend([ @@ -361,13 +361,13 @@ pub(super) async fn load_client_key_budgets( .bind(ids) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load client budgets"))?; + .map_err(|source| postgres_unavailable("load client budgets", source))?; let mut budgets = BTreeMap::new(); for row in rows { let parse = |field| -> StoreResult { row.get::(field) .parse() - .map_err(|_| postgres_unavailable("decode client budget")) + .map_err(|source| postgres_unavailable("decode client budget", source)) }; budgets.insert( row.get::("id"), @@ -388,9 +388,14 @@ pub(super) async fn load_client_key_budgets( ); } for record in records { - record.budget = budgets - .remove(&record.id) - .ok_or_else(|| postgres_unavailable("load client budget policy"))?; + record.budget = + budgets + .remove(&record.id) + .ok_or_else(|| crate::StoreError::Unavailable { + backend: crate::StoreBackend::PostgreSql, + message: "load client budget policy".to_owned(), + source: None, + })?; } Ok(()) } diff --git a/backend/crates/gateway-store/src/postgres/client_keys.rs b/backend/crates/gateway-store/src/postgres/client_keys.rs index 824345a8f..86dd97045 100644 --- a/backend/crates/gateway-store/src/postgres/client_keys.rs +++ b/backend/crates/gateway-store/src/postgres/client_keys.rs @@ -391,7 +391,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository { .build() .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("list client API keys"))?; + .map_err(|source| postgres_unavailable("list client API keys", source))?; let mut items = rows .iter() .map(client_record_from_row) @@ -425,7 +425,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("reveal client API key"))? + .map_err(|source| postgres_unavailable("reveal client API key", source))? .map(client_secret_from_row) .transpose() } @@ -469,7 +469,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("get client API key"))? + .map_err(|source| postgres_unavailable("get client API key", source))? .as_ref() .map(client_record_from_row) .transpose()?; @@ -503,7 +503,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository { .bind(timestamps) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("touch client API keys"))?; + .map_err(|source| postgres_unavailable("touch client API keys", source))?; Ok(result.rows_affected()) } } @@ -678,7 +678,12 @@ impl PgAdminClientKeyStore { .bind(id.as_str()) .fetch_one(&self.keys.pool) .await - .map_err(|_| admin_store_error(ENTITY, postgres_unavailable("read client key status"))) + .map_err(|source| { + admin_store_error( + ENTITY, + postgres_unavailable("read client key status", source), + ) + }) } async fn revision(&self) -> AdminStoreResult { @@ -718,12 +723,11 @@ impl ClientKeyStore for PgAdminClientKeyStore { use gateway_admin::model::client_keys::ClientKeyBudgetMutationOrigin; let map_error = |error| admin_store_error(ENTITY, error); let mut tx = match &origin { - ClientKeyBudgetMutationOrigin::Admin => self - .keys - .pool - .begin() - .await - .map_err(|_| map_error(postgres_unavailable("begin budget limits update")))?, + ClientKeyBudgetMutationOrigin::Admin => { + self.keys.pool.begin().await.map_err(|source| { + map_error(postgres_unavailable("begin budget limits update", source)) + })? + } ClientKeyBudgetMutationOrigin::Plugin(owner) => { super::plugins::begin_plugin_mutation(&self.keys.pool, owner).await? } @@ -732,23 +736,28 @@ impl ClientKeyStore for PgAdminClientKeyStore { sqlx::query("select config_revision from runtime_settings where id=1 for update") .execute(&mut *tx) .await - .map_err(|_| map_error(postgres_unavailable("lock budget limits configuration")))?; + .map_err(|source| { + map_error(postgres_unavailable( + "lock budget limits configuration", + source, + )) + })?; let current: Option<(String, String)> = sqlx::query_as( "select daily_limit_usd::text, weekly_limit_usd::text from client_api_keys where id=$1 for update" ).bind(command.id.as_str()).fetch_optional(&mut *tx).await - .map_err(|_| map_error(postgres_unavailable("lock budget limits key")))?; + .map_err(|source| map_error(postgres_unavailable("lock budget limits key", source)))?; let (daily, weekly) = current.ok_or_else(|| { map_error(StoreError::NotFound { entity: ENTITY, id: command.id.as_str().to_owned(), }) })?; - let daily: Decimal = daily - .parse() - .map_err(|_| map_error(postgres_unavailable("decode daily budget limit")))?; - let weekly: Decimal = weekly - .parse() - .map_err(|_| map_error(postgres_unavailable("decode weekly budget limit")))?; + let daily: Decimal = daily.parse().map_err(|source| { + map_error(postgres_unavailable("decode daily budget limit", source)) + })?; + let weekly: Decimal = weekly.parse().map_err(|source| { + map_error(postgres_unavailable("decode weekly budget limit", source)) + })?; let mut fields = Vec::new(); if command.daily_limit_usd.is_some_and(|value| value != daily) { fields.push("daily_limit_usd".to_owned()); @@ -760,9 +769,12 @@ impl ClientKeyStore for PgAdminClientKeyStore { fields.push("weekly_limit_usd".to_owned()); } if fields.is_empty() { - tx.commit() - .await - .map_err(|_| map_error(postgres_unavailable("finish unchanged budget limits")))?; + tx.commit().await.map_err(|source| { + map_error(postgres_unavailable( + "finish unchanged budget limits", + source, + )) + })?; return Ok(None); } sqlx::query( @@ -776,7 +788,7 @@ impl ClientKeyStore for PgAdminClientKeyStore { .bind(command.weekly_limit_usd.map(|value| value.canonical())) .execute(&mut *tx) .await - .map_err(|_| map_error(postgres_unavailable("update budget limits")))?; + .map_err(|source| map_error(postgres_unavailable("update budget limits", source)))?; let revision = super::bump_config_revision_in_transaction(&mut tx) .await .map_err(map_error)?; @@ -792,9 +804,9 @@ impl ClientKeyStore for PgAdminClientKeyStore { ) .await .map_err(map_error)?; - tx.commit() - .await - .map_err(|_| map_error(postgres_unavailable("commit budget limits update")))?; + tx.commit().await.map_err(|source| { + map_error(postgres_unavailable("commit budget limits update", source)) + })?; Ok(Some(admin_revision(revision)?)) } @@ -1152,7 +1164,7 @@ pub(crate) async fn insert_client_api_key_in_transaction( kind: crate::ConflictKind::InvalidTransition, } } else { - postgres_unavailable("insert client API key in transaction") + postgres_unavailable("insert client API key in transaction", error) } })?; replace_client_api_key_groups_in_transaction(transaction, &key.id, &key.group_ids).await?; @@ -1203,7 +1215,7 @@ pub(crate) async fn update_client_api_key_in_transaction( .bind(sqlx::types::Json(replacement_profiles)) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("update client API key in transaction"))?; + .map_err(|source| postgres_unavailable("update client API key in transaction", source))?; require_changed(result.rows_affected(), &key.id)?; replace_client_api_key_groups_in_transaction(transaction, &key.id, &key.group_ids).await } @@ -1223,7 +1235,7 @@ async fn ensure_client_key_name_available( .bind(id) .fetch_one(&mut **transaction) .await - .map_err(|_| postgres_unavailable("check client API key name"))?; + .map_err(|source| postgres_unavailable("check client API key name", source))?; if duplicate { return Err(StoreError::Conflict { entity: ENTITY, @@ -1246,7 +1258,9 @@ pub(crate) async fn set_client_api_key_enabled_in_transaction( .bind(enabled) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("set client API key state in transaction"))?; + .map_err(|source| { + postgres_unavailable("set client API key state in transaction", source) + })?; require_changed(result.rows_affected(), id) } @@ -1259,7 +1273,7 @@ pub(crate) async fn delete_client_api_key_in_transaction( .bind(id) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("delete client API key in transaction"))?; + .map_err(|source| postgres_unavailable("delete client API key in transaction", source))?; require_changed(result.rows_affected(), id) } @@ -1287,7 +1301,7 @@ async fn replace_client_api_key_groups_in_transaction( .bind(group_ids) .fetch_one(&mut **transaction) .await - .map_err(|_| postgres_unavailable("validate client API key groups"))?; + .map_err(|source| postgres_unavailable("validate client API key groups", source))?; if usize::try_from(count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { entity: "account group", @@ -1299,7 +1313,7 @@ async fn replace_client_api_key_groups_in_transaction( .bind(key_id) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("delete client API key groups"))?; + .map_err(|source| postgres_unavailable("delete client API key groups", source))?; if !group_ids.is_empty() { sqlx::query( "insert into client_api_key_groups @@ -1310,7 +1324,7 @@ async fn replace_client_api_key_groups_in_transaction( .bind(group_ids) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("insert client API key groups"))?; + .map_err(|source| postgres_unavailable("insert client API key groups", source))?; } Ok(()) } @@ -1444,7 +1458,7 @@ async fn count_client_api_keys(pool: &PgPool, search: Option<&str>) -> StoreResu .build_query_scalar::() .fetch_one(pool) .await - .map_err(|_| postgres_unavailable("count client API keys"))?; + .map_err(|source| postgres_unavailable("count client API keys", source))?; to_u64(count) } @@ -1504,7 +1518,7 @@ async fn load_client_key_memberships( .bind(ids) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load client API key memberships"))?; + .map_err(|source| postgres_unavailable("load client API key memberships", source))?; let mut groups = BTreeMap::>::new(); let mut providers = BTreeMap::>::new(); for row in rows { diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index 7105a847e..d77577ce6 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -1,5 +1,7 @@ //! 单行 `model_requests` 生命周期与最终 usage/cost 的 PostgreSQL owner +use crate::core_store_error; + use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde_json::Value; @@ -289,7 +291,7 @@ pub struct ModelRequestFinalization { pub error_kind: Option, pub provider_error_code: Option, pub error_message: Option, - pub raw_upstream_error: Option, + pub error_details: Option, pub continuation_unavailable_reason: Option, pub upstream_connection_id: Option, pub upstream_connection_exit_reason: Option, @@ -492,7 +494,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(request.continuation.requested) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("insert model request"))?; + .map_err(|source| postgres_unavailable("insert model request", source))?; Ok(()) } @@ -581,7 +583,9 @@ impl ModelRequestRepository for PgExecutionStore { .bind(request.continuation.requested) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("insert model request with first attempt"))?; + .map_err(|source| { + postgres_unavailable("insert model request with first attempt", source) + })?; Ok(()) } @@ -643,7 +647,7 @@ impl ModelRequestRepository for PgExecutionStore { )?) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("begin model request attempt"))? + .map_err(|source| postgres_unavailable("begin model request attempt", source))? .ok_or(StoreError::Conflict { entity: ENTITY, id: attempt.model_request_id, @@ -666,7 +670,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(state.as_str()) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("mark upstream send state"))?; + .map_err(|source| postgres_unavailable("mark upstream send state", source))?; Ok(result.rows_affected() == 1) } @@ -689,7 +693,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(client_status_code.map(i32::from)) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("mark downstream committed"))?; + .map_err(|source| postgres_unavailable("mark downstream committed", source))?; Ok(result.rows_affected() == 1) } @@ -708,7 +712,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(i32::from(client_status_code)) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("record client status code"))?; + .map_err(|source| postgres_unavailable("record client status code", source))?; Ok(result.rows_affected() == 1) } @@ -739,7 +743,7 @@ impl ModelRequestRepository for PgExecutionStore { upstream_transport = coalesce($37, upstream_transport), http_version = coalesce($38, http_version), websocket_pool = $39, service_tier = $40, provider_observation_json = $41, - raw_upstream_error = $42, + error_details = $42, continuation_unavailable_reason = $43, upstream_connection_id = $44, upstream_connection_exit_reason = $45, @@ -924,7 +928,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(finalization.websocket_pool) .bind(finalization.service_tier) .bind(finalization.provider_metadata_json.map(sqlx::types::Json)) - .bind(finalization.raw_upstream_error) + .bind(finalization.error_details) .bind(finalization.continuation_unavailable_reason) .bind(finalization.upstream_connection_id) .bind(finalization.upstream_connection_exit_reason) @@ -941,7 +945,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(finalization.billing_snapshot_json.map(sqlx::types::Json)) .fetch_one(&self.pool) .await - .map_err(|_| postgres_unavailable("finalize model request"))?; + .map_err(|source| postgres_unavailable("finalize model request", source))?; Ok(finalized == 1) } @@ -962,7 +966,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(now) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("recover expired model requests"))?; + .map_err(|source| postgres_unavailable("recover expired model requests", source))?; Ok(ModelRequestRecoveryReport { requests: result.rows_affected(), }) @@ -1006,7 +1010,7 @@ impl ExecutionStore for PgExecutionStore { .bind(ttl_ms) .fetch_one(&pool) .await - .map_err(|_| postgres_unavailable("renew model request recovery lease")) + .map_err(|source| postgres_unavailable("renew model request recovery lease", source)) }) }, )) @@ -1105,7 +1109,7 @@ impl ExecutionStore for PgExecutionStore { .retry_after() .map(|duration| u64::try_from(duration.as_millis())) .transpose() - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; super::OpsEventRepository::append_ops_event( &super::PgOpsEventRepository::new(self.pool.clone()), super::OpsEvent { @@ -1124,9 +1128,7 @@ impl ExecutionStore for PgExecutionStore { .map(|model| model.as_str().to_owned()), failure_kind: error.kind().as_str().to_owned(), upstream_send_state: Some(error.send_state().as_str().to_owned()), - raw_upstream_error: error - .raw_upstream_error() - .map(|raw| raw.as_str().to_owned()), + error_details: error.error_details(), status_code: error.upstream_status().or(failure.upstream_status_code), provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()), retry_after_ms, @@ -1135,8 +1137,9 @@ impl ExecutionStore for PgExecutionStore { .map(|id| id.as_str().to_owned()) .or(failure.upstream_request_id), latency_ms: Some( - u64::try_from(failure.latency.as_millis()) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + u64::try_from(failure.latency.as_millis()).map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?, ), message: error.diagnostic().map_or_else( || "intermediate upstream failure".to_owned(), @@ -1169,7 +1172,7 @@ impl ExecutionStore for PgExecutionStore { upstream_model_id: None, failure_kind: error.kind().as_str().to_owned(), upstream_send_state: None, - raw_upstream_error: None, + error_details: None, status_code: None, provider_error_code: error.client_error_code().map(str::to_owned), retry_after_ms: error @@ -1197,9 +1200,9 @@ impl ExecutionStore for PgExecutionStore { .retry_after() .map(|duration| u64::try_from(duration.as_millis())) .transpose() - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; let latency_ms = u64::try_from(failure.latency.as_millis()) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned()); super::OpsEventRepository::append_ops_event( &super::PgOpsEventRepository::new(self.pool.clone()), @@ -1216,9 +1219,7 @@ impl ExecutionStore for PgExecutionStore { upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()), failure_kind: error.kind().as_str().to_owned(), upstream_send_state: Some(error.send_state().as_str().to_owned()), - raw_upstream_error: error - .raw_upstream_error() - .map(|raw| raw.as_str().to_owned()), + error_details: error.error_details(), status_code: error.upstream_status(), provider_error_code, retry_after_ms, @@ -1260,7 +1261,7 @@ impl ExecutionStore for PgExecutionStore { .as_deref() .map(serde_json::from_str::) .transpose() - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; if provider_metadata_json .as_ref() .is_some_and(|metadata| !metadata.is_object()) @@ -1329,11 +1330,13 @@ impl ExecutionStore for PgExecutionStore { .as_deref() .map(serde_json::from_str) .transpose() - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?, error_kind, provider_error_code: finalization.provider_error_code, error_message, - raw_upstream_error: finalization.raw_upstream_error, + error_details: finalization.error_details, continuation_unavailable_reason, upstream_connection_id, upstream_connection_exit_reason, @@ -1429,17 +1432,6 @@ fn require_core_update(updated: bool) -> Result<(), CoreStoreError> { } } -fn core_store_error(error: StoreError) -> CoreStoreError { - let kind = match error { - StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable, - StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict, - StoreError::NotFound { .. } | StoreError::InvalidData { .. } => { - CoreStoreErrorKind::InvalidData - } - }; - CoreStoreError::new(kind) -} - fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest { let (routing_scope, routing_group_refs, routing_group_names_snapshot) = routing_snapshot_row(&request.routing); diff --git a/backend/crates/gateway-store/src/postgres/execution_buffer.rs b/backend/crates/gateway-store/src/postgres/execution_buffer.rs index 86b10ad64..faac9b2a2 100644 --- a/backend/crates/gateway-store/src/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/src/postgres/execution_buffer.rs @@ -784,6 +784,7 @@ impl ExecutionObservationWrite { finalization.provider_metadata_json.as_deref(), finalization.diagnostic_trace_json.as_deref(), finalization.provider_error_code.as_deref(), + finalization.error_details.as_deref(), ]) .saturating_add(error_bytes) .saturating_add(size_of::()) @@ -874,9 +875,13 @@ fn attempt_bytes(attempt: &AttemptRecord) -> usize { } fn provider_error_bytes(error: &ProviderError) -> usize { + use std::error::Error as _; + let mut bytes = text_bytes([ error.upstream_code().map(|value| value.as_str()), error.upstream_request_id().map(|value| value.as_str()), + error.diagnostic().map(|value| value.as_str()), + error.raw_upstream_error().map(|value| value.as_str()), ]); if let Some(client_error) = error.client_visible_upstream_error() { bytes = bytes.saturating_add(text_bytes([ @@ -895,7 +900,33 @@ fn provider_error_bytes(error: &ProviderError) -> usize { .saturating_add(header.value().len()); } } - bytes + bytes.saturating_add(error_source_bytes(error.source())) +} + +/// 原始来源也占据队列内存；只计数而不保存格式化内容，避免复制或打印敏感原因 +fn error_source_bytes(mut source: Option<&(dyn std::error::Error + 'static)>) -> usize { + struct ByteCount(usize); + impl std::fmt::Write for ByteCount { + fn write_str(&mut self, value: &str) -> std::fmt::Result { + self.0 = self.0.saturating_add(value.len()); + Ok(()) + } + } + let mut bytes = ByteCount(0); + for _ in 0..32 { + let Some(error) = source else { return bytes.0 }; + bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error)); + if std::fmt::write(&mut bytes, format_args!("{error}")).is_err() { + return usize::MAX; + } + source = error.source(); + } + // 过深或循环的第三方来源不能让观测写入无限遍历，交给已有超限丢弃计数 + if source.is_some() { + usize::MAX + } else { + bytes.0 + } } fn text_bytes(values: [Option<&str>; N]) -> usize { diff --git a/backend/crates/gateway-store/src/postgres/mod.rs b/backend/crates/gateway-store/src/postgres/mod.rs index 3b9c38afc..d83a67eda 100644 --- a/backend/crates/gateway-store/src/postgres/mod.rs +++ b/backend/crates/gateway-store/src/postgres/mod.rs @@ -8,7 +8,7 @@ use sqlx::{ use crate::{ POSTGRES_IDLE_TRANSACTION_TIMEOUT, POSTGRES_LOCK_TIMEOUT, POSTGRES_STATEMENT_TIMEOUT, Revision, - StoreBackend, StoreError, StorePoolConfig, StoreResult, postgres_unavailable, + StoreError, StorePoolConfig, StoreResult, postgres_unavailable, }; mod account_groups; @@ -59,12 +59,15 @@ pub async fn connect_and_migrate( pool_config: StorePoolConfig, ) -> StoreResult { if database_url.trim().is_empty() { - return Err(postgres_unavailable("connect PostgreSQL")); + return Err(StoreError::InvalidData { + entity: "PostgreSQL configuration", + message: "database URL is empty".to_owned(), + }); } pool_config.validate()?; let connect_options = database_url .parse::() - .map_err(|_| postgres_unavailable("parse PostgreSQL connection options"))?; + .map_err(|source| postgres_unavailable("parse PostgreSQL connection options", source))?; let migration_pool = PgPoolOptions::new() .max_connections(1) .connect_with( @@ -73,13 +76,10 @@ pub async fn connect_and_migrate( .application_name("codex-proxy-rs:migration"), ) .await - .map_err(|_| postgres_unavailable("connect PostgreSQL for migrations"))?; + .map_err(|source| postgres_unavailable("connect PostgreSQL for migrations", source))?; if let Err(error) = MIGRATOR.run(&migration_pool).await { migration_pool.close().await; - return Err(StoreError::Unavailable { - backend: StoreBackend::PostgreSql, - message: format!("apply PostgreSQL migrations: {error}"), - }); + return Err(postgres_unavailable("apply PostgreSQL migrations", error)); } migration_pool.close().await; @@ -94,7 +94,7 @@ pub(crate) async fn connect_read_only( pool_config.validate()?; let options = database_url .parse::() - .map_err(|_| postgres_unavailable("parse PostgreSQL connection options"))?; + .map_err(|source| postgres_unavailable("parse PostgreSQL connection options", source))?; connect_pool(options, pool_config, true).await } @@ -135,7 +135,7 @@ async fn connect_pool( }) .connect_with(connect_options.application_name("codex-proxy-rs")) .await - .map_err(|_| postgres_unavailable("connect PostgreSQL"))?; + .map_err(|source| postgres_unavailable("connect PostgreSQL", source))?; Ok(pool) } @@ -221,11 +221,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository { replacement: ControlPlaneReplacement, ) -> StoreResult { replacement.settings.validate()?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin control plane replacement"))?; + let mut transaction = + self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin control plane replacement", source) + })?; let result = async { // 锁住所有配置写入共同使用的行，版本检查与修改不能被其他事务穿插 let current = sqlx::query_scalar::<_, i64>( @@ -233,7 +232,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository { ) .fetch_one(&mut *transaction) .await - .map_err(|_| postgres_unavailable("lock control plane revision"))?; + .map_err(|source| postgres_unavailable("lock control plane revision", source))?; if u64::try_from(current).ok() != Some(replacement.expected_revision.get()) { return Err(StoreError::Conflict { entity: "runtime settings", @@ -251,17 +250,15 @@ impl ControlPlaneRepository for PgControlPlaneRepository { .await; match result { Ok(snapshot) => { - transaction - .commit() - .await - .map_err(|_| postgres_unavailable("commit control plane replacement"))?; + transaction.commit().await.map_err(|source| { + postgres_unavailable("commit control plane replacement", source) + })?; Ok(snapshot) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| postgres_unavailable("rollback control plane replacement"))?; + transaction.rollback().await.map_err(|source| { + postgres_unavailable("rollback control plane replacement", source) + })?; Err(error) } } @@ -337,11 +334,9 @@ impl PgControlPlaneRepository { mutation: ControlPlaneMutation, mut audit: AdminAuditEvent, ) -> StoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin targeted control plane mutation"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin targeted control plane mutation", source) + })?; let result = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; match mutation { @@ -362,8 +357,8 @@ impl PgControlPlaneRepository { .bind(&key.id) .fetch_one(&mut *transaction) .await - .map_err(|_| { - postgres_unavailable("load client API key routing scope for audit") + .map_err(|source| { + postgres_unavailable("load client API key routing scope for audit", source) })?; if previously_restricted && key.group_ids.is_empty() { audit @@ -393,15 +388,14 @@ impl PgControlPlaneRepository { .await; match result { Ok(revision) => { - transaction - .commit() - .await - .map_err(|_| postgres_unavailable("commit targeted control plane mutation"))?; + transaction.commit().await.map_err(|source| { + postgres_unavailable("commit targeted control plane mutation", source) + })?; Ok(revision) } Err(error) => { - transaction.rollback().await.map_err(|_| { - postgres_unavailable("rollback targeted control plane mutation") + transaction.rollback().await.map_err(|source| { + postgres_unavailable("rollback targeted control plane mutation", source) })?; Err(error) } diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index d678efebb..427c3c6b3 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -778,7 +778,7 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops upstream_request_id: error.upstream_request_id, latency_ms: error.latency_ms, message: error.message, - raw_upstream_error: error.raw_upstream_error, + error_details: error.error_details, client_ip: error.client_ip, user_agent: error.user_agent, reasoning_effort: error.reasoning_effort, @@ -890,7 +890,7 @@ pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult< provider_metadata_json: get::>(row, "provider_observation_json")? .map(|value| serde_json::to_string(&value)) .transpose() - .map_err(|_| postgres_unavailable("encode provider observation"))?, + .map_err(|source| postgres_unavailable("encode provider observation", source))?, attempt_count: to_u32(get(row, "attempt_count")?)?, upstream_send_state: get(row, "upstream_send_state")?, downstream_committed_at: get(row, "downstream_committed_at")?, @@ -977,7 +977,7 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult StoreResult { if !value.is_finite() || value < 0.0 { - return Err(postgres_unavailable("decode latency percentile")); + return Err(crate::StoreError::Unavailable { + backend: crate::StoreBackend::PostgreSql, + message: "decode latency percentile".to_owned(), + source: None, + }); } Ok(Self(value.to_bits())) } @@ -740,7 +744,7 @@ pub struct OpsErrorRecord { pub upstream_request_id: Option, pub latency_ms: Option, pub message: String, - pub raw_upstream_error: Option, + pub error_details: Option, pub client_ip: Option, pub user_agent: Option, pub reasoning_effort: Option, diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs index c3ef4a628..65fe8db0e 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/accounts.rs @@ -92,7 +92,7 @@ pub(crate) async fn provider_account_usage( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load provider account usage"))?; + .map_err(|source| postgres_unavailable("load provider account usage", source))?; let mut observations = Vec::with_capacity(usize::from(query.limit)); let mut costs = HashMap::>::new(); @@ -126,9 +126,11 @@ pub(crate) async fn provider_account_usage( } (0 | 1, 0 | 1) => {} _ => { - return Err(postgres_unavailable( - "decode provider account usage grouping", - )); + return Err(crate::StoreError::Unavailable { + backend: crate::StoreBackend::PostgreSql, + message: "decode provider account usage grouping".to_owned(), + source: None, + }); } } } @@ -188,7 +190,7 @@ pub(crate) async fn provider_account_request_buckets( .bind(range.end) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load provider account request timeline"))?; + .map_err(|source| postgres_unavailable("load provider account request timeline", source))?; let mut observed = HashMap::>::new(); for row in rows { diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs index 48563b045..f67e2a6a2 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/dashboard.rs @@ -99,7 +99,7 @@ pub(crate) async fn request_metrics( .build() .fetch_one(pool) .await - .map_err(|_| postgres_unavailable("load request metrics"))?; + .map_err(|source| postgres_unavailable("load request metrics", source))?; request_metrics_from_row(&row) } @@ -121,7 +121,7 @@ pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult, Vec>::new(); for row in rows { result @@ -470,7 +470,7 @@ pub(crate) async fn attempt_metrics( .build() .fetch_one(pool) .await - .map_err(|_| postgres_unavailable("load attempt metrics"))?; + .map_err(|source| postgres_unavailable("load attempt metrics", source))?; Ok(AttemptMetrics { attempt_count: unsigned(&row, "attempt_count")?, success_count: unsigned(&row, "success_count")?, @@ -505,7 +505,7 @@ pub(crate) async fn request_costs( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load request costs"))? + .map_err(|source| postgres_unavailable("load request costs", source))? .iter() .map(cost_from_row) .collect() @@ -537,7 +537,7 @@ pub(crate) async fn provider_observations( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load provider observations"))?; + .map_err(|source| postgres_unavailable("load provider observations", source))?; rows.iter() .map(|row| { Ok(ProviderObservation { @@ -615,7 +615,7 @@ pub(crate) fn optional_percentile( column: &str, ) -> StoreResult> { row.try_get::, _>(column) - .map_err(|_| postgres_unavailable("decode latency percentile"))? + .map_err(|source| postgres_unavailable("decode latency percentile", source))? .map(PercentileMilliseconds::new) .transpose() } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs index c1364f335..43770afac 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/ops.rs @@ -20,7 +20,7 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source, mr.provider_error_code, mr.client_response_id, mr.upstream_request_id, mr.latency_ms, coalesce(mr.error_message, mr.error_kind) as message, - mr.raw_upstream_error, + mr.error_details, host(mr.client_ip) as client_ip, mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind, mr.subagent_kind, mr.compact, @@ -51,7 +51,7 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source, oe.upstream_send_state, null::integer as client_status_code, oe.status_code as upstream_status_code, oe.provider_error_code, mr.client_response_id, oe.upstream_request_id, - oe.latency_ms, oe.message, oe.raw_upstream_error, + oe.latency_ms, oe.message, oe.error_details, host(mr.client_ip) as client_ip, mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind, mr.subagent_kind, mr.compact, @@ -93,7 +93,7 @@ pub(crate) async fn list_ops_errors( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("list ops errors"))?; + .map_err(|source| postgres_unavailable("list ops errors", source))?; let items = rows .iter() .map(ops_error_from_row) @@ -124,7 +124,7 @@ pub(crate) async fn count_ops_errors( .build_query_scalar::() .fetch_one(pool) .await - .map_err(|_| postgres_unavailable("count ops errors"))?; + .map_err(|source| postgres_unavailable("count ops errors", source))?; to_u64(total) } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index b688a3096..6c3aa6829 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -212,7 +212,7 @@ pub(crate) async fn list_usage_record_items( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("list usage records"))?; + .map_err(|source| postgres_unavailable("list usage records", source))?; let items = rows .iter() .map(usage_list_record_from_row) @@ -237,7 +237,7 @@ pub(crate) async fn count_usage_records( .build_query_scalar::() .fetch_one(pool) .await - .map_err(|_| postgres_unavailable("count usage records"))?; + .map_err(|source| postgres_unavailable("count usage records", source))?; to_u64(total) } @@ -254,7 +254,7 @@ pub(crate) async fn usage_record_detail( .build() .fetch_optional(pool) .await - .map_err(|_| postgres_unavailable("load usage record detail"))? + .map_err(|source| postgres_unavailable("load usage record detail", source))? .ok_or_else(|| StoreError::NotFound { entity: "model request", id: request_id.to_owned(), @@ -276,7 +276,7 @@ pub(crate) async fn usage_record_detail( .bind(request_id) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load usage attempt observations"))?; + .map_err(|source| postgres_unavailable("load usage attempt observations", source))?; let mut attempts = rows .iter() .map(intermediate_attempt_from_row) @@ -286,7 +286,7 @@ pub(crate) async fn usage_record_detail( } let trace: Option = row .try_get("diagnostic_trace_json") - .map_err(|_| postgres_unavailable("decode request trace"))?; + .map_err(|source| postgres_unavailable("decode request trace", source))?; let related_requests = sqlx::query_scalar::<_, Value>( "select jsonb_build_object('requestId', related.id, 'outcome', related.outcome, 'relation', case when related.id = current.recovery_request_id then 'recovered_by' else 'recovers' end, @@ -295,7 +295,7 @@ pub(crate) async fn usage_record_detail( on related.id = current.recovery_request_id or related.recovery_request_id = current.id where current.id = $1 order by related.started_at limit 20" ).bind(request_id).fetch_all(pool).await - .map_err(|_| postgres_unavailable("load related recovery requests"))?; + .map_err(|source| postgres_unavailable("load related recovery requests", source))?; Ok(UsageRecordDetail { request, attempts, @@ -468,7 +468,7 @@ pub(crate) async fn usage_diagnostics( .build() .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load usage diagnostics"))?; + .map_err(|source| postgres_unavailable("load usage diagnostics", source))?; let total_request_count = rows .first() .map(|row| unsigned(row, "total_request_count")) @@ -506,7 +506,13 @@ pub(crate) async fn usage_diagnostics( .push(cost_from_row(row)?); } 0 => {} - _ => return Err(postgres_unavailable("decode usage diagnostic grouping")), + _ => { + return Err(crate::StoreError::Unavailable { + backend: crate::StoreBackend::PostgreSql, + message: "decode usage diagnostic grouping".to_owned(), + source: None, + }); + } } } let mut display_names = match dimension { @@ -577,7 +583,7 @@ pub(crate) async fn diagnostic_account_display_names( .bind(account_ids) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load diagnostic account display names"))?; + .map_err(|source| postgres_unavailable("load diagnostic account display names", source))?; let mut display_names = HashMap::with_capacity(rows.len()); for row in rows { display_names.insert( @@ -599,7 +605,7 @@ pub(crate) async fn diagnostic_api_key_display_names( .bind(key_ids) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load diagnostic api key display names"))?; + .map_err(|source| postgres_unavailable("load diagnostic api key display names", source))?; let mut display_names = HashMap::with_capacity(rows.len()); for row in rows { display_names.insert(get(&row, "id")?, get(&row, "name")?); diff --git a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs index 022d800c9..7ed466534 100644 --- a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs +++ b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs @@ -101,11 +101,12 @@ impl ObservabilityQueryBudget { ); Ok(permit) } - Ok(Err(_)) => Err(StoreError::Unavailable { + Ok(Err(source)) => Err(StoreError::Unavailable { backend: StoreBackend::PostgreSql, message: "observability PostgreSQL connection budget is closed".to_owned(), + source: Some(gateway_core::error::ErrorSource::new(source)), }), - Err(_) => { + Err(source) => { tracing::warn!( query_class = "observability", operation, @@ -116,6 +117,7 @@ impl ObservabilityQueryBudget { Err(StoreError::Unavailable { backend: StoreBackend::PostgreSql, message: "observability PostgreSQL connection budget is exhausted".to_owned(), + source: Some(gateway_core::error::ErrorSource::new(source)), }) } } diff --git a/backend/crates/gateway-store/src/postgres/ops_events.rs b/backend/crates/gateway-store/src/postgres/ops_events.rs index f9c659003..d16548b05 100644 --- a/backend/crates/gateway-store/src/postgres/ops_events.rs +++ b/backend/crates/gateway-store/src/postgres/ops_events.rs @@ -38,7 +38,7 @@ pub struct OpsEvent { pub upstream_model_id: Option, pub failure_kind: String, pub upstream_send_state: Option, - pub raw_upstream_error: Option, + pub error_details: Option, pub status_code: Option, pub provider_error_code: Option, pub retry_after_ms: Option, @@ -119,7 +119,7 @@ impl OpsEventRepository for PgOpsEventRepository { provider_account_name_snapshot, provider_account_email_snapshot, provider_account_authentication_kind_snapshot, failure_kind, upstream_send_state, status_code, provider_error_code, retry_after_ms, - upstream_request_id, latency_ms, message, raw_upstream_error, + upstream_request_id, latency_ms, message, error_details, created_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, @@ -165,11 +165,11 @@ impl OpsEventRepository for PgOpsEventRepository { .map_err(|_| invalid("latency_ms is too large"))?, ) .bind(event.message) - .bind(event.raw_upstream_error) + .bind(event.error_details) .bind(event.created_at) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("append ops event"))?; + .map_err(|source| postgres_unavailable("append ops event", source))?; Ok(()) } } diff --git a/backend/crates/gateway-store/src/postgres/pricing.rs b/backend/crates/gateway-store/src/postgres/pricing.rs index c0afb01a2..2c3eb3e8b 100644 --- a/backend/crates/gateway-store/src/postgres/pricing.rs +++ b/backend/crates/gateway-store/src/postgres/pricing.rs @@ -143,7 +143,7 @@ impl PgControlPlaneRepository { ) -> StoreResult { let (Json(overrides), Json(synced), synced_at) = sqlx::query_as::<_, (Json, Json, Option>)>( "select pricing_overrides_json, pricing_synced_json, pricing_synced_at from runtime_settings where id = 1") - .fetch_one(&self.pool).await.map_err(|_| postgres_unavailable("load model pricing"))?; + .fetch_one(&self.pool).await.map_err(|source| postgres_unavailable("load model pricing", source))?; validate_pricing(&overrides)?; validate_pricing(&synced)?; Ok(gateway_admin::model::pricing::StoredPricing { @@ -162,13 +162,13 @@ impl PgControlPlaneRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin pricing sync"))?; + .map_err(|source| postgres_unavailable("begin pricing sync", source))?; let Json(mut prices) = sqlx::query_scalar::<_, Json>( "select pricing_synced_json from runtime_settings where id = 1 for update", ) .fetch_one(&mut *transaction) .await - .map_err(|_| postgres_unavailable("lock synced model pricing"))?; + .map_err(|source| postgres_unavailable("lock synced model pricing", source))?; for (provider, models) in changes { let stored = prices.entry(provider).or_default(); for (model, price) in models { @@ -182,13 +182,13 @@ impl PgControlPlaneRepository { prices.retain(|_, models| !models.is_empty()); validate_pricing(&prices)?; sqlx::query("update runtime_settings set pricing_synced_json = $1, pricing_synced_at = now() where id = 1") - .bind(Json(prices)).execute(&mut *transaction).await.map_err(|_| postgres_unavailable("sync model pricing"))?; + .bind(Json(prices)).execute(&mut *transaction).await.map_err(|source| postgres_unavailable("sync model pricing", source))?; let revision = bump_config_revision_in_transaction(&mut transaction).await?; append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?; transaction .commit() .await - .map_err(|_| postgres_unavailable("commit pricing sync"))?; + .map_err(|source| postgres_unavailable("commit pricing sync", source))?; Ok(revision) } @@ -201,14 +201,14 @@ impl PgControlPlaneRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin model pricing update"))?; + .map_err(|source| postgres_unavailable("begin model pricing update", source))?; // 锁定当前配置再更新选中项，避免批量操作覆盖其他管理员已提交的模型 let (Json(mut pricing), Json(mut synced)) = sqlx::query_as::<_, (Json, Json)>( "select pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1 for update", ) .fetch_one(&mut *transaction) .await - .map_err(|_| postgres_unavailable("lock model pricing"))?; + .map_err(|source| postgres_unavailable("lock model pricing", source))?; let models = pricing.entry(command.provider.clone()).or_default(); let source = synced.entry(command.provider).or_default(); for model in command.models { @@ -242,13 +242,13 @@ impl PgControlPlaneRepository { .bind(Json(synced)) .execute(&mut *transaction) .await - .map_err(|_| postgres_unavailable("update model pricing"))?; + .map_err(|source| postgres_unavailable("update model pricing", source))?; let revision = bump_config_revision_in_transaction(&mut transaction).await?; append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?; transaction .commit() .await - .map_err(|_| postgres_unavailable("commit model pricing"))?; + .map_err(|source| postgres_unavailable("commit model pricing", source))?; Ok(revision) } } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index 605164e4e..2e80e3c4d 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -123,7 +123,9 @@ impl PgAdminAccountStore { .bind(ends) .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("load provider account quota window usage")) + .map_err(|source| { + postgres_unavailable("load provider account quota window usage", source) + }) }) .await .map_err(|error| admin_store_error(ENTITY, error))?; @@ -270,10 +272,10 @@ impl PgAdminAccountStore { .bind(account_ids) .fetch_all(&self.pool) .await - .map_err(|_| { + .map_err(|source| { admin_store_error( ENTITY, - postgres_unavailable("load account group references"), + postgres_unavailable("load account group references", source), ) })?; let mut groups = BTreeMap::>::new(); @@ -430,7 +432,12 @@ impl AccountStore for PgAdminAccountStore { ) .fetch_one(&self.pool) .await - .map_err(|_| admin_store_error(ENTITY, postgres_unavailable("load account concurrency")))?; + .map_err(|source| { + admin_store_error( + ENTITY, + postgres_unavailable("load account concurrency", source), + ) + })?; let default_concurrency = u64::try_from(default_concurrency).map_err(|_| { AdminStoreError::new( AdminStoreErrorKind::Invalid, @@ -701,8 +708,11 @@ impl AccountStore for PgAdminAccountStore { limit: AccountConcurrencyLimit, context: &MutationContext, ) -> AdminStoreResult> { - let mut transaction = self.pool.begin().await.map_err(|_| { - admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction")) + let mut transaction = self.pool.begin().await.map_err(|source| { + admin_store_error( + ENTITY, + postgres_unavailable("begin concurrency reduction", source), + ) })?; let result = async { @@ -710,7 +720,7 @@ impl AccountStore for PgAdminAccountStore { let default_limit: i64 = sqlx::query_scalar( "select max_concurrent_per_account from runtime_settings where id = 1 for update" ).fetch_one(&mut *transaction).await - .map_err(|_| postgres_unavailable("lock default concurrency"))?; + .map_err(|source| postgres_unavailable("lock default concurrency", source))?; let changed = sqlx::query_scalar::<_, String>( "update provider_accounts set concurrency_limit = $2, updated_at = greatest(now(), updated_at) where id = $1 and enabled = true @@ -722,7 +732,7 @@ impl AccountStore for PgAdminAccountStore { .bind(default_limit) .fetch_optional(&mut *transaction) .await - .map_err(|_| postgres_unavailable("lower account concurrency"))?; + .map_err(|source| postgres_unavailable("lower account concurrency", source))?; if changed.is_none() { return Ok(None); } @@ -919,10 +929,10 @@ impl AccountStore for PgAdminAccountStore { .await .map_err(|error| admin_store_error(ENTITY, error))?; let revision = control_plane.settings.config_revision; - let mut transaction = self.accounts.pool.begin().await.map_err(|_| { + let mut transaction = self.accounts.pool.begin().await.map_err(|source| { admin_store_error( ENTITY, - postgres_unavailable("begin credential export audit"), + postgres_unavailable("begin credential export audit", source), ) })?; let result = async { diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs index 3e2eb4e80..be1135b11 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs @@ -160,7 +160,12 @@ pub(crate) async fn load_admin_account_page( .bind(offset) .fetch_all(pool) .await - .map_err(|_| admin_store_error(ENTITY, postgres_unavailable("load admin account page")))?; + .map_err(|source| { + admin_store_error( + ENTITY, + postgres_unavailable("load admin account page", source), + ) + })?; let metadata = rows.first().ok_or_else(|| { AdminStoreError::new( AdminStoreErrorKind::Unavailable, @@ -211,10 +216,10 @@ async fn validate_group_filter( .bind(group_id.as_str()) .fetch_one(pool) .await - .map_err(|_| { + .map_err(|source| { admin_store_error( ENTITY, - postgres_unavailable("validate admin account group filter"), + postgres_unavailable("validate admin account group filter", source), ) })?; if exists { diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs index 6edfc8a45..cb8cf16cd 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/core_adapter.rs @@ -1,6 +1,7 @@ //! Core `ProviderAccountStore` 端口适配与 core 投影映射 use super::*; +use crate::core_store_error; fn loaded_credential_from_record( record: ProviderAccountRecord, @@ -105,7 +106,7 @@ impl ProviderAccountStore for PgProviderAccountRepository { .bind(i64::from(query.limit().get())) .fetch_all(&self.pool) .await - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?; rows.into_iter() .map(account_record_from_row) .map(|record| { @@ -218,11 +219,12 @@ impl ProviderAccountStore for PgProviderAccountRepository { .bind(preserve_profile) .fetch_optional(&self.pool) .await - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?; match next { Some(next) => Ok(CredentialCasOutcome::Updated( - CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?).map_err( + |source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source), + )?, )), None => Ok(CredentialCasOutcome::Conflict), } @@ -248,12 +250,12 @@ impl ProviderAccountStore for PgProviderAccountRepository { .bind(account_ids) .fetch_all(&self.pool) .await - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?; rows.into_iter() .map(|row| { - let account_id = row - .try_get::("id") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let account_id = row.try_get::("id").map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?; let revision = row .try_get::("credential_revision") .ok() @@ -262,21 +264,27 @@ impl ProviderAccountStore for PgProviderAccountRepository { .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; let quota = row .try_get::, _>("provider_quota_json") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))? + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })? .and_then(|value| value.as_object().cloned()) .map(OpaqueProviderData::new) .unwrap_or_else(|| OpaqueProviderData::new(serde_json::Map::new())); let observed_at = row .try_get::, _>("quota_observed_at") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?; let access = row .try_get::("quota_access_state") .ok() .and_then(|value| QuotaAccessState::parse(&value)) .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; - let evidence = row - .try_get::, _>("quota_evidence") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let evidence = + row.try_get::, _>("quota_evidence") + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?; let evidence = match evidence { Some(value) => Some( QuotaEvidence::parse(&value) @@ -286,10 +294,14 @@ impl ProviderAccountStore for PgProviderAccountRepository { }; let reset_at = row .try_get::>, _>("quota_reset_at") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?; let access_observed_at = row .try_get::>, _>("quota_access_observed_at") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?; let state = QuotaState::from_persisted( access, evidence, @@ -298,11 +310,12 @@ impl ProviderAccountStore for PgProviderAccountRepository { ) .ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; Ok(QuotaObservation { - plan_type: row - .try_get("plan_type") - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, - account_id: CoreProviderAccountId::new(account_id) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?, + plan_type: row.try_get("plan_type").map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?, + account_id: CoreProviderAccountId::new(account_id).map_err(|source| { + CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source) + })?, expected_revision: revision, quota, observed_at: observed_at.into(), diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs index 67ad1a665..b0e61f456 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/quota_forecast.rs @@ -32,7 +32,7 @@ pub(super) async fn load_history( .bind(MAX_FORECAST_HISTORY_POINTS as i32) .fetch_all(pool) .await - .map_err(|_| postgres_unavailable("load quota forecast history")) + .map_err(|source| postgres_unavailable("load quota forecast history", source)) }) .await .map_err(|error| admin_store_error(ENTITY, error))?; diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs index 0b47acbda..4090f40f8 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -105,7 +105,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("load provider account"))?; + .map_err(|source| postgres_unavailable("load provider account", source))?; row.map(account_record_from_row).transpose() } @@ -141,7 +141,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(limit) .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("list plugin accounts"))?; + .map_err(|source| postgres_unavailable("list plugin accounts", source))?; rows.into_iter().map(account_summary_from_row).collect() } @@ -167,7 +167,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(include_disabled) .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("list provider accounts"))?; + .map_err(|source| postgres_unavailable("list provider accounts", source))?; rows.into_iter().map(account_summary_from_row).collect() } @@ -178,7 +178,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin provider account insert"))?; + .map_err(|source| postgres_unavailable("begin provider account insert", source))?; let proxy_id = match account.outbound_proxy.as_ref() { Some(proxy) => Some( super::super::proxies::ensure_proxy_for_url(&mut transaction, proxy, None) @@ -221,11 +221,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.model_access.as_ref().map(sqlx::types::Json)) .execute(&mut *transaction) .await - .map_err(|_| postgres_unavailable("insert provider account"))?; + .map_err(|source| postgres_unavailable("insert provider account", source))?; transaction .commit() .await - .map_err(|_| postgres_unavailable("commit provider account insert"))?; + .map_err(|source| postgres_unavailable("commit provider account insert", source))?; Ok(()) } @@ -243,7 +243,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.plan_type) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("update provider account"))?; + .map_err(|source| postgres_unavailable("update provider account", source))?; Ok(result.rows_affected() == 1) } @@ -279,7 +279,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(update.next_refresh_at) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("compare and swap provider credentials"))? + .map_err(|source| postgres_unavailable("compare and swap provider credentials", source))? .ok_or(StoreError::Conflict { entity: ENTITY, id: update.account_id, @@ -311,7 +311,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(update.message.as_deref()) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("apply provider account state"))?; + .map_err(|source| postgres_unavailable("apply provider account state", source))?; Ok(result.rows_affected() == 1) } @@ -324,7 +324,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(enabled) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("set provider account enabled state"))?; + .map_err(|source| postgres_unavailable("set provider account enabled state", source))?; Ok(result.rows_affected() == 1) } @@ -375,7 +375,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(plan_type) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("compare and swap provider quota"))?; + .map_err(|source| postgres_unavailable("compare and swap provider quota", source))?; Ok(result.rows_affected() == 1) } @@ -406,7 +406,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(state.reset_at().map(DateTime::::from)) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("apply provider quota access"))?; + .map_err(|source| postgres_unavailable("apply provider quota access", source))?; Ok(result.rows_affected() == 1) } @@ -429,7 +429,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(observed_at) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("touch provider quota observation"))?; + .map_err(|source| postgres_unavailable("touch provider quota observation", source))?; Ok(result.rows_affected() == 1) } @@ -439,7 +439,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(id) .execute(&self.pool) .await - .map_err(|_| postgres_unavailable("delete disabled provider account"))?; + .map_err(|source| postgres_unavailable("delete disabled provider account", source))?; Ok(result.rows_affected() == 1) } } @@ -458,7 +458,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { .bind(&scope.provider_kind) .fetch_all(&self.pool) .await - .map_err(|_| postgres_unavailable("export provider accounts"))?; + .map_err(|source| postgres_unavailable("export provider accounts", source))?; let records = rows .into_iter() .map(account_record_from_row) @@ -486,11 +486,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { &self, command: ImportProviderAccounts, ) -> StoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin import"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin provider account admin import", source) + })?; let result = import_provider_accounts_in_transaction(&mut transaction, command).await; finish_admin_transaction(transaction, result, "provider account admin import").await } @@ -499,11 +497,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { &self, command: RotateProviderAccount, ) -> StoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin rotation"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin provider account admin rotation", source) + })?; let result = rotate_provider_account_admin_in_transaction(&mut transaction, command).await; finish_admin_transaction(transaction, result, "provider account admin rotation").await } @@ -516,11 +512,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { if let Some(group_ids) = &command.group_ids { validate_batch_update_group_ids(group_ids)?; } - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin state change"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin provider account admin state change", source) + })?; let result = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; update_provider_accounts_scheduling_in_transaction( @@ -562,11 +556,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { command: RecoverProviderAccount, ) -> StoreResult { validate_admin_account_ids(std::slice::from_ref(&command.account_id))?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin recovery"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin provider account admin recovery", source) + })?; let result = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; let recovered = sqlx::query_scalar::<_, String>( @@ -593,7 +585,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { .bind(&command.account_id) .fetch_optional(&mut *transaction) .await - .map_err(|_| postgres_unavailable("recover provider account state"))? + .map_err(|source| postgres_unavailable("recover provider account state", source))? .ok_or_else(|| StoreError::NotFound { entity: ENTITY, id: command.account_id.clone(), @@ -614,11 +606,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { ) -> StoreResult { command.scope.validate()?; validate_admin_account_ids(&command.account_ids)?; - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| postgres_unavailable("begin provider account admin deletion"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + postgres_unavailable("begin provider account admin deletion", source) + })?; let result = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; delete_provider_accounts_in_transaction( @@ -646,7 +636,7 @@ async fn update_provider_account_notes_in_transaction( .bind(notes.trim()) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("update provider account notes"))?; + .map_err(|source| postgres_unavailable("update provider account notes", source))?; Ok(()) } @@ -666,7 +656,7 @@ async fn replace_account_group_assignments_in_transaction( .bind(&group_ids) .fetch_one(&mut **transaction) .await - .map_err(|_| postgres_unavailable("validate account group assignment"))?; + .map_err(|source| postgres_unavailable("validate account group assignment", source))?; if usize::try_from(known_group_count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { entity: "account group", @@ -678,7 +668,7 @@ async fn replace_account_group_assignments_in_transaction( .bind(account_ids) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("clear account group assignments"))?; + .map_err(|source| postgres_unavailable("clear account group assignments", source))?; if group_ids.is_empty() { return Ok(()); } @@ -693,7 +683,7 @@ async fn replace_account_group_assignments_in_transaction( .bind(account_ids) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("assign accounts to groups"))?; + .map_err(|source| postgres_unavailable("assign accounts to groups", source))?; Ok(()) } @@ -786,7 +776,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( kind: ConflictKind::InvalidTransition, } } else { - postgres_unavailable("upsert provider account in admin transaction") + postgres_unavailable("upsert provider account in admin transaction", error) } })? .ok_or_else(|| StoreError::Conflict { @@ -865,7 +855,7 @@ pub(crate) async fn rotate_provider_account_in_transaction( kind: ConflictKind::InvalidTransition, } } else { - postgres_unavailable("rotate provider account in admin transaction") + postgres_unavailable("rotate provider account in admin transaction", error) } })? .ok_or_else(|| StoreError::Conflict { @@ -911,7 +901,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction( .bind(concurrency_limit.is_some()) .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("set provider accounts state in admin transaction"))? + .map_err(|source| postgres_unavailable("set provider accounts state in admin transaction", source))? .into_iter() .collect::>(); let expected = account_ids.iter().cloned().collect::>(); @@ -939,7 +929,9 @@ pub(crate) async fn delete_provider_accounts_in_transaction( .bind(&scope.provider_kind) .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("delete provider account in admin transaction"))?; + .map_err(|source| { + postgres_unavailable("delete provider account in admin transaction", source) + })?; let deleted = deleted.into_iter().collect::>(); let expected = account_ids.iter().cloned().collect::>(); if deleted == expected { @@ -1022,14 +1014,14 @@ pub(crate) async fn finish_admin_transaction( transaction .commit() .await - .map_err(|_| postgres_unavailable(operation))?; + .map_err(|source| postgres_unavailable(operation, source))?; Ok(value) } Err(error) => { transaction .rollback() .await - .map_err(|_| postgres_unavailable(operation))?; + .map_err(|source| postgres_unavailable(operation, source))?; Err(error) } } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index a2cb5fbfe..7d73af863 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -465,11 +465,11 @@ pub(crate) fn core_account_from_summary( summary: ProviderAccountSummary, ) -> Result { let id = CoreProviderAccountId::new(summary.id) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; let provider = ProviderKind::new(summary.provider_kind) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; let revision = CoreCredentialRevision::new(summary.credential_revision.get()) - .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + .map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?; Ok(CoreProviderAccount::new( id, provider, @@ -501,17 +501,6 @@ pub(crate) fn core_account_from_summary( )) } -pub(crate) fn core_store_error(error: StoreError) -> CoreStoreError { - let kind = match error { - StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable, - StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict, - StoreError::NotFound { .. } | StoreError::InvalidData { .. } => { - CoreStoreErrorKind::InvalidData - } - }; - CoreStoreError::new(kind) -} - pub(crate) fn require_core_update(updated: bool) -> Result<(), CoreStoreError> { if updated { Ok(()) diff --git a/backend/crates/gateway-store/src/postgres/proxies.rs b/backend/crates/gateway-store/src/postgres/proxies.rs index d70b28465..938e8b884 100644 --- a/backend/crates/gateway-store/src/postgres/proxies.rs +++ b/backend/crates/gateway-store/src/postgres/proxies.rs @@ -67,7 +67,7 @@ async fn exclude_active_imports( .bind(id) .fetch_one(&mut **transaction) .await - .map_err(|_| unavailable())?; + .map_err(unavailable)?; if !acquired { return Err(conflict(id)); } @@ -77,8 +77,8 @@ async fn exclude_active_imports( fn store_error(error: StoreError) -> AdminStoreError { admin_store_error(ENTITY, error) } -fn unavailable() -> StoreError { - postgres_unavailable("outbound proxy operation") +fn unavailable(source: impl std::error::Error + Send + Sync + 'static) -> StoreError { + postgres_unavailable("outbound proxy operation", source) } fn conflict(id: &str) -> StoreError { StoreError::Conflict { @@ -213,7 +213,7 @@ async fn transaction_record( .build() .fetch_optional(&mut **transaction) .await - .map_err(|_| unavailable())? + .map_err(unavailable)? .ok_or_else(|| conflict(id))?; record(row) } @@ -234,7 +234,7 @@ async fn save_test( .bind(result.exit_ip.map(|ip| ip.to_string())).bind(result.exit_ipv4.map(|ip| ip.to_string())) .bind(result.exit_ipv6.map(|ip| ip.to_string())).bind(result.message) .bind(sqlx::types::Json(result.location)).bind(detected.map(sqlx::types::Json)) - .execute(&mut **transaction).await.map_err(|_| unavailable())?; + .execute(&mut **transaction).await.map_err(unavailable)?; Ok(()) } @@ -252,7 +252,7 @@ async fn save_location( .bind(location.map(|value| value.region.trim())) .bind(location.map(|value| value.city.trim())) .bind(location.map(|value| value.timezone.name())) - .execute(&mut **transaction).await.map_err(|_| unavailable())?; + .execute(&mut **transaction).await.map_err(unavailable)?; Ok(()) } @@ -264,7 +264,7 @@ async fn lock_url( .bind(proxy.expose_url()) .execute(&mut **transaction) .await - .map_err(|_| unavailable())?; + .map_err(unavailable)?; Ok(()) } @@ -276,7 +276,7 @@ pub(crate) async fn ensure_proxy_for_url( ) -> StoreResult<(String, bool)> { lock_url(transaction, proxy).await?; if let Some(id) = sqlx::query_scalar::<_, String>("select id from outbound_proxies where proxy_url = $1 order by created_at, id limit 1 for share") - .bind(proxy.expose_url()).fetch_optional(&mut **transaction).await.map_err(|_| unavailable())? { + .bind(proxy.expose_url()).fetch_optional(&mut **transaction).await.map_err(unavailable)? { return Ok((id, false)); } let id = format!("proxy_{}", uuid::Uuid::now_v7().simple()); @@ -287,7 +287,7 @@ pub(crate) async fn ensure_proxy_for_url( .bind(proxy.expose_url()) .execute(&mut **transaction) .await - .map_err(|_| unavailable())?; + .map_err(unavailable)?; Ok((id, true)) } @@ -308,7 +308,7 @@ pub(crate) async fn resolve_proxy_selection( .bind(id) .fetch_optional(&mut **transaction) .await - .map_err(|_| unavailable())? + .map_err(unavailable)? .ok_or_else(|| StoreError::NotFound { entity: ENTITY, id: id.clone(), @@ -354,7 +354,7 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let revision = bump_config_revision_in_transaction(&mut transaction) .await .map_err(store_error)?; @@ -369,7 +369,7 @@ impl ProxyStore for PgProxyRepository { .bind(proxy_id) .execute(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; if updated.rows_affected() != 1 { return Err(store_error(conflict(proxy_id))); } @@ -388,7 +388,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; admin_revision(revision) } @@ -403,12 +403,12 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; // 数量和当前页共用只读快照，避免绑定变化使同一次响应的分页事实不一致 sqlx::query("set transaction isolation level repeatable read, read only") .execute(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let total: i64 = sqlx::query_scalar( "select count(a.id) from outbound_proxies p left join provider_accounts a on a.outbound_proxy_id = p.id @@ -420,7 +420,7 @@ impl ProxyStore for PgProxyRepository { .bind(&query.search) .fetch_optional(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))? + .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { entity: ENTITY, @@ -439,7 +439,7 @@ impl ProxyStore for PgProxyRepository { .bind(i64::from(query.page - 1) * i64::from(query.page_size.get())) .fetch_all(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))? + .map_err(|source| store_error(unavailable(source)))? .into_iter() .map(|row| { Ok(ProxyAccountRef { @@ -468,7 +468,7 @@ impl ProxyStore for PgProxyRepository { .bind(&account_ids) .fetch_all(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let mut groups = BTreeMap::>::new(); for (account_id, group_id, name, color, enabled) in rows { groups.entry(account_id).or_default().push(AccountGroupRef { @@ -486,7 +486,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; Ok(ProxyAccountPage { items, total: u64::try_from(total).map_err(|_| store_error(invalid()))?, @@ -505,14 +505,14 @@ impl ProxyStore for PgProxyRepository { .pool .acquire() .await - .map_err(|_| store_error(unavailable()))? + .map_err(|source| store_error(unavailable(source)))? .detach(); let acquired: bool = sqlx::query_scalar("select pg_try_advisory_lock_shared(hashtextextended($1, 739219))") .bind(id) .fetch_one(&mut connection) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; if !acquired { return Err(store_error(conflict(id))); } @@ -524,7 +524,7 @@ impl ProxyStore for PgProxyRepository { .bind(id) .execute(&mut connection) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; return Err(error); } }; @@ -550,7 +550,7 @@ impl ProxyStore for PgProxyRepository { .bind(&query.search) .fetch_one(&self.pool) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let mut builder = QueryBuilder::::new(SELECT); builder .push(" where strpos(lower(p.name), lower(") @@ -563,7 +563,7 @@ impl ProxyStore for PgProxyRepository { .build() .fetch_all(&self.pool) .await - .map_err(|_| store_error(unavailable()))? + .map_err(|source| store_error(unavailable(source)))? .into_iter() .map(record) .collect::>>() @@ -583,7 +583,7 @@ impl ProxyStore for PgProxyRepository { .build() .fetch_optional(&self.pool) .await - .map_err(|_| store_error(unavailable()))? + .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { entity: ENTITY, @@ -602,7 +602,7 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let revision = bump_config_revision_in_transaction(&mut transaction) .await .map_err(store_error)?; @@ -621,7 +621,7 @@ impl ProxyStore for PgProxyRepository { .bind(command.auto_location) .execute(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; if let Some(test) = command.test { let current = transaction_record(&mut transaction, &id) .await @@ -643,7 +643,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; Ok(ProxyMutation { config_revision: admin_revision(revision)?, record: self.get(&id).await?, @@ -659,7 +659,7 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; exclude_active_imports(&mut transaction, &command.id) .await .map_err(store_error)?; @@ -677,7 +677,7 @@ impl ProxyStore for PgProxyRepository { .bind(&command.id) .fetch_one(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; if duplicate { return Err(store_error(conflict(&command.id))); } @@ -697,7 +697,7 @@ impl ProxyStore for PgProxyRepository { where id = $1 and revision = $2") .bind(&command.id).bind(i64::try_from(command.revision.get()).map_err(|_| store_error(invalid()))?) .bind(&command.name).bind(command.proxy.as_ref().map(OutboundProxy::expose_url)).bind(command.auto_location) - .execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?; + .execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?; if changed.rows_affected() != 1 { return Err(store_error(conflict(&command.id))); } @@ -715,7 +715,7 @@ impl ProxyStore for PgProxyRepository { .map_err(store_error)?; } sqlx::query("update provider_accounts a set outbound_proxy_url = p.proxy_url, updated_at = greatest(now(), a.updated_at) from outbound_proxies p where p.id = $1 and a.outbound_proxy_id = p.id and a.outbound_proxy_url is distinct from p.proxy_url") - .bind(&command.id).execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?; + .bind(&command.id).execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?; audit( &mut transaction, context, @@ -729,7 +729,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; Ok(ProxyMutation { config_revision: admin_revision(revision)?, record: self.get(&command.id).await?, @@ -746,7 +746,7 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; exclude_active_imports(&mut transaction, id) .await .map_err(store_error)?; @@ -765,7 +765,7 @@ impl ProxyStore for PgProxyRepository { }) { store_error(conflict(id)) } else { - store_error(unavailable()) + store_error(unavailable(error)) } })?; if result.rows_affected() != 1 { @@ -784,7 +784,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; admin_revision(config_revision) } @@ -799,7 +799,7 @@ impl ProxyStore for PgProxyRepository { .pool .begin() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; exclude_active_imports(&mut transaction, id) .await .map_err(store_error)?; @@ -809,7 +809,7 @@ impl ProxyStore for PgProxyRepository { ) .fetch_one(&mut *transaction) .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; let current = transaction_record(&mut transaction, id) .await .map_err(store_error)?; @@ -818,7 +818,7 @@ impl ProxyStore for PgProxyRepository { transaction .rollback() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; return Err(store_error(conflict(id))); } save_test(&mut transaction, ¤t, result) @@ -826,7 +826,7 @@ impl ProxyStore for PgProxyRepository { .map_err(store_error)?; if current.auto_location { sqlx::query("update outbound_proxies set revision = revision + 1, updated_at = now() where id = $1") - .bind(id).execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?; + .bind(id).execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?; } let updated = transaction_record(&mut transaction, id) .await @@ -852,7 +852,7 @@ impl ProxyStore for PgProxyRepository { transaction .commit() .await - .map_err(|_| store_error(unavailable()))?; + .map_err(|source| store_error(unavailable(source)))?; Ok(ProxyMutation { config_revision: admin_revision(revision)?, record: updated, diff --git a/backend/crates/gateway-store/src/postgres/retention.rs b/backend/crates/gateway-store/src/postgres/retention.rs index 3dd447dc0..3dc209202 100644 --- a/backend/crates/gateway-store/src/postgres/retention.rs +++ b/backend/crates/gateway-store/src/postgres/retention.rs @@ -34,7 +34,7 @@ impl RetentionStore for PgRetentionRepository { ) .fetch_optional(&self.pool) .await - .map_err(|_| unavailable("load retention settings"))? + .map_err(|source| unavailable("load retention settings", source))? .ok_or_else(|| { AdminStoreError::new( AdminStoreErrorKind::NotFound, @@ -89,12 +89,15 @@ impl RetentionStore for PgRetentionRepository { .execute(&self.pool) .await .map(|result| result.rows_affected()) - .map_err(|_| unavailable("delete expired records")) + .map_err(|source| unavailable("delete expired records", source)) } } -fn unavailable(operation: &'static str) -> AdminStoreError { - admin_store_error("retention", postgres_unavailable(operation)) +fn unavailable( + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> AdminStoreError { + admin_store_error("retention", postgres_unavailable(operation, source)) } fn invalid_policy() -> AdminStoreError { diff --git a/backend/crates/gateway-store/src/postgres/runtime_settings.rs b/backend/crates/gateway-store/src/postgres/runtime_settings.rs index d4b1ce55d..7f024b464 100644 --- a/backend/crates/gateway-store/src/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/src/postgres/runtime_settings.rs @@ -248,12 +248,12 @@ impl RuntimeSettingsRepository for PgRuntimeSettingsRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin runtime settings update"))?; + .map_err(|source| postgres_unavailable("begin runtime settings update", source))?; let revision = update_runtime_settings_in_transaction(&mut transaction, &update).await?; transaction .commit() .await - .map_err(|_| postgres_unavailable("commit runtime settings update"))?; + .map_err(|source| postgres_unavailable("commit runtime settings update", source))?; Ok(revision) } } @@ -274,7 +274,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul ) .fetch_optional(pool) .await - .map_err(|_| postgres_unavailable("load runtime settings"))? + .map_err(|source| postgres_unavailable("load runtime settings", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -300,7 +300,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { .bind(slot) .execute(&self.pool) .await - .map_err(|_| provider_unavailable("claim warmup slot"))? + .map_err(|source| crate::provider_unavailable("claim warmup slot", source))? .rows_affected() == 1; Ok(claimed) @@ -325,7 +325,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { .bind(provider.as_str()) .bind(sqlx::types::Json(initial.expose_to_provider())) .fetch_one(&self.pool).await - .map_err(|_| provider_unavailable("initialize Provider request profile"))?; + .map_err(|source| crate::provider_unavailable("initialize Provider request profile", source))?; Ok(gateway_core::account::OpaqueProviderData::new(document.0)) }) } @@ -356,7 +356,9 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { .bind(provider.as_str()) .fetch_all(&self.pool) .await - .map_err(|_| provider_unavailable("load Provider request profile configurations"))?; + .map_err(|source| { + crate::provider_unavailable("load Provider request profile configurations", source) + })?; let expected_revision = i64::try_from(revision.get()) .map_err(|_| provider_invalid("validate Provider request profile revision"))?; if rows.is_empty() @@ -386,7 +388,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { Box::pin(async move { let settings = RuntimeSettingsRepository::load_runtime_settings(self) .await - .map_err(|_| provider_unavailable("load refresh policy"))?; + .map_err(|source| crate::provider_unavailable("load refresh policy", source))?; let concurrency = NonZeroU32::new(settings.refresh_concurrency) .ok_or_else(|| provider_invalid("decode refresh policy"))?; ProviderRefreshPolicy::try_new( @@ -402,7 +404,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { Box::pin(async move { let settings = RuntimeSettingsRepository::load_runtime_settings(self) .await - .map_err(|_| provider_unavailable("load freeze policy"))?; + .map_err(|source| crate::provider_unavailable("load freeze policy", source))?; ProviderFreezePolicy::try_new( settings.account_auto_freeze_enabled, settings.account_auto_freeze_threshold, @@ -421,7 +423,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository { Box::pin(async move { let settings = RuntimeSettingsRepository::load_runtime_settings(self) .await - .map_err(|_| provider_unavailable("load warmup policy"))?; + .map_err(|source| crate::provider_unavailable("load warmup policy", source))?; ProviderWarmupPolicy::try_new( settings.account_warmup_enabled, settings.account_warmup_schedule_time, @@ -449,7 +451,7 @@ pub(crate) async fn load_runtime_settings_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load runtime settings in transaction"))? + .map_err(|source| postgres_unavailable("load runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -564,7 +566,7 @@ pub(crate) async fn update_runtime_settings_in_transaction( .bind(i64::from(update.openai_guardian_reserved_concurrency)) .fetch_optional(&mut **transaction) .await - .map_err(|_| postgres_unavailable("update runtime settings in transaction"))? + .map_err(|source| postgres_unavailable("update runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -583,7 +585,7 @@ pub(crate) async fn bump_config_revision_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| postgres_unavailable("bump config revision in transaction"))? + .map_err(|source| postgres_unavailable("bump config revision in transaction", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -605,7 +607,7 @@ pub(crate) async fn update_admin_api_key_in_transaction( .bind(admin_api_key.as_deref()) .execute(&mut **transaction) .await - .map_err(|_| postgres_unavailable("update admin api key in transaction"))?; + .map_err(|source| postgres_unavailable("update admin api key in transaction", source))?; Ok(()) } @@ -732,10 +734,6 @@ fn invalid_numeric() -> StoreError { } } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/postgres/snapshot.rs b/backend/crates/gateway-store/src/postgres/snapshot.rs index 815946163..b6afead3a 100644 --- a/backend/crates/gateway-store/src/postgres/snapshot.rs +++ b/backend/crates/gateway-store/src/postgres/snapshot.rs @@ -98,11 +98,13 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { .pool .begin() .await - .map_err(|_| postgres_unavailable("begin runtime snapshot"))?; + .map_err(|source| postgres_unavailable("begin runtime snapshot", source))?; sqlx::query("set transaction isolation level repeatable read read only") .execute(&mut *transaction) .await - .map_err(|_| postgres_unavailable("configure runtime snapshot transaction"))?; + .map_err(|source| { + postgres_unavailable("configure runtime snapshot transaction", source) + })?; let (config_revision, settings) = load_settings(&mut transaction).await?; let client_api_keys = load_client_keys(&mut transaction).await?; @@ -112,7 +114,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { transaction .commit() .await - .map_err(|_| postgres_unavailable("commit runtime snapshot"))?; + .map_err(|source| postgres_unavailable("commit runtime snapshot", source))?; let observed_current_revision = RuntimeSnapshotRepository::current_config_revision(self).await?; @@ -133,7 +135,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { ) .fetch_optional(&self.pool) .await - .map_err(|_| postgres_unavailable("read current config revision"))? + .map_err(|source| postgres_unavailable("read current config revision", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -284,7 +286,7 @@ async fn load_settings( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load snapshot settings"))? + .map_err(|source| postgres_unavailable("load snapshot settings", source))? .ok_or_else(|| StoreError::NotFound { entity: "runtime settings", id: "1".to_owned(), @@ -349,7 +351,7 @@ async fn load_client_keys( ) .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load snapshot client policies"))?; + .map_err(|source| postgres_unavailable("load snapshot client policies", source))?; rows.into_iter() .map(|row| { let mut key = ClientApiKeySnapshot::from_persisted(row.0, row.1, row.2, row.3, row.4)?; @@ -367,7 +369,7 @@ async fn load_account_groups( ) .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load snapshot account groups"))?; + .map_err(|source| postgres_unavailable("load snapshot account groups", source))?; rows.into_iter() .map(|(id, name, enabled, fast_mode)| { Ok(SnapshotAccountGroupData { @@ -394,7 +396,7 @@ async fn load_provider_accounts( >("select id, provider_kind, model_access_json from provider_accounts order by id") .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load snapshot provider accounts")) + .map_err(|source| postgres_unavailable("load snapshot provider accounts", source)) .map(|rows| { rows.into_iter() .map( @@ -417,7 +419,7 @@ async fn load_group_memberships( ) .fetch_all(&mut **transaction) .await - .map_err(|_| postgres_unavailable("load snapshot group memberships"))?; + .map_err(|source| postgres_unavailable("load snapshot group memberships", source))?; rows.into_iter() .map(|(group_id, account_id)| { Ok(SnapshotGroupMembershipData { diff --git a/backend/crates/gateway-store/src/redis/artifact_profile.rs b/backend/crates/gateway-store/src/redis/artifact_profile.rs index ee05c7cd9..ea403de04 100644 --- a/backend/crates/gateway-store/src/redis/artifact_profile.rs +++ b/backend/crates/gateway-store/src/redis/artifact_profile.rs @@ -97,7 +97,9 @@ impl ProviderArtifactProfileCachePort for RedisProviderArtifactProfileRepository .arg(ttl_ms) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("replace artifact profile"))?; + .map_err(|source| { + crate::provider_unavailable("replace artifact profile", source) + })?; match outcome { 1 => Ok(true), 0 => Ok(false), @@ -126,7 +128,7 @@ impl ProviderArtifactProfileCachePort for RedisProviderArtifactProfileRepository .arg("profile") .query_async(&mut connection) .await - .map_err(|_| provider_unavailable("read artifact profile"))?; + .map_err(|source| crate::provider_unavailable("read artifact profile", source))?; let (artifact_sequence, verified_at_ms, payload) = values; if artifact_sequence.is_none() && verified_at_ms.is_none() && payload.is_none() { return Ok(None); @@ -171,10 +173,6 @@ fn exact_positive_millis( .ok_or_else(|| provider_invalid(operation)) } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/auth.rs b/backend/crates/gateway-store/src/redis/auth.rs index a1c53d7eb..d2422d094 100644 --- a/backend/crates/gateway-store/src/redis/auth.rs +++ b/backend/crates/gateway-store/src/redis/auth.rs @@ -162,7 +162,7 @@ return current .arg(expiry) .invoke_async(&mut self.connection.clone()) .await - .map_err(|_| redis_unavailable("renew authentication session"))?; + .map_err(|source| redis_unavailable("renew authentication session", source))?; payload.map(|value| decode_session(&value)).transpose() } @@ -173,7 +173,7 @@ return current .arg(key) .query_async::>(&mut connection) .await - .map_err(|_| redis_unavailable("load authentication session"))?; + .map_err(|source| redis_unavailable("load authentication session", source))?; payload.map(|value| decode_session(&value)).transpose() } @@ -193,7 +193,7 @@ return current .arg(expires_at_millis) .query_async::(&mut connection) .await - .map_err(|_| redis_unavailable("store authentication session"))?; + .map_err(|source| redis_unavailable("store authentication session", source))?; Ok(()) } @@ -204,7 +204,7 @@ return current .arg(key) .query_async::>(&mut connection) .await - .map_err(|_| redis_unavailable("delete authentication session"))?; + .map_err(|source| redis_unavailable("delete authentication session", source))?; payload.map(|value| decode_session(&value)).transpose() } @@ -232,7 +232,7 @@ return current .arg(window_millis) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("rate limit authentication login"))?; + .map_err(|source| redis_unavailable("rate limit authentication login", source))?; if retry_after_millis <= 0 { Ok(None) } else { diff --git a/backend/crates/gateway-store/src/redis/client_admission.rs b/backend/crates/gateway-store/src/redis/client_admission.rs index 9166e5f4f..df9039046 100644 --- a/backend/crates/gateway-store/src/redis/client_admission.rs +++ b/backend/crates/gateway-store/src/redis/client_admission.rs @@ -293,7 +293,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { .arg(u8::from(request.allow_concurrency_acquire)) .invoke_async::(&mut connection) .await - .map_err(|_| redis_unavailable("admit client request"))?; + .map_err(|source| redis_unavailable("admit client request", source))?; match code { 0 => Ok(ClientAdmissionDecision::Granted), 1 => Ok(ClientAdmissionDecision::Rejected( @@ -320,7 +320,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { .arg(model_request_id) .query_async::(&mut connection) .await - .map_err(|_| redis_unavailable("release client request"))?; + .map_err(|source| redis_unavailable("release client request", source))?; Ok(removed == 1) } @@ -361,7 +361,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { let (code, restored_recent_requests, restored_running_requests) = invocation .invoke_async::<(i64, u64, u64)>(&mut connection) .await - .map_err(|_| redis_unavailable("restore client admission"))?; + .map_err(|source| redis_unavailable("restore client admission", source))?; if code == -1 { return Err(invalid("request start time is after Redis server time")); } @@ -381,7 +381,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { .arg(&keys) .query_async::(&mut connection) .await - .map_err(|_| redis_unavailable("clear client admission"))?; + .map_err(|source| redis_unavailable("clear client admission", source))?; Ok(()) } } @@ -413,7 +413,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { .arg(redis_duration_millis(ttl)?) .invoke_async::(&mut connection) .await - .map_err(|_| redis_unavailable("renew client request"))?; + .map_err(|source| redis_unavailable("renew client request", source))?; Ok(renewed == 1) }) }, diff --git a/backend/crates/gateway-store/src/redis/credential_cooldown.rs b/backend/crates/gateway-store/src/redis/credential_cooldown.rs index 9cd9bdbcb..d37eb50b9 100644 --- a/backend/crates/gateway-store/src/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/src/redis/credential_cooldown.rs @@ -229,7 +229,7 @@ impl RedisCredentialCooldownRepository { .invoke_async::(&mut connection) .await } - .map_err(|_| redis_unavailable("cache credential cooldown"))?; + .map_err(|source| redis_unavailable("cache credential cooldown", source))?; Ok(written == 1) } @@ -254,7 +254,7 @@ impl RedisCredentialCooldownRepository { .await }; let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) = - result.map_err(|_| redis_unavailable("read credential cooldown"))?; + result.map_err(|source| redis_unavailable("read credential cooldown", source))?; if present == 0 { return Ok(None); } @@ -298,7 +298,7 @@ impl RedisCredentialCooldownRepository { .invoke_async::(&mut connection) .await } - .map_err(|_| redis_unavailable("invalidate credential cooldown"))?; + .map_err(|source| redis_unavailable("invalidate credential cooldown", source))?; Ok(removed == 1) } @@ -310,7 +310,7 @@ impl RedisCredentialCooldownRepository { .arg(-1) .query_async(&mut connection) .await - .map_err(|_| redis_unavailable("list indexed cooldowns")) + .map_err(|source| redis_unavailable("list indexed cooldowns", source)) } pub(crate) async fn active_cooldowns(&self) -> StoreResult { @@ -384,7 +384,7 @@ impl RedisCredentialCooldownRepository { .arg(uuid::Uuid::new_v4().to_string()) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("finish capacity freeze"))?; + .map_err(|source| redis_unavailable("finish capacity freeze", source))?; Ok(changed == 1) } @@ -398,7 +398,7 @@ impl RedisCredentialCooldownRepository { .arg(self.capacity_peak_key(provider_account_id)?) .query_async(&mut connection) .await - .map_err(|_| redis_unavailable("read capacity peak in-flight"))?; + .map_err(|source| redis_unavailable("read capacity peak in-flight", source))?; peak.map(u32::try_from) .transpose() .map_err(|_| invalid("capacity peak in-flight is invalid")) @@ -493,7 +493,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { .arg(100) .query_async(&mut connection) .await - .map_err(|_| redis_unavailable("scan account cooldown keys"))?; + .map_err(|source| redis_unavailable("scan account cooldown keys", source))?; keys.extend(found); cursor = next; if cursor == 0 { @@ -513,7 +513,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository { .arg(provider_account_id) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("delete account cooldowns and index"))?; + .map_err(|source| redis_unavailable("delete account cooldowns and index", source))?; Ok(removed > 0) } } @@ -533,7 +533,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { }; CredentialCooldownRepository::cache_credential_cooldown(self, &record) .await - .map_err(|_| provider_unavailable("cache credential cooldown")) + .map_err(|source| crate::provider_unavailable("cache credential cooldown", source)) }) } @@ -544,7 +544,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { Box::pin(async move { CredentialCooldownRepository::read_credential_cooldown(self, account_id.as_str()) .await - .map_err(|_| provider_unavailable("read credential cooldown"))? + .map_err(|source| crate::provider_unavailable("read credential cooldown", source))? .map(|record| { let account_id = ProviderAccountId::new(record.provider_account_id) .map_err(|_| provider_invalid("decode credential cooldown"))?; @@ -575,7 +575,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { revision, ) .await - .map_err(|_| provider_unavailable("clear credential cooldown")) + .map_err(|source| crate::provider_unavailable("clear credential cooldown", source)) }) } @@ -595,7 +595,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { None, ) .await - .map_err(|_| provider_unavailable("cache scoped credential cooldown")) + .map_err(|source| { + crate::provider_unavailable("cache scoped credential cooldown", source) + }) }) } @@ -612,7 +614,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { None, ) .await - .map_err(|_| provider_unavailable("read scoped credential cooldown"))? + .map_err(|source| { + crate::provider_unavailable("read scoped credential cooldown", source) + })? .map(|(revision, until, _, _)| { Ok(ProviderScopedCooldown::new( account_id.clone(), @@ -642,7 +646,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { None, ) .await - .map_err(|_| provider_unavailable("clear scoped credential cooldown")) + .map_err(|source| { + crate::provider_unavailable("clear scoped credential cooldown", source) + }) }) } @@ -653,7 +659,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { Box::pin(async move { self.delete_account_cooldowns(account_id.as_str()) .await - .map_err(|_| provider_unavailable("clear all credential cooldowns")) + .map_err(|source| { + crate::provider_unavailable("clear all credential cooldowns", source) + }) }) } @@ -680,7 +688,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { .arg(i64::from(in_flight)) .invoke_async(&mut connection) .await - .map_err(|_| provider_unavailable("record capacity failure"))?; + .map_err(|source| crate::provider_unavailable("record capacity failure", source))?; u32::try_from(count).map_err(|_| provider_invalid("decode capacity failure count")) }) } @@ -710,7 +718,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { .arg(account_id.as_str()) .invoke_async(&mut connection) .await - .map_err(|_| provider_unavailable("clear cooldown after success"))?; + .map_err(|source| { + crate::provider_unavailable("clear cooldown after success", source) + })?; Ok(()) }) } @@ -722,7 +732,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { Box::pin(async move { self.read_capacity_peak(account_id.as_str()) .await - .map_err(|_| provider_unavailable("read capacity peak in-flight")) + .map_err(|source| { + crate::provider_unavailable("read capacity peak in-flight", source) + }) }) } } @@ -734,10 +746,6 @@ fn invalid(message: &str) -> StoreError { } } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index 40ebcfd7f..e44b87f08 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -384,7 +384,7 @@ impl RedisCredentialLeaseRepository { .arg(SIGNAL_TTL_MILLIS) .invoke_async::(&mut connection) .await - .map_err(|_| redis_unavailable("advance provider scheduling cursor"))?; + .map_err(|source| redis_unavailable("advance provider scheduling cursor", source))?; cursor .parse::() .map_err(|_| invalid("Redis returned an invalid scheduling cursor")) @@ -458,7 +458,7 @@ impl RedisCredentialLeaseRepository { .arg(signal_ttl) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("acquire credential lease"))?; + .map_err(|source| redis_unavailable("acquire credential lease", source))?; if acquired == 0 { return Ok(LeaseAttempt { grant: None, @@ -485,7 +485,7 @@ impl RedisCredentialLeaseRepository { .key(&keys[2]) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("load credential runtime signal"))?; + .map_err(|source| redis_unavailable("load credential runtime signal", source))?; let in_flight = in_flight .parse::() .map_err(|_| invalid("Redis returned an invalid in-flight count"))?; @@ -532,7 +532,7 @@ impl RedisProviderLeaseCoordinator { self.repository .advance_scheduling_cursor(client_api_key_id, provider_kind) .await - .map_err(|_| provider_unavailable("advance scheduling cursor")) + .map_err(|source| crate::provider_unavailable("advance scheduling cursor", source)) } async fn load_signals( @@ -547,7 +547,7 @@ impl RedisProviderLeaseCoordinator { .repository .credential_runtime_signals(&ids) .await - .map_err(|_| provider_unavailable("load scheduling signals"))?; + .map_err(|source| crate::provider_unavailable("load scheduling signals", source))?; signals .into_iter() .map(|signal| { @@ -579,7 +579,10 @@ impl RedisProviderLeaseCoordinator { ) -> Result { let ttl = request.deadline().bounded(REQUEST_LEASE_TTL); if ttl.is_zero() { - return Err(provider_unavailable("acquire expired scheduling lease")); + return Err(ProviderStoreError::new( + ProviderStoreErrorKind::Unavailable, + "acquire expired scheduling lease", + )); } let acquisition = self .repository @@ -592,13 +595,15 @@ impl RedisProviderLeaseCoordinator { ttl, }) .await - .map_err(|_| provider_unavailable("acquire scheduling lease"))?; + .map_err(|source| crate::provider_unavailable("acquire scheduling lease", source))?; Ok(match acquisition { CredentialBoundedLeaseAcquisition::Acquired(guard) => { ProviderLeaseAcquisition::Acquired(Box::new( guard .maintain(request.deadline(), request.cancellation()) - .map_err(|_| provider_unavailable("maintain scheduling lease"))?, + .map_err(|source| { + crate::provider_unavailable("maintain scheduling lease", source) + })?, )) } CredentialBoundedLeaseAcquisition::Busy { retry_after } => { @@ -622,7 +627,7 @@ impl RedisProviderLeaseCoordinator { ttl: OAUTH_REFRESH_LEASE_TTL, }) .await - .map_err(|_| provider_unavailable("acquire refresh capacity"))?; + .map_err(|source| crate::provider_unavailable("acquire refresh capacity", source))?; Ok(match acquisition { CredentialBoundedLeaseAcquisition::Acquired(guard) => { ProviderLeaseAcquisition::Acquired(Box::new(guard)) @@ -676,7 +681,9 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator { Some(guard) => ProviderLeaseAcquisition::Acquired(Box::new(guard)), None => ProviderLeaseAcquisition::Busy { retry_after: None }, }) - .map_err(|_| provider_unavailable("acquire refresh lease")) + .map_err(|source| { + crate::provider_unavailable("acquire refresh lease", source) + }) } } }) @@ -701,7 +708,9 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator { .repository .credential_runtime_signals(&ids) .await - .map_err(|_| provider_unavailable("load account in-flight signals"))?; + .map_err(|source| { + crate::provider_unavailable("load account in-flight signals", source) + })?; Ok(signals .into_iter() .filter_map(|signal| { @@ -713,10 +722,6 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator { } } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - #[async_trait] impl CredentialLeaseRepository for RedisCredentialLeaseRepository { async fn acquire_credential_lease( @@ -743,7 +748,7 @@ impl CredentialLeaseRepository for RedisCredentialLeaseRepository { .arg(duration_millis(request.ttl)?) .invoke_async(&mut connection) .await - .map_err(|_| redis_unavailable("renew credential lease"))?; + .map_err(|source| redis_unavailable("renew credential lease", source))?; if renewed == 0 { return Ok(None); } @@ -768,7 +773,7 @@ impl CredentialLeaseRepository for RedisCredentialLeaseRepository { .arg(member) .invoke_async::(&mut connection) .await - .map_err(|_| redis_unavailable("release credential lease"))?; + .map_err(|source| redis_unavailable("release credential lease", source))?; Ok(released == 1) } diff --git a/backend/crates/gateway-store/src/redis/credential_state.rs b/backend/crates/gateway-store/src/redis/credential_state.rs index 438e6eae6..42b781169 100644 --- a/backend/crates/gateway-store/src/redis/credential_state.rs +++ b/backend/crates/gateway-store/src/redis/credential_state.rs @@ -178,7 +178,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository { .arg(ttl_ms) .invoke_async::(&mut connection) .await - .map_err(|_| redis_unavailable("cache credential state"))?; + .map_err(|source| redis_unavailable("cache credential state", source))?; Ok(written == 1) } @@ -200,7 +200,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository { .arg("observed_at_ms") .query_async(&mut connection) .await - .map_err(|_| redis_unavailable("read credential state"))?; + .map_err(|source| redis_unavailable("read credential state", source))?; let (Some(revision), Some(enabled), Some(credential_state), Some(observed_at_ms)) = values else { return Ok(None); @@ -228,7 +228,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository { .arg(self.key(provider_account_id)?) .query_async::(&mut connection) .await - .map_err(|_| redis_unavailable("clear credential state"))?; + .map_err(|source| redis_unavailable("clear credential state", source))?; Ok(removed == 1) } } @@ -262,7 +262,7 @@ impl ProviderCatalogCacheRepository for RedisCredentialStateRepository { .arg(ttl_ms) .query_async::<()>(&mut connection) .await - .map_err(|_| redis_unavailable("replace provider catalog cache"))?; + .map_err(|source| redis_unavailable("replace provider catalog cache", source))?; Ok(()) } @@ -275,7 +275,7 @@ impl ProviderCatalogCacheRepository for RedisCredentialStateRepository { .arg(self.catalog_key(key)?) .query_async::>>(&mut connection) .await - .map_err(|_| redis_unavailable("read provider catalog cache"))?; + .map_err(|source| redis_unavailable("read provider catalog cache", source))?; let Some(payload) = payload else { return Ok(None); }; @@ -310,7 +310,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { ) .await .map(|_| ()) - .map_err(|_| provider_unavailable("replace credential state")) + .map_err(|source| crate::provider_unavailable("replace credential state", source)) }) } @@ -323,7 +323,9 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { let cached = CredentialStateRepository::read_credential_state(self, account_id.as_str()) .await - .map_err(|_| provider_unavailable("read credential state"))?; + .map_err(|source| { + crate::provider_unavailable("read credential state", source) + })?; cached .map(|state| { let account_id = ProviderAccountId::new(state.provider_account_id) @@ -351,7 +353,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { Box::pin(async move { CredentialStateRepository::clear_credential_state(self, account_id.as_str()) .await - .map_err(|_| provider_unavailable("clear credential state")) + .map_err(|source| crate::provider_unavailable("clear credential state", source)) }) } @@ -367,14 +369,14 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { .ok_or_else(|| provider_invalid("record refresh backoff"))?; let key = self .backoff_key(account_id.as_str()) - .map_err(|_| provider_unavailable("record refresh backoff"))?; + .map_err(|source| crate::provider_unavailable("record refresh backoff", source))?; let mut connection = self.connection.clone(); let count = Script::new(RECORD_REFRESH_BACKOFF_SCRIPT) .key(key) .arg(window_millis) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("record refresh backoff"))?; + .map_err(|source| crate::provider_unavailable("record refresh backoff", source))?; Ok(u32::try_from(count).unwrap_or(u32::MAX)) }) } @@ -386,13 +388,13 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository { Box::pin(async move { let key = self .backoff_key(account_id.as_str()) - .map_err(|_| provider_unavailable("clear refresh backoff"))?; + .map_err(|source| crate::provider_unavailable("clear refresh backoff", source))?; let mut connection = self.connection.clone(); redis::cmd("DEL") .arg(key) .query_async::(&mut connection) .await - .map_err(|_| provider_unavailable("clear refresh backoff"))?; + .map_err(|source| crate::provider_unavailable("clear refresh backoff", source))?; Ok(()) }) } @@ -421,7 +423,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository { ttl_seconds, ) .await - .map_err(|_| provider_unavailable("replace catalog cache")) + .map_err(|source| crate::provider_unavailable("replace catalog cache", source)) }) } @@ -437,7 +439,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository { }; ProviderCatalogCacheRepository::get_provider_catalog(self, &key) .await - .map_err(|_| provider_unavailable("read catalog cache")) + .map_err(|source| crate::provider_unavailable("read catalog cache", source)) }) } } @@ -456,10 +458,6 @@ fn catalog_invalid(message: &str) -> StoreError { } } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/oauth_pending.rs b/backend/crates/gateway-store/src/redis/oauth_pending.rs index 6685490c5..1bcbaab6d 100644 --- a/backend/crates/gateway-store/src/redis/oauth_pending.rs +++ b/backend/crates/gateway-store/src/redis/oauth_pending.rs @@ -178,7 +178,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository { .arg(ttl_millis) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("create OAuth pending flow"))?; + .map_err(|source| { + crate::provider_unavailable("create OAuth pending flow", source) + })?; Ok(if stored == 1 { OAuthPendingPutOutcome::Stored } else { @@ -209,7 +211,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository { .arg(claim_ttl) .invoke_async(&mut connection) .await - .map_err(|_| provider_unavailable("claim OAuth pending flow"))?; + .map_err(|source| { + crate::provider_unavailable("claim OAuth pending flow", source) + })?; match status { 0 => Ok(OAuthPendingClaimOutcome::NotFound), -1 => Ok(OAuthPendingClaimOutcome::OwnerMismatch), @@ -248,7 +252,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository { .arg(claim) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("release OAuth pending claim"))?; + .map_err(|source| { + crate::provider_unavailable("release OAuth pending claim", source) + })?; Ok(match status { 0 => OAuthPendingReleaseOutcome::NotFound, -1 => OAuthPendingReleaseOutcome::OwnerMismatch, @@ -282,7 +288,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository { .arg(claim) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("consume OAuth pending claim"))?; + .map_err(|source| { + crate::provider_unavailable("consume OAuth pending claim", source) + })?; Ok(match status { 0 => OAuthPendingConsumeOutcome::NotFound, -1 => OAuthPendingConsumeOutcome::OwnerMismatch, @@ -319,10 +327,6 @@ fn ttl_millis(ttl: Duration) -> StoreResult { }) } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/provider_session_affinity.rs b/backend/crates/gateway-store/src/redis/provider_session_affinity.rs index b50436ff1..1f8ae486f 100644 --- a/backend/crates/gateway-store/src/redis/provider_session_affinity.rs +++ b/backend/crates/gateway-store/src/redis/provider_session_affinity.rs @@ -103,7 +103,9 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository { .arg(self.key(provider_kind, key)?) .query_async::>(&mut self.connection.clone()) .await - .map_err(|_| provider_unavailable("load provider session binding"))?; + .map_err(|source| { + crate::provider_unavailable("load provider session binding", source) + })?; raw.as_deref().map(BindingRecord::decode).transpose() }) } @@ -138,7 +140,9 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository { .arg(ttl_millis) .invoke_async::(&mut self.connection.clone()) .await - .map_err(|_| provider_unavailable("admit provider session binding"))?; + .map_err(|source| { + crate::provider_unavailable("admit provider session binding", source) + })?; Ok(applied.then_some(binding)) }) } @@ -155,7 +159,7 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository { .arg(format!("{}:alias", self.key(provider, alias)?)) .query_async::>(&mut self.connection.clone()) .await - .map_err(|_| provider_unavailable("load session alias"))?; + .map_err(|source| crate::provider_unavailable("load session alias", source))?; value .map(|raw| { let record: AliasRecord = serde_json::from_str(&raw) @@ -189,7 +193,7 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository { .arg(&record) .arg(session_affinity_ttl_millis(ttl)?) .invoke_async::(&mut self.connection.clone()).await - .map_err(|_| provider_unavailable("bind session alias")) + .map_err(|source| crate::provider_unavailable("bind session alias", source)) }) } } @@ -202,10 +206,6 @@ fn session_affinity_ttl_millis(ttl: Duration) -> Result .map_err(|_| provider_invalid("validate provider session affinity TTL")) } -fn provider_unavailable(operation: &'static str) -> ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/provider_session_exclusion.rs b/backend/crates/gateway-store/src/redis/provider_session_exclusion.rs index cbbdc5590..50ba82acd 100644 --- a/backend/crates/gateway-store/src/redis/provider_session_exclusion.rs +++ b/backend/crates/gateway-store/src/redis/provider_session_exclusion.rs @@ -119,7 +119,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository { .arg(self.key(provider_kind, key)?) .query_async::>(&mut connection) .await - .map_err(|_| provider_unavailable("load provider session exclusion"))?; + .map_err(|source| { + crate::provider_unavailable("load provider session exclusion", source) + })?; payload.map(decode_state).transpose() }) } @@ -144,7 +146,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository { .arg(ttl_seconds) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("record provider session exclusion"))?; + .map_err(|source| { + crate::provider_unavailable("record provider session exclusion", source) + })?; decode_state(payload) }) } @@ -162,7 +166,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository { .arg(expected_revision) .invoke_async::(&mut connection) .await - .map_err(|_| provider_unavailable("clear provider session exclusion"))?; + .map_err(|source| { + crate::provider_unavailable("clear provider session exclusion", source) + })?; Ok(removed > 0) }) } @@ -185,10 +191,6 @@ fn decode_state(payload: impl AsRef) -> Result ProviderStoreError { - ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation) -} - fn provider_invalid(operation: &'static str) -> ProviderStoreError { ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation) } diff --git a/backend/crates/gateway-store/src/redis/runtime_change.rs b/backend/crates/gateway-store/src/redis/runtime_change.rs index fb92e8bd8..0a743cd91 100644 --- a/backend/crates/gateway-store/src/redis/runtime_change.rs +++ b/backend/crates/gateway-store/src/redis/runtime_change.rs @@ -54,11 +54,11 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository { .client .get_connection_manager() .await - .map_err(|_| redis_unavailable("connect runtime change publisher"))?; + .map_err(|source| redis_unavailable("connect runtime change publisher", source))?; connection .publish::<_, _, i64>(&self.channel, payload) .await - .map_err(|_| redis_unavailable("publish runtime change"))?; + .map_err(|source| redis_unavailable("publish runtime change", source))?; Ok(()) } @@ -67,11 +67,11 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository { .client .get_async_pubsub() .await - .map_err(|_| redis_unavailable("connect runtime change subscriber"))?; + .map_err(|source| redis_unavailable("connect runtime change subscriber", source))?; pubsub .subscribe(&self.channel) .await - .map_err(|_| redis_unavailable("subscribe runtime changes"))?; + .map_err(|source| redis_unavailable("subscribe runtime changes", source))?; let stream = pubsub.into_on_message().map(|message| { let payload = message .get_payload::() diff --git a/backend/crates/gateway-store/src/value.rs b/backend/crates/gateway-store/src/value.rs index e8d422046..6e201acfd 100644 --- a/backend/crates/gateway-store/src/value.rs +++ b/backend/crates/gateway-store/src/value.rs @@ -23,12 +23,13 @@ pub enum ConflictKind { } /// Store adapter 的稳定错误边界 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] pub enum StoreError { #[error("{backend:?} store is unavailable: {message}")] Unavailable { backend: StoreBackend, message: String, + source: Option, }, #[error("{entity} {id} was not found")] NotFound { entity: &'static str, id: String }, @@ -101,7 +102,7 @@ pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> Ad StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid, StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable, }; - AdminStoreError::new(kind, resource, "store operation failed") + AdminStoreError::new(kind, resource, "store operation failed").with_source(error) } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -240,16 +241,46 @@ pub(crate) fn require_nonempty( } } -pub(crate) fn postgres_unavailable(operation: &'static str) -> StoreError { +pub(crate) fn postgres_unavailable( + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> StoreError { StoreError::Unavailable { backend: StoreBackend::PostgreSql, message: operation.to_owned(), + source: Some(gateway_core::error::ErrorSource::new(source)), } } -pub(crate) fn redis_unavailable(operation: &'static str) -> StoreError { +pub(crate) fn redis_unavailable( + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> StoreError { StoreError::Unavailable { backend: StoreBackend::Redis, message: operation.to_owned(), + source: Some(gateway_core::error::ErrorSource::new(source)), } } + +/// Provider 存储端口共用不可用分类，实际后端及原因由来源链保留 +pub(crate) fn provider_unavailable( + operation: &'static str, + source: impl std::error::Error + Send + Sync + 'static, +) -> gateway_core::provider_ports::ProviderStoreError { + gateway_core::provider_ports::ProviderStoreError::caused_by( + gateway_core::provider_ports::ProviderStoreErrorKind::Unavailable, + operation, + source, + ) +} + +pub(crate) fn core_store_error(error: StoreError) -> gateway_core::error::StoreError { + use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind}; + let kind = match error { + StoreError::Unavailable { .. } => StoreErrorKind::Unavailable, + StoreError::Conflict { .. } => StoreErrorKind::Conflict, + StoreError::NotFound { .. } | StoreError::InvalidData { .. } => StoreErrorKind::InvalidData, + }; + CoreStoreError::caused_by(kind, error) +} diff --git a/backend/crates/gateway-store/tests/postgres/connection.rs b/backend/crates/gateway-store/tests/postgres/connection.rs new file mode 100644 index 000000000..9fd564814 --- /dev/null +++ b/backend/crates/gateway-store/tests/postgres/connection.rs @@ -0,0 +1,23 @@ +//! 验证实际基础设施错误在 Store 边界保留来源而不进入普通格式化 + +use std::error::Error as _; + +#[tokio::test] +async fn invalid_postgres_options_preserve_the_sqlx_source() { + let error = gateway_store::postgres::connect_and_migrate( + "postgres://localhost:invalid-port/test", + gateway_store::StorePoolConfig::default(), + ) + .await + .unwrap_err(); + + let source = error.source().expect("PostgreSQL source"); + assert!(source.downcast_ref::().is_some()); + assert!(source.source().is_some()); + assert!( + error + .to_string() + .contains("parse PostgreSQL connection options") + ); + assert!(!format!("{error:?} {error}").contains("invalid-port")); +} diff --git a/backend/crates/gateway-store/tests/postgres/execution.rs b/backend/crates/gateway-store/tests/postgres/execution.rs index 3c1d7e3ab..3f96fb180 100644 --- a/backend/crates/gateway-store/tests/postgres/execution.rs +++ b/backend/crates/gateway-store/tests/postgres/execution.rs @@ -677,7 +677,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization { provider_metadata_json: None, error: None, provider_error_code: None, - raw_upstream_error: None, + error_details: None, failure_observation: Default::default(), retry_after_ms: None, usage: Usage::new(), @@ -829,8 +829,8 @@ async fn core_adapter_persists_opaque_response_ids_as_bytes() { } #[tokio::test] -async fn core_adapter_persists_raw_upstream_error_verbatim() { - let Some(database) = TestDatabase::create("execution_raw_upstream_error").await else { +async fn core_adapter_persists_native_causes_and_verbatim_upstream_details() { + let Some(database) = TestDatabase::create("execution_error_details").await else { return; }; seed_running_request(&database.pool, "req_raw_upstream_error") @@ -838,6 +838,12 @@ async fn core_adapter_persists_raw_upstream_error_verbatim() { .expect("seed model request"); let store = PgExecutionStore::new(database.pool.clone()); let raw = r#"{"error":{"message":"raw upstream body","opaque":"\u0000"}}"#; + let provider_error = gateway_core::error::ProviderError::new( + gateway_core::error::ProviderErrorKind::Unavailable, + UpstreamSendState::Sent, + ) + .with_source(std::io::Error::other("original local cause")) + .with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(raw)); let mut finalization = successful_core_finalization("req_raw_upstream_error"); finalization.outcome = ExecutionOutcome::Failed; finalization.client_status_code = Some(502); @@ -846,19 +852,25 @@ async fn core_adapter_persists_raw_upstream_error_verbatim() { GatewayErrorKind::UpstreamUnavailable, "upstream service is unavailable", )); - finalization.raw_upstream_error = Some(raw.to_owned()); + finalization.error_details = provider_error.error_details(); ExecutionStore::finalize_model_request(&store, finalization) .await - .expect("persist raw upstream error"); + .expect("persist restricted error details"); let persisted: Option = sqlx::query_scalar( - "select raw_upstream_error from model_requests where id = 'req_raw_upstream_error'", + "select error_details from model_requests where id = 'req_raw_upstream_error'", ) .fetch_one(&database.pool) .await - .expect("load raw upstream error"); - assert_eq!(persisted.as_deref(), Some(raw)); + .expect("load restricted error details"); + let persisted: Value = serde_json::from_str(persisted.as_deref().unwrap()).unwrap(); + assert_eq!(persisted["upstream"], raw); + assert_eq!( + persisted["causes"]["messages"], + json!(["original local cause"]) + ); + assert_eq!(persisted["causes"]["truncated"], false); database.close().await; } @@ -1527,7 +1539,7 @@ pub(super) fn early_failure(request: &CoreNewModelRequest) -> CoreModelRequestFi "no available provider", )), provider_error_code: None, - raw_upstream_error: None, + error_details: None, failure_observation: Default::default(), retry_after_ms: None, usage: Usage::new(), @@ -1622,7 +1634,7 @@ async fn zero_attempt_failure_is_queryable_without_fabricating_upstream_facts() "downstream_committed_at", "service_tier", "provider_observation_json", - "raw_upstream_error", + "error_details", "input_tokens", "output_tokens", "total_tokens", @@ -1675,7 +1687,7 @@ async fn zero_attempt_failure_is_queryable_without_fabricating_upstream_facts() assert_eq!(error.upstream_transport, None); assert_eq!(error.upstream_status_code, None); assert_eq!(error.upstream_request_id, None); - assert_eq!(error.raw_upstream_error, None); + assert_eq!(error.error_details, None); // 路由入口不等于已选择 Provider；没有平台事实时不能被平台/attempt 筛选命中 for filter in [ diff --git a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs index ce2fac4d3..def00d8ea 100644 --- a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs @@ -561,34 +561,63 @@ async fn shutdown_drains_observations_already_accepted_by_the_queue() { #[tokio::test] async fn observation_byte_budget_drops_payload_without_waiting_for_the_database() { + for provider_error in [ + ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent) + .with_source(std::io::Error::other("x".repeat(2_048))), + ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent) + .with_upstream_code(OpaqueUpstreamValue::new("x".repeat(2_048))), + ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent) + .with_raw_upstream_error(gateway_core::error::RawUpstreamError::new( + "x".repeat(2_048), + )), + ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent) + .with_diagnostic(gateway_core::error::ProviderDiagnostic::new( + "x".repeat(2_048), + )), + ] { + let inner = Arc::new(RecordingStore::default()); + let (store, _writer) = BufferedExecutionStore::with_limits( + Arc::clone(&inner), + NonZeroUsize::new(8).expect("capacity"), + NonZeroUsize::new(1_024).expect("byte capacity"), + ); + let failure = ProbeFailure { + provider_kind: ProviderKind::new("openai").expect("provider"), + account_id: ProviderAccountId::new("acct_byte_budget").expect("account"), + upstream_model_id: UpstreamModelId::new("gpt-byte-budget").expect("model"), + error: provider_error, + latency: Duration::from_millis(1), + }; + + tokio::time::timeout( + Duration::from_millis(50), + store.record_probe_failure(failure), + ) + .await + .expect("byte budget must not wait") + .expect("byte budget is fail-open"); + + assert!(inner.operations.lock().expect("operations lock").is_empty()); + assert_eq!(store.stats().queued_items, 0); + assert_eq!(store.stats().dropped_total, 1); + } +} + +#[tokio::test] +async fn final_error_body_counts_towards_the_observation_byte_budget() { let inner = Arc::new(RecordingStore::default()); let (store, _writer) = BufferedExecutionStore::with_limits( Arc::clone(&inner), - NonZeroUsize::new(8).expect("capacity"), - NonZeroUsize::new(1_024).expect("byte capacity"), + NonZeroUsize::new(8).unwrap(), + NonZeroUsize::new(1_024).unwrap(), ); - let provider_error = - ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent) - .with_upstream_code(OpaqueUpstreamValue::new("x".repeat(2_048))); - let failure = ProbeFailure { - provider_kind: ProviderKind::new("openai").expect("provider"), - account_id: ProviderAccountId::new("acct_byte_budget").expect("account"), - upstream_model_id: UpstreamModelId::new("gpt-byte-budget").expect("model"), - error: provider_error, - latency: Duration::from_millis(1), - }; - - tokio::time::timeout( - Duration::from_millis(50), - store.record_probe_failure(failure), - ) - .await - .expect("byte budget must not wait") - .expect("byte budget is fail-open"); - - assert!(inner.operations.lock().expect("operations lock").is_empty()); - assert_eq!(store.stats().queued_items, 0); + let request = accepted_request("req_error_body_budget"); + let mut finalization = early_failure(&request); + finalization.error_details = Some("x".repeat(2_048)); + store.finalize_model_request(finalization).await.unwrap(); assert_eq!(store.stats().dropped_total, 1); + assert_eq!(store.stats().queued_bytes, 0); + assert!(inner.operations.lock().unwrap().is_empty()); } #[tokio::test] diff --git a/backend/crates/gateway-store/tests/postgres/mod.rs b/backend/crates/gateway-store/tests/postgres/mod.rs index 6e5477449..c61a22c2f 100644 --- a/backend/crates/gateway-store/tests/postgres/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/mod.rs @@ -18,6 +18,7 @@ mod admission_recovery; mod backup; mod client_budgets; mod client_keys; +mod connection; mod execution; mod execution_buffer; mod health; diff --git a/backend/crates/gateway-store/tests/postgres/observability/mod.rs b/backend/crates/gateway-store/tests/postgres/observability/mod.rs index ed7ad3e50..51bc5190f 100644 --- a/backend/crates/gateway-store/tests/postgres/observability/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/observability/mod.rs @@ -777,7 +777,7 @@ async fn ops_should_include_incomplete_upstream_errors() { client_transport = 'websocket', upstream_transport = 'websocket', downstream_committed_at = completed_at, client_status_code = null, upstream_status_code = 400, - provider_error_code = null, raw_upstream_error = $1 + provider_error_code = null, error_details = $1 where id = 'req_observe_failed'", ) .bind(raw_error) @@ -810,7 +810,7 @@ async fn ops_should_include_incomplete_upstream_errors() { assert_eq!(error.failure_kind, "invalid_request"); assert_eq!(error.upstream_status_code, Some(400)); assert_eq!(error.client_status_code, None); - assert_eq!(error.raw_upstream_error.as_deref(), Some(raw_error)); + assert_eq!(error.error_details.as_deref(), Some(raw_error)); } #[tokio::test] diff --git a/backend/crates/gateway-store/tests/postgres/ops_events.rs b/backend/crates/gateway-store/tests/postgres/ops_events.rs index 3d8da0c95..a6d79a10b 100644 --- a/backend/crates/gateway-store/tests/postgres/ops_events.rs +++ b/backend/crates/gateway-store/tests/postgres/ops_events.rs @@ -18,7 +18,7 @@ fn request_scoped_ops_event_requires_attempt_index() { upstream_model_id: None, failure_kind: "timeout".to_owned(), upstream_send_state: Some("not_sent".to_owned()), - raw_upstream_error: None, + error_details: None, status_code: None, provider_error_code: None, retry_after_ms: None, diff --git a/backend/crates/gateway-store/tests/postgres/schema_integrity.rs b/backend/crates/gateway-store/tests/postgres/schema_integrity.rs index 0ff34fd94..be6952b66 100644 --- a/backend/crates/gateway-store/tests/postgres/schema_integrity.rs +++ b/backend/crates/gateway-store/tests/postgres/schema_integrity.rs @@ -2,6 +2,44 @@ use super::TestDatabase; +#[tokio::test] +async fn error_details_migration_preserves_existing_request_and_event_text() { + let Some(db) = TestDatabase::create_through("error_details_upgrade", 22).await else { + return; + }; + seed_request(&db.pool).await; + let raw = "{ \"error\": {\"code\":\"Vendor.Unknown\",\"message\":\"原始错误\"} }"; + sqlx::query("update model_requests set raw_upstream_error = $1 where id = 'req_integrity'") + .bind(raw) + .execute(&db.pool) + .await + .unwrap(); + sqlx::query( + "insert into ops_events (id, level, component, operation, failure_kind, message, created_at, raw_upstream_error) + values ('error_details_upgrade', 'error', 'provider', 'generate', 'unavailable', 'safe message', now(), $1)", + ).bind(raw).execute(&db.pool).await.unwrap(); + + super::TEST_MIGRATOR.run(&db.pool).await.unwrap(); + let details: Vec = sqlx::query_scalar( + "select error_details from model_requests where id = 'req_integrity' + union all select error_details from ops_events where id = 'error_details_upgrade'", + ) + .fetch_all(&db.pool) + .await + .unwrap(); + assert_eq!(details, vec![raw.to_owned(), raw.to_owned()]); + let old_columns: i64 = sqlx::query_scalar( + "select count(*) from information_schema.columns + where table_schema = current_schema() and table_name in ('model_requests', 'ops_events') + and column_name = 'raw_upstream_error'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(old_columns, 0); + db.close().await; +} + async fn seed_request(pool: &sqlx::PgPool) { sqlx::query( "insert into model_requests ( diff --git a/backend/crates/gateway-store/tests/postgres/snapshots.rs b/backend/crates/gateway-store/tests/postgres/snapshots.rs index 94cd53992..d5cb2be46 100644 --- a/backend/crates/gateway-store/tests/postgres/snapshots.rs +++ b/backend/crates/gateway-store/tests/postgres/snapshots.rs @@ -255,7 +255,7 @@ async fn attempts_should_keep_their_own_account_snapshots() { upstream_model_id: Some("upstream-a".to_owned()), failure_kind: "rate_limited".to_owned(), upstream_send_state: Some("sent".to_owned()), - raw_upstream_error: Some( + error_details: Some( r#"{"error":{"code":"rate_limit","message":"raw marker"}}"#.to_owned(), ), status_code: Some(429), @@ -365,7 +365,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti set outcome = 'failed', error_kind = 'upstream_error', error_message = 'snapshot failure', client_status_code = 502, upstream_status_code = 502, completed_at = $1, - raw_upstream_error = $2 + error_details = $2 where id = 'req_snap_error'", ) .bind(started_at + chrono::Duration::seconds(2)) @@ -387,7 +387,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti upstream_model_id: Some("grok-test".to_owned()), failure_kind: "auth_failed".to_owned(), upstream_send_state: Some("not_sent".to_owned()), - raw_upstream_error: Some(r#"{"error":"raw probe failure"}"#.to_owned()), + error_details: Some(r#"{"error":"raw probe failure"}"#.to_owned()), status_code: Some(401), provider_error_code: Some("invalid_api_key".to_owned()), retry_after_ms: None, @@ -435,7 +435,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti ); assert_eq!(request_error.endpoint.as_deref(), Some("/v1/responses")); assert_eq!( - request_error.raw_upstream_error.as_deref(), + request_error.error_details.as_deref(), Some(r#"{"error":{"message":"raw request failure"}}"#) ); let probe_error = errors @@ -457,7 +457,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti ); assert_eq!(probe_error.endpoint, None); assert_eq!( - probe_error.raw_upstream_error.as_deref(), + probe_error.error_details.as_deref(), Some(r#"{"error":"raw probe failure"}"#) ); diff --git a/backend/crates/providers/openai/src/credential/refresh.rs b/backend/crates/providers/openai/src/credential/refresh.rs index 751756afb..eededdd56 100644 --- a/backend/crates/providers/openai/src/credential/refresh.rs +++ b/backend/crates/providers/openai/src/credential/refresh.rs @@ -7,6 +7,8 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Utc}; use futures::{StreamExt as _, stream}; +use gateway_core::error::ErrorSource; + use gateway_core::account::{ AccountErrorReason, CredentialState, ProviderAccount, ProviderAccountId, ProviderRefreshQuery, }; @@ -393,10 +395,11 @@ impl CodexCredentialRefreshService { CodexCredentialRefreshError, > { let now = SystemTime::now(); - let provider = ProviderKind::new(PROVIDER_NAME) - .map_err(|_| CredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| { + CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source))) + })?; let limit = NonZeroU32::new(MAX_REFRESH_BATCH) - .ok_or(CredentialRepositoryError::InvalidCredentialData)?; + .ok_or(CredentialRepositoryError::InvalidCredentialData(None))?; // 查询窗口按错峰上界（2×margin）放宽，先取回候选超集， // 再在内存中按账号稳定偏移收窄；存储层谓词保持通用窗口语义。 let query = ProviderRefreshQuery::new( @@ -520,7 +523,7 @@ impl CodexCredentialRefreshService { credential_revision: revision.get(), }) } - Err(CredentialRepositoryError::RevisionConflict) => { + Err(CredentialRepositoryError::RevisionConflict(_)) => { Ok(CodexCredentialRefreshOutcome::Stale { account_id: due.account.id().to_string(), }) @@ -544,7 +547,7 @@ impl CodexCredentialRefreshService { } = failure; match self.repository.load_runtime_credential(account).await { Ok(_) => {} - Err(CredentialRepositoryError::RevisionConflict) => { + Err(CredentialRepositoryError::RevisionConflict(_)) => { return Ok(CodexCredentialRefreshOutcome::Stale { account_id: account.id().to_string(), }); @@ -586,7 +589,7 @@ impl CodexCredentialRefreshService { ); Ok(outcome) } - Err(CredentialRepositoryError::RevisionConflict) => { + Err(CredentialRepositoryError::RevisionConflict(_)) => { Ok(CodexCredentialRefreshOutcome::Stale { account_id: account.id().to_string(), }) @@ -640,7 +643,7 @@ impl CodexCredentialRefreshService { ); Ok(true) } - Err(CredentialRepositoryError::RevisionConflict) => Ok(false), + Err(CredentialRepositoryError::RevisionConflict(_)) => Ok(false), Err(error) => Err(error.into()), } } diff --git a/backend/crates/providers/openai/src/credential/repository.rs b/backend/crates/providers/openai/src/credential/repository.rs index c608f561b..7b690f8e5 100644 --- a/backend/crates/providers/openai/src/credential/repository.rs +++ b/backend/crates/providers/openai/src/credential/repository.rs @@ -3,6 +3,8 @@ use std::sync::Arc; use std::time::SystemTime; +use gateway_core::error::ErrorSource; + use gateway_core::account::{ AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision, CredentialState, LoadedCredential, ProviderAccount, ProviderAccountStore, @@ -53,7 +55,7 @@ impl CodexCredentialRepository { let mut data = CodexCredentialCodec::decode_complete(¤t.credential)?; let oauth = data .oauth_mut() - .ok_or(CredentialRepositoryError::InvalidCredentialData)?; + .ok_or(CredentialRepositoryError::InvalidCredentialData(None))?; oauth.access_token = secret.access_token.expose_secret().to_owned(); oauth.refresh_token = secret .refresh_token @@ -73,7 +75,9 @@ impl CodexCredentialRepository { access_token_expires_at, next_refresh_at, ) - .map_err(|_| CredentialRepositoryError::InvalidCredentialData)? + .map_err(|source| { + CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source))) + })? .preserving_profile() .with_account_state( oauth_account_state(account, CredentialState::Ready), @@ -108,7 +112,9 @@ impl CodexCredentialRepository { account.access_token_expires_at(), Some(next_refresh_at), ) - .map_err(|_| CredentialRepositoryError::InvalidCredentialData)? + .map_err(|source| { + CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source))) + })? .preserving_profile(); if let Some(error_reason) = error_reason { update = update.with_account_state( @@ -124,8 +130,9 @@ impl CodexCredentialRepository { pub async fn list_for_provider( &self, ) -> Result, CredentialRepositoryError> { - let provider = ProviderKind::new(PROVIDER_NAME) - .map_err(|_| CredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| { + CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source))) + })?; self.store .list_for_provider(&provider) .await @@ -147,11 +154,11 @@ impl CodexCredentialRepository { loaded: &LoadedCredential, ) -> Result { if loaded.account.provider().as_str() != PROVIDER_NAME { - return Err(CredentialRepositoryError::InvalidCredentialData); + return Err(CredentialRepositoryError::InvalidCredentialData(None)); } let data = CodexCredentialCodec::decode_complete(&loaded.credential)?; if data.authentication_kind() != loaded.account.authentication_kind() { - return Err(CredentialRepositoryError::InvalidCredentialData); + return Err(CredentialRepositoryError::InvalidCredentialData(None)); } CodexCredentialCodec::decode(&loaded.credential).map_err(Into::into) } @@ -161,14 +168,14 @@ impl CodexCredentialRepository { account: &ProviderAccount, ) -> Result { if account.provider().as_str() != PROVIDER_NAME { - return Err(CredentialRepositoryError::InvalidCredentialData); + return Err(CredentialRepositoryError::InvalidCredentialData(None)); } let loaded = self .store .load_credential(account.id(), account.revision()) .await?; if loaded.account != *account { - return Err(CredentialRepositoryError::RevisionConflict); + return Err(CredentialRepositoryError::RevisionConflict(None)); } self.decode_runtime_credential(&loaded) } @@ -205,7 +212,9 @@ impl CodexCredentialRepository { account.access_token_expires_at(), account.next_refresh_at(), ) - .map_err(|_| CredentialRepositoryError::InvalidCredentialData)? + .map_err(|source| { + CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source))) + })? .preserving_profile(); cas_revision(self.store.compare_and_swap_credential(update).await?) } @@ -270,35 +279,34 @@ fn cas_revision( ) -> Result { match outcome { CredentialCasOutcome::Updated(revision) => Ok(revision), - CredentialCasOutcome::Conflict => Err(CredentialRepositoryError::RevisionConflict), + CredentialCasOutcome::Conflict => Err(CredentialRepositoryError::RevisionConflict(None)), } } #[derive(Debug, Error)] pub enum CredentialRepositoryError { #[error("Codex credential data is invalid")] - InvalidCredentialData, + InvalidCredentialData(#[source] Option), #[error("Codex credential revision conflict")] - RevisionConflict, + RevisionConflict(#[source] Option), #[error("provider account store is unavailable")] - Store, + Store(#[source] gateway_core::error::StoreError), } impl From for CredentialRepositoryError { fn from(error: gateway_core::error::StoreError) -> Self { match error.kind() { - gateway_core::error::StoreErrorKind::Conflict => Self::RevisionConflict, - gateway_core::error::StoreErrorKind::Unavailable - | gateway_core::error::StoreErrorKind::InvalidState - | gateway_core::error::StoreErrorKind::InvalidData => Self::Store, - _ => Self::Store, + gateway_core::error::StoreErrorKind::Conflict => { + Self::RevisionConflict(Some(ErrorSource::new(error))) + } + _ => Self::Store(error), } } } impl From for CredentialRepositoryError { - fn from(_: CodexCredentialDataError) -> Self { - Self::InvalidCredentialData + fn from(error: CodexCredentialDataError) -> Self { + Self::InvalidCredentialData(Some(ErrorSource::new(error))) } } diff --git a/backend/crates/providers/openai/src/credential/selector.rs b/backend/crates/providers/openai/src/credential/selector.rs index 2a82dc5cd..ff2827e9b 100644 --- a/backend/crates/providers/openai/src/credential/selector.rs +++ b/backend/crates/providers/openai/src/credential/selector.rs @@ -5,6 +5,8 @@ use std::fmt; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; +use gateway_core::error::ErrorSource; + use gateway_core::account::{ AccountCandidate, AccountCapacitySnapshot, AccountEligibilityPolicy, AccountErrorReason, AccountFeedbackStats, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, @@ -519,7 +521,7 @@ impl CodexCredentialSelector { .store() .get_account(required) .await - .map_err(|_| CredentialSelectionError::Store)? + .map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))? { accounts.push(account); } @@ -555,7 +557,7 @@ impl CodexCredentialSelector { { let runtime = match self.repository.load_runtime_credential(&account).await { Ok(runtime) => runtime, - Err(CredentialRepositoryError::RevisionConflict) => { + Err(CredentialRepositoryError::RevisionConflict(_)) => { retry_account_snapshot( request.attempt, &account, @@ -564,7 +566,7 @@ impl CodexCredentialSelector { continue 'capacity; } // 非固定账号的损坏凭据不能阻断其余账号的传输资格检查 - Err(CredentialRepositoryError::InvalidCredentialData) + Err(CredentialRepositoryError::InvalidCredentialData(_)) if pinned_account.is_none() => { continue; @@ -839,12 +841,12 @@ impl CodexCredentialSelector { .iter() .find(|candidate| candidate.account.id() == selected.account.id()) .map(|candidate| candidate.account.clone()) - .ok_or(CredentialSelectionError::InvalidCredential)?; + .ok_or(CredentialSelectionError::InvalidCredential(None))?; // 额度观测等并发更新会使整个账号快照失效，必须重新选号并校验资格 // 在占用租约和请求间隔前完成校验，避免重读被自己的异步释放挡住 let runtime = match self.repository.load_runtime_credential(&account).await { Ok(runtime) => runtime, - Err(CredentialRepositoryError::RevisionConflict) => { + Err(CredentialRepositoryError::RevisionConflict(_)) => { retry_account_snapshot(request.attempt, &account, &mut snapshot_retries)?; continue 'capacity; } @@ -971,8 +973,8 @@ impl CodexCredentialSelector { ), ) .await - .map_err(|_| binding_store_error())? - .map_err(|_| binding_store_error())?; + .map_err(binding_store_error)? + .map_err(binding_store_error)?; if !applied { return Err(CredentialSelectionError::SessionBound(Box::new( CredentialSelectionError::NoEligibleCredential, @@ -990,8 +992,8 @@ impl CodexCredentialSelector { self.session_affinity.load_alias(&self.provider_kind, turn), ) .await - .map_err(|_| binding_store_error())? - .map_err(|_| binding_store_error()) + .map_err(binding_store_error)? + .map_err(binding_store_error) .map(|key| key.map(CodexSessionAffinity::from_turn_alias)) } @@ -1004,8 +1006,8 @@ impl CodexCredentialSelector { self.session_affinity.load(&self.provider_kind, key), ) .await - .map_err(|_| binding_store_error())? - .map_err(|_| binding_store_error()) + .map_err(binding_store_error)? + .map_err(binding_store_error) } async fn admit_session( @@ -1025,9 +1027,9 @@ impl CodexCredentialSelector { ), ) .await - .map_err(|_| binding_store_error())? + .map_err(binding_store_error)? .map(|binding| binding.is_some()) - .map_err(|_| binding_store_error()) + .map_err(binding_store_error) } async fn prepare_cyber_policy_scope( @@ -1190,7 +1192,7 @@ impl CodexCredentialSelector { .quota .record_confirmed_exhaustion(account, QuotaEvidence::PaymentRequired, None, now) .await - .map_err(|_| CredentialSelectionError::Store), + .map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source))), CodexAccountFailure::UsageLimitExhausted { reset_at } => self .quota .record_confirmed_exhaustion( @@ -1200,13 +1202,13 @@ impl CodexCredentialSelector { now, ) .await - .map_err(|_| CredentialSelectionError::Store), + .map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source))), // 429：临时限流只写运行时冷却，不改变凭据或额度事实 CodexAccountFailure::RateLimited { retry_after } => { self.quota .apply_rate_limit_429(account, retry_after, now) .await - .map_err(|_| CredentialSelectionError::Store)?; + .map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?; Ok(()) } // Cloudflare 挑战：内存退避表（记录风险计数），不写账号事实 @@ -1311,8 +1313,8 @@ impl CodexCredentialSelector { .store() .get_account(account_id) .await - .map_err(|_| CredentialSelectionError::Store)? - .ok_or(CredentialSelectionError::InvalidCredential) + .map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))? + .ok_or(CredentialSelectionError::InvalidCredential(None)) } pub async fn capture_response_cookies( @@ -1440,8 +1442,12 @@ impl CodexCredentialSelector { } } -fn binding_store_error() -> CredentialSelectionError { - CredentialSelectionError::SessionBound(Box::new(CredentialSelectionError::Store)) +fn binding_store_error( + source: impl std::error::Error + Send + Sync + 'static, +) -> CredentialSelectionError { + CredentialSelectionError::SessionBound(Box::new(CredentialSelectionError::Store( + ErrorSource::new(source), + ))) } fn affinity_selection_for_bound_account( @@ -1641,7 +1647,7 @@ pub enum CredentialSelectionError { #[error("session account changed during native continuation")] ContinuationOwnerChanged, #[error("session account binding cannot be bypassed: {0}")] - SessionBound(Box), + SessionBound(#[source] Box), #[error(transparent)] QueueRejected(#[from] QueueRejection), #[error("no eligible Codex account")] @@ -1651,15 +1657,15 @@ pub enum CredentialSelectionError { #[error("Codex account capacity is unavailable")] CapacityUnavailable { retry_after: Option }, #[error("Codex account data is invalid")] - InvalidCredential, + InvalidCredential(#[source] Option), #[error("Codex account changed repeatedly during selection")] AccountSnapshotChanged, #[error("Codex account store is unavailable")] - Store, + Store(#[source] ErrorSource), #[error("Codex account lease runtime is unavailable")] - Coordinator, + Coordinator(#[source] ProviderStoreError), #[error("Codex Cookie policy rejected the value")] - CookiePolicy, + CookiePolicy(#[source] super::cookie::CookiePolicyError), #[error("account scheduling policy rejected the request")] PolicyRejected, #[error("account scheduling policy is unavailable")] @@ -1668,24 +1674,25 @@ pub enum CredentialSelectionError { impl From for CredentialSelectionError { fn from(error: CredentialRepositoryError) -> Self { - match error { - CredentialRepositoryError::InvalidCredentialData => Self::InvalidCredential, - CredentialRepositoryError::RevisionConflict | CredentialRepositoryError::Store => { - Self::Store + match &error { + CredentialRepositoryError::InvalidCredentialData(_) => { + Self::InvalidCredential(Some(ErrorSource::new(error))) } + CredentialRepositoryError::RevisionConflict(_) + | CredentialRepositoryError::Store(_) => Self::Store(ErrorSource::new(error)), } } } impl From for CredentialSelectionError { - fn from(_: ProviderStoreError) -> Self { - Self::Coordinator + fn from(error: ProviderStoreError) -> Self { + Self::Coordinator(error) } } impl From for CredentialSelectionError { - fn from(_: super::cookie::CookiePolicyError) -> Self { - Self::CookiePolicy + fn from(error: super::cookie::CookiePolicyError) -> Self { + Self::CookiePolicy(error) } } diff --git a/backend/crates/providers/openai/src/provider/execution.rs b/backend/crates/providers/openai/src/provider/execution.rs index 7edc0038b..d6af23208 100644 --- a/backend/crates/providers/openai/src/provider/execution.rs +++ b/backend/crates/providers/openai/src/provider/execution.rs @@ -290,9 +290,7 @@ impl CodexProvider { client: self .client_for_request(&context)? .for_account(lease.account()) - .map_err(|_| { - provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) - })? + .map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)? .with_authentication(lease.authentication()) .with_middleware_headers(middleware_headers), response_origin: request.response_origin, diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index 5aac75f98..bc2bd85f5 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -5,7 +5,7 @@ use gateway_core::diagnostics::TraceContext; /// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集 /// -/// 已归一的上游 code 优先；code 缺失时才读取结构化客户端错误的 code/type +/// 原始上游 code 优先；code 缺失时才读取结构化客户端错误的 code/type /// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集 /// 容量拒绝影响短期调度健康度，但不证明账号凭据或额度失效 const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[ @@ -273,7 +273,7 @@ pub(super) fn log_client_upstream_error( body, transport, .. - } => log_raw_upstream_body( + } => log_upstream_error_summary( context, *transport, "http_error_response", @@ -283,7 +283,7 @@ pub(super) fn log_client_upstream_error( body, ), CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => { - log_raw_upstream_body( + log_upstream_error_summary( context, CodexBackendTransport::WebSocket, "websocket_opening_response", @@ -303,7 +303,7 @@ pub(super) fn log_client_upstream_error( .map(|connection_id| connection_id.to_string()) .unwrap_or_default(); let close_reason_bytes = close.reason().map_or(0, str::len); - tracing::warn!( + tracing::debug!( request_id = %context.request_id, account_id = %context.account_id, attempt_index = context.attempt_index, @@ -333,7 +333,7 @@ pub(super) fn log_canonical_upstream_error( let CodexCanonicalError::Upstream(failure) = error else { return; }; - log_raw_upstream_body( + log_upstream_error_summary( context, transport, "responses_error_event", @@ -344,7 +344,7 @@ pub(super) fn log_canonical_upstream_error( ); } -pub(super) fn log_raw_upstream_body( +pub(super) fn log_upstream_error_summary( context: UpstreamErrorLogContext<'_>, transport: CodexBackendTransport, error_kind: &'static str, @@ -357,7 +357,7 @@ pub(super) fn log_raw_upstream_body( .websocket_connection_id .map(|connection_id| connection_id.to_string()) .unwrap_or_default(); - tracing::warn!( + tracing::debug!( request_id = %context.request_id, account_id = %context.account_id, attempt_index = context.attempt_index, @@ -366,11 +366,8 @@ pub(super) fn log_raw_upstream_body( upstream_error_kind = error_kind, upstream_status_code = status_code.unwrap_or_default(), upstream_status_code_present = status_code.is_some(), - upstream_error_code = upstream_code.unwrap_or_default(), upstream_error_code_present = upstream_code.is_some(), - upstream_error_type = upstream_type.unwrap_or_default(), upstream_error_type_present = upstream_type.is_some(), - upstream_error_raw, upstream_error_raw_bytes = upstream_error_raw.len(), "OpenAI upstream returned an error payload" ); @@ -568,8 +565,9 @@ pub(super) fn apply_websocket_recovery_policy( websocket_failure_kind = failure.error.kind().as_str(), websocket_failure_code = failure .error - .upstream_code() - .map_or("", OpaqueUpstreamValue::as_str), + .diagnostic() + .and_then(ProviderDiagnostic::code) + .unwrap_or(""), upstream_status_code = failure.error.upstream_status().unwrap_or_default(), upstream_status_code_present = failure.error.upstream_status().is_some(), transport_requirement = context.requirement.as_str(), @@ -598,8 +596,9 @@ pub(super) fn apply_websocket_recovery_policy( websocket_failure_kind = failure.error.kind().as_str(), websocket_failure_code = failure .error - .upstream_code() - .map_or("", OpaqueUpstreamValue::as_str), + .diagnostic() + .and_then(ProviderDiagnostic::code) + .unwrap_or(""), upstream_send_state = ?send_state, transport_requirement = context.requirement.as_str(), continuation_recovery_action = "client_replay_required", @@ -642,8 +641,9 @@ pub(super) fn apply_websocket_recovery_policy( websocket_failure_kind = failure.error.kind().as_str(), websocket_failure_code = failure .error - .upstream_code() - .map_or("", OpaqueUpstreamValue::as_str), + .diagnostic() + .and_then(ProviderDiagnostic::code) + .unwrap_or(""), upstream_status_code = failure.error.upstream_status().unwrap_or_default(), upstream_status_code_present = failure.error.upstream_status().is_some(), websocket_retry_count = retry_index.get(), @@ -669,8 +669,9 @@ pub(super) fn apply_websocket_recovery_policy( websocket_failure_kind = failure.error.kind().as_str(), websocket_failure_code = failure .error - .upstream_code() - .map_or("", OpaqueUpstreamValue::as_str), + .diagnostic() + .and_then(ProviderDiagnostic::code) + .unwrap_or(""), upstream_status_code = failure.error.upstream_status().unwrap_or_default(), upstream_status_code_present = failure.error.upstream_status().is_some(), websocket_retry_count = context.retry_count, @@ -897,7 +898,7 @@ pub(super) fn map_client_error( } let connect_retry = !local_connection_capacity && matches!(&error, CodexClientError::Http(error) if transient_http_connect(error)); - let mut failure = match error { + let mut failure = match &error { CodexClientError::ConnectionBudgetExhausted => MappedProviderFailure::plain( provider_error(ProviderErrorKind::Timeout, UpstreamSendState::NotSent) .with_connection_retry(gateway_core::engine::AttemptTransport::Fallback), @@ -940,8 +941,9 @@ pub(super) fn map_client_error( uncertain_state, ) .with_status(status.as_u16()); - if let Some(request_id) = diagnostics.request_id { - error = error.with_upstream_request_id(OpaqueUpstreamValue::new(request_id)); + if let Some(request_id) = &diagnostics.request_id { + error = + error.with_upstream_request_id(OpaqueUpstreamValue::new(request_id.clone())); } MappedProviderFailure::plain(error) } @@ -964,7 +966,7 @@ pub(super) fn map_client_error( let mut failure = MappedProviderFailure::plain(continuation_replay_required_error( continuation_unavailable_reason.unwrap_or("scope_unavailable"), )); - if let Some(client_visible_error) = websocket_client_visible_error(&error) { + if let Some(client_visible_error) = websocket_client_visible_error(error) { failure.error = failure .error .with_client_visible_upstream_error(client_visible_error); @@ -973,10 +975,10 @@ pub(super) fn map_client_error( } CodexClientError::WebSocket(error) => { let close_code = error.close_before_terminal().and_then(|close| close.code()); - let client_visible_error = websocket_client_visible_error(&error); + let client_visible_error = websocket_client_visible_error(error); let mut failure = MappedProviderFailure::plain(provider_error( - websocket_error_kind(&error), - websocket_send_state(&error), + websocket_error_kind(error), + websocket_send_state(error), )); if let Some(close_code) = close_code { failure.error = @@ -1043,6 +1045,39 @@ pub(super) fn map_client_error( .error .with_connection_observation(connection_observation); } + // Reqwest 的请求 URL 可能携带认证参数；原因链保留，请求方向的 URL 不进入运维详情 + let mut request_url_redacted = false; + let error = match error { + CodexClientError::Http(source) => { + request_url_redacted = source.url().is_some(); + CodexClientError::Http(source.without_url()) + } + CodexClientError::HttpJson(source) => { + request_url_redacted = source.url().is_some(); + CodexClientError::HttpJson(source.without_url()) + } + CodexClientError::ErrorBodyRead { + source, + status, + diagnostics, + transport, + transport_metrics, + } => { + request_url_redacted = source.url().is_some(); + CodexClientError::ErrorBodyRead { + source: source.without_url(), + status, + diagnostics, + transport, + transport_metrics, + } + } + error => error, + }; + failure.error = failure.error.with_source(error); + if request_url_redacted { + failure.error = failure.error.redact_sensitive_context("request URL"); + } failure.observation = observation; failure } @@ -1119,11 +1154,7 @@ fn client_diagnostic(error: &CodexClientError) -> Option { "request_compression_failed", "OpenAI request compression failed".to_owned(), ), - CodexClientError::CustomCa(_) => ( - "prepare", - "custom_ca_invalid", - "OpenAI custom CA configuration is invalid".to_owned(), - ), + CodexClientError::CustomCa(error) => return Some(custom_ca_diagnostic(error)), CodexClientError::ModelCatalog(_) => ( "prepare", "model_catalog_invalid", @@ -1134,6 +1165,63 @@ fn client_diagnostic(error: &CodexClientError) -> Option { Some(ProviderDiagnostic::new(message).with_classification(stage, code)) } +pub(super) fn profile_diagnostic( + error: &crate::transport::profile::selection::ClientProfileError, +) -> ProviderDiagnostic { + use crate::transport::profile::selection::ClientProfileError; + let code = match error { + ClientProfileError::Invalid => "request_profile_invalid", + ClientProfileError::InvalidUserAgent => "request_profile_user_agent_invalid", + ClientProfileError::CompanionHeadersRequired => "request_profile_headers_required", + ClientProfileError::CompanionHeadersConflict => "request_profile_headers_conflict", + ClientProfileError::ReleaseUnavailable => "request_profile_release_unavailable", + }; + // 此类型的 Display 全部由本地固定文案组成，不包含配置字段 + ProviderDiagnostic::new(error.to_string()).with_classification("prepare", code) +} + +fn custom_ca_diagnostic(error: &crate::transport::tls::CustomCaError) -> ProviderDiagnostic { + use crate::transport::tls::CustomCaError; + let (code, message) = match error { + CustomCaError::ProxyConfiguration => ( + "account_proxy_invalid", + "OpenAI account proxy configuration is invalid".to_owned(), + ), + CustomCaError::ReadCaFile { source_env, .. } => ( + "custom_ca_read_failed", + format!("OpenAI CA certificate file selected by {source_env} could not be read"), + ), + CustomCaError::InvalidCaFile { source_env, .. } => ( + "custom_ca_invalid", + format!("OpenAI CA certificate file selected by {source_env} is not valid PEM"), + ), + CustomCaError::RegisterCertificate { + certificate_index, .. + } + | CustomCaError::RegisterRustlsCertificate { + certificate_index, .. + } => ( + "custom_ca_registration_failed", + format!("OpenAI CA certificate #{certificate_index} could not be registered"), + ), + CustomCaError::BuildClientWithCustomCa { .. } => ( + "custom_ca_client_build_failed", + "OpenAI HTTP client with custom CA certificates could not be built".to_owned(), + ), + CustomCaError::BuildClientWithSystemRoots(_) => ( + "system_tls_client_build_failed", + "OpenAI HTTP client with system root certificates could not be built".to_owned(), + ), + CustomCaError::LoadNativeRoots(_) => ( + "system_tls_roots_unavailable", + "OpenAI system root certificates could not be loaded".to_owned(), + ), + }; + ProviderDiagnostic::new(message) + .with_classification("prepare", code) + .with_io_cause(error) +} + fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnostic { let (stage, code) = match error.diagnostic_cause() { CodexWebSocketExchangeError::Connect(_) => ( @@ -1349,7 +1437,7 @@ pub(super) fn map_upstream_failure( .is_some_and(|status| status.is_client_error()) && is_cyber_policy_code(failure.code.as_deref()); let continuation_failure = failure - .persistable_code() + .upstream_code() .filter(|code| is_history_failure_code(code)) .map(|_| ContinuationFailure::HistoryUnavailable); let send_state = upstream_send_state(failure.send_phase); @@ -1414,7 +1502,7 @@ pub(super) fn map_upstream_failure( max_delay, ); } - if let Some(code) = failure.persistable_code() { + if let Some(code) = failure.upstream_code() { error = error.with_upstream_code(OpaqueUpstreamValue::new(code.to_owned())); } if let Some(request_id) = failure.request_id.as_deref() { @@ -1423,13 +1511,10 @@ pub(super) fn map_upstream_failure( let status = error .upstream_status() .map_or_else(|| "none".to_owned(), |status| status.to_string()); - let code = error - .upstream_code() - .map_or_else(|| "none".to_owned(), |code| code.as_str().to_owned()); let kind = error.kind().as_str(); error = error.with_diagnostic( ProviderDiagnostic::new(format!( - "OpenAI upstream failure: kind={}, status={status}, code={code}", + "OpenAI upstream failure: kind={}, status={status}", kind )) .with_classification("upstream", "upstream_rejected"), @@ -1461,13 +1546,14 @@ pub(super) fn is_cyber_policy_code(code: Option<&str>) -> bool { } pub(super) fn is_history_failure_code(code: &str) -> bool { - matches!( - code, - "previous_response_not_found" - | "invalid_encrypted_content" - | "missing_tool_output" - | "no_tool_output" - ) + [ + "previous_response_not_found", + "invalid_encrypted_content", + "missing_tool_output", + "no_tool_output", + ] + .iter() + .any(|known| code.trim().eq_ignore_ascii_case(known)) } pub(super) const fn provider_error_kind(category: CodexFailureCategory) -> ProviderErrorKind { diff --git a/backend/crates/providers/openai/src/provider/live.rs b/backend/crates/providers/openai/src/provider/live.rs index cd9c27265..16017290d 100644 --- a/backend/crates/providers/openai/src/provider/live.rs +++ b/backend/crates/providers/openai/src/provider/live.rs @@ -361,9 +361,7 @@ impl CodexProvider { client: self .client_for_request(&context)? .for_account(lease.account()) - .map_err(|_| { - provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) - })? + .map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)? .with_authentication(lease.authentication()) .with_middleware_headers(middleware_headers), registry: Arc::clone(&self.live_registry), diff --git a/backend/crates/providers/openai/src/provider/mod.rs b/backend/crates/providers/openai/src/provider/mod.rs index 47c9cddd3..5fe833688 100644 --- a/backend/crates/providers/openai/src/provider/mod.rs +++ b/backend/crates/providers/openai/src/provider/mod.rs @@ -236,8 +236,16 @@ impl CodexProvider { return Ok(self.client.clone()); }; let profile = serde_json::from_value(Value::Object(profile.expose_to_provider().clone())) - .map_err(|_| { + .map_err(|error: serde_json::Error| { + let diagnostic = ProviderDiagnostic::new(format!( + "OpenAI request profile decoding failed: {:?}, line={}, column={}", + error.classify(), + error.line(), + error.column(), + )) + .with_classification("prepare", "request_profile_decode_failed"); provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) + .with_diagnostic(diagnostic) })?; Ok(self.client.clone().with_request_profile(profile)) } @@ -332,19 +340,23 @@ impl Provider for CodexProvider { ) -> Result { let selection = crate::transport::profile::identity::RequestProfileSelection::parse(configuration) - .map_err(|_| { + .map_err(|error| { provider_error( ProviderErrorKind::InvalidRequest, UpstreamSendState::NotSent, ) + .with_diagnostic(profile_diagnostic(&error)) })?; let profile = selection .resolve(self.client.profile_state()) - .map_err(|_| { + .map_err(|error| { provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) + .with_diagnostic(profile_diagnostic(&error)) })?; - crate::transport::profile::selection::object(&profile) - .map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent)) + crate::transport::profile::selection::object(&profile).map_err(|error| { + provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent) + .with_diagnostic(profile_diagnostic(&error)) + }) } fn name(&self) -> &'static str { @@ -984,9 +996,7 @@ impl CodexProvider { client: self .client_for_request(&context)? .for_account(lease.account()) - .map_err(|_| { - provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent) - })? + .map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)? .with_authentication(lease.authentication()) .with_connection_budget(context.connection_budget().clone()) .with_response_control(context.response_control().cloned()) diff --git a/backend/crates/providers/openai/src/provider/observation.rs b/backend/crates/providers/openai/src/provider/observation.rs index f87fe82a3..11a92e1d0 100644 --- a/backend/crates/providers/openai/src/provider/observation.rs +++ b/backend/crates/providers/openai/src/provider/observation.rs @@ -769,10 +769,26 @@ pub(super) fn map_request_error(error: CodexRequestEncodeError) -> ProviderError } pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderError { - match error { + selection_failure(&error).with_source(error) +} + +fn selection_failure(error: &CredentialSelectionError) -> ProviderError { + let diagnostic_code = match error { + CredentialSelectionError::InvalidCredential(_) => Some("account_credential_invalid"), + CredentialSelectionError::Store(_) => Some("account_store_unavailable"), + CredentialSelectionError::Coordinator(_) => Some("account_coordinator_unavailable"), + CredentialSelectionError::CookiePolicy(_) => Some("account_cookie_policy_rejected"), + CredentialSelectionError::PolicyUnavailable => Some("account_policy_unavailable"), + _ => None, + }; + // 这些变体只含固定文案，不格式化可能承载外部正文的其他错误 + let diagnostic = diagnostic_code.map(|code| { + ProviderDiagnostic::new(error.to_string()).with_classification("account_selection", code) + }); + let mapped = match error { CredentialSelectionError::Cancelled => provider_error(ProviderErrorKind::Cancelled, UpstreamSendState::NotSent).with_retry_prohibited(), CredentialSelectionError::ContinuationOwnerChanged => continuation_replay_required_error("scope_unavailable"), - CredentialSelectionError::SessionBound(error) => map_selection_error(*error).with_retry_prohibited(), + CredentialSelectionError::SessionBound(error) => selection_failure(error).with_retry_prohibited(), CredentialSelectionError::QueueRejected(error) => { provider_error(error.provider_kind(), UpstreamSendState::NotSent) } @@ -782,7 +798,7 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr UpstreamSendState::NotSent, ); match retry_after { - Some(retry) => error.with_retry_after(retry), + Some(retry) => error.with_retry_after(*retry), None => error, } } @@ -807,10 +823,10 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr ProviderDiagnostic::new("OpenAI account changed repeatedly during selection") .with_classification("account_selection", "account_snapshot_conflict"), ), - CredentialSelectionError::InvalidCredential - | CredentialSelectionError::Store - | CredentialSelectionError::Coordinator - | CredentialSelectionError::CookiePolicy => provider_error( + CredentialSelectionError::InvalidCredential(_) + | CredentialSelectionError::Store(_) + | CredentialSelectionError::Coordinator(_) + | CredentialSelectionError::CookiePolicy(_) => provider_error( ProviderErrorKind::ProviderInfrastructureUnavailable, UpstreamSendState::NotSent, ), @@ -822,5 +838,9 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr ProviderErrorKind::Unavailable, UpstreamSendState::NotSent, ), + }; + match diagnostic { + Some(diagnostic) => mapped.with_diagnostic(diagnostic), + None => mapped, } } diff --git a/backend/crates/providers/openai/src/transport/diagnostics.rs b/backend/crates/providers/openai/src/transport/diagnostics.rs index 49954179e..f19f27594 100644 --- a/backend/crates/providers/openai/src/transport/diagnostics.rs +++ b/backend/crates/providers/openai/src/transport/diagnostics.rs @@ -25,57 +25,6 @@ const UPSTREAM_TRACE_HEADERS: &[&str] = &[ ]; const IDENTITY_AUTHORIZATION_ERROR_HEADER: &str = "x-openai-authorization-error"; const IDENTITY_ERROR_JSON_HEADER: &str = "x-error-json"; -const PERSISTABLE_UPSTREAM_CODES: &[&str] = &[ - "access_token_expired", - "account_banned", - "account_deactivated", - "account_disabled", - "account_suspended", - "authentication_error", - "billing_limit", - "cyber_policy", - "deactivated_workspace", - "flex_unavailable", - "identity_verification_required", - "insufficient_quota", - "invalid_api_key", - "invalid_encrypted_content", - "invalid_prompt", - "invalid_request", - "missing_tool_output", - "model_not_available", - "model_not_supported", - "no_tool_output", - "organization_disabled", - "payment_required", - "permission_denied", - "previous_response_not_found", - "quota_exceeded", - "quota_exhausted", - "rate_limit_error", - "rate_limit_exceeded", - "rate_limit_reached", - "refresh_token_invalidated", - "server_error", - "server_is_overloaded", - "slow_down", - "token_expired", - "token_invalid", - "token_invalidated", - "token_revoked", - "unauthorized", - "unsupported", - "unsupported_feature", - "usage_limit_reached", - "verification_required", - "websocket_connection_limit_reached", - "workspace_deactivated", - "workspace_member_credits_depleted", - "workspace_member_usage_limit_reached", - "workspace_owner_credits_depleted", - "workspace_owner_usage_limit_reached", -]; - /// 上游拒绝相对业务 payload 的发送阶段 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CodexUpstreamSendPhase { @@ -256,15 +205,8 @@ impl CodexUpstreamFailure { } } - pub(crate) fn persistable_code(&self) -> Option<&'static str> { - self.code - .as_deref() - .and_then(persistable_upstream_code) - .or_else(|| { - self.identity_error_code - .as_deref() - .and_then(persistable_upstream_code) - }) + pub(crate) fn upstream_code(&self) -> Option<&str> { + self.code.as_deref().or(self.identity_error_code.as_deref()) } } @@ -303,7 +245,10 @@ impl ParsedUpstreamError { .or_else(|| value.get("message")) .and_then(Value::as_str) .and_then(non_empty_owned); - let code = code_value.and_then(Value::as_str).and_then(non_empty_owned); + let code = code_value + .and_then(Value::as_str) + .filter(|code| !code.trim().is_empty()) + .map(str::to_owned); let message = client_message .clone() .or_else(|| error.as_str().and_then(non_empty_owned)) @@ -417,8 +362,7 @@ fn decode_identity_error_code(encoded: &str) -> Option { .ok()? .pointer("/error/code")? .as_str() - .map(str::trim) - .filter(|code| !code.is_empty()) + .filter(|code| !code.trim().is_empty()) .map(ToString::to_string) } @@ -571,13 +515,6 @@ fn non_empty_owned(value: &str) -> Option { (!value.is_empty()).then(|| value.to_owned()) } -fn persistable_upstream_code(value: &str) -> Option<&'static str> { - PERSISTABLE_UPSTREAM_CODES - .iter() - .copied() - .find(|known| value.trim().eq_ignore_ascii_case(known)) -} - fn is_model_unsupported(value: &str) -> bool { value.contains("model_not_supported") || value.contains("model_not_available") diff --git a/backend/crates/providers/openai/src/transport/websocket/pump.rs b/backend/crates/providers/openai/src/transport/websocket/pump.rs index 1332e7a21..8b3b031e2 100644 --- a/backend/crates/providers/openai/src/transport/websocket/pump.rs +++ b/backend/crates/providers/openai/src/transport/websocket/pump.rs @@ -710,7 +710,7 @@ fn log_pump_exit( observation: &WebSocketConnectionObservation, backpressure_events: u64, ) { - let detail = reason.detail().unwrap_or_default(); + let detail = reason.metric_reason(); let account_id = context.account_id.as_deref().unwrap_or("unknown"); let conversation_id_hash = context.conversation_id_hash.as_deref().unwrap_or("unknown"); if reason.is_unexpected() { @@ -728,9 +728,6 @@ fn log_pump_exit( ); } else if matches!(reason, PumpExitReason::UpstreamCloseFrame { .. }) { let upstream_close = reason.upstream_close(); - let upstream_error_raw = upstream_close - .and_then(CodexWebSocketCloseError::reason) - .unwrap_or_default(); tracing::info!( websocket_connection_id = %connection_id, account_id = %account_id, @@ -741,7 +738,7 @@ fn log_pump_exit( connection_idle_ms = observation.idle_ms(), pump_exit_detail = detail, upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code), - upstream_error_raw, + upstream_close_reason_bytes = upstream_close.and_then(CodexWebSocketCloseError::reason).map_or(0, str::len), upstream_error_raw_present = upstream_close.is_some_and(|close| close.reason().is_some()), backpressure_events, "Responses WebSocket pump received close frame" diff --git a/backend/crates/providers/openai/tests/credential/contract/mod.rs b/backend/crates/providers/openai/tests/credential/contract/mod.rs index 44f27cb04..28bfabc25 100644 --- a/backend/crates/providers/openai/tests/credential/contract/mod.rs +++ b/backend/crates/providers/openai/tests/credential/contract/mod.rs @@ -674,7 +674,7 @@ fn selector_does_not_retry_credential_store_unavailability() { })) .expect_err("unavailability is not an account snapshot conflict"); - assert!(matches!(error, CredentialSelectionError::Store)); + assert!(matches!(error, CredentialSelectionError::Store(_))); assert_eq!(store.credential_loads(), 1); assert!(leases.requests.lock().expect("lease requests").is_empty()); } @@ -704,7 +704,10 @@ fn selector_does_not_retry_invalid_credential_data() { })) .expect_err("invalid data is not an account snapshot conflict"); - assert!(matches!(error, CredentialSelectionError::InvalidCredential)); + assert!(matches!( + error, + CredentialSelectionError::InvalidCredential(_) + )); assert_eq!(store.credential_loads(), 1); assert!(leases.requests.lock().expect("lease requests").is_empty()); } diff --git a/backend/crates/providers/openai/tests/provider/contract/error_details.rs b/backend/crates/providers/openai/tests/provider/contract/error_details.rs new file mode 100644 index 000000000..d9f23538a --- /dev/null +++ b/backend/crates/providers/openai/tests/provider/contract/error_details.rs @@ -0,0 +1,148 @@ +//! 验证原始上游错误码独立于本地分类、诊断摘要和重试策略 + +use super::*; + +#[tokio::test] +async fn network_failure_preserves_native_causes_without_the_request_url() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let provider = provider_with_base_url_and_retry_budget( + &store, + format!("http://{address}/PRIVATE_REQUEST_PATH"), + 0, + ); + let mut stream = provider + .execute( + planned_request("openai", http_generate_operation()), + context("req_native_cause", CancellationToken::new()), + ) + .await + .unwrap(); + let error = loop { + match stream.next().await { + Some(Ok(_)) => {} + Some(Err(error)) => break error, + None => panic!("connection failure must surface"), + } + }; + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + let details = error.error_details().unwrap(); + assert!(!details.contains("PRIVATE_REQUEST_PATH")); + let details: Value = serde_json::from_str(&details).unwrap(); + assert_eq!(details["redacted"], true); + let mut source = std::error::Error::source(&error); + let mut saw_native = false; + while let Some(cause) = source { + if let Some(http) = cause.downcast_ref::() { + assert!(http.url().is_none()); + assert!(http.is_connect()); + } + if let Some(io) = cause.downcast_ref::() { + saw_native = io.kind() == std::io::ErrorKind::ConnectionRefused; + } + source = cause.source(); + } + assert!( + saw_native, + "the original I/O cause must survive Provider mapping" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn upstream_codes_survive_http_and_websocket_opening_without_normalization() { + let captured = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .json() + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::DEBUG) + .with_writer(captured.clone()) + .finish(); + let _subscriber = tracing::subscriber::set_default(subscriber); + for use_websocket in [false, true] { + for (body_code, header_code, expected_kind) in [ + ( + Some("Vendor.Future.v2"), + None, + ProviderErrorKind::Unavailable, + ), + (Some(" SERVER_ERROR "), None, ProviderErrorKind::Unavailable), + ( + None, + Some("Vendor.Identity.v2"), + ProviderErrorKind::Unavailable, + ), + ( + Some("PrEvIoUs_ReSpOnSe_NoT_FoUnD"), + None, + ProviderErrorKind::ContinuationRecoveryRequired, + ), + ] { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + let server = MockServer::start().await; + let body = json!({"error": {"code": body_code, "message": "PRIVATE_UPSTREAM_DETAIL"}}); + let mut response = ResponseTemplate::new(503).set_body_json(&body); + if let Some(code) = header_code { + response = response.insert_header( + "x-error-json", + STANDARD.encode(json!({"error": {"code": code}}).to_string()), + ); + } + Mock::given(method(if use_websocket { "GET" } else { "POST" })) + .and(path("/codex/responses")) + .respond_with(response) + .mount(&server) + .await; + let operation = if use_websocket { + capacity_websocket_operation() + } else { + http_generate_operation() + }; + let mut stream = provider_with_base_url_and_retry_budget(&store, server.uri(), 0) + .execute( + planned_request("openai", operation), + context("req_original_error_code", CancellationToken::new()), + ) + .await + .expect("prepare stream"); + let error = loop { + match stream.next().await { + Some(Ok(_)) => {} + Some(Err(error)) => break error, + None => panic!("upstream rejection must surface"), + } + }; + let original_code = body_code.or(header_code).unwrap(); + assert_eq!( + error.upstream_code().map(|code| code.as_str()), + Some(original_code) + ); + assert_eq!(error.kind(), expected_kind); + assert_eq!( + serde_json::from_str::(error.raw_upstream_error().unwrap().as_str()) + .unwrap(), + body, + ); + let diagnostic = error.diagnostic().expect("safe diagnostic"); + assert!(!diagnostic.as_str().contains(original_code)); + assert!(!diagnostic.as_str().contains("PRIVATE_UPSTREAM_DETAIL")); + assert!(!format!("{error:?} {error}").contains(original_code)); + } + } + let events = captured.json_events(); + assert!(events.iter().any(|event| { + event["fields"]["message"] == "OpenAI upstream returned an error payload" + })); + let logs = serde_json::to_string(&events).unwrap(); + for private in [ + "Vendor.Future.v2", + "Vendor.Identity.v2", + "PRIVATE_UPSTREAM_DETAIL", + ] { + assert!(!logs.contains(private), "ordinary logs leaked {private}"); + } +} diff --git a/backend/crates/providers/openai/tests/provider/contract/live.rs b/backend/crates/providers/openai/tests/provider/contract/live.rs deleted file mode 100644 index 8ac21819c..000000000 --- a/backend/crates/providers/openai/tests/provider/contract/live.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! 显式真实请求测试的凭据导入，只创建内存账号，不修改本地导出 - -use super::*; - -pub(super) async fn imported_account() -> (Arc, Value) { - use provider_openai::credential::{CodexAccountProfile, CodexOAuthSecret}; - use secrecy::SecretString; - - let file = std::env::var("CPR_LIVE_ACCOUNTS_FILE").expect("credential file path required"); - let document: Value = - serde_json::from_slice(&std::fs::read(file).expect("read credential file")) - .expect("parse credential export"); - let account = document["documents"][0]["document"]["accounts"][0].clone(); - let field = |key| account.get(key).and_then(Value::as_str).map(str::to_owned); - let store = Arc::new(MemoryAccountStore::default()); - store - .seed_oauth_credential(ImportCodexOAuthCredential { - account_id: "acct_provider_contract".into(), - name: "local-live-validation".into(), - enabled: true, - secret: CodexOAuthSecret { - access_token: SecretString::from( - field("accessToken").expect("access token required"), - ), - refresh_token: field("refreshToken").map(SecretString::from), - id_token: field("idToken").map(SecretString::from), - }, - verified_account: CodexAccountProfile { - email: None, - oauth_subject: field("userId").expect("user identity required"), - poid: None, - chatgpt_account_id: field("accountId").expect("account identity required"), - chatgpt_user_id: field("userId").expect("user identity required"), - plan_type: field("planType"), - access_token_expires_at: field("accessTokenExpiresAt") - .and_then(|date| chrono::DateTime::parse_from_rfc3339(&date).ok()) - .map(|date| date.with_timezone(&Utc)), - }, - next_refresh_at: Some(Utc::now() + chrono::Duration::hours(1)), - }) - .await; - (store, account) -} - -pub(super) async fn model(store: &Arc) -> String { - let catalog = CodexCredentialCatalogService::new( - store.repository(), - wire_profile(), - reqwest::Client::builder().no_proxy().build().unwrap(), - OFFICIAL_CODEX_BASE_URL.to_owned(), - catalog_cache(), - ); - let models = catalog - .refresh_account_catalog(&ProviderAccountId::new("acct_provider_contract").unwrap()) - .await - .expect("load supported models"); - std::env::var("CPR_LIVE_MODEL").unwrap_or_else(|_| { - models - .models() - .first() - .expect("available text model") - .clone() - }) -} diff --git a/backend/crates/providers/openai/tests/provider/contract/mod.rs b/backend/crates/providers/openai/tests/provider/contract/mod.rs index 1f2628212..38a3087cd 100644 --- a/backend/crates/providers/openai/tests/provider/contract/mod.rs +++ b/backend/crates/providers/openai/tests/provider/contract/mod.rs @@ -2,7 +2,7 @@ mod account_isolation; mod capacity; -mod live; +mod error_details; mod precommit; mod response_interrupt; mod session_binding; @@ -84,6 +84,88 @@ use crate::support::{ }; use crate::transport::accept_codex_test_websocket; +#[tokio::test] +async fn account_client_preparation_keeps_ca_read_cause_without_exposing_path() { + const CHILD: &str = "CPR_TEST_CA_PREPARATION_FAILURE"; + if std::env::var_os(CHILD).is_none() { + let directory = tempfile::tempdir().unwrap(); + // 环境变量只影响隔离子进程，不能改变并发 TLS 测试的配置 + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", std::thread::current().name().unwrap()]) + .env(CHILD, "1") + .env( + provider_openai::transport::tls::CODEX_CA_CERT_ENV, + directory.path().join("PRIVATE_MISSING_CA.pem"), + ) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_provider_contract").await; + store.set_egress( + "acct_provider_contract", + Some(gateway_core::account::OutboundProxy::parse("http://127.0.0.1:9").unwrap()), + None, + ); + let error = provider_with_base_url(&store, "http://127.0.0.1:9".to_owned()) + .execute( + planned_request("openai", generate_operation()), + fallback_transport_context("req_ca_prepare"), + ) + .await + .err() + .expect("CA failure must occur before returning a stream"); + assert_eq!(error.kind(), ProviderErrorKind::Protocol); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + let snapshot = error.stable_snapshot(); + let diagnostic = snapshot.diagnostic().unwrap(); + assert_eq!(diagnostic.stage(), Some("prepare")); + assert_eq!(diagnostic.code(), Some("io_not_found")); + assert!(diagnostic.as_str().contains("CODEX_CA_CERTIFICATE")); + assert!(diagnostic.as_str().contains("OS cause:")); + assert!(!diagnostic.as_str().contains("PRIVATE_")); + let gateway = gateway_core::error::GatewayError::from_provider(&snapshot); + assert_eq!(gateway.safe_message(), "upstream service is unavailable"); + assert_eq!(gateway.diagnostic(), Some(diagnostic)); +} + +#[tokio::test] +async fn invalid_request_profile_preserves_diagnostic_before_any_upstream_send() { + let store = Arc::new(MemoryAccountStore::default()); + let provider = provider_with_base_url(&store, "http://127.0.0.1:1".to_owned()); + let configuration = OpaqueProviderData::new( + json!({ + "mode": "custom", "userAgent": "PRIVATE_USER_AGENT\n", + }) + .as_object() + .unwrap() + .clone(), + ); + let error = provider + .resolve_request_profile(&configuration) + .unwrap_err(); + assert_eq!(error.kind(), ProviderErrorKind::InvalidRequest); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + let snapshot = error.stable_snapshot(); + let diagnostic = snapshot.diagnostic().unwrap(); + assert_eq!(diagnostic.stage(), Some("prepare")); + assert_eq!( + diagnostic.code(), + Some("request_profile_user_agent_invalid") + ); + assert!(!diagnostic.as_str().contains("PRIVATE_")); + let gateway = gateway_core::error::GatewayError::from_provider(&snapshot); + assert_eq!(gateway.diagnostic(), Some(diagnostic)); +} + #[tokio::test] async fn native_openai_translates_a_non_native_source_before_encoding() { let store = Arc::new(MemoryAccountStore::default()); @@ -3594,6 +3676,22 @@ async fn selection_infrastructure_errors_have_a_distinct_classification() { UpstreamSendState::NotSent, ) ); + let diagnostic = error.diagnostic().unwrap(); + assert_eq!(diagnostic.stage(), Some("account_selection")); + assert_eq!(diagnostic.code(), Some("account_store_unavailable")); + assert_eq!(diagnostic.as_str(), "Codex account store is unavailable"); + use std::error::Error as _; + let snapshot = error.stable_snapshot(); + let mut cause = snapshot.source().expect("selection source"); + while let Some(source) = cause.source() { + cause = source; + } + let original = cause + .downcast_ref::() + .expect("original database cause"); + assert_eq!(original.kind(), std::io::ErrorKind::ConnectionRefused); + assert_eq!(original.to_string(), "PRIVATE_DATABASE_CAUSE"); + assert!(!format!("{error:?} {error}").contains("PRIVATE_DATABASE_CAUSE")); } #[tokio::test] diff --git a/backend/crates/providers/openai/tests/provider/contract/session_binding.rs b/backend/crates/providers/openai/tests/provider/contract/session_binding.rs index 7b0d30af3..e162b7ee0 100644 --- a/backend/crates/providers/openai/tests/provider/contract/session_binding.rs +++ b/backend/crates/providers/openai/tests/provider/contract/session_binding.rs @@ -168,121 +168,6 @@ async fn unknown_image_turn_does_not_infer_a_conversation() { assert_eq!(affinity.binding_count(), 0); } -/// 显式运行才读取本地导出并访问官方服务；只输出状态与测试账号别名 -#[tokio::test] -#[ignore = "requires CPR_LIVE_ACCOUNTS_FILE and sends real requests"] -async fn real_conversation_requests_keep_the_imported_account() { - let (store, account) = super::live::imported_account().await; - let field = |key| account.get(key).and_then(Value::as_str).map(str::to_owned); - let model = super::live::model(&store).await; - let affinity = Arc::new(MemorySessionAffinity::default()); - let provider = provider_with_affinity(&store, affinity.clone()); - let session = uuid::Uuid::new_v4().to_string(); - let turn = uuid::Uuid::new_v4().to_string(); - let response = |thread: &str| { - let request_turn = if thread == session { - format!("root-{turn}") - } else { - turn.clone() - }; - Operation::Generate(GenerateRequest::from_protocol_payload( - ProtocolPayload::json_object("openai", json!({ - "model":model, "instructions":"Reply with OK only.", "stream":true, "store":false, - "input":[{"role":"user","content":[{"type":"input_text","text":"Reply with OK."}]}], - "client_metadata":{"x-codex-turn-metadata":json!({"session_id":session,"thread_id":thread,"turn_id":request_turn}).to_string()} - }).as_object().unwrap().clone()).unwrap().with_context(Map::from_iter([ - ("use_websocket".into(), json!(false)), ("session_id".into(), json!("different-cache-routing-key")) - ])), - )) - }; - let search = Operation::Search(StandaloneSearchRequest::from_raw_json(RawJsonPayload::new("openai", Bytes::from(json!({ - "id":session, "model":model, "commands":{"search_query":[{"q":"site:openai.com ChatGPT"}]}, "max_output_tokens":128 - }).to_string())).unwrap())); - let image = Operation::GenerateImage(ImageRequest::from_raw_json(ImageRequestKind::Generation, RawJsonPayload::new("openai", Bytes::from(json!({ - "prompt":"A plain blue square on a white background.", "model":"gpt-image-2", "n":1, "quality":"low", "size":"1024x1024" - }).to_string())).unwrap().with_context(Map::from_iter([("image_turn_id".into(), json!(turn))])))); - let summarize_error = |stage: &str, error: gateway_core::error::ProviderError| { - let message = error.client_visible_upstream_error().map(|upstream| { - let mut message = upstream.message().to_owned(); - for key in [ - "accessToken", - "refreshToken", - "idToken", - "email", - "accountId", - "userId", - ] { - if let Some(value) = field(key).filter(|value| !value.is_empty()) { - message = message.replace(&value, "[redacted]"); - } - } - message.chars().take(256).collect::() - }); - json!({"stage":stage,"kind":format!("{:?}",error.kind()),"status":error.upstream_status(),"code":error.client_visible_upstream_error().and_then(|error| error.code()),"message":message}) - }; - let mut results = Vec::new(); - for (name, operation) in [ - ("root", response(&session)), - ("child", response("child-thread")), - ("search", search), - ("image-turn-alias", image), - ] { - let request = if matches!(operation, Operation::Generate(_)) { - planned_request_for_model("openai", operation, &model) - } else { - planned_provider_endpoint_request("openai", operation) - }; - let run = async { - let mut stream = provider - .clone() - .execute( - request, - context("req_live_binding_probe", CancellationToken::new()), - ) - .await - .map_err(|error| summarize_error("selection", error))?; - assert_eq!( - stream.metadata().provider_account_id().as_str(), - "acct_provider_contract", - "never send decoy credentials upstream" - ); - let mut events = 0; - while let Some(event) = stream.next().await { - event.map_err(|error| summarize_error("upstream", error))?; - events += 1; - } - Ok::<_, Value>(events) - }; - let result = match timeout(Duration::from_secs(90), run).await { - Ok(Ok(events)) => { - json!({"case":name,"account":"imported-account","ok":true,"events":events}) - } - Ok(Err(error)) => { - json!({"case":name,"account":"imported-account","ok":false,"error":error}) - } - Err(_) => { - json!({"case":name,"account":"imported-account","ok":false,"error":"timeout"}) - } - }; - eprintln!("LIVE_BINDING {result}"); - results.push(result); - if name == "root" { - // 高权重候选验证后续请求没有重新按权重选号，发送前断言阻止误用占位凭据 - create_account(&store, "acct_subagent_b").await; - store.set_scheduling("acct_subagent_b", None, AccountWeight::new(100).unwrap()); - } - } - assert_eq!(affinity.binding_count(), 1); - if let Ok(path) = std::env::var("CPR_LIVE_REPORT") { - std::fs::write(path, serde_json::to_vec_pretty(&results).unwrap()) - .expect("write sanitized report"); - } - assert!( - results.iter().all(|result| result["ok"] == true), - "some real upstream requests failed; see sanitized status report" - ); -} - #[tokio::test] async fn child_waits_for_first_root_claim_and_cancellation_releases_waiter() { let store = Arc::new(MemoryAccountStore::default()); diff --git a/backend/crates/providers/openai/tests/provider/contract/timing.rs b/backend/crates/providers/openai/tests/provider/contract/timing.rs index 1cde4effa..cf8bab362 100644 --- a/backend/crates/providers/openai/tests/provider/contract/timing.rs +++ b/backend/crates/providers/openai/tests/provider/contract/timing.rs @@ -3,7 +3,6 @@ use std::time::Instant; use gateway_core::engine::provider::ProviderStream; -use gateway_core::error::ProviderError; use gateway_core::event::ProviderResponseTimings; use super::*; @@ -174,107 +173,3 @@ async fn structural_frames_wait_for_semantic_output_on_http_and_websocket() { } } } - -/// 只发送短文本请求，输出请求级耗时与用量，不记录凭据、账号身份或响应正文 -#[tokio::test] -#[ignore = "requires CPR_LIVE_ACCOUNTS_FILE and sends real requests"] -async fn real_http_and_websocket_requests_report_output_timings() { - let (store, _) = super::live::imported_account().await; - let provider = provider_with_base_url(&store, OFFICIAL_CODEX_BASE_URL.to_owned()); - let model = super::live::model(&store).await; - for websocket in [false, true] { - let expected_transport = if websocket { "websocket" } else { "http_sse" }; - let mut protocol_context = Map::from_iter([("use_websocket".into(), json!(websocket))]); - if websocket { - // 模拟下游 WebSocket 新链，避免快路径预算到期后自动回退 HTTP - protocol_context.insert( - "downstream_websocket_connection_id".into(), - json!("ws_live_timing"), - ); - } - let mut generate = GenerateRequest::from_protocol_payload( - ProtocolPayload::json_object( - "openai", - json!({ - "model": model, - "instructions": "Reply with the numbers 1 through 30, separated by spaces, and nothing else.", - "stream": true, - "store": false, - "input": [{"role": "user", "content": [{"type": "input_text", "text": "Count now."}]}] - }) - .as_object() - .unwrap() - .clone(), - ) - .unwrap() - .with_context(protocol_context), - ); - if websocket { - // 测试 Provider 未装配持久会话标识，显式提供本地池键，不声明上游续接 ID - generate = generate.with_provider_session_state( - ProviderSessionState::new( - "openai", - Map::from_iter([ - ("account_id".into(), json!("acct_provider_contract")), - ("conversation_id".into(), json!("live_timing_conversation")), - ("continuation_scope".into(), json!("persisted")), - ]), - ) - .unwrap(), - ); - } - let operation = Operation::Generate(generate); - let started_at = Instant::now(); - let run = async { - let mut stream = provider - .clone() - .execute( - planned_request_for_model("openai", operation, &model), - timed_context(started_at, 1), - ) - .await?; - let mut timings = ProviderResponseTimings::default(); - let mut output_tokens = None; - let mut completed = false; - while let Some(event) = stream.next().await { - let event = event?; - if let Some(observation) = event.response_observation() { - assert_eq!(observation.transport().as_str(), expected_transport); - timings = observation.timings(); - } - for fact in event.canonical_facts() { - match fact { - GatewayEvent::Usage(usage) => output_tokens = usage.output_tokens, - GatewayEvent::Completed(_) => completed = true, - _ => {} - } - } - } - Ok::<_, ProviderError>((timings, output_tokens, completed)) - }; - let (timings, output_tokens, completed) = timeout(Duration::from_secs(35), run) - .await - .expect("live request deadline") - .unwrap_or_else(|error| { - panic!( - "live request failed: kind={:?}, status={:?}, code={:?}", - error.kind(), - error.upstream_status(), - error - .client_visible_upstream_error() - .and_then(|error| error.code()) - ); - }); - let total_ms = started_at.elapsed().as_millis(); - let first_event_ms = timings.first_event_ms.expect("first upstream packet"); - let first_token_ms = timings.first_token_ms.expect("first semantic output"); - let first_text_ms = timings.first_text_ms.expect("first text output"); - let output_tokens = output_tokens.expect("upstream output usage"); - assert!(completed && output_tokens > 0); - assert!(first_event_ms <= first_token_ms && first_token_ms <= first_text_ms); - assert!(u128::from(first_text_ms) <= total_ms); - eprintln!( - "LIVE_TIMING model={model} transport={expected_transport} first_event_ms={first_event_ms} first_token_ms={first_token_ms} first_text_ms={first_text_ms} total_ms={total_ms} output_tokens={output_tokens}" - ); - } -} diff --git a/backend/crates/providers/openai/tests/support.rs b/backend/crates/providers/openai/tests/support.rs index 5da82c42b..7512351f3 100644 --- a/backend/crates/providers/openai/tests/support.rs +++ b/backend/crates/providers/openai/tests/support.rs @@ -259,7 +259,13 @@ impl ProviderAccountStore for MemoryAccountStore { provider: &ProviderKind, ) -> Result, StoreError> { if self.fail_provider_listing.load(Ordering::SeqCst) { - return Err(store_error(StoreErrorKind::Unavailable)); + return Err(StoreError::caused_by( + StoreErrorKind::Unavailable, + std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + "PRIVATE_DATABASE_CAUSE", + ), + )); } // 与 Postgres 实现的调度列表语义一致：停用账号不进入常规候选 Ok(self diff --git a/backend/crates/providers/xai/src/provider/failure.rs b/backend/crates/providers/xai/src/provider/failure.rs index 18be7638c..b6a9cf707 100644 --- a/backend/crates/providers/xai/src/provider/failure.rs +++ b/backend/crates/providers/xai/src/provider/failure.rs @@ -6,12 +6,13 @@ use gateway_core::error::ProviderDiagnostic; pub(super) fn is_invalid_encrypted_content_failure(error: &GrokInferenceTransportError) -> bool { error.kind() == GrokInferenceTransportErrorKind::InvalidRequest && error.status() == Some(400) - && error.upstream_code().is_some_and(|code| { - matches!( - code.as_str(), - REASONING_DECODE_FAILED_CODE | "invalid_encrypted_content" - ) - }) + && (error.diagnostic().and_then(ProviderDiagnostic::code) + == Some(REASONING_DECODE_FAILED_CODE) + || error.upstream_code().is_some_and(|code| { + [REASONING_DECODE_FAILED_CODE, "invalid_encrypted_content"] + .iter() + .any(|known| code.as_str().trim().eq_ignore_ascii_case(known)) + })) } pub(super) enum InferenceBoundary { @@ -279,15 +280,20 @@ pub(super) fn transport_error_contains_any( let code = error .upstream_code() .map(|code| code.as_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + let classification = error + .diagnostic() + .and_then(ProviderDiagnostic::code) .unwrap_or_default(); let message = error .client_visible_upstream_error() .map(ClientVisibleUpstreamError::message) .unwrap_or_default() .to_ascii_lowercase(); - signals - .iter() - .any(|signal| code.contains(signal) || message.contains(signal)) + signals.iter().any(|signal| { + classification.contains(signal) || code.contains(signal) || message.contains(signal) + }) } pub(super) async fn map_and_record_stream_transport_failure( @@ -313,15 +319,21 @@ pub(super) fn map_continuation_failure( let is_reasoning_decode_failure = context.continuation_attempt() == ContinuationAttempt::Native && error.kind() == ProviderErrorKind::InvalidRequest && error.upstream_status() == Some(400) - && error - .upstream_code() - .is_some_and(|code| code.as_str() == REASONING_DECODE_FAILED_CODE); + && (error.diagnostic().and_then(ProviderDiagnostic::code) + == Some(REASONING_DECODE_FAILED_CODE) + || error.upstream_code().is_some_and(|code| { + code.as_str() + .trim() + .eq_ignore_ascii_case(REASONING_DECODE_FAILED_CODE) + })); let is_missing_native_response = context.continuation_attempt() == ContinuationAttempt::Native && error.kind() == ProviderErrorKind::InvalidRequest && error.upstream_status() == Some(404) - && error - .upstream_code() - .is_some_and(|code| code.as_str() == RESPONSE_NOT_FOUND_CODE); + && error.upstream_code().is_some_and(|code| { + code.as_str() + .trim() + .eq_ignore_ascii_case(RESPONSE_NOT_FOUND_CODE) + }); if is_reasoning_decode_failure || is_missing_native_response { error .with_continuation_failure(ContinuationFailure::HistoryUnavailable) @@ -519,6 +531,9 @@ pub(super) fn map_transport_error_with_state( if let Some(code) = error.upstream_code().cloned() { mapped = mapped.with_upstream_code(code); } + if let Some(raw) = error.raw_upstream_error().cloned() { + mapped = mapped.with_raw_upstream_error(raw); + } if let Some(detail) = error.client_visible_upstream_error().cloned() { mapped = mapped.with_client_visible_upstream_error(detail); } @@ -539,6 +554,9 @@ pub(super) fn map_transport_error_with_state( if error.sensitive_context_was_redacted() { mapped = mapped.redact_sensitive_context("upstream transport context"); } + if std::error::Error::source(&error).is_some() { + mapped = mapped.with_source(error); + } mapped } diff --git a/backend/crates/providers/xai/src/transport/canonical.rs b/backend/crates/providers/xai/src/transport/canonical.rs index 8a4a6aaa1..2fa90dc26 100644 --- a/backend/crates/providers/xai/src/transport/canonical.rs +++ b/backend/crates/providers/xai/src/transport/canonical.rs @@ -1246,8 +1246,9 @@ fn upstream_event_error(value: &Value) -> ProviderError { _ => ProviderErrorKind::Unavailable, } }; - let mut error = ProviderError::new(kind, UpstreamSendState::Sent) - .redact_sensitive_context("upstream event"); + let mut error = ProviderError::new(kind, UpstreamSendState::Sent).with_raw_upstream_error( + gateway_core::error::RawUpstreamError::new(value.to_string()), + ); if let Some(code) = code { error = error.with_upstream_code(OpaqueUpstreamValue::new(code.to_owned())); } @@ -1271,8 +1272,8 @@ fn upstream_error_field<'a>(value: &'a Value, field: &str) -> Option<&'a str> { .and_then(Value::as_str) } -fn protocol_error(_error: impl std::fmt::Debug) -> ProviderError { - protocol_error_marker() +fn protocol_error(error: impl std::error::Error + Send + Sync + 'static) -> ProviderError { + protocol_error_marker().with_source(error) } fn protocol_error_marker() -> ProviderError { diff --git a/backend/crates/providers/xai/src/transport/inference.rs b/backend/crates/providers/xai/src/transport/inference.rs index bce28254d..84a6d63d6 100644 --- a/backend/crates/providers/xai/src/transport/inference.rs +++ b/backend/crates/providers/xai/src/transport/inference.rs @@ -6,7 +6,10 @@ use std::pin::Pin; use std::time::Duration; use futures::Stream; -use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic}; +use gateway_core::error::{ + ClientVisibleUpstreamError, ErrorSource, OpaqueUpstreamValue, ProviderDiagnostic, + RawUpstreamError, +}; use gateway_core::event::UpstreamHttpVersion; use gateway_core::upstream::UpstreamSendState; use url::Url; @@ -317,11 +320,13 @@ pub enum GrokInferenceTransportErrorKind { Cancelled, } -/// 已分类的 transport 错误，绝不包含上游响应体 -#[derive(Clone, PartialEq, Eq)] +/// 推理 transport 失败，公开分类与受控上游详情分别承载 +#[derive(Clone)] pub struct GrokInferenceTransportError { kind: GrokInferenceTransportErrorKind, + source: Option, diagnostic: Option>, + raw_upstream_error: Option>, send_state: UpstreamSendState, status: Option, retry_after: Option, @@ -340,7 +345,9 @@ impl GrokInferenceTransportError { pub const fn new(kind: GrokInferenceTransportErrorKind, send_state: UpstreamSendState) -> Self { Self { kind, + source: None, diagnostic: None, + raw_upstream_error: None, send_state, status: None, retry_after: None, @@ -370,6 +377,24 @@ impl GrokInferenceTransportError { self.diagnostic.as_deref() } + #[must_use] + pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { + self.source = Some(ErrorSource::new(source)); + self + } + + /// 原始响应仅交给受控诊断，普通格式化不会展开正文 + #[must_use] + pub fn with_raw_upstream_error(mut self, error: RawUpstreamError) -> Self { + self.raw_upstream_error = Some(Box::new(error)); + self + } + + #[must_use] + pub fn raw_upstream_error(&self) -> Option<&RawUpstreamError> { + self.raw_upstream_error.as_deref() + } + /// 附着合法的 HTTP 状态码 #[must_use] pub fn with_status(mut self, status: u16) -> Self { @@ -397,7 +422,7 @@ impl GrokInferenceTransportError { self } - /// 附着从错误 JSON 中提取并清洗后的稳定机器码 + /// 保留错误 JSON 中的原始 code，业务分类不覆盖此值 #[must_use] pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self { self.upstream_code = Some(Box::new(code)); @@ -536,7 +561,11 @@ impl fmt::Display for GrokInferenceTransportError { } } -impl std::error::Error for GrokInferenceTransportError {} +impl std::error::Error for GrokInferenceTransportError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_deref().map(|source| source as _) + } +} /// 推理 transport 返回的 future pub type GrokInferenceTransportFuture<'a> = Pin< diff --git a/backend/crates/providers/xai/src/transport/network.rs b/backend/crates/providers/xai/src/transport/network.rs index ac9aba3e2..d5d719c0b 100644 --- a/backend/crates/providers/xai/src/transport/network.rs +++ b/backend/crates/providers/xai/src/transport/network.rs @@ -462,7 +462,7 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport { UpstreamSendState::Sent, ) .with_transport_metrics(transport_metrics)), - Err(error) => Err(classify_inference_stream_error(&error) + Err(error) => Err(classify_inference_stream_error(error) .with_transport_metrics(transport_metrics)), }; std::future::ready(Some(item)) @@ -1018,8 +1018,19 @@ fn classify_inference_reqwest_error(error: reqwest::Error) -> GrokInferenceTrans UpstreamSendState::Ambiguous, ) }; - GrokInferenceTransportError::new(kind, send_state) - .with_diagnostic(inference_http_diagnostic(&error)) + let failure = GrokInferenceTransportError::new(kind, send_state) + .with_diagnostic(inference_http_diagnostic(&error)); + with_inference_reqwest_source(failure, error) +} + +fn with_inference_reqwest_source( + mut failure: GrokInferenceTransportError, + source: reqwest::Error, +) -> GrokInferenceTransportError { + if source.url().is_some() { + failure = failure.redact_sensitive_context("request URL"); + } + failure.with_source(source.without_url()) } fn classify_model_catalog_reqwest_error(error: reqwest::Error) -> GrokModelCatalogTransportError { @@ -1044,8 +1055,8 @@ fn classify_billing_reqwest_error(error: reqwest::Error) -> GrokBillingTransport GrokBillingTransportError::new(kind) } -fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTransportError { - GrokInferenceTransportError::new( +fn classify_inference_stream_error(error: reqwest::Error) -> GrokInferenceTransportError { + let failure = GrokInferenceTransportError::new( if error.is_timeout() { GrokInferenceTransportErrorKind::Timeout } else { @@ -1053,7 +1064,8 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans }, UpstreamSendState::Sent, ) - .with_diagnostic(inference_http_diagnostic(error)) + .with_diagnostic(inference_http_diagnostic(&error)); + with_inference_reqwest_source(failure, error) } async fn classify_inference_status( @@ -1065,16 +1077,27 @@ async fn classify_inference_status( let http_version = upstream_http_version(response.version()); let request_id = upstream_request_id(&response); let status_code = status.as_u16(); - let body = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await { - Ok(BoundedBody::Body(body)) => body, - Ok(BoundedBody::TooLarge) | Err(_) => { + let (body, body_error) = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await { + Ok(BoundedBody::Body(body)) => (Some(body), None), + Ok(BoundedBody::TooLarge) => { trace.record( "capture.gap", - serde_json::json!({"reason": "error_body_unavailable_or_too_large"}), + serde_json::json!({"reason": "error_body_too_large"}), ); - Vec::new() + (None, None) + } + Err(source) => { + trace.record( + "capture.gap", + serde_json::json!({"reason": "error_body_read_failed"}), + ); + (None, Some(source)) } }; + let raw = body.as_ref().map(|body| { + gateway_core::error::RawUpstreamError::new(String::from_utf8_lossy(body).into_owned()) + }); + let body = body.unwrap_or_default(); trace.capture("upstream.error.body", &body); let metadata = inference_error_metadata(&body); let body_failure = classify_grok_body_failure(&metadata, &body); @@ -1108,17 +1131,23 @@ async fn classify_inference_status( let credential_recovery_required = kind == GrokInferenceTransportErrorKind::Unauthorized; let mut error = GrokInferenceTransportError::new(kind, UpstreamSendState::Sent) .with_status(status_code) - .with_response_facts(http_version, request_id) - .redact_sensitive_context("upstream response body"); - let upstream_code = if status == StatusCode::BAD_REQUEST && reasoning_decode_failed(&metadata) { - Some("reasoning_decode_failed".to_owned()) + .with_response_facts(http_version, request_id); + if let Some(raw) = raw { + error = error.with_raw_upstream_error(raw); + } + let diagnostic_code = if status == StatusCode::BAD_REQUEST && reasoning_decode_failed(&metadata) + { + "reasoning_decode_failed" } else { - metadata - .code - .as_deref() - .and_then(normalize_failure_code) - .or_else(|| body_failure.map(GrokBodyFailure::marker).map(str::to_owned)) + body_failure.map_or("upstream_failure", GrokBodyFailure::marker) }; + error = error.with_diagnostic( + ProviderDiagnostic::new(format!( + "xAI upstream rejected request: status={status_code}" + )) + .with_classification("upstream", diagnostic_code), + ); + let upstream_code = metadata.code; if let Some(message) = metadata.client_message.as_deref() { error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new( scrub_account_fingerprints(message), @@ -1135,6 +1164,9 @@ async fn classify_inference_status( if let Some(retry_after) = retry_after { error = error.with_retry_after(retry_after); } + if let Some(source) = body_error { + error = with_inference_reqwest_source(error, source); + } error } @@ -1287,8 +1319,7 @@ fn first_string(object: &serde_json::Map, fields: &[&str]) -> Opt fields .iter() .find_map(|field| object.get(*field).and_then(Value::as_str)) - .map(str::trim) - .filter(|value| !value.is_empty()) + .filter(|value| !value.trim().is_empty()) .map(ToOwned::to_owned) } @@ -1456,22 +1487,6 @@ fn contains_any(value: &str, signals: &[&str]) -> bool { signals.iter().any(|signal| value.contains(signal)) } -fn normalize_failure_code(value: &str) -> Option { - let mut normalized = String::with_capacity(value.len().min(48)); - for character in value.trim().to_ascii_lowercase().chars() { - if character.is_ascii_alphanumeric() { - normalized.push(character); - } else if matches!(character, '-' | '_' | '.' | ':') { - normalized.push('_'); - } - if normalized.len() >= 48 { - break; - } - } - let normalized = normalized.trim_matches('_'); - (!normalized.is_empty()).then(|| normalized.to_owned()) -} - fn upstream_http_version(version: reqwest::Version) -> UpstreamHttpVersion { match version { reqwest::Version::HTTP_09 => UpstreamHttpVersion::Http09, diff --git a/backend/crates/providers/xai/tests/provider/contract.rs b/backend/crates/providers/xai/tests/provider/contract.rs index aaf300d0a..ce002f5ad 100644 --- a/backend/crates/providers/xai/tests/provider/contract.rs +++ b/backend/crates/providers/xai/tests/provider/contract.rs @@ -2750,7 +2750,10 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() { GrokInferenceTransportErrorKind::ModelQuotaExhausted, UpstreamSendState::Sent, ) - .with_status(403), + .with_status(403) + .with_raw_upstream_error(gateway_core::error::RawUpstreamError::new( + "PRIVATE_QUOTA_ERROR", + )), ); let provider = provider(selector.clone(), transport).await; let mut stream = provider @@ -2764,6 +2767,11 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() { let error = next_provider_error(&mut stream).await; assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted); + assert_eq!( + error.raw_upstream_error().unwrap().as_str(), + "PRIVATE_QUOTA_ERROR" + ); + assert!(!format!("{error:?} {error}").contains("PRIVATE_QUOTA_ERROR")); assert!(error.replay_is_safe()); assert_eq!( selector.feedback.lock().expect("feedback").as_slice(), diff --git a/backend/crates/providers/xai/tests/transport/canonical.rs b/backend/crates/providers/xai/tests/transport/canonical.rs index 6a4515592..5726beef1 100644 --- a/backend/crates/providers/xai/tests/transport/canonical.rs +++ b/backend/crates/providers/xai/tests/transport/canonical.rs @@ -1452,6 +1452,10 @@ fn decoder_should_classify_failed_event_without_retaining_body() { assert_eq!(visible.message(), "secret"); assert_eq!(visible.code(), Some("rate_limit_exceeded")); assert_eq!(visible.error_type(), Some("server_error")); + let raw: serde_json::Value = + serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(raw["error"]["message"], "secret"); + assert_eq!(raw["error"]["code"], "rate_limit_exceeded"); assert!(!format!("{error:?}").contains("secret")); } diff --git a/backend/crates/providers/xai/tests/transport/network.rs b/backend/crates/providers/xai/tests/transport/network.rs index 0284f1792..f7da6e883 100644 --- a/backend/crates/providers/xai/tests/transport/network.rs +++ b/backend/crates/providers/xai/tests/transport/network.rs @@ -27,6 +27,58 @@ use provider_xai::{ use crate::support::{loopback_endpoint_policy, rejecting_account_proxy}; +#[tokio::test] +async fn inference_transport_preserves_verbatim_unknown_codes_without_changing_classification() { + for code in [ + " Vendor.Future:v2-UNRECOGNIZED ", + "长错误码", + "Vendor.".repeat(20).as_str(), + ] { + let server = MockServer::start().await; + let body = json!({"error": {"code": code, "message": "private response detail"}}); + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(503).set_body_json(&body)) + .mount(&server) + .await; + let origin = Url::parse(&server.uri()).unwrap(); + let error = inference_transport(&origin) + .execute(inference_request(&origin)) + .await + .unwrap_err(); + assert_eq!(error.kind(), GrokInferenceTransportErrorKind::Unavailable); + assert_eq!(error.upstream_code().unwrap().as_str(), code); + assert_eq!( + serde_json::from_str::(error.raw_upstream_error().unwrap().as_str()) + .unwrap(), + body + ); + assert_eq!(error.diagnostic().unwrap().code(), Some("upstream_failure")); + assert!(!format!("{error:?} {error}").contains(code)); + } +} + +#[tokio::test] +async fn inference_transport_keeps_native_network_causes_without_request_urls() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let origin = Url::parse(&format!("http://{address}")).unwrap(); + let error = inference_transport(&origin) + .execute(inference_request(&origin)) + .await + .unwrap_err(); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + let source = std::error::Error::source(&error) + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(source.is_connect()); + assert!(source.url().is_none()); + assert!(error.sensitive_context_was_redacted()); + assert!(std::error::Error::source(source).is_some()); +} + #[tokio::test] async fn official_inference_tls_should_advertise_grok_cli_alpn() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -536,7 +588,7 @@ async fn inference_connect_failure_keeps_safe_diagnosis_without_endpoint_or_payl } #[tokio::test] -async fn inference_transport_should_classify_http_failures_without_retaining_bodies() { +async fn inference_transport_should_retain_http_failure_bodies_only_in_explicit_details() { let cases = [ (400, GrokInferenceTransportErrorKind::InvalidRequest), (401, GrokInferenceTransportErrorKind::Unauthorized), @@ -578,7 +630,7 @@ async fn inference_transport_should_classify_http_failures_without_retaining_bod expected_kind, Some(status), UpstreamSendState::Sent, - true, + false, Some(gateway_core::event::UpstreamHttpVersion::Http11), ) ); @@ -588,6 +640,7 @@ async fn inference_transport_should_classify_http_failures_without_retaining_bod ); assert!(error.transport_metrics().headers_ms().is_some()); assert!(error.client_visible_upstream_error().is_none()); + assert_eq!(error.raw_upstream_error().unwrap().as_str(), secret); assert!(!rendered.contains(&secret)); } } @@ -621,7 +674,7 @@ async fn inference_transport_should_expose_safe_flat_json_error_details() { ); assert_eq!( (detail.message(), detail.code(), detail.error_type()), - (message, Some("personal_team_blocked_spending_limit"), None,) + (message, Some("personal-team-blocked:spending-limit"), None,) ); assert!(!rendered.contains(message)); } @@ -656,7 +709,7 @@ async fn inference_transport_should_scrub_nested_json_error_details() { (detail.message(), detail.code(), detail.error_type()), ( "team [redacted] rate limited", - Some("rate_limit_burst"), + Some("rate-limit:burst"), Some("rate_limit_error"), ) ); @@ -789,7 +842,7 @@ async fn inference_transport_should_classify_model_quota_for_403_and_429() { #[tokio::test] async fn inference_transport_should_prefer_structured_quota_fields_over_conflicting_message() { for (code, error_type) in [ - ("quota_exceeded", Some("subscription_free_usage_exhausted")), + ("quota_exceeded", Some("subscription:free-usage-exhausted")), ("rate_limit_exceeded", Some("quota_exceeded")), ] { let server = MockServer::start().await; @@ -845,7 +898,7 @@ async fn inference_transport_should_classify_free_usage_429_as_account_free_quot assert_eq!(error.status(), Some(429)); assert_eq!( error.upstream_code().map(|code| code.as_str()), - Some("subscription_free_usage_exhausted") + Some("subscription:free-usage-exhausted") ); } @@ -855,7 +908,7 @@ async fn inference_transport_should_apply_body_aware_400_failures() { ( json!({"error": {"code": "subscription:free-usage-exhausted", "message": "free usage exhausted"}}), GrokInferenceTransportErrorKind::FreeQuotaExhausted, - Some("subscription_free_usage_exhausted"), + Some("free_usage_exhausted"), ), ( json!({"error": {"message": "selected model is at capacity"}}), @@ -900,7 +953,10 @@ async fn inference_transport_should_apply_body_aware_400_failures() { assert_eq!(error.kind(), expected_kind); assert_eq!( - error.upstream_code().map(|code| code.as_str()), + error + .diagnostic() + .and_then(|diagnostic| diagnostic.code()) + .filter(|code| *code != "upstream_failure"), expected_code ); } @@ -961,8 +1017,9 @@ async fn inference_transport_should_classify_reasoning_decode_rejections() { error.kind(), GrokInferenceTransportErrorKind::InvalidRequest ); + assert!(error.upstream_code().is_none()); assert_eq!( - error.upstream_code().map(|code| code.as_str()), + error.diagnostic().and_then(|diagnostic| diagnostic.code()), Some("reasoning_decode_failed") ); } diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 04ddb7e25..1831d5822 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -20,3 +20,4 @@ fcfae912ddefd54551ebf8e7e733442519d2862d2977134d548ebc1e75c1dc34 0017_unlimited e8c1339166ac02fb2d4a077424ee5f64f6dc20eb4d3f8ec4fcb09c0f2a0e61f6 0020_plugin_trust.sql 22faf2646168275150b1786c0f463f9ff2ca52463a272479e4fcc6f46c8cfa37 0021_runtime_scheduling.sql 77a140ddfb9dfae9b7d7349e5bfc9c8a7e7927bae6e77c1a627b34fcbb714bdb 0022_refresh_margin_and_group_fast_mode.sql +68fe28522870903f9a9c9772e13edda29b9b0f1798d350b1025c273b05d25370 0023_error_details.sql diff --git a/backend/migrations/0023_error_details.sql b/backend/migrations/0023_error_details.sql new file mode 100644 index 000000000..17d3a3efc --- /dev/null +++ b/backend/migrations/0023_error_details.sql @@ -0,0 +1,3 @@ +-- 运维错误详情同时承载本地来源链与上游响应，运行时只使用统一字段 +alter table model_requests rename column raw_upstream_error to error_details; +alter table ops_events rename column raw_upstream_error to error_details; diff --git a/docs/api.md b/docs/api.md index 878bdd5f7..6f0cce983 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1702,6 +1702,12 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可 Key 已删除或未关联时为 `null`，不影响记录返回，不包含密钥原文 +运维错误记录的 `errorDetails` 为受控诊断文本，无详情时为 `null`。 +错误来源快照以 JSON 文本保存：`causes.messages` 按外层到内层排列本地原因，`causes.truncated` +标记来源链截断，`upstream` 保存上游正文原文，`redacted` 标记敏感上下文移除；没有对应来源时为 `null`。 +稳定错误分类、原始上游 code 和该详情分别保留，不以公共错误文案代替原始原因。 +详情不出现在 Key 用量接口、普通错误信封和诊断包导出中；记录仍受异步观测写入与保留周期约束 + 管理端请求列表与详情分别保留 `requestedModel`（客户端请求）、`upstreamModel`（网关发送）与 `upstreamResponseModel`（上游返回）。返回模型缺失时为 `null`，不使用请求或映射模型补齐。 OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告（流内报告可覆盖初始响应头）， @@ -1751,9 +1757,13 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告（流内 未知名称的 `{ bytes, sha256 }` 摘要，或缺失时的 `null`。已保存的 trace 不自动清理或回填， 其中的 `sanitized` 标记不能作为可直接公开的保证 -管理端下载的诊断包 `schemaVersion: 2` 用于人工反馈，不是备份或导入格式。它包含关联 ID、错误分类摘要、 -请求与错误事件各自的状态、attempt、时间线阶段和计时；不自动导出 message/raw error、任意 metadata、 -trace event data、请求响应正文和头部。`availability` 与 `omitted` 明示未采集、不完整或主动省略的内容， +`attempt.failed.data` 的 `kind / sendState / upstreamStatus` 与 `diagnostic.stage / diagnostic.code` +由 Core 错误类型与 Provider 静态诊断生成；`diagnostic.message` 保存安全摘要。展示与导出共用这些字段， +`sendState` 为 `not_sent / sent / ambiguous`，摘要被截断时带有 `truncated` 标记 + +管理端下载的诊断包 `schemaVersion: 3` 用于人工反馈，不是备份或导入格式。它包含关联 ID、错误分类摘要、 +请求与错误事件各自的状态、attempt、时间线阶段、计时和上述失败分类；不自动导出 message/raw error、任意 metadata、 +其他 trace event data、请求响应正文和头部。`availability` 与 `omitted` 明示未采集、不完整或主动省略的内容， `null` 不代表没有发生错误。版本、环境及原始错误片段仍需操作者另行补充并审阅脱敏 错误记录中的“已自动恢复”表示系统关联到了后续成功请求，不会把原来的失败记录改为成功。 diff --git a/docs/architecture.md b/docs/architecture.md index 75e096e61..3c23e5eb4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -501,8 +501,26 @@ client，OIDC 的 JWKS 缓存与单飞归属对应出口状态。自动刷新提 2. **控制面展示错误**：`/api/admin/*` 由 API 统一 HTTP 状态与数值业务码，由 Admin/API owner 提供安全中文 文案。extractor rejection、namespace 404 和 method 405 也使用同一 JSON 信封；任意 Store、Serde、 Provider `Display` 不得直接跨越 HTTP 边界 -3. **原始上游诊断**：只在连接测试、运维错误详情等明确诊断界面展示已经捕获的原始字段，不翻译、不补造， - 也不塞入普通 Admin 错误信封。原始 body 不进入 Debug、普通日志或持久化错误消息 +3. **受控错误详情**：运维错误详情分别保存本地原因链与上游响应，原始字段不翻译、不补造， + 不进入普通 Admin 错误信封、Debug、普通日志或持久化错误摘要。连接测试只展示该接口允许的诊断投影 + +错误分类负责公共响应、重试和账号处置，原始上游 code 独立保存，不受已知分类白名单限制。 +本地异常通过标准 `Error::source` 跨层传播；Core 的不透明来源载体保持底层类型，不依赖数据库或 HTTP 库， +普通格式化不展开来源。Store 统一基础设施不可用的转换，Provider 拥有业务分类， +插件 Runtime 集中投影 RPC、fault 与上游适配事件，调用点只附加原始来源与执行上下文 + +ProviderError 统一生成受控详情，失败 attempt、探测和最终请求复用该出口。 +原始上游正文与本地原因链分开承载，来源链最多保留 32 层、64 KiB 文本，截断明确标记； +请求 URL 等请求方向的敏感上下文在附加来源前移除并标记。错误快照保留来源与错误详情， +协议透传响应和未交付事件仍由当前请求独占。详情计入观测队列的内存预算； +队列满、关闭超时或写入失败仍可能丢失记录，沿用丢弃与写入失败指标，不自动重放不保证幂等的写入 + +推理日志由作出处置的边界记录恢复、放弃或终态；仅传播错误的层不重复打印。 +传输收包使用 debug 摘要，普通日志使用稳定阶段、分类与 request/attempt 关联，不展开原始正文、 +任意上游 code 或关闭原因。受控详情无法写入时不能回退到普通日志打印原文 + +Core 从执行错误生成一份失败诊断，展示与导出共用其中的分类；事件摘要截断时保留有界的失败诊断， +并显式标记截断。完整上游错误仍归运维错误记录，不进入诊断包 Provider 管理适配在仍持有结构化失败事实时选择静态 `public_message`；Admin 将其转为可安全展示的 `AdminError.message`，API 保留其中的 502／503 具体原因，无公开消息时才使用通用回退。认证与未知 diff --git a/frontend/src/api/modules/usage.ts b/frontend/src/api/modules/usage.ts index e1984a9cd..b2a64bc28 100644 --- a/frontend/src/api/modules/usage.ts +++ b/frontend/src/api/modules/usage.ts @@ -209,7 +209,18 @@ export interface RequestTraceEvent { exchangeId: number | null stage: string count: number - data: Record + data: Record & { + /** attempt.failed 由 Core 从 ProviderError 生成，分类与诊断只有这一份来源 */ + kind?: string + sendState?: 'not_sent' | 'sent' | 'ambiguous' + upstreamStatus?: number | null + diagnostic?: { + stage: string | null + code: string | null + message: string + truncated?: boolean + } | null + } } export interface RequestTrace { @@ -303,7 +314,7 @@ export interface OpsError { subagentKind: string | null compact: boolean | null message: string - rawUpstreamError: string | null + errorDetails: string | null metadata: OpsErrorMetadata createdAt: string createdAtDisplay: string diff --git a/frontend/src/views/usage/components/OpsErrorDetailModal.vue b/frontend/src/views/usage/components/OpsErrorDetailModal.vue index 67a615c8f..1488d647b 100644 --- a/frontend/src/views/usage/components/OpsErrorDetailModal.vue +++ b/frontend/src/views/usage/components/OpsErrorDetailModal.vue @@ -140,11 +140,11 @@ function visibleFields(items: DetailField[]) { -
+
diff --git a/frontend/src/views/usage/utils/diagnosticsBundle.ts b/frontend/src/views/usage/utils/diagnosticsBundle.ts index a7f33f8f4..8065c1b24 100644 --- a/frontend/src/views/usage/utils/diagnosticsBundle.ts +++ b/frontend/src/views/usage/utils/diagnosticsBundle.ts @@ -1,5 +1,5 @@ import type { OpsError, UsageRecordDetail } from '@/api' -import { opsErrorSummary } from './opsErrorPresentation' +import { failureClassText } from './opsErrorPresentation' export function requestDiagnosticsBundle( requestId: string, @@ -14,7 +14,7 @@ export function requestDiagnosticsBundle( const request = record ?? error return { - schemaVersion: 2, + schemaVersion: 3, source: 'usage.request_diagnostics', exportPolicy: 'allowlisted_fields_without_payloads', request: { @@ -40,10 +40,9 @@ export function requestDiagnosticsBundle( requestId: error.requestId, source: error.metadata.source, component: error.metadata.component, - summary: opsErrorSummary(error), - summarySource: 'failure_class_or_provider_error_code', + summary: failureClassText(error.failureClass), + summarySource: 'failure_class', failureClass: error.failureClass, - providerErrorCode: error.providerErrorCode, upstreamSendState: error.upstreamSendState, attemptIndex: error.attemptIndex, clientStatusCode: error.clientStatusCode, @@ -54,7 +53,7 @@ export function requestDiagnosticsBundle( createdAt: error.createdAt, } : null, - // 旧 trace 的 sanitized 标记不是安全保证；不遍历或复制任何 event.data。 + // 只导出 Core 从错误类型生成的分类，正文与其他事件 data 不进入诊断包 trace: trace ? { schemaVersion: trace.schemaVersion, @@ -70,6 +69,15 @@ export function requestDiagnosticsBundle( exchangeId: event.exchangeId, stage: event.stage, count: event.count, + failure: event.stage === 'attempt.failed' + ? { + kind: event.data.kind, + sendState: event.data.sendState, + stage: event.data.diagnostic?.stage ?? null, + code: event.data.diagnostic?.code ?? null, + upstreamStatus: event.data.upstreamStatus, + } + : null, })), } : null, @@ -83,7 +91,6 @@ export function requestDiagnosticsBundle( outcome: attempt.outcome, downstreamCommitted: attempt.downstreamCommitted, statusCode: attempt.statusCode, - providerErrorCode: attempt.providerErrorCode, failureClass: attempt.failureClass, accountId: attempt.accountId, firstTokenMs: attempt.firstTokenMs, @@ -100,7 +107,13 @@ export function requestDiagnosticsBundle( availability: { requestDetail: record ? 'available' : 'unavailable', errorSummary: error ? 'available' : 'not_available_for_selected_request', - trace: !record ? 'unknown' : trace ? 'stages_only' : 'not_recorded', + trace: !record + ? 'unknown' + : !trace + ? 'not_recorded' + : trace.events.some(event => event.stage === 'attempt.failed') + ? 'stages_and_failure_classifications' + : 'stages_only', traceEventsDropped: trace?.droppedEvents ?? null, attemptsComplete: record?.attemptsComplete ?? null, relatedRequests: record ? 'available' : 'unknown', @@ -108,19 +121,21 @@ export function requestDiagnosticsBundle( }, omitted: [ 'error.message', - 'error.rawUpstreamError', + 'error.errorDetails', + 'error.providerErrorCode', + 'attempts.providerErrorCode', 'request.message', 'request.metadata', - 'trace.events.data', + 'trace.events.data_except_failure_classification', 'request_and_response_headers_and_bodies', 'user_supplied_model_names', 'account_names_emails_and_credential_names', 'client_ip_user_agent_and_api_key', ], notes: [ - 'schemaVersion 2 仅供人工排障，null 表示未知或未采集，不代表没有发生错误', - '错误摘要复用列表的分类展示，不是上游错误原文，error 为 null 时请结合 attempts 的分类与状态', - '时间线仅导出阶段、顺序和计时，不含事件 data，历史 sanitized 标记不作为安全依据', + 'schemaVersion 3 仅供人工排障，null 表示未知或未采集，不代表没有发生错误', + '错误摘要只使用稳定分类，不含上游错误码或原文，error 为 null 时请结合 attempts 的分类与状态', + '时间线导出阶段、顺序、计时及 Core 记录的失败分类，不含错误原文和其他事件 data', 'attemptsComplete 不为 true 时，尝试列表不完整，没有时间线的旧记录无法补回未采集事件', '未自动采集网关版本、客户端版本、部署环境和故障发生时区，请另行补充', '关联 ID 仍可能属于内部信息，分享前请审阅，原始错误、正文和日志需另行审阅脱敏，勿直接公开',