fix(diagnostics): preserve failure causes across request tracing

Retain preparation and account-selection diagnostics, preserve plugin fault
details, and record failures before attempt startup. Keep failure
classification when trace summaries are truncated.

Carry native error sources through Core, infrastructure availability errors,
OpenAI credential selection, provider inference, and plugin upstream adapters.
Preserve original upstream codes independently of failure classification and
project bounded cause chains and upstream bodies into protected error details.

Rename persisted and admin API error detail fields, keep raw details out of
ordinary logs and diagnostic exports, and account for retained error data in
the observation queue memory budget. Add regression coverage for source
retention, safe output, and migration upgrades; remove account-dependent live
tests. Document the implemented diagnostic boundaries.

Refs #357

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Zyy
2026-10-06 18:19:04 +08:00
co-authored by Codex
parent b27877c99b
commit a5e4041295
102 changed files with 2311 additions and 1333 deletions
@@ -851,7 +851,7 @@ pub struct OpsError {
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub message: String,
pub raw_upstream_error: Option<String>,
pub error_details: Option<String>,
pub client_ip: Option<String>,
pub user_agent: Option<String>,
pub reasoning_effort: Option<String>,
@@ -52,12 +52,13 @@ pub enum AdminStoreErrorKind {
}
/// 隐藏数据库实现细节的持久化错误
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[derive(Debug, Clone, thiserror::Error)]
#[error("{resource} store operation failed: {message}")]
pub struct AdminStoreError {
kind: AdminStoreErrorKind,
resource: &'static str,
message: String,
source: Option<gateway_core::error::ErrorSource>,
}
impl AdminStoreError {
@@ -71,9 +72,17 @@ impl AdminStoreError {
kind,
resource,
message: message.into(),
source: None,
}
}
/// 来源留在内部诊断链,公开消息不展开基础设施错误
#[must_use]
pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
self.source = Some(gateway_core::error::ErrorSource::new(source));
self
}
#[must_use]
pub const fn kind(&self) -> AdminStoreErrorKind {
self.kind
@@ -617,7 +617,7 @@ pub(crate) fn ops_error_view(
subagent_kind: error.subagent_kind,
compact: error.compact,
message: error.message,
raw_upstream_error: error.raw_upstream_error,
error_details: error.error_details,
metadata: OpsErrorMetadataView {
source: error.source,
component: error.component,
@@ -814,8 +814,8 @@ pub struct OpsErrorView {
pub subagent_kind: Option<String>,
pub compact: Option<bool>,
pub message: String,
/// 上游错误正文或 WebSocket close/error frame 原文;不做脱敏
pub raw_upstream_error: Option<String>,
/// 受控错误详情,分别承载本地原因链与上游错误正文
pub error_details: Option<String>,
pub metadata: OpsErrorMetadataView,
pub created_at: DateTime<Utc>,
pub created_at_display: String,
@@ -466,7 +466,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
upstream_request_id: None,
latency_ms: Some(90),
message: "snapshot error".to_owned(),
raw_upstream_error: Some(
error_details: Some(
r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(),
),
client_ip: Some("203.0.113.8".to_owned()),
@@ -526,7 +526,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"userAgent": value["data"]["items"][0]["userAgent"],
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
"upstreamSendState": value["data"]["items"][0]["upstreamSendState"],
"rawUpstreamError": value["data"]["items"][0]["rawUpstreamError"],
"errorDetails": value["data"]["items"][0]["errorDetails"],
"continuationUnavailableReason": value["data"]["items"][0]["metadata"]["continuationUnavailableReason"],
"upstreamConnectionExitReason": value["data"]["items"][0]["metadata"]["upstreamConnectionExitReason"],
"upstreamConnectionAgeMs": value["data"]["items"][0]["metadata"]["upstreamConnectionAgeMs"],
@@ -547,7 +547,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
"userAgent": "codex-cli/0.144.0",
"providerErrorCode": "upstream",
"upstreamSendState": "sent",
"rawUpstreamError": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
"errorDetails": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
"continuationUnavailableReason": "reused_connection_lost",
"upstreamConnectionExitReason": "tcp_reset",
"upstreamConnectionAgeMs": 12000,
@@ -183,7 +183,7 @@ fn error_record() -> OpsError {
upstream_request_id: Some("private-sentinel".to_owned()),
latency_ms: Some(420),
message: "private-sentinel".to_owned(),
raw_upstream_error: Some("private-sentinel".to_owned()),
error_details: Some("private-sentinel".to_owned()),
client_ip: Some("192.0.2.42".to_owned()),
user_agent: Some("key-usage-test/1.0".to_owned()),
reasoning_effort: Some("xhigh".to_owned()),
@@ -10,6 +10,8 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde::Serialize;
use serde_json::{Value, json};
use crate::error::ProviderError;
use super::capture::{bounded, diagnostic_event_json, diagnostic_event_type};
use super::{body_fingerprint, diagnostic_headers};
@@ -123,6 +125,22 @@ impl TraceContext {
self.push(stage, data, false);
}
/// 诊断正文与导出分类来自同一失败,导出路径不接收任意 JSON 或上游文案
pub fn record_provider_failure(&self, error: &ProviderError) {
if !self.is_enabled() {
return;
}
let data = json!({
"kind": error.kind().as_str(), "sendState": error.send_state().as_str(),
"diagnostic": error.diagnostic().map(|diagnostic| json!({
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
})),
"upstreamStatus": error.upstream_status(),
"upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()),
});
self.push("attempt.failed", data, false);
}
/// 同一个头部边界只采集一次:默认脱敏,显式开启 dump 时另存完整字节
/// facts 必须为调用方构造的安全对象;仅受控诊断头可明文保留,完整头部另存 dump
pub fn headers<'a>(
@@ -243,7 +261,7 @@ impl TraceContext {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Some(json!({
"schemaVersion": 1, "requestId": state.request_id,
"schemaVersion": 2, "requestId": state.request_id,
"startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence,
"droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS,
"captureMode": "sanitized", "events": state.events,
@@ -258,10 +276,35 @@ impl TraceContext {
let Some(state) = &self.state else { return };
let mut encoded = serde_json::to_vec(&data).unwrap_or_default();
if encoded.len() > MAX_DATA_BYTES {
data = json!({"truncated": true, "summary": body_fingerprint(&encoded),
let mut summary = json!({"truncated": true, "summary": body_fingerprint(&encoded),
"eventType": data.get("eventType"), "body": data.get("body"),
"jsonValid": data.get("jsonValid"),
});
if stage == "attempt.failed" {
// 过长诊断的指纹不能挤掉已经由 Provider 分类的失败原因
// 保持有界,完整诊断仍由终态错误记录保存
summary = json!({"truncated": true, "summary": body_fingerprint(&encoded)});
for key in ["kind", "sendState"] {
if let Some(value) = data.get(key).and_then(Value::as_str) {
summary[key] = json!(bounded(value, 64));
}
}
summary["upstreamStatus"] =
json!(data.get("upstreamStatus").and_then(Value::as_u64));
if let Some(diagnostic) = data.get("diagnostic").and_then(Value::as_object) {
let mut kept = json!({});
for (key, limit) in [("stage", 96), ("code", 96), ("message", 256)] {
if let Some(value) = diagnostic.get(key).and_then(Value::as_str) {
kept[key] = json!(bounded(value, limit));
if value.chars().count() > limit {
kept["truncated"] = json!(true);
}
}
}
summary["diagnostic"] = kept;
}
}
data = summary;
encoded = serde_json::to_vec(&data).unwrap_or_default();
}
let mut state = state
@@ -398,7 +398,7 @@ struct FailureFinalization {
upstream_status_code: Option<u16>,
upstream_request_id: Option<String>,
provider_error_code: Option<String>,
raw_upstream_error: Option<String>,
error_details: Option<String>,
retry_after_ms: Option<u64>,
observation: ModelRequestFailureObservation,
}
@@ -523,6 +523,10 @@ where
self.trace.clone()
}
pub(super) fn request_id(&self) -> &ModelRequestId {
&self.request_id
}
/// 读取下一条 canonical event;首条未提交事件会携带 commit 要求
///
/// # Errors
@@ -956,7 +960,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -1026,6 +1030,9 @@ where
}
Ok(None) => {}
Err(error) => {
self.trace
.attempt(self.attempts)
.record_provider_failure(&error);
self.finish_provider_error(&error).await?;
return Err(provider_engine_error(error));
}
@@ -1101,7 +1108,7 @@ where
ProviderBoundary::Result(result) => match *result {
Ok(stream) => stream,
Err(error) => {
record_trace_error(&attempt_trace, &error);
attempt_trace.record_provider_failure(&error);
let continuation_retry = !error.retry_is_prohibited()
&& self.prepare_unavailable_native_continuation_replay(&error);
let candidate_retry = !error.retry_is_prohibited() && !continuation_retry
@@ -1200,7 +1207,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -1233,7 +1240,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -1256,7 +1263,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -1282,7 +1289,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -1438,7 +1445,9 @@ where
) -> Result<StreamErrorOutcome, EngineError> {
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
// 均只接触已剥离的稳定错误字段
record_trace_error(&self.trace.attempt(self.attempts), &error);
self.trace
.attempt(self.attempts)
.record_provider_failure(&error);
let mut atomic_client_events = error.take_atomic_client_events();
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
if self.request_observation.is_some() {
@@ -1961,7 +1970,7 @@ where
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
error: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
failure_observation: ModelRequestFailureObservation::default(),
retry_after_ms: None,
usage: self.observation.usage.clone(),
@@ -2002,9 +2011,7 @@ where
upstream_status_code: error.upstream_status(),
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()),
raw_upstream_error: error
.raw_upstream_error()
.map(|raw| raw.as_str().to_owned()),
error_details: error.error_details(),
retry_after_ms: error.retry_after().map(duration_ms),
observation: ModelRequestFailureObservation {
continuation_unavailable_reason: error
@@ -2065,7 +2072,7 @@ where
upstream_status_code: None,
upstream_request_id: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
retry_after_ms: None,
observation: ModelRequestFailureObservation::default(),
})
@@ -2128,7 +2135,7 @@ where
diagnostic_trace_json: self.trace.snapshot().map(|value| value.to_string()),
error: Some(finalization.error),
provider_error_code: finalization.provider_error_code,
raw_upstream_error: finalization.raw_upstream_error,
error_details: finalization.error_details,
failure_observation: finalization.observation,
retry_after_ms: finalization.retry_after_ms,
usage: self.observation.usage.clone(),
@@ -2489,19 +2496,3 @@ const fn escalate_send_state(a: UpstreamSendState, b: UpstreamSendState) -> Upst
fn duration_ms(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
fn record_trace_error(trace: &TraceContext, error: &ProviderError) {
trace.record("attempt.failed", json!({
"kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()),
"diagnostic": error.diagnostic().map(|diagnostic| json!({
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
})),
"upstreamStatus": error.upstream_status(),
"upstreamRequestId": error.upstream_request_id().map(|id| id.as_str()),
"upstreamCode": error.upstream_code().map(|code| code.as_str()),
"rawError": error.raw_upstream_error().map(|raw| {
let value = serde_json::from_str(raw.as_str()).unwrap_or_else(|_| json!(raw.as_str()));
crate::diagnostics::diagnostic_json(&value)
}),
}));
}
@@ -1310,8 +1310,12 @@ impl DefaultExecutionService {
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
pin_mut!(write, timeout);
select_biased! {
result = write => { if result.is_err() { tracing::warn!("入口拒绝观测写入失败"); } },
_ = timeout => tracing::warn!("入口拒绝观测写入超时"),
result = write => {
if let Err(error) = result {
tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", error_kind = ?error.kind(), "入口拒绝观测写入失败");
}
},
_ = timeout => tracing::warn!(request_id = request_id.as_str(), operation = "record_entry_rejection", "入口拒绝观测写入超时"),
}
}
if let (Some(observation), Err(error)) = (&request_observation, &result) {
@@ -2091,7 +2095,8 @@ impl DefaultExecutionService {
failure_kind = provider_error.kind().as_str(),
send_state = ?provider_error.send_state(),
upstream_status = ?provider_error.upstream_status(),
provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()),
diagnostic_stage = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.stage()),
diagnostic_code = ?provider_error.diagnostic().and_then(|diagnostic| diagnostic.code()),
latency_ms,
"账号连接测试失败"
);
@@ -2477,8 +2482,9 @@ struct AdmissionLease {
}
async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) {
let request_id = charge.request_id.clone();
if let Err(error) = port.settle(charge).await {
tracing::error!(%error, "Client budget settlement failed; storage will retry on the next request");
tracing::error!(request_id = request_id.as_str(), operation = "settle_client_budget", %error, "Client budget settlement failed; storage will retry on the next request");
}
}
@@ -2490,7 +2496,7 @@ impl AdmissionLease {
.release(&self.client_api_key_id, &self.model_request_id)
.await
{
tracing::warn!(%error, "Client admission 释放失败,依赖租约 TTL 收敛");
tracing::warn!(request_id = self.model_request_id.as_str(), key_id = self.client_api_key_id.as_str(), operation = "release_client_admission", %error, "Client admission 释放失败,依赖租约 TTL 收敛");
}
self.armed = false;
}
@@ -2621,7 +2627,7 @@ impl DefaultExecutionSession {
async fn finalize_detached(&mut self) {
if let Err(error) = self.core.cancel_and_finalize().await {
tracing::warn!(%error, "Detached execution 终态收敛失败");
tracing::warn!(request_id = self.core.request_id().as_str(), operation = "finalize_detached_execution", %error, "Detached execution 终态收敛失败");
}
self.settle_if_finalized().await;
}
@@ -934,7 +934,7 @@ pub struct ModelRequestFinalization {
pub error: Option<GatewayError>,
pub provider_error_code: Option<String>,
/// Provider 返回的原始错误正文或 WebSocket close/error frame
pub raw_upstream_error: Option<String>,
pub error_details: Option<String>,
pub failure_observation: ModelRequestFailureObservation,
pub retry_after_ms: Option<u64>,
pub usage: Usage,
@@ -1,5 +1,9 @@
//! 网关核心使用的稳定错误分类
mod source;
pub use source::ErrorSource;
use std::fmt;
use std::num::NonZeroU32;
use std::time::Duration;
@@ -231,6 +235,8 @@ impl ProviderDiagnostic {
while let Some(error) = source {
if let Some(error) = error.downcast_ref::<std::io::Error>() {
let code = match error.kind() {
std::io::ErrorKind::NotFound => Some("io_not_found"),
std::io::ErrorKind::PermissionDenied => Some("io_permission_denied"),
std::io::ErrorKind::ConnectionRefused => Some("connection_refused"),
std::io::ErrorKind::ConnectionReset => Some("tcp_reset"),
std::io::ErrorKind::ConnectionAborted => Some("connection_aborted"),
@@ -244,6 +250,15 @@ impl ProviderDiagnostic {
if let Some(code) = code {
self.message.push_str("; I/O cause: ");
self.message.push_str(code);
if let Some(os_code) = error.raw_os_error() {
// 只重建操作系统错误,不能格式化可能包含 URL、路径或正文的自定义 I/O 错误
use std::fmt::Write as _;
let _ = write!(
self.message,
"; OS cause: {}",
std::io::Error::from_raw_os_error(os_code)
);
}
self.code = Some(code);
break;
}
@@ -277,7 +292,7 @@ impl fmt::Debug for ProviderDiagnostic {
/// 运维错误详情中按原样展示的上游错误返回
///
/// 该值只接收响应方向的错误正文或 WebSocket close/error frame;不得放入请求
/// 该值只接收响应方向的错误正文、WebSocket close/error frame 或上游适配插件 fault;不得放入请求
/// Authorization、Cookie 或客户端输入
/// 它会被持久化并由管理端显式返回,但普通
/// `Debug`/`Display` 不输出正文,避免在非运维日志中重复扩散
@@ -427,8 +442,9 @@ impl fmt::Debug for ClientVisibleUpstreamResponse {
///
/// 稳定诊断字段不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
/// code/request ID/response ID
/// Adapter 若捕获到可能含 secret 的上下文,只能调用
/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记
/// 原始错误来源与上游响应正文仅由 [`ProviderError::error_details`] 提取到受控运维详情
/// 请求方向的凭据等敏感上下文必须先移除,并通过
/// [`ProviderError::redact_sensitive_context`] 留下脱敏标记
///
/// 唯一例外是 [`ProviderError::with_atomic_client_events`]:它只承载尚未交付客户端、
/// 必须与本错误一起完成重试判断的协议事件
@@ -450,6 +466,7 @@ pub struct ProviderError {
retry_same_account: bool,
sensitive_context_redacted: bool,
diagnostic: Option<Box<ProviderDiagnostic>>,
source: Option<ErrorSource>,
raw_upstream_error: Option<Box<RawUpstreamError>>,
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
client_visible_upstream_response: Option<Box<ClientVisibleUpstreamResponse>>,
@@ -491,6 +508,7 @@ impl ProviderError {
retry_same_account: false,
sensitive_context_redacted: false,
diagnostic: None,
source: None,
raw_upstream_error: None,
client_visible_upstream_error: None,
client_visible_upstream_response: None,
@@ -507,7 +525,7 @@ impl ProviderError {
self
}
/// 附加 adapter 已分类为安全的上游错误 code
/// 保留原始上游错误 code,不参与普通格式化或替代稳定分类
#[must_use]
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
self.upstream_values_mut().code = Some(code);
@@ -845,6 +863,29 @@ impl ProviderError {
self.diagnostic.as_deref()
}
/// 包装底层失败;来源只供显式诊断访问,不改变分类和发送事实
#[must_use]
pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
self.source = Some(ErrorSource::new(source));
self
}
/// 运维详情的唯一投影入口,来源链与上游正文分别保留,不进入公共响应或诊断包
#[must_use]
pub fn error_details(&self) -> Option<String> {
if self.source.is_none() && self.raw_upstream_error.is_none() {
return None;
}
Some(
serde_json::json!({
"causes": self.source.as_ref().map(ErrorSource::snapshot),
"upstream": self.raw_upstream_error().map(RawUpstreamError::as_str),
"redacted": self.sensitive_context_was_redacted(),
})
.to_string(),
)
}
/// 返回将由运维错误详情原样展示的上游错误返回
#[must_use]
pub fn raw_upstream_error(&self) -> Option<&RawUpstreamError> {
@@ -882,7 +923,7 @@ impl ProviderError {
/// 复制可安全进入诊断和持久化边界的稳定错误事实
///
/// 请求局部的原始上游响应与尚未提交的客户端事件不会进入快照
/// 保留原因链和上游错误详情;供协议透传的完整响应与尚未提交的客户端事件不进入快照
#[must_use]
pub fn stable_snapshot(&self) -> Self {
Self {
@@ -901,6 +942,7 @@ impl ProviderError {
retry_same_account: self.retry_same_account,
sensitive_context_redacted: self.sensitive_context_redacted,
diagnostic: self.diagnostic.clone(),
source: self.source.clone(),
raw_upstream_error: self.raw_upstream_error.clone(),
client_visible_upstream_error: self.client_visible_upstream_error.clone(),
client_visible_upstream_response: None,
@@ -916,10 +958,7 @@ impl fmt::Debug for ProviderError {
.field("kind", &self.kind)
.field("send_state", &self.send_state)
.field("upstream_status", &self.upstream_status)
.field(
"upstream_code",
&self.upstream_code().map(|_| "<classified-safe>"),
)
.field("upstream_code", &self.upstream_code().map(|_| "<opaque>"))
.field(
"upstream_request_id",
&self.upstream_request_id().map(|_| "<classified-safe>"),
@@ -986,7 +1025,11 @@ impl fmt::Display for ProviderError {
}
}
impl std::error::Error for ProviderError {}
impl std::error::Error for ProviderError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source.as_deref().map(|source| source as _)
}
}
/// 对客户端协议稳定的网关错误分类
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -1277,18 +1320,31 @@ pub enum StoreErrorKind {
InvalidData,
}
/// Core port 返回的脱敏存储错误
#[derive(Debug, Clone, PartialEq, Eq, Error)]
/// Core port 返回的存储错误,普通格式化不展开原始来源
#[derive(Debug, Clone, Error)]
#[error("execution store failed: {kind:?}")]
pub struct StoreError {
kind: StoreErrorKind,
source: Option<ErrorSource>,
}
impl StoreError {
/// 创建不携带数据库正文的 store 错误
/// 创建只有分类的本地存储错误;包装实际异常使用 caused_by
#[must_use]
pub const fn new(kind: StoreErrorKind) -> Self {
Self { kind }
Self { kind, source: None }
}
/// 包装已有存储错误并保留来源
#[must_use]
pub fn caused_by(
kind: StoreErrorKind,
source: impl std::error::Error + Send + Sync + 'static,
) -> Self {
Self {
kind,
source: Some(ErrorSource::new(source)),
}
}
/// 返回稳定错误分类
@@ -0,0 +1,71 @@
//! 跨领域边界共享原始错误来源,不在普通格式化中展开详情
use std::{error::Error, fmt, ops::Deref, sync::Arc};
/// 保留底层错误的类型和来源链,避免领域端口依赖具体基础设施
#[derive(Clone)]
pub struct ErrorSource(Arc<dyn Error + Send + Sync>);
impl ErrorSource {
#[must_use]
pub fn new(error: impl Error + Send + Sync + 'static) -> Self {
Self(Arc::new(error))
}
/// 仅供受控错误详情落盘,限制链深度和总文本大小并显式标记缺口
pub(super) fn snapshot(&self) -> serde_json::Value {
let mut messages = Vec::new();
let mut source: Option<&(dyn Error + 'static)> = Some(self.0.as_ref());
let mut remaining = 64 * 1024;
let mut truncated = false;
for _ in 0..32 {
let Some(error) = source else { break };
let mut message = BoundedMessage {
text: String::new(),
remaining,
};
if fmt::write(&mut message, format_args!("{error}")).is_err() {
truncated = true;
}
remaining = message.remaining;
messages.push(message.text);
source = error.source();
if truncated || remaining == 0 {
break;
}
}
serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()})
}
}
struct BoundedMessage {
text: String,
remaining: usize,
}
impl fmt::Write for BoundedMessage {
fn write_str(&mut self, value: &str) -> fmt::Result {
let end = value.floor_char_boundary(self.remaining.min(value.len()));
self.text.push_str(&value[..end]);
self.remaining -= end;
if end == value.len() {
Ok(())
} else {
Err(fmt::Error)
}
}
}
impl Deref for ErrorSource {
type Target = dyn Error + Send + Sync;
fn deref(&self) -> &Self::Target {
self.0.as_ref()
}
}
impl fmt::Debug for ErrorSource {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ErrorSource(<restricted>)")
}
}
@@ -28,17 +28,36 @@ pub enum ProviderStoreErrorKind {
}
/// Provider 存储端口的脱敏错误
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[derive(Debug, Clone, thiserror::Error)]
#[error("provider store {operation} failed: {kind:?}")]
pub struct ProviderStoreError {
kind: ProviderStoreErrorKind,
operation: &'static str,
source: Option<crate::error::ErrorSource>,
}
impl ProviderStoreError {
#[must_use]
pub const fn new(kind: ProviderStoreErrorKind, operation: &'static str) -> Self {
Self { kind, operation }
Self {
kind,
operation,
source: None,
}
}
/// 包装基础设施失败,分类与原始来源分别保留
#[must_use]
pub fn caused_by(
kind: ProviderStoreErrorKind,
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> Self {
Self {
kind,
operation,
source: Some(crate::error::ErrorSource::new(source)),
}
}
#[must_use]
@@ -6,6 +6,8 @@ use std::{
};
use gateway_core::diagnostics::{TraceContext, body_fingerprint};
use gateway_core::error::{ProviderDiagnostic, ProviderError, ProviderErrorKind, RawUpstreamError};
use gateway_core::upstream::UpstreamSendState;
use serde_json::json;
use tracing::{
Event, Metadata, Subscriber,
@@ -47,6 +49,55 @@ fn bounded_history_preserves_start_failure_and_final_result() {
);
}
#[test]
fn provider_failure_trace_keeps_classification_when_diagnostic_is_truncated() {
let trace = TraceContext::new("req_export");
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
.with_diagnostic(
ProviderDiagnostic::new("Safe diagnostic".repeat(500))
.with_classification("prepare", "account_proxy_invalid"),
)
.with_raw_upstream_error(RawUpstreamError::new("PRIVATE_RAW"));
trace.attempt(3).record_provider_failure(&error);
let snapshot = trace.snapshot().unwrap();
let data = &snapshot["events"][0]["data"];
assert_eq!(data["kind"], "unavailable");
assert_eq!(data["sendState"], "not_sent");
assert_eq!(data["diagnostic"]["stage"], "prepare");
assert_eq!(data["diagnostic"]["code"], "account_proxy_invalid");
assert_eq!(data["diagnostic"]["truncated"], true);
assert!(!snapshot.to_string().contains("PRIVATE_RAW"));
}
#[test]
fn oversized_failure_keeps_classification_and_marks_truncated_message() {
let trace = TraceContext::new("req_oversized_failure");
trace.attempt(2).record("attempt.failed", json!({
"kind": "transport", "sendState": "not_sent", "upstreamStatus": 502,
"diagnostic": {"stage": "connect", "code": "connection_refused", "message": "连接被拒绝".repeat(2000)},
"rawError": {"summary": "x".repeat(10_000)},
}));
let snapshot = trace.snapshot().unwrap();
let event = &snapshot["events"][0];
assert_eq!(event["attemptIndex"], 2);
let data = &event["data"];
assert_eq!(data["truncated"], true);
assert_eq!(data["kind"], "transport");
assert_eq!(data["sendState"], "not_sent");
assert_eq!(data["upstreamStatus"], 502);
assert_eq!(data["diagnostic"]["stage"], "connect");
assert_eq!(data["diagnostic"]["code"], "connection_refused");
assert_eq!(data["diagnostic"]["truncated"], true);
assert!(
data["diagnostic"]["message"]
.as_str()
.unwrap()
.starts_with("连接被拒绝")
);
assert!(serde_json::to_vec(data).unwrap().len() <= 4096);
assert!(data.get("rawError").is_none());
}
#[test]
fn shared_attempts_and_exchanges_coalesce_delta_without_losing_metadata() {
let trace = TraceContext::new("req_trace");
@@ -314,6 +365,18 @@ fn ordinary_trace_logs_and_serialized_snapshots_share_the_sanitized_capture_boun
);
exchange.capture_named_event("upstream.event", b"{}", Some("PRIVATE_SSE_EVENT"));
exchange.wire_event("openai", Some("PRIVATE_WIRE_EVENT"));
exchange.record_provider_failure(
&ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_upstream_code(gateway_core::error::OpaqueUpstreamValue::new(
"PRIVATE_CODE",
))
.with_raw_upstream_error(RawUpstreamError::new("PRIVATE_UPSTREAM_BODY"))
.with_source(std::io::Error::other("PRIVATE_NATIVE_CAUSE"))
.with_diagnostic(
ProviderDiagnostic::new("upstream rejected request")
.with_classification("upstream", "upstream_failure"),
),
);
exchange.headers(
"upstream.response.headers",
json!({"status": 429}),
@@ -65,6 +65,7 @@ struct FinalState {
image_output_tokens: Option<u64>,
image_generation_succeeded: Option<bool>,
provider_error_code: Option<String>,
error_details: Option<String>,
retry_after_ms: Option<u64>,
latency_ms: Option<u64>,
client_response_id: Option<String>,
@@ -232,6 +233,7 @@ impl ExecutionStore for FakeStore {
image_output_tokens: finalization.usage.image_output_tokens,
image_generation_succeeded: finalization.image_generation_succeeded,
provider_error_code: finalization.provider_error_code,
error_details: finalization.error_details,
retry_after_ms: finalization.retry_after_ms,
latency_ms: finalization.timings.latency_ms,
client_response_id: finalization.client_response_id,
@@ -297,6 +299,7 @@ struct ScriptedProvider {
profile_generation: AtomicUsize,
default_profile_calls: AtomicUsize,
default_profile: Mutex<Option<gateway_core::account::OpaqueProviderData>>,
default_profile_error: Mutex<Option<ProviderError>>,
scripts: Mutex<VecDeque<Script>>,
contexts: Mutex<Vec<AttemptContext>>,
operations: Mutex<Vec<Operation>>,
@@ -323,6 +326,7 @@ impl ScriptedProvider {
profile_generation: AtomicUsize::new(1),
default_profile_calls: AtomicUsize::new(0),
default_profile: Mutex::new(None),
default_profile_error: Mutex::new(None),
scripts: Mutex::new(scripts.into()),
contexts: Mutex::new(Vec::new()),
operations: Mutex::new(Vec::new()),
@@ -349,6 +353,9 @@ impl Provider for ScriptedProvider {
&self,
) -> Result<Option<gateway_core::account::OpaqueProviderData>, ProviderError> {
self.default_profile_calls.fetch_add(1, Ordering::SeqCst);
if let Some(error) = self.default_profile_error.lock().unwrap().take() {
return Err(error);
}
Ok(self.default_profile.lock().unwrap().clone())
}
@@ -2849,6 +2856,7 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
ProviderErrorKind::RateLimited,
UpstreamSendState::Sent,
)
.with_source(std::io::Error::other("ORIGINAL_RATE_LIMIT_CAUSE"))
.with_status(429)
.with_upstream_code(OpaqueUpstreamValue::new("rate_limit_exceeded"))
.with_retry_after(Duration::from_secs(30))
@@ -2894,6 +2902,12 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
state.finalizations[0].provider_error_code.as_deref(),
Some("rate_limit_exceeded")
);
let details: Value =
serde_json::from_str(state.finalizations[0].error_details.as_deref().unwrap()).unwrap();
assert_eq!(
details["causes"]["messages"],
json!(["ORIGINAL_RATE_LIMIT_CAUSE"])
);
}
#[test]
@@ -4905,6 +4919,59 @@ fn global_location_and_group_fast_policy_reach_every_account_retry() {
}
}
#[test]
fn profile_preparation_failure_is_traced_and_finalized_without_starting_an_attempt() {
let operation = generate_operation();
let route_plan = plan(&operation);
let (coordinator, store, provider) = coordinator(vec![]);
*provider.default_profile_error.lock().unwrap() = Some(
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
.with_diagnostic(
gateway_core::error::ProviderDiagnostic::new("Profile release unavailable")
.with_classification("prepare", "request_profile_release_unavailable"),
),
);
let mut session = block_on(coordinator.start(
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
operation,
route_plan,
None,
None,
CancellationToken::new(),
))
.unwrap();
assert!(block_on(session.collect_uncommitted()).is_err());
assert!(provider.contexts.lock().unwrap().is_empty());
let state = store.state.lock().unwrap();
assert_eq!(state.finalizations.len(), 1);
let trace: Value = serde_json::from_str(
state.finalizations[0]
.diagnostic_trace_json
.as_deref()
.unwrap(),
)
.unwrap();
let events = trace["events"].as_array().unwrap();
let failure = events
.iter()
.find(|event| event["stage"] == "attempt.failed")
.unwrap();
assert_eq!(failure["attemptIndex"], 0);
assert_eq!(
failure["data"]["diagnostic"]["code"],
"request_profile_release_unavailable"
);
assert_eq!(
failure["data"]["diagnostic"]["message"],
"Profile release unavailable"
);
assert!(
events
.iter()
.all(|event| event["stage"] != "attempt.started")
);
}
#[test]
fn first_resolved_profile_is_frozen_across_account_retries() {
use gateway_core::account::OpaqueProviderData;
@@ -9,6 +9,91 @@ use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHead
use gateway_core::upstream::UpstreamSendState;
use serde_json::json;
#[test]
fn source_chain_survives_domain_conversion_and_snapshot_without_entering_formatting() {
use gateway_core::error::{StoreError, StoreErrorKind};
use gateway_core::provider_ports::{ProviderStoreError, ProviderStoreErrorKind};
use std::error::Error as _;
let store = StoreError::caused_by(
StoreErrorKind::Unavailable,
std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "PRIVATE_DB_CAUSE"),
);
let port =
ProviderStoreError::caused_by(ProviderStoreErrorKind::Unavailable, "select account", store);
let error = ProviderError::new(
ProviderErrorKind::ProviderInfrastructureUnavailable,
UpstreamSendState::NotSent,
)
.with_source(port);
let snapshot = error.stable_snapshot();
drop(error);
let port = snapshot
.source()
.unwrap()
.downcast_ref::<ProviderStoreError>()
.unwrap();
let store = port.source().unwrap().downcast_ref::<StoreError>().unwrap();
let original = store
.source()
.unwrap()
.downcast_ref::<std::io::Error>()
.unwrap();
assert_eq!(original.kind(), std::io::ErrorKind::ConnectionRefused);
assert_eq!(original.to_string(), "PRIVATE_DB_CAUSE");
assert!(
!format!("{snapshot:?} {snapshot} {port:?} {port} {store:?} {store}")
.contains("PRIVATE_DB_CAUSE")
);
assert_eq!(snapshot.send_state(), UpstreamSendState::NotSent);
}
#[test]
fn restricted_details_preserve_causes_and_verbatim_upstream_body_separately() {
let body = "{ \"error\": {\"code\":\"Vendor.Unknown\",\"message\":\"PRIVATE_BODY\"} }";
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
.with_source(std::io::Error::other("PRIVATE_CAUSE"))
.with_raw_upstream_error(RawUpstreamError::new(body));
let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap();
assert_eq!(details["upstream"], body);
assert_eq!(details["causes"]["messages"], json!(["PRIVATE_CAUSE"]));
assert_eq!(details["causes"]["truncated"], false);
assert!(!format!("{error:?} {error}").contains("PRIVATE_"));
assert!(
ProviderError::new(ProviderErrorKind::Cancelled, UpstreamSendState::NotSent)
.error_details()
.is_none()
);
}
#[test]
fn restricted_cause_snapshot_bounds_unicode_and_cycles_with_an_explicit_marker() {
let error = ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::NotSent)
.with_source(std::io::Error::other("原始原因".repeat(20_000)));
let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap();
assert_eq!(details["causes"]["truncated"], true);
assert!(details["causes"]["messages"][0].as_str().unwrap().len() <= 64 * 1024);
#[derive(Debug)]
struct CyclicError;
impl std::fmt::Display for CyclicError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("cyclic source")
}
}
impl std::error::Error for CyclicError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(self)
}
}
let error = ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::NotSent)
.with_source(CyclicError);
let details: serde_json::Value = serde_json::from_str(&error.error_details().unwrap()).unwrap();
assert_eq!(details["causes"]["truncated"], true);
assert_eq!(details["causes"]["messages"].as_array().unwrap().len(), 32);
}
#[test]
fn provider_error_debug_should_not_expose_sensitive_context() {
let secret = "sk-do-not-log-this";
@@ -55,6 +140,28 @@ fn classified_provider_diagnostic_survives_stable_snapshot_without_entering_debu
assert!(!format!("{gateway:?}").contains(message));
}
#[test]
fn io_diagnostic_keeps_os_cause_without_copying_custom_error_text() {
let os_error = std::fs::File::open(
std::env::temp_dir().join(format!("cpr-missing-ca-{}/missing.pem", std::process::id())),
)
.unwrap_err();
let diagnostic = ProviderDiagnostic::new("CA read failed")
.with_classification("prepare", "custom_ca_read_failed")
.with_io_cause(&os_error);
assert!(diagnostic.as_str().contains(&os_error.to_string()));
assert_eq!(diagnostic.stage(), Some("prepare"));
assert_eq!(diagnostic.code(), Some("io_not_found"));
let private = std::io::Error::new(
std::io::ErrorKind::ConnectionRefused,
"PRIVATE_URL_AND_CREDENTIAL",
);
let diagnostic = ProviderDiagnostic::new("Connect failed").with_io_cause(&private);
assert_eq!(diagnostic.code(), Some("connection_refused"));
assert!(!diagnostic.as_str().contains("PRIVATE_"));
}
#[test]
fn raw_upstream_error_survives_stable_snapshot_without_entering_debug() {
let raw = r#"{"error":{"message":"verbatim upstream marker"}}"#;
@@ -1,24 +1,22 @@
//! 校验并解码插件上游事件,转换为宿主执行事件与错误
//! 校验插件流式事件,生成领域事件并保持序列约束
use std::time::Duration;
use super::failure::{failure_error, invalid};
use bytes::Bytes;
use gateway_core::{
engine::upstream_adapter::UpstreamAccountConnection,
error::{ClientVisibleUpstreamError, ProviderError, ProviderErrorKind},
error::ProviderError,
event::{
ContentItem, ContentKind, EventSequenceValidator, FinishReason, GatewayEvent,
ProtocolWireEvent, ProviderEvent, ProviderResponseObservation, ReasoningDelta,
ResponseMeta, TextDelta, ToolCallDelta,
},
metering::Usage,
upstream::{OpaqueUpstreamValue, UpstreamSendState, UpstreamTransport},
upstream::{UpstreamSendState, UpstreamTransport},
};
use gateway_plugin_sdk::call::{
model::{CanonicalEvent, WireEvent, WirePayload},
upstream_adapter::{
UpstreamAdapterEvent, UpstreamContinuation, UpstreamFailure, UpstreamFailureKind,
},
upstream_adapter::{UpstreamAdapterEvent, UpstreamContinuation},
};
use gateway_protocol::openai::sse::SseEventDecoder;
@@ -57,7 +55,8 @@ impl EventDecoder {
account: &dyn UpstreamAccountConnection,
sent: UpstreamSendState,
) -> Result<DecodedEvent, EventDecodeError> {
let message = UpstreamAdapterEvent::decode(payload).map_err(|_| invalid(sent))?;
let message = UpstreamAdapterEvent::decode(payload)
.map_err(|source| invalid(sent).with_source(source))?;
if self.completed || message.event.facts.len() > 64 {
return Err(invalid(sent).into());
}
@@ -185,7 +184,9 @@ impl EventDecoder {
account.calculate_cost(self.service_tier.as_deref(), &self.usage)
{
let cost = GatewayEvent::CalculatedCost(cost);
self.sequence.observe(&cost).map_err(|_| invalid(sent))?;
self.sequence
.observe(&cost)
.map_err(|source| invalid(sent).with_source(source))?;
facts.push(cost);
}
self.completed = true;
@@ -204,7 +205,9 @@ impl EventDecoder {
}))
}
};
self.sequence.observe(&fact).map_err(|_| invalid(sent))?;
self.sequence
.observe(&fact)
.map_err(|source| invalid(sent).with_source(source))?;
facts.push(fact);
}
if message.continuation.is_some() && !self.completed {
@@ -219,10 +222,11 @@ impl EventDecoder {
};
if let Some(tier) = &self.service_tier {
let observation = ProviderResponseObservation::new(
UpstreamTransport::new(transport).map_err(|_| invalid(sent))?,
UpstreamTransport::new(transport)
.map_err(|source| invalid(sent).with_source(source))?,
)
.try_with_service_tier(tier.clone())
.map_err(|_| invalid(sent))?;
.map_err(|source| invalid(sent).with_source(source))?;
event.attach_observation(observation);
}
Ok(DecodedEvent {
@@ -232,7 +236,9 @@ impl EventDecoder {
}
pub(super) fn finish(&self, sent: UpstreamSendState) -> Result<(), ProviderError> {
self.sequence.finish().map_err(|_| invalid(sent))
self.sequence
.finish()
.map_err(|source| invalid(sent).with_source(source))
}
}
@@ -307,14 +313,15 @@ fn decode_wire(
ProtocolWireEvent::raw_sse(protocol, Bytes::from(frame))
}
WirePayload::RawJson { body } => {
serde_json::from_slice::<serde_json::Value>(&body).map_err(|_| invalid(sent))?;
serde_json::from_slice::<serde_json::Value>(&body)
.map_err(|source| invalid(sent).with_source(source))?;
ProtocolWireEvent::raw_json(protocol, Bytes::from(body))
}
WirePayload::RawBody { body } => {
ProtocolWireEvent::raw_http_body(protocol, Bytes::from(body))
}
};
result.map_err(|_| invalid(sent))
result.map_err(|source| invalid(sent).with_source(source))
}
fn single_sse(
@@ -327,51 +334,3 @@ fn single_sse(
}
frames.pop().ok_or_else(|| invalid(sent))
}
fn failure_error(
failure: UpstreamFailure,
sent: UpstreamSendState,
) -> Result<ProviderError, ProviderError> {
if failure.message.len() > 64 * 1024
|| failure.code.as_ref().is_some_and(|code| code.len() > 256)
|| failure
.status
.is_some_and(|status| !(400..=599).contains(&status))
|| failure
.retry_after_ms
.is_some_and(|delay| delay > 24 * 60 * 60 * 1000)
{
return Err(invalid(sent));
}
let kind = match failure.kind {
UpstreamFailureKind::InvalidRequest => ProviderErrorKind::InvalidRequest,
UpstreamFailureKind::Unsupported => ProviderErrorKind::Unsupported,
UpstreamFailureKind::Unauthorized => ProviderErrorKind::Unauthorized,
UpstreamFailureKind::PermissionDenied => ProviderErrorKind::PermissionDenied,
UpstreamFailureKind::RateLimited => ProviderErrorKind::RateLimited,
UpstreamFailureKind::QuotaExhausted => ProviderErrorKind::QuotaExhausted,
UpstreamFailureKind::Timeout => ProviderErrorKind::Timeout,
UpstreamFailureKind::Unavailable => ProviderErrorKind::Unavailable,
UpstreamFailureKind::Protocol => ProviderErrorKind::Protocol,
};
let mut error = ProviderError::new(kind, sent);
if let Some(code) = &failure.code {
error = error.with_upstream_code(OpaqueUpstreamValue::new(code.clone()));
}
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
failure.message,
failure.code,
None,
));
if let Some(status) = failure.status {
error = error.with_status(status);
}
if let Some(delay) = failure.retry_after_ms {
error = error.with_retry_after(Duration::from_millis(delay));
}
Ok(error)
}
pub(super) fn invalid(sent: UpstreamSendState) -> ProviderError {
ProviderError::new(ProviderErrorKind::Protocol, sent)
}
@@ -10,7 +10,7 @@ use gateway_core::{
upstream::UpstreamSendState,
};
use gateway_plugin_sdk::{
CallContext, ErrorCode, PluginFault, Stage,
CallContext, Stage,
call::upstream_adapter::{
ContinuationScope, EXECUTE_METHOD, UpstreamAdapterRequest, UpstreamContinuation,
UpstreamTransport,
@@ -20,10 +20,11 @@ use serde::{Deserialize, Serialize};
use super::{
PluginUpstreamAdapter,
event::{DecodedEvent, EventDecodeError, EventDecoder, invalid},
event::{DecodedEvent, EventDecodeError, EventDecoder},
failure::{fault_error, invalid, rpc_error},
};
use crate::{
RpcError, RpcStream,
RpcStream,
callback::{
CallbackScope,
upstream::{ConnectionOwner, ManagedUpstream},
@@ -240,10 +241,10 @@ impl Execution {
invocation
.operation
.middleware_body()
.map_err(|_| invalid())?
.map_err(|source| invalid().with_source(source))?
.to_vec(),
)
.map_err(|_| invalid())?;
.map_err(|source| invalid().with_source(source))?;
invocation.context.trace().record("plugin.upstream", serde_json::json!({ "instanceId": self.adapter.instance_id, "adapterId": declaration.id, "generation": context.generation, "transport": declaration.transport.as_str() }));
let stream = tokio::select! {
biased;
@@ -252,7 +253,7 @@ impl Execution {
};
if stream.initial.result != serde_json::json!({}) || !stream.initial.payload.is_empty() {
self.adapter.session.invalid_response(Stage::Upstream);
return Err(super::event::invalid(managed.send_state.get()));
return Err(super::failure::invalid(managed.send_state.get()));
}
Ok(ActiveCall {
context,
@@ -292,7 +293,7 @@ fn previous_continuation(
}
let stored: StoredContinuation =
serde_json::from_value(serde_json::Value::Object(state.payload().clone()))
.map_err(|_| invalid())?;
.map_err(|source| invalid().with_source(source))?;
if stored.client_key_id != owner.client_key_id
|| stored.account_id != owner.account_id
|| stored.credential_revision != owner.credential_revision
@@ -325,7 +326,7 @@ async fn attach_continuation(
if continuation.upstream_response_id.is_empty()
|| continuation.upstream_response_id.len() > 512
|| serde_json::to_vec(&continuation)
.map_err(|_| invalid())?
.map_err(|source| invalid().with_source(source))?
.len()
> 32 * 1024
{
@@ -350,12 +351,13 @@ async fn attach_continuation(
account_id: invocation.account.account_id().as_str().to_owned(),
credential_revision: invocation.account.credential_revision().get(),
};
let serde_json::Value::Object(payload) = serde_json::to_value(state).map_err(|_| invalid())?
let serde_json::Value::Object(payload) =
serde_json::to_value(state).map_err(|source| invalid().with_source(source))?
else {
return Err(invalid());
};
let state = ProviderSessionState::new(adapter.declaration.provider.as_str(), payload)
.map_err(|_| invalid())?
.map_err(|source| invalid().with_source(source))?
.with_extension_owner(extension_owner(adapter, &active.context, local));
decoded.event.attach_session_update(state);
Ok(())
@@ -375,25 +377,3 @@ fn extension_owner(
connection_local,
}
}
fn rpc_error(error: RpcError, sent: UpstreamSendState) -> ProviderError {
match error {
RpcError::Remote(fault) => fault_error(fault, sent),
RpcError::Timeout => ProviderError::new(ProviderErrorKind::Timeout, sent),
RpcError::Cancelled => ProviderError::new(ProviderErrorKind::Cancelled, sent),
_ => ProviderError::new(ProviderErrorKind::Unavailable, sent),
}
}
fn fault_error(fault: PluginFault, sent: UpstreamSendState) -> ProviderError {
let kind = match fault.code {
ErrorCode::Timeout => ProviderErrorKind::Timeout,
ErrorCode::Cancelled => ProviderErrorKind::Cancelled,
ErrorCode::Unsupported => ProviderErrorKind::Unsupported,
ErrorCode::InvalidInput | ErrorCode::Rejected | ErrorCode::PermissionDenied => {
ProviderErrorKind::InvalidRequest
}
_ => ProviderErrorKind::Unavailable,
};
ProviderError::new(kind, sent)
}
@@ -0,0 +1,115 @@
//! 插件上游适配的唯一错误投影边界,保留来源并生成安全分类
use std::time::Duration;
use crate::RpcError;
use gateway_core::{
error::{
ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic, ProviderError,
ProviderErrorKind, RawUpstreamError,
},
upstream::UpstreamSendState,
};
use gateway_plugin_sdk::{
ErrorCode, PluginFault,
call::upstream_adapter::{UpstreamFailure, UpstreamFailureKind},
};
pub(super) fn rpc_error(error: RpcError, sent: UpstreamSendState) -> ProviderError {
let (code, message) = error.diagnostic();
let kind = match error {
RpcError::Remote(fault) => return fault_error(fault, sent),
RpcError::Timeout => ProviderErrorKind::Timeout,
RpcError::Cancelled => ProviderErrorKind::Cancelled,
_ => ProviderErrorKind::Unavailable,
};
ProviderError::new(kind, sent)
.with_diagnostic(ProviderDiagnostic::new(message).with_classification("plugin_rpc", code))
.with_source(error)
}
pub(super) fn fault_error(fault: PluginFault, sent: UpstreamSendState) -> ProviderError {
let kind = match fault.code {
ErrorCode::Timeout => ProviderErrorKind::Timeout,
ErrorCode::Cancelled => ProviderErrorKind::Cancelled,
ErrorCode::Unsupported => ProviderErrorKind::Unsupported,
ErrorCode::InvalidInput | ErrorCode::Rejected | ErrorCode::PermissionDenied => {
ProviderErrorKind::InvalidRequest
}
_ => ProviderErrorKind::Unavailable,
};
// 仅保存响应方向的插件 fault,普通日志与公共错误继续使用宿主分类
let raw = serde_json::json!({"source": "plugin_upstream_adapter", "fault": fault}).to_string();
ProviderError::new(kind, sent)
.with_diagnostic(
ProviderDiagnostic::new(format!(
"Plugin upstream adapter returned a fault: {:?}",
fault.code,
))
.with_classification("plugin_rpc", "remote"),
)
.with_raw_upstream_error(RawUpstreamError::new(raw))
}
pub(super) fn failure_error(
failure: UpstreamFailure,
sent: UpstreamSendState,
) -> Result<ProviderError, ProviderError> {
if failure.message.len() > 64 * 1024
|| failure.code.as_ref().is_some_and(|code| code.len() > 256)
|| failure
.status
.is_some_and(|status| !(400..=599).contains(&status))
|| failure
.retry_after_ms
.is_some_and(|delay| delay > 24 * 60 * 60 * 1000)
{
return Err(invalid(sent));
}
let kind = match failure.kind {
UpstreamFailureKind::InvalidRequest => ProviderErrorKind::InvalidRequest,
UpstreamFailureKind::Unsupported => ProviderErrorKind::Unsupported,
UpstreamFailureKind::Unauthorized => ProviderErrorKind::Unauthorized,
UpstreamFailureKind::PermissionDenied => ProviderErrorKind::PermissionDenied,
UpstreamFailureKind::RateLimited => ProviderErrorKind::RateLimited,
UpstreamFailureKind::QuotaExhausted => ProviderErrorKind::QuotaExhausted,
UpstreamFailureKind::Timeout => ProviderErrorKind::Timeout,
UpstreamFailureKind::Unavailable => ProviderErrorKind::Unavailable,
UpstreamFailureKind::Protocol => ProviderErrorKind::Protocol,
};
// 插件已经解析的上游错误与客户端投影分别承载,消费 message 前保存原始返回
let raw =
serde_json::to_string(&failure).map_err(|source| invalid(sent).with_source(source))?;
let mut error = ProviderError::new(kind, sent)
.with_raw_upstream_error(RawUpstreamError::new(raw))
.with_diagnostic(
ProviderDiagnostic::new(format!(
"Plugin upstream failure: kind={}, status={:?}",
kind.as_str(),
failure.status,
))
.with_classification("upstream", "plugin_upstream_failure"),
);
if let Some(code) = &failure.code {
error = error.with_upstream_code(OpaqueUpstreamValue::new(code.clone()));
}
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
failure.message,
failure.code,
None,
));
if let Some(status) = failure.status {
error = error.with_status(status);
}
if let Some(delay) = failure.retry_after_ms {
error = error.with_retry_after(Duration::from_millis(delay));
}
Ok(error)
}
pub(super) fn invalid(sent: UpstreamSendState) -> ProviderError {
ProviderError::new(ProviderErrorKind::Protocol, sent).with_diagnostic(
ProviderDiagnostic::new("Plugin upstream adapter returned an invalid event")
.with_classification("plugin_protocol", "invalid_event"),
)
}
@@ -2,6 +2,7 @@
mod event;
mod execution;
mod failure;
mod registration;
pub(crate) mod target;
@@ -69,6 +69,44 @@ pub enum RpcError {
Remote(PluginFault),
}
impl RpcError {
/// 会话状态与请求错误共用安全分类,不展开插件返回正文
pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) {
match self {
RpcError::Start(_) => ("process_start", "插件进程启动失败"),
RpcError::Handshake => ("handshake", "插件握手失败"),
RpcError::Protocol => ("protocol", "插件协议错误"),
RpcError::InvalidResponse(stage) => (
"invalid_response",
match stage {
Stage::Registration => "注册阶段:插件返回了无效响应",
Stage::Configuration => "配置阶段:插件返回了无效响应",
Stage::Authentication => "认证阶段:插件返回了无效响应",
Stage::Routing => "路由阶段:插件返回了无效响应",
Stage::Scheduling => "调度阶段:插件返回了无效响应",
Stage::Retry => "重试阶段:插件返回了无效响应",
Stage::Http => "HTTP 中间件:插件返回了无效响应",
Stage::Service => "服务中间件:插件返回了无效响应",
Stage::WebSocket => "WebSocket 中间件:插件返回了无效响应",
Stage::Request => "请求中间件:插件返回了无效响应",
Stage::Attempt => "尝试中间件:插件返回了无效响应",
Stage::Upstream => "上游适配阶段:插件返回了无效响应",
Stage::Observation => "观察阶段:插件返回了无效响应",
Stage::Management | Stage::PublicManagement => "管理阶段:插件返回了无效响应",
Stage::CommandLine => "命令行阶段:插件返回了无效响应",
Stage::Maintenance => "维护阶段:插件返回了无效响应",
},
),
RpcError::Closed => ("closed", "插件进程或传输已停止"),
RpcError::Timeout => ("timeout", "插件调用超时"),
RpcError::Cancelled => ("cancelled", "插件会话已取消"),
RpcError::Capacity => ("capacity", "插件调用容量已耗尽"),
RpcError::Context => ("context", "插件调用上下文无效"),
RpcError::Remote(_) => ("remote", "插件返回错误"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RpcSessionDiagnostic {
Ready,
@@ -657,40 +695,7 @@ impl RpcSession {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(error) = &state.failure {
let (code, message) = match error {
RpcError::Start(_) => ("process_start", "插件进程启动失败"),
RpcError::Handshake => ("handshake", "插件握手失败"),
RpcError::Protocol => ("protocol", "插件协议错误"),
RpcError::InvalidResponse(stage) => (
"invalid_response",
match stage {
Stage::Registration => "注册阶段:插件返回了无效响应",
Stage::Configuration => "配置阶段:插件返回了无效响应",
Stage::Authentication => "认证阶段:插件返回了无效响应",
Stage::Routing => "路由阶段:插件返回了无效响应",
Stage::Scheduling => "调度阶段:插件返回了无效响应",
Stage::Retry => "重试阶段:插件返回了无效响应",
Stage::Http => "HTTP 中间件:插件返回了无效响应",
Stage::Service => "服务中间件:插件返回了无效响应",
Stage::WebSocket => "WebSocket 中间件:插件返回了无效响应",
Stage::Request => "请求中间件:插件返回了无效响应",
Stage::Attempt => "尝试中间件:插件返回了无效响应",
Stage::Upstream => "上游适配阶段:插件返回了无效响应",
Stage::Observation => "观察阶段:插件返回了无效响应",
Stage::Management | Stage::PublicManagement => {
"管理阶段:插件返回了无效响应"
}
Stage::CommandLine => "命令行阶段:插件返回了无效响应",
Stage::Maintenance => "维护阶段:插件返回了无效响应",
},
),
RpcError::Closed => ("closed", "插件进程或传输已停止"),
RpcError::Timeout => ("timeout", "插件调用超时"),
RpcError::Cancelled => ("cancelled", "插件会话已取消"),
RpcError::Capacity => ("capacity", "插件调用容量已耗尽"),
RpcError::Context => ("context", "插件调用上下文无效"),
RpcError::Remote(_) => ("remote", "插件返回错误"),
};
let (code, message) = error.diagnostic();
return RpcSessionDiagnostic::Failed { code, message };
}
if self.slots.is_closed() {
@@ -378,6 +378,15 @@ async fn adapter_failure_preserves_upstream_diagnostics_and_native_feedback() {
"rate_limit_exceeded"
);
assert_eq!(error.retry_after(), Some(Duration::from_millis(1500)));
let snapshot = error.stable_snapshot();
let raw: Value = serde_json::from_str(snapshot.raw_upstream_error().unwrap().as_str()).unwrap();
assert_eq!(raw["message"], "fixture rate limit");
assert_eq!(raw["code"], "rate_limit_exceeded");
assert_eq!(raw["status"], 429);
let diagnostic = snapshot.diagnostic().unwrap();
assert_eq!(diagnostic.code(), Some("plugin_upstream_failure"));
assert!(!diagnostic.as_str().contains("fixture rate limit"));
assert!(!format!("{snapshot:?}").contains("fixture rate limit"));
assert!(generation.is_ready(), "上游 429 不能停止插件");
assert!(generation.can_serve());
assert_eq!(account.failures.load(Ordering::SeqCst), 1);
@@ -464,6 +473,18 @@ async fn adapter_does_not_publish_completed_when_rpc_fails_after_terminal_frame(
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
assert_eq!(error.send_state(), UpstreamSendState::Sent);
let mut error = error;
let raw: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap();
assert_eq!(raw["source"], "plugin_upstream_adapter");
assert_eq!(raw["fault"]["message"], "fixture terminal fault");
assert_eq!(error.diagnostic().unwrap().stage(), Some("plugin_rpc"));
assert!(
!error
.diagnostic()
.unwrap()
.as_str()
.contains("fixture terminal fault")
);
assert!(!format!("{error:?}").contains("fixture terminal fault"));
let facts = error
.take_atomic_client_events()
.into_iter()
@@ -960,38 +981,56 @@ async fn upstream_adapter_can_dispatch_http_after_its_instance_entered_the_scope
#[tokio::test]
async fn malformed_adapter_events_stop_only_the_plugin_session() {
use gateway_admin::ports::plugins::PluginRuntimeDiagnostics as _;
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_body_string("{}"))
.mount(&server)
.await;
let mut config = configuration(&server.uri(), false);
config["upstream_events"] = json!([{
"event":{"facts":[{"type":"completed","id":"never-started","model":"native-model","reason":"stop"}]}
}]);
let (cache, store, runtime) = setup(config).await;
let generation = ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
.await
.unwrap();
let mut stream = execute(&generation, Account::new(), "key-one", None);
assert_eq!(
stream.next().await.unwrap().unwrap_err().kind(),
ProviderErrorKind::Protocol
);
assert!(!generation.is_ready());
assert!(generation.can_serve());
let snapshot = store.snapshot.lock().unwrap().clone();
assert!(snapshot.instances[0].enabled);
let diagnostics = runtime
.runtime_diagnostics(&snapshot, Some(1), Some(&generation))
.await
.unwrap();
assert_eq!(
diagnostics["instance-one"].failure.as_ref().unwrap().code,
"invalid_response"
);
drop(stream);
drop(generation);
runtime.shutdown().await;
super::wait_until_empty(cache.path()).await;
use std::error::Error as _;
for (fact, has_sequence_error) in [
(
json!({"type":"completed","id":"never-started","model":"native-model","reason":"stop"}),
false,
),
(
json!({"type":"text_delta","index":0,"text":"delta before start"}),
true,
),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_body_string("{}"))
.mount(&server)
.await;
let mut config = configuration(&server.uri(), false);
config["upstream_events"] = json!([{"event":{"facts":[fact]}}]);
let (cache, store, runtime) = setup(config).await;
let generation =
ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
.await
.unwrap();
let mut stream = execute(&generation, Account::new(), "key-one", None);
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.kind(), ProviderErrorKind::Protocol);
if has_sequence_error {
assert!(
error
.source()
.unwrap()
.is::<gateway_core::event::EventSequenceError>()
);
}
assert_eq!(error.diagnostic().unwrap().code(), Some("invalid_event"));
assert!(!generation.is_ready());
assert!(generation.can_serve());
let snapshot = store.snapshot.lock().unwrap().clone();
assert!(snapshot.instances[0].enabled);
let diagnostics = runtime
.runtime_diagnostics(&snapshot, Some(1), Some(&generation))
.await
.unwrap();
assert_eq!(
diagnostics["instance-one"].failure.as_ref().unwrap().code,
"invalid_response"
);
drop(stream);
drop(generation);
runtime.shutdown().await;
super::wait_until_empty(cache.path()).await;
}
}
@@ -24,15 +24,16 @@ impl StagingArea {
///
/// 目录创建或权限设置失败时返回 [`StoreError`]
pub fn open(base_dir: PathBuf, max_archive_bytes: u64) -> StoreResult<Self> {
std::fs::create_dir_all(&base_dir).map_err(|_| unavailable("create staging directory"))?;
std::fs::create_dir_all(&base_dir)
.map_err(|source| unavailable("create staging directory", source))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt as _;
std::fs::set_permissions(&base_dir, std::fs::Permissions::from_mode(0o700))
.map_err(|_| unavailable("set staging directory permissions"))?;
.map_err(|source| unavailable("set staging directory permissions", source))?;
}
let metadata = std::fs::metadata(&base_dir)
.map_err(|_| unavailable("read staging directory metadata"))?;
.map_err(|source| unavailable("read staging directory metadata", source))?;
if !metadata.is_dir() {
return Err(invalid("staging path is not a directory"));
}
@@ -70,7 +71,7 @@ impl StagingArea {
/// 剩余空间不足或读取失败时返回 [`crate::StoreError`]
pub fn ensure_capacity(&self) -> StoreResult<()> {
let free = fs2::available_space(&self.base_dir)
.map_err(|_| unavailable("read staging free space"))?;
.map_err(|source| unavailable("read staging free space", source))?;
if free < MIN_STAGING_FREE_BYTES {
return Err(StoreError::InvalidData {
entity: "backup staging",
@@ -93,8 +94,11 @@ impl StagingArea {
}
}
fn unavailable(operation: &'static str) -> StoreError {
crate::postgres_unavailable(operation)
fn unavailable(
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> StoreError {
crate::postgres_unavailable(operation, source)
}
fn invalid(message: &str) -> StoreError {
+2 -2
View File
@@ -113,11 +113,11 @@ async fn connect(
config.pool.acquire_timeout(),
)?;
let redis_client = ::redis::Client::open(config.redis_url()?)
.map_err(|_| redis_unavailable("create Redis client"))?;
.map_err(|source| redis_unavailable("create Redis client", source))?;
let redis_connection = redis_client
.get_connection_manager()
.await
.map_err(|_| redis_unavailable("connect Redis manager"))?;
.map_err(|source| redis_unavailable("connect Redis manager", source))?;
let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone()));
let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new(
@@ -90,10 +90,9 @@ impl PgAccountGroupRepository {
where
F: for<'a> FnOnce(&'a mut Transaction<'_, Postgres>) -> BoxFuture<'a, StoreResult<()>>,
{
let mut transaction =
self.pool.begin().await.map_err(|_| {
admin_store_error(ENTITY, unavailable("begin account group mutation"))
})?;
let mut transaction = self.pool.begin().await.map_err(|source| {
admin_store_error(ENTITY, unavailable("begin account group mutation", source))
})?;
let result = async {
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
mutation(&mut transaction).await?;
@@ -103,14 +102,17 @@ impl PgAccountGroupRepository {
.await;
match result {
Ok(revision) => {
transaction.commit().await.map_err(|_| {
admin_store_error(ENTITY, unavailable("commit account group mutation"))
transaction.commit().await.map_err(|source| {
admin_store_error(ENTITY, unavailable("commit account group mutation", source))
})?;
admin_revision(revision)
}
Err(error) => {
transaction.rollback().await.map_err(|_| {
admin_store_error(ENTITY, unavailable("rollback account group mutation"))
transaction.rollback().await.map_err(|source| {
admin_store_error(
ENTITY,
unavailable("rollback account group mutation", source),
)
})?;
Err(admin_store_error(ENTITY, error))
}
@@ -139,7 +141,9 @@ impl AccountGroupStore for PgAccountGroupRepository {
.build()
.fetch_all(&self.pool)
.await
.map_err(|_| admin_store_error(ENTITY, unavailable("list account groups")))?;
.map_err(|source| {
admin_store_error(ENTITY, unavailable("list account groups", source))
})?;
let mut items = rows
.iter()
.map(group_record)
@@ -201,7 +205,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
.bind(group_ids)
.fetch_all(&self.pool)
.await
.map_err(|_| admin_store_error(ENTITY, unavailable("load account group members")))?;
.map_err(|source| admin_store_error(ENTITY, unavailable("load account group members", source)))?;
rows.into_iter()
.map(|row| {
let group_id = AccountGroupId::new(
@@ -336,7 +340,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
.bind(command.enabled)
.execute(&mut **transaction)
.await
.map_err(|_| unavailable("set account group state"))?;
.map_err(|source| unavailable("set account group state", source))?;
require_one(result.rows_affected(), command.id.as_str())
})
})
@@ -373,7 +377,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
.bind(command.id.as_str())
.execute(&mut **transaction)
.await
.map_err(|_| unavailable("delete account group"))?;
.map_err(|source| unavailable("delete account group", source))?;
if result.rows_affected() == 1 {
return Ok(());
}
@@ -383,7 +387,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
.bind(command.id.as_str())
.fetch_one(&mut **transaction)
.await
.map_err(|_| unavailable("check account group delete conflict"))?;
.map_err(|source| unavailable("check account group delete conflict", source))?;
if exists {
Err(conflict(command.id.as_str()))
} else {
@@ -454,7 +458,7 @@ async fn count_groups(pool: &PgPool, query: &AccountGroupListQuery) -> StoreResu
.build_query_scalar::<i64>()
.fetch_one(pool)
.await
.map_err(|_| unavailable("count account groups"))?;
.map_err(|source| unavailable("count account groups", source))?;
u64::try_from(count).map_err(|_| invalid("negative account group count"))
}
@@ -466,7 +470,7 @@ async fn load_record(pool: &PgPool, id: &str) -> StoreResult<Option<AccountGroup
.build()
.fetch_optional(pool)
.await
.map_err(|_| unavailable("load account group"))?
.map_err(|source| unavailable("load account group", source))?
.as_ref()
.map(group_record)
.transpose()
@@ -571,7 +575,7 @@ async fn group_costs(
)
.fetch_all(pool)
.await
.map_err(|_| unavailable("load account group costs"))?;
.map_err(|source| unavailable("load account group costs", source))?;
rows.into_iter()
.map(|row| {
let group_id: String = row
@@ -644,7 +648,7 @@ fn map_group_write_error(error: sqlx::Error, id: &str) -> StoreError {
{
conflict(id)
} else {
unavailable("write account group")
unavailable("write account group", error)
}
}
@@ -678,8 +682,11 @@ fn invalid(message: &str) -> StoreError {
}
}
fn unavailable(message: &'static str) -> StoreError {
postgres_unavailable(message)
fn unavailable(
message: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> StoreError {
postgres_unavailable(message, source)
}
/// 原生管理与插件自有分组共用字段校验和写入规则
@@ -120,7 +120,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
.bind(admin_user_id)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("read admin password hash"))
.map_err(|source| postgres_unavailable("read admin password hash", source))
}
async fn change_password(
@@ -136,7 +136,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin password change"))?;
.map_err(|source| postgres_unavailable("begin password change", source))?;
let changed = sqlx::query(
"update admin_users set password_hash = $3, updated_at = now()
where id = $1 and password_hash = $2",
@@ -146,7 +146,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
.bind(password_hash)
.execute(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("change admin password"))?
.map_err(|source| postgres_unavailable("change admin password", source))?
.rows_affected()
== 1;
if !changed {
@@ -156,7 +156,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit password change"))?;
.map_err(|source| postgres_unavailable("commit password change", source))?;
Ok(true)
}
@@ -176,7 +176,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
.bind(password_hash)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("create admin password hash"))?;
.map_err(|source| postgres_unavailable("create admin password hash", source))?;
Ok(result.rows_affected() == 1)
}
@@ -201,7 +201,7 @@ impl AdminSecurityAuditRepository for PgAdminSecurityAuditRepository {
.bind(event.created_at)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("append admin audit event"))?;
.map_err(|source| postgres_unavailable("append admin audit event", source))?;
Ok(())
}
}
@@ -238,7 +238,7 @@ pub(crate) async fn append_admin_audit_event_in_transaction(
.bind(event.created_at)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("append admin audit event in transaction"))?;
.map_err(|source| postgres_unavailable("append admin audit event in transaction", source))?;
Ok(())
}
@@ -72,7 +72,7 @@ impl ClientAdmissionRecoveryRepository for PgClientAdmissionRecoveryRepository {
.bind(window_started_at)
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("load client admission recovery"))?;
.map_err(|source| postgres_unavailable("load client admission recovery", source))?;
let mut recoveries = BTreeMap::<String, ClientAdmissionRecovery>::new();
for (client_api_key_ref, model_request_id, started_at, deadline_at, outcome) in rows {
let recovery = recoveries
@@ -841,7 +841,7 @@ async fn append_audit_in_transaction(
.bind(event.created_at)
.execute(&mut **transaction)
.await
.map_err(|_| crate::postgres_unavailable("append backup audit event"))?;
.map_err(|source| crate::postgres_unavailable("append backup audit event", source))?;
Ok(())
}
@@ -30,10 +30,10 @@ pub(super) async fn reset_client_key_budget(
context: &MutationContext,
) -> AdminStoreResult<()> {
let mut tx = match &origin {
ClientKeyBudgetMutationOrigin::Admin => pool.begin().await.map_err(|_| {
ClientKeyBudgetMutationOrigin::Admin => pool.begin().await.map_err(|source| {
crate::admin_store_error(
"client API key budget",
postgres_unavailable("begin budget reset"),
postgres_unavailable("begin budget reset", source),
)
})?,
ClientKeyBudgetMutationOrigin::Plugin(owner) => {
@@ -43,10 +43,10 @@ pub(super) async fn reset_client_key_budget(
reset_client_key_budget_in_transaction(&mut tx, &command, context)
.await
.map_err(|error| crate::admin_store_error("client API key budget", error))?;
tx.commit().await.map_err(|_| {
tx.commit().await.map_err(|source| {
crate::admin_store_error(
"client API key budget",
postgres_unavailable("commit budget reset"),
postgres_unavailable("commit budget reset", source),
)
})
}
@@ -62,7 +62,7 @@ async fn reset_client_key_budget_in_transaction(
.bind(command.id.as_str())
.fetch_optional(&mut **tx)
.await
.map_err(|_| postgres_unavailable("lock budget reset key"))?;
.map_err(|source| postgres_unavailable("lock budget reset key", source))?;
if exists.is_none() {
return Err(StoreError::NotFound {
entity: "client API key",
@@ -95,7 +95,7 @@ async fn reset_client_key_budget_in_transaction(
.bind(reset_at)
.execute(&mut **tx)
.await
.map_err(|_| postgres_unavailable("reset client budget"))?;
.map_err(|source| postgres_unavailable("reset client budget", source))?;
let mut fields = Vec::new();
if daily {
fields.extend([
@@ -361,13 +361,13 @@ pub(super) async fn load_client_key_budgets(
.bind(ids)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load client budgets"))?;
.map_err(|source| postgres_unavailable("load client budgets", source))?;
let mut budgets = BTreeMap::new();
for row in rows {
let parse = |field| -> StoreResult<Decimal> {
row.get::<String, _>(field)
.parse()
.map_err(|_| postgres_unavailable("decode client budget"))
.map_err(|source| postgres_unavailable("decode client budget", source))
};
budgets.insert(
row.get::<String, _>("id"),
@@ -388,9 +388,14 @@ pub(super) async fn load_client_key_budgets(
);
}
for record in records {
record.budget = budgets
.remove(&record.id)
.ok_or_else(|| postgres_unavailable("load client budget policy"))?;
record.budget =
budgets
.remove(&record.id)
.ok_or_else(|| crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "load client budget policy".to_owned(),
source: None,
})?;
}
Ok(())
}
@@ -391,7 +391,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository {
.build()
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("list client API keys"))?;
.map_err(|source| postgres_unavailable("list client API keys", source))?;
let mut items = rows
.iter()
.map(client_record_from_row)
@@ -425,7 +425,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository {
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("reveal client API key"))?
.map_err(|source| postgres_unavailable("reveal client API key", source))?
.map(client_secret_from_row)
.transpose()
}
@@ -469,7 +469,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository {
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("get client API key"))?
.map_err(|source| postgres_unavailable("get client API key", source))?
.as_ref()
.map(client_record_from_row)
.transpose()?;
@@ -503,7 +503,7 @@ impl ClientApiKeyRepository for PgClientApiKeyRepository {
.bind(timestamps)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("touch client API keys"))?;
.map_err(|source| postgres_unavailable("touch client API keys", source))?;
Ok(result.rows_affected())
}
}
@@ -678,7 +678,12 @@ impl PgAdminClientKeyStore {
.bind(id.as_str())
.fetch_one(&self.keys.pool)
.await
.map_err(|_| admin_store_error(ENTITY, postgres_unavailable("read client key status")))
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("read client key status", source),
)
})
}
async fn revision(&self) -> AdminStoreResult<gateway_admin::model::Revision> {
@@ -718,12 +723,11 @@ impl ClientKeyStore for PgAdminClientKeyStore {
use gateway_admin::model::client_keys::ClientKeyBudgetMutationOrigin;
let map_error = |error| admin_store_error(ENTITY, error);
let mut tx = match &origin {
ClientKeyBudgetMutationOrigin::Admin => self
.keys
.pool
.begin()
.await
.map_err(|_| map_error(postgres_unavailable("begin budget limits update")))?,
ClientKeyBudgetMutationOrigin::Admin => {
self.keys.pool.begin().await.map_err(|source| {
map_error(postgres_unavailable("begin budget limits update", source))
})?
}
ClientKeyBudgetMutationOrigin::Plugin(owner) => {
super::plugins::begin_plugin_mutation(&self.keys.pool, owner).await?
}
@@ -732,23 +736,28 @@ impl ClientKeyStore for PgAdminClientKeyStore {
sqlx::query("select config_revision from runtime_settings where id=1 for update")
.execute(&mut *tx)
.await
.map_err(|_| map_error(postgres_unavailable("lock budget limits configuration")))?;
.map_err(|source| {
map_error(postgres_unavailable(
"lock budget limits configuration",
source,
))
})?;
let current: Option<(String, String)> = sqlx::query_as(
"select daily_limit_usd::text, weekly_limit_usd::text from client_api_keys where id=$1 for update"
).bind(command.id.as_str()).fetch_optional(&mut *tx).await
.map_err(|_| map_error(postgres_unavailable("lock budget limits key")))?;
.map_err(|source| map_error(postgres_unavailable("lock budget limits key", source)))?;
let (daily, weekly) = current.ok_or_else(|| {
map_error(StoreError::NotFound {
entity: ENTITY,
id: command.id.as_str().to_owned(),
})
})?;
let daily: Decimal = daily
.parse()
.map_err(|_| map_error(postgres_unavailable("decode daily budget limit")))?;
let weekly: Decimal = weekly
.parse()
.map_err(|_| map_error(postgres_unavailable("decode weekly budget limit")))?;
let daily: Decimal = daily.parse().map_err(|source| {
map_error(postgres_unavailable("decode daily budget limit", source))
})?;
let weekly: Decimal = weekly.parse().map_err(|source| {
map_error(postgres_unavailable("decode weekly budget limit", source))
})?;
let mut fields = Vec::new();
if command.daily_limit_usd.is_some_and(|value| value != daily) {
fields.push("daily_limit_usd".to_owned());
@@ -760,9 +769,12 @@ impl ClientKeyStore for PgAdminClientKeyStore {
fields.push("weekly_limit_usd".to_owned());
}
if fields.is_empty() {
tx.commit()
.await
.map_err(|_| map_error(postgres_unavailable("finish unchanged budget limits")))?;
tx.commit().await.map_err(|source| {
map_error(postgres_unavailable(
"finish unchanged budget limits",
source,
))
})?;
return Ok(None);
}
sqlx::query(
@@ -776,7 +788,7 @@ impl ClientKeyStore for PgAdminClientKeyStore {
.bind(command.weekly_limit_usd.map(|value| value.canonical()))
.execute(&mut *tx)
.await
.map_err(|_| map_error(postgres_unavailable("update budget limits")))?;
.map_err(|source| map_error(postgres_unavailable("update budget limits", source)))?;
let revision = super::bump_config_revision_in_transaction(&mut tx)
.await
.map_err(map_error)?;
@@ -792,9 +804,9 @@ impl ClientKeyStore for PgAdminClientKeyStore {
)
.await
.map_err(map_error)?;
tx.commit()
.await
.map_err(|_| map_error(postgres_unavailable("commit budget limits update")))?;
tx.commit().await.map_err(|source| {
map_error(postgres_unavailable("commit budget limits update", source))
})?;
Ok(Some(admin_revision(revision)?))
}
@@ -1152,7 +1164,7 @@ pub(crate) async fn insert_client_api_key_in_transaction(
kind: crate::ConflictKind::InvalidTransition,
}
} else {
postgres_unavailable("insert client API key in transaction")
postgres_unavailable("insert client API key in transaction", error)
}
})?;
replace_client_api_key_groups_in_transaction(transaction, &key.id, &key.group_ids).await?;
@@ -1203,7 +1215,7 @@ pub(crate) async fn update_client_api_key_in_transaction(
.bind(sqlx::types::Json(replacement_profiles))
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("update client API key in transaction"))?;
.map_err(|source| postgres_unavailable("update client API key in transaction", source))?;
require_changed(result.rows_affected(), &key.id)?;
replace_client_api_key_groups_in_transaction(transaction, &key.id, &key.group_ids).await
}
@@ -1223,7 +1235,7 @@ async fn ensure_client_key_name_available(
.bind(id)
.fetch_one(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("check client API key name"))?;
.map_err(|source| postgres_unavailable("check client API key name", source))?;
if duplicate {
return Err(StoreError::Conflict {
entity: ENTITY,
@@ -1246,7 +1258,9 @@ pub(crate) async fn set_client_api_key_enabled_in_transaction(
.bind(enabled)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("set client API key state in transaction"))?;
.map_err(|source| {
postgres_unavailable("set client API key state in transaction", source)
})?;
require_changed(result.rows_affected(), id)
}
@@ -1259,7 +1273,7 @@ pub(crate) async fn delete_client_api_key_in_transaction(
.bind(id)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("delete client API key in transaction"))?;
.map_err(|source| postgres_unavailable("delete client API key in transaction", source))?;
require_changed(result.rows_affected(), id)
}
@@ -1287,7 +1301,7 @@ async fn replace_client_api_key_groups_in_transaction(
.bind(group_ids)
.fetch_one(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("validate client API key groups"))?;
.map_err(|source| postgres_unavailable("validate client API key groups", source))?;
if usize::try_from(count).ok() != Some(group_ids.len()) {
return Err(StoreError::NotFound {
entity: "account group",
@@ -1299,7 +1313,7 @@ async fn replace_client_api_key_groups_in_transaction(
.bind(key_id)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("delete client API key groups"))?;
.map_err(|source| postgres_unavailable("delete client API key groups", source))?;
if !group_ids.is_empty() {
sqlx::query(
"insert into client_api_key_groups
@@ -1310,7 +1324,7 @@ async fn replace_client_api_key_groups_in_transaction(
.bind(group_ids)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("insert client API key groups"))?;
.map_err(|source| postgres_unavailable("insert client API key groups", source))?;
}
Ok(())
}
@@ -1444,7 +1458,7 @@ async fn count_client_api_keys(pool: &PgPool, search: Option<&str>) -> StoreResu
.build_query_scalar::<i64>()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("count client API keys"))?;
.map_err(|source| postgres_unavailable("count client API keys", source))?;
to_u64(count)
}
@@ -1504,7 +1518,7 @@ async fn load_client_key_memberships(
.bind(ids)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load client API key memberships"))?;
.map_err(|source| postgres_unavailable("load client API key memberships", source))?;
let mut groups = BTreeMap::<String, Vec<ClientApiKeyGroupRecord>>::new();
let mut providers = BTreeMap::<String, BTreeSet<String>>::new();
for row in rows {
@@ -1,5 +1,7 @@
//! 单行 `model_requests` 生命周期与最终 usage/cost 的 PostgreSQL owner
use crate::core_store_error;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde_json::Value;
@@ -289,7 +291,7 @@ pub struct ModelRequestFinalization {
pub error_kind: Option<String>,
pub provider_error_code: Option<String>,
pub error_message: Option<String>,
pub raw_upstream_error: Option<String>,
pub error_details: Option<String>,
pub continuation_unavailable_reason: Option<String>,
pub upstream_connection_id: Option<String>,
pub upstream_connection_exit_reason: Option<String>,
@@ -492,7 +494,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request"))?;
.map_err(|source| postgres_unavailable("insert model request", source))?;
Ok(())
}
@@ -581,7 +583,9 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(request.continuation.requested)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
.map_err(|source| {
postgres_unavailable("insert model request with first attempt", source)
})?;
Ok(())
}
@@ -643,7 +647,7 @@ impl ModelRequestRepository for PgExecutionStore {
)?)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("begin model request attempt"))?
.map_err(|source| postgres_unavailable("begin model request attempt", source))?
.ok_or(StoreError::Conflict {
entity: ENTITY,
id: attempt.model_request_id,
@@ -666,7 +670,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(state.as_str())
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("mark upstream send state"))?;
.map_err(|source| postgres_unavailable("mark upstream send state", source))?;
Ok(result.rows_affected() == 1)
}
@@ -689,7 +693,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(client_status_code.map(i32::from))
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("mark downstream committed"))?;
.map_err(|source| postgres_unavailable("mark downstream committed", source))?;
Ok(result.rows_affected() == 1)
}
@@ -708,7 +712,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(i32::from(client_status_code))
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("record client status code"))?;
.map_err(|source| postgres_unavailable("record client status code", source))?;
Ok(result.rows_affected() == 1)
}
@@ -739,7 +743,7 @@ impl ModelRequestRepository for PgExecutionStore {
upstream_transport = coalesce($37, upstream_transport),
http_version = coalesce($38, http_version), websocket_pool = $39,
service_tier = $40, provider_observation_json = $41,
raw_upstream_error = $42,
error_details = $42,
continuation_unavailable_reason = $43,
upstream_connection_id = $44,
upstream_connection_exit_reason = $45,
@@ -924,7 +928,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(finalization.websocket_pool)
.bind(finalization.service_tier)
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
.bind(finalization.raw_upstream_error)
.bind(finalization.error_details)
.bind(finalization.continuation_unavailable_reason)
.bind(finalization.upstream_connection_id)
.bind(finalization.upstream_connection_exit_reason)
@@ -941,7 +945,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(finalization.billing_snapshot_json.map(sqlx::types::Json))
.fetch_one(&self.pool)
.await
.map_err(|_| postgres_unavailable("finalize model request"))?;
.map_err(|source| postgres_unavailable("finalize model request", source))?;
Ok(finalized == 1)
}
@@ -962,7 +966,7 @@ impl ModelRequestRepository for PgExecutionStore {
.bind(now)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("recover expired model requests"))?;
.map_err(|source| postgres_unavailable("recover expired model requests", source))?;
Ok(ModelRequestRecoveryReport {
requests: result.rows_affected(),
})
@@ -1006,7 +1010,7 @@ impl ExecutionStore for PgExecutionStore {
.bind(ttl_ms)
.fetch_one(&pool)
.await
.map_err(|_| postgres_unavailable("renew model request recovery lease"))
.map_err(|source| postgres_unavailable("renew model request recovery lease", source))
})
},
))
@@ -1105,7 +1109,7 @@ impl ExecutionStore for PgExecutionStore {
.retry_after()
.map(|duration| u64::try_from(duration.as_millis()))
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
super::OpsEventRepository::append_ops_event(
&super::PgOpsEventRepository::new(self.pool.clone()),
super::OpsEvent {
@@ -1124,9 +1128,7 @@ impl ExecutionStore for PgExecutionStore {
.map(|model| model.as_str().to_owned()),
failure_kind: error.kind().as_str().to_owned(),
upstream_send_state: Some(error.send_state().as_str().to_owned()),
raw_upstream_error: error
.raw_upstream_error()
.map(|raw| raw.as_str().to_owned()),
error_details: error.error_details(),
status_code: error.upstream_status().or(failure.upstream_status_code),
provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()),
retry_after_ms,
@@ -1135,8 +1137,9 @@ impl ExecutionStore for PgExecutionStore {
.map(|id| id.as_str().to_owned())
.or(failure.upstream_request_id),
latency_ms: Some(
u64::try_from(failure.latency.as_millis())
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
u64::try_from(failure.latency.as_millis()).map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?,
),
message: error.diagnostic().map_or_else(
|| "intermediate upstream failure".to_owned(),
@@ -1169,7 +1172,7 @@ impl ExecutionStore for PgExecutionStore {
upstream_model_id: None,
failure_kind: error.kind().as_str().to_owned(),
upstream_send_state: None,
raw_upstream_error: None,
error_details: None,
status_code: None,
provider_error_code: error.client_error_code().map(str::to_owned),
retry_after_ms: error
@@ -1197,9 +1200,9 @@ impl ExecutionStore for PgExecutionStore {
.retry_after()
.map(|duration| u64::try_from(duration.as_millis()))
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
let latency_ms = u64::try_from(failure.latency.as_millis())
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
super::OpsEventRepository::append_ops_event(
&super::PgOpsEventRepository::new(self.pool.clone()),
@@ -1216,9 +1219,7 @@ impl ExecutionStore for PgExecutionStore {
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
failure_kind: error.kind().as_str().to_owned(),
upstream_send_state: Some(error.send_state().as_str().to_owned()),
raw_upstream_error: error
.raw_upstream_error()
.map(|raw| raw.as_str().to_owned()),
error_details: error.error_details(),
status_code: error.upstream_status(),
provider_error_code,
retry_after_ms,
@@ -1260,7 +1261,7 @@ impl ExecutionStore for PgExecutionStore {
.as_deref()
.map(serde_json::from_str::<Value>)
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
if provider_metadata_json
.as_ref()
.is_some_and(|metadata| !metadata.is_object())
@@ -1329,11 +1330,13 @@ impl ExecutionStore for PgExecutionStore {
.as_deref()
.map(serde_json::from_str)
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?,
error_kind,
provider_error_code: finalization.provider_error_code,
error_message,
raw_upstream_error: finalization.raw_upstream_error,
error_details: finalization.error_details,
continuation_unavailable_reason,
upstream_connection_id,
upstream_connection_exit_reason,
@@ -1429,17 +1432,6 @@ fn require_core_update(updated: bool) -> Result<(), CoreStoreError> {
}
}
fn core_store_error(error: StoreError) -> CoreStoreError {
let kind = match error {
StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable,
StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict,
StoreError::NotFound { .. } | StoreError::InvalidData { .. } => {
CoreStoreErrorKind::InvalidData
}
};
CoreStoreError::new(kind)
}
fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
let (routing_scope, routing_group_refs, routing_group_names_snapshot) =
routing_snapshot_row(&request.routing);
@@ -784,6 +784,7 @@ impl ExecutionObservationWrite {
finalization.provider_metadata_json.as_deref(),
finalization.diagnostic_trace_json.as_deref(),
finalization.provider_error_code.as_deref(),
finalization.error_details.as_deref(),
])
.saturating_add(error_bytes)
.saturating_add(size_of::<ModelRequestFinalization>())
@@ -874,9 +875,13 @@ fn attempt_bytes(attempt: &AttemptRecord) -> usize {
}
fn provider_error_bytes(error: &ProviderError) -> usize {
use std::error::Error as _;
let mut bytes = text_bytes([
error.upstream_code().map(|value| value.as_str()),
error.upstream_request_id().map(|value| value.as_str()),
error.diagnostic().map(|value| value.as_str()),
error.raw_upstream_error().map(|value| value.as_str()),
]);
if let Some(client_error) = error.client_visible_upstream_error() {
bytes = bytes.saturating_add(text_bytes([
@@ -895,7 +900,33 @@ fn provider_error_bytes(error: &ProviderError) -> usize {
.saturating_add(header.value().len());
}
}
bytes
bytes.saturating_add(error_source_bytes(error.source()))
}
/// 原始来源也占据队列内存;只计数而不保存格式化内容,避免复制或打印敏感原因
fn error_source_bytes(mut source: Option<&(dyn std::error::Error + 'static)>) -> usize {
struct ByteCount(usize);
impl std::fmt::Write for ByteCount {
fn write_str(&mut self, value: &str) -> std::fmt::Result {
self.0 = self.0.saturating_add(value.len());
Ok(())
}
}
let mut bytes = ByteCount(0);
for _ in 0..32 {
let Some(error) = source else { return bytes.0 };
bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error));
if std::fmt::write(&mut bytes, format_args!("{error}")).is_err() {
return usize::MAX;
}
source = error.source();
}
// 过深或循环的第三方来源不能让观测写入无限遍历,交给已有超限丢弃计数
if source.is_some() {
usize::MAX
} else {
bytes.0
}
}
fn text_bytes<const N: usize>(values: [Option<&str>; N]) -> usize {
@@ -8,7 +8,7 @@ use sqlx::{
use crate::{
POSTGRES_IDLE_TRANSACTION_TIMEOUT, POSTGRES_LOCK_TIMEOUT, POSTGRES_STATEMENT_TIMEOUT, Revision,
StoreBackend, StoreError, StorePoolConfig, StoreResult, postgres_unavailable,
StoreError, StorePoolConfig, StoreResult, postgres_unavailable,
};
mod account_groups;
@@ -59,12 +59,15 @@ pub async fn connect_and_migrate(
pool_config: StorePoolConfig,
) -> StoreResult<PgPool> {
if database_url.trim().is_empty() {
return Err(postgres_unavailable("connect PostgreSQL"));
return Err(StoreError::InvalidData {
entity: "PostgreSQL configuration",
message: "database URL is empty".to_owned(),
});
}
pool_config.validate()?;
let connect_options = database_url
.parse::<PgConnectOptions>()
.map_err(|_| postgres_unavailable("parse PostgreSQL connection options"))?;
.map_err(|source| postgres_unavailable("parse PostgreSQL connection options", source))?;
let migration_pool = PgPoolOptions::new()
.max_connections(1)
.connect_with(
@@ -73,13 +76,10 @@ pub async fn connect_and_migrate(
.application_name("codex-proxy-rs:migration"),
)
.await
.map_err(|_| postgres_unavailable("connect PostgreSQL for migrations"))?;
.map_err(|source| postgres_unavailable("connect PostgreSQL for migrations", source))?;
if let Err(error) = MIGRATOR.run(&migration_pool).await {
migration_pool.close().await;
return Err(StoreError::Unavailable {
backend: StoreBackend::PostgreSql,
message: format!("apply PostgreSQL migrations: {error}"),
});
return Err(postgres_unavailable("apply PostgreSQL migrations", error));
}
migration_pool.close().await;
@@ -94,7 +94,7 @@ pub(crate) async fn connect_read_only(
pool_config.validate()?;
let options = database_url
.parse::<PgConnectOptions>()
.map_err(|_| postgres_unavailable("parse PostgreSQL connection options"))?;
.map_err(|source| postgres_unavailable("parse PostgreSQL connection options", source))?;
connect_pool(options, pool_config, true).await
}
@@ -135,7 +135,7 @@ async fn connect_pool(
})
.connect_with(connect_options.application_name("codex-proxy-rs"))
.await
.map_err(|_| postgres_unavailable("connect PostgreSQL"))?;
.map_err(|source| postgres_unavailable("connect PostgreSQL", source))?;
Ok(pool)
}
@@ -221,11 +221,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
replacement: ControlPlaneReplacement,
) -> StoreResult<ControlPlaneSnapshot> {
replacement.settings.validate()?;
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin control plane replacement"))?;
let mut transaction =
self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin control plane replacement", source)
})?;
let result = async {
// 锁住所有配置写入共同使用的行,版本检查与修改不能被其他事务穿插
let current = sqlx::query_scalar::<_, i64>(
@@ -233,7 +232,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
)
.fetch_one(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("lock control plane revision"))?;
.map_err(|source| postgres_unavailable("lock control plane revision", source))?;
if u64::try_from(current).ok() != Some(replacement.expected_revision.get()) {
return Err(StoreError::Conflict {
entity: "runtime settings",
@@ -251,17 +250,15 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
.await;
match result {
Ok(snapshot) => {
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit control plane replacement"))?;
transaction.commit().await.map_err(|source| {
postgres_unavailable("commit control plane replacement", source)
})?;
Ok(snapshot)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| postgres_unavailable("rollback control plane replacement"))?;
transaction.rollback().await.map_err(|source| {
postgres_unavailable("rollback control plane replacement", source)
})?;
Err(error)
}
}
@@ -337,11 +334,9 @@ impl PgControlPlaneRepository {
mutation: ControlPlaneMutation,
mut audit: AdminAuditEvent,
) -> StoreResult<Revision> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin targeted control plane mutation"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin targeted control plane mutation", source)
})?;
let result = async {
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
match mutation {
@@ -362,8 +357,8 @@ impl PgControlPlaneRepository {
.bind(&key.id)
.fetch_one(&mut *transaction)
.await
.map_err(|_| {
postgres_unavailable("load client API key routing scope for audit")
.map_err(|source| {
postgres_unavailable("load client API key routing scope for audit", source)
})?;
if previously_restricted && key.group_ids.is_empty() {
audit
@@ -393,15 +388,14 @@ impl PgControlPlaneRepository {
.await;
match result {
Ok(revision) => {
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit targeted control plane mutation"))?;
transaction.commit().await.map_err(|source| {
postgres_unavailable("commit targeted control plane mutation", source)
})?;
Ok(revision)
}
Err(error) => {
transaction.rollback().await.map_err(|_| {
postgres_unavailable("rollback targeted control plane mutation")
transaction.rollback().await.map_err(|source| {
postgres_unavailable("rollback targeted control plane mutation", source)
})?;
Err(error)
}
@@ -778,7 +778,7 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
upstream_request_id: error.upstream_request_id,
latency_ms: error.latency_ms,
message: error.message,
raw_upstream_error: error.raw_upstream_error,
error_details: error.error_details,
client_ip: error.client_ip,
user_agent: error.user_agent,
reasoning_effort: error.reasoning_effort,
@@ -890,7 +890,7 @@ pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<
provider_metadata_json: get::<Option<serde_json::Value>>(row, "provider_observation_json")?
.map(|value| serde_json::to_string(&value))
.transpose()
.map_err(|_| postgres_unavailable("encode provider observation"))?,
.map_err(|source| postgres_unavailable("encode provider observation", source))?,
attempt_count: to_u32(get(row, "attempt_count")?)?,
upstream_send_state: get(row, "upstream_send_state")?,
downstream_committed_at: get(row, "downstream_committed_at")?,
@@ -977,7 +977,7 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
upstream_request_id: get(row, "upstream_request_id")?,
latency_ms: optional_unsigned(row, "latency_ms")?,
message: get(row, "message")?,
raw_upstream_error: get(row, "raw_upstream_error")?,
error_details: get(row, "error_details")?,
client_ip: get(row, "client_ip")?,
user_agent: get(row, "user_agent")?,
reasoning_effort: get(row, "reasoning_effort")?,
@@ -259,7 +259,11 @@ pub struct PercentileMilliseconds(u64);
impl PercentileMilliseconds {
pub(crate) fn new(value: f64) -> StoreResult<Self> {
if !value.is_finite() || value < 0.0 {
return Err(postgres_unavailable("decode latency percentile"));
return Err(crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "decode latency percentile".to_owned(),
source: None,
});
}
Ok(Self(value.to_bits()))
}
@@ -740,7 +744,7 @@ pub struct OpsErrorRecord {
pub upstream_request_id: Option<String>,
pub latency_ms: Option<u64>,
pub message: String,
pub raw_upstream_error: Option<String>,
pub error_details: Option<String>,
pub client_ip: Option<String>,
pub user_agent: Option<String>,
pub reasoning_effort: Option<String>,
@@ -92,7 +92,7 @@ pub(crate) async fn provider_account_usage(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account usage"))?;
.map_err(|source| postgres_unavailable("load provider account usage", source))?;
let mut observations = Vec::with_capacity(usize::from(query.limit));
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
@@ -126,9 +126,11 @@ pub(crate) async fn provider_account_usage(
}
(0 | 1, 0 | 1) => {}
_ => {
return Err(postgres_unavailable(
"decode provider account usage grouping",
));
return Err(crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "decode provider account usage grouping".to_owned(),
source: None,
});
}
}
}
@@ -188,7 +190,7 @@ pub(crate) async fn provider_account_request_buckets(
.bind(range.end)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider account request timeline"))?;
.map_err(|source| postgres_unavailable("load provider account request timeline", source))?;
let mut observed = HashMap::<String, BTreeMap<u64, u64>>::new();
for row in rows {
@@ -99,7 +99,7 @@ pub(crate) async fn request_metrics(
.build()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load request metrics"))?;
.map_err(|source| postgres_unavailable("load request metrics", source))?;
request_metrics_from_row(&row)
}
@@ -121,7 +121,7 @@ pub(crate) async fn dashboard_totals(pool: &PgPool) -> StoreResult<DashboardTota
.build()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load dashboard totals"))?;
.map_err(|source| postgres_unavailable("load dashboard totals", source))?;
Ok(DashboardTotals {
request_count: unsigned(&row, "request_count")?,
input_tokens: unsigned(&row, "input_tokens")?,
@@ -259,7 +259,7 @@ async fn request_metric_series_inner(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request metric series"))?;
.map_err(|source| postgres_unavailable("load request metric series", source))?;
let mut points = BTreeMap::new();
for row in rows {
@@ -321,7 +321,7 @@ pub(crate) fn calculated_usage_billing_facts(
// 费用逐条累加与行顺序无关,避免全量物化和不必要的数据库排序
let mut rows = query.build().fetch(pool);
while let Some(row) = rows.try_next().await
.map_err(|_| postgres_unavailable("load calculated usage billing facts"))?
.map_err(|source| postgres_unavailable("load calculated usage billing facts", source))?
{
yield calculated_usage_billing_fact_from_row(&row)?;
}
@@ -354,7 +354,7 @@ pub(crate) async fn request_costs_by_bucket(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request costs by bucket"))?;
.map_err(|source| postgres_unavailable("load request costs by bucket", source))?;
let mut result = BTreeMap::<DateTime<Utc>, Vec<CurrencyCostTotal>>::new();
for row in rows {
result
@@ -470,7 +470,7 @@ pub(crate) async fn attempt_metrics(
.build()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("load attempt metrics"))?;
.map_err(|source| postgres_unavailable("load attempt metrics", source))?;
Ok(AttemptMetrics {
attempt_count: unsigned(&row, "attempt_count")?,
success_count: unsigned(&row, "success_count")?,
@@ -505,7 +505,7 @@ pub(crate) async fn request_costs(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load request costs"))?
.map_err(|source| postgres_unavailable("load request costs", source))?
.iter()
.map(cost_from_row)
.collect()
@@ -537,7 +537,7 @@ pub(crate) async fn provider_observations(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load provider observations"))?;
.map_err(|source| postgres_unavailable("load provider observations", source))?;
rows.iter()
.map(|row| {
Ok(ProviderObservation {
@@ -615,7 +615,7 @@ pub(crate) fn optional_percentile(
column: &str,
) -> StoreResult<Option<PercentileMilliseconds>> {
row.try_get::<Option<f64>, _>(column)
.map_err(|_| postgres_unavailable("decode latency percentile"))?
.map_err(|source| postgres_unavailable("decode latency percentile", source))?
.map(PercentileMilliseconds::new)
.transpose()
}
@@ -20,7 +20,7 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
mr.latency_ms,
coalesce(mr.error_message, mr.error_kind) as message,
mr.raw_upstream_error,
mr.error_details,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
@@ -51,7 +51,7 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
oe.upstream_send_state,
null::integer as client_status_code, oe.status_code as upstream_status_code,
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
oe.latency_ms, oe.message, oe.raw_upstream_error,
oe.latency_ms, oe.message, oe.error_details,
host(mr.client_ip) as client_ip, mr.user_agent,
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
mr.subagent_kind, mr.compact,
@@ -93,7 +93,7 @@ pub(crate) async fn list_ops_errors(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("list ops errors"))?;
.map_err(|source| postgres_unavailable("list ops errors", source))?;
let items = rows
.iter()
.map(ops_error_from_row)
@@ -124,7 +124,7 @@ pub(crate) async fn count_ops_errors(
.build_query_scalar::<i64>()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("count ops errors"))?;
.map_err(|source| postgres_unavailable("count ops errors", source))?;
to_u64(total)
}
@@ -212,7 +212,7 @@ pub(crate) async fn list_usage_record_items(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("list usage records"))?;
.map_err(|source| postgres_unavailable("list usage records", source))?;
let items = rows
.iter()
.map(usage_list_record_from_row)
@@ -237,7 +237,7 @@ pub(crate) async fn count_usage_records(
.build_query_scalar::<i64>()
.fetch_one(pool)
.await
.map_err(|_| postgres_unavailable("count usage records"))?;
.map_err(|source| postgres_unavailable("count usage records", source))?;
to_u64(total)
}
@@ -254,7 +254,7 @@ pub(crate) async fn usage_record_detail(
.build()
.fetch_optional(pool)
.await
.map_err(|_| postgres_unavailable("load usage record detail"))?
.map_err(|source| postgres_unavailable("load usage record detail", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "model request",
id: request_id.to_owned(),
@@ -276,7 +276,7 @@ pub(crate) async fn usage_record_detail(
.bind(request_id)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load usage attempt observations"))?;
.map_err(|source| postgres_unavailable("load usage attempt observations", source))?;
let mut attempts = rows
.iter()
.map(intermediate_attempt_from_row)
@@ -286,7 +286,7 @@ pub(crate) async fn usage_record_detail(
}
let trace: Option<Value> = row
.try_get("diagnostic_trace_json")
.map_err(|_| postgres_unavailable("decode request trace"))?;
.map_err(|source| postgres_unavailable("decode request trace", source))?;
let related_requests = sqlx::query_scalar::<_, Value>(
"select jsonb_build_object('requestId', related.id, 'outcome', related.outcome,
'relation', case when related.id = current.recovery_request_id then 'recovered_by' else 'recovers' end,
@@ -295,7 +295,7 @@ pub(crate) async fn usage_record_detail(
on related.id = current.recovery_request_id or related.recovery_request_id = current.id
where current.id = $1 order by related.started_at limit 20"
).bind(request_id).fetch_all(pool).await
.map_err(|_| postgres_unavailable("load related recovery requests"))?;
.map_err(|source| postgres_unavailable("load related recovery requests", source))?;
Ok(UsageRecordDetail {
request,
attempts,
@@ -468,7 +468,7 @@ pub(crate) async fn usage_diagnostics(
.build()
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load usage diagnostics"))?;
.map_err(|source| postgres_unavailable("load usage diagnostics", source))?;
let total_request_count = rows
.first()
.map(|row| unsigned(row, "total_request_count"))
@@ -506,7 +506,13 @@ pub(crate) async fn usage_diagnostics(
.push(cost_from_row(row)?);
}
0 => {}
_ => return Err(postgres_unavailable("decode usage diagnostic grouping")),
_ => {
return Err(crate::StoreError::Unavailable {
backend: crate::StoreBackend::PostgreSql,
message: "decode usage diagnostic grouping".to_owned(),
source: None,
});
}
}
}
let mut display_names = match dimension {
@@ -577,7 +583,7 @@ pub(crate) async fn diagnostic_account_display_names(
.bind(account_ids)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load diagnostic account display names"))?;
.map_err(|source| postgres_unavailable("load diagnostic account display names", source))?;
let mut display_names = HashMap::with_capacity(rows.len());
for row in rows {
display_names.insert(
@@ -599,7 +605,7 @@ pub(crate) async fn diagnostic_api_key_display_names(
.bind(key_ids)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load diagnostic api key display names"))?;
.map_err(|source| postgres_unavailable("load diagnostic api key display names", source))?;
let mut display_names = HashMap::with_capacity(rows.len());
for row in rows {
display_names.insert(get(&row, "id")?, get(&row, "name")?);
@@ -101,11 +101,12 @@ impl ObservabilityQueryBudget {
);
Ok(permit)
}
Ok(Err(_)) => Err(StoreError::Unavailable {
Ok(Err(source)) => Err(StoreError::Unavailable {
backend: StoreBackend::PostgreSql,
message: "observability PostgreSQL connection budget is closed".to_owned(),
source: Some(gateway_core::error::ErrorSource::new(source)),
}),
Err(_) => {
Err(source) => {
tracing::warn!(
query_class = "observability",
operation,
@@ -116,6 +117,7 @@ impl ObservabilityQueryBudget {
Err(StoreError::Unavailable {
backend: StoreBackend::PostgreSql,
message: "observability PostgreSQL connection budget is exhausted".to_owned(),
source: Some(gateway_core::error::ErrorSource::new(source)),
})
}
}
@@ -38,7 +38,7 @@ pub struct OpsEvent {
pub upstream_model_id: Option<String>,
pub failure_kind: String,
pub upstream_send_state: Option<String>,
pub raw_upstream_error: Option<String>,
pub error_details: Option<String>,
pub status_code: Option<u16>,
pub provider_error_code: Option<String>,
pub retry_after_ms: Option<u64>,
@@ -119,7 +119,7 @@ impl OpsEventRepository for PgOpsEventRepository {
provider_account_name_snapshot, provider_account_email_snapshot,
provider_account_authentication_kind_snapshot,
failure_kind, upstream_send_state, status_code, provider_error_code, retry_after_ms,
upstream_request_id, latency_ms, message, raw_upstream_error,
upstream_request_id, latency_ms, message, error_details,
created_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
@@ -165,11 +165,11 @@ impl OpsEventRepository for PgOpsEventRepository {
.map_err(|_| invalid("latency_ms is too large"))?,
)
.bind(event.message)
.bind(event.raw_upstream_error)
.bind(event.error_details)
.bind(event.created_at)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("append ops event"))?;
.map_err(|source| postgres_unavailable("append ops event", source))?;
Ok(())
}
}
@@ -143,7 +143,7 @@ impl PgControlPlaneRepository {
) -> StoreResult<gateway_admin::model::pricing::StoredPricing> {
let (Json(overrides), Json(synced), synced_at) = sqlx::query_as::<_, (Json<PricingOverrides>, Json<PricingOverrides>, Option<chrono::DateTime<chrono::Utc>>)>(
"select pricing_overrides_json, pricing_synced_json, pricing_synced_at from runtime_settings where id = 1")
.fetch_one(&self.pool).await.map_err(|_| postgres_unavailable("load model pricing"))?;
.fetch_one(&self.pool).await.map_err(|source| postgres_unavailable("load model pricing", source))?;
validate_pricing(&overrides)?;
validate_pricing(&synced)?;
Ok(gateway_admin::model::pricing::StoredPricing {
@@ -162,13 +162,13 @@ impl PgControlPlaneRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin pricing sync"))?;
.map_err(|source| postgres_unavailable("begin pricing sync", source))?;
let Json(mut prices) = sqlx::query_scalar::<_, Json<PricingOverrides>>(
"select pricing_synced_json from runtime_settings where id = 1 for update",
)
.fetch_one(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("lock synced model pricing"))?;
.map_err(|source| postgres_unavailable("lock synced model pricing", source))?;
for (provider, models) in changes {
let stored = prices.entry(provider).or_default();
for (model, price) in models {
@@ -182,13 +182,13 @@ impl PgControlPlaneRepository {
prices.retain(|_, models| !models.is_empty());
validate_pricing(&prices)?;
sqlx::query("update runtime_settings set pricing_synced_json = $1, pricing_synced_at = now() where id = 1")
.bind(Json(prices)).execute(&mut *transaction).await.map_err(|_| postgres_unavailable("sync model pricing"))?;
.bind(Json(prices)).execute(&mut *transaction).await.map_err(|source| postgres_unavailable("sync model pricing", source))?;
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?;
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit pricing sync"))?;
.map_err(|source| postgres_unavailable("commit pricing sync", source))?;
Ok(revision)
}
@@ -201,14 +201,14 @@ impl PgControlPlaneRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin model pricing update"))?;
.map_err(|source| postgres_unavailable("begin model pricing update", source))?;
// 锁定当前配置再更新选中项,避免批量操作覆盖其他管理员已提交的模型
let (Json(mut pricing), Json(mut synced)) = sqlx::query_as::<_, (Json<PricingOverrides>, Json<PricingOverrides>)>(
"select pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1 for update",
)
.fetch_one(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("lock model pricing"))?;
.map_err(|source| postgres_unavailable("lock model pricing", source))?;
let models = pricing.entry(command.provider.clone()).or_default();
let source = synced.entry(command.provider).or_default();
for model in command.models {
@@ -242,13 +242,13 @@ impl PgControlPlaneRepository {
.bind(Json(synced))
.execute(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("update model pricing"))?;
.map_err(|source| postgres_unavailable("update model pricing", source))?;
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?;
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit model pricing"))?;
.map_err(|source| postgres_unavailable("commit model pricing", source))?;
Ok(revision)
}
}
@@ -123,7 +123,9 @@ impl PgAdminAccountStore {
.bind(ends)
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("load provider account quota window usage"))
.map_err(|source| {
postgres_unavailable("load provider account quota window usage", source)
})
})
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
@@ -270,10 +272,10 @@ impl PgAdminAccountStore {
.bind(account_ids)
.fetch_all(&self.pool)
.await
.map_err(|_| {
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("load account group references"),
postgres_unavailable("load account group references", source),
)
})?;
let mut groups = BTreeMap::<String, Vec<AccountGroupRef>>::new();
@@ -430,7 +432,12 @@ impl AccountStore for PgAdminAccountStore {
)
.fetch_one(&self.pool)
.await
.map_err(|_| admin_store_error(ENTITY, postgres_unavailable("load account concurrency")))?;
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("load account concurrency", source),
)
})?;
let default_concurrency = u64::try_from(default_concurrency).map_err(|_| {
AdminStoreError::new(
AdminStoreErrorKind::Invalid,
@@ -701,8 +708,11 @@ impl AccountStore for PgAdminAccountStore {
limit: AccountConcurrencyLimit,
context: &MutationContext,
) -> AdminStoreResult<Option<AccountUpdateResult>> {
let mut transaction = self.pool.begin().await.map_err(|_| {
admin_store_error(ENTITY, postgres_unavailable("begin concurrency reduction"))
let mut transaction = self.pool.begin().await.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("begin concurrency reduction", source),
)
})?;
let result =
async {
@@ -710,7 +720,7 @@ impl AccountStore for PgAdminAccountStore {
let default_limit: i64 = sqlx::query_scalar(
"select max_concurrent_per_account from runtime_settings where id = 1 for update"
).fetch_one(&mut *transaction).await
.map_err(|_| postgres_unavailable("lock default concurrency"))?;
.map_err(|source| postgres_unavailable("lock default concurrency", source))?;
let changed = sqlx::query_scalar::<_, String>(
"update provider_accounts set concurrency_limit = $2, updated_at = greatest(now(), updated_at)
where id = $1 and enabled = true
@@ -722,7 +732,7 @@ impl AccountStore for PgAdminAccountStore {
.bind(default_limit)
.fetch_optional(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("lower account concurrency"))?;
.map_err(|source| postgres_unavailable("lower account concurrency", source))?;
if changed.is_none() {
return Ok(None);
}
@@ -919,10 +929,10 @@ impl AccountStore for PgAdminAccountStore {
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
let revision = control_plane.settings.config_revision;
let mut transaction = self.accounts.pool.begin().await.map_err(|_| {
let mut transaction = self.accounts.pool.begin().await.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("begin credential export audit"),
postgres_unavailable("begin credential export audit", source),
)
})?;
let result = async {
@@ -160,7 +160,12 @@ pub(crate) async fn load_admin_account_page(
.bind(offset)
.fetch_all(pool)
.await
.map_err(|_| admin_store_error(ENTITY, postgres_unavailable("load admin account page")))?;
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("load admin account page", source),
)
})?;
let metadata = rows.first().ok_or_else(|| {
AdminStoreError::new(
AdminStoreErrorKind::Unavailable,
@@ -211,10 +216,10 @@ async fn validate_group_filter(
.bind(group_id.as_str())
.fetch_one(pool)
.await
.map_err(|_| {
.map_err(|source| {
admin_store_error(
ENTITY,
postgres_unavailable("validate admin account group filter"),
postgres_unavailable("validate admin account group filter", source),
)
})?;
if exists {
@@ -1,6 +1,7 @@
//! Core `ProviderAccountStore` 端口适配与 core 投影映射
use super::*;
use crate::core_store_error;
fn loaded_credential_from_record(
record: ProviderAccountRecord,
@@ -105,7 +106,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.bind(i64::from(query.limit().get()))
.fetch_all(&self.pool)
.await
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?;
rows.into_iter()
.map(account_record_from_row)
.map(|record| {
@@ -218,11 +219,12 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.bind(preserve_profile)
.fetch_optional(&self.pool)
.await
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?;
match next {
Some(next) => Ok(CredentialCasOutcome::Updated(
CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?)
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
CoreCredentialRevision::new(to_u64(next).map_err(core_store_error)?).map_err(
|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source),
)?,
)),
None => Ok(CredentialCasOutcome::Conflict),
}
@@ -248,12 +250,12 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.bind(account_ids)
.fetch_all(&self.pool)
.await
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::Unavailable))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::Unavailable, source))?;
rows.into_iter()
.map(|row| {
let account_id = row
.try_get::<String, _>("id")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let account_id = row.try_get::<String, _>("id").map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?;
let revision = row
.try_get::<i64, _>("credential_revision")
.ok()
@@ -262,21 +264,27 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let quota = row
.try_get::<Option<serde_json::Value>, _>("provider_quota_json")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?
.and_then(|value| value.as_object().cloned())
.map(OpaqueProviderData::new)
.unwrap_or_else(|| OpaqueProviderData::new(serde_json::Map::new()));
let observed_at = row
.try_get::<DateTime<Utc>, _>("quota_observed_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?;
let access = row
.try_get::<String, _>("quota_access_state")
.ok()
.and_then(|value| QuotaAccessState::parse(&value))
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let evidence = row
.try_get::<Option<String>, _>("quota_evidence")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let evidence =
row.try_get::<Option<String>, _>("quota_evidence")
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?;
let evidence = match evidence {
Some(value) => Some(
QuotaEvidence::parse(&value)
@@ -286,10 +294,14 @@ impl ProviderAccountStore for PgProviderAccountRepository {
};
let reset_at = row
.try_get::<Option<DateTime<Utc>>, _>("quota_reset_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?;
let access_observed_at = row
.try_get::<Option<DateTime<Utc>>, _>("quota_access_observed_at")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?;
let state = QuotaState::from_persisted(
access,
evidence,
@@ -298,11 +310,12 @@ impl ProviderAccountStore for PgProviderAccountRepository {
)
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
Ok(QuotaObservation {
plan_type: row
.try_get("plan_type")
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
account_id: CoreProviderAccountId::new(account_id)
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
plan_type: row.try_get("plan_type").map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?,
account_id: CoreProviderAccountId::new(account_id).map_err(|source| {
CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source)
})?,
expected_revision: revision,
quota,
observed_at: observed_at.into(),
@@ -32,7 +32,7 @@ pub(super) async fn load_history(
.bind(MAX_FORECAST_HISTORY_POINTS as i32)
.fetch_all(pool)
.await
.map_err(|_| postgres_unavailable("load quota forecast history"))
.map_err(|source| postgres_unavailable("load quota forecast history", source))
})
.await
.map_err(|error| admin_store_error(ENTITY, error))?;
@@ -105,7 +105,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("load provider account"))?;
.map_err(|source| postgres_unavailable("load provider account", source))?;
row.map(account_record_from_row).transpose()
}
@@ -141,7 +141,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("list plugin accounts"))?;
.map_err(|source| postgres_unavailable("list plugin accounts", source))?;
rows.into_iter().map(account_summary_from_row).collect()
}
@@ -167,7 +167,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(include_disabled)
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("list provider accounts"))?;
.map_err(|source| postgres_unavailable("list provider accounts", source))?;
rows.into_iter().map(account_summary_from_row).collect()
}
@@ -178,7 +178,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account insert"))?;
.map_err(|source| postgres_unavailable("begin provider account insert", source))?;
let proxy_id = match account.outbound_proxy.as_ref() {
Some(proxy) => Some(
super::super::proxies::ensure_proxy_for_url(&mut transaction, proxy, None)
@@ -221,11 +221,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(account.model_access.as_ref().map(sqlx::types::Json))
.execute(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("insert provider account"))?;
.map_err(|source| postgres_unavailable("insert provider account", source))?;
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit provider account insert"))?;
.map_err(|source| postgres_unavailable("commit provider account insert", source))?;
Ok(())
}
@@ -243,7 +243,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(account.plan_type)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("update provider account"))?;
.map_err(|source| postgres_unavailable("update provider account", source))?;
Ok(result.rows_affected() == 1)
}
@@ -279,7 +279,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(update.next_refresh_at)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("compare and swap provider credentials"))?
.map_err(|source| postgres_unavailable("compare and swap provider credentials", source))?
.ok_or(StoreError::Conflict {
entity: ENTITY,
id: update.account_id,
@@ -311,7 +311,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(update.message.as_deref())
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("apply provider account state"))?;
.map_err(|source| postgres_unavailable("apply provider account state", source))?;
Ok(result.rows_affected() == 1)
}
@@ -324,7 +324,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(enabled)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("set provider account enabled state"))?;
.map_err(|source| postgres_unavailable("set provider account enabled state", source))?;
Ok(result.rows_affected() == 1)
}
@@ -375,7 +375,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(plan_type)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("compare and swap provider quota"))?;
.map_err(|source| postgres_unavailable("compare and swap provider quota", source))?;
Ok(result.rows_affected() == 1)
}
@@ -406,7 +406,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(state.reset_at().map(DateTime::<Utc>::from))
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("apply provider quota access"))?;
.map_err(|source| postgres_unavailable("apply provider quota access", source))?;
Ok(result.rows_affected() == 1)
}
@@ -429,7 +429,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(observed_at)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("touch provider quota observation"))?;
.map_err(|source| postgres_unavailable("touch provider quota observation", source))?;
Ok(result.rows_affected() == 1)
}
@@ -439,7 +439,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
.bind(id)
.execute(&self.pool)
.await
.map_err(|_| postgres_unavailable("delete disabled provider account"))?;
.map_err(|source| postgres_unavailable("delete disabled provider account", source))?;
Ok(result.rows_affected() == 1)
}
}
@@ -458,7 +458,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
.bind(&scope.provider_kind)
.fetch_all(&self.pool)
.await
.map_err(|_| postgres_unavailable("export provider accounts"))?;
.map_err(|source| postgres_unavailable("export provider accounts", source))?;
let records = rows
.into_iter()
.map(account_record_from_row)
@@ -486,11 +486,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
&self,
command: ImportProviderAccounts,
) -> StoreResult<ProviderAccountAdminImport> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account admin import"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin provider account admin import", source)
})?;
let result = import_provider_accounts_in_transaction(&mut transaction, command).await;
finish_admin_transaction(transaction, result, "provider account admin import").await
}
@@ -499,11 +497,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
&self,
command: RotateProviderAccount,
) -> StoreResult<ProviderAccountAdminRotation> {
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account admin rotation"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin provider account admin rotation", source)
})?;
let result = rotate_provider_account_admin_in_transaction(&mut transaction, command).await;
finish_admin_transaction(transaction, result, "provider account admin rotation").await
}
@@ -516,11 +512,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
if let Some(group_ids) = &command.group_ids {
validate_batch_update_group_ids(group_ids)?;
}
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account admin state change"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin provider account admin state change", source)
})?;
let result = async {
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
update_provider_accounts_scheduling_in_transaction(
@@ -562,11 +556,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
command: RecoverProviderAccount,
) -> StoreResult<Revision> {
validate_admin_account_ids(std::slice::from_ref(&command.account_id))?;
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account admin recovery"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin provider account admin recovery", source)
})?;
let result = async {
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
let recovered = sqlx::query_scalar::<_, String>(
@@ -593,7 +585,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
.bind(&command.account_id)
.fetch_optional(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("recover provider account state"))?
.map_err(|source| postgres_unavailable("recover provider account state", source))?
.ok_or_else(|| StoreError::NotFound {
entity: ENTITY,
id: command.account_id.clone(),
@@ -614,11 +606,9 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
) -> StoreResult<Revision> {
command.scope.validate()?;
validate_admin_account_ids(&command.account_ids)?;
let mut transaction = self
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin provider account admin deletion"))?;
let mut transaction = self.pool.begin().await.map_err(|source| {
postgres_unavailable("begin provider account admin deletion", source)
})?;
let result = async {
let revision = bump_config_revision_in_transaction(&mut transaction).await?;
delete_provider_accounts_in_transaction(
@@ -646,7 +636,7 @@ async fn update_provider_account_notes_in_transaction(
.bind(notes.trim())
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("update provider account notes"))?;
.map_err(|source| postgres_unavailable("update provider account notes", source))?;
Ok(())
}
@@ -666,7 +656,7 @@ async fn replace_account_group_assignments_in_transaction(
.bind(&group_ids)
.fetch_one(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("validate account group assignment"))?;
.map_err(|source| postgres_unavailable("validate account group assignment", source))?;
if usize::try_from(known_group_count).ok() != Some(group_ids.len()) {
return Err(StoreError::NotFound {
entity: "account group",
@@ -678,7 +668,7 @@ async fn replace_account_group_assignments_in_transaction(
.bind(account_ids)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("clear account group assignments"))?;
.map_err(|source| postgres_unavailable("clear account group assignments", source))?;
if group_ids.is_empty() {
return Ok(());
}
@@ -693,7 +683,7 @@ async fn replace_account_group_assignments_in_transaction(
.bind(account_ids)
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("assign accounts to groups"))?;
.map_err(|source| postgres_unavailable("assign accounts to groups", source))?;
Ok(())
}
@@ -786,7 +776,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
kind: ConflictKind::InvalidTransition,
}
} else {
postgres_unavailable("upsert provider account in admin transaction")
postgres_unavailable("upsert provider account in admin transaction", error)
}
})?
.ok_or_else(|| StoreError::Conflict {
@@ -865,7 +855,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
kind: ConflictKind::InvalidTransition,
}
} else {
postgres_unavailable("rotate provider account in admin transaction")
postgres_unavailable("rotate provider account in admin transaction", error)
}
})?
.ok_or_else(|| StoreError::Conflict {
@@ -911,7 +901,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction(
.bind(concurrency_limit.is_some())
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("set provider accounts state in admin transaction"))?
.map_err(|source| postgres_unavailable("set provider accounts state in admin transaction", source))?
.into_iter()
.collect::<BTreeSet<_>>();
let expected = account_ids.iter().cloned().collect::<BTreeSet<_>>();
@@ -939,7 +929,9 @@ pub(crate) async fn delete_provider_accounts_in_transaction(
.bind(&scope.provider_kind)
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("delete provider account in admin transaction"))?;
.map_err(|source| {
postgres_unavailable("delete provider account in admin transaction", source)
})?;
let deleted = deleted.into_iter().collect::<BTreeSet<_>>();
let expected = account_ids.iter().cloned().collect::<BTreeSet<_>>();
if deleted == expected {
@@ -1022,14 +1014,14 @@ pub(crate) async fn finish_admin_transaction<T>(
transaction
.commit()
.await
.map_err(|_| postgres_unavailable(operation))?;
.map_err(|source| postgres_unavailable(operation, source))?;
Ok(value)
}
Err(error) => {
transaction
.rollback()
.await
.map_err(|_| postgres_unavailable(operation))?;
.map_err(|source| postgres_unavailable(operation, source))?;
Err(error)
}
}
@@ -465,11 +465,11 @@ pub(crate) fn core_account_from_summary(
summary: ProviderAccountSummary,
) -> Result<CoreProviderAccount, CoreStoreError> {
let id = CoreProviderAccountId::new(summary.id)
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
let provider = ProviderKind::new(summary.provider_kind)
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
let revision = CoreCredentialRevision::new(summary.credential_revision.get())
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
.map_err(|source| CoreStoreError::caused_by(CoreStoreErrorKind::InvalidData, source))?;
Ok(CoreProviderAccount::new(
id,
provider,
@@ -501,17 +501,6 @@ pub(crate) fn core_account_from_summary(
))
}
pub(crate) fn core_store_error(error: StoreError) -> CoreStoreError {
let kind = match error {
StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable,
StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict,
StoreError::NotFound { .. } | StoreError::InvalidData { .. } => {
CoreStoreErrorKind::InvalidData
}
};
CoreStoreError::new(kind)
}
pub(crate) fn require_core_update(updated: bool) -> Result<(), CoreStoreError> {
if updated {
Ok(())
@@ -67,7 +67,7 @@ async fn exclude_active_imports(
.bind(id)
.fetch_one(&mut **transaction)
.await
.map_err(|_| unavailable())?;
.map_err(unavailable)?;
if !acquired {
return Err(conflict(id));
}
@@ -77,8 +77,8 @@ async fn exclude_active_imports(
fn store_error(error: StoreError) -> AdminStoreError {
admin_store_error(ENTITY, error)
}
fn unavailable() -> StoreError {
postgres_unavailable("outbound proxy operation")
fn unavailable(source: impl std::error::Error + Send + Sync + 'static) -> StoreError {
postgres_unavailable("outbound proxy operation", source)
}
fn conflict(id: &str) -> StoreError {
StoreError::Conflict {
@@ -213,7 +213,7 @@ async fn transaction_record(
.build()
.fetch_optional(&mut **transaction)
.await
.map_err(|_| unavailable())?
.map_err(unavailable)?
.ok_or_else(|| conflict(id))?;
record(row)
}
@@ -234,7 +234,7 @@ async fn save_test(
.bind(result.exit_ip.map(|ip| ip.to_string())).bind(result.exit_ipv4.map(|ip| ip.to_string()))
.bind(result.exit_ipv6.map(|ip| ip.to_string())).bind(result.message)
.bind(sqlx::types::Json(result.location)).bind(detected.map(sqlx::types::Json))
.execute(&mut **transaction).await.map_err(|_| unavailable())?;
.execute(&mut **transaction).await.map_err(unavailable)?;
Ok(())
}
@@ -252,7 +252,7 @@ async fn save_location(
.bind(location.map(|value| value.region.trim()))
.bind(location.map(|value| value.city.trim()))
.bind(location.map(|value| value.timezone.name()))
.execute(&mut **transaction).await.map_err(|_| unavailable())?;
.execute(&mut **transaction).await.map_err(unavailable)?;
Ok(())
}
@@ -264,7 +264,7 @@ async fn lock_url(
.bind(proxy.expose_url())
.execute(&mut **transaction)
.await
.map_err(|_| unavailable())?;
.map_err(unavailable)?;
Ok(())
}
@@ -276,7 +276,7 @@ pub(crate) async fn ensure_proxy_for_url(
) -> StoreResult<(String, bool)> {
lock_url(transaction, proxy).await?;
if let Some(id) = sqlx::query_scalar::<_, String>("select id from outbound_proxies where proxy_url = $1 order by created_at, id limit 1 for share")
.bind(proxy.expose_url()).fetch_optional(&mut **transaction).await.map_err(|_| unavailable())? {
.bind(proxy.expose_url()).fetch_optional(&mut **transaction).await.map_err(unavailable)? {
return Ok((id, false));
}
let id = format!("proxy_{}", uuid::Uuid::now_v7().simple());
@@ -287,7 +287,7 @@ pub(crate) async fn ensure_proxy_for_url(
.bind(proxy.expose_url())
.execute(&mut **transaction)
.await
.map_err(|_| unavailable())?;
.map_err(unavailable)?;
Ok((id, true))
}
@@ -308,7 +308,7 @@ pub(crate) async fn resolve_proxy_selection(
.bind(id)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| unavailable())?
.map_err(unavailable)?
.ok_or_else(|| StoreError::NotFound {
entity: ENTITY,
id: id.clone(),
@@ -354,7 +354,7 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let revision = bump_config_revision_in_transaction(&mut transaction)
.await
.map_err(store_error)?;
@@ -369,7 +369,7 @@ impl ProxyStore for PgProxyRepository {
.bind(proxy_id)
.execute(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
if updated.rows_affected() != 1 {
return Err(store_error(conflict(proxy_id)));
}
@@ -388,7 +388,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
admin_revision(revision)
}
@@ -403,12 +403,12 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
// 数量和当前页共用只读快照,避免绑定变化使同一次响应的分页事实不一致
sqlx::query("set transaction isolation level repeatable read, read only")
.execute(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let total: i64 = sqlx::query_scalar(
"select count(a.id) from outbound_proxies p
left join provider_accounts a on a.outbound_proxy_id = p.id
@@ -420,7 +420,7 @@ impl ProxyStore for PgProxyRepository {
.bind(&query.search)
.fetch_optional(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?
.map_err(|source| store_error(unavailable(source)))?
.ok_or_else(|| {
store_error(StoreError::NotFound {
entity: ENTITY,
@@ -439,7 +439,7 @@ impl ProxyStore for PgProxyRepository {
.bind(i64::from(query.page - 1) * i64::from(query.page_size.get()))
.fetch_all(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?
.map_err(|source| store_error(unavailable(source)))?
.into_iter()
.map(|row| {
Ok(ProxyAccountRef {
@@ -468,7 +468,7 @@ impl ProxyStore for PgProxyRepository {
.bind(&account_ids)
.fetch_all(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let mut groups = BTreeMap::<String, Vec<AccountGroupRef>>::new();
for (account_id, group_id, name, color, enabled) in rows {
groups.entry(account_id).or_default().push(AccountGroupRef {
@@ -486,7 +486,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
Ok(ProxyAccountPage {
items,
total: u64::try_from(total).map_err(|_| store_error(invalid()))?,
@@ -505,14 +505,14 @@ impl ProxyStore for PgProxyRepository {
.pool
.acquire()
.await
.map_err(|_| store_error(unavailable()))?
.map_err(|source| store_error(unavailable(source)))?
.detach();
let acquired: bool =
sqlx::query_scalar("select pg_try_advisory_lock_shared(hashtextextended($1, 739219))")
.bind(id)
.fetch_one(&mut connection)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
if !acquired {
return Err(store_error(conflict(id)));
}
@@ -524,7 +524,7 @@ impl ProxyStore for PgProxyRepository {
.bind(id)
.execute(&mut connection)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
return Err(error);
}
};
@@ -550,7 +550,7 @@ impl ProxyStore for PgProxyRepository {
.bind(&query.search)
.fetch_one(&self.pool)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let mut builder = QueryBuilder::<Postgres>::new(SELECT);
builder
.push(" where strpos(lower(p.name), lower(")
@@ -563,7 +563,7 @@ impl ProxyStore for PgProxyRepository {
.build()
.fetch_all(&self.pool)
.await
.map_err(|_| store_error(unavailable()))?
.map_err(|source| store_error(unavailable(source)))?
.into_iter()
.map(record)
.collect::<StoreResult<Vec<_>>>()
@@ -583,7 +583,7 @@ impl ProxyStore for PgProxyRepository {
.build()
.fetch_optional(&self.pool)
.await
.map_err(|_| store_error(unavailable()))?
.map_err(|source| store_error(unavailable(source)))?
.ok_or_else(|| {
store_error(StoreError::NotFound {
entity: ENTITY,
@@ -602,7 +602,7 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let revision = bump_config_revision_in_transaction(&mut transaction)
.await
.map_err(store_error)?;
@@ -621,7 +621,7 @@ impl ProxyStore for PgProxyRepository {
.bind(command.auto_location)
.execute(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
if let Some(test) = command.test {
let current = transaction_record(&mut transaction, &id)
.await
@@ -643,7 +643,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
Ok(ProxyMutation {
config_revision: admin_revision(revision)?,
record: self.get(&id).await?,
@@ -659,7 +659,7 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
exclude_active_imports(&mut transaction, &command.id)
.await
.map_err(store_error)?;
@@ -677,7 +677,7 @@ impl ProxyStore for PgProxyRepository {
.bind(&command.id)
.fetch_one(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
if duplicate {
return Err(store_error(conflict(&command.id)));
}
@@ -697,7 +697,7 @@ impl ProxyStore for PgProxyRepository {
where id = $1 and revision = $2")
.bind(&command.id).bind(i64::try_from(command.revision.get()).map_err(|_| store_error(invalid()))?)
.bind(&command.name).bind(command.proxy.as_ref().map(OutboundProxy::expose_url)).bind(command.auto_location)
.execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?;
.execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?;
if changed.rows_affected() != 1 {
return Err(store_error(conflict(&command.id)));
}
@@ -715,7 +715,7 @@ impl ProxyStore for PgProxyRepository {
.map_err(store_error)?;
}
sqlx::query("update provider_accounts a set outbound_proxy_url = p.proxy_url, updated_at = greatest(now(), a.updated_at) from outbound_proxies p where p.id = $1 and a.outbound_proxy_id = p.id and a.outbound_proxy_url is distinct from p.proxy_url")
.bind(&command.id).execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?;
.bind(&command.id).execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?;
audit(
&mut transaction,
context,
@@ -729,7 +729,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
Ok(ProxyMutation {
config_revision: admin_revision(revision)?,
record: self.get(&command.id).await?,
@@ -746,7 +746,7 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
exclude_active_imports(&mut transaction, id)
.await
.map_err(store_error)?;
@@ -765,7 +765,7 @@ impl ProxyStore for PgProxyRepository {
}) {
store_error(conflict(id))
} else {
store_error(unavailable())
store_error(unavailable(error))
}
})?;
if result.rows_affected() != 1 {
@@ -784,7 +784,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
admin_revision(config_revision)
}
@@ -799,7 +799,7 @@ impl ProxyStore for PgProxyRepository {
.pool
.begin()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
exclude_active_imports(&mut transaction, id)
.await
.map_err(store_error)?;
@@ -809,7 +809,7 @@ impl ProxyStore for PgProxyRepository {
)
.fetch_one(&mut *transaction)
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
let current = transaction_record(&mut transaction, id)
.await
.map_err(store_error)?;
@@ -818,7 +818,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.rollback()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
return Err(store_error(conflict(id)));
}
save_test(&mut transaction, &current, result)
@@ -826,7 +826,7 @@ impl ProxyStore for PgProxyRepository {
.map_err(store_error)?;
if current.auto_location {
sqlx::query("update outbound_proxies set revision = revision + 1, updated_at = now() where id = $1")
.bind(id).execute(&mut *transaction).await.map_err(|_| store_error(unavailable()))?;
.bind(id).execute(&mut *transaction).await.map_err(|source| store_error(unavailable(source)))?;
}
let updated = transaction_record(&mut transaction, id)
.await
@@ -852,7 +852,7 @@ impl ProxyStore for PgProxyRepository {
transaction
.commit()
.await
.map_err(|_| store_error(unavailable()))?;
.map_err(|source| store_error(unavailable(source)))?;
Ok(ProxyMutation {
config_revision: admin_revision(revision)?,
record: updated,
@@ -34,7 +34,7 @@ impl RetentionStore for PgRetentionRepository {
)
.fetch_optional(&self.pool)
.await
.map_err(|_| unavailable("load retention settings"))?
.map_err(|source| unavailable("load retention settings", source))?
.ok_or_else(|| {
AdminStoreError::new(
AdminStoreErrorKind::NotFound,
@@ -89,12 +89,15 @@ impl RetentionStore for PgRetentionRepository {
.execute(&self.pool)
.await
.map(|result| result.rows_affected())
.map_err(|_| unavailable("delete expired records"))
.map_err(|source| unavailable("delete expired records", source))
}
}
fn unavailable(operation: &'static str) -> AdminStoreError {
admin_store_error("retention", postgres_unavailable(operation))
fn unavailable(
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> AdminStoreError {
admin_store_error("retention", postgres_unavailable(operation, source))
}
fn invalid_policy() -> AdminStoreError {
@@ -248,12 +248,12 @@ impl RuntimeSettingsRepository for PgRuntimeSettingsRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin runtime settings update"))?;
.map_err(|source| postgres_unavailable("begin runtime settings update", source))?;
let revision = update_runtime_settings_in_transaction(&mut transaction, &update).await?;
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit runtime settings update"))?;
.map_err(|source| postgres_unavailable("commit runtime settings update", source))?;
Ok(revision)
}
}
@@ -274,7 +274,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
)
.fetch_optional(pool)
.await
.map_err(|_| postgres_unavailable("load runtime settings"))?
.map_err(|source| postgres_unavailable("load runtime settings", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -300,7 +300,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
.bind(slot)
.execute(&self.pool)
.await
.map_err(|_| provider_unavailable("claim warmup slot"))?
.map_err(|source| crate::provider_unavailable("claim warmup slot", source))?
.rows_affected()
== 1;
Ok(claimed)
@@ -325,7 +325,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
.bind(provider.as_str())
.bind(sqlx::types::Json(initial.expose_to_provider()))
.fetch_one(&self.pool).await
.map_err(|_| provider_unavailable("initialize Provider request profile"))?;
.map_err(|source| crate::provider_unavailable("initialize Provider request profile", source))?;
Ok(gateway_core::account::OpaqueProviderData::new(document.0))
})
}
@@ -356,7 +356,9 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
.bind(provider.as_str())
.fetch_all(&self.pool)
.await
.map_err(|_| provider_unavailable("load Provider request profile configurations"))?;
.map_err(|source| {
crate::provider_unavailable("load Provider request profile configurations", source)
})?;
let expected_revision = i64::try_from(revision.get())
.map_err(|_| provider_invalid("validate Provider request profile revision"))?;
if rows.is_empty()
@@ -386,7 +388,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
Box::pin(async move {
let settings = RuntimeSettingsRepository::load_runtime_settings(self)
.await
.map_err(|_| provider_unavailable("load refresh policy"))?;
.map_err(|source| crate::provider_unavailable("load refresh policy", source))?;
let concurrency = NonZeroU32::new(settings.refresh_concurrency)
.ok_or_else(|| provider_invalid("decode refresh policy"))?;
ProviderRefreshPolicy::try_new(
@@ -402,7 +404,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
Box::pin(async move {
let settings = RuntimeSettingsRepository::load_runtime_settings(self)
.await
.map_err(|_| provider_unavailable("load freeze policy"))?;
.map_err(|source| crate::provider_unavailable("load freeze policy", source))?;
ProviderFreezePolicy::try_new(
settings.account_auto_freeze_enabled,
settings.account_auto_freeze_threshold,
@@ -421,7 +423,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
Box::pin(async move {
let settings = RuntimeSettingsRepository::load_runtime_settings(self)
.await
.map_err(|_| provider_unavailable("load warmup policy"))?;
.map_err(|source| crate::provider_unavailable("load warmup policy", source))?;
ProviderWarmupPolicy::try_new(
settings.account_warmup_enabled,
settings.account_warmup_schedule_time,
@@ -449,7 +451,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load runtime settings in transaction"))?
.map_err(|source| postgres_unavailable("load runtime settings in transaction", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -564,7 +566,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
.bind(i64::from(update.openai_guardian_reserved_concurrency))
.fetch_optional(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("update runtime settings in transaction"))?
.map_err(|source| postgres_unavailable("update runtime settings in transaction", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -583,7 +585,7 @@ pub(crate) async fn bump_config_revision_in_transaction(
)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("bump config revision in transaction"))?
.map_err(|source| postgres_unavailable("bump config revision in transaction", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -605,7 +607,7 @@ pub(crate) async fn update_admin_api_key_in_transaction(
.bind(admin_api_key.as_deref())
.execute(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("update admin api key in transaction"))?;
.map_err(|source| postgres_unavailable("update admin api key in transaction", source))?;
Ok(())
}
@@ -732,10 +734,6 @@ fn invalid_numeric() -> StoreError {
}
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -98,11 +98,13 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
.pool
.begin()
.await
.map_err(|_| postgres_unavailable("begin runtime snapshot"))?;
.map_err(|source| postgres_unavailable("begin runtime snapshot", source))?;
sqlx::query("set transaction isolation level repeatable read read only")
.execute(&mut *transaction)
.await
.map_err(|_| postgres_unavailable("configure runtime snapshot transaction"))?;
.map_err(|source| {
postgres_unavailable("configure runtime snapshot transaction", source)
})?;
let (config_revision, settings) = load_settings(&mut transaction).await?;
let client_api_keys = load_client_keys(&mut transaction).await?;
@@ -112,7 +114,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
transaction
.commit()
.await
.map_err(|_| postgres_unavailable("commit runtime snapshot"))?;
.map_err(|source| postgres_unavailable("commit runtime snapshot", source))?;
let observed_current_revision =
RuntimeSnapshotRepository::current_config_revision(self).await?;
@@ -133,7 +135,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
)
.fetch_optional(&self.pool)
.await
.map_err(|_| postgres_unavailable("read current config revision"))?
.map_err(|source| postgres_unavailable("read current config revision", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -284,7 +286,7 @@ async fn load_settings(
)
.fetch_optional(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load snapshot settings"))?
.map_err(|source| postgres_unavailable("load snapshot settings", source))?
.ok_or_else(|| StoreError::NotFound {
entity: "runtime settings",
id: "1".to_owned(),
@@ -349,7 +351,7 @@ async fn load_client_keys(
)
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load snapshot client policies"))?;
.map_err(|source| postgres_unavailable("load snapshot client policies", source))?;
rows.into_iter()
.map(|row| {
let mut key = ClientApiKeySnapshot::from_persisted(row.0, row.1, row.2, row.3, row.4)?;
@@ -367,7 +369,7 @@ async fn load_account_groups(
)
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load snapshot account groups"))?;
.map_err(|source| postgres_unavailable("load snapshot account groups", source))?;
rows.into_iter()
.map(|(id, name, enabled, fast_mode)| {
Ok(SnapshotAccountGroupData {
@@ -394,7 +396,7 @@ async fn load_provider_accounts(
>("select id, provider_kind, model_access_json from provider_accounts order by id")
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load snapshot provider accounts"))
.map_err(|source| postgres_unavailable("load snapshot provider accounts", source))
.map(|rows| {
rows.into_iter()
.map(
@@ -417,7 +419,7 @@ async fn load_group_memberships(
)
.fetch_all(&mut **transaction)
.await
.map_err(|_| postgres_unavailable("load snapshot group memberships"))?;
.map_err(|source| postgres_unavailable("load snapshot group memberships", source))?;
rows.into_iter()
.map(|(group_id, account_id)| {
Ok(SnapshotGroupMembershipData {
@@ -97,7 +97,9 @@ impl ProviderArtifactProfileCachePort for RedisProviderArtifactProfileRepository
.arg(ttl_ms)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("replace artifact profile"))?;
.map_err(|source| {
crate::provider_unavailable("replace artifact profile", source)
})?;
match outcome {
1 => Ok(true),
0 => Ok(false),
@@ -126,7 +128,7 @@ impl ProviderArtifactProfileCachePort for RedisProviderArtifactProfileRepository
.arg("profile")
.query_async(&mut connection)
.await
.map_err(|_| provider_unavailable("read artifact profile"))?;
.map_err(|source| crate::provider_unavailable("read artifact profile", source))?;
let (artifact_sequence, verified_at_ms, payload) = values;
if artifact_sequence.is_none() && verified_at_ms.is_none() && payload.is_none() {
return Ok(None);
@@ -171,10 +173,6 @@ fn exact_positive_millis(
.ok_or_else(|| provider_invalid(operation))
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -162,7 +162,7 @@ return current
.arg(expiry)
.invoke_async(&mut self.connection.clone())
.await
.map_err(|_| redis_unavailable("renew authentication session"))?;
.map_err(|source| redis_unavailable("renew authentication session", source))?;
payload.map(|value| decode_session(&value)).transpose()
}
@@ -173,7 +173,7 @@ return current
.arg(key)
.query_async::<Option<String>>(&mut connection)
.await
.map_err(|_| redis_unavailable("load authentication session"))?;
.map_err(|source| redis_unavailable("load authentication session", source))?;
payload.map(|value| decode_session(&value)).transpose()
}
@@ -193,7 +193,7 @@ return current
.arg(expires_at_millis)
.query_async::<String>(&mut connection)
.await
.map_err(|_| redis_unavailable("store authentication session"))?;
.map_err(|source| redis_unavailable("store authentication session", source))?;
Ok(())
}
@@ -204,7 +204,7 @@ return current
.arg(key)
.query_async::<Option<String>>(&mut connection)
.await
.map_err(|_| redis_unavailable("delete authentication session"))?;
.map_err(|source| redis_unavailable("delete authentication session", source))?;
payload.map(|value| decode_session(&value)).transpose()
}
@@ -232,7 +232,7 @@ return current
.arg(window_millis)
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("rate limit authentication login"))?;
.map_err(|source| redis_unavailable("rate limit authentication login", source))?;
if retry_after_millis <= 0 {
Ok(None)
} else {
@@ -293,7 +293,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository {
.arg(u8::from(request.allow_concurrency_acquire))
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("admit client request"))?;
.map_err(|source| redis_unavailable("admit client request", source))?;
match code {
0 => Ok(ClientAdmissionDecision::Granted),
1 => Ok(ClientAdmissionDecision::Rejected(
@@ -320,7 +320,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository {
.arg(model_request_id)
.query_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("release client request"))?;
.map_err(|source| redis_unavailable("release client request", source))?;
Ok(removed == 1)
}
@@ -361,7 +361,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository {
let (code, restored_recent_requests, restored_running_requests) = invocation
.invoke_async::<(i64, u64, u64)>(&mut connection)
.await
.map_err(|_| redis_unavailable("restore client admission"))?;
.map_err(|source| redis_unavailable("restore client admission", source))?;
if code == -1 {
return Err(invalid("request start time is after Redis server time"));
}
@@ -381,7 +381,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository {
.arg(&keys)
.query_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("clear client admission"))?;
.map_err(|source| redis_unavailable("clear client admission", source))?;
Ok(())
}
}
@@ -413,7 +413,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository {
.arg(redis_duration_millis(ttl)?)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("renew client request"))?;
.map_err(|source| redis_unavailable("renew client request", source))?;
Ok(renewed == 1)
})
},
@@ -229,7 +229,7 @@ impl RedisCredentialCooldownRepository {
.invoke_async::<i64>(&mut connection)
.await
}
.map_err(|_| redis_unavailable("cache credential cooldown"))?;
.map_err(|source| redis_unavailable("cache credential cooldown", source))?;
Ok(written == 1)
}
@@ -254,7 +254,7 @@ impl RedisCredentialCooldownRepository {
.await
};
let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) =
result.map_err(|_| redis_unavailable("read credential cooldown"))?;
result.map_err(|source| redis_unavailable("read credential cooldown", source))?;
if present == 0 {
return Ok(None);
}
@@ -298,7 +298,7 @@ impl RedisCredentialCooldownRepository {
.invoke_async::<i64>(&mut connection)
.await
}
.map_err(|_| redis_unavailable("invalidate credential cooldown"))?;
.map_err(|source| redis_unavailable("invalidate credential cooldown", source))?;
Ok(removed == 1)
}
@@ -310,7 +310,7 @@ impl RedisCredentialCooldownRepository {
.arg(-1)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("list indexed cooldowns"))
.map_err(|source| redis_unavailable("list indexed cooldowns", source))
}
pub(crate) async fn active_cooldowns(&self) -> StoreResult<AccountRuntimeSnapshot> {
@@ -384,7 +384,7 @@ impl RedisCredentialCooldownRepository {
.arg(uuid::Uuid::new_v4().to_string())
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("finish capacity freeze"))?;
.map_err(|source| redis_unavailable("finish capacity freeze", source))?;
Ok(changed == 1)
}
@@ -398,7 +398,7 @@ impl RedisCredentialCooldownRepository {
.arg(self.capacity_peak_key(provider_account_id)?)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("read capacity peak in-flight"))?;
.map_err(|source| redis_unavailable("read capacity peak in-flight", source))?;
peak.map(u32::try_from)
.transpose()
.map_err(|_| invalid("capacity peak in-flight is invalid"))
@@ -493,7 +493,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
.arg(100)
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("scan account cooldown keys"))?;
.map_err(|source| redis_unavailable("scan account cooldown keys", source))?;
keys.extend(found);
cursor = next;
if cursor == 0 {
@@ -513,7 +513,7 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
.arg(provider_account_id)
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("delete account cooldowns and index"))?;
.map_err(|source| redis_unavailable("delete account cooldowns and index", source))?;
Ok(removed > 0)
}
}
@@ -533,7 +533,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
};
CredentialCooldownRepository::cache_credential_cooldown(self, &record)
.await
.map_err(|_| provider_unavailable("cache credential cooldown"))
.map_err(|source| crate::provider_unavailable("cache credential cooldown", source))
})
}
@@ -544,7 +544,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
Box::pin(async move {
CredentialCooldownRepository::read_credential_cooldown(self, account_id.as_str())
.await
.map_err(|_| provider_unavailable("read credential cooldown"))?
.map_err(|source| crate::provider_unavailable("read credential cooldown", source))?
.map(|record| {
let account_id = ProviderAccountId::new(record.provider_account_id)
.map_err(|_| provider_invalid("decode credential cooldown"))?;
@@ -575,7 +575,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
revision,
)
.await
.map_err(|_| provider_unavailable("clear credential cooldown"))
.map_err(|source| crate::provider_unavailable("clear credential cooldown", source))
})
}
@@ -595,7 +595,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
None,
)
.await
.map_err(|_| provider_unavailable("cache scoped credential cooldown"))
.map_err(|source| {
crate::provider_unavailable("cache scoped credential cooldown", source)
})
})
}
@@ -612,7 +614,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
None,
)
.await
.map_err(|_| provider_unavailable("read scoped credential cooldown"))?
.map_err(|source| {
crate::provider_unavailable("read scoped credential cooldown", source)
})?
.map(|(revision, until, _, _)| {
Ok(ProviderScopedCooldown::new(
account_id.clone(),
@@ -642,7 +646,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
None,
)
.await
.map_err(|_| provider_unavailable("clear scoped credential cooldown"))
.map_err(|source| {
crate::provider_unavailable("clear scoped credential cooldown", source)
})
})
}
@@ -653,7 +659,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
Box::pin(async move {
self.delete_account_cooldowns(account_id.as_str())
.await
.map_err(|_| provider_unavailable("clear all credential cooldowns"))
.map_err(|source| {
crate::provider_unavailable("clear all credential cooldowns", source)
})
})
}
@@ -680,7 +688,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.arg(i64::from(in_flight))
.invoke_async(&mut connection)
.await
.map_err(|_| provider_unavailable("record capacity failure"))?;
.map_err(|source| crate::provider_unavailable("record capacity failure", source))?;
u32::try_from(count).map_err(|_| provider_invalid("decode capacity failure count"))
})
}
@@ -710,7 +718,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.arg(account_id.as_str())
.invoke_async(&mut connection)
.await
.map_err(|_| provider_unavailable("clear cooldown after success"))?;
.map_err(|source| {
crate::provider_unavailable("clear cooldown after success", source)
})?;
Ok(())
})
}
@@ -722,7 +732,9 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
Box::pin(async move {
self.read_capacity_peak(account_id.as_str())
.await
.map_err(|_| provider_unavailable("read capacity peak in-flight"))
.map_err(|source| {
crate::provider_unavailable("read capacity peak in-flight", source)
})
})
}
}
@@ -734,10 +746,6 @@ fn invalid(message: &str) -> StoreError {
}
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -384,7 +384,7 @@ impl RedisCredentialLeaseRepository {
.arg(SIGNAL_TTL_MILLIS)
.invoke_async::<String>(&mut connection)
.await
.map_err(|_| redis_unavailable("advance provider scheduling cursor"))?;
.map_err(|source| redis_unavailable("advance provider scheduling cursor", source))?;
cursor
.parse::<u64>()
.map_err(|_| invalid("Redis returned an invalid scheduling cursor"))
@@ -458,7 +458,7 @@ impl RedisCredentialLeaseRepository {
.arg(signal_ttl)
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("acquire credential lease"))?;
.map_err(|source| redis_unavailable("acquire credential lease", source))?;
if acquired == 0 {
return Ok(LeaseAttempt {
grant: None,
@@ -485,7 +485,7 @@ impl RedisCredentialLeaseRepository {
.key(&keys[2])
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("load credential runtime signal"))?;
.map_err(|source| redis_unavailable("load credential runtime signal", source))?;
let in_flight = in_flight
.parse::<u32>()
.map_err(|_| invalid("Redis returned an invalid in-flight count"))?;
@@ -532,7 +532,7 @@ impl RedisProviderLeaseCoordinator {
self.repository
.advance_scheduling_cursor(client_api_key_id, provider_kind)
.await
.map_err(|_| provider_unavailable("advance scheduling cursor"))
.map_err(|source| crate::provider_unavailable("advance scheduling cursor", source))
}
async fn load_signals(
@@ -547,7 +547,7 @@ impl RedisProviderLeaseCoordinator {
.repository
.credential_runtime_signals(&ids)
.await
.map_err(|_| provider_unavailable("load scheduling signals"))?;
.map_err(|source| crate::provider_unavailable("load scheduling signals", source))?;
signals
.into_iter()
.map(|signal| {
@@ -579,7 +579,10 @@ impl RedisProviderLeaseCoordinator {
) -> Result<ProviderLeaseAcquisition, ProviderStoreError> {
let ttl = request.deadline().bounded(REQUEST_LEASE_TTL);
if ttl.is_zero() {
return Err(provider_unavailable("acquire expired scheduling lease"));
return Err(ProviderStoreError::new(
ProviderStoreErrorKind::Unavailable,
"acquire expired scheduling lease",
));
}
let acquisition = self
.repository
@@ -592,13 +595,15 @@ impl RedisProviderLeaseCoordinator {
ttl,
})
.await
.map_err(|_| provider_unavailable("acquire scheduling lease"))?;
.map_err(|source| crate::provider_unavailable("acquire scheduling lease", source))?;
Ok(match acquisition {
CredentialBoundedLeaseAcquisition::Acquired(guard) => {
ProviderLeaseAcquisition::Acquired(Box::new(
guard
.maintain(request.deadline(), request.cancellation())
.map_err(|_| provider_unavailable("maintain scheduling lease"))?,
.map_err(|source| {
crate::provider_unavailable("maintain scheduling lease", source)
})?,
))
}
CredentialBoundedLeaseAcquisition::Busy { retry_after } => {
@@ -622,7 +627,7 @@ impl RedisProviderLeaseCoordinator {
ttl: OAUTH_REFRESH_LEASE_TTL,
})
.await
.map_err(|_| provider_unavailable("acquire refresh capacity"))?;
.map_err(|source| crate::provider_unavailable("acquire refresh capacity", source))?;
Ok(match acquisition {
CredentialBoundedLeaseAcquisition::Acquired(guard) => {
ProviderLeaseAcquisition::Acquired(Box::new(guard))
@@ -676,7 +681,9 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator {
Some(guard) => ProviderLeaseAcquisition::Acquired(Box::new(guard)),
None => ProviderLeaseAcquisition::Busy { retry_after: None },
})
.map_err(|_| provider_unavailable("acquire refresh lease"))
.map_err(|source| {
crate::provider_unavailable("acquire refresh lease", source)
})
}
}
})
@@ -701,7 +708,9 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator {
.repository
.credential_runtime_signals(&ids)
.await
.map_err(|_| provider_unavailable("load account in-flight signals"))?;
.map_err(|source| {
crate::provider_unavailable("load account in-flight signals", source)
})?;
Ok(signals
.into_iter()
.filter_map(|signal| {
@@ -713,10 +722,6 @@ impl ProviderLeasePort for RedisProviderLeaseCoordinator {
}
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
#[async_trait]
impl CredentialLeaseRepository for RedisCredentialLeaseRepository {
async fn acquire_credential_lease(
@@ -743,7 +748,7 @@ impl CredentialLeaseRepository for RedisCredentialLeaseRepository {
.arg(duration_millis(request.ttl)?)
.invoke_async(&mut connection)
.await
.map_err(|_| redis_unavailable("renew credential lease"))?;
.map_err(|source| redis_unavailable("renew credential lease", source))?;
if renewed == 0 {
return Ok(None);
}
@@ -768,7 +773,7 @@ impl CredentialLeaseRepository for RedisCredentialLeaseRepository {
.arg(member)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("release credential lease"))?;
.map_err(|source| redis_unavailable("release credential lease", source))?;
Ok(released == 1)
}
@@ -178,7 +178,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
.arg(ttl_ms)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("cache credential state"))?;
.map_err(|source| redis_unavailable("cache credential state", source))?;
Ok(written == 1)
}
@@ -200,7 +200,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
.arg("observed_at_ms")
.query_async(&mut connection)
.await
.map_err(|_| redis_unavailable("read credential state"))?;
.map_err(|source| redis_unavailable("read credential state", source))?;
let (Some(revision), Some(enabled), Some(credential_state), Some(observed_at_ms)) = values
else {
return Ok(None);
@@ -228,7 +228,7 @@ impl CredentialStateRepository for RedisCredentialStateRepository {
.arg(self.key(provider_account_id)?)
.query_async::<i64>(&mut connection)
.await
.map_err(|_| redis_unavailable("clear credential state"))?;
.map_err(|source| redis_unavailable("clear credential state", source))?;
Ok(removed == 1)
}
}
@@ -262,7 +262,7 @@ impl ProviderCatalogCacheRepository for RedisCredentialStateRepository {
.arg(ttl_ms)
.query_async::<()>(&mut connection)
.await
.map_err(|_| redis_unavailable("replace provider catalog cache"))?;
.map_err(|source| redis_unavailable("replace provider catalog cache", source))?;
Ok(())
}
@@ -275,7 +275,7 @@ impl ProviderCatalogCacheRepository for RedisCredentialStateRepository {
.arg(self.catalog_key(key)?)
.query_async::<Option<Vec<u8>>>(&mut connection)
.await
.map_err(|_| redis_unavailable("read provider catalog cache"))?;
.map_err(|source| redis_unavailable("read provider catalog cache", source))?;
let Some(payload) = payload else {
return Ok(None);
};
@@ -310,7 +310,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
)
.await
.map(|_| ())
.map_err(|_| provider_unavailable("replace credential state"))
.map_err(|source| crate::provider_unavailable("replace credential state", source))
})
}
@@ -323,7 +323,9 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
let cached =
CredentialStateRepository::read_credential_state(self, account_id.as_str())
.await
.map_err(|_| provider_unavailable("read credential state"))?;
.map_err(|source| {
crate::provider_unavailable("read credential state", source)
})?;
cached
.map(|state| {
let account_id = ProviderAccountId::new(state.provider_account_id)
@@ -351,7 +353,7 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
Box::pin(async move {
CredentialStateRepository::clear_credential_state(self, account_id.as_str())
.await
.map_err(|_| provider_unavailable("clear credential state"))
.map_err(|source| crate::provider_unavailable("clear credential state", source))
})
}
@@ -367,14 +369,14 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
.ok_or_else(|| provider_invalid("record refresh backoff"))?;
let key = self
.backoff_key(account_id.as_str())
.map_err(|_| provider_unavailable("record refresh backoff"))?;
.map_err(|source| crate::provider_unavailable("record refresh backoff", source))?;
let mut connection = self.connection.clone();
let count = Script::new(RECORD_REFRESH_BACKOFF_SCRIPT)
.key(key)
.arg(window_millis)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("record refresh backoff"))?;
.map_err(|source| crate::provider_unavailable("record refresh backoff", source))?;
Ok(u32::try_from(count).unwrap_or(u32::MAX))
})
}
@@ -386,13 +388,13 @@ impl ProviderCredentialStatePort for RedisCredentialStateRepository {
Box::pin(async move {
let key = self
.backoff_key(account_id.as_str())
.map_err(|_| provider_unavailable("clear refresh backoff"))?;
.map_err(|source| crate::provider_unavailable("clear refresh backoff", source))?;
let mut connection = self.connection.clone();
redis::cmd("DEL")
.arg(key)
.query_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("clear refresh backoff"))?;
.map_err(|source| crate::provider_unavailable("clear refresh backoff", source))?;
Ok(())
})
}
@@ -421,7 +423,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository {
ttl_seconds,
)
.await
.map_err(|_| provider_unavailable("replace catalog cache"))
.map_err(|source| crate::provider_unavailable("replace catalog cache", source))
})
}
@@ -437,7 +439,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository {
};
ProviderCatalogCacheRepository::get_provider_catalog(self, &key)
.await
.map_err(|_| provider_unavailable("read catalog cache"))
.map_err(|source| crate::provider_unavailable("read catalog cache", source))
})
}
}
@@ -456,10 +458,6 @@ fn catalog_invalid(message: &str) -> StoreError {
}
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -178,7 +178,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository {
.arg(ttl_millis)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("create OAuth pending flow"))?;
.map_err(|source| {
crate::provider_unavailable("create OAuth pending flow", source)
})?;
Ok(if stored == 1 {
OAuthPendingPutOutcome::Stored
} else {
@@ -209,7 +211,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository {
.arg(claim_ttl)
.invoke_async(&mut connection)
.await
.map_err(|_| provider_unavailable("claim OAuth pending flow"))?;
.map_err(|source| {
crate::provider_unavailable("claim OAuth pending flow", source)
})?;
match status {
0 => Ok(OAuthPendingClaimOutcome::NotFound),
-1 => Ok(OAuthPendingClaimOutcome::OwnerMismatch),
@@ -248,7 +252,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository {
.arg(claim)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("release OAuth pending claim"))?;
.map_err(|source| {
crate::provider_unavailable("release OAuth pending claim", source)
})?;
Ok(match status {
0 => OAuthPendingReleaseOutcome::NotFound,
-1 => OAuthPendingReleaseOutcome::OwnerMismatch,
@@ -282,7 +288,9 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository {
.arg(claim)
.invoke_async::<i64>(&mut connection)
.await
.map_err(|_| provider_unavailable("consume OAuth pending claim"))?;
.map_err(|source| {
crate::provider_unavailable("consume OAuth pending claim", source)
})?;
Ok(match status {
0 => OAuthPendingConsumeOutcome::NotFound,
-1 => OAuthPendingConsumeOutcome::OwnerMismatch,
@@ -319,10 +327,6 @@ fn ttl_millis(ttl: Duration) -> StoreResult<u64> {
})
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -103,7 +103,9 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository {
.arg(self.key(provider_kind, key)?)
.query_async::<Option<String>>(&mut self.connection.clone())
.await
.map_err(|_| provider_unavailable("load provider session binding"))?;
.map_err(|source| {
crate::provider_unavailable("load provider session binding", source)
})?;
raw.as_deref().map(BindingRecord::decode).transpose()
})
}
@@ -138,7 +140,9 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository {
.arg(ttl_millis)
.invoke_async::<bool>(&mut self.connection.clone())
.await
.map_err(|_| provider_unavailable("admit provider session binding"))?;
.map_err(|source| {
crate::provider_unavailable("admit provider session binding", source)
})?;
Ok(applied.then_some(binding))
})
}
@@ -155,7 +159,7 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository {
.arg(format!("{}:alias", self.key(provider, alias)?))
.query_async::<Option<String>>(&mut self.connection.clone())
.await
.map_err(|_| provider_unavailable("load session alias"))?;
.map_err(|source| crate::provider_unavailable("load session alias", source))?;
value
.map(|raw| {
let record: AliasRecord = serde_json::from_str(&raw)
@@ -189,7 +193,7 @@ impl ProviderSessionAffinityPort for RedisProviderSessionAffinityRepository {
.arg(&record)
.arg(session_affinity_ttl_millis(ttl)?)
.invoke_async::<bool>(&mut self.connection.clone()).await
.map_err(|_| provider_unavailable("bind session alias"))
.map_err(|source| crate::provider_unavailable("bind session alias", source))
})
}
}
@@ -202,10 +206,6 @@ fn session_affinity_ttl_millis(ttl: Duration) -> Result<u64, ProviderStoreError>
.map_err(|_| provider_invalid("validate provider session affinity TTL"))
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -119,7 +119,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository {
.arg(self.key(provider_kind, key)?)
.query_async::<Option<String>>(&mut connection)
.await
.map_err(|_| provider_unavailable("load provider session exclusion"))?;
.map_err(|source| {
crate::provider_unavailable("load provider session exclusion", source)
})?;
payload.map(decode_state).transpose()
})
}
@@ -144,7 +146,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository {
.arg(ttl_seconds)
.invoke_async::<String>(&mut connection)
.await
.map_err(|_| provider_unavailable("record provider session exclusion"))?;
.map_err(|source| {
crate::provider_unavailable("record provider session exclusion", source)
})?;
decode_state(payload)
})
}
@@ -162,7 +166,9 @@ impl ProviderSessionExclusionPort for RedisProviderSessionExclusionRepository {
.arg(expected_revision)
.invoke_async::<u64>(&mut connection)
.await
.map_err(|_| provider_unavailable("clear provider session exclusion"))?;
.map_err(|source| {
crate::provider_unavailable("clear provider session exclusion", source)
})?;
Ok(removed > 0)
})
}
@@ -185,10 +191,6 @@ fn decode_state(payload: impl AsRef<str>) -> Result<ProviderSessionExclusions, P
))
}
fn provider_unavailable(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::Unavailable, operation)
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -54,11 +54,11 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository {
.client
.get_connection_manager()
.await
.map_err(|_| redis_unavailable("connect runtime change publisher"))?;
.map_err(|source| redis_unavailable("connect runtime change publisher", source))?;
connection
.publish::<_, _, i64>(&self.channel, payload)
.await
.map_err(|_| redis_unavailable("publish runtime change"))?;
.map_err(|source| redis_unavailable("publish runtime change", source))?;
Ok(())
}
@@ -67,11 +67,11 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository {
.client
.get_async_pubsub()
.await
.map_err(|_| redis_unavailable("connect runtime change subscriber"))?;
.map_err(|source| redis_unavailable("connect runtime change subscriber", source))?;
pubsub
.subscribe(&self.channel)
.await
.map_err(|_| redis_unavailable("subscribe runtime changes"))?;
.map_err(|source| redis_unavailable("subscribe runtime changes", source))?;
let stream = pubsub.into_on_message().map(|message| {
let payload = message
.get_payload::<String>()
+35 -4
View File
@@ -23,12 +23,13 @@ pub enum ConflictKind {
}
/// Store adapter 的稳定错误边界
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[derive(Debug, Clone, thiserror::Error)]
pub enum StoreError {
#[error("{backend:?} store is unavailable: {message}")]
Unavailable {
backend: StoreBackend,
message: String,
source: Option<gateway_core::error::ErrorSource>,
},
#[error("{entity} {id} was not found")]
NotFound { entity: &'static str, id: String },
@@ -101,7 +102,7 @@ pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> Ad
StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid,
StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable,
};
AdminStoreError::new(kind, resource, "store operation failed")
AdminStoreError::new(kind, resource, "store operation failed").with_source(error)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -240,16 +241,46 @@ pub(crate) fn require_nonempty(
}
}
pub(crate) fn postgres_unavailable(operation: &'static str) -> StoreError {
pub(crate) fn postgres_unavailable(
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> StoreError {
StoreError::Unavailable {
backend: StoreBackend::PostgreSql,
message: operation.to_owned(),
source: Some(gateway_core::error::ErrorSource::new(source)),
}
}
pub(crate) fn redis_unavailable(operation: &'static str) -> StoreError {
pub(crate) fn redis_unavailable(
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> StoreError {
StoreError::Unavailable {
backend: StoreBackend::Redis,
message: operation.to_owned(),
source: Some(gateway_core::error::ErrorSource::new(source)),
}
}
/// Provider 存储端口共用不可用分类,实际后端及原因由来源链保留
pub(crate) fn provider_unavailable(
operation: &'static str,
source: impl std::error::Error + Send + Sync + 'static,
) -> gateway_core::provider_ports::ProviderStoreError {
gateway_core::provider_ports::ProviderStoreError::caused_by(
gateway_core::provider_ports::ProviderStoreErrorKind::Unavailable,
operation,
source,
)
}
pub(crate) fn core_store_error(error: StoreError) -> gateway_core::error::StoreError {
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind};
let kind = match error {
StoreError::Unavailable { .. } => StoreErrorKind::Unavailable,
StoreError::Conflict { .. } => StoreErrorKind::Conflict,
StoreError::NotFound { .. } | StoreError::InvalidData { .. } => StoreErrorKind::InvalidData,
};
CoreStoreError::caused_by(kind, error)
}
@@ -0,0 +1,23 @@
//! 验证实际基础设施错误在 Store 边界保留来源而不进入普通格式化
use std::error::Error as _;
#[tokio::test]
async fn invalid_postgres_options_preserve_the_sqlx_source() {
let error = gateway_store::postgres::connect_and_migrate(
"postgres://localhost:invalid-port/test",
gateway_store::StorePoolConfig::default(),
)
.await
.unwrap_err();
let source = error.source().expect("PostgreSQL source");
assert!(source.downcast_ref::<sqlx::Error>().is_some());
assert!(source.source().is_some());
assert!(
error
.to_string()
.contains("parse PostgreSQL connection options")
);
assert!(!format!("{error:?} {error}").contains("invalid-port"));
}
@@ -677,7 +677,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
provider_metadata_json: None,
error: None,
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
failure_observation: Default::default(),
retry_after_ms: None,
usage: Usage::new(),
@@ -829,8 +829,8 @@ async fn core_adapter_persists_opaque_response_ids_as_bytes() {
}
#[tokio::test]
async fn core_adapter_persists_raw_upstream_error_verbatim() {
let Some(database) = TestDatabase::create("execution_raw_upstream_error").await else {
async fn core_adapter_persists_native_causes_and_verbatim_upstream_details() {
let Some(database) = TestDatabase::create("execution_error_details").await else {
return;
};
seed_running_request(&database.pool, "req_raw_upstream_error")
@@ -838,6 +838,12 @@ async fn core_adapter_persists_raw_upstream_error_verbatim() {
.expect("seed model request");
let store = PgExecutionStore::new(database.pool.clone());
let raw = r#"{"error":{"message":"raw upstream body","opaque":"\u0000"}}"#;
let provider_error = gateway_core::error::ProviderError::new(
gateway_core::error::ProviderErrorKind::Unavailable,
UpstreamSendState::Sent,
)
.with_source(std::io::Error::other("original local cause"))
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(raw));
let mut finalization = successful_core_finalization("req_raw_upstream_error");
finalization.outcome = ExecutionOutcome::Failed;
finalization.client_status_code = Some(502);
@@ -846,19 +852,25 @@ async fn core_adapter_persists_raw_upstream_error_verbatim() {
GatewayErrorKind::UpstreamUnavailable,
"upstream service is unavailable",
));
finalization.raw_upstream_error = Some(raw.to_owned());
finalization.error_details = provider_error.error_details();
ExecutionStore::finalize_model_request(&store, finalization)
.await
.expect("persist raw upstream error");
.expect("persist restricted error details");
let persisted: Option<String> = sqlx::query_scalar(
"select raw_upstream_error from model_requests where id = 'req_raw_upstream_error'",
"select error_details from model_requests where id = 'req_raw_upstream_error'",
)
.fetch_one(&database.pool)
.await
.expect("load raw upstream error");
assert_eq!(persisted.as_deref(), Some(raw));
.expect("load restricted error details");
let persisted: Value = serde_json::from_str(persisted.as_deref().unwrap()).unwrap();
assert_eq!(persisted["upstream"], raw);
assert_eq!(
persisted["causes"]["messages"],
json!(["original local cause"])
);
assert_eq!(persisted["causes"]["truncated"], false);
database.close().await;
}
@@ -1527,7 +1539,7 @@ pub(super) fn early_failure(request: &CoreNewModelRequest) -> CoreModelRequestFi
"no available provider",
)),
provider_error_code: None,
raw_upstream_error: None,
error_details: None,
failure_observation: Default::default(),
retry_after_ms: None,
usage: Usage::new(),
@@ -1622,7 +1634,7 @@ async fn zero_attempt_failure_is_queryable_without_fabricating_upstream_facts()
"downstream_committed_at",
"service_tier",
"provider_observation_json",
"raw_upstream_error",
"error_details",
"input_tokens",
"output_tokens",
"total_tokens",
@@ -1675,7 +1687,7 @@ async fn zero_attempt_failure_is_queryable_without_fabricating_upstream_facts()
assert_eq!(error.upstream_transport, None);
assert_eq!(error.upstream_status_code, None);
assert_eq!(error.upstream_request_id, None);
assert_eq!(error.raw_upstream_error, None);
assert_eq!(error.error_details, None);
// 路由入口不等于已选择 Provider;没有平台事实时不能被平台/attempt 筛选命中
for filter in [
@@ -561,34 +561,63 @@ async fn shutdown_drains_observations_already_accepted_by_the_queue() {
#[tokio::test]
async fn observation_byte_budget_drops_payload_without_waiting_for_the_database() {
for provider_error in [
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
.with_source(std::io::Error::other("x".repeat(2_048))),
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
.with_upstream_code(OpaqueUpstreamValue::new("x".repeat(2_048))),
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
"x".repeat(2_048),
)),
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
.with_diagnostic(gateway_core::error::ProviderDiagnostic::new(
"x".repeat(2_048),
)),
] {
let inner = Arc::new(RecordingStore::default());
let (store, _writer) = BufferedExecutionStore::with_limits(
Arc::clone(&inner),
NonZeroUsize::new(8).expect("capacity"),
NonZeroUsize::new(1_024).expect("byte capacity"),
);
let failure = ProbeFailure {
provider_kind: ProviderKind::new("openai").expect("provider"),
account_id: ProviderAccountId::new("acct_byte_budget").expect("account"),
upstream_model_id: UpstreamModelId::new("gpt-byte-budget").expect("model"),
error: provider_error,
latency: Duration::from_millis(1),
};
tokio::time::timeout(
Duration::from_millis(50),
store.record_probe_failure(failure),
)
.await
.expect("byte budget must not wait")
.expect("byte budget is fail-open");
assert!(inner.operations.lock().expect("operations lock").is_empty());
assert_eq!(store.stats().queued_items, 0);
assert_eq!(store.stats().dropped_total, 1);
}
}
#[tokio::test]
async fn final_error_body_counts_towards_the_observation_byte_budget() {
let inner = Arc::new(RecordingStore::default());
let (store, _writer) = BufferedExecutionStore::with_limits(
Arc::clone(&inner),
NonZeroUsize::new(8).expect("capacity"),
NonZeroUsize::new(1_024).expect("byte capacity"),
NonZeroUsize::new(8).unwrap(),
NonZeroUsize::new(1_024).unwrap(),
);
let provider_error =
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
.with_upstream_code(OpaqueUpstreamValue::new("x".repeat(2_048)));
let failure = ProbeFailure {
provider_kind: ProviderKind::new("openai").expect("provider"),
account_id: ProviderAccountId::new("acct_byte_budget").expect("account"),
upstream_model_id: UpstreamModelId::new("gpt-byte-budget").expect("model"),
error: provider_error,
latency: Duration::from_millis(1),
};
tokio::time::timeout(
Duration::from_millis(50),
store.record_probe_failure(failure),
)
.await
.expect("byte budget must not wait")
.expect("byte budget is fail-open");
assert!(inner.operations.lock().expect("operations lock").is_empty());
assert_eq!(store.stats().queued_items, 0);
let request = accepted_request("req_error_body_budget");
let mut finalization = early_failure(&request);
finalization.error_details = Some("x".repeat(2_048));
store.finalize_model_request(finalization).await.unwrap();
assert_eq!(store.stats().dropped_total, 1);
assert_eq!(store.stats().queued_bytes, 0);
assert!(inner.operations.lock().unwrap().is_empty());
}
#[tokio::test]
@@ -18,6 +18,7 @@ mod admission_recovery;
mod backup;
mod client_budgets;
mod client_keys;
mod connection;
mod execution;
mod execution_buffer;
mod health;
@@ -777,7 +777,7 @@ async fn ops_should_include_incomplete_upstream_errors() {
client_transport = 'websocket', upstream_transport = 'websocket',
downstream_committed_at = completed_at,
client_status_code = null, upstream_status_code = 400,
provider_error_code = null, raw_upstream_error = $1
provider_error_code = null, error_details = $1
where id = 'req_observe_failed'",
)
.bind(raw_error)
@@ -810,7 +810,7 @@ async fn ops_should_include_incomplete_upstream_errors() {
assert_eq!(error.failure_kind, "invalid_request");
assert_eq!(error.upstream_status_code, Some(400));
assert_eq!(error.client_status_code, None);
assert_eq!(error.raw_upstream_error.as_deref(), Some(raw_error));
assert_eq!(error.error_details.as_deref(), Some(raw_error));
}
#[tokio::test]
@@ -18,7 +18,7 @@ fn request_scoped_ops_event_requires_attempt_index() {
upstream_model_id: None,
failure_kind: "timeout".to_owned(),
upstream_send_state: Some("not_sent".to_owned()),
raw_upstream_error: None,
error_details: None,
status_code: None,
provider_error_code: None,
retry_after_ms: None,
@@ -2,6 +2,44 @@
use super::TestDatabase;
#[tokio::test]
async fn error_details_migration_preserves_existing_request_and_event_text() {
let Some(db) = TestDatabase::create_through("error_details_upgrade", 22).await else {
return;
};
seed_request(&db.pool).await;
let raw = "{ \"error\": {\"code\":\"Vendor.Unknown\",\"message\":\"原始错误\"} }";
sqlx::query("update model_requests set raw_upstream_error = $1 where id = 'req_integrity'")
.bind(raw)
.execute(&db.pool)
.await
.unwrap();
sqlx::query(
"insert into ops_events (id, level, component, operation, failure_kind, message, created_at, raw_upstream_error)
values ('error_details_upgrade', 'error', 'provider', 'generate', 'unavailable', 'safe message', now(), $1)",
).bind(raw).execute(&db.pool).await.unwrap();
super::TEST_MIGRATOR.run(&db.pool).await.unwrap();
let details: Vec<String> = sqlx::query_scalar(
"select error_details from model_requests where id = 'req_integrity'
union all select error_details from ops_events where id = 'error_details_upgrade'",
)
.fetch_all(&db.pool)
.await
.unwrap();
assert_eq!(details, vec![raw.to_owned(), raw.to_owned()]);
let old_columns: i64 = sqlx::query_scalar(
"select count(*) from information_schema.columns
where table_schema = current_schema() and table_name in ('model_requests', 'ops_events')
and column_name = 'raw_upstream_error'",
)
.fetch_one(&db.pool)
.await
.unwrap();
assert_eq!(old_columns, 0);
db.close().await;
}
async fn seed_request(pool: &sqlx::PgPool) {
sqlx::query(
"insert into model_requests (
@@ -255,7 +255,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
upstream_model_id: Some("upstream-a".to_owned()),
failure_kind: "rate_limited".to_owned(),
upstream_send_state: Some("sent".to_owned()),
raw_upstream_error: Some(
error_details: Some(
r#"{"error":{"code":"rate_limit","message":"raw marker"}}"#.to_owned(),
),
status_code: Some(429),
@@ -365,7 +365,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
set outcome = 'failed', error_kind = 'upstream_error',
error_message = 'snapshot failure', client_status_code = 502,
upstream_status_code = 502, completed_at = $1,
raw_upstream_error = $2
error_details = $2
where id = 'req_snap_error'",
)
.bind(started_at + chrono::Duration::seconds(2))
@@ -387,7 +387,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
upstream_model_id: Some("grok-test".to_owned()),
failure_kind: "auth_failed".to_owned(),
upstream_send_state: Some("not_sent".to_owned()),
raw_upstream_error: Some(r#"{"error":"raw probe failure"}"#.to_owned()),
error_details: Some(r#"{"error":"raw probe failure"}"#.to_owned()),
status_code: Some(401),
provider_error_code: Some("invalid_api_key".to_owned()),
retry_after_ms: None,
@@ -435,7 +435,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
);
assert_eq!(request_error.endpoint.as_deref(), Some("/v1/responses"));
assert_eq!(
request_error.raw_upstream_error.as_deref(),
request_error.error_details.as_deref(),
Some(r#"{"error":{"message":"raw request failure"}}"#)
);
let probe_error = errors
@@ -457,7 +457,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
);
assert_eq!(probe_error.endpoint, None);
assert_eq!(
probe_error.raw_upstream_error.as_deref(),
probe_error.error_details.as_deref(),
Some(r#"{"error":"raw probe failure"}"#)
);
@@ -7,6 +7,8 @@ use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use futures::{StreamExt as _, stream};
use gateway_core::error::ErrorSource;
use gateway_core::account::{
AccountErrorReason, CredentialState, ProviderAccount, ProviderAccountId, ProviderRefreshQuery,
};
@@ -393,10 +395,11 @@ impl CodexCredentialRefreshService {
CodexCredentialRefreshError,
> {
let now = SystemTime::now();
let provider = ProviderKind::new(PROVIDER_NAME)
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?;
let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| {
CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source)))
})?;
let limit = NonZeroU32::new(MAX_REFRESH_BATCH)
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
.ok_or(CredentialRepositoryError::InvalidCredentialData(None))?;
// 查询窗口按错峰上界(2×margin)放宽,先取回候选超集,
// 再在内存中按账号稳定偏移收窄;存储层谓词保持通用窗口语义。
let query = ProviderRefreshQuery::new(
@@ -520,7 +523,7 @@ impl CodexCredentialRefreshService {
credential_revision: revision.get(),
})
}
Err(CredentialRepositoryError::RevisionConflict) => {
Err(CredentialRepositoryError::RevisionConflict(_)) => {
Ok(CodexCredentialRefreshOutcome::Stale {
account_id: due.account.id().to_string(),
})
@@ -544,7 +547,7 @@ impl CodexCredentialRefreshService {
} = failure;
match self.repository.load_runtime_credential(account).await {
Ok(_) => {}
Err(CredentialRepositoryError::RevisionConflict) => {
Err(CredentialRepositoryError::RevisionConflict(_)) => {
return Ok(CodexCredentialRefreshOutcome::Stale {
account_id: account.id().to_string(),
});
@@ -586,7 +589,7 @@ impl CodexCredentialRefreshService {
);
Ok(outcome)
}
Err(CredentialRepositoryError::RevisionConflict) => {
Err(CredentialRepositoryError::RevisionConflict(_)) => {
Ok(CodexCredentialRefreshOutcome::Stale {
account_id: account.id().to_string(),
})
@@ -640,7 +643,7 @@ impl CodexCredentialRefreshService {
);
Ok(true)
}
Err(CredentialRepositoryError::RevisionConflict) => Ok(false),
Err(CredentialRepositoryError::RevisionConflict(_)) => Ok(false),
Err(error) => Err(error.into()),
}
}
@@ -3,6 +3,8 @@
use std::sync::Arc;
use std::time::SystemTime;
use gateway_core::error::ErrorSource;
use gateway_core::account::{
AccountErrorReason, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
CredentialRevision, CredentialState, LoadedCredential, ProviderAccount, ProviderAccountStore,
@@ -53,7 +55,7 @@ impl CodexCredentialRepository {
let mut data = CodexCredentialCodec::decode_complete(&current.credential)?;
let oauth = data
.oauth_mut()
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
.ok_or(CredentialRepositoryError::InvalidCredentialData(None))?;
oauth.access_token = secret.access_token.expose_secret().to_owned();
oauth.refresh_token = secret
.refresh_token
@@ -73,7 +75,9 @@ impl CodexCredentialRepository {
access_token_expires_at,
next_refresh_at,
)
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?
.map_err(|source| {
CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source)))
})?
.preserving_profile()
.with_account_state(
oauth_account_state(account, CredentialState::Ready),
@@ -108,7 +112,9 @@ impl CodexCredentialRepository {
account.access_token_expires_at(),
Some(next_refresh_at),
)
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?
.map_err(|source| {
CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source)))
})?
.preserving_profile();
if let Some(error_reason) = error_reason {
update = update.with_account_state(
@@ -124,8 +130,9 @@ impl CodexCredentialRepository {
pub async fn list_for_provider(
&self,
) -> Result<Vec<ProviderAccount>, CredentialRepositoryError> {
let provider = ProviderKind::new(PROVIDER_NAME)
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?;
let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| {
CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source)))
})?;
self.store
.list_for_provider(&provider)
.await
@@ -147,11 +154,11 @@ impl CodexCredentialRepository {
loaded: &LoadedCredential,
) -> Result<CodexRuntimeCredential, CredentialRepositoryError> {
if loaded.account.provider().as_str() != PROVIDER_NAME {
return Err(CredentialRepositoryError::InvalidCredentialData);
return Err(CredentialRepositoryError::InvalidCredentialData(None));
}
let data = CodexCredentialCodec::decode_complete(&loaded.credential)?;
if data.authentication_kind() != loaded.account.authentication_kind() {
return Err(CredentialRepositoryError::InvalidCredentialData);
return Err(CredentialRepositoryError::InvalidCredentialData(None));
}
CodexCredentialCodec::decode(&loaded.credential).map_err(Into::into)
}
@@ -161,14 +168,14 @@ impl CodexCredentialRepository {
account: &ProviderAccount,
) -> Result<CodexRuntimeCredential, CredentialRepositoryError> {
if account.provider().as_str() != PROVIDER_NAME {
return Err(CredentialRepositoryError::InvalidCredentialData);
return Err(CredentialRepositoryError::InvalidCredentialData(None));
}
let loaded = self
.store
.load_credential(account.id(), account.revision())
.await?;
if loaded.account != *account {
return Err(CredentialRepositoryError::RevisionConflict);
return Err(CredentialRepositoryError::RevisionConflict(None));
}
self.decode_runtime_credential(&loaded)
}
@@ -205,7 +212,9 @@ impl CodexCredentialRepository {
account.access_token_expires_at(),
account.next_refresh_at(),
)
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?
.map_err(|source| {
CredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(source)))
})?
.preserving_profile();
cas_revision(self.store.compare_and_swap_credential(update).await?)
}
@@ -270,35 +279,34 @@ fn cas_revision(
) -> Result<CredentialRevision, CredentialRepositoryError> {
match outcome {
CredentialCasOutcome::Updated(revision) => Ok(revision),
CredentialCasOutcome::Conflict => Err(CredentialRepositoryError::RevisionConflict),
CredentialCasOutcome::Conflict => Err(CredentialRepositoryError::RevisionConflict(None)),
}
}
#[derive(Debug, Error)]
pub enum CredentialRepositoryError {
#[error("Codex credential data is invalid")]
InvalidCredentialData,
InvalidCredentialData(#[source] Option<ErrorSource>),
#[error("Codex credential revision conflict")]
RevisionConflict,
RevisionConflict(#[source] Option<ErrorSource>),
#[error("provider account store is unavailable")]
Store,
Store(#[source] gateway_core::error::StoreError),
}
impl From<gateway_core::error::StoreError> for CredentialRepositoryError {
fn from(error: gateway_core::error::StoreError) -> Self {
match error.kind() {
gateway_core::error::StoreErrorKind::Conflict => Self::RevisionConflict,
gateway_core::error::StoreErrorKind::Unavailable
| gateway_core::error::StoreErrorKind::InvalidState
| gateway_core::error::StoreErrorKind::InvalidData => Self::Store,
_ => Self::Store,
gateway_core::error::StoreErrorKind::Conflict => {
Self::RevisionConflict(Some(ErrorSource::new(error)))
}
_ => Self::Store(error),
}
}
}
impl From<CodexCredentialDataError> for CredentialRepositoryError {
fn from(_: CodexCredentialDataError) -> Self {
Self::InvalidCredentialData
fn from(error: CodexCredentialDataError) -> Self {
Self::InvalidCredentialData(Some(ErrorSource::new(error)))
}
}
@@ -5,6 +5,8 @@ use std::fmt;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use gateway_core::error::ErrorSource;
use gateway_core::account::{
AccountCandidate, AccountCapacitySnapshot, AccountEligibilityPolicy, AccountErrorReason,
AccountFeedbackStats, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext,
@@ -519,7 +521,7 @@ impl CodexCredentialSelector {
.store()
.get_account(required)
.await
.map_err(|_| CredentialSelectionError::Store)?
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?
{
accounts.push(account);
}
@@ -555,7 +557,7 @@ impl CodexCredentialSelector {
{
let runtime = match self.repository.load_runtime_credential(&account).await {
Ok(runtime) => runtime,
Err(CredentialRepositoryError::RevisionConflict) => {
Err(CredentialRepositoryError::RevisionConflict(_)) => {
retry_account_snapshot(
request.attempt,
&account,
@@ -564,7 +566,7 @@ impl CodexCredentialSelector {
continue 'capacity;
}
// 非固定账号的损坏凭据不能阻断其余账号的传输资格检查
Err(CredentialRepositoryError::InvalidCredentialData)
Err(CredentialRepositoryError::InvalidCredentialData(_))
if pinned_account.is_none() =>
{
continue;
@@ -839,12 +841,12 @@ impl CodexCredentialSelector {
.iter()
.find(|candidate| candidate.account.id() == selected.account.id())
.map(|candidate| candidate.account.clone())
.ok_or(CredentialSelectionError::InvalidCredential)?;
.ok_or(CredentialSelectionError::InvalidCredential(None))?;
// 额度观测等并发更新会使整个账号快照失效,必须重新选号并校验资格
// 在占用租约和请求间隔前完成校验,避免重读被自己的异步释放挡住
let runtime = match self.repository.load_runtime_credential(&account).await {
Ok(runtime) => runtime,
Err(CredentialRepositoryError::RevisionConflict) => {
Err(CredentialRepositoryError::RevisionConflict(_)) => {
retry_account_snapshot(request.attempt, &account, &mut snapshot_retries)?;
continue 'capacity;
}
@@ -971,8 +973,8 @@ impl CodexCredentialSelector {
),
)
.await
.map_err(|_| binding_store_error())?
.map_err(|_| binding_store_error())?;
.map_err(binding_store_error)?
.map_err(binding_store_error)?;
if !applied {
return Err(CredentialSelectionError::SessionBound(Box::new(
CredentialSelectionError::NoEligibleCredential,
@@ -990,8 +992,8 @@ impl CodexCredentialSelector {
self.session_affinity.load_alias(&self.provider_kind, turn),
)
.await
.map_err(|_| binding_store_error())?
.map_err(|_| binding_store_error())
.map_err(binding_store_error)?
.map_err(binding_store_error)
.map(|key| key.map(CodexSessionAffinity::from_turn_alias))
}
@@ -1004,8 +1006,8 @@ impl CodexCredentialSelector {
self.session_affinity.load(&self.provider_kind, key),
)
.await
.map_err(|_| binding_store_error())?
.map_err(|_| binding_store_error())
.map_err(binding_store_error)?
.map_err(binding_store_error)
}
async fn admit_session(
@@ -1025,9 +1027,9 @@ impl CodexCredentialSelector {
),
)
.await
.map_err(|_| binding_store_error())?
.map_err(binding_store_error)?
.map(|binding| binding.is_some())
.map_err(|_| binding_store_error())
.map_err(binding_store_error)
}
async fn prepare_cyber_policy_scope(
@@ -1190,7 +1192,7 @@ impl CodexCredentialSelector {
.quota
.record_confirmed_exhaustion(account, QuotaEvidence::PaymentRequired, None, now)
.await
.map_err(|_| CredentialSelectionError::Store),
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source))),
CodexAccountFailure::UsageLimitExhausted { reset_at } => self
.quota
.record_confirmed_exhaustion(
@@ -1200,13 +1202,13 @@ impl CodexCredentialSelector {
now,
)
.await
.map_err(|_| CredentialSelectionError::Store),
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source))),
// 429:临时限流只写运行时冷却,不改变凭据或额度事实
CodexAccountFailure::RateLimited { retry_after } => {
self.quota
.apply_rate_limit_429(account, retry_after, now)
.await
.map_err(|_| CredentialSelectionError::Store)?;
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?;
Ok(())
}
// Cloudflare 挑战:内存退避表(记录风险计数),不写账号事实
@@ -1311,8 +1313,8 @@ impl CodexCredentialSelector {
.store()
.get_account(account_id)
.await
.map_err(|_| CredentialSelectionError::Store)?
.ok_or(CredentialSelectionError::InvalidCredential)
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?
.ok_or(CredentialSelectionError::InvalidCredential(None))
}
pub async fn capture_response_cookies(
@@ -1440,8 +1442,12 @@ impl CodexCredentialSelector {
}
}
fn binding_store_error() -> CredentialSelectionError {
CredentialSelectionError::SessionBound(Box::new(CredentialSelectionError::Store))
fn binding_store_error(
source: impl std::error::Error + Send + Sync + 'static,
) -> CredentialSelectionError {
CredentialSelectionError::SessionBound(Box::new(CredentialSelectionError::Store(
ErrorSource::new(source),
)))
}
fn affinity_selection_for_bound_account(
@@ -1641,7 +1647,7 @@ pub enum CredentialSelectionError {
#[error("session account changed during native continuation")]
ContinuationOwnerChanged,
#[error("session account binding cannot be bypassed: {0}")]
SessionBound(Box<CredentialSelectionError>),
SessionBound(#[source] Box<CredentialSelectionError>),
#[error(transparent)]
QueueRejected(#[from] QueueRejection),
#[error("no eligible Codex account")]
@@ -1651,15 +1657,15 @@ pub enum CredentialSelectionError {
#[error("Codex account capacity is unavailable")]
CapacityUnavailable { retry_after: Option<Duration> },
#[error("Codex account data is invalid")]
InvalidCredential,
InvalidCredential(#[source] Option<ErrorSource>),
#[error("Codex account changed repeatedly during selection")]
AccountSnapshotChanged,
#[error("Codex account store is unavailable")]
Store,
Store(#[source] ErrorSource),
#[error("Codex account lease runtime is unavailable")]
Coordinator,
Coordinator(#[source] ProviderStoreError),
#[error("Codex Cookie policy rejected the value")]
CookiePolicy,
CookiePolicy(#[source] super::cookie::CookiePolicyError),
#[error("account scheduling policy rejected the request")]
PolicyRejected,
#[error("account scheduling policy is unavailable")]
@@ -1668,24 +1674,25 @@ pub enum CredentialSelectionError {
impl From<CredentialRepositoryError> for CredentialSelectionError {
fn from(error: CredentialRepositoryError) -> Self {
match error {
CredentialRepositoryError::InvalidCredentialData => Self::InvalidCredential,
CredentialRepositoryError::RevisionConflict | CredentialRepositoryError::Store => {
Self::Store
match &error {
CredentialRepositoryError::InvalidCredentialData(_) => {
Self::InvalidCredential(Some(ErrorSource::new(error)))
}
CredentialRepositoryError::RevisionConflict(_)
| CredentialRepositoryError::Store(_) => Self::Store(ErrorSource::new(error)),
}
}
}
impl From<ProviderStoreError> for CredentialSelectionError {
fn from(_: ProviderStoreError) -> Self {
Self::Coordinator
fn from(error: ProviderStoreError) -> Self {
Self::Coordinator(error)
}
}
impl From<super::cookie::CookiePolicyError> for CredentialSelectionError {
fn from(_: super::cookie::CookiePolicyError) -> Self {
Self::CookiePolicy
fn from(error: super::cookie::CookiePolicyError) -> Self {
Self::CookiePolicy(error)
}
}
@@ -290,9 +290,7 @@ impl CodexProvider {
client: self
.client_for_request(&context)?
.for_account(lease.account())
.map_err(|_| {
provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
})?
.map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)?
.with_authentication(lease.authentication())
.with_middleware_headers(middleware_headers),
response_origin: request.response_origin,
@@ -5,7 +5,7 @@ use gateway_core::diagnostics::TraceContext;
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集
///
/// 已归一的上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type
/// 原始上游 code 优先;code 缺失时才读取结构化客户端错误的 code/type
/// 新增错误、裸 HTTP 状态和内部错误 kind 默认都不会进入该闭集
/// 容量拒绝影响短期调度健康度,但不证明账号凭据或额度失效
const OPENAI_ACCOUNT_SCORE_FAILURE_REASONS: &[&str] = &[
@@ -273,7 +273,7 @@ pub(super) fn log_client_upstream_error(
body,
transport,
..
} => log_raw_upstream_body(
} => log_upstream_error_summary(
context,
*transport,
"http_error_response",
@@ -283,7 +283,7 @@ pub(super) fn log_client_upstream_error(
body,
),
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
log_raw_upstream_body(
log_upstream_error_summary(
context,
CodexBackendTransport::WebSocket,
"websocket_opening_response",
@@ -303,7 +303,7 @@ pub(super) fn log_client_upstream_error(
.map(|connection_id| connection_id.to_string())
.unwrap_or_default();
let close_reason_bytes = close.reason().map_or(0, str::len);
tracing::warn!(
tracing::debug!(
request_id = %context.request_id,
account_id = %context.account_id,
attempt_index = context.attempt_index,
@@ -333,7 +333,7 @@ pub(super) fn log_canonical_upstream_error(
let CodexCanonicalError::Upstream(failure) = error else {
return;
};
log_raw_upstream_body(
log_upstream_error_summary(
context,
transport,
"responses_error_event",
@@ -344,7 +344,7 @@ pub(super) fn log_canonical_upstream_error(
);
}
pub(super) fn log_raw_upstream_body(
pub(super) fn log_upstream_error_summary(
context: UpstreamErrorLogContext<'_>,
transport: CodexBackendTransport,
error_kind: &'static str,
@@ -357,7 +357,7 @@ pub(super) fn log_raw_upstream_body(
.websocket_connection_id
.map(|connection_id| connection_id.to_string())
.unwrap_or_default();
tracing::warn!(
tracing::debug!(
request_id = %context.request_id,
account_id = %context.account_id,
attempt_index = context.attempt_index,
@@ -366,11 +366,8 @@ pub(super) fn log_raw_upstream_body(
upstream_error_kind = error_kind,
upstream_status_code = status_code.unwrap_or_default(),
upstream_status_code_present = status_code.is_some(),
upstream_error_code = upstream_code.unwrap_or_default(),
upstream_error_code_present = upstream_code.is_some(),
upstream_error_type = upstream_type.unwrap_or_default(),
upstream_error_type_present = upstream_type.is_some(),
upstream_error_raw,
upstream_error_raw_bytes = upstream_error_raw.len(),
"OpenAI upstream returned an error payload"
);
@@ -568,8 +565,9 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_failure_kind = failure.error.kind().as_str(),
websocket_failure_code = failure
.error
.upstream_code()
.map_or("", OpaqueUpstreamValue::as_str),
.diagnostic()
.and_then(ProviderDiagnostic::code)
.unwrap_or(""),
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
upstream_status_code_present = failure.error.upstream_status().is_some(),
transport_requirement = context.requirement.as_str(),
@@ -598,8 +596,9 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_failure_kind = failure.error.kind().as_str(),
websocket_failure_code = failure
.error
.upstream_code()
.map_or("", OpaqueUpstreamValue::as_str),
.diagnostic()
.and_then(ProviderDiagnostic::code)
.unwrap_or(""),
upstream_send_state = ?send_state,
transport_requirement = context.requirement.as_str(),
continuation_recovery_action = "client_replay_required",
@@ -642,8 +641,9 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_failure_kind = failure.error.kind().as_str(),
websocket_failure_code = failure
.error
.upstream_code()
.map_or("", OpaqueUpstreamValue::as_str),
.diagnostic()
.and_then(ProviderDiagnostic::code)
.unwrap_or(""),
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
upstream_status_code_present = failure.error.upstream_status().is_some(),
websocket_retry_count = retry_index.get(),
@@ -669,8 +669,9 @@ pub(super) fn apply_websocket_recovery_policy(
websocket_failure_kind = failure.error.kind().as_str(),
websocket_failure_code = failure
.error
.upstream_code()
.map_or("", OpaqueUpstreamValue::as_str),
.diagnostic()
.and_then(ProviderDiagnostic::code)
.unwrap_or(""),
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
upstream_status_code_present = failure.error.upstream_status().is_some(),
websocket_retry_count = context.retry_count,
@@ -897,7 +898,7 @@ pub(super) fn map_client_error(
}
let connect_retry = !local_connection_capacity
&& matches!(&error, CodexClientError::Http(error) if transient_http_connect(error));
let mut failure = match error {
let mut failure = match &error {
CodexClientError::ConnectionBudgetExhausted => MappedProviderFailure::plain(
provider_error(ProviderErrorKind::Timeout, UpstreamSendState::NotSent)
.with_connection_retry(gateway_core::engine::AttemptTransport::Fallback),
@@ -940,8 +941,9 @@ pub(super) fn map_client_error(
uncertain_state,
)
.with_status(status.as_u16());
if let Some(request_id) = diagnostics.request_id {
error = error.with_upstream_request_id(OpaqueUpstreamValue::new(request_id));
if let Some(request_id) = &diagnostics.request_id {
error =
error.with_upstream_request_id(OpaqueUpstreamValue::new(request_id.clone()));
}
MappedProviderFailure::plain(error)
}
@@ -964,7 +966,7 @@ pub(super) fn map_client_error(
let mut failure = MappedProviderFailure::plain(continuation_replay_required_error(
continuation_unavailable_reason.unwrap_or("scope_unavailable"),
));
if let Some(client_visible_error) = websocket_client_visible_error(&error) {
if let Some(client_visible_error) = websocket_client_visible_error(error) {
failure.error = failure
.error
.with_client_visible_upstream_error(client_visible_error);
@@ -973,10 +975,10 @@ pub(super) fn map_client_error(
}
CodexClientError::WebSocket(error) => {
let close_code = error.close_before_terminal().and_then(|close| close.code());
let client_visible_error = websocket_client_visible_error(&error);
let client_visible_error = websocket_client_visible_error(error);
let mut failure = MappedProviderFailure::plain(provider_error(
websocket_error_kind(&error),
websocket_send_state(&error),
websocket_error_kind(error),
websocket_send_state(error),
));
if let Some(close_code) = close_code {
failure.error =
@@ -1043,6 +1045,39 @@ pub(super) fn map_client_error(
.error
.with_connection_observation(connection_observation);
}
// Reqwest 的请求 URL 可能携带认证参数;原因链保留,请求方向的 URL 不进入运维详情
let mut request_url_redacted = false;
let error = match error {
CodexClientError::Http(source) => {
request_url_redacted = source.url().is_some();
CodexClientError::Http(source.without_url())
}
CodexClientError::HttpJson(source) => {
request_url_redacted = source.url().is_some();
CodexClientError::HttpJson(source.without_url())
}
CodexClientError::ErrorBodyRead {
source,
status,
diagnostics,
transport,
transport_metrics,
} => {
request_url_redacted = source.url().is_some();
CodexClientError::ErrorBodyRead {
source: source.without_url(),
status,
diagnostics,
transport,
transport_metrics,
}
}
error => error,
};
failure.error = failure.error.with_source(error);
if request_url_redacted {
failure.error = failure.error.redact_sensitive_context("request URL");
}
failure.observation = observation;
failure
}
@@ -1119,11 +1154,7 @@ fn client_diagnostic(error: &CodexClientError) -> Option<ProviderDiagnostic> {
"request_compression_failed",
"OpenAI request compression failed".to_owned(),
),
CodexClientError::CustomCa(_) => (
"prepare",
"custom_ca_invalid",
"OpenAI custom CA configuration is invalid".to_owned(),
),
CodexClientError::CustomCa(error) => return Some(custom_ca_diagnostic(error)),
CodexClientError::ModelCatalog(_) => (
"prepare",
"model_catalog_invalid",
@@ -1134,6 +1165,63 @@ fn client_diagnostic(error: &CodexClientError) -> Option<ProviderDiagnostic> {
Some(ProviderDiagnostic::new(message).with_classification(stage, code))
}
pub(super) fn profile_diagnostic(
error: &crate::transport::profile::selection::ClientProfileError,
) -> ProviderDiagnostic {
use crate::transport::profile::selection::ClientProfileError;
let code = match error {
ClientProfileError::Invalid => "request_profile_invalid",
ClientProfileError::InvalidUserAgent => "request_profile_user_agent_invalid",
ClientProfileError::CompanionHeadersRequired => "request_profile_headers_required",
ClientProfileError::CompanionHeadersConflict => "request_profile_headers_conflict",
ClientProfileError::ReleaseUnavailable => "request_profile_release_unavailable",
};
// 此类型的 Display 全部由本地固定文案组成,不包含配置字段
ProviderDiagnostic::new(error.to_string()).with_classification("prepare", code)
}
fn custom_ca_diagnostic(error: &crate::transport::tls::CustomCaError) -> ProviderDiagnostic {
use crate::transport::tls::CustomCaError;
let (code, message) = match error {
CustomCaError::ProxyConfiguration => (
"account_proxy_invalid",
"OpenAI account proxy configuration is invalid".to_owned(),
),
CustomCaError::ReadCaFile { source_env, .. } => (
"custom_ca_read_failed",
format!("OpenAI CA certificate file selected by {source_env} could not be read"),
),
CustomCaError::InvalidCaFile { source_env, .. } => (
"custom_ca_invalid",
format!("OpenAI CA certificate file selected by {source_env} is not valid PEM"),
),
CustomCaError::RegisterCertificate {
certificate_index, ..
}
| CustomCaError::RegisterRustlsCertificate {
certificate_index, ..
} => (
"custom_ca_registration_failed",
format!("OpenAI CA certificate #{certificate_index} could not be registered"),
),
CustomCaError::BuildClientWithCustomCa { .. } => (
"custom_ca_client_build_failed",
"OpenAI HTTP client with custom CA certificates could not be built".to_owned(),
),
CustomCaError::BuildClientWithSystemRoots(_) => (
"system_tls_client_build_failed",
"OpenAI HTTP client with system root certificates could not be built".to_owned(),
),
CustomCaError::LoadNativeRoots(_) => (
"system_tls_roots_unavailable",
"OpenAI system root certificates could not be loaded".to_owned(),
),
};
ProviderDiagnostic::new(message)
.with_classification("prepare", code)
.with_io_cause(error)
}
fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnostic {
let (stage, code) = match error.diagnostic_cause() {
CodexWebSocketExchangeError::Connect(_) => (
@@ -1349,7 +1437,7 @@ pub(super) fn map_upstream_failure(
.is_some_and(|status| status.is_client_error())
&& is_cyber_policy_code(failure.code.as_deref());
let continuation_failure = failure
.persistable_code()
.upstream_code()
.filter(|code| is_history_failure_code(code))
.map(|_| ContinuationFailure::HistoryUnavailable);
let send_state = upstream_send_state(failure.send_phase);
@@ -1414,7 +1502,7 @@ pub(super) fn map_upstream_failure(
max_delay,
);
}
if let Some(code) = failure.persistable_code() {
if let Some(code) = failure.upstream_code() {
error = error.with_upstream_code(OpaqueUpstreamValue::new(code.to_owned()));
}
if let Some(request_id) = failure.request_id.as_deref() {
@@ -1423,13 +1511,10 @@ pub(super) fn map_upstream_failure(
let status = error
.upstream_status()
.map_or_else(|| "none".to_owned(), |status| status.to_string());
let code = error
.upstream_code()
.map_or_else(|| "none".to_owned(), |code| code.as_str().to_owned());
let kind = error.kind().as_str();
error = error.with_diagnostic(
ProviderDiagnostic::new(format!(
"OpenAI upstream failure: kind={}, status={status}, code={code}",
"OpenAI upstream failure: kind={}, status={status}",
kind
))
.with_classification("upstream", "upstream_rejected"),
@@ -1461,13 +1546,14 @@ pub(super) fn is_cyber_policy_code(code: Option<&str>) -> bool {
}
pub(super) fn is_history_failure_code(code: &str) -> bool {
matches!(
code,
"previous_response_not_found"
| "invalid_encrypted_content"
| "missing_tool_output"
| "no_tool_output"
)
[
"previous_response_not_found",
"invalid_encrypted_content",
"missing_tool_output",
"no_tool_output",
]
.iter()
.any(|known| code.trim().eq_ignore_ascii_case(known))
}
pub(super) const fn provider_error_kind(category: CodexFailureCategory) -> ProviderErrorKind {
@@ -361,9 +361,7 @@ impl CodexProvider {
client: self
.client_for_request(&context)?
.for_account(lease.account())
.map_err(|_| {
provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
})?
.map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)?
.with_authentication(lease.authentication())
.with_middleware_headers(middleware_headers),
registry: Arc::clone(&self.live_registry),
@@ -236,8 +236,16 @@ impl CodexProvider {
return Ok(self.client.clone());
};
let profile = serde_json::from_value(Value::Object(profile.expose_to_provider().clone()))
.map_err(|_| {
.map_err(|error: serde_json::Error| {
let diagnostic = ProviderDiagnostic::new(format!(
"OpenAI request profile decoding failed: {:?}, line={}, column={}",
error.classify(),
error.line(),
error.column(),
))
.with_classification("prepare", "request_profile_decode_failed");
provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent)
.with_diagnostic(diagnostic)
})?;
Ok(self.client.clone().with_request_profile(profile))
}
@@ -332,19 +340,23 @@ impl Provider for CodexProvider {
) -> Result<gateway_core::account::OpaqueProviderData, ProviderError> {
let selection =
crate::transport::profile::identity::RequestProfileSelection::parse(configuration)
.map_err(|_| {
.map_err(|error| {
provider_error(
ProviderErrorKind::InvalidRequest,
UpstreamSendState::NotSent,
)
.with_diagnostic(profile_diagnostic(&error))
})?;
let profile = selection
.resolve(self.client.profile_state())
.map_err(|_| {
.map_err(|error| {
provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
.with_diagnostic(profile_diagnostic(&error))
})?;
crate::transport::profile::selection::object(&profile)
.map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent))
crate::transport::profile::selection::object(&profile).map_err(|error| {
provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent)
.with_diagnostic(profile_diagnostic(&error))
})
}
fn name(&self) -> &'static str {
@@ -984,9 +996,7 @@ impl CodexProvider {
client: self
.client_for_request(&context)?
.for_account(lease.account())
.map_err(|_| {
provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
})?
.map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)?
.with_authentication(lease.authentication())
.with_connection_budget(context.connection_budget().clone())
.with_response_control(context.response_control().cloned())
@@ -769,10 +769,26 @@ pub(super) fn map_request_error(error: CodexRequestEncodeError) -> ProviderError
}
pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
match error {
selection_failure(&error).with_source(error)
}
fn selection_failure(error: &CredentialSelectionError) -> ProviderError {
let diagnostic_code = match error {
CredentialSelectionError::InvalidCredential(_) => Some("account_credential_invalid"),
CredentialSelectionError::Store(_) => Some("account_store_unavailable"),
CredentialSelectionError::Coordinator(_) => Some("account_coordinator_unavailable"),
CredentialSelectionError::CookiePolicy(_) => Some("account_cookie_policy_rejected"),
CredentialSelectionError::PolicyUnavailable => Some("account_policy_unavailable"),
_ => None,
};
// 这些变体只含固定文案,不格式化可能承载外部正文的其他错误
let diagnostic = diagnostic_code.map(|code| {
ProviderDiagnostic::new(error.to_string()).with_classification("account_selection", code)
});
let mapped = match error {
CredentialSelectionError::Cancelled => provider_error(ProviderErrorKind::Cancelled, UpstreamSendState::NotSent).with_retry_prohibited(),
CredentialSelectionError::ContinuationOwnerChanged => continuation_replay_required_error("scope_unavailable"),
CredentialSelectionError::SessionBound(error) => map_selection_error(*error).with_retry_prohibited(),
CredentialSelectionError::SessionBound(error) => selection_failure(error).with_retry_prohibited(),
CredentialSelectionError::QueueRejected(error) => {
provider_error(error.provider_kind(), UpstreamSendState::NotSent)
}
@@ -782,7 +798,7 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr
UpstreamSendState::NotSent,
);
match retry_after {
Some(retry) => error.with_retry_after(retry),
Some(retry) => error.with_retry_after(*retry),
None => error,
}
}
@@ -807,10 +823,10 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr
ProviderDiagnostic::new("OpenAI account changed repeatedly during selection")
.with_classification("account_selection", "account_snapshot_conflict"),
),
CredentialSelectionError::InvalidCredential
| CredentialSelectionError::Store
| CredentialSelectionError::Coordinator
| CredentialSelectionError::CookiePolicy => provider_error(
CredentialSelectionError::InvalidCredential(_)
| CredentialSelectionError::Store(_)
| CredentialSelectionError::Coordinator(_)
| CredentialSelectionError::CookiePolicy(_) => provider_error(
ProviderErrorKind::ProviderInfrastructureUnavailable,
UpstreamSendState::NotSent,
),
@@ -822,5 +838,9 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr
ProviderErrorKind::Unavailable,
UpstreamSendState::NotSent,
),
};
match diagnostic {
Some(diagnostic) => mapped.with_diagnostic(diagnostic),
None => mapped,
}
}
@@ -25,57 +25,6 @@ const UPSTREAM_TRACE_HEADERS: &[&str] = &[
];
const IDENTITY_AUTHORIZATION_ERROR_HEADER: &str = "x-openai-authorization-error";
const IDENTITY_ERROR_JSON_HEADER: &str = "x-error-json";
const PERSISTABLE_UPSTREAM_CODES: &[&str] = &[
"access_token_expired",
"account_banned",
"account_deactivated",
"account_disabled",
"account_suspended",
"authentication_error",
"billing_limit",
"cyber_policy",
"deactivated_workspace",
"flex_unavailable",
"identity_verification_required",
"insufficient_quota",
"invalid_api_key",
"invalid_encrypted_content",
"invalid_prompt",
"invalid_request",
"missing_tool_output",
"model_not_available",
"model_not_supported",
"no_tool_output",
"organization_disabled",
"payment_required",
"permission_denied",
"previous_response_not_found",
"quota_exceeded",
"quota_exhausted",
"rate_limit_error",
"rate_limit_exceeded",
"rate_limit_reached",
"refresh_token_invalidated",
"server_error",
"server_is_overloaded",
"slow_down",
"token_expired",
"token_invalid",
"token_invalidated",
"token_revoked",
"unauthorized",
"unsupported",
"unsupported_feature",
"usage_limit_reached",
"verification_required",
"websocket_connection_limit_reached",
"workspace_deactivated",
"workspace_member_credits_depleted",
"workspace_member_usage_limit_reached",
"workspace_owner_credits_depleted",
"workspace_owner_usage_limit_reached",
];
/// 上游拒绝相对业务 payload 的发送阶段
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CodexUpstreamSendPhase {
@@ -256,15 +205,8 @@ impl CodexUpstreamFailure {
}
}
pub(crate) fn persistable_code(&self) -> Option<&'static str> {
self.code
.as_deref()
.and_then(persistable_upstream_code)
.or_else(|| {
self.identity_error_code
.as_deref()
.and_then(persistable_upstream_code)
})
pub(crate) fn upstream_code(&self) -> Option<&str> {
self.code.as_deref().or(self.identity_error_code.as_deref())
}
}
@@ -303,7 +245,10 @@ impl ParsedUpstreamError {
.or_else(|| value.get("message"))
.and_then(Value::as_str)
.and_then(non_empty_owned);
let code = code_value.and_then(Value::as_str).and_then(non_empty_owned);
let code = code_value
.and_then(Value::as_str)
.filter(|code| !code.trim().is_empty())
.map(str::to_owned);
let message = client_message
.clone()
.or_else(|| error.as_str().and_then(non_empty_owned))
@@ -417,8 +362,7 @@ fn decode_identity_error_code(encoded: &str) -> Option<String> {
.ok()?
.pointer("/error/code")?
.as_str()
.map(str::trim)
.filter(|code| !code.is_empty())
.filter(|code| !code.trim().is_empty())
.map(ToString::to_string)
}
@@ -571,13 +515,6 @@ fn non_empty_owned(value: &str) -> Option<String> {
(!value.is_empty()).then(|| value.to_owned())
}
fn persistable_upstream_code(value: &str) -> Option<&'static str> {
PERSISTABLE_UPSTREAM_CODES
.iter()
.copied()
.find(|known| value.trim().eq_ignore_ascii_case(known))
}
fn is_model_unsupported(value: &str) -> bool {
value.contains("model_not_supported")
|| value.contains("model_not_available")
@@ -710,7 +710,7 @@ fn log_pump_exit(
observation: &WebSocketConnectionObservation,
backpressure_events: u64,
) {
let detail = reason.detail().unwrap_or_default();
let detail = reason.metric_reason();
let account_id = context.account_id.as_deref().unwrap_or("unknown");
let conversation_id_hash = context.conversation_id_hash.as_deref().unwrap_or("unknown");
if reason.is_unexpected() {
@@ -728,9 +728,6 @@ fn log_pump_exit(
);
} else if matches!(reason, PumpExitReason::UpstreamCloseFrame { .. }) {
let upstream_close = reason.upstream_close();
let upstream_error_raw = upstream_close
.and_then(CodexWebSocketCloseError::reason)
.unwrap_or_default();
tracing::info!(
websocket_connection_id = %connection_id,
account_id = %account_id,
@@ -741,7 +738,7 @@ fn log_pump_exit(
connection_idle_ms = observation.idle_ms(),
pump_exit_detail = detail,
upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code),
upstream_error_raw,
upstream_close_reason_bytes = upstream_close.and_then(CodexWebSocketCloseError::reason).map_or(0, str::len),
upstream_error_raw_present = upstream_close.is_some_and(|close| close.reason().is_some()),
backpressure_events,
"Responses WebSocket pump received close frame"
@@ -674,7 +674,7 @@ fn selector_does_not_retry_credential_store_unavailability() {
}))
.expect_err("unavailability is not an account snapshot conflict");
assert!(matches!(error, CredentialSelectionError::Store));
assert!(matches!(error, CredentialSelectionError::Store(_)));
assert_eq!(store.credential_loads(), 1);
assert!(leases.requests.lock().expect("lease requests").is_empty());
}
@@ -704,7 +704,10 @@ fn selector_does_not_retry_invalid_credential_data() {
}))
.expect_err("invalid data is not an account snapshot conflict");
assert!(matches!(error, CredentialSelectionError::InvalidCredential));
assert!(matches!(
error,
CredentialSelectionError::InvalidCredential(_)
));
assert_eq!(store.credential_loads(), 1);
assert!(leases.requests.lock().expect("lease requests").is_empty());
}
@@ -0,0 +1,148 @@
//! 验证原始上游错误码独立于本地分类、诊断摘要和重试策略
use super::*;
#[tokio::test]
async fn network_failure_preserves_native_causes_without_the_request_url() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
drop(listener);
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
let provider = provider_with_base_url_and_retry_budget(
&store,
format!("http://{address}/PRIVATE_REQUEST_PATH"),
0,
);
let mut stream = provider
.execute(
planned_request("openai", http_generate_operation()),
context("req_native_cause", CancellationToken::new()),
)
.await
.unwrap();
let error = loop {
match stream.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("connection failure must surface"),
}
};
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
let details = error.error_details().unwrap();
assert!(!details.contains("PRIVATE_REQUEST_PATH"));
let details: Value = serde_json::from_str(&details).unwrap();
assert_eq!(details["redacted"], true);
let mut source = std::error::Error::source(&error);
let mut saw_native = false;
while let Some(cause) = source {
if let Some(http) = cause.downcast_ref::<reqwest::Error>() {
assert!(http.url().is_none());
assert!(http.is_connect());
}
if let Some(io) = cause.downcast_ref::<std::io::Error>() {
saw_native = io.kind() == std::io::ErrorKind::ConnectionRefused;
}
source = cause.source();
}
assert!(
saw_native,
"the original I/O cause must survive Provider mapping"
);
}
#[tokio::test(flavor = "current_thread")]
async fn upstream_codes_survive_http_and_websocket_opening_without_normalization() {
let captured = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.json()
.without_time()
.with_ansi(false)
.with_max_level(tracing::Level::DEBUG)
.with_writer(captured.clone())
.finish();
let _subscriber = tracing::subscriber::set_default(subscriber);
for use_websocket in [false, true] {
for (body_code, header_code, expected_kind) in [
(
Some("Vendor.Future.v2"),
None,
ProviderErrorKind::Unavailable,
),
(Some(" SERVER_ERROR "), None, ProviderErrorKind::Unavailable),
(
None,
Some("Vendor.Identity.v2"),
ProviderErrorKind::Unavailable,
),
(
Some("PrEvIoUs_ReSpOnSe_NoT_FoUnD"),
None,
ProviderErrorKind::ContinuationRecoveryRequired,
),
] {
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
let server = MockServer::start().await;
let body = json!({"error": {"code": body_code, "message": "PRIVATE_UPSTREAM_DETAIL"}});
let mut response = ResponseTemplate::new(503).set_body_json(&body);
if let Some(code) = header_code {
response = response.insert_header(
"x-error-json",
STANDARD.encode(json!({"error": {"code": code}}).to_string()),
);
}
Mock::given(method(if use_websocket { "GET" } else { "POST" }))
.and(path("/codex/responses"))
.respond_with(response)
.mount(&server)
.await;
let operation = if use_websocket {
capacity_websocket_operation()
} else {
http_generate_operation()
};
let mut stream = provider_with_base_url_and_retry_budget(&store, server.uri(), 0)
.execute(
planned_request("openai", operation),
context("req_original_error_code", CancellationToken::new()),
)
.await
.expect("prepare stream");
let error = loop {
match stream.next().await {
Some(Ok(_)) => {}
Some(Err(error)) => break error,
None => panic!("upstream rejection must surface"),
}
};
let original_code = body_code.or(header_code).unwrap();
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some(original_code)
);
assert_eq!(error.kind(), expected_kind);
assert_eq!(
serde_json::from_str::<Value>(error.raw_upstream_error().unwrap().as_str())
.unwrap(),
body,
);
let diagnostic = error.diagnostic().expect("safe diagnostic");
assert!(!diagnostic.as_str().contains(original_code));
assert!(!diagnostic.as_str().contains("PRIVATE_UPSTREAM_DETAIL"));
assert!(!format!("{error:?} {error}").contains(original_code));
}
}
let events = captured.json_events();
assert!(events.iter().any(|event| {
event["fields"]["message"] == "OpenAI upstream returned an error payload"
}));
let logs = serde_json::to_string(&events).unwrap();
for private in [
"Vendor.Future.v2",
"Vendor.Identity.v2",
"PRIVATE_UPSTREAM_DETAIL",
] {
assert!(!logs.contains(private), "ordinary logs leaked {private}");
}
}
@@ -1,64 +0,0 @@
//! 显式真实请求测试的凭据导入,只创建内存账号,不修改本地导出
use super::*;
pub(super) async fn imported_account() -> (Arc<MemoryAccountStore>, Value) {
use provider_openai::credential::{CodexAccountProfile, CodexOAuthSecret};
use secrecy::SecretString;
let file = std::env::var("CPR_LIVE_ACCOUNTS_FILE").expect("credential file path required");
let document: Value =
serde_json::from_slice(&std::fs::read(file).expect("read credential file"))
.expect("parse credential export");
let account = document["documents"][0]["document"]["accounts"][0].clone();
let field = |key| account.get(key).and_then(Value::as_str).map(str::to_owned);
let store = Arc::new(MemoryAccountStore::default());
store
.seed_oauth_credential(ImportCodexOAuthCredential {
account_id: "acct_provider_contract".into(),
name: "local-live-validation".into(),
enabled: true,
secret: CodexOAuthSecret {
access_token: SecretString::from(
field("accessToken").expect("access token required"),
),
refresh_token: field("refreshToken").map(SecretString::from),
id_token: field("idToken").map(SecretString::from),
},
verified_account: CodexAccountProfile {
email: None,
oauth_subject: field("userId").expect("user identity required"),
poid: None,
chatgpt_account_id: field("accountId").expect("account identity required"),
chatgpt_user_id: field("userId").expect("user identity required"),
plan_type: field("planType"),
access_token_expires_at: field("accessTokenExpiresAt")
.and_then(|date| chrono::DateTime::parse_from_rfc3339(&date).ok())
.map(|date| date.with_timezone(&Utc)),
},
next_refresh_at: Some(Utc::now() + chrono::Duration::hours(1)),
})
.await;
(store, account)
}
pub(super) async fn model(store: &Arc<MemoryAccountStore>) -> String {
let catalog = CodexCredentialCatalogService::new(
store.repository(),
wire_profile(),
reqwest::Client::builder().no_proxy().build().unwrap(),
OFFICIAL_CODEX_BASE_URL.to_owned(),
catalog_cache(),
);
let models = catalog
.refresh_account_catalog(&ProviderAccountId::new("acct_provider_contract").unwrap())
.await
.expect("load supported models");
std::env::var("CPR_LIVE_MODEL").unwrap_or_else(|_| {
models
.models()
.first()
.expect("available text model")
.clone()
})
}
@@ -2,7 +2,7 @@
mod account_isolation;
mod capacity;
mod live;
mod error_details;
mod precommit;
mod response_interrupt;
mod session_binding;
@@ -84,6 +84,88 @@ use crate::support::{
};
use crate::transport::accept_codex_test_websocket;
#[tokio::test]
async fn account_client_preparation_keeps_ca_read_cause_without_exposing_path() {
const CHILD: &str = "CPR_TEST_CA_PREPARATION_FAILURE";
if std::env::var_os(CHILD).is_none() {
let directory = tempfile::tempdir().unwrap();
// 环境变量只影响隔离子进程,不能改变并发 TLS 测试的配置
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", std::thread::current().name().unwrap()])
.env(CHILD, "1")
.env(
provider_openai::transport::tls::CODEX_CA_CERT_ENV,
directory.path().join("PRIVATE_MISSING_CA.pem"),
)
.output()
.unwrap();
assert!(
output.status.success(),
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
return;
}
let store = Arc::new(MemoryAccountStore::default());
create_account(&store, "acct_provider_contract").await;
store.set_egress(
"acct_provider_contract",
Some(gateway_core::account::OutboundProxy::parse("http://127.0.0.1:9").unwrap()),
None,
);
let error = provider_with_base_url(&store, "http://127.0.0.1:9".to_owned())
.execute(
planned_request("openai", generate_operation()),
fallback_transport_context("req_ca_prepare"),
)
.await
.err()
.expect("CA failure must occur before returning a stream");
assert_eq!(error.kind(), ProviderErrorKind::Protocol);
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
let snapshot = error.stable_snapshot();
let diagnostic = snapshot.diagnostic().unwrap();
assert_eq!(diagnostic.stage(), Some("prepare"));
assert_eq!(diagnostic.code(), Some("io_not_found"));
assert!(diagnostic.as_str().contains("CODEX_CA_CERTIFICATE"));
assert!(diagnostic.as_str().contains("OS cause:"));
assert!(!diagnostic.as_str().contains("PRIVATE_"));
let gateway = gateway_core::error::GatewayError::from_provider(&snapshot);
assert_eq!(gateway.safe_message(), "upstream service is unavailable");
assert_eq!(gateway.diagnostic(), Some(diagnostic));
}
#[tokio::test]
async fn invalid_request_profile_preserves_diagnostic_before_any_upstream_send() {
let store = Arc::new(MemoryAccountStore::default());
let provider = provider_with_base_url(&store, "http://127.0.0.1:1".to_owned());
let configuration = OpaqueProviderData::new(
json!({
"mode": "custom", "userAgent": "PRIVATE_USER_AGENT\n",
})
.as_object()
.unwrap()
.clone(),
);
let error = provider
.resolve_request_profile(&configuration)
.unwrap_err();
assert_eq!(error.kind(), ProviderErrorKind::InvalidRequest);
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
let snapshot = error.stable_snapshot();
let diagnostic = snapshot.diagnostic().unwrap();
assert_eq!(diagnostic.stage(), Some("prepare"));
assert_eq!(
diagnostic.code(),
Some("request_profile_user_agent_invalid")
);
assert!(!diagnostic.as_str().contains("PRIVATE_"));
let gateway = gateway_core::error::GatewayError::from_provider(&snapshot);
assert_eq!(gateway.diagnostic(), Some(diagnostic));
}
#[tokio::test]
async fn native_openai_translates_a_non_native_source_before_encoding() {
let store = Arc::new(MemoryAccountStore::default());
@@ -3594,6 +3676,22 @@ async fn selection_infrastructure_errors_have_a_distinct_classification() {
UpstreamSendState::NotSent,
)
);
let diagnostic = error.diagnostic().unwrap();
assert_eq!(diagnostic.stage(), Some("account_selection"));
assert_eq!(diagnostic.code(), Some("account_store_unavailable"));
assert_eq!(diagnostic.as_str(), "Codex account store is unavailable");
use std::error::Error as _;
let snapshot = error.stable_snapshot();
let mut cause = snapshot.source().expect("selection source");
while let Some(source) = cause.source() {
cause = source;
}
let original = cause
.downcast_ref::<std::io::Error>()
.expect("original database cause");
assert_eq!(original.kind(), std::io::ErrorKind::ConnectionRefused);
assert_eq!(original.to_string(), "PRIVATE_DATABASE_CAUSE");
assert!(!format!("{error:?} {error}").contains("PRIVATE_DATABASE_CAUSE"));
}
#[tokio::test]
@@ -168,121 +168,6 @@ async fn unknown_image_turn_does_not_infer_a_conversation() {
assert_eq!(affinity.binding_count(), 0);
}
/// 显式运行才读取本地导出并访问官方服务;只输出状态与测试账号别名
#[tokio::test]
#[ignore = "requires CPR_LIVE_ACCOUNTS_FILE and sends real requests"]
async fn real_conversation_requests_keep_the_imported_account() {
let (store, account) = super::live::imported_account().await;
let field = |key| account.get(key).and_then(Value::as_str).map(str::to_owned);
let model = super::live::model(&store).await;
let affinity = Arc::new(MemorySessionAffinity::default());
let provider = provider_with_affinity(&store, affinity.clone());
let session = uuid::Uuid::new_v4().to_string();
let turn = uuid::Uuid::new_v4().to_string();
let response = |thread: &str| {
let request_turn = if thread == session {
format!("root-{turn}")
} else {
turn.clone()
};
Operation::Generate(GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object("openai", json!({
"model":model, "instructions":"Reply with OK only.", "stream":true, "store":false,
"input":[{"role":"user","content":[{"type":"input_text","text":"Reply with OK."}]}],
"client_metadata":{"x-codex-turn-metadata":json!({"session_id":session,"thread_id":thread,"turn_id":request_turn}).to_string()}
}).as_object().unwrap().clone()).unwrap().with_context(Map::from_iter([
("use_websocket".into(), json!(false)), ("session_id".into(), json!("different-cache-routing-key"))
])),
))
};
let search = Operation::Search(StandaloneSearchRequest::from_raw_json(RawJsonPayload::new("openai", Bytes::from(json!({
"id":session, "model":model, "commands":{"search_query":[{"q":"site:openai.com ChatGPT"}]}, "max_output_tokens":128
}).to_string())).unwrap()));
let image = Operation::GenerateImage(ImageRequest::from_raw_json(ImageRequestKind::Generation, RawJsonPayload::new("openai", Bytes::from(json!({
"prompt":"A plain blue square on a white background.", "model":"gpt-image-2", "n":1, "quality":"low", "size":"1024x1024"
}).to_string())).unwrap().with_context(Map::from_iter([("image_turn_id".into(), json!(turn))]))));
let summarize_error = |stage: &str, error: gateway_core::error::ProviderError| {
let message = error.client_visible_upstream_error().map(|upstream| {
let mut message = upstream.message().to_owned();
for key in [
"accessToken",
"refreshToken",
"idToken",
"email",
"accountId",
"userId",
] {
if let Some(value) = field(key).filter(|value| !value.is_empty()) {
message = message.replace(&value, "[redacted]");
}
}
message.chars().take(256).collect::<String>()
});
json!({"stage":stage,"kind":format!("{:?}",error.kind()),"status":error.upstream_status(),"code":error.client_visible_upstream_error().and_then(|error| error.code()),"message":message})
};
let mut results = Vec::new();
for (name, operation) in [
("root", response(&session)),
("child", response("child-thread")),
("search", search),
("image-turn-alias", image),
] {
let request = if matches!(operation, Operation::Generate(_)) {
planned_request_for_model("openai", operation, &model)
} else {
planned_provider_endpoint_request("openai", operation)
};
let run = async {
let mut stream = provider
.clone()
.execute(
request,
context("req_live_binding_probe", CancellationToken::new()),
)
.await
.map_err(|error| summarize_error("selection", error))?;
assert_eq!(
stream.metadata().provider_account_id().as_str(),
"acct_provider_contract",
"never send decoy credentials upstream"
);
let mut events = 0;
while let Some(event) = stream.next().await {
event.map_err(|error| summarize_error("upstream", error))?;
events += 1;
}
Ok::<_, Value>(events)
};
let result = match timeout(Duration::from_secs(90), run).await {
Ok(Ok(events)) => {
json!({"case":name,"account":"imported-account","ok":true,"events":events})
}
Ok(Err(error)) => {
json!({"case":name,"account":"imported-account","ok":false,"error":error})
}
Err(_) => {
json!({"case":name,"account":"imported-account","ok":false,"error":"timeout"})
}
};
eprintln!("LIVE_BINDING {result}");
results.push(result);
if name == "root" {
// 高权重候选验证后续请求没有重新按权重选号,发送前断言阻止误用占位凭据
create_account(&store, "acct_subagent_b").await;
store.set_scheduling("acct_subagent_b", None, AccountWeight::new(100).unwrap());
}
}
assert_eq!(affinity.binding_count(), 1);
if let Ok(path) = std::env::var("CPR_LIVE_REPORT") {
std::fs::write(path, serde_json::to_vec_pretty(&results).unwrap())
.expect("write sanitized report");
}
assert!(
results.iter().all(|result| result["ok"] == true),
"some real upstream requests failed; see sanitized status report"
);
}
#[tokio::test]
async fn child_waits_for_first_root_claim_and_cancellation_releases_waiter() {
let store = Arc::new(MemoryAccountStore::default());
@@ -3,7 +3,6 @@
use std::time::Instant;
use gateway_core::engine::provider::ProviderStream;
use gateway_core::error::ProviderError;
use gateway_core::event::ProviderResponseTimings;
use super::*;
@@ -174,107 +173,3 @@ async fn structural_frames_wait_for_semantic_output_on_http_and_websocket() {
}
}
}
/// 只发送短文本请求,输出请求级耗时与用量,不记录凭据、账号身份或响应正文
#[tokio::test]
#[ignore = "requires CPR_LIVE_ACCOUNTS_FILE and sends real requests"]
async fn real_http_and_websocket_requests_report_output_timings() {
let (store, _) = super::live::imported_account().await;
let provider = provider_with_base_url(&store, OFFICIAL_CODEX_BASE_URL.to_owned());
let model = super::live::model(&store).await;
for websocket in [false, true] {
let expected_transport = if websocket { "websocket" } else { "http_sse" };
let mut protocol_context = Map::from_iter([("use_websocket".into(), json!(websocket))]);
if websocket {
// 模拟下游 WebSocket 新链,避免快路径预算到期后自动回退 HTTP
protocol_context.insert(
"downstream_websocket_connection_id".into(),
json!("ws_live_timing"),
);
}
let mut generate = GenerateRequest::from_protocol_payload(
ProtocolPayload::json_object(
"openai",
json!({
"model": model,
"instructions": "Reply with the numbers 1 through 30, separated by spaces, and nothing else.",
"stream": true,
"store": false,
"input": [{"role": "user", "content": [{"type": "input_text", "text": "Count now."}]}]
})
.as_object()
.unwrap()
.clone(),
)
.unwrap()
.with_context(protocol_context),
);
if websocket {
// 测试 Provider 未装配持久会话标识,显式提供本地池键,不声明上游续接 ID
generate = generate.with_provider_session_state(
ProviderSessionState::new(
"openai",
Map::from_iter([
("account_id".into(), json!("acct_provider_contract")),
("conversation_id".into(), json!("live_timing_conversation")),
("continuation_scope".into(), json!("persisted")),
]),
)
.unwrap(),
);
}
let operation = Operation::Generate(generate);
let started_at = Instant::now();
let run = async {
let mut stream = provider
.clone()
.execute(
planned_request_for_model("openai", operation, &model),
timed_context(started_at, 1),
)
.await?;
let mut timings = ProviderResponseTimings::default();
let mut output_tokens = None;
let mut completed = false;
while let Some(event) = stream.next().await {
let event = event?;
if let Some(observation) = event.response_observation() {
assert_eq!(observation.transport().as_str(), expected_transport);
timings = observation.timings();
}
for fact in event.canonical_facts() {
match fact {
GatewayEvent::Usage(usage) => output_tokens = usage.output_tokens,
GatewayEvent::Completed(_) => completed = true,
_ => {}
}
}
}
Ok::<_, ProviderError>((timings, output_tokens, completed))
};
let (timings, output_tokens, completed) = timeout(Duration::from_secs(35), run)
.await
.expect("live request deadline")
.unwrap_or_else(|error| {
panic!(
"live request failed: kind={:?}, status={:?}, code={:?}",
error.kind(),
error.upstream_status(),
error
.client_visible_upstream_error()
.and_then(|error| error.code())
);
});
let total_ms = started_at.elapsed().as_millis();
let first_event_ms = timings.first_event_ms.expect("first upstream packet");
let first_token_ms = timings.first_token_ms.expect("first semantic output");
let first_text_ms = timings.first_text_ms.expect("first text output");
let output_tokens = output_tokens.expect("upstream output usage");
assert!(completed && output_tokens > 0);
assert!(first_event_ms <= first_token_ms && first_token_ms <= first_text_ms);
assert!(u128::from(first_text_ms) <= total_ms);
eprintln!(
"LIVE_TIMING model={model} transport={expected_transport} first_event_ms={first_event_ms} first_token_ms={first_token_ms} first_text_ms={first_text_ms} total_ms={total_ms} output_tokens={output_tokens}"
);
}
}
@@ -259,7 +259,13 @@ impl ProviderAccountStore for MemoryAccountStore {
provider: &ProviderKind,
) -> Result<Vec<ProviderAccount>, StoreError> {
if self.fail_provider_listing.load(Ordering::SeqCst) {
return Err(store_error(StoreErrorKind::Unavailable));
return Err(StoreError::caused_by(
StoreErrorKind::Unavailable,
std::io::Error::new(
std::io::ErrorKind::ConnectionRefused,
"PRIVATE_DATABASE_CAUSE",
),
));
}
// 与 Postgres 实现的调度列表语义一致:停用账号不进入常规候选
Ok(self
@@ -6,12 +6,13 @@ use gateway_core::error::ProviderDiagnostic;
pub(super) fn is_invalid_encrypted_content_failure(error: &GrokInferenceTransportError) -> bool {
error.kind() == GrokInferenceTransportErrorKind::InvalidRequest
&& error.status() == Some(400)
&& error.upstream_code().is_some_and(|code| {
matches!(
code.as_str(),
REASONING_DECODE_FAILED_CODE | "invalid_encrypted_content"
)
})
&& (error.diagnostic().and_then(ProviderDiagnostic::code)
== Some(REASONING_DECODE_FAILED_CODE)
|| error.upstream_code().is_some_and(|code| {
[REASONING_DECODE_FAILED_CODE, "invalid_encrypted_content"]
.iter()
.any(|known| code.as_str().trim().eq_ignore_ascii_case(known))
}))
}
pub(super) enum InferenceBoundary {
@@ -279,15 +280,20 @@ pub(super) fn transport_error_contains_any(
let code = error
.upstream_code()
.map(|code| code.as_str())
.unwrap_or_default()
.to_ascii_lowercase();
let classification = error
.diagnostic()
.and_then(ProviderDiagnostic::code)
.unwrap_or_default();
let message = error
.client_visible_upstream_error()
.map(ClientVisibleUpstreamError::message)
.unwrap_or_default()
.to_ascii_lowercase();
signals
.iter()
.any(|signal| code.contains(signal) || message.contains(signal))
signals.iter().any(|signal| {
classification.contains(signal) || code.contains(signal) || message.contains(signal)
})
}
pub(super) async fn map_and_record_stream_transport_failure(
@@ -313,15 +319,21 @@ pub(super) fn map_continuation_failure(
let is_reasoning_decode_failure = context.continuation_attempt() == ContinuationAttempt::Native
&& error.kind() == ProviderErrorKind::InvalidRequest
&& error.upstream_status() == Some(400)
&& error
.upstream_code()
.is_some_and(|code| code.as_str() == REASONING_DECODE_FAILED_CODE);
&& (error.diagnostic().and_then(ProviderDiagnostic::code)
== Some(REASONING_DECODE_FAILED_CODE)
|| error.upstream_code().is_some_and(|code| {
code.as_str()
.trim()
.eq_ignore_ascii_case(REASONING_DECODE_FAILED_CODE)
}));
let is_missing_native_response = context.continuation_attempt() == ContinuationAttempt::Native
&& error.kind() == ProviderErrorKind::InvalidRequest
&& error.upstream_status() == Some(404)
&& error
.upstream_code()
.is_some_and(|code| code.as_str() == RESPONSE_NOT_FOUND_CODE);
&& error.upstream_code().is_some_and(|code| {
code.as_str()
.trim()
.eq_ignore_ascii_case(RESPONSE_NOT_FOUND_CODE)
});
if is_reasoning_decode_failure || is_missing_native_response {
error
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
@@ -519,6 +531,9 @@ pub(super) fn map_transport_error_with_state(
if let Some(code) = error.upstream_code().cloned() {
mapped = mapped.with_upstream_code(code);
}
if let Some(raw) = error.raw_upstream_error().cloned() {
mapped = mapped.with_raw_upstream_error(raw);
}
if let Some(detail) = error.client_visible_upstream_error().cloned() {
mapped = mapped.with_client_visible_upstream_error(detail);
}
@@ -539,6 +554,9 @@ pub(super) fn map_transport_error_with_state(
if error.sensitive_context_was_redacted() {
mapped = mapped.redact_sensitive_context("upstream transport context");
}
if std::error::Error::source(&error).is_some() {
mapped = mapped.with_source(error);
}
mapped
}
@@ -1246,8 +1246,9 @@ fn upstream_event_error(value: &Value) -> ProviderError {
_ => ProviderErrorKind::Unavailable,
}
};
let mut error = ProviderError::new(kind, UpstreamSendState::Sent)
.redact_sensitive_context("upstream event");
let mut error = ProviderError::new(kind, UpstreamSendState::Sent).with_raw_upstream_error(
gateway_core::error::RawUpstreamError::new(value.to_string()),
);
if let Some(code) = code {
error = error.with_upstream_code(OpaqueUpstreamValue::new(code.to_owned()));
}
@@ -1271,8 +1272,8 @@ fn upstream_error_field<'a>(value: &'a Value, field: &str) -> Option<&'a str> {
.and_then(Value::as_str)
}
fn protocol_error(_error: impl std::fmt::Debug) -> ProviderError {
protocol_error_marker()
fn protocol_error(error: impl std::error::Error + Send + Sync + 'static) -> ProviderError {
protocol_error_marker().with_source(error)
}
fn protocol_error_marker() -> ProviderError {
@@ -6,7 +6,10 @@ use std::pin::Pin;
use std::time::Duration;
use futures::Stream;
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic};
use gateway_core::error::{
ClientVisibleUpstreamError, ErrorSource, OpaqueUpstreamValue, ProviderDiagnostic,
RawUpstreamError,
};
use gateway_core::event::UpstreamHttpVersion;
use gateway_core::upstream::UpstreamSendState;
use url::Url;
@@ -317,11 +320,13 @@ pub enum GrokInferenceTransportErrorKind {
Cancelled,
}
/// 已分类的 transport 错误,绝不包含上游响应体
#[derive(Clone, PartialEq, Eq)]
/// 推理 transport 失败,公开分类与受控上游详情分别承载
#[derive(Clone)]
pub struct GrokInferenceTransportError {
kind: GrokInferenceTransportErrorKind,
source: Option<ErrorSource>,
diagnostic: Option<Box<ProviderDiagnostic>>,
raw_upstream_error: Option<Box<RawUpstreamError>>,
send_state: UpstreamSendState,
status: Option<u16>,
retry_after: Option<Duration>,
@@ -340,7 +345,9 @@ impl GrokInferenceTransportError {
pub const fn new(kind: GrokInferenceTransportErrorKind, send_state: UpstreamSendState) -> Self {
Self {
kind,
source: None,
diagnostic: None,
raw_upstream_error: None,
send_state,
status: None,
retry_after: None,
@@ -370,6 +377,24 @@ impl GrokInferenceTransportError {
self.diagnostic.as_deref()
}
#[must_use]
pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
self.source = Some(ErrorSource::new(source));
self
}
/// 原始响应仅交给受控诊断,普通格式化不会展开正文
#[must_use]
pub fn with_raw_upstream_error(mut self, error: RawUpstreamError) -> Self {
self.raw_upstream_error = Some(Box::new(error));
self
}
#[must_use]
pub fn raw_upstream_error(&self) -> Option<&RawUpstreamError> {
self.raw_upstream_error.as_deref()
}
/// 附着合法的 HTTP 状态码
#[must_use]
pub fn with_status(mut self, status: u16) -> Self {
@@ -397,7 +422,7 @@ impl GrokInferenceTransportError {
self
}
/// 附着从错误 JSON 中提取并清洗后的稳定机器码
/// 保留错误 JSON 中的原始 code,业务分类不覆盖此值
#[must_use]
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
self.upstream_code = Some(Box::new(code));
@@ -536,7 +561,11 @@ impl fmt::Display for GrokInferenceTransportError {
}
}
impl std::error::Error for GrokInferenceTransportError {}
impl std::error::Error for GrokInferenceTransportError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source.as_deref().map(|source| source as _)
}
}
/// 推理 transport 返回的 future
pub type GrokInferenceTransportFuture<'a> = Pin<
@@ -462,7 +462,7 @@ impl GrokInferenceTransport for ReqwestGrokInferenceTransport {
UpstreamSendState::Sent,
)
.with_transport_metrics(transport_metrics)),
Err(error) => Err(classify_inference_stream_error(&error)
Err(error) => Err(classify_inference_stream_error(error)
.with_transport_metrics(transport_metrics)),
};
std::future::ready(Some(item))
@@ -1018,8 +1018,19 @@ fn classify_inference_reqwest_error(error: reqwest::Error) -> GrokInferenceTrans
UpstreamSendState::Ambiguous,
)
};
GrokInferenceTransportError::new(kind, send_state)
.with_diagnostic(inference_http_diagnostic(&error))
let failure = GrokInferenceTransportError::new(kind, send_state)
.with_diagnostic(inference_http_diagnostic(&error));
with_inference_reqwest_source(failure, error)
}
fn with_inference_reqwest_source(
mut failure: GrokInferenceTransportError,
source: reqwest::Error,
) -> GrokInferenceTransportError {
if source.url().is_some() {
failure = failure.redact_sensitive_context("request URL");
}
failure.with_source(source.without_url())
}
fn classify_model_catalog_reqwest_error(error: reqwest::Error) -> GrokModelCatalogTransportError {
@@ -1044,8 +1055,8 @@ fn classify_billing_reqwest_error(error: reqwest::Error) -> GrokBillingTransport
GrokBillingTransportError::new(kind)
}
fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTransportError {
GrokInferenceTransportError::new(
fn classify_inference_stream_error(error: reqwest::Error) -> GrokInferenceTransportError {
let failure = GrokInferenceTransportError::new(
if error.is_timeout() {
GrokInferenceTransportErrorKind::Timeout
} else {
@@ -1053,7 +1064,8 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans
},
UpstreamSendState::Sent,
)
.with_diagnostic(inference_http_diagnostic(error))
.with_diagnostic(inference_http_diagnostic(&error));
with_inference_reqwest_source(failure, error)
}
async fn classify_inference_status(
@@ -1065,16 +1077,27 @@ async fn classify_inference_status(
let http_version = upstream_http_version(response.version());
let request_id = upstream_request_id(&response);
let status_code = status.as_u16();
let body = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await {
Ok(BoundedBody::Body(body)) => body,
Ok(BoundedBody::TooLarge) | Err(_) => {
let (body, body_error) = match collect_bounded(response, MAX_ERROR_BODY_BYTES).await {
Ok(BoundedBody::Body(body)) => (Some(body), None),
Ok(BoundedBody::TooLarge) => {
trace.record(
"capture.gap",
serde_json::json!({"reason": "error_body_unavailable_or_too_large"}),
serde_json::json!({"reason": "error_body_too_large"}),
);
Vec::new()
(None, None)
}
Err(source) => {
trace.record(
"capture.gap",
serde_json::json!({"reason": "error_body_read_failed"}),
);
(None, Some(source))
}
};
let raw = body.as_ref().map(|body| {
gateway_core::error::RawUpstreamError::new(String::from_utf8_lossy(body).into_owned())
});
let body = body.unwrap_or_default();
trace.capture("upstream.error.body", &body);
let metadata = inference_error_metadata(&body);
let body_failure = classify_grok_body_failure(&metadata, &body);
@@ -1108,17 +1131,23 @@ async fn classify_inference_status(
let credential_recovery_required = kind == GrokInferenceTransportErrorKind::Unauthorized;
let mut error = GrokInferenceTransportError::new(kind, UpstreamSendState::Sent)
.with_status(status_code)
.with_response_facts(http_version, request_id)
.redact_sensitive_context("upstream response body");
let upstream_code = if status == StatusCode::BAD_REQUEST && reasoning_decode_failed(&metadata) {
Some("reasoning_decode_failed".to_owned())
.with_response_facts(http_version, request_id);
if let Some(raw) = raw {
error = error.with_raw_upstream_error(raw);
}
let diagnostic_code = if status == StatusCode::BAD_REQUEST && reasoning_decode_failed(&metadata)
{
"reasoning_decode_failed"
} else {
metadata
.code
.as_deref()
.and_then(normalize_failure_code)
.or_else(|| body_failure.map(GrokBodyFailure::marker).map(str::to_owned))
body_failure.map_or("upstream_failure", GrokBodyFailure::marker)
};
error = error.with_diagnostic(
ProviderDiagnostic::new(format!(
"xAI upstream rejected request: status={status_code}"
))
.with_classification("upstream", diagnostic_code),
);
let upstream_code = metadata.code;
if let Some(message) = metadata.client_message.as_deref() {
error = error.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
scrub_account_fingerprints(message),
@@ -1135,6 +1164,9 @@ async fn classify_inference_status(
if let Some(retry_after) = retry_after {
error = error.with_retry_after(retry_after);
}
if let Some(source) = body_error {
error = with_inference_reqwest_source(error, source);
}
error
}
@@ -1287,8 +1319,7 @@ fn first_string(object: &serde_json::Map<String, Value>, fields: &[&str]) -> Opt
fields
.iter()
.find_map(|field| object.get(*field).and_then(Value::as_str))
.map(str::trim)
.filter(|value| !value.is_empty())
.filter(|value| !value.trim().is_empty())
.map(ToOwned::to_owned)
}
@@ -1456,22 +1487,6 @@ fn contains_any(value: &str, signals: &[&str]) -> bool {
signals.iter().any(|signal| value.contains(signal))
}
fn normalize_failure_code(value: &str) -> Option<String> {
let mut normalized = String::with_capacity(value.len().min(48));
for character in value.trim().to_ascii_lowercase().chars() {
if character.is_ascii_alphanumeric() {
normalized.push(character);
} else if matches!(character, '-' | '_' | '.' | ':') {
normalized.push('_');
}
if normalized.len() >= 48 {
break;
}
}
let normalized = normalized.trim_matches('_');
(!normalized.is_empty()).then(|| normalized.to_owned())
}
fn upstream_http_version(version: reqwest::Version) -> UpstreamHttpVersion {
match version {
reqwest::Version::HTTP_09 => UpstreamHttpVersion::Http09,
@@ -2750,7 +2750,10 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() {
GrokInferenceTransportErrorKind::ModelQuotaExhausted,
UpstreamSendState::Sent,
)
.with_status(403),
.with_status(403)
.with_raw_upstream_error(gateway_core::error::RawUpstreamError::new(
"PRIVATE_QUOTA_ERROR",
)),
);
let provider = provider(selector.clone(), transport).await;
let mut stream = provider
@@ -2764,6 +2767,11 @@ async fn model_quota_transport_failure_preserves_model_scoped_feedback() {
let error = next_provider_error(&mut stream).await;
assert_eq!(error.kind(), ProviderErrorKind::QuotaExhausted);
assert_eq!(
error.raw_upstream_error().unwrap().as_str(),
"PRIVATE_QUOTA_ERROR"
);
assert!(!format!("{error:?} {error}").contains("PRIVATE_QUOTA_ERROR"));
assert!(error.replay_is_safe());
assert_eq!(
selector.feedback.lock().expect("feedback").as_slice(),
@@ -1452,6 +1452,10 @@ fn decoder_should_classify_failed_event_without_retaining_body() {
assert_eq!(visible.message(), "secret");
assert_eq!(visible.code(), Some("rate_limit_exceeded"));
assert_eq!(visible.error_type(), Some("server_error"));
let raw: serde_json::Value =
serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap();
assert_eq!(raw["error"]["message"], "secret");
assert_eq!(raw["error"]["code"], "rate_limit_exceeded");
assert!(!format!("{error:?}").contains("secret"));
}
@@ -27,6 +27,58 @@ use provider_xai::{
use crate::support::{loopback_endpoint_policy, rejecting_account_proxy};
#[tokio::test]
async fn inference_transport_preserves_verbatim_unknown_codes_without_changing_classification() {
for code in [
" Vendor.Future:v2-UNRECOGNIZED ",
"长错误码",
"Vendor.".repeat(20).as_str(),
] {
let server = MockServer::start().await;
let body = json!({"error": {"code": code, "message": "private response detail"}});
Mock::given(method("POST"))
.and(path("/v1/responses"))
.respond_with(ResponseTemplate::new(503).set_body_json(&body))
.mount(&server)
.await;
let origin = Url::parse(&server.uri()).unwrap();
let error = inference_transport(&origin)
.execute(inference_request(&origin))
.await
.unwrap_err();
assert_eq!(error.kind(), GrokInferenceTransportErrorKind::Unavailable);
assert_eq!(error.upstream_code().unwrap().as_str(), code);
assert_eq!(
serde_json::from_str::<serde_json::Value>(error.raw_upstream_error().unwrap().as_str())
.unwrap(),
body
);
assert_eq!(error.diagnostic().unwrap().code(), Some("upstream_failure"));
assert!(!format!("{error:?} {error}").contains(code));
}
}
#[tokio::test]
async fn inference_transport_keeps_native_network_causes_without_request_urls() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
drop(listener);
let origin = Url::parse(&format!("http://{address}")).unwrap();
let error = inference_transport(&origin)
.execute(inference_request(&origin))
.await
.unwrap_err();
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
let source = std::error::Error::source(&error)
.unwrap()
.downcast_ref::<reqwest::Error>()
.unwrap();
assert!(source.is_connect());
assert!(source.url().is_none());
assert!(error.sensitive_context_was_redacted());
assert!(std::error::Error::source(source).is_some());
}
#[tokio::test]
async fn official_inference_tls_should_advertise_grok_cli_alpn() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -536,7 +588,7 @@ async fn inference_connect_failure_keeps_safe_diagnosis_without_endpoint_or_payl
}
#[tokio::test]
async fn inference_transport_should_classify_http_failures_without_retaining_bodies() {
async fn inference_transport_should_retain_http_failure_bodies_only_in_explicit_details() {
let cases = [
(400, GrokInferenceTransportErrorKind::InvalidRequest),
(401, GrokInferenceTransportErrorKind::Unauthorized),
@@ -578,7 +630,7 @@ async fn inference_transport_should_classify_http_failures_without_retaining_bod
expected_kind,
Some(status),
UpstreamSendState::Sent,
true,
false,
Some(gateway_core::event::UpstreamHttpVersion::Http11),
)
);
@@ -588,6 +640,7 @@ async fn inference_transport_should_classify_http_failures_without_retaining_bod
);
assert!(error.transport_metrics().headers_ms().is_some());
assert!(error.client_visible_upstream_error().is_none());
assert_eq!(error.raw_upstream_error().unwrap().as_str(), secret);
assert!(!rendered.contains(&secret));
}
}
@@ -621,7 +674,7 @@ async fn inference_transport_should_expose_safe_flat_json_error_details() {
);
assert_eq!(
(detail.message(), detail.code(), detail.error_type()),
(message, Some("personal_team_blocked_spending_limit"), None,)
(message, Some("personal-team-blocked:spending-limit"), None,)
);
assert!(!rendered.contains(message));
}
@@ -656,7 +709,7 @@ async fn inference_transport_should_scrub_nested_json_error_details() {
(detail.message(), detail.code(), detail.error_type()),
(
"team [redacted] rate limited",
Some("rate_limit_burst"),
Some("rate-limit:burst"),
Some("rate_limit_error"),
)
);
@@ -789,7 +842,7 @@ async fn inference_transport_should_classify_model_quota_for_403_and_429() {
#[tokio::test]
async fn inference_transport_should_prefer_structured_quota_fields_over_conflicting_message() {
for (code, error_type) in [
("quota_exceeded", Some("subscription_free_usage_exhausted")),
("quota_exceeded", Some("subscription:free-usage-exhausted")),
("rate_limit_exceeded", Some("quota_exceeded")),
] {
let server = MockServer::start().await;
@@ -845,7 +898,7 @@ async fn inference_transport_should_classify_free_usage_429_as_account_free_quot
assert_eq!(error.status(), Some(429));
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
Some("subscription_free_usage_exhausted")
Some("subscription:free-usage-exhausted")
);
}
@@ -855,7 +908,7 @@ async fn inference_transport_should_apply_body_aware_400_failures() {
(
json!({"error": {"code": "subscription:free-usage-exhausted", "message": "free usage exhausted"}}),
GrokInferenceTransportErrorKind::FreeQuotaExhausted,
Some("subscription_free_usage_exhausted"),
Some("free_usage_exhausted"),
),
(
json!({"error": {"message": "selected model is at capacity"}}),
@@ -900,7 +953,10 @@ async fn inference_transport_should_apply_body_aware_400_failures() {
assert_eq!(error.kind(), expected_kind);
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
error
.diagnostic()
.and_then(|diagnostic| diagnostic.code())
.filter(|code| *code != "upstream_failure"),
expected_code
);
}
@@ -961,8 +1017,9 @@ async fn inference_transport_should_classify_reasoning_decode_rejections() {
error.kind(),
GrokInferenceTransportErrorKind::InvalidRequest
);
assert!(error.upstream_code().is_none());
assert_eq!(
error.upstream_code().map(|code| code.as_str()),
error.diagnostic().and_then(|diagnostic| diagnostic.code()),
Some("reasoning_decode_failed")
);
}
+1
View File
@@ -20,3 +20,4 @@ fcfae912ddefd54551ebf8e7e733442519d2862d2977134d548ebc1e75c1dc34 0017_unlimited
e8c1339166ac02fb2d4a077424ee5f64f6dc20eb4d3f8ec4fcb09c0f2a0e61f6 0020_plugin_trust.sql
22faf2646168275150b1786c0f463f9ff2ca52463a272479e4fcc6f46c8cfa37 0021_runtime_scheduling.sql
77a140ddfb9dfae9b7d7349e5bfc9c8a7e7927bae6e77c1a627b34fcbb714bdb 0022_refresh_margin_and_group_fast_mode.sql
68fe28522870903f9a9c9772e13edda29b9b0f1798d350b1025c273b05d25370 0023_error_details.sql
@@ -0,0 +1,3 @@
-- 运维错误详情同时承载本地来源链与上游响应,运行时只使用统一字段
alter table model_requests rename column raw_upstream_error to error_details;
alter table ops_events rename column raw_upstream_error to error_details;
+13 -3
View File
@@ -1702,6 +1702,12 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可
Key 已删除或未关联时为 `null`,不影响记录返回,不包含密钥原文
运维错误记录的 `errorDetails` 为受控诊断文本,无详情时为 `null`。
错误来源快照以 JSON 文本保存:`causes.messages` 按外层到内层排列本地原因,`causes.truncated`
标记来源链截断,`upstream` 保存上游正文原文,`redacted` 标记敏感上下文移除;没有对应来源时为 `null`。
稳定错误分类、原始上游 code 和该详情分别保留,不以公共错误文案代替原始原因。
详情不出现在 Key 用量接口、普通错误信封和诊断包导出中;记录仍受异步观测写入与保留周期约束
管理端请求列表与详情分别保留 `requestedModel`(客户端请求)、`upstreamModel`(网关发送)与
`upstreamResponseModel`(上游返回)。返回模型缺失时为 `null`,不使用请求或映射模型补齐。
OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告(流内报告可覆盖初始响应头),
@@ -1751,9 +1757,13 @@ OpenAI 优先采用服务端 `openai-model` / `x-openai-model` 报告(流内
未知名称的 `{ bytes, sha256 }` 摘要,或缺失时的 `null`。已保存的 trace 不自动清理或回填,
其中的 `sanitized` 标记不能作为可直接公开的保证
管理端下载的诊断包 `schemaVersion: 2` 用于人工反馈,不是备份或导入格式。它包含关联 ID、错误分类摘要、
请求与错误事件各自的状态、attempt、时间线阶段和计时;不自动导出 message/raw error、任意 metadata、
trace event data、请求响应正文和头部。`availability` 与 `omitted` 明示未采集、不完整或主动省略的内容,
`attempt.failed.data` 的 `kind / sendState / upstreamStatus` 与 `diagnostic.stage / diagnostic.code`
由 Core 错误类型与 Provider 静态诊断生成;`diagnostic.message` 保存安全摘要。展示与导出共用这些字段,
`sendState` 为 `not_sent / sent / ambiguous`,摘要被截断时带有 `truncated` 标记
管理端下载的诊断包 `schemaVersion: 3` 用于人工反馈,不是备份或导入格式。它包含关联 ID、错误分类摘要、
请求与错误事件各自的状态、attempt、时间线阶段、计时和上述失败分类;不自动导出 message/raw error、任意 metadata、
其他 trace event data、请求响应正文和头部。`availability` 与 `omitted` 明示未采集、不完整或主动省略的内容,
`null` 不代表没有发生错误。版本、环境及原始错误片段仍需操作者另行补充并审阅脱敏
错误记录中的“已自动恢复”表示系统关联到了后续成功请求,不会把原来的失败记录改为成功。
+20 -2
View File
@@ -501,8 +501,26 @@ client,OIDC 的 JWKS 缓存与单飞归属对应出口状态。自动刷新提
2. **控制面展示错误**:`/api/admin/*` 由 API 统一 HTTP 状态与数值业务码,由 Admin/API owner 提供安全中文
文案。extractor rejection、namespace 404 和 method 405 也使用同一 JSON 信封;任意 Store、Serde、
Provider `Display` 不得直接跨越 HTTP 边界
3. **原始上游诊断**:只在连接测试、运维错误详情等明确诊断界面展示已经捕获的原始字段,不翻译、不补造,
也不塞入普通 Admin 错误信封。原始 body 不进入 Debug、普通日志或持久化错误消息
3. **受控错误详情**:运维错误详情分别保存本地原因链与上游响应,原始字段不翻译、不补造,
不进入普通 Admin 错误信封、Debug、普通日志或持久化错误摘要。连接测试只展示该接口允许的诊断投影
错误分类负责公共响应、重试和账号处置,原始上游 code 独立保存,不受已知分类白名单限制。
本地异常通过标准 `Error::source` 跨层传播;Core 的不透明来源载体保持底层类型,不依赖数据库或 HTTP 库,
普通格式化不展开来源。Store 统一基础设施不可用的转换,Provider 拥有业务分类,
插件 Runtime 集中投影 RPC、fault 与上游适配事件,调用点只附加原始来源与执行上下文
ProviderError 统一生成受控详情,失败 attempt、探测和最终请求复用该出口。
原始上游正文与本地原因链分开承载,来源链最多保留 32 层、64 KiB 文本,截断明确标记;
请求 URL 等请求方向的敏感上下文在附加来源前移除并标记。错误快照保留来源与错误详情,
协议透传响应和未交付事件仍由当前请求独占。详情计入观测队列的内存预算;
队列满、关闭超时或写入失败仍可能丢失记录,沿用丢弃与写入失败指标,不自动重放不保证幂等的写入
推理日志由作出处置的边界记录恢复、放弃或终态;仅传播错误的层不重复打印。
传输收包使用 debug 摘要,普通日志使用稳定阶段、分类与 request/attempt 关联,不展开原始正文、
任意上游 code 或关闭原因。受控详情无法写入时不能回退到普通日志打印原文
Core 从执行错误生成一份失败诊断,展示与导出共用其中的分类;事件摘要截断时保留有界的失败诊断,
并显式标记截断。完整上游错误仍归运维错误记录,不进入诊断包
Provider 管理适配在仍持有结构化失败事实时选择静态 `public_message`;Admin 将其转为可安全展示的
`AdminError.message`,API 保留其中的 502/503 具体原因,无公开消息时才使用通用回退。认证与未知
+13 -2
View File
@@ -209,7 +209,18 @@ export interface RequestTraceEvent {
exchangeId: number | null
stage: string
count: number
data: Record<string, unknown>
data: Record<string, unknown> & {
/** attempt.failed 由 Core 从 ProviderError 生成,分类与诊断只有这一份来源 */
kind?: string
sendState?: 'not_sent' | 'sent' | 'ambiguous'
upstreamStatus?: number | null
diagnostic?: {
stage: string | null
code: string | null
message: string
truncated?: boolean
} | null
}
}
export interface RequestTrace {
@@ -303,7 +314,7 @@ export interface OpsError {
subagentKind: string | null
compact: boolean | null
message: string
rawUpstreamError: string | null
errorDetails: string | null
metadata: OpsErrorMetadata
createdAt: string
createdAtDisplay: string

Some files were not shown because too many files have changed in this diff Show More