From d922c3af604431e5c25898e71a2204191d735587 Mon Sep 17 00:00:00 2001 From: Zyy Date: Tue, 6 Oct 2026 20:26:46 +0800 Subject: [PATCH] fix(diagnostics): preserve error chains and avoid logging backpressure Preserve native failure causes through store, provider, plugin, admin, and worker boundaries, including secondary cleanup failures. Route operational failures through the shared diagnostics sink with protected error details. Use bounded nonblocking file logging with batched writes and reserved warning/error capacity. Reserve observation capacity for failures and request lifecycle records, and rate-limit overload warnings. Validate error propagation and blocked-output HTTP, admin, SSE, and WebSocket behavior; keep the local migration plan out of version control. Co-authored-by: Codex --- backend/apps/gateway/src/bootstrap/server.rs | 3 +- .../crates/gateway-admin/src/backup/task.rs | 11 +- backend/crates/gateway-admin/src/model/mod.rs | 35 ++- .../src/model/provider_credentials.rs | 2 +- .../gateway-admin/src/ports/provider.rs | 31 ++- .../crates/gateway-admin/src/ports/store.rs | 14 +- .../crates/gateway-admin/src/use_case/mod.rs | 20 +- .../tests/model/provider_credentials.rs | 10 +- .../gateway-api/src/admin/diagnostics.rs | 42 ++++ backend/crates/gateway-api/src/admin/mod.rs | 1 + backend/crates/gateway-api/src/admin/wire.rs | 6 + backend/crates/gateway-api/src/lib.rs | 5 + .../crates/gateway-api/src/openai/error.rs | 7 +- .../tests/admin/error_diagnostics.rs | 61 +++++ backend/crates/gateway-api/tests/admin/mod.rs | 2 + .../crates/gateway-api/tests/architecture.rs | 2 + .../crates/gateway-api/tests/openai/auth.rs | 2 + .../crates/gateway-api/tests/openai/error.rs | 51 ++-- .../tests/openai/live/websocket.rs | 1 + .../crates/gateway-api/tests/openai/mod.rs | 19 ++ .../websocket/connection/middleware.rs | 2 + .../responses/websocket/connection/mod.rs | 1 + .../openai/responses/websocket/interrupt.rs | 1 + .../tests/openai/responses/websocket/mod.rs | 2 + backend/crates/gateway-api/tests/support.rs | 15 ++ .../gateway-core/src/diagnostics/failure.rs | 54 +++++ .../gateway-core/src/diagnostics/mod.rs | 4 +- .../gateway-core/src/engine/admission.rs | 8 +- .../crates/gateway-core/src/engine/budget.rs | 4 +- .../gateway-core/src/engine/execution.rs | 76 +++++- .../crates/gateway-core/src/error/details.rs | 43 ++++ backend/crates/gateway-core/src/error/mod.rs | 53 +++-- .../crates/gateway-core/src/error/source.rs | 131 ++++++++-- backend/crates/gateway-core/src/lib.rs | 4 + .../crates/gateway-core/src/provider_ports.rs | 12 +- .../crates/gateway-core/src/runtime/mod.rs | 5 +- backend/crates/gateway-core/src/task.rs | 37 ++- .../gateway-core/tests/engine/admission.rs | 4 +- .../gateway-core/tests/engine/execution.rs | 83 ++++++- .../crates/gateway-core/tests/error/mod.rs | 31 +++ backend/crates/gateway-core/tests/main.rs | 13 + backend/crates/gateway-host/src/lib.rs | 3 +- .../crates/gateway-host/src/logging/sink.rs | 157 ++++++++++-- .../crates/gateway-host/src/logging/writer.rs | 17 +- backend/crates/gateway-host/src/workers.rs | 86 ++++--- .../crates/gateway-host/tests/logging/sink.rs | 131 ++++++++++ .../tests/plugin_distribution/github.rs | 13 +- backend/crates/gateway-host/tests/workers.rs | 119 +++++++-- .../src/adapter/command_line/execution.rs | 8 +- .../runtime/src/adapter/policy/middleware.rs | 2 +- .../runtime/src/rpc/dispatch.rs | 25 +- .../gateway-plugin/runtime/src/rpc/session.rs | 100 ++++---- .../runtime/src/stream/bridge.rs | 4 +- .../runtime/src/stream/flow_control.rs | 13 +- .../runtime/tests/generation/maintenance.rs | 42 +++- .../runtime/tests/rpc/session.rs | 42 +++- .../runtime/tests/support/worker/mod.rs | 14 +- .../gateway-plugin/sdk/src/client/frame.rs | 6 +- .../crates/gateway-plugin/sdk/src/message.rs | 2 +- .../gateway-plugin/sdk/tests/message.rs | 21 ++ .../gateway-store/src/backup/staging.rs | 2 + backend/crates/gateway-store/src/bundle.rs | 12 +- backend/crates/gateway-store/src/config.rs | 9 + .../src/postgres/account_groups.rs | 78 +++--- .../src/postgres/admin_security_audit.rs | 6 +- .../src/postgres/admission_recovery.rs | 8 +- .../gateway-store/src/postgres/backup.rs | 225 +++++++++--------- .../src/postgres/client_budgets.rs | 52 ++-- .../gateway-store/src/postgres/client_keys.rs | 98 +++++--- .../gateway-store/src/postgres/execution.rs | 58 ++++- .../src/postgres/execution_buffer.rs | 135 +++++++---- .../crates/gateway-store/src/postgres/mod.rs | 20 +- .../src/postgres/observability/mapping.rs | 17 +- .../postgres/observability/queries/usage.rs | 1 + .../postgres/observability/query_budget.rs | 2 + .../gateway-store/src/postgres/ops_events.rs | 7 +- .../gateway-store/src/postgres/pricing.rs | 1 + .../provider_accounts/admin_adapter.rs | 1 + .../provider_accounts/admin_queries.rs | 1 + .../provider_accounts/authorization.rs | 43 ++-- .../postgres/provider_accounts/repository.rs | 23 +- .../src/postgres/provider_accounts/rows.rs | 21 +- .../gateway-store/src/postgres/proxies.rs | 144 +++++++---- .../src/postgres/runtime_settings.rs | 8 + .../gateway-store/src/postgres/snapshot.rs | 19 +- .../crates/gateway-store/src/redis/auth.rs | 1 + .../src/redis/client_admission.rs | 18 +- .../src/redis/coordination_buffer.rs | 56 +++-- .../src/redis/credential_cooldown.rs | 9 +- .../src/redis/credential_leases.rs | 23 +- .../src/redis/credential_state.rs | 2 + backend/crates/gateway-store/src/redis/mod.rs | 1 + .../gateway-store/src/redis/oauth_pending.rs | 1 + .../gateway-store/src/redis/runtime_change.rs | 7 +- .../gateway-store/src/redis/worker_lease.rs | 12 +- backend/crates/gateway-store/src/value.rs | 101 ++++++-- backend/crates/gateway-store/src/workers.rs | 5 +- backend/crates/gateway-store/tests/main.rs | 13 + .../tests/postgres/client_budgets.rs | 42 ++++ .../tests/postgres/execution_buffer.rs | 156 ++++++++++++ .../tests/postgres/query_budget.rs | 1 + .../tests/redis/coordination_buffer.rs | 3 + backend/crates/providers/openai/src/admin.rs | 1 + .../providers/openai/src/credential/admin.rs | 38 +-- .../openai/src/credential/diagnostics.rs | 52 ++++ .../providers/openai/src/credential/mod.rs | 2 + .../openai/src/credential/refresh.rs | 54 +++-- .../openai/src/credential/token_client.rs | 177 ++++++++++++-- backend/crates/providers/openai/src/lib.rs | 2 + .../providers/openai/src/provider/failure.rs | 23 +- .../providers/openai/src/provider/mod.rs | 4 +- .../providers/openai/src/provider/workers.rs | 27 +-- .../crates/providers/openai/tests/admin.rs | 15 ++ .../openai/tests/credential/admin.rs | 18 ++ .../openai/tests/credential/refresh.rs | 53 +++++ .../openai/tests/credential/token_client.rs | 55 ++++- backend/crates/providers/openai/tests/main.rs | 13 + .../openai/tests/provider/contract/mod.rs | 25 +- backend/crates/providers/xai/src/admin.rs | 10 +- .../providers/xai/src/credential/catalog.rs | 6 +- .../providers/xai/src/credential/refresh.rs | 17 +- .../xai/src/credential/repository.rs | 87 ++++--- .../providers/xai/src/credential/selector.rs | 68 ++++-- .../providers/xai/src/provider/failure.rs | 23 +- .../providers/xai/src/provider/workers.rs | 7 +- .../providers/xai/src/transport/session.rs | 6 +- backend/crates/providers/xai/tests/admin.rs | 20 +- .../xai/tests/credential/contract.rs | 4 +- .../xai/tests/credential/repository.rs | 4 +- .../providers/xai/tests/provider/contract.rs | 57 ++++- deploy/README.md | 12 +- docs/api.md | 3 +- docs/architecture.md | 28 ++- 133 files changed, 3144 insertions(+), 922 deletions(-) create mode 100644 backend/crates/gateway-api/src/admin/diagnostics.rs create mode 100644 backend/crates/gateway-api/tests/admin/error_diagnostics.rs create mode 100644 backend/crates/gateway-core/src/diagnostics/failure.rs create mode 100644 backend/crates/gateway-core/src/error/details.rs create mode 100644 backend/crates/providers/openai/src/credential/diagnostics.rs diff --git a/backend/apps/gateway/src/bootstrap/server.rs b/backend/apps/gateway/src/bootstrap/server.rs index 927f295a4..569f032ec 100644 --- a/backend/apps/gateway/src/bootstrap/server.rs +++ b/backend/apps/gateway/src/bootstrap/server.rs @@ -117,6 +117,7 @@ pub async fn run() -> Result<(), BootstrapError> { probes, host.worker_health(), host.connection_lifecycle(), + store.diagnostics(), ) ) .with_middleware({ @@ -155,7 +156,7 @@ pub async fn run() -> Result<(), BootstrapError> { )); or_shutdown!( plugin_runtime, - host.start_workers(plan, store.worker_leader_lease()) + host.start_workers(plan, store.worker_leader_lease(), store.diagnostics()) ); host.report_startup_ready("Workers"); let served = host.serve(api.router()).await; diff --git a/backend/crates/gateway-admin/src/backup/task.rs b/backend/crates/gateway-admin/src/backup/task.rs index d2656afac..06417bb27 100644 --- a/backend/crates/gateway-admin/src/backup/task.rs +++ b/backend/crates/gateway-admin/src/backup/task.rs @@ -28,7 +28,7 @@ use crate::{ impl From for WorkerTaskError { fn from(error: BackupError) -> Self { - WorkerTaskError::safe(error.message()) + WorkerTaskError::safe(error.message()).with_source(error) } } @@ -125,8 +125,9 @@ impl BackupTask { if cron.is_empty() { return Ok(()); } - let schedule = BackupSchedule::parse(cron, self.timezone) - .map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?; + let schedule = BackupSchedule::parse(cron, self.timezone).map_err(|source| { + WorkerTaskError::safe("backup schedule is invalid").with_source(source) + })?; if settings.schedule_timezone.as_deref() != Some(timezone) { // 时区切换只初始化未来游标,不把旧时区计划当作应补跑任务 @@ -685,9 +686,9 @@ fn status_transition( } fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError { - WorkerTaskError::safe(format!("backup store failed: {error}")) + WorkerTaskError::safe("backup store failed").with_source(error) } fn infra_error(error: BackupError) -> WorkerTaskError { - WorkerTaskError::safe(error.message()) + WorkerTaskError::safe(error.message()).with_source(error) } diff --git a/backend/crates/gateway-admin/src/model/mod.rs b/backend/crates/gateway-admin/src/model/mod.rs index 6d7ce6eb1..ca4f0b8ce 100644 --- a/backend/crates/gateway-admin/src/model/mod.rs +++ b/backend/crates/gateway-admin/src/model/mod.rs @@ -39,12 +39,31 @@ pub enum AdminErrorKind { Internal, } -/// 不携带基础设施细节、可安全返回给管理员的管理用例错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +impl AdminErrorKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Invalid => "invalid", + Self::Forbidden => "forbidden", + Self::Unauthorized => "unauthorized", + Self::NotFound => "not_found", + Self::Conflict => "conflict", + Self::RateLimited => "rate_limited", + Self::BadGateway => "bad_gateway", + Self::UpstreamResultUnknown => "upstream_result_unknown", + Self::Unavailable => "unavailable", + Self::Internal => "internal", + } + } +} + +/// 管理用例的安全消息与内部原因分离,原始来源不进入响应 +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct AdminError { kind: AdminErrorKind, message: String, + source: Option, } impl AdminError { @@ -53,9 +72,21 @@ impl AdminError { Self { kind, message: message.into(), + source: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + gateway_core::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } + #[must_use] pub const fn kind(&self) -> AdminErrorKind { self.kind diff --git a/backend/crates/gateway-admin/src/model/provider_credentials.rs b/backend/crates/gateway-admin/src/model/provider_credentials.rs index 27456c462..a8a04185a 100644 --- a/backend/crates/gateway-admin/src/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/src/model/provider_credentials.rs @@ -862,7 +862,7 @@ pub struct ProviderSubscription { } /// 按需汇聚的个人信息;资料查询失败不丢弃可用的订阅结果 -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub struct AccountPersonalInfo { pub profile: Result, pub subscription: Option, diff --git a/backend/crates/gateway-admin/src/ports/provider.rs b/backend/crates/gateway-admin/src/ports/provider.rs index 182e038e0..0079f2610 100644 --- a/backend/crates/gateway-admin/src/ports/provider.rs +++ b/backend/crates/gateway-admin/src/ports/provider.rs @@ -46,11 +46,12 @@ pub enum ProviderAdminErrorKind { /// `message` 是 Provider 局部诊断,可能包含原始上游正文;通用管理用例不得自动把它作为公开文案 /// 只有明确拥有原始诊断合同的调用方才能读取,`Debug` 始终只记录是否存在 /// `public_message` 则是明确标记为可公开的静态提示,不允许携带动态上游材料 -#[derive(Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Clone, thiserror::Error)] #[error("provider admin operation failed: {kind:?}")] pub struct ProviderAdminError { kind: ProviderAdminErrorKind, - message: Option, + #[source] + diagnostic: Option>, public_message: Option<&'static str>, } @@ -59,7 +60,7 @@ impl std::fmt::Debug for ProviderAdminError { formatter .debug_struct("ProviderAdminError") .field("kind", &self.kind) - .field("message", &self.message.as_ref().map(|_| "")) + .field("message", &self.message().map(|_| "")) .field("public_message", &self.public_message) .finish() } @@ -70,14 +71,22 @@ impl ProviderAdminError { pub const fn new(kind: ProviderAdminErrorKind) -> Self { Self { kind, - message: None, + diagnostic: None, public_message: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).source = + Some(source.into()); + self + } + #[must_use] pub fn with_message(mut self, message: impl Into) -> Self { - self.message = Some(message.into()); + Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).message = + Some(message.into()); self } @@ -100,10 +109,20 @@ impl ProviderAdminError { #[must_use] pub fn message(&self) -> Option<&str> { - self.message.as_deref() + self.diagnostic + .as_ref() + .and_then(|diagnostic| diagnostic.message.as_deref()) } } +// Provider 局部消息与 typed 来源共用一个不公开的上下文,避免跨用例转换后丢失 +#[derive(Debug, Clone, Default, thiserror::Error)] +#[error("{}", message.as_deref().unwrap_or("provider admin dependency failed"))] +struct ProviderAdminDiagnostic { + message: Option, + source: Option, +} + /// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力 /// /// 数据变更由 Provider 返回 prepared facts;config revision、审计与 PostgreSQL 事务 diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 00b0b9203..16cad9b77 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -78,8 +78,18 @@ impl AdminStoreError { /// 来源留在内部诊断链,公开消息不展开基础设施错误 #[must_use] - pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { - self.source = Some(gateway_core::error::ErrorSource::new(source)); + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + /// 清理失败不覆盖最初操作的分类与原因 + #[must_use] + pub fn with_cleanup(mut self, cleanup: impl Into) -> Self { + self.source = Some(gateway_core::error::ErrorSource::cleanup( + self.source.take(), + cleanup, + )); self } diff --git a/backend/crates/gateway-admin/src/use_case/mod.rs b/backend/crates/gateway-admin/src/use_case/mod.rs index 5a3d86002..8ed47dccd 100644 --- a/backend/crates/gateway-admin/src/use_case/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/mod.rs @@ -51,7 +51,6 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError | AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict, AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable, }; - tracing::warn!(resource, error_kind = ?error.kind(), "admin store operation failed"); let message = match kind { AdminErrorKind::Invalid => "请求参数不合法", AdminErrorKind::NotFound => "请求的资源不存在", @@ -59,7 +58,10 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError AdminErrorKind::Unavailable => "依赖服务暂不可用", _ => "服务内部错误", }; - AdminError::new(kind, message) + AdminError::new(kind, message).with_source(OperationFailure { + operation: resource, + source: error.into(), + }) } fn map_provider_error( @@ -79,7 +81,6 @@ fn map_provider_error( ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway, ProviderAdminErrorKind::Internal => AdminErrorKind::Internal, }; - tracing::warn!(resource, error_kind = ?error.kind(), "admin provider operation failed"); let message = match kind { AdminErrorKind::Invalid => "Provider 请求不合法", AdminErrorKind::NotFound => "Provider 资源不存在", @@ -90,7 +91,10 @@ fn map_provider_error( AdminErrorKind::Internal => "服务内部错误", _ => "Provider 操作失败", }; - AdminError::new(kind, error.public_message().unwrap_or(message)) + AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure { + operation: resource, + source: error.into(), + }) } async fn publish_committed( @@ -454,3 +458,11 @@ async fn delete_credentials( account_ids, }) } + +// 用例的操作名补充端口资源上下文,底层分类与原始原因仍由来源持有 +#[derive(Debug, thiserror::Error)] +#[error("admin operation failed: {operation}")] +struct OperationFailure { + operation: &'static str, + source: gateway_core::error::ErrorSource, +} diff --git a/backend/crates/gateway-admin/tests/model/provider_credentials.rs b/backend/crates/gateway-admin/tests/model/provider_credentials.rs index b2ab6e5ce..2465ba2ff 100644 --- a/backend/crates/gateway-admin/tests/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/tests/model/provider_credentials.rs @@ -223,8 +223,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() { assert_eq!( PendingAuthorizationMutation::from_storage_v1(Value::Object( expected.to_storage_v1() - )), - Ok(expected.clone()), + )) + .unwrap(), + expected.clone(), ); let with_proxy = expected .with_outbound_proxy(Some( @@ -237,8 +238,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() { assert_eq!( PendingAuthorizationMutation::from_storage_v1(Value::Object( with_proxy.to_storage_v1() - )), - Ok(with_proxy) + )) + .unwrap(), + with_proxy ); } } diff --git a/backend/crates/gateway-api/src/admin/diagnostics.rs b/backend/crates/gateway-api/src/admin/diagnostics.rs new file mode 100644 index 000000000..aa5e9ccdb --- /dev/null +++ b/backend/crates/gateway-api/src/admin/diagnostics.rs @@ -0,0 +1,42 @@ +//! 管理请求在 HTTP 出口记录一次受控详情,响应只保留安全投影 + +use std::sync::Arc; + +use axum::{ + extract::{Request, State}, + middleware::Next, + response::Response, +}; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; +use tower_http::request_id::RequestId; + +pub(crate) async fn record_failure( + State(diagnostics): State>, + request: Request, + next: Next, +) -> Response { + let correlation_id = request + .extensions() + .get::() + .and_then(|id| id.header_value().to_str().ok()) + .map(str::to_owned); + let mut response = next.run(request).await; + if let Some(error) = response + .extensions_mut() + .remove::() + && let Some(details) = error.error_details() + { + let mut failure = OperationalFailure::new( + "admin", + "http_request", + error.kind().as_str(), + error.message(), + ); + failure.correlation_id = correlation_id; + failure.details = Some(details); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!("admin diagnostic could not be recorded"); + } + } + response +} diff --git a/backend/crates/gateway-api/src/admin/mod.rs b/backend/crates/gateway-api/src/admin/mod.rs index d7f11e980..8fb803cd7 100644 --- a/backend/crates/gateway-api/src/admin/mod.rs +++ b/backend/crates/gateway-api/src/admin/mod.rs @@ -15,6 +15,7 @@ pub mod accounts; pub mod auth; pub mod backups; pub mod client_keys; +pub(crate) mod diagnostics; mod extract; pub mod observability; mod plugins; diff --git a/backend/crates/gateway-api/src/admin/wire.rs b/backend/crates/gateway-api/src/admin/wire.rs index e8f455225..0092b88b5 100644 --- a/backend/crates/gateway-api/src/admin/wire.rs +++ b/backend/crates/gateway-api/src/admin/wire.rs @@ -170,6 +170,7 @@ pub struct AdminError { status: StatusCode, body: AdminErrorBody, retry_after_seconds: Option, + diagnostic: Option, } #[derive(Debug, Clone, Copy)] @@ -266,6 +267,7 @@ impl AdminError { status, body: AdminErrorBody::new(code, message), retry_after_seconds: None, + diagnostic: None, } } @@ -380,12 +382,16 @@ pub(crate) fn map_admin_service_error(error: gateway_admin::model::AdminError) - { response.body.message = error.message().to_owned(); } + response.diagnostic = Some(error); response } impl IntoResponse for AdminError { fn into_response(self) -> Response { let mut response = (self.status, Json(self.body)).into_response(); + if let Some(error) = self.diagnostic { + response.extensions_mut().insert(error); + } if let Some(seconds) = self.retry_after_seconds && let Ok(value) = axum::http::HeaderValue::from_str(&seconds.to_string()) { diff --git a/backend/crates/gateway-api/src/lib.rs b/backend/crates/gateway-api/src/lib.rs index 4ecd8cc2b..0f0c4559f 100644 --- a/backend/crates/gateway-api/src/lib.rs +++ b/backend/crates/gateway-api/src/lib.rs @@ -170,6 +170,7 @@ pub fn initialize( probes: Vec>, worker_health: Arc, lifecycle: Arc, + diagnostics: Arc, ) -> Result { // 配置加载已解析环境变量与相对路径,初始化只校验,避免覆盖最终目录 config.validate().map_err(ApiError::Config)?; @@ -256,6 +257,10 @@ pub fn initialize( headers_ms = latency.as_millis(), "HTTP response headers ready"); }); let router = router + .layer(axum::middleware::from_fn_with_state( + diagnostics, + admin::diagnostics::record_failure, + )) .layer(PropagateRequestIdLayer::new(request_id_header.clone())) .layer(trace_layer) .layer(SetRequestIdLayer::new( diff --git a/backend/crates/gateway-api/src/openai/error.rs b/backend/crates/gateway-api/src/openai/error.rs index de8088f25..d6bf964bd 100644 --- a/backend/crates/gateway-api/src/openai/error.rs +++ b/backend/crates/gateway-api/src/openai/error.rs @@ -165,8 +165,11 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError { "no upstream provider is currently available for this request", ) } - EngineError::Store(_) - | EngineError::ProviderMetadataMismatch + EngineError::Store(source) => { + GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed") + .with_source(source.clone()) + } + EngineError::ProviderMetadataMismatch | EngineError::ContinuationPinMismatch | EngineError::RequiredAccountMismatch | EngineError::AccountOutsideClientScope diff --git a/backend/crates/gateway-api/tests/admin/error_diagnostics.rs b/backend/crates/gateway-api/tests/admin/error_diagnostics.rs new file mode 100644 index 000000000..f2be747e9 --- /dev/null +++ b/backend/crates/gateway-api/tests/admin/error_diagnostics.rs @@ -0,0 +1,61 @@ +//! 验证管理端来源经用例和 HTTP 出口到达受控记录,公开响应和扩展不泄漏 + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use std::sync::Arc; +use tower::ServiceExt; + +#[tokio::test] +async fn store_failure_is_recorded_once_with_the_incoming_request_id() { + let fixture = super::AdminTestFixture::new().await; + fixture.auth.insert_session("valid-session"); + let diagnostics = Arc::new(crate::support::RecordingDiagnostics::default()); + let bundle = crate::openai::api_bundle_with_diagnostics( + fixture.services, + gateway_api::ApiConfig { + asset_directory: std::env::temp_dir(), + request_timeout_seconds: None, + cors_allowed_origins: Vec::new(), + request_id_header: "x-request-id".to_owned(), + }, + diagnostics.clone(), + ); + let response = bundle + .router() + .oneshot( + Request::builder() + .uri("/api/admin/accounts") + .header("cookie", "cpr_session=valid-session") + .header("x-request-id", "admin-native-cause-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!( + response + .extensions() + .get::() + .is_none() + ); + let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("PRIVATE_ADMIN_NATIVE_CAUSE")); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!(failures[0].kind, "unavailable"); + assert_eq!( + failures[0].correlation_id.as_deref(), + Some("admin-native-cause-test") + ); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_ADMIN_NATIVE_CAUSE") + ); +} diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index 823d89f12..e454f44d6 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -86,6 +86,7 @@ mod account_groups; mod accounts; mod auth; mod client_keys; +mod error_diagnostics; mod errors; mod observability; mod proxies; @@ -1673,6 +1674,7 @@ fn unavailable(resource: &'static str) -> AdminStoreError { resource, "unused test port", ) + .with_source(std::io::Error::other("PRIVATE_ADMIN_NATIVE_CAUSE")) } fn not_found(resource: &'static str) -> AdminStoreError { diff --git a/backend/crates/gateway-api/tests/architecture.rs b/backend/crates/gateway-api/tests/architecture.rs index 5a08380cf..24ca76d0a 100644 --- a/backend/crates/gateway-api/tests/architecture.rs +++ b/backend/crates/gateway-api/tests/architecture.rs @@ -45,6 +45,7 @@ fn source_tree_should_match_frozen_machine_manifest() { "src/admin/auth.rs", "src/admin/backups.rs", "src/admin/client_keys.rs", + "src/admin/diagnostics.rs", "src/admin/extract.rs", "src/admin/mod.rs", "src/admin/observability/mod.rs", @@ -124,6 +125,7 @@ fn test_tree_should_match_frozen_rust_mirror() { "tests/admin/auth.rs", "tests/admin/client_keys.rs", "tests/admin/errors.rs", + "tests/admin/error_diagnostics.rs", "tests/admin/mod.rs", "tests/admin/observability/mod.rs", "tests/admin/observability/query.rs", diff --git a/backend/crates/gateway-api/tests/openai/auth.rs b/backend/crates/gateway-api/tests/openai/auth.rs index 79ef73c15..72dfbaf20 100644 --- a/backend/crates/gateway-api/tests/openai/auth.rs +++ b/backend/crates/gateway-api/tests/openai/auth.rs @@ -479,6 +479,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission( Arc::new(super::UnusedAdmissions), Arc::new(super::UnusedContinuation), Arc::new(super::IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let admin = crate::admin::AdminTestFixture::new().await; let bundle = gateway_api::initialize( @@ -493,6 +494,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission( vec![], Arc::new(super::EmptyWorkerHealth), Arc::new(super::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap(); let baseline = bundle.dispatcher(); diff --git a/backend/crates/gateway-api/tests/openai/error.rs b/backend/crates/gateway-api/tests/openai/error.rs index 467ad58fd..a7cf044f8 100644 --- a/backend/crates/gateway-api/tests/openai/error.rs +++ b/backend/crates/gateway-api/tests/openai/error.rs @@ -304,12 +304,14 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::AccountCapacityUnavailable, - "all eligible upstream accounts are temporarily busy" - ) + projected.kind(), + GatewayErrorKind::AccountCapacityUnavailable + ); + assert_eq!( + projected.safe_message(), + "all eligible upstream accounts are temporarily busy" ); } @@ -352,12 +354,14 @@ fn provider_infrastructure_failure_should_have_a_distinct_client_contract() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::ProviderInfrastructureUnavailable, - "provider account infrastructure is temporarily unavailable" - ) + projected.kind(), + GatewayErrorKind::ProviderInfrastructureUnavailable + ); + assert_eq!( + projected.safe_message(), + "provider account infrastructure is temporarily unavailable" ); } @@ -368,12 +372,11 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::NoAvailableProvider); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::NoAvailableProvider, - "no upstream provider is currently available for this request" - ) + projected.safe_message(), + "no upstream provider is currently available for this request" ); } @@ -499,13 +502,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() { ] { let error = EngineError::Provider(ProviderError::new(kind, UpstreamSendState::Ambiguous)); - assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::UpstreamUnavailable, - "upstream service is unavailable" - ) - ); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::UpstreamUnavailable); + assert_eq!(projected.safe_message(), "upstream service is unavailable"); } } @@ -513,10 +512,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() { fn engine_store_error_should_collapse_to_safe_internal_error() { let error = EngineError::Store(StoreError::new(StoreErrorKind::Unavailable)); - assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed") - ); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::Internal); + assert_eq!(projected.safe_message(), "gateway execution failed"); } #[test] fn openai_error_response_should_preserve_only_safe_contract_fields() { @@ -669,6 +667,7 @@ mod model_routing { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); let response = crate::openai::api_router(Arc::new(execution)) .await diff --git a/backend/crates/gateway-api/tests/openai/live/websocket.rs b/backend/crates/gateway-api/tests/openai/live/websocket.rs index 6e0f1ed65..74e9fe0b8 100644 --- a/backend/crates/gateway-api/tests/openai/live/websocket.rs +++ b/backend/crates/gateway-api/tests/openai/live/websocket.rs @@ -138,6 +138,7 @@ async fn blocked_relay( Vec::new(), Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(Lifecycle(trace.clone())), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .router(); diff --git a/backend/crates/gateway-api/tests/openai/mod.rs b/backend/crates/gateway-api/tests/openai/mod.rs index 4628e8bee..42e34aca5 100644 --- a/backend/crates/gateway-api/tests/openai/mod.rs +++ b/backend/crates/gateway-api/tests/openai/mod.rs @@ -92,6 +92,7 @@ pub(super) fn api_router_with_admin_and_client( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); api_router_with_admin_and_execution(admin, execution) } @@ -106,6 +107,18 @@ pub(super) fn api_router_with_config( pub(super) fn api_bundle( admin: gateway_admin::AdminServices, config: gateway_api::ApiConfig, +) -> gateway_api::ApiBundle { + api_bundle_with_diagnostics( + admin, + config, + Arc::new(crate::support::RecordingDiagnostics::default()), + ) +} + +pub(crate) fn api_bundle_with_diagnostics( + admin: gateway_admin::AdminServices, + config: gateway_api::ApiConfig, + diagnostics: Arc, ) -> gateway_api::ApiBundle { let execution = Arc::new(DefaultExecutionService::new( RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")), @@ -114,6 +127,7 @@ pub(super) fn api_bundle( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); gateway_api::initialize( config, @@ -122,6 +136,7 @@ pub(super) fn api_bundle( Vec::new(), Arc::new(EmptyWorkerHealth), Arc::new(TestLifecycle::default()), + diagnostics, ) .unwrap() } @@ -138,6 +153,7 @@ fn api_router_with_config_and_execution( Vec::new(), Arc::new(EmptyWorkerHealth), Arc::new(TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .expect("API bundle") .router() @@ -180,6 +196,7 @@ async fn api_router_with_origins_and_worker_health( Vec::new(), worker_health, Arc::new(TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .expect("API bundle") .router() @@ -214,6 +231,7 @@ pub(super) fn authenticated_client_for_provider_with_limit( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); source .authenticate(plaintext) @@ -244,6 +262,7 @@ pub(super) fn authenticated_client_with_min_versions( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); source .authenticate(plaintext) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs index 92e97b45f..e54596e90 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs @@ -165,6 +165,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake Arc::new(crate::openai::UnusedAdmissions), Arc::new(crate::openai::UnusedContinuation), Arc::new(crate::openai::IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let admin = crate::admin::AdminTestFixture::new().await; let plan = Arc::new(SettingsPlan::default()); @@ -187,6 +188,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake vec![], Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(crate::openai::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .with_middleware(move |_| Some(frozen.clone())) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs index 385c51972..a2eb4b412 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs @@ -761,6 +761,7 @@ async fn start_active_response_with_middleware( Vec::new(), Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(crate::openai::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .with_middleware(move |_| Some(plan.clone())) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs index 9b83dcd4b..03ac069d5 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs @@ -162,6 +162,7 @@ async fn interrupt_is_live_scoped_and_keeps_queued_creates_serial() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let app = api_router(execution).await; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs index fd6e5b871..49adf16b0 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs @@ -1108,6 +1108,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele ports.clone(), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .with_budget(ports.clone()), ); @@ -1125,6 +1126,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele Vec::new(), Arc::new(EmptyWorkerHealth), lifecycle.clone(), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .router(); diff --git a/backend/crates/gateway-api/tests/support.rs b/backend/crates/gateway-api/tests/support.rs index 67c757589..65aeccdd0 100644 --- a/backend/crates/gateway-api/tests/support.rs +++ b/backend/crates/gateway-api/tests/support.rs @@ -159,3 +159,18 @@ impl ClientKeyVerifier for AcceptingVerifier { } } } + +#[derive(Default)] +pub(crate) struct RecordingDiagnostics( + pub std::sync::Mutex>, +); +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-core/src/diagnostics/failure.rs b/backend/crates/gateway-core/src/diagnostics/failure.rs new file mode 100644 index 000000000..4c55a4ce0 --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/failure.rs @@ -0,0 +1,54 @@ +//! 控制面与后台操作的受控故障记录,不伪造模型请求或上游发送事实 + +use async_trait::async_trait; + +use crate::{ + account::ProviderAccountId, + error::{ErrorDetails, OpaqueUpstreamValue, StoreError}, + identity::ProviderKind, +}; + +#[derive(Debug)] +pub struct OperationalFailure { + pub occurred_at: std::time::SystemTime, + pub component: &'static str, + pub operation: &'static str, + pub kind: &'static str, + pub message: String, + pub correlation_id: Option, + pub provider_kind: Option, + pub account_id: Option, + pub upstream_status: Option, + pub upstream_code: Option, + pub details: Option, +} + +impl OperationalFailure { + #[must_use] + pub fn new( + component: &'static str, + operation: &'static str, + kind: &'static str, + message: impl Into, + ) -> Self { + Self { + occurred_at: std::time::SystemTime::now(), + component, + operation, + kind, + message: message.into(), + correlation_id: None, + provider_kind: None, + account_id: None, + upstream_status: None, + upstream_code: None, + details: None, + } + } +} + +/// 记录遵循既有观测队列的 best-effort 合同,失败不应替换业务结果 +#[async_trait] +pub trait OperationalDiagnostics: Send + Sync { + async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError>; +} diff --git a/backend/crates/gateway-core/src/diagnostics/mod.rs b/backend/crates/gateway-core/src/diagnostics/mod.rs index 338f6866a..24bd0287c 100644 --- a/backend/crates/gateway-core/src/diagnostics/mod.rs +++ b/backend/crates/gateway-core/src/diagnostics/mod.rs @@ -1,10 +1,12 @@ -//! 有界、请求局部的诊断证据;不参与路由、重试或客户端协议决策 +//! 请求诊断与操作故障的观测合同,不参与路由、重试或客户端协议决策 mod capture; +mod failure; mod selection; mod stream; mod trace; pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json}; +pub use failure::{OperationalDiagnostics, OperationalFailure}; pub use stream::{StreamCapture, StreamFormat}; pub use trace::TraceContext; diff --git a/backend/crates/gateway-core/src/engine/admission.rs b/backend/crates/gateway-core/src/engine/admission.rs index 72eaa46a4..d0eaa6ec3 100644 --- a/backend/crates/gateway-core/src/engine/admission.rs +++ b/backend/crates/gateway-core/src/engine/admission.rs @@ -57,9 +57,9 @@ pub struct ClientAdmissionRestoreResult { pub restored_running_requests: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("client admission store is unavailable")] -pub struct ClientAdmissionError; +pub struct ClientAdmissionError(#[source] pub Option); pub trait ClientAdmissionPort: Send + Sync { /// 仅续期已经准入的并发槽位,不重复消费 RPM @@ -118,10 +118,10 @@ pub async fn restore_client_admission_startup( let expired = execution .recover_expired(now) .await - .map_err(|_| ClientAdmissionError)?; + .map_err(|source| ClientAdmissionError(Some(source.into())))?; let since = now .checked_sub(Duration::from_secs(61)) - .ok_or(ClientAdmissionError)?; + .ok_or(ClientAdmissionError(None))?; let recoveries = recovery.load_recovery(since).await?; let restored_clients = u64::try_from(recoveries.len()).unwrap_or(u64::MAX); let mut restored_recent_requests = 0_u64; diff --git a/backend/crates/gateway-core/src/engine/budget.rs b/backend/crates/gateway-core/src/engine/budget.rs index 101de66e2..e42587073 100644 --- a/backend/crates/gateway-core/src/engine/budget.rs +++ b/backend/crates/gateway-core/src/engine/budget.rs @@ -39,9 +39,9 @@ pub struct ClientBudgetCharge { pub completed_at: SystemTime, } -#[derive(Debug, Clone, Copy, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("client budget store is unavailable")] -pub struct ClientBudgetError; +pub struct ClientBudgetError(#[source] pub Option); pub trait ClientBudgetPort: Send + Sync { /// 原子检查当前限额与已用金额,不创建预扣费或待结算记录 diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index 7610d53d5..29975b7f9 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -739,6 +739,7 @@ pub struct DefaultExecutionService { snapshots: RuntimeSnapshotHandle, /// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events observations: Arc, + diagnostics: Arc, providers: ProviderRegistry, admissions: Arc, admission_waiting: ConcurrencyWaitQueue, @@ -763,10 +764,12 @@ impl DefaultExecutionService { admissions: Arc, continuation: Arc, client_api_key_usage: Arc, + diagnostics: Arc, ) -> Self { Self { snapshots, observations: execution, + diagnostics, providers, admissions, admission_waiting: ConcurrencyWaitQueue::default(), @@ -1325,7 +1328,12 @@ impl DefaultExecutionService { && let Some(guard) = start_guard.take() { guard - .release_failed(self.budget.as_deref(), request_id, budget_key_id) + .release_failed( + self.budget.as_deref(), + request_id, + budget_key_id, + self.diagnostics.as_ref(), + ) .await; } result @@ -1498,6 +1506,7 @@ impl DefaultExecutionService { amount_usd: crate::metering::Decimal::ZERO, completed_at: SystemTime::now(), }, + self.diagnostics.as_ref(), ) .await; } @@ -1518,6 +1527,7 @@ impl DefaultExecutionService { active_request, Arc::clone(&self.continuation), self.budget.clone(), + self.diagnostics.clone(), )), }) } @@ -1872,6 +1882,7 @@ impl DefaultExecutionService { CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), budget); let mut admission = AdmissionLease { port: Arc::clone(&self.admissions), + diagnostics: self.diagnostics.clone(), client_api_key_id: key.clone(), model_request_id: request_id.clone(), armed: false, @@ -1902,7 +1913,7 @@ impl DefaultExecutionService { pin_mut!(acquire, timeout, cancelled); let decision = select_biased! { () = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "request admission was cancelled")), - result = acquire => result.map_err(|_| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable"))?, + result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable").with_source(source))?, _ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")), }; match decision { @@ -2477,14 +2488,26 @@ struct AdmissionLease { renewal: Option>, armed: bool, port: Arc, + diagnostics: Arc, client_api_key_id: ClientApiKeyId, model_request_id: ModelRequestId, } -async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) { +async fn settle_budget( + port: &dyn ClientBudgetPort, + charge: ClientBudgetCharge, + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, +) { let request_id = charge.request_id.clone(); if let Err(error) = port.settle(charge).await { - tracing::error!(request_id = request_id.as_str(), operation = "settle_client_budget", %error, "Client budget settlement failed; storage will retry on the next request"); + record_resource_failure( + diagnostics, + &request_id, + "settle_client_budget", + "Client budget settlement failed; storage will retry on the next request", + error, + ) + .await; } } @@ -2496,7 +2519,14 @@ impl AdmissionLease { .release(&self.client_api_key_id, &self.model_request_id) .await { - tracing::warn!(request_id = self.model_request_id.as_str(), key_id = self.client_api_key_id.as_str(), operation = "release_client_admission", %error, "Client admission 释放失败,依赖租约 TTL 收敛"); + record_resource_failure( + self.diagnostics.as_ref(), + &self.model_request_id, + "release_client_admission", + "Client admission release failed; lease TTL remains active", + error, + ) + .await; } self.armed = false; } @@ -2537,6 +2567,7 @@ impl ExecutionStartGuard { budget: Option<&dyn ClientBudgetPort>, request_id: ModelRequestId, key_id: ClientApiKeyId, + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, ) { if let Some(active_request) = self.active_request { active_request.release(); @@ -2550,6 +2581,7 @@ impl ExecutionStartGuard { amount_usd: crate::metering::Decimal::ZERO, completed_at: SystemTime::now(), }, + diagnostics, ) .await; } @@ -2565,6 +2597,7 @@ struct DefaultExecutionSession { continuation: Arc, continuation_recorded: bool, budget: Option>, + diagnostics: Arc, } impl DefaultExecutionSession { @@ -2574,6 +2607,7 @@ impl DefaultExecutionSession { active_request: Option, continuation: Arc, budget: Option>, + diagnostics: Arc, ) -> Self { Self { core, @@ -2583,6 +2617,7 @@ impl DefaultExecutionSession { continuation, continuation_recorded: false, budget, + diagnostics, } } @@ -2595,12 +2630,13 @@ impl DefaultExecutionSession { } let budget = self.budget.take(); let charge = self.core.budget_charge(); + let diagnostics = self.diagnostics.clone(); // 在首次 await 前把完整清理责任留在会话内 // 事件等待被取消后,后续 poll // 或 detach 继续同一个 future,既不丢失费用,也不重启已完成的结算 self.cleanup = Some(Box::pin(async move { if let Some(budget) = budget { - settle_budget(budget.as_ref(), charge).await; + settle_budget(budget.as_ref(), charge, diagnostics.as_ref()).await; } admission.release().await; })); @@ -2923,6 +2959,34 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError { "no provider is available", ) } + EngineError::Store(source) => { + GatewayError::new(GatewayErrorKind::Internal, "request execution failed") + .with_source(source.clone()) + } _ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"), } } + +async fn record_resource_failure( + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, + request_id: &ModelRequestId, + operation: &'static str, + message: &'static str, + source: impl Into, +) { + let mut failure = crate::diagnostics::OperationalFailure::new( + "core", + operation, + "resource_unavailable", + message, + ); + failure.correlation_id = Some(request_id.as_str().to_owned()); + failure.details = crate::error::ErrorDetails::capture(Some(&source.into()), None, false); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!( + request_id = request_id.as_str(), + operation, + "resource diagnostic could not be recorded" + ); + } +} diff --git a/backend/crates/gateway-core/src/error/details.rs b/backend/crates/gateway-core/src/error/details.rs new file mode 100644 index 000000000..6b35ee6b9 --- /dev/null +++ b/backend/crates/gateway-core/src/error/details.rs @@ -0,0 +1,43 @@ +//! 原因链与上游正文的受控诊断投影,不参与普通错误格式化 + +use super::{ErrorSource, RawUpstreamError}; + +#[derive(Clone)] +pub struct ErrorDetails(String); + +impl ErrorDetails { + #[must_use] + pub fn capture( + source: Option<&ErrorSource>, + upstream: Option<&RawUpstreamError>, + redacted: bool, + ) -> Option { + if source.is_none() && upstream.is_none() { + return None; + } + Some(Self( + serde_json::json!({ + "causes": source.map(ErrorSource::snapshot), + "upstream": upstream.map(RawUpstreamError::as_str), + "redacted": redacted, + }) + .to_string(), + )) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + #[must_use] + pub fn into_string(self) -> String { + self.0 + } +} + +impl std::fmt::Debug for ErrorDetails { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("ErrorDetails()") + } +} diff --git a/backend/crates/gateway-core/src/error/mod.rs b/backend/crates/gateway-core/src/error/mod.rs index d860eb504..cc97f3ffa 100644 --- a/backend/crates/gateway-core/src/error/mod.rs +++ b/backend/crates/gateway-core/src/error/mod.rs @@ -1,6 +1,8 @@ //! 网关核心使用的稳定错误分类 +mod details; mod source; +pub use details::ErrorDetails; pub use source::ErrorSource; @@ -865,25 +867,20 @@ impl ProviderError { /// 包装底层失败;来源只供显式诊断访问,不改变分类和发送事实 #[must_use] - pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { - self.source = Some(ErrorSource::new(source)); + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); self } /// 运维详情的唯一投影入口,来源链与上游正文分别保留,不进入公共响应或诊断包 #[must_use] pub fn error_details(&self) -> Option { - if self.source.is_none() && self.raw_upstream_error.is_none() { - return None; - } - Some( - serde_json::json!({ - "causes": self.source.as_ref().map(ErrorSource::snapshot), - "upstream": self.raw_upstream_error().map(RawUpstreamError::as_str), - "redacted": self.sensitive_context_was_redacted(), - }) - .to_string(), + ErrorDetails::capture( + self.source.as_ref(), + self.raw_upstream_error(), + self.sensitive_context_was_redacted(), ) + .map(ErrorDetails::into_string) } /// 返回将由运维错误详情原样展示的上游错误返回 @@ -1094,14 +1091,15 @@ impl GatewayErrorKind { } /// 协议无关、可安全暴露的网关错误 -#[derive(Clone, PartialEq, Eq, Error)] +#[derive(Clone, Error)] #[error("{message}")] pub struct GatewayError { kind: GatewayErrorKind, message: &'static str, diagnostic: Option>, - client_visible_upstream_error: Option, + client_visible_upstream_error: Option>, client_details: Option>, + source: Option, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -1120,9 +1118,21 @@ impl GatewayError { diagnostic: None, client_visible_upstream_error: None, client_details: None, + source: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + ErrorDetails::capture(self.source.as_ref(), None, false) + } + #[must_use] pub fn with_client_code(mut self, code: &'static str) -> Self { self.client_details @@ -1232,7 +1242,7 @@ impl GatewayError { /// 附加只供请求方协议展示的结构化上游错误 #[must_use] pub fn with_client_visible_upstream_error(mut self, error: ClientVisibleUpstreamError) -> Self { - self.client_visible_upstream_error = Some(error); + self.client_visible_upstream_error = Some(Box::new(error)); self } @@ -1259,7 +1269,7 @@ impl GatewayError { #[must_use] pub fn client_message(&self) -> &str { self.client_visible_upstream_error - .as_ref() + .as_deref() .map_or(self.message, ClientVisibleUpstreamError::message) } @@ -1267,7 +1277,7 @@ impl GatewayError { #[must_use] pub fn client_error_type(&self) -> Option<&str> { self.client_visible_upstream_error - .as_ref() + .as_deref() .and_then(ClientVisibleUpstreamError::error_type) } @@ -1275,7 +1285,7 @@ impl GatewayError { #[must_use] pub fn client_error_code(&self) -> Option<&str> { self.client_visible_upstream_error - .as_ref() + .as_deref() .and_then(ClientVisibleUpstreamError::code) .or_else(|| { self.client_details @@ -1337,13 +1347,10 @@ impl StoreError { /// 包装已有存储错误并保留来源 #[must_use] - pub fn caused_by( - kind: StoreErrorKind, - source: impl std::error::Error + Send + Sync + 'static, - ) -> Self { + pub fn caused_by(kind: StoreErrorKind, source: impl Into) -> Self { Self { kind, - source: Some(ErrorSource::new(source)), + source: Some(source.into()), } } diff --git a/backend/crates/gateway-core/src/error/source.rs b/backend/crates/gateway-core/src/error/source.rs index 54a848c67..974581b2c 100644 --- a/backend/crates/gateway-core/src/error/source.rs +++ b/backend/crates/gateway-core/src/error/source.rs @@ -12,32 +12,125 @@ impl ErrorSource { Self(Arc::new(error)) } + /// 清理失败是附属事实,标准来源链仍指向最初失败 + #[must_use] + pub fn with_cleanup(self, cleanup: impl Into) -> Self { + Self::new(CleanupFailure { + primary: Some(self), + cleanup: cleanup.into(), + }) + } + + /// 附属清理错误与可选的原始底层原因分别保留 + #[must_use] + pub fn cleanup(primary: Option, cleanup: impl Into) -> Self { + Self::new(CleanupFailure { + primary, + cleanup: cleanup.into(), + }) + } + /// 仅供受控错误详情落盘,限制链深度和总文本大小并显式标记缺口 pub(super) fn snapshot(&self) -> serde_json::Value { - let mut messages = Vec::new(); - let mut source: Option<&(dyn Error + 'static)> = Some(self.0.as_ref()); - let mut remaining = 64 * 1024; - let mut truncated = false; - for _ in 0..32 { - let Some(error) = source else { break }; - let mut message = BoundedMessage { - text: String::new(), - remaining, - }; - if fmt::write(&mut message, format_args!("{error}")).is_err() { - truncated = true; - } - remaining = message.remaining; - messages.push(message.text); - source = error.source(); - if truncated || remaining == 0 { - break; + snapshot(self.0.as_ref(), &mut 32, &mut (64 * 1024)) + } + + /// 观测队列估算来源占用,包含附属清理失败;过深或循环的来源按超限处理 + #[must_use] + pub fn estimated_chain_bytes(source: Option<&(dyn Error + 'static)>) -> usize { + struct ByteCount(usize); + impl fmt::Write for ByteCount { + fn write_str(&mut self, value: &str) -> fmt::Result { + self.0 = self.0.saturating_add(value.len()); + Ok(()) } } - serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()}) + let mut pending = Vec::from_iter(source); + let mut bytes = ByteCount(0); + for _ in 0..32 { + let Some(error) = pending.pop() else { + return bytes.0; + }; + bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error)); + if fmt::write(&mut bytes, format_args!("{error}")).is_err() { + return usize::MAX; + } + if let Some(cleanup) = error.downcast_ref::() { + pending.push(cleanup.cleanup.0.as_ref()); + } + if let Some(source) = error.source() { + pending.push(source); + } + } + if pending.is_empty() { + bytes.0 + } else { + usize::MAX + } } } +impl From for ErrorSource { + fn from(error: E) -> Self { + Self::new(error) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("operation failed and cleanup also failed")] +struct CleanupFailure { + #[source] + primary: Option, + cleanup: ErrorSource, +} + +fn snapshot( + error: &(dyn Error + 'static), + nodes: &mut usize, + bytes: &mut usize, +) -> serde_json::Value { + let mut messages = Vec::new(); + let mut cleanup_sources = Vec::new(); + let mut source = Some(error); + let mut truncated = false; + while *nodes > 0 { + let Some(error) = source else { break }; + *nodes -= 1; + if let Some(failure) = error.downcast_ref::() { + cleanup_sources.push(failure.cleanup.0.as_ref()); + source = failure + .primary + .as_ref() + .map(|primary| primary.0.as_ref() as &(dyn Error + 'static)); + continue; + } + let mut message = BoundedMessage { + text: String::new(), + remaining: *bytes, + }; + if fmt::write(&mut message, format_args!("{error}")).is_err() { + truncated = true; + } + *bytes = message.remaining; + messages.push(message.text); + source = error.source(); + if truncated || *bytes == 0 { + break; + } + } + let mut result = + serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()}); + if !cleanup_sources.is_empty() { + result["cleanup"] = serde_json::Value::Array( + cleanup_sources + .into_iter() + .map(|error| snapshot(error, nodes, bytes)) + .collect(), + ); + } + result +} + struct BoundedMessage { text: String, remaining: usize, diff --git a/backend/crates/gateway-core/src/lib.rs b/backend/crates/gateway-core/src/lib.rs index b125510f7..7fd908809 100644 --- a/backend/crates/gateway-core/src/lib.rs +++ b/backend/crates/gateway-core/src/lib.rs @@ -53,6 +53,7 @@ use task::{WorkerContribution, WorkerDefinitionError}; #[derive(Clone)] pub struct CoreStorePorts { execution: Arc, + diagnostics: Arc, admissions: Arc, admission_recovery: Arc, continuation: Arc, @@ -77,9 +78,11 @@ impl CoreStorePorts { Arc, ), client_api_key_usage: Arc, + diagnostics: Arc, ) -> Self { Self { execution, + diagnostics, admissions, admission_recovery, continuation, @@ -201,6 +204,7 @@ fn build_execution_service( Arc::clone(&ports.admissions), Arc::clone(&ports.continuation), Arc::clone(&ports.client_api_key_usage), + Arc::clone(&ports.diagnostics), ); if let Some(budget) = &ports.budget { service = service.with_budget(Arc::clone(budget)); diff --git a/backend/crates/gateway-core/src/provider_ports.rs b/backend/crates/gateway-core/src/provider_ports.rs index 6034824b5..5d8d19421 100644 --- a/backend/crates/gateway-core/src/provider_ports.rs +++ b/backend/crates/gateway-core/src/provider_ports.rs @@ -51,12 +51,12 @@ impl ProviderStoreError { pub fn caused_by( kind: ProviderStoreErrorKind, operation: &'static str, - source: impl std::error::Error + Send + Sync + 'static, + source: impl Into, ) -> Self { Self { kind, operation, - source: Some(crate::error::ErrorSource::new(source)), + source: Some(source.into()), } } @@ -1494,6 +1494,7 @@ pub struct ProviderStorePorts { cooldowns: Arc, runtime_policy: Arc, oauth_pending: Arc, + diagnostics: Arc, } impl ProviderStorePorts { @@ -1511,6 +1512,7 @@ impl ProviderStorePorts { cooldowns: Arc, runtime_policy: Arc, oauth_pending: Arc, + diagnostics: Arc, ) -> Self { Self { accounts, @@ -1524,9 +1526,15 @@ impl ProviderStorePorts { cooldowns, runtime_policy, oauth_pending, + diagnostics, } } + #[must_use] + pub fn diagnostics(&self) -> Arc { + self.diagnostics.clone() + } + #[must_use] pub fn accounts(&self) -> Arc { Arc::clone(&self.accounts) diff --git a/backend/crates/gateway-core/src/runtime/mod.rs b/backend/crates/gateway-core/src/runtime/mod.rs index b60cece46..5391cb6f7 100644 --- a/backend/crates/gateway-core/src/runtime/mod.rs +++ b/backend/crates/gateway-core/src/runtime/mod.rs @@ -454,7 +454,10 @@ impl ScheduledTask for RuntimeSnapshotReconciliationTask { .refresh_with_mode(RefreshMode::Reconcile) .await .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("runtime snapshot reconciliation failed")) + .map_err(|source| { + WorkerTaskError::safe("runtime snapshot reconciliation failed") + .with_source(source) + }) }) } } diff --git a/backend/crates/gateway-core/src/task.rs b/backend/crates/gateway-core/src/task.rs index f45d90ffa..52fe06eac 100644 --- a/backend/crates/gateway-core/src/task.rs +++ b/backend/crates/gateway-core/src/task.rs @@ -485,10 +485,11 @@ pub enum WorkerDefinitionError { } /// 不暴露基础设施原文或 lease resource 的错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct WorkerLeaseError { message: String, + source: Option, } impl WorkerLeaseError { @@ -496,6 +497,7 @@ impl WorkerLeaseError { pub fn safe(message: impl Into) -> Self { Self { message: message.into(), + source: None, } } @@ -503,13 +505,24 @@ impl WorkerLeaseError { pub fn as_safe_str(&self) -> &str { &self.message } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + crate::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } } /// 不暴露 Provider 原文或存储细节的后台任务错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct WorkerTaskError { message: String, + source: Option, } impl WorkerTaskError { @@ -517,6 +530,7 @@ impl WorkerTaskError { pub fn safe(message: impl Into) -> Self { Self { message: message.into(), + source: None, } } @@ -524,6 +538,25 @@ impl WorkerTaskError { pub fn as_safe_str(&self) -> &str { &self.message } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + crate::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } + + #[must_use] + pub fn with_cleanup(mut self, cleanup: impl Into) -> Self { + self.source = Some(crate::error::ErrorSource::cleanup( + self.source.take(), + cleanup, + )); + self + } } fn valid_owner(owner: &str) -> bool { diff --git a/backend/crates/gateway-core/tests/engine/admission.rs b/backend/crates/gateway-core/tests/engine/admission.rs index ba8808f49..7a3bbe59c 100644 --- a/backend/crates/gateway-core/tests/engine/admission.rs +++ b/backend/crates/gateway-core/tests/engine/admission.rs @@ -72,7 +72,7 @@ fn client_admission_startup_recovery_should_fail_closed_at_each_boundary() { assert!( restore_client_admission_startup( &RecoveryStore::success(0), - &ScriptedRecoveries::new(Err(ClientAdmissionError)), + &ScriptedRecoveries::new(Err(ClientAdmissionError(None))), &RecordingAdmissions::default(), now, ) @@ -212,7 +212,7 @@ impl ClientAdmissionPort for RecordingAdmissions { ) -> BoxFuture<'_, Result> { Box::pin(async move { if self.fail_restore { - return Err(ClientAdmissionError); + return Err(ClientAdmissionError(None)); } self.restored.lock().expect("restored").push(recovery); Ok(ClientAdmissionRestoreResult { diff --git a/backend/crates/gateway-core/tests/engine/execution.rs b/backend/crates/gateway-core/tests/engine/execution.rs index f13f5eb12..3ae6cce71 100644 --- a/backend/crates/gateway-core/tests/engine/execution.rs +++ b/backend/crates/gateway-core/tests/engine/execution.rs @@ -99,7 +99,9 @@ impl ClientBudgetPort for Budget { assert!(self.active.load(Ordering::SeqCst)); self.charges.lock().unwrap().push(charge); if self.fail_settlement { - Err(ClientBudgetError) + Err(ClientBudgetError(Some( + std::io::Error::other("PRIVATE_SETTLEMENT_CAUSE").into(), + ))) } else { Ok(()) } @@ -115,6 +117,7 @@ fn service(admissions: Arc, budget: Arc) -> DefaultExecution admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) } @@ -324,6 +327,7 @@ fn reused_client_uses_updated_limits_for_each_execution() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let limited = RateLimits { @@ -360,6 +364,7 @@ fn reused_client_cannot_start_after_key_disable_or_snapshot_suspension() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let next = request(&service, ClientTransport::WebSocket); if suspend { @@ -463,6 +468,7 @@ fn reused_client_is_reauthenticated_by_the_current_frontend_plan_without_identit admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_frontend_authentication(authentication.clone()); let client = service @@ -554,6 +560,7 @@ fn early_failure_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) } @@ -911,6 +918,7 @@ fn retry_policy_can_stop_but_cannot_bypass_replay_safety() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policies); let mut started = service @@ -1169,6 +1177,7 @@ fn completed_parent_rejects_new_nested_execution_and_cancels_an_active_child() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); @@ -1360,6 +1369,7 @@ fn native_response_processing_uses_the_real_translation_boundary() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_observers(observer_index); @@ -1510,6 +1520,7 @@ fn response_translation_zero_output_preserves_canonical_usage_and_finalization() Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let mut started = service @@ -1663,6 +1674,7 @@ fn native_response_failure_after_downstream_commit_is_not_replayed() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let mut started = service @@ -1735,6 +1747,7 @@ fn model_routing_policy_selects_native_provider() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); let mut started = service @@ -1788,6 +1801,7 @@ fn model_routing_reject_stops_before_provider_execution() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); assert!(matches!( @@ -1875,6 +1889,7 @@ fn model_routing_cannot_expand_the_frozen_key_model_scope() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); @@ -2022,6 +2037,7 @@ fn scheduler_reject_is_a_terminal_policy_rejection_before_upstream_send() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index) .with_request_observers(observer_index); @@ -2292,6 +2308,7 @@ fn observation_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) .with_request_observers(observer_index); @@ -2329,6 +2346,7 @@ fn fallback_exhaustion_observation_uses_the_last_actual_provider_not_the_candida Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_observers(observer_index); let mut started = service @@ -2393,6 +2411,7 @@ fn routing_external_effect_stops_a_not_sent_provider_retry() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index) .with_request_observers(observer_index); @@ -2579,6 +2598,20 @@ fn charged_service( admissions: Arc, budget: Option>, fail: bool, +) -> DefaultExecutionService { + charged_service_with_diagnostics( + admissions, + budget, + fail, + Arc::new(crate::RecordingDiagnostics::default()), + ) +} + +fn charged_service_with_diagnostics( + admissions: Arc, + budget: Option>, + fail: bool, + diagnostics: Arc, ) -> DefaultExecutionService { let service = DefaultExecutionService::new( RuntimeSnapshotHandle::new(start_snapshot()), @@ -2591,6 +2624,7 @@ fn charged_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + diagnostics, ); match budget { Some(budget) => service.with_budget(budget), @@ -2826,7 +2860,13 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() { fail_settlement, ..Default::default() }); - let service = charged_service(admissions.clone(), Some(budget.clone()), true); + let diagnostics = Arc::new(crate::RecordingDiagnostics::default()); + let service = charged_service_with_diagnostics( + admissions.clone(), + Some(budget.clone()), + true, + diagnostics.clone(), + ); let mut started = service .start(request(&service, ClientTransport::WebSocket)) .await @@ -2840,6 +2880,24 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() { started.session.detach_finalize().await; // Store 端口已接管精确费用后,结算错误不能改写 Provider 错误或触发第二次结算 assert_cleanup_completed(&admissions, &budget, &started.request_id); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), usize::from(fail_settlement)); + if let Some(failure) = failures.first() { + assert_eq!(failure.operation, "settle_client_budget"); + assert_eq!( + failure.correlation_id.as_deref(), + Some(started.request_id.as_str()) + ); + assert!( + failure + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_SETTLEMENT_CAUSE") + ); + assert!(!format!("{failure:?}").contains("PRIVATE_SETTLEMENT_CAUSE")); + } } }); } @@ -2923,6 +2981,7 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -2954,6 +3013,7 @@ fn probe_failures_should_be_observable_without_a_model_request_row() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -2999,6 +3059,7 @@ fn provider_local_probe_failure_should_remain_distinct_from_upstream() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3041,6 +3102,7 @@ fn diagnostic_probe_does_not_apply_data_plane_account_model_policy() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3072,6 +3134,7 @@ fn probe_observation_store_failure_preserves_the_provider_error() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3195,6 +3258,7 @@ fn client_catalog_forwards_scope_maps_whole_objects_and_omits_unroutable_models( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").expect("authenticate"); let result = block_on(service.client_model_catalog(&client, "codex", "0.154.0")); @@ -3350,6 +3414,7 @@ fn successful_authentication_should_record_client_key_usage() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); service @@ -3369,6 +3434,7 @@ fn client_key_verification_should_not_record_client_key_usage() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); let key_id = service @@ -3390,6 +3456,7 @@ fn request_verification_should_apply_entry_authentication_without_recording_key_ Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service @@ -3423,6 +3490,7 @@ fn assert_provider_endpoint_observation(model: Option<&str>) { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_usage_test") @@ -3476,6 +3544,7 @@ fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -3522,6 +3591,7 @@ fn continuation_owned_by_another_client_api_key_should_fail_closed() { Arc::new(UnusedAdmissions), Arc::new(RejectedContinuation::OwnershipMismatch), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -3549,6 +3619,7 @@ fn invalid_continuation_record_should_not_be_forwarded_as_an_external_handle() { Arc::new(UnusedAdmissions), Arc::new(RejectedContinuation::InvalidData), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -4024,6 +4095,7 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let running = service .start(request(&service, ClientTransport::HttpSse)) @@ -4133,6 +4205,7 @@ fn queue_service( admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); (service, admissions) } @@ -4257,6 +4330,7 @@ fn reused_websocket_client_gets_group_fast_policy_from_each_new_request_snapshot admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); for (revision, fast_mode) in [ @@ -4306,6 +4380,7 @@ fn request_settings_recompute_routing_and_admission_without_changing_sibling_or_ admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let mut modified = service.prepare_verified_execution(client.clone()).unwrap(); @@ -4478,6 +4553,7 @@ fn repeated_connection_failures_never_block_later_requests_for_the_provider() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); for _ in 0..5 { let client = service @@ -4528,6 +4604,7 @@ fn entry_settings_freeze_authentication_and_rebase_only_explicit_overrides() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let baseline = service.request_settings().unwrap(); let mut values = serde_json::to_value(baseline.values()).unwrap(); @@ -4670,6 +4747,7 @@ fn unchanged_and_precompiled_request_settings_reuse_the_frozen_snapshot() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let mut prepared = service.prepare_verified_execution(client).unwrap(); @@ -4767,6 +4845,7 @@ fn child_settings_recompute_key_scope_without_inheriting_parent_defaults() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let parent = service .prepare_execution(service.authenticate("sk_parent").unwrap()) diff --git a/backend/crates/gateway-core/tests/error/mod.rs b/backend/crates/gateway-core/tests/error/mod.rs index d95d24486..a264b4729 100644 --- a/backend/crates/gateway-core/tests/error/mod.rs +++ b/backend/crates/gateway-core/tests/error/mod.rs @@ -351,3 +351,34 @@ fn client_visible_upstream_error_should_preserve_opaque_structured_fields() { assert_eq!(detail.error_type(), Some(error_type.as_str())); assert!(!format!("{detail:?}").contains(&message)); } + +#[test] +fn cleanup_failure_keeps_the_primary_cause_and_separate_bounded_details() { + use gateway_core::error::{ErrorDetails, ErrorSource}; + let primary = ErrorSource::new(std::io::Error::other("PRIMARY_NATIVE_FAILURE")); + let combined = primary.with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE")); + assert_eq!( + combined.source().unwrap().to_string(), + "PRIMARY_NATIVE_FAILURE" + ); + let details = ErrorDetails::capture(Some(&combined), None, false).unwrap(); + let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap(); + assert_eq!( + value["causes"]["messages"], + json!(["PRIMARY_NATIVE_FAILURE"]) + ); + assert_eq!( + value["causes"]["cleanup"][0]["messages"], + json!(["ROLLBACK_NATIVE_FAILURE"]) + ); + assert_eq!(value["causes"]["truncated"], false); + assert!(!format!("{combined:?} {details:?}").contains("NATIVE_FAILURE")); + + let oversized = ErrorSource::new(std::io::Error::other("界".repeat(24_000))) + .with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE")); + let details = ErrorDetails::capture(Some(&oversized), None, false).unwrap(); + let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap(); + assert_eq!(value["causes"]["truncated"], true); + assert_eq!(value["causes"]["cleanup"][0]["truncated"], true); + assert!(details.as_str().len() < 66_000); +} diff --git a/backend/crates/gateway-core/tests/main.rs b/backend/crates/gateway-core/tests/main.rs index 395730830..48d0a3cde 100644 --- a/backend/crates/gateway-core/tests/main.rs +++ b/backend/crates/gateway-core/tests/main.rs @@ -21,3 +21,16 @@ mod task; mod upstream; mod time; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-host/src/lib.rs b/backend/crates/gateway-host/src/lib.rs index b409adf35..91efed550 100644 --- a/backend/crates/gateway-host/src/lib.rs +++ b/backend/crates/gateway-host/src/lib.rs @@ -156,8 +156,9 @@ impl HostBundle { &self, plan: Vec, lease: Arc, + diagnostics: Arc, ) -> Result<(), HostError> { - self.workers.start(plan, lease)?; + self.workers.start(plan, lease, diagnostics)?; Ok(()) } diff --git a/backend/crates/gateway-host/src/logging/sink.rs b/backend/crates/gateway-host/src/logging/sink.rs index fe093b946..5f42b984f 100644 --- a/backend/crates/gateway-host/src/logging/sink.rs +++ b/backend/crates/gateway-host/src/logging/sink.rs @@ -1,7 +1,7 @@ -//! 有界文件日志队列:拥堵时背压,正常退出时排空;写入失败显式影响健康状态 +//! 非阻塞文件日志接收与后台批量写入,错误记录保留独立容量,关闭时排空 use std::io::{self, Write}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, mpsc}; use std::thread::{self, JoinHandle}; @@ -11,10 +11,70 @@ use tracing_subscriber::fmt::MakeWriter; use super::writer::RotatingLogWriter; -const QUEUE_CAPACITY: usize = 64; +const QUEUE_CAPACITY: usize = 4_096; +const QUEUE_BYTES: usize = 64 * 1024 * 1024; +const ERROR_CAPACITY: usize = 512; +const ERROR_BYTES: usize = 8 * 1024 * 1024; +const WRITE_BATCH: usize = 64; + +// 通道不另设容量;预留在复制正文之前完成,并随记录释放,包含正在写入的记录 +struct Budget { + items: AtomicUsize, + bytes: AtomicUsize, + maximum_items: usize, + maximum_bytes: usize, +} + +impl Budget { + fn new(maximum_items: usize, maximum_bytes: usize) -> Arc { + Arc::new(Self { + items: AtomicUsize::new(0), + bytes: AtomicUsize::new(0), + maximum_items, + maximum_bytes, + }) + } + + fn reserve(self: &Arc, bytes: usize) -> Option { + self.items + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |items| { + items + .checked_add(1) + .filter(|items| *items <= self.maximum_items) + }) + .ok()?; + if self + .bytes + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| { + used.checked_add(bytes) + .filter(|used| *used <= self.maximum_bytes) + }) + .is_err() + { + self.items.fetch_sub(1, Ordering::AcqRel); + return None; + } + Some(Reservation { + budget: Arc::clone(self), + bytes, + }) + } +} + +struct Reservation { + budget: Arc, + bytes: usize, +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.bytes.fetch_sub(self.bytes, Ordering::AcqRel); + self.budget.items.fetch_sub(1, Ordering::AcqRel); + } +} enum Command { - Write(Vec), + Write(Vec, Reservation), Shutdown, } @@ -22,6 +82,8 @@ enum Command { pub(super) struct LogHealth { write_failures: AtomicU64, maintenance_failures: AtomicU64, + rejected_records: AtomicU64, + rejected_errors: AtomicU64, } impl LogHealth { @@ -36,6 +98,14 @@ impl LogHealth { eprintln!("file_logging: archive/retention maintenance failed ({kind:?})"); } } + + fn rejected(&self, important: bool) { + self.rejected_records.fetch_add(1, Ordering::Relaxed); + if important { + self.rejected_errors.fetch_add(1, Ordering::Relaxed); + } + // 接收路径只更新状态,不能回退到 stderr 或递归打印新的拥堵日志 + } } impl HealthProbe for LogHealth { @@ -46,9 +116,11 @@ impl HealthProbe for LogHealth { fn check(&self) -> BoxFuture<'_, HealthState> { Box::pin(async move { let failures = self.write_failures.load(Ordering::Relaxed); - if failures > 0 { + let rejected = self.rejected_records.load(Ordering::Relaxed); + let errors = self.rejected_errors.load(Ordering::Relaxed); + if failures > 0 || rejected > 0 { return HealthState::Unhealthy(format!( - "log completeness compromised: {failures} write/flush failures since startup" + "log completeness compromised: {failures} write/flush failures, {rejected} rejected records ({errors} warnings/errors) since startup" )); } let failures = self.maintenance_failures.load(Ordering::Relaxed); @@ -64,12 +136,15 @@ impl HealthProbe for LogHealth { #[derive(Clone)] pub(super) struct FileLogSink { - sender: mpsc::SyncSender, + sender: mpsc::Sender, + regular: Arc, + errors: Arc, + important: bool, health: Arc, } pub(super) struct FileLogGuard { - sender: mpsc::SyncSender, + sender: mpsc::Sender, worker: Option>, health: Arc, } @@ -80,20 +155,36 @@ impl FileLogSink { name: &'static str, health: Arc, ) -> io::Result<(Self, FileLogGuard)> { - let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY); + let (sender, receiver) = mpsc::channel(); let worker_health = Arc::clone(&health); let worker = thread::Builder::new() .name(name.to_owned()) .spawn(move || { - while let Ok(command) = receiver.recv() { - match command { - Command::Write(bytes) => { - if let Err(error) = writer.write_all(&bytes) { - worker_health.write_failed(error.kind()); + let mut reservations = Vec::with_capacity(WRITE_BATCH); + 'consume: while let Ok(mut command) = receiver.recv() { + for index in 0..WRITE_BATCH { + match command { + Command::Write(bytes, reservation) => { + if let Err(error) = writer.write_all(&bytes) { + worker_health.write_failed(error.kind()); + } + reservations.push(reservation); } + Command::Shutdown => break 'consume, + } + if index + 1 == WRITE_BATCH { + break; + } + match receiver.try_recv() { + Ok(next) => command = next, + Err(_) => break, } - Command::Shutdown => break, } + if let Err(error) = writer.flush() { + worker_health.write_failed(error.kind()); + } + // 缓冲刷入文件后才归还额度,慢输出不能让在途记录逃逸预算 + reservations.clear(); } if let Err(error) = writer.sync() { worker_health.write_failed(error.kind()); @@ -102,6 +193,9 @@ impl FileLogSink { Ok(( Self { sender: sender.clone(), + regular: Budget::new(QUEUE_CAPACITY, QUEUE_BYTES), + errors: Budget::new(ERROR_CAPACITY, ERROR_BYTES), + important: false, health: Arc::clone(&health), }, FileLogGuard { @@ -119,29 +213,44 @@ impl<'a> MakeWriter<'a> for FileLogSink { fn make_writer(&'a self) -> Self::Writer { self.clone() } + + fn make_writer_for(&'a self, metadata: &tracing::Metadata<'_>) -> Self::Writer { + let mut writer = self.clone(); + writer.important = *metadata.level() <= tracing::Level::WARN; + writer + } } impl Write for FileLogSink { fn write(&mut self, buffer: &[u8]) -> io::Result { - self.sender - .send(Command::Write(buffer.to_vec())) - .map_err(|_| { - self.health.write_failed(io::ErrorKind::BrokenPipe); - io::Error::new(io::ErrorKind::BrokenPipe, "file log worker unavailable") - })?; + let bytes = buffer.len().saturating_add(std::mem::size_of::()); + let reservation = self + .regular + .reserve(bytes) + .or_else(|| self.important.then(|| self.errors.reserve(bytes)).flatten()); + if let Some(reservation) = reservation { + if self + .sender + .send(Command::Write(buffer.to_vec(), reservation)) + .is_err() + { + self.health.rejected(self.important); + } + } else { + self.health.rejected(self.important); + } + // tracing 的 writer 失败回退可能同步写 stderr;缺口由独立健康状态报告 Ok(buffer.len()) } fn flush(&mut self) -> io::Result<()> { - // 文件不额外缓冲,由守卫负责队列排空与同步的生命周期 Ok(()) } } impl Drop for FileLogGuard { fn drop(&mut self) { - // 关闭指令排在所有已接收记录之后 - // 等待线程退出,确保正常关闭时不会因超时而丢弃队列中的日志 + // 仅进程关闭时等待已接收日志落盘,请求侧不承担排空责任 let _ = self.sender.send(Command::Shutdown); if self .worker diff --git a/backend/crates/gateway-host/src/logging/writer.rs b/backend/crates/gateway-host/src/logging/writer.rs index c0561fc83..70743a727 100644 --- a/backend/crates/gateway-host/src/logging/writer.rs +++ b/backend/crates/gateway-host/src/logging/writer.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; -use std::io::{self, Write}; +use std::io::{self, BufWriter, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -20,7 +20,7 @@ pub(super) struct RotatingLogWriter { date: NaiveDate, segment: usize, bytes_written: u64, - file: File, + file: BufWriter, health: Arc, } @@ -58,7 +58,7 @@ impl RotatingLogWriter { date, segment, bytes_written, - file, + file: BufWriter::with_capacity(64 * 1024, file), health, }; // 重启后恢复未完成归档,正在写入的文件不参与压缩 @@ -84,7 +84,7 @@ impl RotatingLogWriter { let previous = self .directory .join(log_file_name(self.prefix, self.date, self.segment)); - self.file.sync_all()?; + self.sync()?; let segment = if day_changed { managed_log_files(&self.directory, self.prefix)? .into_iter() @@ -100,10 +100,10 @@ impl RotatingLogWriter { .join(log_file_name(self.prefix, date, segment)); let file = open_log_segment(&path)?; // 新文件打开成功后才能提交轮转状态 - self.file = file; + self.file = BufWriter::with_capacity(64 * 1024, file); self.date = date; self.segment = segment; - self.bytes_written = self.file.metadata()?.len(); + self.bytes_written = self.file.get_ref().metadata()?.len(); if let Err(error) = compress_log_file(&previous) { self.health.maintenance_failed(error.kind()); } @@ -121,7 +121,8 @@ impl RotatingLogWriter { } pub(super) fn sync(&mut self) -> io::Result<()> { - self.file.sync_all() + self.file.flush()?; + self.file.get_ref().sync_all() } } @@ -129,7 +130,7 @@ impl Write for RotatingLogWriter { fn write(&mut self, buffer: &[u8]) -> io::Result { self.rotate_if_required(buffer.len())?; if let Err(error) = self.file.write_all(buffer) { - self.bytes_written = self.file.metadata()?.len(); + self.bytes_written = self.file.get_ref().metadata()?.len(); return Err(error); } self.bytes_written = self.bytes_written.saturating_add(buffer.len() as u64); diff --git a/backend/crates/gateway-host/src/workers.rs b/backend/crates/gateway-host/src/workers.rs index 4d2136f70..989f5f785 100644 --- a/backend/crates/gateway-host/src/workers.rs +++ b/backend/crates/gateway-host/src/workers.rs @@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use futures::FutureExt as _; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; use gateway_core::health::{ WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState, }; @@ -48,6 +49,7 @@ impl WorkerSupervisor { &self, plan: Vec, leader_lease: Arc, + diagnostics: Arc, ) -> Result<(), WorkerStartError> { if self .started @@ -56,7 +58,7 @@ impl WorkerSupervisor { { return Err(WorkerStartError::AlreadyStarted); } - let result = self.start_inner(plan, leader_lease); + let result = self.start_inner(plan, leader_lease, diagnostics); if result.is_err() { self.started.store(false, Ordering::Release); } @@ -67,6 +69,7 @@ impl WorkerSupervisor { &self, plan: Vec, leader_lease: Arc, + diagnostics: Arc, ) -> Result<(), WorkerStartError> { tokio::runtime::Handle::try_current().map_err(|_| WorkerStartError::RuntimeUnavailable)?; let WorkerPlan { @@ -91,14 +94,14 @@ impl WorkerSupervisor { lease, Arc::clone(&leader_lease), self.cancellation.clone(), - Arc::clone(&self.health), + (Arc::clone(&self.health), Arc::clone(&diagnostics)), )), WorkerRunnable::Daemon { restart, task } => tokio::spawn(supervise_daemon( id.clone(), task, restart, self.cancellation.clone(), - Arc::clone(&self.health), + (Arc::clone(&self.health), Arc::clone(&diagnostics)), )), }; (id, handle) @@ -360,8 +363,9 @@ async fn supervise_scheduled( lease: Option, leader_lease: Arc, cancellation: CancellationToken, - health: Arc, + observation: (Arc, Arc), ) { + let (health, diagnostics) = observation; let mut backoff = schedule.initial_backoff(); loop { if cancellation.is_cancelled() { @@ -397,7 +401,8 @@ async fn supervise_scheduled( .min(schedule.maximum_backoff()) } ScheduledOutcome::Failed(error) => { - let failures = health.failed(&id, error.clone()); + record_worker_failure(diagnostics.as_ref(), &id, &error).await; + let failures = health.failed(&id, error.as_safe_str().to_owned()); let delay = take_backoff(&mut backoff, schedule.maximum_backoff()); tracing::warn!( worker = %id, @@ -438,8 +443,10 @@ async fn acquire_and_run( run_leased_cycle(id, task, guard, schedule, cancellation, health).await } Ok(Ok(WorkerLeaseAcquisition::Busy { retry_after })) => ScheduledOutcome::Busy(retry_after), - Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()), - Err(_) => ScheduledOutcome::Failed("leader lease port panicked".to_owned()), + Ok(Err(error)) => { + ScheduledOutcome::Failed(WorkerTaskError::safe(error.as_safe_str()).with_source(error)) + } + Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease port panicked")), } } @@ -478,7 +485,9 @@ async fn run_leased_cycle( ) -> ScheduledOutcome { let fencing_token = match std::panic::catch_unwind(AssertUnwindSafe(|| guard.fencing_token())) { Ok(token) => token, - Err(_) => return ScheduledOutcome::Failed("leader lease guard panicked".to_owned()), + Err(_) => { + return ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease guard panicked")); + } }; health.running(id, Some(fencing_token)); let cycle_cancel = CancellationToken::new(); @@ -518,38 +527,39 @@ async fn run_leased_cycle( }; match release_guard(guard, schedule.leader_lease_renewal_interval()).await { Ok(()) => outcome, - Err(error) if matches!(outcome, ScheduledOutcome::ShuttingDown) => { - tracing::warn!(worker = %id, error, "leader lease 释放失败"); - outcome - } - Err(error) => ScheduledOutcome::Failed(error), + Err(cleanup) => match outcome { + ScheduledOutcome::Failed(error) => { + ScheduledOutcome::Failed(error.with_cleanup(cleanup)) + } + _ => ScheduledOutcome::Failed(cleanup), + }, } } async fn renew_guard( guard: &mut dyn WorkerLeaderLeaseGuard, timeout: Duration, -) -> Result<(), String> { +) -> Result<(), WorkerTaskError> { let future = AssertUnwindSafe(guard.renew()).catch_unwind(); match tokio::time::timeout(timeout, future).await { Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()), - Ok(Err(_)) => Err("leader lease renewal panicked".to_owned()), - Err(_) => Err("leader lease renewal timed out".to_owned()), + Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)), + Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease renewal panicked")), + Err(_) => Err(WorkerTaskError::safe("leader lease renewal timed out")), } } async fn release_guard( guard: Box, timeout: Duration, -) -> Result<(), String> { +) -> Result<(), WorkerTaskError> { let future = std::panic::catch_unwind(AssertUnwindSafe(|| guard.release())) - .map_err(|_| "leader lease release panicked".to_owned())?; + .map_err(|_| WorkerTaskError::safe("leader lease release panicked"))?; match tokio::time::timeout(timeout, AssertUnwindSafe(future).catch_unwind()).await { Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()), - Ok(Err(_)) => Err("leader lease release panicked".to_owned()), - Err(_) => Err("leader lease release timed out".to_owned()), + Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)), + Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease release panicked")), + Err(_) => Err(WorkerTaskError::safe("leader lease release timed out")), } } @@ -558,8 +568,9 @@ async fn supervise_daemon( task: Box, restart: DaemonRestartPolicy, cancellation: CancellationToken, - health: Arc, + observation: (Arc, Arc), ) { + let (health, diagnostics) = observation; let mut backoff = restart.initial_backoff(); loop { if cancellation.is_cancelled() { @@ -579,12 +590,13 @@ async fn supervise_daemon( }; let Some(result) = result else { break }; let error = match result { - Ok(Ok(())) => "daemon exited unexpectedly".to_owned(), - Ok(Err(error)) => error.as_safe_str().to_owned(), - Err(_) => "daemon panicked".to_owned(), + Ok(Ok(())) => WorkerTaskError::safe("daemon exited unexpectedly"), + Ok(Err(error)) => error, + Err(_) => WorkerTaskError::safe("daemon panicked"), }; let delay = take_backoff(&mut backoff, restart.maximum_backoff()); - let failures = health.failed(&id, error.clone()); + record_worker_failure(diagnostics.as_ref(), &id, &error).await; + let failures = health.failed(&id, error.as_safe_str().to_owned()); tracing::warn!( worker = %id, failures, @@ -603,7 +615,7 @@ async fn supervise_daemon( enum ScheduledOutcome { Succeeded, Busy(Option), - Failed(String), + Failed(WorkerTaskError), ShuttingDown, } @@ -612,8 +624,8 @@ fn task_result( ) -> ScheduledOutcome { match result { Ok(Ok(())) => ScheduledOutcome::Succeeded, - Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()), - Err(_) => ScheduledOutcome::Failed("worker panicked".to_owned()), + Ok(Err(error)) => ScheduledOutcome::Failed(error), + Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("worker panicked")), } } @@ -641,3 +653,17 @@ fn lock_unpoisoned(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } + +async fn record_worker_failure( + diagnostics: &dyn OperationalDiagnostics, + id: &WorkerId, + error: &WorkerTaskError, +) { + let mut failure = + OperationalFailure::new("worker", "run", "worker_failed", error.as_safe_str()); + failure.correlation_id = Some(id.to_string()); + failure.details = error.error_details(); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(worker = %id, "worker diagnostic could not be recorded"); + } +} diff --git a/backend/crates/gateway-host/tests/logging/sink.rs b/backend/crates/gateway-host/tests/logging/sink.rs index edb52cffc..59f8e7f73 100644 --- a/backend/crates/gateway-host/tests/logging/sink.rs +++ b/backend/crates/gateway-host/tests/logging/sink.rs @@ -110,3 +110,134 @@ fn write_failure_marks_log_completeness_unhealthy_even_after_writes_recover() { assert!(body.contains("before-failure") && body.contains("after-recovery")); assert!(!body.contains("cannot-write")); } + +#[cfg(unix)] +#[test] +fn blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors() { + const CHILD: &str = "logging::sink::blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors"; + if env::var_os(CHILD_PROCESS_ENV).is_some() { + let directory = PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut config = logging_config(directory.clone(), false); + config.logging.oauth_recovery = false; + config.logging.file.max_file_size_mb = 8; + let bundle = runtime.block_on(gateway_host::initialize(config)).unwrap(); + let health = bundle.logging_health_probe(); + let date = gateway_core::time::DeploymentTimeZone::default() + .local(chrono::Utc::now()) + .date_naive(); + let fifo = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.1.log")); + assert!( + Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success() + ); + let (release, released) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + // 超时只用于让旧的同步实现能退出测试,接口必须在解除阻塞之前完成 + let _ = released.recv_timeout(Duration::from_secs(3)); + let mut body = String::new(); + fs::File::open(fifo) + .unwrap() + .read_to_string(&mut body) + .unwrap(); + body + }); + let payload = "x".repeat(8 * 1024 * 1024); + tracing::info!(target: APPLICATION_LOG_TARGET, payload, "rotate before requests"); + tracing::info!(target: APPLICATION_LOG_TARGET, "block on opening next segment"); + runtime.block_on(async { + let initial = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.log")); + tokio::time::timeout(Duration::from_secs(2), async { + while fs::metadata(&initial).unwrap().len() < 8 * 1024 * 1024 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }).await.unwrap(); + use futures::{SinkExt as _, StreamExt as _}; + let app = axum::Router::new() + .route("/v1/log-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="v1", "critical_after_congestion"); + "ok" + })) + .route("/admin/log-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="admin", "critical_after_congestion"); + "ok" + })) + .route("/v1/stream-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="stream", "critical_after_congestion"); + axum::response::Sse::new(futures::stream::iter([ + Ok::<_, std::convert::Infallible>(axum::response::sse::Event::default().data("first")), + Ok(axum::response::sse::Event::default().data("last")), + ])) + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + let started = std::time::Instant::now(); + for sequence in 0..5_000 { + tracing::info!(target: APPLICATION_LOG_TARGET, sequence, "congested regular log"); + } + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + for path in ["/v1/log-test", "/admin/log-test", "/v1/stream-test"] { + let response = client.get(format!("http://{address}{path}")).send().await.unwrap(); + assert!(response.status().is_success()); + let body = response.text().await.unwrap(); + if path.ends_with("stream-test") { + assert!(body.contains("first") && body.contains("last")); + } else { + assert_eq!(body, "ok"); + } + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let websocket_server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + tracing::error!(target: APPLICATION_LOG_TARGET, path="websocket", "critical_after_congestion"); + socket.send(tokio_tungstenite::tungstenite::Message::Text("frame".into())).await.unwrap(); + }); + let stream = tokio::net::TcpStream::connect(address).await.unwrap(); + let (mut websocket, _) = tokio_tungstenite::client_async(format!("ws://{address}/"), stream).await.unwrap(); + assert_eq!(websocket.next().await.unwrap().unwrap().into_text().unwrap(), "frame"); + websocket_server.await.unwrap(); + let elapsed = started.elapsed(); + assert!(elapsed < Duration::from_secs(1), "interfaces waited for logging: {elapsed:?}"); + eprintln!("paused log output: 5000 submissions and HTTP/admin/SSE/WS completed in {elapsed:?}"); + let HealthState::Unhealthy(message) = health.check().await else { panic!("queue overflow must be visible") }; + assert!(message.contains("0 warnings/errors"), "{message}"); + server.abort(); + }); + release.send(()).unwrap(); + drop(bundle); + let body = reader.join().unwrap(); + assert_eq!( + body.lines() + .filter(|line| line.contains("critical_after_congestion")) + .count(), + 4 + ); + for line in body.lines() { + serde_json::from_str::(line).unwrap(); + } + return; + } + let directory = tempfile::tempdir().unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args(["--exact", CHILD, "--nocapture"]) + .env(CHILD_PROCESS_ENV, "1") + .env(LOG_DIRECTORY_ENV, directory.path()) + .env("RUST_LOG", "off,logging_test_application=info") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); +} diff --git a/backend/crates/gateway-host/tests/plugin_distribution/github.rs b/backend/crates/gateway-host/tests/plugin_distribution/github.rs index d71c65d41..1fd014f27 100644 --- a/backend/crates/gateway-host/tests/plugin_distribution/github.rs +++ b/backend/crates/gateway-host/tests/plugin_distribution/github.rs @@ -198,13 +198,12 @@ async fn rate_limit_applies_to_other_repositories_using_the_same_identity_and_eg .kind(), AdminErrorKind::RateLimited ); - assert_eq!( - distribution - .query_release(query(None), vec![], None) - .await - .unwrap_err(), - failure - ); + let cached_failure = distribution + .query_release(query(None), vec![], None) + .await + .unwrap_err(); + assert_eq!(cached_failure.kind(), failure.kind()); + assert_eq!(cached_failure.message(), failure.message()); assert_eq!(server.received_requests().await.unwrap().len(), 1); } diff --git a/backend/crates/gateway-host/tests/workers.rs b/backend/crates/gateway-host/tests/workers.rs index 91988de7c..9ff6ecb3b 100644 --- a/backend/crates/gateway-host/tests/workers.rs +++ b/backend/crates/gateway-host/tests/workers.rs @@ -54,7 +54,8 @@ impl ScheduledTask for CountingTask { Box::pin(async move { let run = self.runs.fetch_add(1, Ordering::SeqCst) + 1; if run <= self.fail_until { - Err(WorkerTaskError::safe("expected test failure")) + Err(WorkerTaskError::safe("expected test failure") + .with_source(std::io::Error::other("PRIVATE_WORKER_NATIVE_CAUSE"))) } else { Ok(()) } @@ -343,9 +344,11 @@ async fn registry_composes_multiple_owners_but_rejects_duplicate_identity() { let supervisor = supervisor(); assert!(matches!( - supervisor.start(plan, Arc::new(FakeLeasePort::default())), - Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai" - )); + supervisor.start(plan, Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), + Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai" + )); } #[tokio::test] @@ -360,7 +363,11 @@ async fn only_database_fact_reasons_can_disable_nonexistent_workers() { ))); assert!(matches!( - supervisor().start(plan, Arc::new(FakeLeasePort::default())), + supervisor().start( + plan, + Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), Err(WorkerStartError::KindDisabled(WorkerKind::OpsFlush)) )); } @@ -373,9 +380,11 @@ async fn registry_refuses_start_when_any_real_owner_kind_is_missing() { .collect(); assert!(matches!( - supervisor().start(plan, Arc::new(FakeLeasePort::default())), - Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention] - )); + supervisor().start(plan, Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), + Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention] + )); } #[tokio::test] @@ -383,7 +392,11 @@ async fn disabled_database_facts_never_execute_or_acquire_a_lease() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(complete_plan(Vec::new()), leases.clone()) + .start( + complete_plan(Vec::new()), + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| leases.requests().len() >= ACTIVE_KINDS.len()).await; let snapshot = supervisor.health_source().snapshot(); @@ -422,7 +435,11 @@ async fn one_kind_runs_all_registered_owner_tasks() { ]); let supervisor = supervisor(); supervisor - .start(plan, Arc::new(FakeLeasePort::default())) + .start( + plan, + Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| second.runs.load(Ordering::SeqCst) >= 1).await; @@ -456,6 +473,7 @@ async fn leader_guard_lives_through_cycle_and_fence_reaches_owner() { .start( target_plan(task.clone(), long_interval_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.entered.notified().await; @@ -478,6 +496,7 @@ async fn long_cycle_renews_leader_lease_before_ttl() { .start( target_plan(task.clone(), renewal_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.entered.notified().await; @@ -500,7 +519,11 @@ async fn renewal_failure_cancels_old_cycle_before_another_leader_cycle_starts() leases.renew_script(target_worker(), [RenewStep::Error]); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), renewal_schedule()), leases) + .start( + target_plan(task.clone(), renewal_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); yield_until(|| task.starts.load(Ordering::SeqCst) == 1).await; tokio::time::advance(Duration::from_millis(10)).await; @@ -520,6 +543,7 @@ async fn supervisor_uses_exponential_backoff_and_recovers_health() { .start( target_plan(task.clone(), backoff_schedule()), Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await; @@ -548,7 +572,11 @@ async fn task_and_lease_port_panics_are_isolated_and_retried() { ); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), backoff_schedule()), leases) + .start( + target_plan(task.clone(), backoff_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); tokio::task::yield_now().await; tokio::time::advance(Duration::from_millis(10)).await; @@ -578,7 +606,11 @@ async fn lease_busy_is_healthy_standby_and_retries_without_running_owner() { ); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), backoff_schedule()), leases) + .start( + target_plan(task.clone(), backoff_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); yield_until(|| task_health(&supervisor, &target_worker()).state == WorkerRuntimeState::Standby) .await; @@ -598,7 +630,11 @@ async fn cooperative_shutdown_propagates_cancel_before_dropping_all_guards() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(all_active_plan(task.clone()), leases.clone()) + .start( + all_active_plan(task.clone()), + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| task.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len()).await; supervisor.shutdown(Duration::from_secs(1)).await; @@ -619,6 +655,7 @@ async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() { .start( target_plan(task.clone(), long_interval_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.notification.notified().await; @@ -666,7 +703,11 @@ async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(plan, leases.clone()) + .start( + plan, + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); hung.notification.notified().await; wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await; @@ -691,6 +732,7 @@ async fn running_worker_becomes_unhealthy_when_last_success_is_stale() { .start( target_plan(task.clone(), freshness_schedule()), Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.second_cycle.notified().await; @@ -898,3 +940,50 @@ fn lock_unpoisoned(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} + +#[tokio::test] +async fn worker_boundary_records_one_native_failure_and_recovers_health() { + let supervisor = supervisor(); + let task = CountingTask::new(1); + let diagnostics = Arc::new(RecordingDiagnostics::default()); + supervisor + .start( + target_plan(task.clone(), test_schedule()), + Arc::new(FakeLeasePort::default()), + diagnostics.clone(), + ) + .unwrap(); + wait_until(|| task.runs.load(Ordering::SeqCst) >= 2).await; + supervisor.shutdown(Duration::from_secs(1)).await; + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].correlation_id.as_deref(), + Some(target_worker().to_string().as_str()) + ); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_WORKER_NATIVE_CAUSE") + ); + assert!( + !format!("{:?}", supervisor.health_source().snapshot()) + .contains("PRIVATE_WORKER_NATIVE_CAUSE") + ); +} diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs index f89a4cd25..fb224f993 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs @@ -135,10 +135,12 @@ fn rpc_failure(error: RpcError) -> &'static str { match error { RpcError::Timeout => "插件响应超时", RpcError::Cancelled => "调用已取消", - RpcError::Closed | RpcError::Start(_) => "插件进程不可用", - RpcError::Handshake | RpcError::Protocol | RpcError::InvalidResponse(_) => "插件协议错误", + RpcError::Closed(_) | RpcError::Start(_) => "插件进程不可用", + RpcError::Handshake(_) | RpcError::Protocol(_) | RpcError::InvalidResponse(..) => { + "插件协议错误" + } RpcError::Capacity => "插件调用容量不足", - RpcError::Context => "调用上下文无效", + RpcError::Context(_) => "调用上下文无效", RpcError::Remote(_) => "插件拒绝命令", } } diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs b/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs index 2aeed2a12..121f16d97 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs @@ -202,7 +202,7 @@ async fn invoke_middleware( ); let (protocol, headers, payload) = match invocation.request_parts() { Ok(projection) => projection, - Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context).await, + Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context(None)).await, }; let mut call_context = invocation_ports.session.context( match entry.mount { diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs b/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs index 9a6388bf8..b5b30217e 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs @@ -46,7 +46,7 @@ async fn write_loop( frame = data.recv() => frame, }; let Some(frame) = frame else { - shared.fail(RpcError::Closed); + shared.fail(RpcError::Closed(None)); return; }; // 撤销尚未发送的调用时丢弃排队帧;已开始写入的 Call 必须先于 Cancel @@ -64,8 +64,10 @@ async fn write_loop( _ = stopped.changed() => return, result = write_frame(&mut writer, &frame) => result, }; - if written.is_err() { - shared.fail(RpcError::Closed); + if let Err(source) = written { + shared.fail(RpcError::Closed(Some( + gateway_core::error::ErrorSource::new(source), + ))); return; } } @@ -87,9 +89,14 @@ async fn read_loop( _ = stopped.changed() => return, frame = read_frame(&mut reader) => frame, }; - let Ok(frame) = frame else { - shared.fail(RpcError::Closed); - return; + let frame = match frame { + Ok(frame) => frame, + Err(source) => { + shared.fail(RpcError::Closed(Some( + gateway_core::error::ErrorSource::new(source), + ))); + return; + } }; let result = match frame.message { Message::Cancelled { id } if frame.payload.is_empty() => { @@ -114,7 +121,7 @@ async fn read_loop( params, } => { if id == 0 || id % 2 != 0 || id <= last_callback { - Err(RpcError::Protocol) + Err(RpcError::Protocol(None)) } else if let Some(context) = shared.context(parent_id) { last_callback = id; if let Some(permit) = shared.try_callback_slot() { @@ -171,10 +178,10 @@ async fn read_loop( })); Ok(()) } else { - Err(RpcError::Protocol) + Err(RpcError::Protocol(None)) } } - _ => Err(RpcError::Protocol), + _ => Err(RpcError::Protocol(None)), }; if let Err(error) = result { shared.fail(error); diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs index 2bd0af2c9..70c401968 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs @@ -45,18 +45,18 @@ impl Default for RpcLimits { } } -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] pub enum RpcError { #[error("plugin process could not be started: {0}")] Start(#[source] ProcessStartError), #[error("plugin handshake does not match the prepared instance")] - Handshake, + Handshake(#[source] Option), #[error("plugin protocol is invalid")] - Protocol, + Protocol(#[source] Option), #[error("plugin returned an invalid response at {0:?}")] - InvalidResponse(Stage), + InvalidResponse(Stage, #[source] Option), #[error("plugin process or transport has stopped")] - Closed, + Closed(#[source] Option), #[error("plugin call exceeded its deadline")] Timeout, #[error("plugin session was cancelled")] @@ -64,7 +64,7 @@ pub enum RpcError { #[error("plugin call capacity is exhausted")] Capacity, #[error("plugin call context is invalid")] - Context, + Context(#[source] Option), #[error("plugin returned an error")] Remote(PluginFault), } @@ -74,9 +74,9 @@ impl RpcError { pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) { match self { RpcError::Start(_) => ("process_start", "插件进程启动失败"), - RpcError::Handshake => ("handshake", "插件握手失败"), - RpcError::Protocol => ("protocol", "插件协议错误"), - RpcError::InvalidResponse(stage) => ( + RpcError::Handshake(_) => ("handshake", "插件握手失败"), + RpcError::Protocol(_) => ("protocol", "插件协议错误"), + RpcError::InvalidResponse(stage, _) => ( "invalid_response", match stage { Stage::Registration => "注册阶段:插件返回了无效响应", @@ -97,11 +97,11 @@ impl RpcError { Stage::Maintenance => "维护阶段:插件返回了无效响应", }, ), - RpcError::Closed => ("closed", "插件进程或传输已停止"), + RpcError::Closed(_) => ("closed", "插件进程或传输已停止"), RpcError::Timeout => ("timeout", "插件调用超时"), RpcError::Cancelled => ("cancelled", "插件会话已取消"), RpcError::Capacity => ("capacity", "插件调用容量已耗尽"), - RpcError::Context => ("context", "插件调用上下文无效"), + RpcError::Context(_) => ("context", "插件调用上下文无效"), RpcError::Remote(_) => ("remote", "插件返回错误"), } } @@ -186,7 +186,7 @@ pub struct RpcSession { impl RpcSession { /// 只用于已确认来自插件的合同错误,不接收业务错误或原始响应内容 pub(crate) fn invalid_response(&self, stage: Stage) -> RpcError { - let error = RpcError::InvalidResponse(stage); + let error = RpcError::InvalidResponse(stage, None); self.shared.fail(error.clone()); error } @@ -196,7 +196,14 @@ impl RpcSession { stage: Stage, value: serde_json::Value, ) -> Result { - serde_json::from_value(value).map_err(|_| self.invalid_response(stage)) + serde_json::from_value(value).map_err(|source| { + let error = RpcError::InvalidResponse( + stage, + Some(gateway_core::error::ErrorSource::new(source)), + ); + self.shared.fail(error.clone()); + error + }) } pub(crate) fn capability_version( @@ -292,16 +299,16 @@ impl Shared { pub fn finish(&self, id: u64, result: Result) -> Result<(), RpcError> { let mut state = self.state(); - let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?; if call.cancellation.is_some() { return Ok(()); } if call.stream.is_some() && result.is_ok() { - let initial = call.result.take().ok_or(RpcError::Protocol)?; + let initial = call.result.take().ok_or(RpcError::Protocol(None))?; call.handler.streaming(&call.context); let _ = initial.send(result); } else { - let pending = state.calls.remove(&id).ok_or(RpcError::Protocol)?; + let pending = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?; drop(state); pending.finish(result); } @@ -310,16 +317,16 @@ impl Shared { pub fn stream_chunk(&self, id: u64, sequence: u64, payload: Vec) -> Result<(), RpcError> { let mut state = self.state(); - let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?; if call.cancellation.is_some() { return Ok(()); } if call.result.is_some() { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } call.stream .as_mut() - .ok_or(RpcError::Protocol)? + .ok_or(RpcError::Protocol(None))? .receive(sequence, payload) } @@ -332,11 +339,11 @@ impl Shared { { return Ok(()); } - let call = state.calls.remove(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?; drop(state); if call.stream.is_none() || call.result.is_some() { - call.finish(Err(RpcError::Protocol)); - return Err(RpcError::Protocol); + call.finish(Err(RpcError::Protocol(None))); + return Err(RpcError::Protocol(None)); } let result = error.map_or_else( || { @@ -359,7 +366,7 @@ impl Shared { }; call.stream .as_mut() - .ok_or(RpcError::Protocol)? + .ok_or(RpcError::Protocol(None))? .release(bytes)?; drop(state); self.send_control(Frame::control(Message::Credit { @@ -441,7 +448,7 @@ impl Shared { .get(&id) .is_some_and(|call| call.cancellation.is_some()) { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } state.calls.remove(&id); Ok(()) @@ -540,10 +547,12 @@ impl RpcSession { || limits.handshake_timeout.is_zero() || limits.maximum_call_timeout.is_zero() { - return Err(RpcError::Context); + return Err(RpcError::Context(None)); } let manifest = package.package().manifest(); - let plugin_id = manifest.plugin_id().map_err(|_| RpcError::Handshake)?; + let plugin_id = manifest.plugin_id().map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; if handshake.protocol_version != PROTOCOL_VERSION || manifest .package @@ -557,7 +566,7 @@ impl RpcSession { || handshake.incarnation.is_empty() || handshake.generation == 0 { - return Err(RpcError::Handshake); + return Err(RpcError::Handshake(None)); } let deadline = tokio::time::Instant::now() + limits.handshake_timeout; let process = loop { @@ -597,10 +606,12 @@ impl RpcSession { }), ) .await - .map_err(|_| RpcError::Handshake)?; - let reply = read_frame(&mut output) - .await - .map_err(|_| RpcError::Handshake)?; + .map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; + let reply = read_frame(&mut output).await.map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; match reply.message { Message::Ready { protocol_version, @@ -611,7 +622,7 @@ impl RpcSession { { Ok(()) } - _ => Err(RpcError::Handshake), + _ => Err(RpcError::Handshake(None)), } }) .await @@ -661,7 +672,7 @@ impl RpcSession { if !monitor.expected_stop.load(Ordering::Acquire) { tracing::warn!(instance_id, exit = ?reason, "插件进程意外停止"); } - monitor.fail(RpcError::Closed); + monitor.fail(RpcError::Closed(None)); // 等进程退出后再回收其文件,避免 Windows 上删除正在执行的制品 drop(package); }); @@ -738,7 +749,7 @@ impl RpcSession { || method.is_empty() || method.len() > 128 { - return Err(RpcError::Context); + return Err(RpcError::Context(None)); } Ok(tokio::time::Instant::now() + Duration::from_millis(context.timeout_ms)) } @@ -807,7 +818,9 @@ impl RpcSession { payload, }; // 本地元数据错误不能进入写队列并关闭整个会话 - validate_frame(&frame).map_err(|_| RpcError::Context)?; + validate_frame(&frame).map_err(|source| { + RpcError::Context(Some(gateway_core::error::ErrorSource::new(source))) + })?; let started = self.begin_call(context, deadline, stream)?; let mut guard = CallGuard { shared: Arc::clone(&self.shared), @@ -815,7 +828,10 @@ impl RpcSession { armed: true, }; let sent = tokio::time::timeout_at(started.deadline, async { - self.data.send(frame).await.map_err(|_| RpcError::Closed)?; + self.data + .send(frame) + .await + .map_err(|_| RpcError::Closed(None))?; if let Some((bytes, frames)) = credit { // 初始信用跟在 Call 后走同一有序队列,不能被控制队列提前发送 self.data @@ -825,7 +841,7 @@ impl RpcSession { frames, })) .await - .map_err(|_| RpcError::Closed)?; + .map_err(|_| RpcError::Closed(None))?; } Ok::<(), RpcError>(()) }) @@ -851,9 +867,9 @@ impl RpcSession { result? } Ok(Err(_)) => { - self.shared.cancel(started.id, RpcError::Closed); + self.shared.cancel(started.id, RpcError::Closed(None)); guard.armed = false; - return Err(RpcError::Closed); + return Err(RpcError::Closed(None)); } Err(_) => { self.shared.cancel(started.id, RpcError::Timeout); @@ -935,7 +951,7 @@ impl RpcSession { .await .is_err() { - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); // 强制停止没有第二段宽限;退出和回调析构是返回前必须确认的事实 self.settle_shutdown().await; } @@ -944,7 +960,7 @@ impl RpcSession { async fn settle_shutdown(&self) { self.shared.process.exited().await; // 不等待监控任务调度;先撤销父调用并中止其回调,再确认回调 future 已析构 - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); self.shared .wait_for_callbacks(self.limits.maximum_callbacks) .await; @@ -954,7 +970,7 @@ impl RpcSession { impl Drop for RpcSession { fn drop(&mut self) { self.shared.expect_stop(); - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); } } diff --git a/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs b/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs index b942b7bb7..500431236 100644 --- a/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs +++ b/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs @@ -43,7 +43,7 @@ impl StreamIngress { self.window.receive(sequence, payload.len())?; self.chunks .try_send(payload) - .map_err(|_| RpcError::Protocol) + .map_err(|_| RpcError::Protocol(None)) } pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> { @@ -86,7 +86,7 @@ impl RpcStream { let Some(terminal) = self.terminal.take() else { return Ok(None); }; - terminal.await.map_err(|_| RpcError::Closed)??; + terminal.await.map_err(|_| RpcError::Closed(None))??; Ok(None) } Err(_) => { diff --git a/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs b/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs index 79d274d46..042b9fe19 100644 --- a/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs +++ b/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs @@ -19,22 +19,25 @@ impl ReceiveWindow { } pub fn receive(&mut self, sequence: u64, bytes: usize) -> Result<(), RpcError> { - let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol)?; + let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol(None))?; if sequence != self.next_sequence || bytes == 0 || bytes > self.bytes || self.frames == 0 { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } self.next_sequence = self .next_sequence .checked_add(1) - .ok_or(RpcError::Protocol)?; + .ok_or(RpcError::Protocol(None))?; self.bytes -= bytes; self.frames -= 1; Ok(()) } pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> { - self.bytes = self.bytes.checked_add(bytes).ok_or(RpcError::Protocol)?; - self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol)?; + self.bytes = self + .bytes + .checked_add(bytes) + .ok_or(RpcError::Protocol(None))?; + self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol(None))?; Ok(()) } } diff --git a/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs b/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs index 20e5846ed..2bf95b3a6 100644 --- a/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs +++ b/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs @@ -5,6 +5,7 @@ use gateway_admin::{ model::{ account_groups::{AccountGroupColor, CreateAccountGroup}, plugin_resources::{GroupMembersChange, ManagedKeyConfig, PluginResourceOwner}, + plugins::instances::PluginInstanceRuntimeStatus, }, ports::plugin_resources::PluginResourceAccess, }; @@ -112,6 +113,7 @@ async fn revision(environment: &Environment) -> gateway_admin::model::Revision { async fn start( environment: &Environment, + phase: &str, ) -> ( Arc, gateway_core::CoreBundle, @@ -120,6 +122,32 @@ async fn start( tokio::task::JoinHandle<()>, ) { let (runtime, core) = environment.runtime().await; + // 集合就绪允许保留隔离的启动失败实例;维护测试必须确认插件本身正在运行 + let snapshot = core.snapshots().snapshot_for_diagnostics().unwrap(); + let extensions = snapshot.extensions().unwrap(); + let instances = environment + .store + .admin_ports() + .plugins() + .load_instances() + .await + .unwrap(); + let diagnostics = gateway_admin::ports::plugins::PluginRuntimeDiagnostics::runtime_diagnostics( + runtime.as_ref(), + &instances, + Some(snapshot.revision().get()), + Some(extensions), + ) + .await + .unwrap(); + assert!( + !diagnostics.is_empty() + && diagnostics + .values() + .all(|instance| instance.status == PluginInstanceRuntimeStatus::Running), + "plugin was not ready during {phase}: {diagnostics:?}" + ); + drop(snapshot); let resources = gateway_admin::initialize_plugin_resources( environment.store.admin_ports().plugin_resources(), core.snapshot_control(), @@ -141,7 +169,7 @@ async fn start( (runtime, core, resources, stop, task) } -async fn wait_done(path: &std::path::Path, after: usize) -> Value { +async fn wait_done(path: &std::path::Path, after: usize, phase: &str) -> Value { tokio::time::timeout(Duration::from_secs(15), async { loop { let entries: Vec = std::fs::read_to_string(path) @@ -162,7 +190,7 @@ async fn wait_done(path: &std::path::Path, after: usize) -> Value { .await .unwrap_or_else(|error| { panic!( - "plugin reconciliation did not finish: {error}; observations: {}", + "plugin reconciliation did not finish during {phase}: {error}; observations: {}", std::fs::read_to_string(path).unwrap_or_default(), ) }) @@ -178,7 +206,7 @@ async fn malformed_maintenance_response_does_not_stop_the_host_worker() { json!({"maintenance_fixture":true,"invalid_response_method":"plugin.reconcile"}), ) .await; - let (runtime, core, resources, stop, task) = start(&environment).await; + let (runtime, core, resources, stop, task) = start(&environment, "malformed response").await; tokio::time::timeout(Duration::from_secs(5), async { while core .snapshots() @@ -220,8 +248,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou let marker = environment.directory.path().join("maintenance.jsonl"); let failed = environment.directory.path().join("failed-once"); environment.install_plugin(json!({"maintenance_fixture":true, "maintenance_marker":marker, "maintenance_fail_once":failed})).await; - let (runtime, core, resources, stop, task) = start(&environment).await; - let first = wait_done(&marker, 0).await; + let (runtime, core, resources, stop, task) = start(&environment, "initial startup").await; + let first = wait_done(&marker, 0, "initial startup").await; assert!(failed.exists()); let group_id = gateway_core::routing::AccountGroupId::new(first["group"].as_str().unwrap()).unwrap(); @@ -284,8 +312,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou // 离线新增账号后重新启动真实插件子进程;稳定资源键必须复用原分组与 Key let offline = environment.account(None).await; std::fs::write(&marker, "").unwrap(); - let (runtime, core, resources, stop, task) = start(&environment).await; - let restored = wait_done(&marker, 0).await; + let (runtime, core, resources, stop, task) = start(&environment, "restoration").await; + let restored = wait_done(&marker, 0, "restoration").await; assert_eq!(restored["group"], first["group"]); assert_eq!(restored["key"], first["key"]); let members = environment diff --git a/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs b/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs index 2292a82fa..2ca74a003 100644 --- a/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs @@ -325,7 +325,7 @@ async fn shutdown_returns_after_in_flight_callback_future_is_dropped() { .await .expect("session shutdown"); assert!(callbacks.dropped.load(Ordering::Acquire)); - assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed))); + assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed(_)))); } #[tokio::test] @@ -416,7 +416,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() { .await .err() .expect("oversized call must fail"); - assert_eq!(error, RpcError::Context); + assert!(matches!(error, RpcError::Context(_))); assert!(matches!( session .call_stream( @@ -426,7 +426,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() { payload, ) .await, - Err(RpcError::Context) + Err(RpcError::Context(_)) )); assert_eq!(callbacks.begun.load(Ordering::Relaxed), begun); assert!(session.is_ready()); @@ -653,7 +653,7 @@ async fn unknown_parent_cannot_borrow_another_calls_resources() { vec![], ) .await; - assert!(matches!(reply, Err(RpcError::Protocol))); + assert!(matches!(reply, Err(RpcError::Protocol(_)))); assert_eq!(callbacks.called.load(Ordering::Relaxed), 0); } @@ -712,12 +712,16 @@ async fn child_crash_fails_the_call_without_waiting_for_its_deadline() { let result = tokio::time::timeout(Duration::from_secs(2), call) .await .unwrap(); - assert!(matches!(result, Err(RpcError::Closed))); + assert!(matches!(result, Err(RpcError::Closed(_)))); } #[tokio::test] async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() { - for method in ["malformed_truncated_frame", "malformed_frame_length"] { + for method in [ + "malformed_truncated_frame", + "malformed_frame_length", + "malformed_metadata", + ] { let callbacks = Arc::new(LifecycleCallbacks::default()); let (_cache, session) = session(Arc::clone(&callbacks)).await; let result = tokio::time::timeout( @@ -731,7 +735,29 @@ async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() { ) .await .expect("malformed worker output must fail within the process boundary"); - assert!(matches!(result, Err(RpcError::Closed))); + let error = result.err().expect("malformed frame failure"); + assert!(matches!(error, RpcError::Closed(Some(_)))); + if method == "malformed_metadata" { + use std::error::Error as _; + let frame = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(frame.source().unwrap().is::()); + let provider = gateway_core::error::ProviderError::new( + gateway_core::error::ProviderErrorKind::Protocol, + gateway_core::upstream::UpstreamSendState::Ambiguous, + ) + .with_source(error); + assert!( + provider + .error_details() + .unwrap() + .contains("PRIVATE_INVALID_FRAME_KIND") + ); + assert!(!format!("{provider:?} {provider}").contains("PRIVATE_INVALID_FRAME_KIND")); + } assert!(!session.is_ready()); assert_eq!(callbacks.begun.load(Ordering::Relaxed), 1); assert_eq!(callbacks.finished.load(Ordering::Relaxed), 1); @@ -786,7 +812,7 @@ async fn stream_window_and_sequence_violations_fail_closed() { ) .await .unwrap(); - assert!(matches!(stream.next().await, Err(RpcError::Protocol))); + assert!(matches!(stream.next().await, Err(RpcError::Protocol(_)))); session.shutdown(Duration::from_secs(1)).await; } } diff --git a/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs b/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs index f3fbb3ade..fce20bbbc 100644 --- a/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs +++ b/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs @@ -1368,9 +1368,19 @@ async fn main() { last_call = id; if matches!( method.as_str(), - "malformed_truncated_frame" | "malformed_frame_length" + "malformed_truncated_frame" | "malformed_frame_length" | "malformed_metadata" ) { - let bytes: &[u8] = if method == "malformed_truncated_frame" { + let mut malformed_metadata = Vec::new(); + if method == "malformed_metadata" { + let metadata = br#"{"type":"PRIVATE_INVALID_FRAME_KIND"}"#; + malformed_metadata + .extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + malformed_metadata.extend_from_slice(&0_u64.to_be_bytes()); + malformed_metadata.extend_from_slice(metadata); + } + let bytes: &[u8] = if method == "malformed_metadata" { + &malformed_metadata + } else if method == "malformed_truncated_frame" { // 声明 16 字节元数据,却只写入一个字节后退出 &[0, 0, 0, 16, 0, 0, 0, 0, b'{'] } else { diff --git a/backend/crates/gateway-plugin/sdk/src/client/frame.rs b/backend/crates/gateway-plugin/sdk/src/client/frame.rs index e7e666216..68707cf3d 100644 --- a/backend/crates/gateway-plugin/sdk/src/client/frame.rs +++ b/backend/crates/gateway-plugin/sdk/src/client/frame.rs @@ -14,7 +14,7 @@ pub async fn read_frame(reader: &mut R) -> Result( writer: &mut W, frame: &Frame, ) -> Result<(), FrameError> { - let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?; + let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?; check_metadata_length(metadata.len())?; let metadata_len = u32::try_from(metadata.len()).map_err(|_| FrameError::Length)?; let payload_len = u64::try_from(frame.payload.len()).map_err(|_| FrameError::Length)?; @@ -49,7 +49,7 @@ pub async fn write_frame( /// 入队前校验本地元数据,避免单次编码失败中断共享传输 pub fn validate_frame(frame: &Frame) -> Result<(), FrameError> { - let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?; + let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?; check_metadata_length(metadata.len()) } diff --git a/backend/crates/gateway-plugin/sdk/src/message.rs b/backend/crates/gateway-plugin/sdk/src/message.rs index dc1693ae6..8c4ddedfa 100644 --- a/backend/crates/gateway-plugin/sdk/src/message.rs +++ b/backend/crates/gateway-plugin/sdk/src/message.rs @@ -116,7 +116,7 @@ pub enum FrameError { #[error("plugin frame exceeds its limit or has invalid lengths")] Length, #[error("plugin frame metadata is invalid")] - Metadata, + Metadata(#[source] serde_json::Error), #[error("plugin transport is closed or incomplete")] Io(#[from] std::io::Error), } diff --git a/backend/crates/gateway-plugin/sdk/tests/message.rs b/backend/crates/gateway-plugin/sdk/tests/message.rs index 6265d0028..b82f97715 100644 --- a/backend/crates/gateway-plugin/sdk/tests/message.rs +++ b/backend/crates/gateway-plugin/sdk/tests/message.rs @@ -22,3 +22,24 @@ fn diagnostics_do_not_render_peer_payloads_or_error_details() { }; assert!(!format!("{frame:?} {fault:?}").contains(secret)); } + +#[tokio::test] +async fn malformed_frame_metadata_retains_the_native_json_location() { + use std::error::Error as _; + let metadata = br#"{"type": "PRIVATE_UNKNOWN_MESSAGE_KIND"}"#; + let mut wire = Vec::new(); + wire.extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + wire.extend_from_slice(&0_u64.to_be_bytes()); + wire.extend_from_slice(metadata); + let error = gateway_plugin_sdk::client::read_frame(&mut wire.as_slice()) + .await + .unwrap_err(); + let source = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert_eq!(source.line(), 1); + assert!(source.column() > 0); + assert!(source.is_data()); +} diff --git a/backend/crates/gateway-store/src/backup/staging.rs b/backend/crates/gateway-store/src/backup/staging.rs index b20f83611..3e5326722 100644 --- a/backend/crates/gateway-store/src/backup/staging.rs +++ b/backend/crates/gateway-store/src/backup/staging.rs @@ -74,6 +74,7 @@ impl StagingArea { .map_err(|source| unavailable("read staging free space", source))?; if free < MIN_STAGING_FREE_BYTES { return Err(StoreError::InvalidData { + source: None, entity: "backup staging", message: "staging disk space is below 1 GiB".to_owned(), }); @@ -103,6 +104,7 @@ fn unavailable( fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "backup staging", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs index 4a6406a21..9cbf61a4b 100644 --- a/backend/crates/gateway-store/src/bundle.rs +++ b/backend/crates/gateway-store/src/bundle.rs @@ -7,6 +7,7 @@ use super::*; /// 已完成连接、迁移与 hydration 的 Store 能力集合 pub struct StoreBundle { + diagnostics: Arc, admin_ports: AdminStorePorts, core_ports: CoreStorePorts, provider_ports: ProviderStorePorts, @@ -19,6 +20,11 @@ pub struct StoreBundle { } impl StoreBundle { + #[must_use] + pub fn diagnostics(&self) -> Arc { + self.diagnostics.clone() + } + #[must_use] pub fn admin_ports(&self) -> AdminStorePorts { self.admin_ports.clone() @@ -199,6 +205,7 @@ async fn connect( let (execution, execution_writer) = postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository)); let execution = Arc::new(execution); + let diagnostics: Arc = execution.clone(); let (client_key_usage, client_key_usage_writer) = postgres::PgClientApiKeyUsageSink::new(pool.clone()); let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone())); @@ -212,7 +219,7 @@ async fn connect( REDIS_NAMESPACE, )?); let (admissions, admission_release_writer) = - redis::BufferedClientAdmissionPort::new(admissions); + redis::BufferedClientAdmissionPort::new(admissions, diagnostics.clone()); let core_ports = CoreStorePorts::new( execution, ( @@ -230,6 +237,7 @@ async fn connect( )?), ), Arc::new(client_key_usage), + diagnostics.clone(), ) .with_budget(Arc::new( postgres::PgClientBudgetStore::new(pool.clone()).with_timezone(config.timezone), @@ -250,6 +258,7 @@ async fn connect( cooldowns, runtime_policy, oauth_pending, + diagnostics.clone(), ); let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new( credential_leases, @@ -283,6 +292,7 @@ async fn connect( ), }; Ok(StoreBundle { + diagnostics, admin_ports, core_ports, provider_ports, diff --git a/backend/crates/gateway-store/src/config.rs b/backend/crates/gateway-store/src/config.rs index f772b3739..fffb15284 100644 --- a/backend/crates/gateway-store/src/config.rs +++ b/backend/crates/gateway-store/src/config.rs @@ -51,6 +51,7 @@ impl StorePoolConfig { pub(crate) fn validate(&self) -> StoreResult<()> { if self.max_connections < 2 || self.acquire_timeout_seconds == 0 { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "pool.max_connections must be at least 2 and acquire timeout must be positive" @@ -81,6 +82,7 @@ impl StoreConfig { pub fn resolve_and_validate(&mut self, runtime_data_dir: &Path) -> StoreResult<()> { if runtime_data_dir.as_os_str().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "runtime_data_dir must not be empty".to_owned(), }); @@ -107,6 +109,7 @@ impl StoreConfig { self.pool.validate()?; if self.backup_staging_dir.as_os_str().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "runtime_data_dir was not resolved".to_owned(), }); @@ -132,12 +135,14 @@ impl StoreConfig { pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult> { match std::env::var(name) { Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("environment variable {name} is empty"), }), Ok(value) => Ok(Some(value)), Err(std::env::VarError::NotPresent) => Ok(None), Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("environment variable {name} is not Unicode"), }), @@ -167,6 +172,7 @@ impl StoreConnectionConfig { if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.password must be exactly 48 hexadecimal characters"), }); @@ -176,17 +182,20 @@ impl StoreConnectionConfig { fn connection_url(&self, field: &'static str) -> StoreResult { let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url is invalid"), })?; if url.password().is_some() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url must not contain a password"), }); } url.set_password(Some(&self.password)) .map_err(|()| StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url cannot carry credentials"), })?; diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index fcae8fe90..04022aa5b 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -93,7 +93,7 @@ impl PgAccountGroupRepository { let mut transaction = self.pool.begin().await.map_err(|source| { admin_store_error(ENTITY, unavailable("begin account group mutation", source)) })?; - let result = async { + let result: StoreResult<_> = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; mutation(&mut transaction).await?; append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?; @@ -108,12 +108,10 @@ impl PgAccountGroupRepository { admin_revision(revision) } Err(error) => { - transaction.rollback().await.map_err(|source| { - admin_store_error( - ENTITY, - unavailable("rollback account group mutation", source), - ) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(admin_store_error(ENTITY, error)) } } @@ -210,14 +208,18 @@ impl AccountGroupStore for PgAccountGroupRepository { .map(|row| { let group_id = AccountGroupId::new( row.try_get::("account_group_id") - .map_err(|_| invalid("invalid group ID"))?, + .map_err(|source| invalid("invalid group ID").with_source(source))?, ) - .map_err(|_| invalid("invalid group ID"))?; + .map_err(|source| invalid("invalid group ID").with_source(source))?; let default_slots = u64::try_from( row.try_get::("max_concurrent_per_account") - .map_err(|_| invalid("invalid default account concurrency"))?, + .map_err(|source| { + invalid("invalid default account concurrency").with_source(source) + })?, ) - .map_err(|_| invalid("invalid default account concurrency"))?; + .map_err(|source| { + invalid("invalid default account concurrency").with_source(source) + })?; let account = account_summary_from_row(row)?; Ok(AccountGroupMemberFact { group_id, @@ -459,7 +461,8 @@ async fn count_groups(pool: &PgPool, query: &AccountGroupListQuery) -> StoreResu .fetch_one(pool) .await .map_err(|source| unavailable("count account groups", source))?; - u64::try_from(count).map_err(|_| invalid("negative account group count")) + u64::try_from(count) + .map_err(|source| invalid("negative account group count").with_source(source)) } async fn load_record(pool: &PgPool, id: &str) -> StoreResult> { @@ -479,33 +482,35 @@ async fn load_record(pool: &PgPool, id: &str) -> StoreResult StoreResult { let provider_counts: serde_json::Value = row .try_get("provider_counts") - .map_err(|_| invalid("invalid provider counts"))?; + .map_err(|source| invalid("invalid provider counts").with_source(source))?; let provider_counts = serde_json::from_value::>(provider_counts) - .map_err(|_| invalid("invalid provider counts"))?; + .map_err(|source| invalid("invalid provider counts").with_source(source))?; Ok(AccountGroupRecord { fast_mode: FastMode::parse( row.try_get("fast_mode") - .map_err(|_| invalid("invalid fast_mode"))?, + .map_err(|source| invalid("invalid fast_mode").with_source(source))?, ) .ok_or_else(|| invalid("invalid fast_mode"))?, id: AccountGroupId::new( row.try_get::("id") - .map_err(|_| invalid("invalid id"))?, + .map_err(|source| invalid("invalid id").with_source(source))?, ) - .map_err(|_| invalid("invalid id"))?, - name: row.try_get("name").map_err(|_| invalid("invalid name"))?, + .map_err(|source| invalid("invalid id").with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid("invalid name").with_source(source))?, description: row .try_get("description") - .map_err(|_| invalid("invalid description"))?, + .map_err(|source| invalid("invalid description").with_source(source))?, color: AccountGroupColor::parse( row.try_get::("color") - .map_err(|_| invalid("invalid color"))? + .map_err(|source| invalid("invalid color").with_source(source))? .as_str(), ) .ok_or_else(|| invalid("invalid color"))?, enabled: row .try_get("enabled") - .map_err(|_| invalid("invalid enabled"))?, + .map_err(|source| invalid("invalid enabled").with_source(source))?, member_count: count_value(row, "member_count")?, provider_counts, client_key_count: count_value(row, "client_key_count")?, @@ -519,16 +524,17 @@ fn group_record(row: &sqlx::postgres::PgRow) -> StoreResult total_slots: Some(0), }, usage: AccountGroupUsage { - today_usd: DecimalAmount::from_str("0").map_err(|_| invalid("invalid zero cost"))?, + today_usd: DecimalAmount::from_str("0") + .map_err(|source| invalid("invalid zero cost").with_source(source))?, retained_total_usd: DecimalAmount::from_str("0") - .map_err(|_| invalid("invalid zero cost"))?, + .map_err(|source| invalid("invalid zero cost").with_source(source))?, }, created_at: row .try_get("created_at") - .map_err(|_| invalid("invalid created_at"))?, + .map_err(|source| invalid("invalid created_at").with_source(source))?, updated_at: row .try_get("updated_at") - .map_err(|_| invalid("invalid updated_at"))?, + .map_err(|source| invalid("invalid updated_at").with_source(source))?, }) } @@ -580,20 +586,21 @@ async fn group_costs( .map(|row| { let group_id: String = row .try_get("group_id") - .map_err(|_| invalid("invalid group ID"))?; + .map_err(|source| invalid("invalid group ID").with_source(source))?; let today = row .try_get::("today_usd") - .map_err(|_| invalid("invalid today cost"))?; + .map_err(|source| invalid("invalid today cost").with_source(source))?; let retained_total = row .try_get::("retained_total_usd") - .map_err(|_| invalid("invalid retained total cost"))?; + .map_err(|source| invalid("invalid retained total cost").with_source(source))?; Ok(( group_id, AccountGroupUsage { today_usd: DecimalAmount::from_str(&today) - .map_err(|_| invalid("invalid today cost"))?, - retained_total_usd: DecimalAmount::from_str(&retained_total) - .map_err(|_| invalid("invalid retained total cost"))?, + .map_err(|source| invalid("invalid today cost").with_source(source))?, + retained_total_usd: DecimalAmount::from_str(&retained_total).map_err( + |source| invalid("invalid retained total cost").with_source(source), + )?, }, )) }) @@ -603,9 +610,9 @@ async fn group_costs( fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult { u64::try_from( row.try_get::(field) - .map_err(|_| invalid("invalid count"))?, + .map_err(|source| invalid("invalid count").with_source(source))?, ) - .map_err(|_| invalid("negative count")) + .map_err(|source| invalid("negative count").with_source(source)) } fn validate_page_query(query: &AccountGroupListQuery) -> AdminStoreResult<()> { @@ -658,6 +665,7 @@ fn not_found(id: &str) -> AdminStoreError { fn not_found_store(id: &str) -> StoreError { StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), } @@ -665,6 +673,7 @@ fn not_found_store(id: &str) -> StoreError { fn conflict(id: &str) -> StoreError { StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: ConflictKind::InvalidTransition, @@ -677,6 +686,7 @@ fn invalid_admin(message: &str) -> AdminStoreError { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } @@ -695,7 +705,7 @@ pub(crate) async fn insert_account_group_in_transaction( command: &NewAccountGroup, ) -> StoreResult<()> { validate_group_fields(&command.name, command.description.as_deref()) - .map_err(|_| invalid("invalid account group fields"))?; + .map_err(|source| invalid("invalid account group fields").with_source(source))?; sqlx::query( "insert into account_groups (id, name, description, color, fast_mode, enabled, created_at, updated_at) diff --git a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs index ad8d5f2eb..814589c66 100644 --- a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs +++ b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs @@ -214,8 +214,9 @@ pub(crate) async fn append_admin_audit_event_in_transaction( event.config_revision = revision .into() .map(|revision| { - i64::try_from(revision.get()) - .map_err(|_| invalid("config revision exceeds PostgreSQL bigint")) + i64::try_from(revision.get()).map_err(|source| { + invalid("config revision exceeds PostgreSQL bigint").with_source(source) + }) }) .transpose()?; event.validate()?; @@ -244,6 +245,7 @@ pub(crate) async fn append_admin_audit_event_in_transaction( fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/admission_recovery.rs b/backend/crates/gateway-store/src/postgres/admission_recovery.rs index 217bf4c96..98ceaedea 100644 --- a/backend/crates/gateway-store/src/postgres/admission_recovery.rs +++ b/backend/crates/gateway-store/src/postgres/admission_recovery.rs @@ -110,18 +110,18 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository { Box::pin(async move { self.load_client_admission_recovery(DateTime::::from(since)) .await - .map_err(|_| ClientAdmissionError)? + .map_err(|source| ClientAdmissionError(Some(source.into())))? .into_iter() .map(|recovery| { let client_api_key_id = ClientApiKeyId::new(recovery.client_api_key_ref) - .map_err(|_| ClientAdmissionError)?; + .map_err(|source| ClientAdmissionError(Some(source.into())))?; let recent_requests = recovery .recent_requests .into_iter() .map(|request| { Ok(RecentAdmissionFact { model_request_id: ModelRequestId::new(request.model_request_id) - .map_err(|_| ClientAdmissionError)?, + .map_err(|source| ClientAdmissionError(Some(source.into())))?, started_at: request.started_at.into(), }) }) @@ -132,7 +132,7 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository { .map(|request| { Ok(RunningAdmissionFact { model_request_id: ModelRequestId::new(request.model_request_id) - .map_err(|_| ClientAdmissionError)?, + .map_err(|source| ClientAdmissionError(Some(source.into())))?, expires_at: request.deadline_at.into(), }) }) diff --git a/backend/crates/gateway-store/src/postgres/backup.rs b/backend/crates/gateway-store/src/postgres/backup.rs index 243cf4b3d..8f4a0a83d 100644 --- a/backend/crates/gateway-store/src/postgres/backup.rs +++ b/backend/crates/gateway-store/src/postgres/backup.rs @@ -55,12 +55,10 @@ impl BackupRepository for PgBackupRepository { command: UpdateBackupStorageCommand, context: &MutationContext, ) -> AdminStoreResult<(BackupSettings, AdminRevision)> { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup storage update"))?; - let result = async { + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup storage update").with_source(source) + })?; + let result: AdminStoreResult<_> = async { let current = lock_settings_in_transaction(&mut transaction).await?; let mut changed = storage_changed_fields(¤t, &command); if changed.is_empty() { @@ -68,11 +66,14 @@ impl BackupRepository for PgBackupRepository { sqlx::query_scalar("select config_revision from runtime_settings where id = 1") .fetch_one(&mut *transaction) .await - .map_err(|_| store_unavailable("load unchanged backup revision"))?; + .map_err(|source| { + store_unavailable("load unchanged backup revision").with_source(source) + })?; let revision = AdminRevision::new( - u64::try_from(revision).map_err(|_| store_invalid("config revision"))?, + u64::try_from(revision) + .map_err(|source| store_invalid("config revision").with_source(source))?, ) - .map_err(|_| store_invalid("config revision"))?; + .map_err(|source| store_invalid("config revision").with_source(source))?; return Ok((current, revision)); } ensure_storage_identity_stable(¤t, &command, &mut transaction).await?; @@ -106,7 +107,7 @@ impl BackupRepository for PgBackupRepository { .bind(command.force_path_style) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("update backup storage"))?; + .map_err(|source| store_unavailable("update backup storage").with_source(source))?; let store_revision = bump_config_revision_in_transaction(&mut transaction) .await @@ -135,17 +136,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(result) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup storage update"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup storage update").with_source(source) + })?; Ok(result) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup storage update"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -158,11 +158,9 @@ impl BackupRepository for PgBackupRepository { context: &MutationContext, timezone: gateway_core::time::DeploymentTimeZone, ) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup schedule update"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup schedule update").with_source(source) + })?; let result = async { sqlx::query( "update backup_settings @@ -183,7 +181,7 @@ impl BackupRepository for PgBackupRepository { .bind(next_run_at) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("update backup schedule"))?; + .map_err(|source| store_unavailable("update backup schedule").with_source(source))?; let audit = crate::mutation_audit( context, @@ -205,17 +203,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(settings) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup schedule update"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup schedule update").with_source(source) + })?; Ok(settings) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup schedule update"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -231,20 +228,23 @@ impl BackupRepository for PgBackupRepository { set last_verified_at = $2, updated_at = now() where id = 1 and storage_revision = $1", ) - .bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?) + .bind( + i64::try_from(storage_revision) + .map_err(|source| store_invalid("storage revision").with_source(source))?, + ) .bind(at) .execute(&self.pool) .await - .map_err(|_| store_unavailable("record backup storage verification"))?; + .map_err(|source| { + store_unavailable("record backup storage verification").with_source(source) + })?; Ok(result.rows_affected() == 1) } async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup record insert"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup record insert").with_source(source) + })?; let result = async { let _ = lock_settings_in_transaction(&mut transaction).await?; insert_queued_in_transaction(&mut transaction, &seed).await @@ -252,17 +252,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(record) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup record insert"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup record insert").with_source(source) + })?; Ok(record) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup record insert"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -276,11 +275,10 @@ impl BackupRepository for PgBackupRepository { expected_timezone: &str, expected_next_run_at: Option>, ) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin scheduled insert"))?; + let mut transaction = + self.pool.begin().await.map_err(|source| { + store_unavailable("begin scheduled insert").with_source(source) + })?; // 先锁定并核对计划,防止配置变更后仍排入旧计划任务 let advanced = sqlx::query( "update backup_settings set next_run_at = $1 @@ -293,52 +291,47 @@ impl BackupRepository for PgBackupRepository { .bind(expected_next_run_at) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("guard scheduled insert"))? + .map_err(|source| store_unavailable("guard scheduled insert").with_source(source))? .rows_affected() > 0; if !advanced { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback stale schedule"))?; + transaction.rollback().await.map_err(|source| { + store_unavailable("rollback stale schedule").with_source(source) + })?; return Ok(false); } // 唯一约束冲突只跳过本次任务;保存点避免连带撤销游标推进 let mut insertion = transaction .begin() .await - .map_err(|_| store_unavailable("begin scheduled savepoint"))?; + .map_err(|source| store_unavailable("begin scheduled savepoint").with_source(source))?; let inserted = match insert_queued_in_transaction(&mut insertion, &seed).await { Ok(_) => { - insertion - .commit() - .await - .map_err(|_| store_unavailable("release scheduled savepoint"))?; + insertion.commit().await.map_err(|source| { + store_unavailable("release scheduled savepoint").with_source(source) + })?; true } Err(error) if is_active_or_scheduled_conflict(&error) => { - insertion - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled savepoint"))?; + insertion.rollback().await.map_err(|source| { + store_unavailable("rollback scheduled savepoint").with_source(source) + })?; false } - Err(error) => { - insertion - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled savepoint"))?; - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled insert"))?; + Err(mut error) => { + if let Err(cleanup) = insertion.rollback().await { + error = error.with_cleanup(cleanup); + } + if let Err(cleanup) = transaction.rollback().await { + error = error.with_cleanup(cleanup); + } return Err(error); } }; transaction .commit() .await - .map_err(|_| store_unavailable("commit scheduled insert"))?; + .map_err(|source| store_unavailable("commit scheduled insert").with_source(source))?; Ok(inserted) } @@ -361,7 +354,7 @@ impl BackupRepository for PgBackupRepository { .bind(trigger) .fetch_one(&self.pool) .await - .map_err(|_| store_unavailable("count backup records"))?; + .map_err(|source| store_unavailable("count backup records").with_source(source))?; let rows = sqlx::query( "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, @@ -376,10 +369,13 @@ impl BackupRepository for PgBackupRepository { .bind(status) .bind(trigger) .bind(i64::from(limit)) - .bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?) + .bind( + i64::try_from(offset) + .map_err(|source| store_invalid("page offset").with_source(source))?, + ) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list backup records"))?; + .map_err(|source| store_unavailable("list backup records").with_source(source))?; let items = rows .iter() @@ -387,7 +383,8 @@ impl BackupRepository for PgBackupRepository { .collect::, _>>()?; Ok(BackupRecordPage { items, - total: u64::try_from(total).map_err(|_| store_invalid("record count"))?, + total: u64::try_from(total) + .map_err(|source| store_invalid("record count").with_source(source))?, page: query.page, page_size: query.page_size, }) @@ -403,7 +400,7 @@ impl BackupRepository for PgBackupRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("load backup record"))? + .map_err(|source| store_unavailable("load backup record").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -419,7 +416,9 @@ impl BackupRepository for PgBackupRepository { ) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list intermediate backup records"))?; + .map_err(|source| { + store_unavailable("list intermediate backup records").with_source(source) + })?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -439,7 +438,7 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list pending deletions"))?; + .map_err(|source| store_unavailable("list pending deletions").with_source(source))?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -463,7 +462,7 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list expired backups"))?; + .map_err(|source| store_unavailable("list expired backups").with_source(source))?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -497,7 +496,7 @@ impl BackupRepository for PgBackupRepository { .bind(now) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("claim queued backup task"))? + .map_err(|source| store_unavailable("claim queued backup task").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -513,7 +512,7 @@ impl BackupRepository for PgBackupRepository { .size_bytes .map(i64::try_from) .transpose() - .map_err(|_| store_invalid("size bytes"))?; + .map_err(|source| store_invalid("size bytes").with_source(source))?; sqlx::query( "update backup_records set status = $1, @@ -542,7 +541,7 @@ impl BackupRepository for PgBackupRepository { .bind(transition.from().as_str()) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("transition backup task"))? + .map_err(|source| store_unavailable("transition backup task").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -564,7 +563,7 @@ impl BackupRepository for PgBackupRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("transition backup to deleting"))? + .map_err(|source| store_unavailable("transition backup to deleting").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -575,7 +574,7 @@ impl BackupRepository for PgBackupRepository { .execute(&self.pool) .await .map(|_| ()) - .map_err(|_| store_unavailable("delete backup record")) + .map_err(|source| store_unavailable("delete backup record").with_source(source)) } async fn advance_schedule_cursor( @@ -602,7 +601,9 @@ impl BackupRepository for PgBackupRepository { .bind(timezone) .execute(&self.pool) .await - .map_err(|_| store_unavailable("advance backup schedule cursor"))?; + .map_err(|source| { + store_unavailable("advance backup schedule cursor").with_source(source) + })?; Ok(result.rows_affected() == 1) } @@ -623,7 +624,9 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list scheduled completed backups"))?; + .map_err(|source| { + store_unavailable("list scheduled completed backups").with_source(source) + })?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -649,7 +652,7 @@ async fn lock_settings_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| store_unavailable("lock backup settings"))? + .map_err(|source| store_unavailable("lock backup settings").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -666,7 +669,7 @@ async fn load_settings_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| store_unavailable("load backup settings in transaction"))? + .map_err(|source| store_unavailable("load backup settings in transaction").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -681,7 +684,7 @@ async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult ) .fetch_optional(pool) .await - .map_err(|_| store_unavailable("load backup settings"))? + .map_err(|source| store_unavailable("load backup settings").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -761,7 +764,9 @@ async fn ensure_storage_identity_stable( sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)") .fetch_one(&mut **transaction) .await - .map_err(|_| store_unavailable("check backup records existence"))?; + .map_err(|source| { + store_unavailable("check backup records existence").with_source(source) + })?; if records_exist { return Err(AdminStoreError::new( AdminStoreErrorKind::Conflict, @@ -815,7 +820,8 @@ async fn append_audit_in_transaction( ) -> StoreResult<()> { event.config_revision = revision .map(|revision| { - i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData { + i64::try_from(revision.get()).map_err(|source| StoreError::InvalidData { + source: Some(source.into()), entity: "backup", message: "revision overflow".to_owned(), }) @@ -849,7 +855,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult, _>("size_bytes"))? .map(u64::try_from) .transpose() - .map_err(|_| store_invalid("size_bytes"))?; + .map_err(|source| store_invalid("size_bytes").with_source(source))?; Ok(BackupRecord { id: decode(row.try_get("id"))?, trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?) @@ -861,7 +867,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult("attempt_count"))?) - .map_err(|_| store_invalid("attempt_count"))?, + .map_err(|source| store_invalid("attempt_count").with_source(source))?, error_code: decode(row.try_get("error_code"))?, error_message: decode(row.try_get("error_message"))?, started_at: decode(row.try_get("started_at"))?, @@ -874,7 +880,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult AdminStoreResult { let storage_revision = u64::try_from(decode(row.try_get::("storage_revision"))?) - .map_err(|_| store_invalid("storage_revision"))?; + .map_err(|source| store_invalid("storage_revision").with_source(source))?; let secret = decode(row.try_get::, _>("secret_access_key"))?; Ok(BackupSettings { storage_revision, @@ -889,9 +895,9 @@ fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult("retention_days"))?) - .map_err(|_| store_invalid("retention_days"))?, + .map_err(|source| store_invalid("retention_days").with_source(source))?, retention_count: u32::try_from(decode(row.try_get::("retention_count"))?) - .map_err(|_| store_invalid("retention_count"))?, + .map_err(|source| store_invalid("retention_count").with_source(source))?, next_run_at: decode(row.try_get("next_run_at"))?, last_verified_at: decode(row.try_get("last_verified_at"))?, updated_at: decode(row.try_get("updated_at"))?, @@ -911,22 +917,7 @@ fn decode(result: Result) -> AdminStoreResult { /// StoreError → AdminStoreError 的显式映射 fn map_admin_error(error: StoreError) -> AdminStoreError { - match error { - StoreError::Unavailable { .. } => AdminStoreError::new( - AdminStoreErrorKind::Unavailable, - "backup", - "backup store unavailable", - ), - StoreError::NotFound { entity, .. } => { - AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found") - } - StoreError::Conflict { entity, .. } => { - AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict") - } - StoreError::InvalidData { entity, message } => { - AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message) - } - } + crate::admin_store_error("backup", error) } fn store_unavailable(operation: &'static str) -> AdminStoreError { diff --git a/backend/crates/gateway-store/src/postgres/client_budgets.rs b/backend/crates/gateway-store/src/postgres/client_budgets.rs index d1e7dad32..190880430 100644 --- a/backend/crates/gateway-store/src/postgres/client_budgets.rs +++ b/backend/crates/gateway-store/src/postgres/client_budgets.rs @@ -65,6 +65,7 @@ async fn reset_client_key_budget_in_transaction( .map_err(|source| postgres_unavailable("lock budget reset key", source))?; if exists.is_none() { return Err(StoreError::NotFound { + source: None, entity: "client API key", id: command.id.as_str().to_owned(), }); @@ -158,9 +159,15 @@ impl PgClientBudgetStore { .cloned() .collect::>(); for charge in retries { - self.settle(charge).await.map_err(|_| unavailable())?; + self.settle(charge) + .await + .map_err(|source| unavailable().with_source(source))?; } - let mut tx = self.pool.begin().await.map_err(|_| unavailable())?; + let mut tx = self + .pool + .begin() + .await + .map_err(|source| unavailable().with_source(source))?; let row = sqlx::query( "select daily_limit_usd::text, weekly_limit_usd::text, enabled from client_api_keys where id = $1 for update", @@ -168,7 +175,7 @@ impl PgClientBudgetStore { .bind(key_id.as_str()) .fetch_optional(&mut *tx) .await - .map_err(|_| unavailable())? + .map_err(|source| unavailable().with_source(source))? .ok_or_else(|| { GatewayError::new( GatewayErrorKind::Unauthorized, @@ -185,16 +192,16 @@ impl PgClientBudgetStore { daily_usd: row .get::("daily_limit_usd") .parse() - .map_err(|_| unavailable())?, + .map_err(|source| unavailable().with_source(source))?, weekly_usd: row .get::("weekly_limit_usd") .parse() - .map_err(|_| unavailable())?, + .map_err(|source| unavailable().with_source(source))?, }; let now = Utc::now(); advance_windows(&mut tx, key_id.as_str(), now, now, self.timezone) .await - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; if limits.is_limited() { let window = sqlx::query( "select daily_used_usd::text, weekly_used_usd::text, daily_end, weekly_end @@ -203,15 +210,15 @@ impl PgClientBudgetStore { .bind(key_id.as_str()) .fetch_one(&mut *tx) .await - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let daily: Decimal = window .get::("daily_used_usd") .parse() - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let weekly: Decimal = window .get::("weekly_used_usd") .parse() - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let daily_exceeded = limits.daily_usd != Decimal::ZERO && daily >= limits.daily_usd; let weekly_exceeded = limits.weekly_usd != Decimal::ZERO && weekly >= limits.weekly_usd; if daily_exceeded || weekly_exceeded { @@ -235,11 +242,17 @@ impl PgClientBudgetStore { .with_retry_after(retry)); } } - tx.commit().await.map_err(|_| unavailable()) + tx.commit() + .await + .map_err(|source| unavailable().with_source(source)) } async fn settle_inner(&self, charge: &ClientBudgetCharge) -> Result<(), ClientBudgetError> { - let mut tx = self.pool.begin().await.map_err(|_| ClientBudgetError)?; + let mut tx = self + .pool + .begin() + .await + .map_err(|source| ClientBudgetError(Some(source.into())))?; // 与准入统一先锁 Key,再写窗口和费用,串行化同一 Key 的并发结算 let key = sqlx::query_scalar::<_, String>( "select id from client_api_keys where id = $1 for update", @@ -247,12 +260,14 @@ impl PgClientBudgetStore { .bind(charge.key_id.as_str()) .fetch_optional(&mut *tx) .await - .map_err(|_| ClientBudgetError)?; + .map_err(|source| ClientBudgetError(Some(source.into())))?; let Some(key) = key else { return Ok(()) }; // 删除 Key 时也会删除其费用记录 settle_in_transaction(&mut tx, &key, charge, self.timezone) .await - .map_err(|_| ClientBudgetError)?; - tx.commit().await.map_err(|_| ClientBudgetError) + .map_err(|source| ClientBudgetError(Some(source.into())))?; + tx.commit() + .await + .map_err(|source| ClientBudgetError(Some(source.into()))) } } @@ -296,7 +311,14 @@ impl ClientBudgetPort for PgClientBudgetStore { fn settle(&self, charge: ClientBudgetCharge) -> BoxFuture<'_, Result<(), ClientBudgetError>> { Box::pin(async move { let result = self.settle_inner(&charge).await; - let mut retry = self.retry.lock().map_err(|_| ClientBudgetError)?; + // 重试缓存的锁异常不能覆盖已捕获的账本失败 + let mut retry = self.retry.lock().map_err(|_| { + result + .as_ref() + .err() + .cloned() + .unwrap_or(ClientBudgetError(None)) + })?; if result.is_err() { retry.insert(charge.request_id.as_str().to_owned(), charge); } else { diff --git a/backend/crates/gateway-store/src/postgres/client_keys.rs b/backend/crates/gateway-store/src/postgres/client_keys.rs index 86dd97045..fbe4baeb9 100644 --- a/backend/crates/gateway-store/src/postgres/client_keys.rs +++ b/backend/crates/gateway-store/src/postgres/client_keys.rs @@ -74,13 +74,17 @@ impl ClientApiKeySnapshot { ) -> StoreResult { Ok(Self { request_profiles: std::collections::BTreeMap::new(), - id: ClientApiKeyId::new(id).map_err(|_| invalid("persisted key ID is invalid"))?, - plaintext_key: PlaintextClientApiKey::new(key) - .map_err(|_| invalid("persisted plaintext key is invalid"))?, + id: ClientApiKeyId::new(id) + .map_err(|source| invalid("persisted key ID is invalid").with_source(source))?, + plaintext_key: PlaintextClientApiKey::new(key).map_err(|source| { + invalid("persisted plaintext key is invalid").with_source(source) + })?, group_ids: group_ids .into_iter() .map(|id| { - AccountGroupId::new(id).map_err(|_| invalid("persisted group ID is invalid")) + AccountGroupId::new(id).map_err(|source| { + invalid("persisted group ID is invalid").with_source(source) + }) }) .collect::>>()?, limits: RateLimits { @@ -703,6 +707,7 @@ impl PgAdminClientKeyStore { admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: ENTITY, id: id.as_str().to_owned(), }, @@ -748,6 +753,7 @@ impl ClientKeyStore for PgAdminClientKeyStore { .map_err(|source| map_error(postgres_unavailable("lock budget limits key", source)))?; let (daily, weekly) = current.ok_or_else(|| { map_error(StoreError::NotFound { + source: None, entity: ENTITY, id: command.id.as_str().to_owned(), }) @@ -1159,6 +1165,7 @@ pub(crate) async fn insert_client_api_key_in_transaction( && database.constraint() == Some("client_api_keys_key_sha256_idx") }) { StoreError::Conflict { + source: None, entity: ENTITY, id: key.id.clone(), kind: crate::ConflictKind::InvalidTransition, @@ -1238,6 +1245,7 @@ async fn ensure_client_key_name_available( .map_err(|source| postgres_unavailable("check client API key name", source))?; if duplicate { return Err(StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: crate::ConflictKind::DuplicateName, @@ -1282,6 +1290,7 @@ fn require_changed(rows_affected: u64, id: &str) -> StoreResult<()> { Ok(()) } else { Err(StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), }) @@ -1304,6 +1313,7 @@ async fn replace_client_api_key_groups_in_transaction( .map_err(|source| postgres_unavailable("validate client API key groups", source))?; if usize::try_from(count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { + source: None, entity: "account group", id: "client API key group selection".to_owned(), }); @@ -1342,8 +1352,9 @@ fn validate_group_ids(group_ids: &[String]) -> StoreResult<()> { } fn validate_key(key: &str) -> StoreResult<()> { - PlaintextClientApiKey::validate(key) - .map_err(|_| invalid("key must contain nonempty visible ASCII characters")) + PlaintextClientApiKey::validate(key).map_err(|source| { + invalid("key must contain nonempty visible ASCII characters").with_source(source) + }) } fn validate_request_profiles<'a>( @@ -1381,7 +1392,8 @@ fn decode_request_profiles( .into_iter() .map(|(provider, profile)| { Ok(( - ProviderKind::new(provider).map_err(|_| invalid("invalid request profile key"))?, + ProviderKind::new(provider) + .map_err(|source| invalid("invalid request profile key").with_source(source))?, OpaqueProviderData::new(profile), )) }) @@ -1404,7 +1416,7 @@ fn client_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult StoreResult>, >, _>("request_profile_overrides") - .map_err(|_| invalid("invalid request profiles"))? + .map_err(|source| invalid("invalid request profiles").with_source(source))? .0, )?, - id: row.try_get("id").map_err(|_| invalid("invalid id"))?, - name: row.try_get("name").map_err(|_| invalid("invalid name"))?, - label: row.try_get("label").map_err(|_| invalid("invalid label"))?, - groups: serde_json::from_value(groups).map_err(|_| invalid("invalid groups"))?, + id: row + .try_get("id") + .map_err(|source| invalid("invalid id").with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid("invalid name").with_source(source))?, + label: row + .try_get("label") + .map_err(|source| invalid("invalid label").with_source(source))?, + groups: serde_json::from_value(groups) + .map_err(|source| invalid("invalid groups").with_source(source))?, provider_kinds: row .try_get("provider_kinds") - .map_err(|_| invalid("invalid provider kinds"))?, + .map_err(|source| invalid("invalid provider kinds").with_source(source))?, prefix: row .try_get("prefix") - .map_err(|_| invalid("invalid prefix"))?, + .map_err(|source| invalid("invalid prefix").with_source(source))?, enabled: row .try_get("enabled") - .map_err(|_| invalid("invalid enabled"))?, + .map_err(|source| invalid("invalid enabled").with_source(source))?, max_concurrency: to_u64( row.try_get("max_concurrency") - .map_err(|_| invalid("invalid max concurrency"))?, + .map_err(|source| invalid("invalid max concurrency").with_source(source))?, )?, requests_per_minute: to_u64( row.try_get("requests_per_minute") - .map_err(|_| invalid("invalid requests per minute"))?, + .map_err(|source| invalid("invalid requests per minute").with_source(source))?, )?, budget: ClientBudgetStatus::default(), last_used_at: row .try_get("last_used_at") - .map_err(|_| invalid("invalid last used at"))?, + .map_err(|source| invalid("invalid last used at").with_source(source))?, created_at: row .try_get("created_at") - .map_err(|_| invalid("invalid created at"))?, + .map_err(|source| invalid("invalid created at").with_source(source))?, updated_at: row .try_get("updated_at") - .map_err(|_| invalid("invalid updated at"))?, + .map_err(|source| invalid("invalid updated at").with_source(source))?, }, ) } @@ -1525,34 +1544,38 @@ async fn load_client_key_memberships( use sqlx::Row as _; let key_id: String = row .try_get("key_id") - .map_err(|_| invalid("invalid client API key membership"))?; + .map_err(|source| invalid("invalid client API key membership").with_source(source))?; let group_id: Option = row .try_get("group_id") - .map_err(|_| invalid("invalid client API key group"))?; + .map_err(|source| invalid("invalid client API key group").with_source(source))?; if let Some(group_id) = group_id { groups .entry(key_id.clone()) .or_default() .push(ClientApiKeyGroupRecord { id: group_id, - name: row - .try_get("group_name") - .map_err(|_| invalid("invalid client API key group name"))?, - color: row - .try_get("group_color") - .map_err(|_| invalid("invalid client API key group color"))?, - enabled: row - .try_get("group_enabled") - .map_err(|_| invalid("invalid client API key group state"))?, + name: row.try_get("group_name").map_err(|source| { + invalid("invalid client API key group name").with_source(source) + })?, + color: row.try_get("group_color").map_err(|source| { + invalid("invalid client API key group color").with_source(source) + })?, + enabled: row.try_get("group_enabled").map_err(|source| { + invalid("invalid client API key group state").with_source(source) + })?, }); providers.entry(key_id).or_default().extend( row.try_get::, _>("group_provider_kinds") - .map_err(|_| invalid("invalid client API key provider kinds"))?, + .map_err(|source| { + invalid("invalid client API key provider kinds").with_source(source) + })?, ); } else { providers.entry(key_id).or_default().extend( row.try_get::, _>("global_provider_kinds") - .map_err(|_| invalid("invalid global provider kinds"))?, + .map_err(|source| { + invalid("invalid global provider kinds").with_source(source) + })?, ); } } @@ -1649,15 +1672,18 @@ fn push_client_key_order(statement: &mut QueryBuilder, sort: ClientApi } fn to_i64(value: u64) -> StoreResult { - i64::try_from(value).map_err(|_| invalid("numeric limit exceeds PostgreSQL bigint")) + i64::try_from(value) + .map_err(|source| invalid("numeric limit exceeds PostgreSQL bigint").with_source(source)) } fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("persisted numeric limit is negative")) + u64::try_from(value) + .map_err(|source| invalid("persisted numeric limit is negative").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index d77577ce6..9ea7c305f 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -631,7 +631,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(attempt.http_version) .bind( i32::try_from(attempt.attempt_count) - .map_err(|_| invalid("attempt_count is too large"))?, + .map_err(|source| invalid("attempt_count is too large").with_source(source))?, ) .bind(optional_i64( attempt.account_selection_wait_ms, @@ -649,11 +649,13 @@ impl ModelRequestRepository for PgExecutionStore { .await .map_err(|source| postgres_unavailable("begin model request attempt", source))? .ok_or(StoreError::Conflict { + source: None, entity: ENTITY, id: attempt.model_request_id, kind: ConflictKind::DownstreamAlreadyCommitted, })?; - u32::try_from(count).map_err(|_| invalid("attempt_count is invalid")) + u32::try_from(count) + .map_err(|source| invalid("attempt_count is invalid").with_source(source)) } async fn mark_upstream_send_state( @@ -847,7 +849,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(finalization.upstream_send_state.as_str()) .bind( i32::try_from(finalization.attempt_count) - .map_err(|_| invalid("attempt_count is too large"))?, + .map_err(|source| invalid("attempt_count is too large").with_source(source))?, ) .bind(finalization.downstream_committed_at) .bind(finalization.client_status_code.map(i32::from)) @@ -992,6 +994,7 @@ impl ExecutionStore for PgExecutionStore { Box::pin(async move { let ttl_ms = i64::try_from(ttl.as_millis()).map_err(|_| { crate::StoreError::InvalidData { + source: None, entity: "model request", message: "lease TTL is invalid".to_owned(), } @@ -1172,7 +1175,9 @@ impl ExecutionStore for PgExecutionStore { upstream_model_id: None, failure_kind: error.kind().as_str().to_owned(), upstream_send_state: None, - error_details: None, + error_details: error + .error_details() + .map(gateway_core::error::ErrorDetails::into_string), status_code: None, provider_error_code: error.client_error_code().map(str::to_owned), retry_after_ms: error @@ -1596,12 +1601,55 @@ fn validate_connection_observation(finalization: &ModelRequestFinalization) -> S } fn to_i64(value: u64, field: &'static str) -> StoreResult { - i64::try_from(value).map_err(|_| invalid(field)) + i64::try_from(value).map_err(|source| invalid(field).with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } } + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for PgExecutionStore { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), CoreStoreError> { + use super::{OpsEvent, OpsEventLevel, OpsEventRepository, PgOpsEventRepository}; + let event = OpsEvent { + id: Uuid::now_v7().to_string(), + model_request_id: None, + attempt_index: None, + level: OpsEventLevel::Warning, + component: failure.component.to_owned(), + operation: failure.operation.to_owned(), + provider_kind: failure.provider_kind.map(|kind| kind.as_str().to_owned()), + provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()), + provider_account_ref: failure.account_id.map(|id| id.as_str().to_owned()), + upstream_model_id: None, + failure_kind: failure.kind.to_owned(), + upstream_send_state: None, + error_details: failure + .details + .map(gateway_core::error::ErrorDetails::into_string), + status_code: failure.upstream_status, + provider_error_code: failure.upstream_code.map(|code| code.as_str().to_owned()), + retry_after_ms: None, + upstream_request_id: None, + latency_ms: None, + message: serde_json::json!({ + "correlationId": failure.correlation_id, + "message": failure.message, + }) + .to_string(), + created_at: failure.occurred_at.into(), + }; + PgOpsEventRepository::new(self.pool.clone()) + .append_ops_event(event) + .await + .map_err(core_store_error) + } +} diff --git a/backend/crates/gateway-store/src/postgres/execution_buffer.rs b/backend/crates/gateway-store/src/postgres/execution_buffer.rs index faac9b2a2..61990b64c 100644 --- a/backend/crates/gateway-store/src/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/src/postgres/execution_buffer.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant, SystemTime}; use async_trait::async_trait; use futures::future::try_join_all; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; use gateway_core::engine::{ AttemptRecord, EntryRejection, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, @@ -41,8 +42,8 @@ pub struct ExecutionBufferStats { } struct ExecutionBufferState { - maximum_queued_items: usize, - maximum_queued_bytes: usize, + regular_item_capacity: usize, + regular_byte_capacity: usize, queued_items: AtomicUsize, queued_bytes: AtomicUsize, enqueued_total: AtomicU64, @@ -53,10 +54,10 @@ struct ExecutionBufferState { } impl ExecutionBufferState { - fn new(maximum_queued_items: NonZeroUsize, maximum_queued_bytes: NonZeroUsize) -> Self { + fn new(regular_item_capacity: NonZeroUsize, regular_byte_capacity: NonZeroUsize) -> Self { Self { - maximum_queued_items: maximum_queued_items.get(), - maximum_queued_bytes: maximum_queued_bytes.get(), + regular_item_capacity: regular_item_capacity.get(), + regular_byte_capacity: regular_byte_capacity.get(), queued_items: AtomicUsize::new(0), queued_bytes: AtomicUsize::new(0), enqueued_total: AtomicU64::new(0), @@ -67,12 +68,21 @@ impl ExecutionBufferState { } } - fn reserve(&self, bytes: usize) -> Result<(), ReservationFailure> { + fn reserve(&self, bytes: usize, critical: bool) -> Result<(), ReservationFailure> { + // 常规额度之外保留四分之一给失败与请求生命周期,普通进度更新不能消耗 + let item_limit = self.regular_item_capacity.saturating_add(if critical { + self.regular_item_capacity / 4 + } else { + 0 + }); + let byte_limit = self.regular_byte_capacity.saturating_add(if critical { + self.regular_byte_capacity / 4 + } else { + 0 + }); self.queued_items .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { - current - .checked_add(1) - .filter(|next| *next <= self.maximum_queued_items) + current.checked_add(1).filter(|next| *next <= item_limit) }) .map_err(|_| ReservationFailure::ItemCapacity)?; let bytes_reserved = self @@ -80,7 +90,7 @@ impl ExecutionBufferState { .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { current .checked_add(bytes) - .filter(|next| *next <= self.maximum_queued_bytes) + .filter(|next| *next <= byte_limit) }) .is_ok(); if !bytes_reserved { @@ -189,16 +199,17 @@ impl BufferedExecutionStore { ) } + /// 参数限制常规接收额度;失败与生命周期记录可使用额外四分之一预留 #[must_use] pub fn with_limits( inner: Arc, capacity: NonZeroUsize, - maximum_queued_bytes: NonZeroUsize, + regular_byte_capacity: NonZeroUsize, ) -> (Self, ExecutionObservationWriter) { let lane_count = PERSISTENCE_LANES.min(capacity.get()); let (senders, receivers): (Vec<_>, Vec<_>) = (0..lane_count).map(|_| mpsc::unbounded_channel()).unzip(); - let state = Arc::new(ExecutionBufferState::new(capacity, maximum_queued_bytes)); + let state = Arc::new(ExecutionBufferState::new(capacity, regular_byte_capacity)); ( Self { inner: Arc::clone(&inner), @@ -224,16 +235,19 @@ impl BufferedExecutionStore { .estimated_bytes() .saturating_add(write.request_id().map_or(0, str::len)) .max(1); - if let Err(failure) = self.state.reserve(estimated_bytes) { + if let Err(failure) = self.state.reserve(estimated_bytes, write.is_critical()) { self.state.record_dropped(1); let stats = self.state.snapshot(); + if !stats.dropped_total.is_power_of_two() { + return; + } tracing::warn!( operation = write.operation(), request_id = ?write.request_id(), reason = failure.reason(), estimated_bytes, - maximum_queued_items = self.state.maximum_queued_items, - maximum_queued_bytes = self.state.maximum_queued_bytes, + regular_item_capacity = self.state.regular_item_capacity, + regular_byte_capacity = self.state.regular_byte_capacity, queued_items = stats.queued_items, queued_bytes = stats.queued_bytes, dropped_total = stats.dropped_total, @@ -253,6 +267,9 @@ impl BufferedExecutionStore { drop(queued); self.state.record_dropped(1); let stats = self.state.snapshot(); + if !stats.dropped_total.is_power_of_two() { + return; + } tracing::warn!( operation, request_id = ?request_id, @@ -285,10 +302,20 @@ fn request_lane(request_id: &str, lane_count: usize) -> usize { usize::try_from(hash % u64::try_from(lane_count).unwrap_or(u64::MAX)).unwrap_or(0) } +#[async_trait] +impl OperationalDiagnostics for BufferedExecutionStore { + async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError> { + self.enqueue(ExecutionObservationWrite::OperationalFailure(Box::new( + failure, + ))); + Ok(()) + } +} + #[async_trait] impl ExecutionStore for BufferedExecutionStore where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { fn maintain_request( &self, @@ -455,7 +482,7 @@ impl ExecutionBufferIdle { impl DaemonTask for ExecutionObservationWriter where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { fn run( &self, @@ -488,7 +515,7 @@ async fn run_lane( shutdown_deadline: &OnceLock, ) -> Result<(), WorkerTaskError> where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { let mut receiver = receiver.lock().await; loop { @@ -548,7 +575,7 @@ fn shared_shutdown_deadline(deadline: &OnceLock) -> Instant { async fn persist_queued(mut queued: QueuedExecutionObservation, store: &S) where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { let operation = queued.operation(); let request_id = queued.request_id().map(ToOwned::to_owned); @@ -570,6 +597,9 @@ where Err(error) => { state.record_write_failure(); let stats = state.snapshot(); + if !stats.write_failure_total.is_power_of_two() { + return; + } tracing::warn!( operation, request_id = ?request_id, @@ -588,7 +618,7 @@ async fn drain_on_shutdown( state: &ExecutionBufferState, deadline: Instant, ) where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { receiver.close(); let queued_at_shutdown = receiver.len(); @@ -695,10 +725,20 @@ enum ExecutionObservationWrite { IntermediateFailure(Box), ProbeFailure(Box), EntryRejection(Box), + OperationalFailure(Box), Finalize(Box), } impl ExecutionObservationWrite { + const fn is_critical(&self) -> bool { + !matches!( + self, + Self::MarkSendState { .. } + | Self::MarkDownstreamCommitted { .. } + | Self::RecordClientStatus { .. } + ) + } + const fn operation(&self) -> &'static str { match self { Self::Create(_) => "create_model_request", @@ -710,6 +750,7 @@ impl ExecutionObservationWrite { Self::IntermediateFailure(_) => "record_intermediate_failure", Self::ProbeFailure(_) => "record_probe_failure", Self::EntryRejection(_) => "record_entry_rejection", + Self::OperationalFailure(_) => "record_operational_failure", Self::Finalize(_) => "finalize_model_request", } } @@ -724,6 +765,7 @@ impl ExecutionObservationWrite { | Self::RecordClientStatus { request_id, .. } => Some(request_id.as_str()), Self::IntermediateFailure(failure) => Some(failure.request_id.as_str()), Self::ProbeFailure(_) | Self::EntryRejection(_) => None, + Self::OperationalFailure(failure) => failure.correlation_id.as_deref(), Self::Finalize(finalization) => Some(finalization.request_id.as_str()), } } @@ -756,13 +798,26 @@ impl ExecutionObservationWrite { rejection.error.client_error_code(), rejection.error.client_error_type(), ]) - .saturating_add(size_of::()), + .saturating_add(size_of::()) + .saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes( + std::error::Error::source(&rejection.error), + )), Self::ProbeFailure(failure) => text_bytes([ Some(failure.provider_kind.as_str()), Some(failure.account_id.as_str()), Some(failure.upstream_model_id.as_str()), ]) .saturating_add(provider_error_bytes(&failure.error)), + Self::OperationalFailure(failure) => { + size_of::().saturating_add(text_bytes([ + Some(failure.message.as_str()), + failure.correlation_id.as_deref(), + failure.provider_kind.as_ref().map(|kind| kind.as_str()), + failure.account_id.as_ref().map(|id| id.as_str()), + failure.upstream_code.as_ref().map(|code| code.as_str()), + failure.details.as_ref().map(|details| details.as_str()), + ])) + } Self::Finalize(finalization) => { let error_bytes = finalization.error.as_ref().map_or(0, |error| { text_bytes([ @@ -770,6 +825,11 @@ impl ExecutionObservationWrite { error.client_error_code(), error.client_error_type(), ]) + .saturating_add( + gateway_core::error::ErrorSource::estimated_chain_bytes( + std::error::Error::source(error), + ), + ) }); text_bytes([ Some(finalization.request_id.as_str()), @@ -794,9 +854,10 @@ impl ExecutionObservationWrite { async fn persist(self, store: &S) -> Result<(), StoreError> where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { match self { + Self::OperationalFailure(failure) => store.record_failure(*failure).await, Self::Create(request) => store.create_model_request(*request).await, Self::Attempt(attempt) => store.record_attempt(*attempt).await, Self::CreateWithAttempt(write) => { @@ -900,33 +961,9 @@ fn provider_error_bytes(error: &ProviderError) -> usize { .saturating_add(header.value().len()); } } - bytes.saturating_add(error_source_bytes(error.source())) -} - -/// 原始来源也占据队列内存;只计数而不保存格式化内容,避免复制或打印敏感原因 -fn error_source_bytes(mut source: Option<&(dyn std::error::Error + 'static)>) -> usize { - struct ByteCount(usize); - impl std::fmt::Write for ByteCount { - fn write_str(&mut self, value: &str) -> std::fmt::Result { - self.0 = self.0.saturating_add(value.len()); - Ok(()) - } - } - let mut bytes = ByteCount(0); - for _ in 0..32 { - let Some(error) = source else { return bytes.0 }; - bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error)); - if std::fmt::write(&mut bytes, format_args!("{error}")).is_err() { - return usize::MAX; - } - source = error.source(); - } - // 过深或循环的第三方来源不能让观测写入无限遍历,交给已有超限丢弃计数 - if source.is_some() { - usize::MAX - } else { - bytes.0 - } + bytes.saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes( + error.source(), + )) } fn text_bytes(values: [Option<&str>; N]) -> usize { diff --git a/backend/crates/gateway-store/src/postgres/mod.rs b/backend/crates/gateway-store/src/postgres/mod.rs index d83a67eda..36e5773c1 100644 --- a/backend/crates/gateway-store/src/postgres/mod.rs +++ b/backend/crates/gateway-store/src/postgres/mod.rs @@ -60,6 +60,7 @@ pub async fn connect_and_migrate( ) -> StoreResult { if database_url.trim().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "PostgreSQL configuration", message: "database URL is empty".to_owned(), }); @@ -225,7 +226,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository { self.pool.begin().await.map_err(|source| { postgres_unavailable("begin control plane replacement", source) })?; - let result = async { + let result: StoreResult<_> = async { // 锁住所有配置写入共同使用的行,版本检查与修改不能被其他事务穿插 let current = sqlx::query_scalar::<_, i64>( "select config_revision from runtime_settings where id = 1 for update", @@ -235,6 +236,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository { .map_err(|source| postgres_unavailable("lock control plane revision", source))?; if u64::try_from(current).ok() != Some(replacement.expected_revision.get()) { return Err(StoreError::Conflict { + source: None, entity: "runtime settings", id: "1".to_owned(), kind: crate::ConflictKind::StaleRevision, @@ -256,9 +258,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository { Ok(snapshot) } Err(error) => { - transaction.rollback().await.map_err(|source| { - postgres_unavailable("rollback control plane replacement", source) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -337,7 +340,7 @@ impl PgControlPlaneRepository { let mut transaction = self.pool.begin().await.map_err(|source| { postgres_unavailable("begin targeted control plane mutation", source) })?; - let result = async { + let result: StoreResult<_> = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; match mutation { ControlPlaneMutation::CreateClientApiKey(key) => { @@ -394,9 +397,10 @@ impl PgControlPlaneRepository { Ok(revision) } Err(error) => { - transaction.rollback().await.map_err(|source| { - postgres_unavailable("rollback targeted control plane mutation", source) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 427c3c6b3..ce36cb383 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -212,6 +212,7 @@ pub(crate) fn admin_latency_percentiles( pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError { observability_error(StoreError::InvalidData { + source: None, entity: "observability latency percentile", message: "latency percentile is outside the admin contract".to_owned(), }) @@ -265,6 +266,7 @@ pub(crate) fn admin_decimal_amount( ) -> AdminStoreResult { admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| { observability_error(StoreError::InvalidData { + source: None, entity: "observability decimal amount", message: "amount is outside the admin numeric contract".to_owned(), }) @@ -449,6 +451,7 @@ pub(crate) fn admin_usage_list_record( (None, None) => None, _ => { return Err(observability_error(StoreError::InvalidData { + source: None, entity: "observability request billing", message: "cost amount and currency must be present together".to_owned(), })); @@ -522,6 +525,7 @@ pub(crate) fn admin_usage_record( (None, None) => None, _ => { return Err(observability_error(StoreError::InvalidData { + source: None, entity: "observability request billing", message: "cost amount and currency must be present together".to_owned(), })); @@ -610,6 +614,7 @@ pub(crate) fn admin_request_outcome( ) -> AdminStoreResult { admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| { observability_error(StoreError::InvalidData { + source: None, entity: "observability request outcome", message: "invalid request outcome".to_owned(), }) @@ -1042,7 +1047,7 @@ pub(crate) fn opaque_response_id( get::>>(row, column)? .map(String::from_utf8) .transpose() - .map_err(|_| invalid(column)) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> { @@ -1075,7 +1080,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) - where T: sqlx::Decode<'r, Postgres> + sqlx::Type, { - row.try_get(column).map_err(|_| invalid(column)) + row.try_get(column) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult { @@ -1104,15 +1110,18 @@ pub(crate) fn optional_status( } pub(crate) fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("numeric observation is negative")) + u64::try_from(value) + .map_err(|source| invalid("numeric observation is negative").with_source(source)) } pub(crate) fn to_u32(value: i32) -> StoreResult { - u32::try_from(value).map_err(|_| invalid("integer observation is negative")) + u32::try_from(value) + .map_err(|source| invalid("integer observation is negative").with_source(source)) } pub(crate) fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "observability", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index 6c3aa6829..3fa1402bd 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -256,6 +256,7 @@ pub(crate) async fn usage_record_detail( .await .map_err(|source| postgres_unavailable("load usage record detail", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "model request", id: request_id.to_owned(), })?; diff --git a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs index 7ed466534..1c2f24c0a 100644 --- a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs +++ b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs @@ -23,11 +23,13 @@ impl ObservabilityQueryBudget { pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult { if max_connections == 0 || wait_timeout.is_zero() { return Err(StoreError::InvalidData { + source: None, entity: "observability query budget", message: "requires a positive connection count and wait timeout".to_owned(), }); } let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData { + source: None, entity: "observability query budget", message: "connection budget does not fit this platform".to_owned(), })?; diff --git a/backend/crates/gateway-store/src/postgres/ops_events.rs b/backend/crates/gateway-store/src/postgres/ops_events.rs index d16548b05..c6d3fb7c5 100644 --- a/backend/crates/gateway-store/src/postgres/ops_events.rs +++ b/backend/crates/gateway-store/src/postgres/ops_events.rs @@ -136,7 +136,7 @@ impl OpsEventRepository for PgOpsEventRepository { .attempt_index .map(i32::try_from) .transpose() - .map_err(|_| invalid("attempt index is too large"))?, + .map_err(|source| invalid("attempt index is too large").with_source(source))?, ) .bind(event.level.as_str()) .bind(event.component) @@ -154,7 +154,7 @@ impl OpsEventRepository for PgOpsEventRepository { .retry_after_ms .map(i64::try_from) .transpose() - .map_err(|_| invalid("retry_after_ms is too large"))?, + .map_err(|source| invalid("retry_after_ms is too large").with_source(source))?, ) .bind(event.upstream_request_id) .bind( @@ -162,7 +162,7 @@ impl OpsEventRepository for PgOpsEventRepository { .latency_ms .map(i64::try_from) .transpose() - .map_err(|_| invalid("latency_ms is too large"))?, + .map_err(|source| invalid("latency_ms is too large").with_source(source))?, ) .bind(event.message) .bind(event.error_details) @@ -176,6 +176,7 @@ impl OpsEventRepository for PgOpsEventRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/pricing.rs b/backend/crates/gateway-store/src/postgres/pricing.rs index 2c3eb3e8b..a8ed63cfa 100644 --- a/backend/crates/gateway-store/src/postgres/pricing.rs +++ b/backend/crates/gateway-store/src/postgres/pricing.rs @@ -132,6 +132,7 @@ pub(crate) fn validate_pricing(pricing: &PricingOverrides) -> StoreResult<()> { fn invalid_pricing() -> StoreError { StoreError::InvalidData { + source: None, entity: "model pricing", message: "invalid model pricing".to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index 2e80e3c4d..36cf5a6d5 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -211,6 +211,7 @@ impl PgAdminAccountStore { admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: ENTITY, id: account_id.to_owned(), }, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs index be1135b11..b2c9c03d2 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs @@ -228,6 +228,7 @@ async fn validate_group_filter( Err(admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: "account group", id: group_id.as_str().to_owned(), }, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs index bd93fc841..f52ea4aae 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs @@ -34,21 +34,25 @@ impl PgAdminAccountStore { { return Err(invalid_receipt()); } - let mut transaction = self.pool.begin().await.map_err(|_| unavailable_receipt())?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| unavailable_receipt().with_source(source))?; let outcome = commit_authorization_in_transaction(&mut transaction, command, context).await; match outcome { Ok(outcome) => { transaction .commit() .await - .map_err(|_| unavailable_receipt())?; + .map_err(|source| unavailable_receipt().with_source(source))?; Ok(outcome) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| unavailable_receipt())?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -71,7 +75,7 @@ async fn commit_authorization_in_transaction( .bind(lock) .execute(&mut **transaction) .await - .map_err(|_| unavailable_receipt())?; + .map_err(|source| unavailable_receipt().with_source(source))?; if let Some(result) = load_receipt(&mut **transaction, key).await? { return Ok(AuthorizationCommitResult { result, @@ -140,12 +144,12 @@ async fn commit_authorization_in_transaction( }]; sqlx::query("insert into authorization_receipts(provider_kind, flow_digest, owner_digest, config_revision, accounts_json) values ($1,$2,$3,$4,$5)") .bind(key.provider_kind().as_str()).bind(key.flow_digest()).bind(key.owner_digest()) - .bind(i64::try_from(result.config_revision.get()).map_err(|_| invalid_receipt())?) - .bind(serde_json::to_value(accounts).map_err(|_| invalid_receipt())?) - .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?; + .bind(i64::try_from(result.config_revision.get()).map_err(|source| invalid_receipt().with_source(source))?) + .bind(serde_json::to_value(accounts).map_err(|source| invalid_receipt().with_source(source))?) + .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?; // 保留期长于 OAuth pending 上限;每次提交有界回收,避免引入第二套维护任务 sqlx::query("delete from authorization_receipts where ctid in (select ctid from authorization_receipts where expires_at <= now() order by expires_at limit 128 for update skip locked)") - .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?; + .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?; Ok(AuthorizationCommitResult { result, newly_committed: true, @@ -157,25 +161,28 @@ async fn load_receipt<'a>( key: &AuthorizationReceiptKey, ) -> AdminStoreResult> { let row: Option<(String, i64, serde_json::Value)> = sqlx::query_as("select owner_digest, config_revision, accounts_json from authorization_receipts where provider_kind=$1 and flow_digest=$2 and expires_at>now()") - .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|_| unavailable_receipt())?; + .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|source| unavailable_receipt().with_source(source))?; let Some((owner, revision, accounts)) = row else { return Ok(None); }; if owner != key.owner_digest() { return Err(invalid_receipt()); } - let [account]: [StoredAccount; 1] = - serde_json::from_value(accounts).map_err(|_| unavailable_receipt())?; + let [account]: [StoredAccount; 1] = serde_json::from_value(accounts) + .map_err(|source| unavailable_receipt().with_source(source))?; Ok(Some(CredentialMutationResult { config_revision: AdminRevision::new( - u64::try_from(revision).map_err(|_| unavailable_receipt())?, + u64::try_from(revision).map_err(|source| unavailable_receipt().with_source(source))?, ) - .map_err(|_| unavailable_receipt())?, + .map_err(|source| unavailable_receipt().with_source(source))?, account_id: CoreProviderAccountId::new(account.account_id) - .map_err(|_| unavailable_receipt())?, + .map_err(|source| unavailable_receipt().with_source(source))?, credential_revision: account .credential_revision - .map(|revision| AdminRevision::new(revision).map_err(|_| unavailable_receipt())) + .map(|revision| { + AdminRevision::new(revision) + .map_err(|source| unavailable_receipt().with_source(source)) + }) .transpose()?, })) } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs index 4090f40f8..00fadcb22 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -213,7 +213,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.concurrency_limit.map(|limit| i64::from(limit.get()))) - .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?) + .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?) .bind(credential_state.as_str()) .bind(account.credential_observed_at) .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url())) @@ -281,6 +281,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .await .map_err(|source| postgres_unavailable("compare and swap provider credentials", source))? .ok_or(StoreError::Conflict { + source: None, entity: ENTITY, id: update.account_id, kind: ConflictKind::StaleRevision, @@ -587,6 +588,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { .await .map_err(|source| postgres_unavailable("recover provider account state", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: ENTITY, id: command.account_id.clone(), })?; @@ -659,6 +661,7 @@ async fn replace_account_group_assignments_in_transaction( .map_err(|source| postgres_unavailable("validate account group assignment", source))?; if usize::try_from(known_group_count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { + source: None, entity: "account group", id: "one or more group IDs".to_owned(), }); @@ -757,7 +760,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.concurrency_limit.map(|limit| i64::from(limit.get()))) - .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?) + .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?) .bind(credential_state.as_str()) .bind(account.credential_observed_at) .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url())) @@ -771,6 +774,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( .is_some_and(sqlx::error::DatabaseError::is_unique_violation) { StoreError::Conflict { + source: Some(error.into()), entity: ENTITY, id: account.id.clone(), kind: ConflictKind::InvalidTransition, @@ -780,6 +784,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( } })? .ok_or_else(|| StoreError::Conflict { + source: None, entity: ENTITY, id: account.id.clone(), kind: ConflictKind::InvalidTransition, @@ -850,6 +855,7 @@ pub(crate) async fn rotate_provider_account_in_transaction( .is_some_and(sqlx::error::DatabaseError::is_unique_violation) { StoreError::Conflict { + source: Some(error.into()), entity: ENTITY, id: update.account_id.clone(), kind: ConflictKind::InvalidTransition, @@ -859,6 +865,7 @@ pub(crate) async fn rotate_provider_account_in_transaction( } })? .ok_or_else(|| StoreError::Conflict { + source: None, entity: ENTITY, id: update.account_id.clone(), kind: ConflictKind::StaleRevision, @@ -893,7 +900,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction( .bind(account_ids) .bind(enabled) .bind(concurrency_limit.flatten().map(|limit| i64::from(limit.get()))) - .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|_| invalid("invalid weight"))?) + .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|source| invalid("invalid weight").with_source(source))?) .bind(outbound_proxy.is_some()) .bind(proxy.as_ref().map(gateway_core::account::OutboundProxy::expose_url)) .bind(proxy_id) @@ -909,6 +916,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction( Ok(()) } else { Err(StoreError::NotFound { + source: None, entity: ENTITY, id: "one or more provider account IDs".to_owned(), }) @@ -1018,10 +1026,10 @@ pub(crate) async fn finish_admin_transaction( Ok(value) } Err(error) => { - transaction - .rollback() - .await - .map_err(|source| postgres_unavailable(operation, source))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -1042,6 +1050,7 @@ pub(super) async fn import_provider_accounts_in_transaction( .await?; if current.as_ref() != Some(&binding.proxy) { return Err(StoreError::Conflict { + source: None, entity: "outbound proxy", id: binding.id.clone(), kind: ConflictKind::StaleRevision, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index 7d73af863..85ebcec68 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -446,12 +446,12 @@ pub(crate) fn account_record_from_row( let credentials = JsonObject::try_from_value( "provider_credentials_json", row.try_get("provider_credentials_json") - .map_err(|_| invalid("invalid credentials JSON"))?, + .map_err(|source| invalid("invalid credentials JSON").with_source(source))?, CREDENTIALS_MAX_BYTES, )?; let quota = row .try_get::, _>("provider_quota_json") - .map_err(|_| invalid("invalid quota JSON"))? + .map_err(|source| invalid("invalid quota JSON").with_source(source))? .map(|value| JsonObject::try_from_value("provider_quota_json", value, QUOTA_MAX_BYTES)) .transpose()?; Ok(ProviderAccountRecord { @@ -514,10 +514,10 @@ pub(crate) fn account_summary_from_row( ) -> StoreResult { let revision = row .try_get::("credential_revision") - .map_err(|_| invalid("invalid credential revision"))?; + .map_err(|source| invalid("invalid credential revision").with_source(source))?; let credential_state = row .try_get::("credential_state") - .map_err(|_| invalid("invalid credential_state"))?; + .map_err(|source| invalid("invalid credential_state").with_source(source))?; let quota_access_state = parse_quota_access_state(&get::(&row, "quota_access_state")?)?; let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?; let quota_access_observed_at = get::>>(&row, "quota_access_observed_at")?; @@ -546,7 +546,7 @@ pub(crate) fn account_summary_from_row( outbound_proxy: get::>(&row, "outbound_proxy_url")? .map(|url| { gateway_core::account::OutboundProxy::parse(&url) - .map_err(|_| invalid("invalid outbound proxy")) + .map_err(|source| invalid("invalid outbound proxy").with_source(source)) }) .transpose()?, id: get(&row, "id")?, @@ -584,7 +584,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) - where T: sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, { - row.try_get(column).map_err(|_| invalid(column)) + row.try_get(column) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn validate_object_size( @@ -594,8 +595,9 @@ pub(crate) fn validate_object_size( ) -> StoreResult<()> { let size = serde_json::to_vec(&object.as_value()) .map_err(|error| StoreError::InvalidData { + source: Some(error.into()), entity: ENTITY, - message: error.to_string(), + message: "JSON encoding failed".to_owned(), })? .len(); if size > max { @@ -606,15 +608,16 @@ pub(crate) fn validate_object_size( } pub(crate) fn to_i64(value: u64) -> StoreResult { - i64::try_from(value).map_err(|_| invalid("revision is too large")) + i64::try_from(value).map_err(|source| invalid("revision is too large").with_source(source)) } pub(crate) fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("revision must be positive")) + u64::try_from(value).map_err(|source| invalid("revision must be positive").with_source(source)) } pub(crate) fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/proxies.rs b/backend/crates/gateway-store/src/postgres/proxies.rs index 938e8b884..e7cf5267b 100644 --- a/backend/crates/gateway-store/src/postgres/proxies.rs +++ b/backend/crates/gateway-store/src/postgres/proxies.rs @@ -82,6 +82,7 @@ fn unavailable(source: impl std::error::Error + Send + Sync + 'static) -> StoreE } fn conflict(id: &str) -> StoreError { StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: ConflictKind::InvalidTransition, @@ -89,39 +90,61 @@ fn conflict(id: &str) -> StoreError { } fn invalid() -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: "invalid proxy record".to_owned(), } } fn record(row: PgRow) -> StoreResult { - let success: Option = row.try_get("last_test_success").map_err(|_| invalid())?; - let ip: Option = row.try_get("last_test_ip").map_err(|_| invalid())?; - let ipv4: Option = row.try_get("last_test_ipv4").map_err(|_| invalid())?; - let ipv6: Option = row.try_get("last_test_ipv6").map_err(|_| invalid())?; - let latency: Option = row.try_get("last_test_latency_ms").map_err(|_| invalid())?; + let success: Option = row + .try_get("last_test_success") + .map_err(|source| invalid().with_source(source))?; + let ip: Option = row + .try_get("last_test_ip") + .map_err(|source| invalid().with_source(source))?; + let ipv4: Option = row + .try_get("last_test_ipv4") + .map_err(|source| invalid().with_source(source))?; + let ipv6: Option = row + .try_get("last_test_ipv6") + .map_err(|source| invalid().with_source(source))?; + let latency: Option = row + .try_get("last_test_latency_ms") + .map_err(|source| invalid().with_source(source))?; Ok(ProxyRecord { - auto_location: row.try_get("auto_location").map_err(|_| invalid())?, + auto_location: row + .try_get("auto_location") + .map_err(|source| invalid().with_source(source))?, detected_location: detected_location_from_row(&row)?, location: location_from_row(&row)?, - id: row.try_get("id").map_err(|_| invalid())?, - name: row.try_get("name").map_err(|_| invalid())?, + id: row + .try_get("id") + .map_err(|source| invalid().with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid().with_source(source))?, proxy: OutboundProxy::parse( &row.try_get::("proxy_url") - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, revision: AdminRevision::new( - u64::try_from(row.try_get::("revision").map_err(|_| invalid())?) - .map_err(|_| invalid())?, + u64::try_from( + row.try_get::("revision") + .map_err(|source| invalid().with_source(source))?, + ) + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, account_count: u64::try_from( row.try_get::("account_count") - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, - last_test_at: row.try_get("last_test_at").map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, + last_test_at: row + .try_get("last_test_at") + .map_err(|source| invalid().with_source(source))?, last_test: success .map(|success| -> StoreResult<_> { Ok(ProxyTestResult { @@ -129,48 +152,60 @@ fn record(row: PgRow) -> StoreResult { .try_get::, _>( "last_location_detection_json", ) - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .0, success, latency_ms: u64::try_from(latency.ok_or_else(invalid)?) - .map_err(|_| invalid())?, - exit_ip: ip.map(|ip| ip.parse().map_err(|_| invalid())).transpose()?, + .map_err(|source| invalid().with_source(source))?, + exit_ip: ip + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) + .transpose()?, exit_ipv4: ipv4 - .map(|ip| ip.parse().map_err(|_| invalid())) + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) .transpose()?, exit_ipv6: ipv6 - .map(|ip| ip.parse().map_err(|_| invalid())) + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) .transpose()?, message: row .try_get::, _>("last_test_message") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .unwrap_or_default(), }) }) .transpose()?, - created_at: row.try_get("created_at").map_err(|_| invalid())?, - updated_at: row.try_get("updated_at").map_err(|_| invalid())?, + created_at: row + .try_get("created_at") + .map_err(|source| invalid().with_source(source))?, + updated_at: row + .try_get("updated_at") + .map_err(|source| invalid().with_source(source))?, }) } pub(crate) fn location_from_row( row: &PgRow, ) -> StoreResult> { - let country: Option = row.try_get("location_country").map_err(|_| invalid())?; + let country: Option = row + .try_get("location_country") + .map_err(|source| invalid().with_source(source))?; country .map(|country| { gateway_core::account::RequestLocation { country, - region: row.try_get("location_region").map_err(|_| invalid())?, - city: row.try_get("location_city").map_err(|_| invalid())?, + region: row + .try_get("location_region") + .map_err(|source| invalid().with_source(source))?, + city: row + .try_get("location_city") + .map_err(|source| invalid().with_source(source))?, timezone: row .try_get::("location_timezone") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .parse() - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, } .normalized() - .map_err(|_| invalid()) + .map_err(|source| invalid().with_source(source)) }) .transpose() } @@ -178,10 +213,13 @@ pub(crate) fn location_from_row( fn detected_location_from_row(row: &PgRow) -> StoreResult> { let detected = row .try_get::>, _>("detected_location_json") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .map(|value| value.0); if let Some(value) = &detected { - value.location.validate().map_err(|_| invalid())?; + value + .location + .validate() + .map_err(|source| invalid().with_source(source))?; } Ok(detected) } @@ -191,7 +229,7 @@ pub(crate) fn effective_location_from_row( ) -> StoreResult> { if row .try_get::, _>("auto_location") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .unwrap_or(false) { Ok(detected_location_from_row(row)?.map(|value| value.location)) @@ -225,12 +263,15 @@ async fn save_test( ) -> StoreResult<()> { let detected = current.detected_location_after_test(&result); if let Some(value) = &detected { - value.location.validate().map_err(|_| invalid())?; + value + .location + .validate() + .map_err(|source| invalid().with_source(source))?; } sqlx::query("update outbound_proxies set last_test_at = now(), last_test_success = $2, last_test_latency_ms = $3, last_test_ip = $4, last_test_ipv4 = $5, last_test_ipv6 = $6, last_test_message = $7, last_location_detection_json = $8, detected_location_json = $9 where id = $1") - .bind(¤t.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|_| invalid())?) + .bind(¤t.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|source| invalid().with_source(source))?) .bind(result.exit_ip.map(|ip| ip.to_string())).bind(result.exit_ipv4.map(|ip| ip.to_string())) .bind(result.exit_ipv6.map(|ip| ip.to_string())).bind(result.message) .bind(sqlx::types::Json(result.location)).bind(detected.map(sqlx::types::Json)) @@ -244,7 +285,9 @@ async fn save_location( location: Option<&gateway_core::account::RequestLocation>, ) -> StoreResult<()> { if let Some(location) = location { - location.validate().map_err(|_| invalid())?; + location + .validate() + .map_err(|source| invalid().with_source(source))?; } sqlx::query("update outbound_proxies set location_country = $2, location_region = $3, location_city = $4, location_timezone = $5 where id = $1") .bind(id) @@ -310,12 +353,13 @@ pub(crate) async fn resolve_proxy_selection( .await .map_err(unavailable)? .ok_or_else(|| StoreError::NotFound { + source: None, entity: ENTITY, id: id.clone(), })?; Ok(( Some(id.clone()), - Some(OutboundProxy::parse(&value).map_err(|_| invalid())?), + Some(OutboundProxy::parse(&value).map_err(|source| invalid().with_source(source))?), )) } } @@ -423,6 +467,7 @@ impl ProxyStore for PgProxyRepository { .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { + source: None, entity: ENTITY, id: query.proxy_id.clone(), }) @@ -443,15 +488,15 @@ impl ProxyStore for PgProxyRepository { .into_iter() .map(|row| { Ok(ProxyAccountRef { - id: row.try_get("id").map_err(|_| invalid())?, - name: row.try_get("name").map_err(|_| invalid())?, - email: row.try_get("email").map_err(|_| invalid())?, - provider_kind: row.try_get("provider_kind").map_err(|_| invalid())?, - authentication_kind: row.try_get("authentication_kind").map_err(|_| invalid())?, - plan_type: row.try_get("plan_type").map_err(|_| invalid())?, + id: row.try_get("id").map_err(|source| invalid().with_source(source))?, + name: row.try_get("name").map_err(|source| invalid().with_source(source))?, + email: row.try_get("email").map_err(|source| invalid().with_source(source))?, + provider_kind: row.try_get("provider_kind").map_err(|source| invalid().with_source(source))?, + authentication_kind: row.try_get("authentication_kind").map_err(|source| invalid().with_source(source))?, + plan_type: row.try_get("plan_type").map_err(|source| invalid().with_source(source))?, plan_type_display: None, groups: Vec::new(), - enabled: row.try_get("enabled").map_err(|_| invalid())?, + enabled: row.try_get("enabled").map_err(|source| invalid().with_source(source))?, }) }) .collect::>>() @@ -586,6 +631,7 @@ impl ProxyStore for PgProxyRepository { .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), }) @@ -815,11 +861,11 @@ impl ProxyStore for PgProxyRepository { .map_err(store_error)?; if current.revision != revision { // Drop 只排队回滚,返回冲突前需释放事务锁,避免误挡紧接着的编辑 - transaction - .rollback() - .await - .map_err(|source| store_error(unavailable(source)))?; - return Err(store_error(conflict(id))); + let error = match transaction.rollback().await { + Ok(()) => conflict(id), + Err(cleanup) => conflict(id).with_cleanup(cleanup), + }; + return Err(store_error(error)); } save_test(&mut transaction, ¤t, result) .await diff --git a/backend/crates/gateway-store/src/postgres/runtime_settings.rs b/backend/crates/gateway-store/src/postgres/runtime_settings.rs index 7f024b464..2b0b27707 100644 --- a/backend/crates/gateway-store/src/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/src/postgres/runtime_settings.rs @@ -206,6 +206,7 @@ impl RuntimeSettingsUpdate { }) { return Err(StoreError::InvalidData { + source: None, entity: "runtime settings", message: "settings violate the frozen runtime constraints".to_owned(), }); @@ -276,6 +277,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul .await .map_err(|source| postgres_unavailable("load runtime settings", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -453,6 +455,7 @@ pub(crate) async fn load_runtime_settings_in_transaction( .await .map_err(|source| postgres_unavailable("load runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -568,6 +571,7 @@ pub(crate) async fn update_runtime_settings_in_transaction( .await .map_err(|source| postgres_unavailable("update runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -587,6 +591,7 @@ pub(crate) async fn bump_config_revision_in_transaction( .await .map_err(|source| postgres_unavailable("bump config revision in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -715,6 +720,7 @@ fn to_u32(value: i64) -> StoreResult { fn invalid_location() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "request location is invalid".to_owned(), } @@ -722,6 +728,7 @@ fn invalid_location() -> StoreError { fn invalid_request_profile() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "Provider request profile key is invalid".to_owned(), } @@ -729,6 +736,7 @@ fn invalid_request_profile() -> StoreError { fn invalid_numeric() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "numeric field is outside its supported range".to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/snapshot.rs b/backend/crates/gateway-store/src/postgres/snapshot.rs index b6afead3a..23b6c84e7 100644 --- a/backend/crates/gateway-store/src/postgres/snapshot.rs +++ b/backend/crates/gateway-store/src/postgres/snapshot.rs @@ -137,6 +137,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { .await .map_err(|source| postgres_unavailable("read current config revision", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -288,6 +289,7 @@ async fn load_settings( .await .map_err(|source| postgres_unavailable("load snapshot settings", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -375,7 +377,8 @@ async fn load_account_groups( Ok(SnapshotAccountGroupData { fast_mode: FastMode::parse(&fast_mode) .ok_or_else(|| invalid("invalid fast_mode"))?, - id: AccountGroupId::new(id).map_err(|_| invalid("invalid account group id"))?, + id: AccountGroupId::new(id) + .map_err(|source| invalid("invalid account group id").with_source(source))?, name, enabled, }) @@ -424,7 +427,7 @@ async fn load_group_memberships( .map(|(group_id, account_id)| { Ok(SnapshotGroupMembershipData { group_id: AccountGroupId::new(group_id) - .map_err(|_| invalid("invalid membership group id"))?, + .map_err(|source| invalid("invalid membership group id").with_source(source))?, account_id, }) }) @@ -436,15 +439,18 @@ fn revision_from_i64(value: i64) -> StoreResult { } fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("numeric snapshot field is negative")) + u64::try_from(value) + .map_err(|source| invalid("numeric snapshot field is negative").with_source(source)) } fn to_u32(value: i64) -> StoreResult { - u32::try_from(value).map_err(|_| invalid("numeric snapshot field is outside u32")) + u32::try_from(value) + .map_err(|source| invalid("numeric snapshot field is outside u32").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime snapshot", message: message.to_owned(), } @@ -459,8 +465,9 @@ fn decode_request_profiles( .into_iter() .map(|(kind, document)| { Ok(( - gateway_core::routing::ProviderKind::new(kind) - .map_err(|_| invalid("invalid request profile provider"))?, + gateway_core::routing::ProviderKind::new(kind).map_err(|source| { + invalid("invalid request profile provider").with_source(source) + })?, gateway_core::account::OpaqueProviderData::new(document), )) }) diff --git a/backend/crates/gateway-store/src/redis/auth.rs b/backend/crates/gateway-store/src/redis/auth.rs index d2422d094..d4dff2c57 100644 --- a/backend/crates/gateway-store/src/redis/auth.rs +++ b/backend/crates/gateway-store/src/redis/auth.rs @@ -284,6 +284,7 @@ fn decode_session(value: &str) -> StoreResult { fn auth_invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "authentication state", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/client_admission.rs b/backend/crates/gateway-store/src/redis/client_admission.rs index df9039046..889b62bd5 100644 --- a/backend/crates/gateway-store/src/redis/client_admission.rs +++ b/backend/crates/gateway-store/src/redis/client_admission.rs @@ -280,7 +280,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { request.validate()?; let keys = self.keys(&request.client_api_key_ref)?; let lease_ttl_ms = u64::try_from(request.lease_ttl.as_millis()) - .map_err(|_| invalid("lease TTL is too large"))?; + .map_err(|source| invalid("lease TTL is too large").with_source(source))?; let mut connection = self.connection.clone(); let code = Script::new(ADMIT_SCRIPT) .key(&keys[0]) @@ -462,7 +462,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { CoreAdmissionDecision::Rejected(CoreAdmissionRejection::ConcurrencyLimited) } }) - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } @@ -474,7 +474,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { Box::pin(async move { self.release_client_request(client_api_key_id.as_str(), model_request_id.as_str()) .await - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } @@ -508,13 +508,14 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { restored_recent_requests: restored.restored_recent_requests, restored_running_requests: restored.restored_running_requests, }) - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "client admission", message: message.to_owned(), } @@ -525,18 +526,19 @@ fn validate_recovery_request_id(model_request_id: &str) -> StoreResult<()> { } fn redis_duration_millis(duration: Duration) -> StoreResult { - let milliseconds = - u64::try_from(duration.as_millis()).map_err(|_| invalid("lease TTL is too large"))?; + let milliseconds = u64::try_from(duration.as_millis()) + .map_err(|source| invalid("lease TTL is too large").with_source(source))?; redis_integer(milliseconds, "lease TTL") } fn redis_timestamp_millis(timestamp: DateTime, field: &str) -> StoreResult { - let milliseconds = u64::try_from(timestamp.timestamp_millis()).map_err(|_| invalid(field))?; + let milliseconds = u64::try_from(timestamp.timestamp_millis()) + .map_err(|source| invalid(field).with_source(source))?; redis_integer(milliseconds, field) } fn redis_len(length: usize, field: &str) -> StoreResult { - let value = u64::try_from(length).map_err(|_| invalid(field))?; + let value = u64::try_from(length).map_err(|source| invalid(field).with_source(source))?; redis_integer(value, field) } diff --git a/backend/crates/gateway-store/src/redis/coordination_buffer.rs b/backend/crates/gateway-store/src/redis/coordination_buffer.rs index aa138d672..18f818342 100644 --- a/backend/crates/gateway-store/src/redis/coordination_buffer.rs +++ b/backend/crates/gateway-store/src/redis/coordination_buffer.rs @@ -32,10 +32,14 @@ pub struct BufferedClientAdmissionPort { impl BufferedClientAdmissionPort { #[must_use] - pub fn new(inner: Arc) -> (Self, ClientAdmissionReleaseWriter) { + pub fn new( + inner: Arc, + diagnostics: Arc, + ) -> (Self, ClientAdmissionReleaseWriter) { Self::with_capacity( inner, NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"), + diagnostics, ) } @@ -43,6 +47,7 @@ impl BufferedClientAdmissionPort { pub fn with_capacity( inner: Arc, capacity: NonZeroUsize, + diagnostics: Arc, ) -> (Self, ClientAdmissionReleaseWriter) { let (sender, receiver) = mpsc::channel(capacity.get()); ( @@ -52,6 +57,7 @@ impl BufferedClientAdmissionPort { }, ClientAdmissionReleaseWriter { inner, + diagnostics, receiver: Mutex::new(receiver), }, ) @@ -127,6 +133,7 @@ struct AdmissionRelease { } pub struct ClientAdmissionReleaseWriter { + diagnostics: Arc, inner: Arc, receiver: Mutex>, } @@ -138,7 +145,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter { loop { let release = tokio::select! { () = cancellation.cancelled() => { - drain_admission_releases(&mut receiver, self.inner.as_ref()).await; + drain_admission_releases(&mut receiver, self.inner.as_ref(), self.diagnostics.as_ref()).await; return Ok(()); }, release = receiver.recv() => release, @@ -148,13 +155,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter { "client admission release queue closed", )); }; - if let Err(error) = self - .inner - .release(&release.client_api_key_id, &release.model_request_id) - .await - { - tracing::warn!(%error, "Client admission 后台释放失败,依赖租约 TTL 收敛"); - } + release_admission(self.inner.as_ref(), self.diagnostics.as_ref(), release).await; } }) } @@ -163,21 +164,46 @@ impl DaemonTask for ClientAdmissionReleaseWriter { async fn drain_admission_releases( receiver: &mut mpsc::Receiver, port: &dyn ClientAdmissionPort, + diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics, ) { receiver.close(); let started_at = Instant::now(); while let Some(release) = receiver.recv().await { let remaining = SHUTDOWN_DRAIN_TIMEOUT.saturating_sub(started_at.elapsed()); if remaining.is_zero() - || tokio::time::timeout( - remaining, - port.release(&release.client_api_key_id, &release.model_request_id), - ) - .await - .is_err() + || tokio::time::timeout(remaining, release_admission(port, diagnostics, release)) + .await + .is_err() { tracing::warn!("Client admission 关闭排空超时,剩余租约依赖 TTL 收敛"); break; } } } + +async fn release_admission( + port: &dyn ClientAdmissionPort, + diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics, + release: AdmissionRelease, +) { + if let Err(source) = port + .release(&release.client_api_key_id, &release.model_request_id) + .await + { + let mut failure = gateway_core::diagnostics::OperationalFailure::new( + "store", + "release_client_admission", + "admission_unavailable", + "Client admission release failed; lease TTL remains active", + ); + failure.correlation_id = Some(release.model_request_id.as_str().to_owned()); + failure.details = + gateway_core::error::ErrorDetails::capture(Some(&source.into()), None, false); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!( + request_id = release.model_request_id.as_str(), + "admission diagnostic could not be recorded" + ); + } + } +} diff --git a/backend/crates/gateway-store/src/redis/credential_cooldown.rs b/backend/crates/gateway-store/src/redis/credential_cooldown.rs index d37eb50b9..ba412f6f1 100644 --- a/backend/crates/gateway-store/src/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/src/redis/credential_cooldown.rs @@ -260,10 +260,10 @@ impl RedisCredentialCooldownRepository { } let revision = revision .parse::() - .map_err(|_| invalid("cached cooldown revision is invalid"))?; + .map_err(|source| invalid("cached cooldown revision is invalid").with_source(source))?; let until_ms = until_ms .parse::() - .map_err(|_| invalid("cached cooldown expiry is invalid"))?; + .map_err(|source| invalid("cached cooldown expiry is invalid").with_source(source))?; let cooldown_until = DateTime::from_timestamp_millis(until_ms) .ok_or_else(|| invalid("cached cooldown expiry is invalid"))?; let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit); @@ -350,7 +350,7 @@ impl RedisCredentialCooldownRepository { account_id, gateway_admin::model::accounts::AccountFreeze { credential_revision: gateway_admin::model::Revision::new(revision.get()) - .map_err(|_| invalid("freeze revision"))?, + .map_err(|source| invalid("freeze revision").with_source(source))?, until, generation, requires_probe: kind.requires_probe(), @@ -401,7 +401,7 @@ impl RedisCredentialCooldownRepository { .map_err(|source| redis_unavailable("read capacity peak in-flight", source))?; peak.map(u32::try_from) .transpose() - .map_err(|_| invalid("capacity peak in-flight is invalid")) + .map_err(|source| invalid("capacity peak in-flight is invalid").with_source(source)) } } @@ -741,6 +741,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential cooldown", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index e44b87f08..1887f1b35 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -247,6 +247,7 @@ impl CredentialLeaseGuard { .grant .clone() .ok_or_else(|| crate::StoreError::InvalidData { + source: None, entity: "credential lease", message: "lease has already been released".to_owned(), })?; @@ -385,9 +386,9 @@ impl RedisCredentialLeaseRepository { .invoke_async::(&mut connection) .await .map_err(|source| redis_unavailable("advance provider scheduling cursor", source))?; - cursor - .parse::() - .map_err(|_| invalid("Redis returned an invalid scheduling cursor")) + cursor.parse::().map_err(|source| { + invalid("Redis returned an invalid scheduling cursor").with_source(source) + }) } fn keys(&self, request: &CredentialLeaseRequest) -> StoreResult<[String; 3]> { @@ -486,9 +487,9 @@ impl RedisCredentialLeaseRepository { .invoke_async(&mut connection) .await .map_err(|source| redis_unavailable("load credential runtime signal", source))?; - let in_flight = in_flight - .parse::() - .map_err(|_| invalid("Redis returned an invalid in-flight count"))?; + let in_flight = in_flight.parse::().map_err(|source| { + invalid("Redis returned an invalid in-flight count").with_source(source) + })?; let last_started_at = if last_started == "0" { None } else { @@ -830,7 +831,7 @@ struct LeaseAttempt { fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult { let fence = fence .parse::() - .map_err(|_| invalid("Redis returned an invalid fencing token"))?; + .map_err(|source| invalid("Redis returned an invalid fencing token").with_source(source))?; Ok(CredentialLeaseGrant { lease_id, fencing_token: Revision::new(fence)?, @@ -841,7 +842,7 @@ fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult StoreResult> { let milliseconds = value .parse::() - .map_err(|_| invalid("Redis returned an invalid timestamp"))?; + .map_err(|source| invalid("Redis returned an invalid timestamp").with_source(source))?; DateTime::from_timestamp_millis(milliseconds) .ok_or_else(|| invalid("Redis returned an out-of-range timestamp")) } @@ -850,7 +851,7 @@ fn duration(value: &str) -> StoreResult { value .parse::() .map(Duration::from_millis) - .map_err(|_| invalid("Redis returned an invalid retry interval")) + .map_err(|source| invalid("Redis returned an invalid retry interval").with_source(source)) } fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) -> StoreResult<()> { @@ -863,11 +864,13 @@ fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) -> fn duration_millis(value: Duration) -> StoreResult { supported_duration(value, true, "duration")?; - u64::try_from(value.as_millis()).map_err(|_| invalid("duration is too large")) + u64::try_from(value.as_millis()) + .map_err(|source| invalid("duration is too large").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential lease", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/credential_state.rs b/backend/crates/gateway-store/src/redis/credential_state.rs index 42b781169..23149b242 100644 --- a/backend/crates/gateway-store/src/redis/credential_state.rs +++ b/backend/crates/gateway-store/src/redis/credential_state.rs @@ -446,6 +446,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential state cache", message: message.to_owned(), } @@ -453,6 +454,7 @@ fn invalid(message: &str) -> StoreError { fn catalog_invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "provider catalog cache", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/mod.rs b/backend/crates/gateway-store/src/redis/mod.rs index 9e09ab0fa..721654506 100644 --- a/backend/crates/gateway-store/src/redis/mod.rs +++ b/backend/crates/gateway-store/src/redis/mod.rs @@ -48,6 +48,7 @@ pub(crate) fn namespace(value: &str) -> StoreResult { .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) { return Err(StoreError::InvalidData { + source: None, entity: "Redis namespace", message: "namespace contains unsupported characters".to_owned(), }); diff --git a/backend/crates/gateway-store/src/redis/oauth_pending.rs b/backend/crates/gateway-store/src/redis/oauth_pending.rs index 1bcbaab6d..60e8b95b5 100644 --- a/backend/crates/gateway-store/src/redis/oauth_pending.rs +++ b/backend/crates/gateway-store/src/redis/oauth_pending.rs @@ -322,6 +322,7 @@ fn ttl_millis(ttl: Duration) -> StoreResult { .ok() .filter(|value| *value > 0 && *value <= MAX_REDIS_EXACT_INTEGER) .ok_or_else(|| StoreError::InvalidData { + source: None, entity: "OAuth pending flow", message: "TTL is outside the supported range".to_owned(), }) diff --git a/backend/crates/gateway-store/src/redis/runtime_change.rs b/backend/crates/gateway-store/src/redis/runtime_change.rs index 0a743cd91..72c500d4d 100644 --- a/backend/crates/gateway-store/src/redis/runtime_change.rs +++ b/backend/crates/gateway-store/src/redis/runtime_change.rs @@ -73,9 +73,9 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository { .await .map_err(|source| redis_unavailable("subscribe runtime changes", source))?; let stream = pubsub.into_on_message().map(|message| { - let payload = message - .get_payload::() - .map_err(|_| invalid("runtime change payload is not UTF-8"))?; + let payload = message.get_payload::().map_err(|source| { + invalid("runtime change payload is not UTF-8").with_source(source) + })?; let wire: RuntimeChangeWire = serde_json::from_str(&payload).map_err(|error| invalid(&error.to_string()))?; wire.try_into() @@ -154,6 +154,7 @@ impl TryFrom for RuntimeChange { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime change", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/worker_lease.rs b/backend/crates/gateway-store/src/redis/worker_lease.rs index d37db1beb..4ebd50714 100644 --- a/backend/crates/gateway-store/src/redis/worker_lease.rs +++ b/backend/crates/gateway-store/src/redis/worker_lease.rs @@ -52,7 +52,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard { current, ) .await - .map_err(|_| WorkerLeaseError::safe("worker lease renewal failed"))? + .map_err(|source| { + WorkerLeaseError::safe("worker lease renewal failed").with_source(source) + })? .ok_or_else(|| WorkerLeaseError::safe("worker lease was lost"))?; self.grant = Some(renewed); Ok(()) @@ -72,7 +74,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard { CredentialLeaseRepository::release_credential_lease(&repository, &request, &grant) .await .map(|_| ()) - .map_err(|_| WorkerLeaseError::safe("worker lease release failed")) + .map_err(|source| { + WorkerLeaseError::safe("worker lease release failed").with_source(source) + }) }) } } @@ -94,7 +98,9 @@ impl WorkerLeaderLeasePort for RedisWorkerLeaderLeasePort { &lease_request, ) .await - .map_err(|_| WorkerLeaseError::safe("worker lease acquisition failed"))?; + .map_err(|source| { + WorkerLeaseError::safe("worker lease acquisition failed").with_source(source) + })?; let Some(grant) = grant else { return Ok(WorkerLeaseAcquisition::Busy { retry_after: None }); }; diff --git a/backend/crates/gateway-store/src/value.rs b/backend/crates/gateway-store/src/value.rs index 6e201acfd..4b794ddc3 100644 --- a/backend/crates/gateway-store/src/value.rs +++ b/backend/crates/gateway-store/src/value.rs @@ -32,20 +32,87 @@ pub enum StoreError { source: Option, }, #[error("{entity} {id} was not found")] - NotFound { entity: &'static str, id: String }, + NotFound { + entity: &'static str, + id: String, + source: Option, + }, #[error("store conflict for {entity} {id}: {kind:?}")] Conflict { entity: &'static str, id: String, kind: ConflictKind, + source: Option, }, #[error("invalid persisted {entity}: {message}")] InvalidData { entity: &'static str, message: String, + source: Option, }, } +impl StoreError { + /// 本地校验可没有来源;转换已有失败时附加真实原因 + #[must_use] + pub(crate) fn with_source( + mut self, + error: impl Into, + ) -> Self { + let error = error.into(); + match &mut self { + Self::Unavailable { source, .. } + | Self::InvalidData { source, .. } + | Self::Conflict { source, .. } + | Self::NotFound { source, .. } => *source = Some(error), + } + self + } + + pub(crate) fn with_cleanup( + mut self, + cleanup: impl Into, + ) -> Self { + let source = match &mut self { + Self::Unavailable { source, .. } + | Self::InvalidData { source, .. } + | Self::Conflict { source, .. } + | Self::NotFound { source, .. } => source, + }; + *source = Some(gateway_core::error::ErrorSource::cleanup( + source.take(), + cleanup, + )); + self + } + + fn admin_kind(&self) -> AdminStoreErrorKind { + match self { + Self::NotFound { .. } => AdminStoreErrorKind::NotFound, + Self::Conflict { + kind: ConflictKind::StaleRevision, + .. + } => AdminStoreErrorKind::StaleRevision, + Self::Conflict { + kind: ConflictKind::DuplicateName, + .. + } => AdminStoreErrorKind::DuplicateName, + Self::Conflict { .. } => AdminStoreErrorKind::Conflict, + Self::InvalidData { .. } => AdminStoreErrorKind::Invalid, + Self::Unavailable { .. } => AdminStoreErrorKind::Unavailable, + } + } + + fn core_kind(&self) -> gateway_core::error::StoreErrorKind { + use gateway_core::error::StoreErrorKind; + match self { + Self::Unavailable { .. } => StoreErrorKind::Unavailable, + Self::Conflict { .. } => StoreErrorKind::Conflict, + Self::NotFound { .. } | Self::InvalidData { .. } => StoreErrorKind::InvalidData, + } + } +} + pub type StoreResult = Result; pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult { Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error)) @@ -88,21 +155,7 @@ pub(crate) fn mutation_audit( } pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> AdminStoreError { - let kind = match error { - StoreError::NotFound { .. } => AdminStoreErrorKind::NotFound, - StoreError::Conflict { - kind: ConflictKind::StaleRevision, - .. - } => AdminStoreErrorKind::StaleRevision, - StoreError::Conflict { - kind: ConflictKind::DuplicateName, - .. - } => AdminStoreErrorKind::DuplicateName, - StoreError::Conflict { .. } => AdminStoreErrorKind::Conflict, - StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid, - StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable, - }; - AdminStoreError::new(kind, resource, "store operation failed").with_source(error) + AdminStoreError::new(error.admin_kind(), resource, "store operation failed").with_source(error) } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -113,6 +166,7 @@ impl Revision { NonZeroU64::new(value) .map(Self) .ok_or_else(|| StoreError::InvalidData { + source: None, entity: "revision", message: "must be greater than zero".to_owned(), }) @@ -160,6 +214,7 @@ impl FromStr for DecimalAmount { }); if !valid { return Err(StoreError::InvalidData { + source: None, entity: "decimal amount", message: "expected a non-negative numeric(20,10) value".to_owned(), }); @@ -190,18 +245,21 @@ impl JsonObject { ) -> StoreResult { let serialized_bytes = serde_json::to_vec(&value) .map_err(|error| StoreError::InvalidData { + source: Some(error.into()), entity, - message: error.to_string(), + message: "JSON encoding failed".to_owned(), })? .len(); let Value::Object(fields) = value else { return Err(StoreError::InvalidData { + source: None, entity, message: "top-level JSON value must be an object".to_owned(), }); }; if serialized_bytes > max_serialized_bytes { return Err(StoreError::InvalidData { + source: None, entity, message: format!("serialized JSON exceeds {max_serialized_bytes} bytes"), }); @@ -233,6 +291,7 @@ pub(crate) fn require_nonempty( ) -> StoreResult<()> { if value.trim().is_empty() { Err(StoreError::InvalidData { + source: None, entity, message: format!("{field} must not be empty"), }) @@ -276,11 +335,5 @@ pub(crate) fn provider_unavailable( } pub(crate) fn core_store_error(error: StoreError) -> gateway_core::error::StoreError { - use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind}; - let kind = match error { - StoreError::Unavailable { .. } => StoreErrorKind::Unavailable, - StoreError::Conflict { .. } => StoreErrorKind::Conflict, - StoreError::NotFound { .. } | StoreError::InvalidData { .. } => StoreErrorKind::InvalidData, - }; - CoreStoreError::caused_by(kind, error) + gateway_core::error::StoreError::caused_by(error.core_kind(), error) } diff --git a/backend/crates/gateway-store/src/workers.rs b/backend/crates/gateway-store/src/workers.rs index df7463b58..d3fc6bb24 100644 --- a/backend/crates/gateway-store/src/workers.rs +++ b/backend/crates/gateway-store/src/workers.rs @@ -181,6 +181,7 @@ pub(crate) fn worker_definition_error( error: gateway_core::task::WorkerDefinitionError, ) -> StoreError { StoreError::InvalidData { + source: None, entity: "store worker plan", message: error.to_string(), } @@ -202,7 +203,9 @@ impl ScheduledTask for StaleModelRequestRecoveryTask { ) .await .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("stale request recovery failed")) + .map_err(|source| { + WorkerTaskError::safe("stale request recovery failed").with_source(source) + }) }) } } diff --git a/backend/crates/gateway-store/tests/main.rs b/backend/crates/gateway-store/tests/main.rs index ce7b14283..965a96667 100644 --- a/backend/crates/gateway-store/tests/main.rs +++ b/backend/crates/gateway-store/tests/main.rs @@ -6,3 +6,16 @@ mod config; mod postgres; mod redis; mod support; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-store/tests/postgres/client_budgets.rs b/backend/crates/gateway-store/tests/postgres/client_budgets.rs index 03334f105..2704c998a 100644 --- a/backend/crates/gateway-store/tests/postgres/client_budgets.rs +++ b/backend/crates/gateway-store/tests/postgres/client_budgets.rs @@ -1209,3 +1209,45 @@ async fn timezone_cutover_preserves_open_windows_and_resumes_without_overlap() { ); database.close().await; } + +#[tokio::test] +async fn failed_settlement_retains_native_database_code_through_the_next_admission() { + let Some(database) = TestDatabase::create("budget_native_failure").await else { + return; + }; + sqlx::query("alter table client_api_keys rename to hidden_test_keys") + .execute(&database.pool) + .await + .unwrap(); + let budgets = PgClientBudgetStore::new(database.pool.clone()); + let error = budgets + .settle(charge("missing-key", "native-error", "1")) + .await + .unwrap_err(); + fn assert_native_code(error: &(dyn std::error::Error + 'static)) { + let mut source = Some(error); + while let Some(error) = source { + if let Some(sqlx::Error::Database(error)) = error.downcast_ref::() { + assert_eq!(error.code().as_deref(), Some("42P01")); + return; + } + source = error.source(); + } + panic!("native database failure was lost"); + } + assert_native_code(&error); + let retried = budgets.admit(key_id("missing-key")).await.unwrap_err(); + assert_eq!( + retried.kind(), + GatewayErrorKind::ProviderInfrastructureUnavailable + ); + assert_native_code(&retried); + assert!( + retried + .error_details() + .unwrap() + .as_str() + .contains("client_api_keys") + ); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs index def00d8ea..d42496cfd 100644 --- a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs @@ -969,3 +969,159 @@ async fn zero_attempt_postgres_write_failures_are_not_retried_and_do_not_stop_th assert_eq!(recovered.trace, None); database.close().await; } + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingStore { + async fn record_failure( + &self, + _: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), StoreError> { + Ok(()) + } +} + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for BlockingStore { + async fn record_failure( + &self, + _: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), StoreError> { + Ok(()) + } +} + +#[tokio::test] +async fn operational_failure_uses_the_bounded_queue_without_creating_a_model_request() { + use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure}; + use gateway_core::error::{ErrorDetails, ErrorSource}; + let Some(database) = TestDatabase::create("operational_details").await else { + return; + }; + let inner = Arc::new(PgExecutionStore::new(database.pool.clone())); + let (store, writer) = + BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(1).unwrap()); + let mut failure = + OperationalFailure::new("admin", "http_request", "unavailable", "safe summary"); + failure.correlation_id = Some("admin-request-123".to_owned()); + failure.details = ErrorDetails::capture( + Some(&ErrorSource::new(std::io::Error::other( + "PRIVATE_STORE_CAUSE", + ))), + None, + false, + ); + let occurred_at = failure.occurred_at; + store.record_failure(failure).await.unwrap(); + store + .record_failure(OperationalFailure::new( + "worker", + "run", + "worker_failed", + "safe", + )) + .await + .unwrap(); + assert_eq!(store.stats().queued_items, 1); + assert_eq!(store.stats().dropped_total, 1); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + writer.run(cancellation).await.unwrap(); + let row: ( + Option, + String, + String, + chrono::DateTime, + ) = sqlx::query_as( + "select model_request_id, message, error_details, created_at + from ops_events where component = 'admin'", + ) + .fetch_one(&database.pool) + .await + .unwrap(); + assert!(row.0.is_none()); + let message: serde_json::Value = serde_json::from_str(&row.1).unwrap(); + assert_eq!(message["correlationId"], "admin-request-123"); + assert!(!row.1.contains("PRIVATE_STORE_CAUSE")); + assert!(row.2.contains("PRIVATE_STORE_CAUSE")); + assert_eq!( + row.3.timestamp_micros(), + chrono::DateTime::::from(occurred_at).timestamp_micros() + ); + let requests: i64 = sqlx::query_scalar("select count(*) from model_requests") + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(requests, 0); + database.close().await; +} + +#[tokio::test] +async fn progress_congestion_preserves_failure_reserve_without_waiting() { + use futures::FutureExt as _; + use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure}; + use gateway_core::error::{ErrorDetails, ErrorSource}; + let Some(database) = TestDatabase::create("failure_reserve").await else { + return; + }; + let inner = Arc::new(PgExecutionStore::new(database.pool.clone())); + let mut request_ids = Vec::new(); + for index in 0..4 { + let request = accepted_request(&format!("req_reserved_failure_{index}")); + request_ids.push(request.id.clone()); + inner.create_model_request(request).await.unwrap(); + } + let request_id = &request_ids[0]; + let (store, writer) = + BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(4).unwrap()); + // 不启动消费者,确定性地填满普通额度;每次调用必须首次 poll 就返回 + for (request_id, status) in request_ids.iter().zip([200, 201, 202, 203]) { + store + .record_client_status(request_id, status) + .now_or_never() + .expect("enqueue waited") + .unwrap(); + } + let mut failure = + OperationalFailure::new("admin", "http_request", "unavailable", "safe failure"); + failure.correlation_id = Some(request_id.as_str().to_owned()); + failure.details = ErrorDetails::capture( + Some(&ErrorSource::new(std::io::Error::other( + "ORIGINAL_RESERVED_CAUSE", + ))), + None, + false, + ); + store + .record_failure(failure) + .now_or_never() + .expect("failure waited for queue capacity") + .unwrap(); + assert_eq!(store.stats().queued_items, 5); + assert_eq!(store.stats().dropped_total, 0); + store + .record_client_status(request_id, 204) + .now_or_never() + .expect("overflow waited") + .unwrap(); + assert_eq!(store.stats().queued_items, 5); + assert_eq!(store.stats().dropped_total, 1); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + writer.run(cancelled).await.unwrap(); + assert_eq!(store.stats().persisted_total, 5); + let detail: String = + sqlx::query_scalar("select error_details from ops_events where component = 'admin'") + .fetch_one(&database.pool) + .await + .unwrap(); + assert!(detail.contains("ORIGINAL_RESERVED_CAUSE")); + let status: i32 = + sqlx::query_scalar("select client_status_code from model_requests where id = $1") + .bind(request_id.as_str()) + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(status, 200); + assert_eq!(store.stats().queued_bytes, 0); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/query_budget.rs b/backend/crates/gateway-store/tests/postgres/query_budget.rs index 4c18a90fa..8459a25d9 100644 --- a/backend/crates/gateway-store/tests/postgres/query_budget.rs +++ b/backend/crates/gateway-store/tests/postgres/query_budget.rs @@ -19,6 +19,7 @@ async fn query_stream_holds_budget_until_completion_error_or_drop() { None, Some(Ok(2)), Some(Err(StoreError::InvalidData { + source: None, entity: "test stream", message: "decode failed".to_owned(), })), diff --git a/backend/crates/gateway-store/tests/redis/coordination_buffer.rs b/backend/crates/gateway-store/tests/redis/coordination_buffer.rs index d65ccd38a..121a3b351 100644 --- a/backend/crates/gateway-store/tests/redis/coordination_buffer.rs +++ b/backend/crates/gateway-store/tests/redis/coordination_buffer.rs @@ -141,6 +141,7 @@ async fn full_recoverable_coordination_queues_should_drop_writes_without_waiting let (admissions, _admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(1).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_buffer_test").expect("client key"); let request = ModelRequestId::new("req_buffer_test").expect("request ID"); @@ -167,6 +168,7 @@ async fn redis_coordination_writers_should_flush_each_side_effect() { let (admissions, admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(8).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_writer_test").expect("client key"); let request = ModelRequestId::new("req_writer_test").expect("request ID"); @@ -207,6 +209,7 @@ async fn awaited_buffered_release_should_not_publish_capacity_before_writer_runs let (admissions, admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(8).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_capacity_handoff").expect("client key"); let request = |index| ClientAdmissionRequest { diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index 2a3deccf3..3291c843e 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -1632,6 +1632,7 @@ fn map_store_error(error: gateway_core::error::StoreError) -> ProviderAdminError StoreErrorKind::Unavailable => ProviderAdminErrorKind::Unavailable, _ => ProviderAdminErrorKind::Internal, }) + .with_source(error) } fn map_credential_admin_error(error: CodexCredentialAdminError) -> ProviderAdminError { diff --git a/backend/crates/providers/openai/src/credential/admin.rs b/backend/crates/providers/openai/src/credential/admin.rs index 0353eec8b..335763569 100644 --- a/backend/crates/providers/openai/src/credential/admin.rs +++ b/backend/crates/providers/openai/src/credential/admin.rs @@ -864,6 +864,7 @@ pub struct CodexCredentialAdminService { personal_access_token_client: Option>, leases: Arc, runtime_policy: Arc, + diagnostics: Arc, } impl fmt::Debug for CodexCredentialAdminService { @@ -886,12 +887,14 @@ impl CodexCredentialAdminService { refresher: Arc, leases: Arc, runtime_policy: Arc, + diagnostics: Arc, ) -> Self { Self { refresher, personal_access_token_client: None, leases, runtime_policy, + diagnostics, } } @@ -959,9 +962,20 @@ impl CodexCredentialAdminService { refresh_token.expose_secret(), current.account.outbound_proxy(), ) - .await - .inspect_err(|error| log_manual_refresh_failure(&account_id, error)) - .map_err(map_refresh_failure)?; + .await; + let tokens = match tokens { + Ok(tokens) => tokens, + Err(error) => { + super::diagnostics::record_refresh_failure( + self.diagnostics.as_ref(), + &account_id, + "manual_refresh", + &error, + ) + .await; + return Err(map_refresh_failure(error)); + } + }; let access_token_expires_at = match tokens.access_token.as_deref() { Some(access_token) => parse_access_token_expiration(access_token), None => current @@ -1236,7 +1250,9 @@ fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError { RefreshFailure::RetryableTransport { .. } => CodexCredentialAdminError::RefreshUnavailable, // Worker 的 Transport 分类还承担 401 退避;管理提示只按已收到的响应事实细分, // 不改变后台刷新策略,也不把明确失败响应误报为租约冲突或执行结果未知 - RefreshFailure::Transport { message, upstream } => match upstream { + RefreshFailure::Transport { + message, upstream, .. + } => match upstream { Some(upstream) => CodexCredentialAdminError::RefreshUpstream { status: upstream.status(), code: upstream.code().map(str::to_owned), @@ -1247,20 +1263,6 @@ fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError { } } -fn log_manual_refresh_failure(account_id: &ProviderAccountId, error: &RefreshFailure) { - let upstream = error.upstream(); - tracing::warn!( - account_id = %account_id, - failure_class = error.classification(), - upstream_message = ?error.message(), - upstream_status = ?upstream.map(super::token_client::RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(super::token_client::RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(super::token_client::RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(super::token_client::RefreshUpstreamFailure::body), - "OpenAI OAuth manual refresh failed" - ); -} - fn parse_import_document( payload: &Value, default_proxy: Option<&gateway_core::account::OutboundProxy>, diff --git a/backend/crates/providers/openai/src/credential/diagnostics.rs b/backend/crates/providers/openai/src/credential/diagnostics.rs new file mode 100644 index 000000000..b848ef7da --- /dev/null +++ b/backend/crates/providers/openai/src/credential/diagnostics.rs @@ -0,0 +1,52 @@ +//! OAuth 刷新失败的受控观测投影,普通日志只记录分类和关联 + +use super::token_client::RefreshFailure; +use gateway_core::{ + account::ProviderAccountId, + diagnostics::{OperationalDiagnostics, OperationalFailure}, + error::{ErrorDetails, ErrorSource, OpaqueUpstreamValue, RawUpstreamError}, + identity::ProviderKind, +}; + +pub(crate) async fn record_refresh_failure( + diagnostics: &dyn OperationalDiagnostics, + account_id: &ProviderAccountId, + operation: &'static str, + error: &RefreshFailure, +) { + let upstream = error.upstream(); + let mut failure = OperationalFailure::new( + "oauth", + operation, + error.classification(), + error.to_string(), + ); + failure.provider_kind = Some(ProviderKind::new("openai").expect("static provider")); + failure.account_id = Some(account_id.clone()); + failure.upstream_status = upstream.map(|value| value.status()); + failure.upstream_code = upstream + .and_then(|value| value.code()) + .map(|code| OpaqueUpstreamValue::new(code.to_owned())); + let source = ErrorSource::new(RefreshDiagnostic(error.clone())); + let body = upstream.map(|value| RawUpstreamError::new(value.body())); + failure.details = ErrorDetails::capture(Some(&source), body.as_ref(), error.redacted()); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(account_id = %account_id, operation, "OAuth diagnostic could not be recorded"); + } +} + +#[derive(Debug)] +struct RefreshDiagnostic(RefreshFailure); +impl std::fmt::Display for RefreshDiagnostic { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0.message() { + Some(message) => formatter.write_str(message), + None => std::fmt::Display::fmt(&self.0, formatter), + } + } +} +impl std::error::Error for RefreshDiagnostic { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} diff --git a/backend/crates/providers/openai/src/credential/mod.rs b/backend/crates/providers/openai/src/credential/mod.rs index a1fb5bbb7..435044416 100644 --- a/backend/crates/providers/openai/src/credential/mod.rs +++ b/backend/crates/providers/openai/src/credential/mod.rs @@ -76,3 +76,5 @@ pub use types::{ CodexCredentialData, CodexCredentialPrincipal, CodexOAuthCredentialData, CodexOAuthSecret, ResponsesTransport, RuntimeCodexCookie, UpsertCodexCookie, }; + +mod diagnostics; diff --git a/backend/crates/providers/openai/src/credential/refresh.rs b/backend/crates/providers/openai/src/credential/refresh.rs index eededdd56..40d91b741 100644 --- a/backend/crates/providers/openai/src/credential/refresh.rs +++ b/backend/crates/providers/openai/src/credential/refresh.rs @@ -95,7 +95,6 @@ fn log_refresh_deferred( access_token_expires_at: Option, attempt: u32, reason: &'static str, - upstream_message: Option<&str>, upstream: Option<&RefreshUpstreamFailure>, retry_at: SystemTime, ) { @@ -103,11 +102,11 @@ fn log_refresh_deferred( account_id = %account_id, attempt, reason, - upstream_message = ?upstream_message, + upstream_status = ?upstream.map(RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(RefreshUpstreamFailure::body), + + + retry_at = %DateTime::::from(retry_at), access_token_expires_at = ?access_token_expires_at.map(DateTime::::from), recovery_deadline = ?refresh_recovery_deadline(access_token_expires_at) @@ -193,6 +192,7 @@ pub struct CodexCredentialRefreshService { leases: Arc, credential_state: Arc, runtime_policy: Arc, + diagnostics: Arc, } impl CodexCredentialRefreshService { @@ -202,6 +202,7 @@ impl CodexCredentialRefreshService { leases: Arc, credential_state: Arc, runtime_policy: Arc, + diagnostics: Arc, ) -> Self { Self { repository, @@ -209,6 +210,7 @@ impl CodexCredentialRefreshService { leases, credential_state, runtime_policy, + diagnostics, } } @@ -239,11 +241,21 @@ impl CodexCredentialRefreshService { match result { Ok(outcome) => outcomes.push(outcome), Err(error) => { - tracing::warn!( - account_id = %account_id, - error = %error, - "OpenAI OAuth refresh attempt failed" + let mut failure = gateway_core::diagnostics::OperationalFailure::new( + "oauth", + "scheduled_refresh", + "refresh_storage", + "OpenAI OAuth refresh attempt failed", ); + failure.account_id = ProviderAccountId::new(account_id.clone()).ok(); + failure.details = gateway_core::error::ErrorDetails::capture( + Some(&ErrorSource::new(error)), + None, + false, + ); + if self.diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(account_id = %account_id, "OAuth diagnostic could not be recorded"); + } outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id }); } } @@ -294,6 +306,15 @@ impl CodexCredentialRefreshService { .refresher .refresh_with_proxy(refresh_token.expose_secret(), due.account.outbound_proxy()) .await; + if let Err(failure) = &refresh_result { + super::diagnostics::record_refresh_failure( + self.diagnostics.as_ref(), + due.account.id(), + "scheduled_refresh", + failure, + ) + .await; + } if recovery_window_exhausted && let Err(failure) = &refresh_result { let message = failure.message().map(str::to_owned); let upstream = failure.upstream(); @@ -356,7 +377,7 @@ impl CodexCredentialRefreshService { ) .await } - Err(RefreshFailure::RetryableTransport { message }) => { + Err(RefreshFailure::RetryableTransport { message, .. }) => { if self .defer_refresh(&due.account, "transport-not-sent", Some(&message), None) .await? @@ -366,7 +387,9 @@ impl CodexCredentialRefreshService { Ok(CodexCredentialRefreshOutcome::Stale { account_id }) } } - Err(RefreshFailure::Transport { message, upstream }) => { + Err(RefreshFailure::Transport { + message, upstream, .. + }) => { // 上游瞬态(401/429/5xx/超时/畸形响应等)保留现有凭据、 // 记录最近一次失败并推进有界退避 if self @@ -576,11 +599,11 @@ impl CodexCredentialRefreshService { account_id = %account.id(), ?credential_state, reason, - upstream_message = ?message.as_deref(), + upstream_status = ?upstream.map(RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(RefreshUpstreamFailure::body), + + + access_token_expires_at = ?account.access_token_expires_at() .map(DateTime::::from), recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at()) @@ -637,7 +660,6 @@ impl CodexCredentialRefreshService { account.access_token_expires_at(), attempt, reason, - upstream_message, upstream, retry_at, ); diff --git a/backend/crates/providers/openai/src/credential/token_client.rs b/backend/crates/providers/openai/src/credential/token_client.rs index 0560e415d..177b77da9 100644 --- a/backend/crates/providers/openai/src/credential/token_client.rs +++ b/backend/crates/providers/openai/src/credential/token_client.rs @@ -81,7 +81,7 @@ impl fmt::Debug for TokenPair { } /// Codex token 刷新的稳定失败分类 -#[derive(Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Clone, thiserror::Error)] pub enum RefreshFailure { #[error("refresh token is invalid or expired")] InvalidGrant { @@ -94,11 +94,19 @@ pub enum RefreshFailure { upstream: Option>, }, #[error("refresh transport failed before server processing")] - RetryableTransport { message: String }, + RetryableTransport { + message: String, + #[source] + source: Option, + redacted: bool, + }, #[error("refresh transport failed after possible server processing")] Transport { message: Option, upstream: Option>, + #[source] + source: Option, + redacted: bool, }, } @@ -120,7 +128,7 @@ impl RefreshFailure { Self::InvalidGrant { message, .. } | Self::Banned { message, .. } | Self::Transport { message, .. } => message.as_deref(), - Self::RetryableTransport { message } => Some(message), + Self::RetryableTransport { message, .. } => Some(message), } } @@ -134,6 +142,47 @@ impl RefreshFailure { } } + pub(crate) fn redacted(&self) -> bool { + self.upstream() + .is_some_and(RefreshUpstreamFailure::redacted) + || match self { + Self::RetryableTransport { redacted, .. } | Self::Transport { redacted, .. } => { + *redacted + } + _ => false, + } + } + + fn redact_refresh_token(&mut self, secret: &str) { + if secret.is_empty() { + return; + } + match self { + Self::InvalidGrant { message, upstream } + | Self::Banned { message, upstream } + | Self::Transport { + message, upstream, .. + } => { + if let Some(message) = message { + *message = message.replace(secret, "[REDACTED]"); + } + if let Some(upstream) = upstream { + for value in [&mut upstream.body] + .into_iter() + .chain(upstream.code.iter_mut()) + .chain(upstream.error_type.iter_mut()) + { + if value.contains(secret) { + *value = value.replace(secret, "[REDACTED]"); + upstream.redacted = true; + } + } + } + } + Self::RetryableTransport { .. } => {} + } + } + #[must_use] pub const fn classification(&self) -> &'static str { match self { @@ -147,18 +196,28 @@ impl RefreshFailure { /// 当前 OAuth 刷新请求收到的完整非成功响应 /// -/// 该值不持久化;`Debug` 不输出正文 -/// 调用方仅在受控刷新失败日志中显式记录正文 +/// 原文只进入受控诊断详情;普通日志不展开正文或任意上游值 #[derive(Clone, PartialEq, Eq)] pub struct RefreshUpstreamFailure { status: u16, code: Option, error_type: Option, body: String, + redacted: bool, } impl RefreshUpstreamFailure { fn new(status: StatusCode, body: &[u8], error: Option<&RefreshErrorResponse>) -> Self { + let mut text = String::from_utf8_lossy(body).into_owned(); + let redacted = if let Ok(mut value) = serde_json::from_slice::(body) { + let changed = redact_oauth_tokens(&mut value); + if changed { + text = value.to_string(); + } + changed + } else { + false + }; Self { status: status.as_u16(), code: error @@ -167,10 +226,16 @@ impl RefreshUpstreamFailure { error_type: error .and_then(RefreshErrorResponse::error_type) .map(str::to_owned), - body: String::from_utf8_lossy(body).into_owned(), + body: text, + redacted, } } + #[must_use] + pub const fn redacted(&self) -> bool { + self.redacted + } + #[must_use] pub const fn status(&self) -> u16 { self.status @@ -197,8 +262,11 @@ impl fmt::Debug for RefreshUpstreamFailure { formatter .debug_struct("RefreshUpstreamFailure") .field("status", &self.status) - .field("code", &self.code) - .field("error_type", &self.error_type) + .field("code", &self.code.as_ref().map(|_| "")) + .field( + "error_type", + &self.error_type.as_ref().map(|_| ""), + ) .field("body", &"") .finish() } @@ -222,6 +290,8 @@ pub trait TokenRefresher: Send + Sync + 'static { fn proxy_refresh_failure() -> RefreshFailure { RefreshFailure::RetryableTransport { + redacted: false, + source: None, message: "account OAuth egress unavailable".to_owned(), } } @@ -503,13 +573,19 @@ impl TokenRefresher for OpenAiTokenClient { proxy: Option<&gateway_core::account::OutboundProxy>, ) -> Result { self.with_proxy(proxy) - .map_err(|_| proxy_refresh_failure())? + .map_err(|source| RefreshFailure::RetryableTransport { + redacted: false, + message: "account OAuth egress unavailable".to_owned(), + source: Some(source.into()), + })? .refresh(refresh_token) .await } async fn refresh(&self, refresh_token: &str) -> Result { - let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|_| { + let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|source| { RefreshFailure::Transport { + redacted: false, + source: Some(source.into()), message: Some("OpenAI OAuth refresh profile is invalid".to_owned()), upstream: None, } @@ -525,12 +601,16 @@ impl TokenRefresher for OpenAiTokenClient { }) .send() .await - .map_err(|error| refresh_transport_failure(&error))?; + .map_err(refresh_transport_failure)?; let (status, body) = read_bounded_response(response).await?; if !status.is_success() { - return Err(classify_refresh_failure(status, &body)); + let mut failure = classify_refresh_failure(status, &body); + failure.redact_refresh_token(refresh_token); + return Err(failure); } - parse_token_pair(&body).map_err(|()| RefreshFailure::Transport { + parse_token_pair(&body).map_err(|source| RefreshFailure::Transport { + redacted: true, + source: Some(source), message: Some("OpenAI OAuth refresh returned an invalid success response".to_owned()), upstream: None, }) @@ -604,8 +684,10 @@ async fn read_bounded_response( .chunk() .await .map_err(|error| RefreshFailure::Transport { - message: Some(error.to_string()), + redacted: error.url().is_some(), + message: Some("OpenAI OAuth response body read failed".to_owned()), upstream: None, + source: Some(error.without_url().into()), })? { let next_len = body @@ -617,6 +699,8 @@ async fn read_bounded_response( "OpenAI OAuth response exceeded {MAX_OAUTH_RESPONSE_BYTES} bytes" )), upstream: None, + source: None, + redacted: true, })?; body.reserve(next_len.saturating_sub(body.len())); body.extend_from_slice(&chunk); @@ -624,14 +708,19 @@ async fn read_bounded_response( Ok((status, body)) } -fn parse_token_pair(body: &[u8]) -> Result { - let tokens = serde_json::from_slice::(body).map_err(|_| ())?; +#[derive(Debug, thiserror::Error)] +#[error("OpenAI OAuth ID token claims are invalid")] +struct InvalidRefreshClaims; + +fn parse_token_pair(body: &[u8]) -> Result { + let tokens = serde_json::from_slice::(body) + .map_err(gateway_core::error::ErrorSource::new)?; if tokens .id_token .as_deref() .is_some_and(|token| super::types::parse_chatgpt_jwt_claims(token).is_err()) { - return Err(()); + return Err(InvalidRefreshClaims.into()); } Ok(TokenPair { access_token: tokens.access_token, @@ -662,6 +751,8 @@ fn classify_refresh_failure(status: StatusCode, body: &[u8]) -> RefreshFailure { // 显式 401 先进入有界恢复退避,避免瞬时授权故障直接失效账号 if status == StatusCode::UNAUTHORIZED { return RefreshFailure::Transport { + redacted: false, + source: None, message, upstream: upstream(), }; @@ -687,19 +778,30 @@ fn classify_refresh_failure(status: StatusCode, body: &[u8]) -> RefreshFailure { }; } RefreshFailure::Transport { + redacted: false, + source: None, message, upstream: upstream(), } } -fn refresh_transport_failure(error: &reqwest::Error) -> RefreshFailure { - let message = error.to_string(); - if is_safe_to_retry_refresh_transport(error) { - RefreshFailure::RetryableTransport { message } +fn refresh_transport_failure(error: reqwest::Error) -> RefreshFailure { + let retryable = is_safe_to_retry_refresh_transport(&error); + let redacted = error.url().is_some(); + let source = Some(error.without_url().into()); + let message = "OpenAI OAuth transport failed".to_owned(); + if retryable { + RefreshFailure::RetryableTransport { + message, + source, + redacted, + } } else { RefreshFailure::Transport { message: Some(message), upstream: None, + source, + redacted, } } } @@ -718,3 +820,36 @@ fn is_safe_to_retry_refresh_transport(error: &reqwest::Error) -> bool { || message.contains("network is unreachable") || message.contains("tls handshake") } + +fn redact_oauth_tokens(value: &mut serde_json::Value) -> bool { + let mut redacted = false; + match value { + serde_json::Value::Object(fields) => { + for (key, value) in fields { + let sensitive = [ + "access_token", + "refresh_token", + "id_token", + "client_secret", + "authorization", + "cookie", + ] + .iter() + .any(|field| key.eq_ignore_ascii_case(field)); + if sensitive && !value.is_null() { + *value = serde_json::Value::String("[REDACTED]".to_owned()); + redacted = true; + } else { + redacted |= redact_oauth_tokens(value); + } + } + } + serde_json::Value::Array(values) => { + for value in values { + redacted |= redact_oauth_tokens(value); + } + } + _ => {} + } + redacted +} diff --git a/backend/crates/providers/openai/src/lib.rs b/backend/crates/providers/openai/src/lib.rs index a48c02f27..d1506bbf0 100644 --- a/backend/crates/providers/openai/src/lib.rs +++ b/backend/crates/providers/openai/src/lib.rs @@ -199,6 +199,7 @@ pub async fn initialize( Arc::clone(&refresher), Arc::clone(&leases), Arc::clone(&runtime_policy), + ports.diagnostics(), ) .with_personal_access_token_client(token_client), ); @@ -208,6 +209,7 @@ pub async fn initialize( Arc::clone(&leases), credential_state, Arc::clone(&runtime_policy), + ports.diagnostics(), )); let pending = Arc::new(OpenAiOAuthPendingStore::new( ports.oauth_pending(), diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index bc2bd85f5..2fb6226ab 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -710,9 +710,6 @@ pub(super) fn continuation_replay_required_error(reason: &'static str) -> Provid ContinuationRecoveryDisposition::ClientReplayRequired, ) .with_continuation_unavailable_reason(reason) - .with_upstream_code(OpaqueUpstreamValue::new( - PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned(), - )) .with_diagnostic(ProviderDiagnostic::new( "previous response is unavailable in the selected upstream scope; client replay is required", )) @@ -974,30 +971,20 @@ pub(super) fn map_client_error( failure } CodexClientError::WebSocket(error) => { - let close_code = error.close_before_terminal().and_then(|close| close.code()); let client_visible_error = websocket_client_visible_error(error); let mut failure = MappedProviderFailure::plain(provider_error( websocket_error_kind(error), websocket_send_state(error), )); - if let Some(close_code) = close_code { - failure.error = - failure - .error - .with_upstream_code(OpaqueUpstreamValue::new(format!( - "websocket_close_{close_code}" - ))); - } if let CodexWebSocketExchangeError::ConnectionLimitReached(upstream) = error.classified() { - failure.error = - failure + failure.error = failure.error.with_replay_safe(); + if let Some(code) = upstream.upstream_code.as_ref() { + failure.error = failure .error - .with_replay_safe() - .with_upstream_code(OpaqueUpstreamValue::new( - WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(), - )); + .with_upstream_code(OpaqueUpstreamValue::new(code.clone())); + } if let Some(status) = upstream.explicit_status_code { failure.error = failure.error.with_status(status); } diff --git a/backend/crates/providers/openai/src/provider/mod.rs b/backend/crates/providers/openai/src/provider/mod.rs index 5fe833688..bb34992e6 100644 --- a/backend/crates/providers/openai/src/provider/mod.rs +++ b/backend/crates/providers/openai/src/provider/mod.rs @@ -82,9 +82,7 @@ use crate::transport::protocol::responses::{ CodexResponsesRequest, PREVIOUS_RESPONSE_NOT_FOUND_CODE, PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE, PreviousResponseScope, ResponseEventSignals, TransportRequirement, transport_requirement, }; -use crate::transport::protocol::websocket::{ - WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE, websocket_response_create_payload_len, -}; +use crate::transport::protocol::websocket::websocket_response_create_payload_len; use crate::transport::request::{ CodexRequestEncodeError, RequestAccountScope, align_structured_location_fields, clear_request_turn_state, encode_generate_request, scope_request_to_account, diff --git a/backend/crates/providers/openai/src/provider/workers.rs b/backend/crates/providers/openai/src/provider/workers.rs index 96c4c07ea..c48342aec 100644 --- a/backend/crates/providers/openai/src/provider/workers.rs +++ b/backend/crates/providers/openai/src/provider/workers.rs @@ -141,8 +141,7 @@ impl ScheduledTask for OpenAiOAuthRefreshTask { return Ok(()); } let outcomes = self.service.refresh_due().await.map_err(|error| { - tracing::error!(error = %error, "OpenAI OAuth refresh cycle failed"); - WorkerTaskError::safe("OpenAI OAuth refresh failed") + WorkerTaskError::safe("OpenAI OAuth refresh failed").with_source(error) })?; let mut refreshed = 0_u64; let mut invalidated = 0_u64; @@ -259,11 +258,8 @@ impl ScheduledTask for OpenAiQuotaTask { } Ok(_) => {} Err(error) => { - tracing::warn!( - error = %error, - "OpenAI quota synchronization failed" - ); - return Err(WorkerTaskError::safe("OpenAI quota synchronization failed")); + return Err(WorkerTaskError::safe("OpenAI quota synchronization failed") + .with_source(error)); } } Ok(()) @@ -279,12 +275,10 @@ impl ScheduledTask for OpenAiCatalogTask { } let result = match self.catalog.refresh_catalogs().await { Ok(_) | Err(CodexCredentialCatalogError::NoEligibleCredential) => Ok(()), - Err(error) => { - tracing::warn!(error = %error, "OpenAI model catalog refresh failed"); - Err(WorkerTaskError::safe( - "OpenAI model catalog synchronization failed", - )) - } + Err(error) => Err(WorkerTaskError::safe( + "OpenAI model catalog synchronization failed", + ) + .with_source(error)), }; // 成功周期尾部追加 [0, 20%) 单侧随机抖动,打破固定周期轮询特征; // 放在周期尾部避免延迟冷启动首轮刷新,失败路径交给宿主退避不叠加。 @@ -385,8 +379,7 @@ impl ScheduledTask for OpenAiWarmupTask { .load_warmup_policy() .await .map_err(|error| { - tracing::warn!(error = %error, "OpenAI warmup policy load failed"); - WorkerTaskError::safe("OpenAI warmup policy load failed") + WorkerTaskError::safe("OpenAI warmup policy load failed").with_source(error) })?; if !policy.enabled() { return Ok(()); @@ -424,7 +417,9 @@ impl ScheduledTask for OpenAiWarmupTask { .runtime_policy() .claim_warmup_slot(self.timezone, slot) .await - .map_err(|_| WorkerTaskError::safe("warmup slot is unavailable"))? + .map_err(|source| { + WorkerTaskError::safe("warmup slot is unavailable").with_source(source) + })? { return Ok(()); } diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index 7fad73770..9946579d9 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -1729,6 +1729,7 @@ fn provider_ports_with_catalog( Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), pending, + Arc::new(RecordingDiagnostics::default()), ) } @@ -2530,6 +2531,7 @@ mod errors { Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ) } @@ -2736,3 +2738,16 @@ async fn oauth_transport_settings_preserve_tokens_refresh_schedule_and_health() json!({"transport":"http"}).as_object().unwrap() ); } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/openai/tests/credential/admin.rs b/backend/crates/providers/openai/tests/credential/admin.rs index 008c0c356..6ad1aad92 100644 --- a/backend/crates/providers/openai/tests/credential/admin.rs +++ b/backend/crates/providers/openai/tests/credential/admin.rs @@ -30,6 +30,7 @@ async fn sub2api_import_resolves_distinct_proxy_bindings_and_encodes_credentials Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service.prepare_import_document(serde_json::json!({"data": { "accounts": [ @@ -64,6 +65,7 @@ async fn import_default_proxy_preserves_explicit_account_exits() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let default_proxy = gateway_core::account::OutboundProxy::parse("http://default.example:8080").unwrap(); @@ -112,6 +114,7 @@ async fn sub2api_invalid_proxy_bindings_are_rejected_before_any_token_refresh() Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let proxy = serde_json::json!({"proxy_key": "bound", "protocol": "http", "host": "127.0.0.1", "port": 18080, "status": "active"}); let mut cases = vec![ @@ -152,6 +155,7 @@ async fn direct_import_persists_opaque_tokens_without_profile_or_token_validatio Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -193,6 +197,7 @@ async fn direct_import_accepts_snake_case_oauth_token_aliases() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -246,6 +251,7 @@ async fn direct_import_projects_access_token_jwt_expiry_without_persisting_refre Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -286,6 +292,7 @@ fn pat_service(server: &MockServer) -> CodexCredentialAdminService { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_personal_access_token_client(Arc::new(client)) } @@ -397,6 +404,7 @@ async fn pat_import_times_out_without_falling_back_to_document_identity() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_personal_access_token_client(Arc::new(client)); let error = service @@ -419,6 +427,7 @@ async fn pat_import_requires_a_validation_client() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = service .prepare_import_document(serde_json::json!({ @@ -679,6 +688,7 @@ async fn oauth_import_uses_official_chatgpt_access_token_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -707,6 +717,7 @@ async fn oauth_import_uses_official_plan_alias_projection_and_user_id_fallback() Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -726,6 +737,7 @@ async fn oauth_import_should_preserve_pro_max_and_future_plan_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for plan in ["prolite", "pro", "promax", "future_plan"] { let access_token = test_jwt(serde_json::json!({"https://api.openai.com/auth":{ @@ -757,6 +769,7 @@ async fn oauth_import_uses_id_token_then_access_token_for_missing_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -780,6 +793,7 @@ async fn oauth_import_does_not_use_top_level_identity_fields() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -805,6 +819,7 @@ async fn oauth_import_rejects_legacy_bare_token_field() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = service .prepare_import_document(serde_json::json!({ @@ -869,6 +884,7 @@ async fn api_key_import_export_preserves_target_without_oauth_exchange() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let imported = service.prepare_import_document(serde_json::json!({ "provider": "openai", "authentication_kind": "api_key", "name": "relay", "base_url": "https://relay.example/custom/v2", "api_key": "sk-test-only" @@ -922,6 +938,7 @@ async fn api_key_import_rejects_unsafe_urls_and_empty_keys() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for (url, key) in [ ("http://remote.example/v1", "sk-test"), @@ -947,6 +964,7 @@ async fn sub2api_api_key_import_preserves_versioned_and_explicit_responses_paths Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for (base, expected) in [ ("https://example.com", "https://example.com/v1"), diff --git a/backend/crates/providers/openai/tests/credential/refresh.rs b/backend/crates/providers/openai/tests/credential/refresh.rs index 17f5e1dc8..1ae8824ff 100644 --- a/backend/crates/providers/openai/tests/credential/refresh.rs +++ b/backend/crates/providers/openai/tests/credential/refresh.rs @@ -97,6 +97,8 @@ impl TokenRefresher for SingleUseRefresher { .expect("refresh response lock") .take() .ok_or_else(|| RefreshFailure::Transport { + redacted: false, + source: None, message: Some("test refresh response is exhausted".to_owned()), upstream: None, }) @@ -220,6 +222,7 @@ fn refresh_service( Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), runtime_policy, + Arc::new(crate::RecordingDiagnostics::default()), ) } @@ -302,6 +305,7 @@ async fn scheduled_refresh_stops_retrying_rejected_disabled_credentials() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), MutableRuntimePolicy::new(Duration::from_secs(5 * 60)), + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_disabled_rejected"; seed_refreshable_account(&store, account_id, SystemTime::now(), None).await; @@ -507,6 +511,7 @@ async fn scheduled_refresh_persists_the_original_upstream_error_message() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_invalid_refresh_token"; seed_refreshable_account( @@ -570,6 +575,7 @@ async fn scheduled_refresh_persists_retryable_message_inside_the_two_hour_window Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_retryable_unauthorized"; let expires_at = SystemTime::now() @@ -614,6 +620,8 @@ async fn scheduled_refresh_uses_the_final_message_after_the_two_hour_window() { store.repository(), Arc::new(FailingRefresher { failure: RefreshFailure::Transport { + redacted: false, + source: None, message: Some(upstream_message.to_owned()), upstream: None, }, @@ -621,6 +629,7 @@ async fn scheduled_refresh_uses_the_final_message_after_the_two_hour_window() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_exhausted_unauthorized"; seed_refreshable_account( @@ -935,6 +944,7 @@ async fn quota_rejection_during_oauth_refresh_preserves_terminal_result() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), MutableRuntimePolicy::new(Duration::from_secs(5 * 60)), + Arc::new(crate::RecordingDiagnostics::default()), ); let outcomes = service.refresh_due().await.expect("refresh cycle"); assert!( @@ -952,3 +962,46 @@ async fn quota_rejection_during_oauth_refresh_preserves_terminal_result() { ); } } + +#[tokio::test] +async fn scheduled_refresh_records_native_failure_once_and_keeps_retry_state() { + let store = Arc::new(MemoryAccountStore::default()); + let diagnostics = Arc::new(crate::RecordingDiagnostics::default()); + let service = CodexCredentialRefreshService::new( + store.repository(), + Arc::new(FailingRefresher { + failure: RefreshFailure::Transport { + redacted: false, + message: Some("safe transport summary".to_owned()), + upstream: None, + source: Some(std::io::Error::other("PRIVATE_OAUTH_NATIVE_CAUSE").into()), + }, + }), + Arc::new(RefreshLeases), + Arc::new(RefreshCredentialState), + MutableRuntimePolicy::new(Duration::from_secs(300)), + diagnostics.clone(), + ); + seed_refreshable_account(&store, "acct_diagnostic", SystemTime::now(), None).await; + let outcomes = service.refresh_due().await.unwrap(); + assert!(matches!( + outcomes.as_slice(), + [CodexCredentialRefreshOutcome::Transient { .. }] + )); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].account_id.as_ref().unwrap().as_str(), + "acct_diagnostic" + ); + assert_eq!(failures[0].operation, "scheduled_refresh"); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_OAUTH_NATIVE_CAUSE") + ); + assert!(!format!("{:?}", failures[0]).contains("PRIVATE_OAUTH_NATIVE_CAUSE")); +} diff --git a/backend/crates/providers/openai/tests/credential/token_client.rs b/backend/crates/providers/openai/tests/credential/token_client.rs index fe23fb365..f0581546d 100644 --- a/backend/crates/providers/openai/tests/credential/token_client.rs +++ b/backend/crates/providers/openai/tests/credential/token_client.rs @@ -423,7 +423,10 @@ async fn generic_invalid_grant_should_remain_transient_like_official_codex() { let failure = refresh_failure(400, body).await; let diagnostic = format!("{failure:?} {failure}"); - let RefreshFailure::Transport { message, upstream } = failure else { + let RefreshFailure::Transport { + message, upstream, .. + } = failure + else { panic!("unknown official refresh code must remain transient"); }; assert_eq!(message, None); @@ -445,7 +448,10 @@ async fn unauthorized_should_preserve_upstream_details_and_remain_retryable() { }"#; let failure = refresh_failure(401, body).await; - let RefreshFailure::Transport { message, upstream } = failure else { + let RefreshFailure::Transport { + message, upstream, .. + } = failure + else { panic!("production policy gives every 401 a bounded recovery window"); }; assert_eq!(message.as_deref(), Some("Invalid refresh token.")); @@ -548,6 +554,7 @@ fn assert_transport_failure( let RefreshFailure::Transport { message: actual_message, upstream, + .. } = failure else { panic!("status {status} must classify as transient"); @@ -572,3 +579,47 @@ async fn refresh_failure(status: u16, body: &str) -> RefreshFailure { .await .expect_err("refresh must fail") } + +#[tokio::test] +async fn oauth_failure_retains_unknown_code_and_redacts_token_material() { + let server = MockServer::start().await; + Mock::given(method("POST")).and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(503).set_body_json(serde_json::json!({ + "error": {"code":"FUTURE_OAUTH_FAILURE","message":"request PRIVATE_REFRESH_INPUT failed"}, + "access_token":"PRIVATE_ACCESS_OUTPUT", "nested":[{"id_token":"PRIVATE_ID_OUTPUT"}, {"refresh_token":"PRIVATE_SECOND_OUTPUT"}] + }))).mount(&server).await; + let error = client(&server) + .refresh("PRIVATE_REFRESH_INPUT") + .await + .unwrap_err(); + assert_eq!(error.classification(), "transport-ambiguous"); + let upstream = error.upstream().unwrap(); + assert_eq!(upstream.status(), 503); + assert_eq!(upstream.code(), Some("FUTURE_OAUTH_FAILURE")); + assert!(upstream.redacted()); + assert!(upstream.body().contains("FUTURE_OAUTH_FAILURE")); + assert!(!upstream.body().contains("PRIVATE_")); + assert!(!format!("{error:?} {error}").contains("FUTURE_OAUTH_FAILURE")); +} + +#[tokio::test] +async fn malformed_success_retains_json_source_without_retaining_token_body() { + use std::error::Error as _; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("{\"access_token\":\"PRIVATE_ACCESS\", invalid"), + ) + .mount(&server) + .await; + let error = client(&server).refresh("refresh").await.unwrap_err(); + let source = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(source.column() > 0); + assert!(error.upstream().is_none()); + assert!(!format!("{error:?} {error}").contains("PRIVATE_ACCESS")); +} diff --git a/backend/crates/providers/openai/tests/main.rs b/backend/crates/providers/openai/tests/main.rs index 4d5a35f07..1c5432eb6 100644 --- a/backend/crates/providers/openai/tests/main.rs +++ b/backend/crates/providers/openai/tests/main.rs @@ -7,3 +7,16 @@ mod jitter; mod provider; mod support; mod transport; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/openai/tests/provider/contract/mod.rs b/backend/crates/providers/openai/tests/provider/contract/mod.rs index 38a3087cd..de7ec2c1a 100644 --- a/backend/crates/providers/openai/tests/provider/contract/mod.rs +++ b/backend/crates/providers/openai/tests/provider/contract/mod.rs @@ -3846,10 +3846,11 @@ async fn repeated_message_too_big_closes_keep_session_on_websocket() { assert_eq!(detail.message(), "message too big"); assert_eq!(detail.code(), Some("message_too_big")); assert_eq!(detail.error_type(), Some("invalid_request_error")); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1009") - ); + assert!(error.upstream_code().is_none()); + let close: Value = + serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1009); } let second_operation = Operation::Generate(generate_with_persisted_session_context( @@ -4653,10 +4654,10 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation() assert!(saw_websocket_observation); assert_eq!(error.send_state(), UpstreamSendState::Ambiguous); assert_eq!(error.pre_delivery_retry(), None); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1000") - ); + assert!(error.upstream_code().is_none()); + let close: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1000); assert_eq!( error.diagnostic().and_then(|diagnostic| diagnostic.code()), Some("upstream_close") @@ -5956,10 +5957,10 @@ async fn websocket_turn_state_metadata_close_does_not_authorize_replay() { assert_eq!(error.send_state(), UpstreamSendState::Ambiguous); assert!(!error.replay_is_safe()); assert_eq!(error.pre_delivery_retry(), None); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1000") - ); + assert!(error.upstream_code().is_none()); + let close: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1000); assert_eq!( error.diagnostic().map(|diagnostic| diagnostic.as_str()), Some( diff --git a/backend/crates/providers/xai/src/admin.rs b/backend/crates/providers/xai/src/admin.rs index 39415b468..448d3bc66 100644 --- a/backend/crates/providers/xai/src/admin.rs +++ b/backend/crates/providers/xai/src/admin.rs @@ -1491,21 +1491,23 @@ fn map_store_error(error: gateway_core::error::StoreError) -> ProviderAdminError StoreErrorKind::Unavailable => ProviderAdminErrorKind::Unavailable, _ => ProviderAdminErrorKind::Internal, }) + .with_source(error) } fn map_repository_error(error: GrokCredentialRepositoryError) -> ProviderAdminError { use GrokCredentialRepositoryError as Error; - provider_error(match error { + provider_error(match &error { Error::InvalidInput(_) | Error::WrongProviderKind | Error::IdentityRebind - | Error::InvalidCredentialData => ProviderAdminErrorKind::Invalid, + | Error::InvalidCredentialData(_) => ProviderAdminErrorKind::Invalid, Error::CredentialNotFound => ProviderAdminErrorKind::NotFound, - Error::StaleCredentialRevision | Error::Conflict | Error::RevisionOverflow => { + Error::StaleCredentialRevision(_) | Error::Conflict(_) | Error::RevisionOverflow => { ProviderAdminErrorKind::Conflict } - Error::Store => ProviderAdminErrorKind::Unavailable, + Error::Store(_) => ProviderAdminErrorKind::Unavailable, }) + .with_source(error) } fn map_refresh_error(error: GrokCredentialRefreshError) -> ProviderAdminError { diff --git a/backend/crates/providers/xai/src/credential/catalog.rs b/backend/crates/providers/xai/src/credential/catalog.rs index 92bd13cc8..35dfb578f 100644 --- a/backend/crates/providers/xai/src/credential/catalog.rs +++ b/backend/crates/providers/xai/src/credential/catalog.rs @@ -1394,13 +1394,13 @@ fn map_quota_repository_error( RepositoryError::CredentialNotFound | RepositoryError::WrongProviderKind => { GrokQuotaError::AccountUnavailable } - RepositoryError::StaleCredentialRevision | RepositoryError::Conflict => { + RepositoryError::StaleCredentialRevision(_) | RepositoryError::Conflict(_) => { GrokQuotaError::StaleCredentialSnapshot } - RepositoryError::Store => GrokQuotaError::Store, + RepositoryError::Store(_) => GrokQuotaError::Store, RepositoryError::InvalidInput(_) | RepositoryError::IdentityRebind - | RepositoryError::InvalidCredentialData + | RepositoryError::InvalidCredentialData(_) | RepositoryError::RevisionOverflow => GrokQuotaError::InvalidData, } } diff --git a/backend/crates/providers/xai/src/credential/refresh.rs b/backend/crates/providers/xai/src/credential/refresh.rs index 9f84ef1fd..fde6a45ee 100644 --- a/backend/crates/providers/xai/src/credential/refresh.rs +++ b/backend/crates/providers/xai/src/credential/refresh.rs @@ -401,7 +401,7 @@ impl GrokCredentialRefreshService { let subject = loaded .account .upstream_user_id() - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_owned(); let credential = DueGrokCredential { account_id: account_id.clone(), @@ -487,7 +487,7 @@ impl GrokCredentialRefreshService { let subject = loaded .account .upstream_user_id() - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_owned(); let prepared = GrokCredentialAdmin .prepare_rotation(&RotateManagedGrokCredential { @@ -918,10 +918,13 @@ impl GrokCredentialRepository { .cloned() .collect::>(); let now = SystemTime::now(); - let provider = ProviderKind::new(PROVIDER_NAME) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let limit = NonZeroU32::new(MAX_REFRESH_BATCH) - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)?; + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))?; let query = ProviderRefreshQuery::new( provider, now.checked_add(policy.margin()).unwrap_or(now), @@ -1015,7 +1018,7 @@ fn stale_repository_error(error: &GrokCredentialRepositoryError) -> bool { matches!( error, GrokCredentialRepositoryError::CredentialNotFound - | GrokCredentialRepositoryError::Conflict - | GrokCredentialRepositoryError::StaleCredentialRevision + | GrokCredentialRepositoryError::Conflict(_) + | GrokCredentialRepositoryError::StaleCredentialRevision(_) ) } diff --git a/backend/crates/providers/xai/src/credential/repository.rs b/backend/crates/providers/xai/src/credential/repository.rs index 9a82aa20f..3e4446961 100644 --- a/backend/crates/providers/xai/src/credential/repository.rs +++ b/backend/crates/providers/xai/src/credential/repository.rs @@ -256,7 +256,7 @@ impl GrokCredentialAdmin { .account .access_token_expires_at() .map(DateTime::::from) - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_rfc3339(), ), ); @@ -398,7 +398,7 @@ impl GrokCredentialRepository { .await .map_err(map_store_error)?; let CredentialCasOutcome::Updated(revision) = outcome else { - return Err(GrokCredentialRepositoryError::StaleCredentialRevision); + return Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)); }; Ok(GrokCredentialRecord { @@ -435,7 +435,11 @@ impl GrokCredentialRepository { current.account.access_token_expires_at(), Some(next_refresh_at), ) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; match self .store .compare_and_swap_credential(update) @@ -444,7 +448,7 @@ impl GrokCredentialRepository { { CredentialCasOutcome::Updated(_) => Ok(()), CredentialCasOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -501,8 +505,11 @@ impl GrokCredentialRepository { pub(crate) async fn list_loaded_for_provider( &self, ) -> Result, GrokCredentialRepositoryError> { - let provider = ProviderKind::new(XAI_PROVIDER_KIND) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(XAI_PROVIDER_KIND).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let accounts = self .store .list_for_provider(&provider) @@ -520,8 +527,11 @@ impl GrokCredentialRepository { pub(crate) async fn list_accounts_for_provider( &self, ) -> Result, GrokCredentialRepositoryError> { - let provider = ProviderKind::new(XAI_PROVIDER_KIND) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(XAI_PROVIDER_KIND).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let accounts = self .store .list_for_provider(&provider) @@ -568,7 +578,7 @@ impl GrokCredentialRepository { match outcome { QuotaWriteOutcome::Updated => Ok(()), QuotaWriteOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -591,7 +601,7 @@ impl GrokCredentialRepository { match outcome { QuotaWriteOutcome::Updated => Ok(()), QuotaWriteOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -673,12 +683,20 @@ fn encode_secret( scope: &secret.scope, refresh_token_expires_at: account.refresh_token_expires_at, }) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let Value::Object(object) = value else { - return Err(GrokCredentialRepositoryError::InvalidCredentialData); + return Err(GrokCredentialRepositoryError::InvalidCredentialData(None)); }; if serde_json::to_vec(&object) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)? + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .len() > MAX_SECRET_BYTES { @@ -698,7 +716,7 @@ fn prepare_rotation( validate_secret(secret)?; ensure_xai(current)?; if current.revision() != expected_revision { - return Err(GrokCredentialRepositoryError::StaleCredentialRevision); + return Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)); } if current.upstream_user_id() != Some(verified_account.subject.as_str()) || current.upstream_account_id() != verified_account.upstream_account_id.as_deref() @@ -730,7 +748,11 @@ fn prepare_rotation( Some(to_system_time(verified_account.access_token_expires_at)), None, ) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; Ok(PreparedGrokCredentialRotation::new(profile, credential)) } @@ -738,15 +760,18 @@ fn decode_secret( credential: &PlaintextCredential, ) -> Result { let value = Value::Object(credential.expose_to_provider().clone()); - let wire: OwnedGrokOAuthSecretWire = serde_json::from_value(value) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let wire: OwnedGrokOAuthSecretWire = serde_json::from_value(value).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if wire.schema_version != CREDENTIAL_SCHEMA_VERSION || wire.auth_method != OAUTH_AUTH_METHOD || wire.access_token.is_empty() || wire.refresh_token.is_empty() || validate_scope(&wire.scope).is_err() { - return Err(GrokCredentialRepositoryError::InvalidCredentialData); + return Err(GrokCredentialRepositoryError::InvalidCredentialData(None)); } Ok(wire) } @@ -861,13 +886,15 @@ fn ensure_xai(account: &ProviderAccount) -> Result<(), GrokCredentialRepositoryE } fn map_store_error(error: gateway_core::error::StoreError) -> GrokCredentialRepositoryError { + use gateway_core::error::ErrorSource; match error.kind() { - StoreErrorKind::Conflict => GrokCredentialRepositoryError::Conflict, - StoreErrorKind::InvalidData | StoreErrorKind::InvalidState => { - GrokCredentialRepositoryError::InvalidCredentialData + StoreErrorKind::Conflict => { + GrokCredentialRepositoryError::Conflict(Some(ErrorSource::new(error))) } - StoreErrorKind::Unavailable => GrokCredentialRepositoryError::Store, - _ => GrokCredentialRepositoryError::Store, + StoreErrorKind::InvalidData | StoreErrorKind::InvalidState => { + GrokCredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(error))) + } + _ => GrokCredentialRepositoryError::Store(error), } } @@ -875,7 +902,9 @@ fn map_rotation_load_error( error: gateway_core::error::StoreError, ) -> GrokCredentialRepositoryError { if error.kind() == StoreErrorKind::Conflict { - GrokCredentialRepositoryError::StaleCredentialRevision + GrokCredentialRepositoryError::StaleCredentialRevision(Some( + gateway_core::error::ErrorSource::new(error), + )) } else { map_store_error(error) } @@ -886,7 +915,7 @@ fn to_system_time(value: DateTime) -> SystemTime { } /// xAI Provider account adapter 的脱敏错误 -#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[derive(Debug, Error, Clone)] pub enum GrokCredentialRepositoryError { #[error("invalid xAI credential input: {0}")] InvalidInput(&'static str), @@ -895,15 +924,15 @@ pub enum GrokCredentialRepositoryError { #[error("provider account belongs to another Provider")] WrongProviderKind, #[error("credential revision is stale")] - StaleCredentialRevision, + StaleCredentialRevision(#[source] Option), #[error("verified xAI identity cannot be rebound")] IdentityRebind, #[error("xAI credential data is invalid")] - InvalidCredentialData, + InvalidCredentialData(#[source] Option), #[error("xAI credential mutation conflicts with current state")] - Conflict, + Conflict(#[source] Option), #[error("credential revision overflow")] RevisionOverflow, #[error("provider account store is unavailable")] - Store, + Store(#[source] gateway_core::error::StoreError), } diff --git a/backend/crates/providers/xai/src/credential/selector.rs b/backend/crates/providers/xai/src/credential/selector.rs index 375eaf7db..f089575a0 100644 --- a/backend/crates/providers/xai/src/credential/selector.rs +++ b/backend/crates/providers/xai/src/credential/selector.rs @@ -103,19 +103,27 @@ impl GrokAccountSessionSelector { let diagnostic = request.eligibility() == AccountEligibilityPolicy::BypassForDiagnostic; // store 侧常规调度列表不包含停用账号;管理端诊断要对固定账号执行真实上游 // 验证,这里把不在列表里的 required 账号显式补回候选 - let mut accounts = self - .repository - .list_accounts_for_provider() - .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + let mut accounts = + self.repository + .list_accounts_for_provider() + .await + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if diagnostic && let Some(required) = request.required_account() && !accounts.iter().any(|account| account.id() == required) - && let Some(account) = self - .repository - .account_by_id(required) - .await - .map_err(|_| GrokSessionSelectorError::Unavailable)? + && let Some(account) = + self.repository + .account_by_id(required) + .await + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? && account.provider() == &self.provider_kind { accounts.push(account); @@ -184,7 +192,11 @@ impl GrokAccountSessionSelector { &account_ids, ) .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let runtime_cooldowns = if diagnostic { std::collections::BTreeMap::new() } else { @@ -202,7 +214,7 @@ impl GrokAccountSessionSelector { .signals() .get(account.id()) .cloned() - .ok_or(GrokSessionSelectorError::Unavailable)? + .ok_or(GrokSessionSelectorError::Unavailable(None))? .with_provider_quota(self.quota.scheduling_signals(&account)) .with_runtime_health(health.0, health.1); Ok(AccountCandidate { account, signals }) @@ -293,7 +305,11 @@ impl GrokAccountSessionSelector { .with_cancellation(request.cancellation().clone()), )) .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let guard = match lease { ProviderLeaseAcquisition::Acquired(guard) => guard, ProviderLeaseAcquisition::Busy { @@ -310,21 +326,29 @@ impl GrokAccountSessionSelector { .repository .load(&selected_id, selected_revision) .await - .map_err(|_| GrokSessionSelectorError::InvalidSession)?; + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if !diagnostic && loaded .refresh_token_expires_at .is_some_and(|expires_at| expires_at <= Utc::now()) { - return Err(GrokSessionSelectorError::InvalidSession); + return Err(GrokSessionSelectorError::InvalidSession(None)); } let binding = GrokSessionBinding::new(selected_id.as_str()) - .map_err(|_| GrokSessionSelectorError::InvalidSession)? + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .with_outbound_proxy(loaded.account.outbound_proxy().cloned()); let upstream_user_id = loaded .account .upstream_user_id() - .ok_or(GrokSessionSelectorError::InvalidSession)?; + .ok_or(GrokSessionSelectorError::InvalidSession(None))?; let session = SelectedGrokSession::new( selected_id, selected_revision, @@ -337,7 +361,11 @@ impl GrokAccountSessionSelector { binding, guard, ) - .map_err(|_| GrokSessionSelectorError::InvalidSession)? + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .with_capacity_snapshot( capacity.map(AccountCapacitySnapshot::with_acquired_request), ); @@ -498,8 +526,8 @@ impl GrokAccountSessionSelector { { Ok(()) => return, Err( - GrokCredentialRepositoryError::Conflict - | GrokCredentialRepositoryError::StaleCredentialRevision, + GrokCredentialRepositoryError::Conflict(_) + | GrokCredentialRepositoryError::StaleCredentialRevision(_), ) => {} Err(error) => { tracing::warn!( diff --git a/backend/crates/providers/xai/src/provider/failure.rs b/backend/crates/providers/xai/src/provider/failure.rs index b6a9cf707..8a038c0c0 100644 --- a/backend/crates/providers/xai/src/provider/failure.rs +++ b/backend/crates/providers/xai/src/provider/failure.rs @@ -398,22 +398,31 @@ pub(super) fn map_request_error(error: GrokRequestEncodeError) -> ProviderError /// 将选择阶段失败映射为带结构化 code 与 retry_after 的 Provider 错误 pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderError { + let mapped = selection_failure(&error); + if std::error::Error::source(&error).is_some() { + mapped.with_source(error) + } else { + mapped + } +} + +fn selection_failure(error: &GrokSessionSelectorError) -> ProviderError { let (retry_after, message, code) = match error { GrokSessionSelectorError::QueueRejected(error) => { return provider_error(error.provider_kind(), UpstreamSendState::NotSent); } GrokSessionSelectorError::AccountCoolingDown { retry_after } => ( - retry_after, - cooling_down_message(retry_after), + *retry_after, + cooling_down_message(*retry_after), "account_cooling_down", ), GrokSessionSelectorError::ModelCoolingDown { retry_after } => ( - retry_after, - model_cooling_down_message(retry_after), + *retry_after, + model_cooling_down_message(*retry_after), "model_cooling_down", ), GrokSessionSelectorError::CapacityUnavailable { retry_after } => ( - retry_after, + *retry_after, "account is at its concurrency or request-interval limit".to_owned(), "account_capacity_busy", ), @@ -422,7 +431,7 @@ pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderEr "no account is eligible for the requested model".to_owned(), "no_eligible_account", ), - GrokSessionSelectorError::Unavailable => ( + GrokSessionSelectorError::Unavailable(_) => ( None, "account scheduling state is temporarily unreadable".to_owned(), "account_selector_unavailable", @@ -436,7 +445,7 @@ pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderEr GrokSessionSelectorError::PolicyUnavailable => { return provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent); } - GrokSessionSelectorError::InvalidSession => { + GrokSessionSelectorError::InvalidSession(_) => { return provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent); } }; diff --git a/backend/crates/providers/xai/src/provider/workers.rs b/backend/crates/providers/xai/src/provider/workers.rs index 48233005e..0ea175b4d 100644 --- a/backend/crates/providers/xai/src/provider/workers.rs +++ b/backend/crates/providers/xai/src/provider/workers.rs @@ -107,8 +107,7 @@ impl ScheduledTask for XaiOAuthRefreshTask { return Ok(()); } let outcomes = self.service.refresh_due().await.map_err(|error| { - tracing::error!(error = %error, "xAI OAuth refresh cycle failed"); - WorkerTaskError::safe("xAI OAuth refresh failed") + WorkerTaskError::safe("xAI OAuth refresh failed").with_source(error) })?; let failures = outcomes .iter() @@ -151,7 +150,9 @@ impl ScheduledTask for XaiQuotaCatalogTask { .accounts .list_for_provider(&self.provider_kind) .await - .map_err(|_| WorkerTaskError::safe("xAI Provider accounts unavailable"))?; + .map_err(|source| { + WorkerTaskError::safe("xAI Provider accounts unavailable").with_source(source) + })?; let mut failures = 0_u64; let now = SystemTime::now(); let accounts = self.reserve_periodic_refreshes(accounts, now); diff --git a/backend/crates/providers/xai/src/transport/session.rs b/backend/crates/providers/xai/src/transport/session.rs index 4859a9cf9..90ce8bd93 100644 --- a/backend/crates/providers/xai/src/transport/session.rs +++ b/backend/crates/providers/xai/src/transport/session.rs @@ -468,7 +468,7 @@ pub trait GrokSessionSelector: Send + Sync { } /// 不含密钥的选择器失败 -#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[derive(Debug, thiserror::Error)] pub enum GrokSessionSelectorError { #[error(transparent)] QueueRejected(#[from] gateway_core::concurrency::QueueRejection), @@ -495,10 +495,10 @@ pub enum GrokSessionSelectorError { }, /// 会话元数据或 Provider 持有的明文密钥非法 #[error("Grok Build session data is invalid")] - InvalidSession, + InvalidSession(#[source] Option), /// 选择器依赖的后端服务不可用 #[error("Grok Build session selector is unavailable")] - Unavailable, + Unavailable(#[source] Option), /// 插件调度策略明确拒绝本次请求 #[error("account scheduling policy rejected the request")] PolicyRejected, diff --git a/backend/crates/providers/xai/tests/admin.rs b/backend/crates/providers/xai/tests/admin.rs index de667cb1b..c0620025a 100644 --- a/backend/crates/providers/xai/tests/admin.rs +++ b/backend/crates/providers/xai/tests/admin.rs @@ -87,6 +87,7 @@ async fn account_unavailable_clears_real_selector_cooldowns_only_for_deleted_acc cooldowns.clone(), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ); let repository = GrokCredentialRepository::new(store.clone()); let selector = GrokAccountSessionSelector::new( @@ -191,7 +192,9 @@ async fn xai_quota_catalog_worker_treats_empty_account_pool_as_idle() { .await .expect("xAI bundle"); - assert_eq!(run_quota_catalog_cycle(&mut bundle).await, Ok(())); + run_quota_catalog_cycle(&mut bundle) + .await + .expect("idle quota cycle"); } #[tokio::test] @@ -650,6 +653,7 @@ fn provider_ports_with_catalog( Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), pending, + Arc::new(RecordingDiagnostics::default()), ) } @@ -1174,6 +1178,7 @@ mod errors { Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ); let bundle = provider_xai::initialize(ports).await.unwrap(); let error = bundle @@ -1285,3 +1290,16 @@ async fn client_profile_preview_and_dashboard_use_saved_configuration() { assert!(dashboard.release.is_none()); assert!(dashboard.verified_at.is_none()); } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/xai/tests/credential/contract.rs b/backend/crates/providers/xai/tests/credential/contract.rs index 179629c61..916bd545a 100644 --- a/backend/crates/providers/xai/tests/credential/contract.rs +++ b/backend/crates/providers/xai/tests/credential/contract.rs @@ -317,12 +317,12 @@ async fn invalid_token_lifetime_is_rejected_before_store_write() { let (store, _) = repository(); let mut input = create_input("lifetime", "subject-lifetime"); input.account.refresh_token_expires_at = Some(input.account.access_token_expires_at); - assert_eq!( + assert!(matches!( GrokCredentialAdmin.prepare_import(&input), Err(GrokCredentialRepositoryError::InvalidInput( "token_lifetime" )) - ); + )); assert_eq!(store.len(), 0); } diff --git a/backend/crates/providers/xai/tests/credential/repository.rs b/backend/crates/providers/xai/tests/credential/repository.rs index c1119c108..ecf740dd3 100644 --- a/backend/crates/providers/xai/tests/credential/repository.rs +++ b/backend/crates/providers/xai/tests/credential/repository.rs @@ -12,10 +12,10 @@ fn repository_rejects_identity_that_cannot_be_sent_as_official_header() { let mut input = create_input("invalid", "subject"); input.account.subject = "subject-with-非-ascii".to_owned(); - assert_eq!( + assert!(matches!( GrokCredentialAdmin.prepare_import(&input), Err(GrokCredentialRepositoryError::InvalidInput("subject")) - ); + )); } #[tokio::test] diff --git a/backend/crates/providers/xai/tests/provider/contract.rs b/backend/crates/providers/xai/tests/provider/contract.rs index ce002f5ad..90fe1b11e 100644 --- a/backend/crates/providers/xai/tests/provider/contract.rs +++ b/backend/crates/providers/xai/tests/provider/contract.rs @@ -888,7 +888,7 @@ impl GrokSessionSelector for StubSelector { GrokSessionBinding::new("acct_provider").expect("binding"), (), ) - .map_err(|_| GrokSessionSelectorError::InvalidSession) + .map_err(|_| GrokSessionSelectorError::InvalidSession(None)) }) } @@ -938,7 +938,7 @@ impl GrokSessionSelector for SequencedAccountSelector { GrokSessionBinding::new("acct_provider").expect("binding"), (), ) - .map_err(|_| GrokSessionSelectorError::InvalidSession) + .map_err(|_| GrokSessionSelectorError::InvalidSession(None)) }) } @@ -3140,7 +3140,7 @@ async fn selection_failures_carry_a_distinguishable_client_error_code() { "no_eligible_account", ), ( - GrokSessionSelectorError::Unavailable, + GrokSessionSelectorError::Unavailable(None), "account_selector_unavailable", ), ]; @@ -3174,6 +3174,57 @@ async fn selection_failures_carry_a_distinguishable_client_error_code() { } } +#[tokio::test] +async fn selection_failure_preserves_native_cause_without_exposing_it_to_clients() { + use gateway_core::error::{ErrorSource, StoreError, StoreErrorKind}; + let source = StoreError::caused_by( + StoreErrorKind::Unavailable, + std::io::Error::other("PRIVATE_XAI_DATABASE_CAUSE"), + ); + let transport = StubInferenceTransport::success(); + let provider = provider( + StubSelector::failing(GrokSessionSelectorError::Unavailable(Some( + ErrorSource::new(source), + ))), + transport.clone(), + ) + .await; + let error = match provider + .execute( + provider_request("xai"), + context(CancellationToken::new(), None), + ) + .await + { + Ok(_) => panic!("selection must fail"), + Err(error) => error, + }; + assert_eq!(error.kind(), ProviderErrorKind::Unavailable); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + assert!( + error + .error_details() + .expect("protected details") + .contains("PRIVATE_XAI_DATABASE_CAUSE") + ); + let mut cause = std::error::Error::source(&error); + let mut native = false; + while let Some(error) = cause { + native |= error.downcast_ref::().is_some(); + cause = error.source(); + } + assert!(native); + assert!(!format!("{error:?} {error}").contains("PRIVATE_XAI_DATABASE_CAUSE")); + assert!( + !error + .client_visible_upstream_error() + .expect("safe detail") + .message() + .contains("PRIVATE_XAI_DATABASE_CAUSE") + ); + assert_eq!(transport.calls.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn native_previous_response_is_rejected_before_selection() { let selector = StubSelector::success(); diff --git a/deploy/README.md b/deploy/README.md index 4a2a220d3..b237db3f5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -439,18 +439,22 @@ Provider schema 以明文 JSON 保存于 PostgreSQL。Redis 只保存可重建 关闭的分片压缩为 `.log.gz`;成功压缩、同步并发布归档后才删除原文件,保留原修改时间。 清理发生在启动和轮转时;空闲期间过期文件可能暂时多保留。检索时须同时读取 `.log` 与 `.log.gz` -报文开关为 `host.logging.request_dump`,默认关闭;开启时原始报文按块完整写入独立文件, +报文开关为 `host.logging.request_dump`,默认关闭;开启时原始报文按块写入独立文件, +缓冲拒收或写入失败可能造成缺块,需结合 `file_logging` 健康状态判断完整性。 数据库请求诊断仍是有界摘要,不能用摘要事件数代替全量报文完整性 OAuth 恢复开关为 `host.logging.oauth_recovery`,默认关闭,与普通文件日志开关分别控制: - 开启后,OpenAI 每次成功取得 AT 和可用 RT 时,在资料补全及数据库写入前写入独立的 `oauth-recovery` 文件集 -- 该通道不受普通日志级别过滤,关闭时也不会改写到普通日志或 stdout;有界队列满时等待写入 +- 该通道不受普通日志级别过滤,关闭时也不会改写到普通日志或 stdout;接收端不等待后台写入 恢复记录和请求转储含原始凭据,`.runtime/logs` 须按敏感数据控制访问和备份 -文件写入使用有界队列背压,正常退出会排空队列并同步文件。`file_logging` 健康探针在写入/同步失败后 -报告 `Unhealthy`,本次进程内恢复写入也不会清除已有缺口;压缩或清理失败报告 `Degraded`。 +每个文件输出使用非阻塞队列,普通额度为 4096 条 / 64 MiB,WARN / ERROR 另有 512 条 / 8 MiB 预留, +额度包含正在写入的记录。后台批量缓冲写文件,正常退出会排空已接收的队列并同步。 +持续拥堵、超大记录或输出关闭导致的拒收会损失整条记录,所有接口均不会为日志等待; +`file_logging` 健康探针在拒收或写入/同步失败后报告 `Unhealthy`, +本次进程内恢复写入也不会清除已有缺口;压缩或清理失败报告 `Degraded`。 断电、强杀或磁盘故障可能造成日志丢失;需要恢复已删除文件时,应使用独立备份。 容量不足时不会提前删除保留窗口内日志,必须根据完整日期的压缩后实际用量规划空间,并监控磁盘余量和 健康探针。Docker stdout 的独立轮转不承担应用文件日志的完整保留承诺 diff --git a/docs/api.md b/docs/api.md index 6f0cce983..9a7bf0b61 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1704,7 +1704,8 @@ Key 已删除或未关联时为 `null`,不影响记录返回,不包含密钥 运维错误记录的 `errorDetails` 为受控诊断文本,无详情时为 `null`。 错误来源快照以 JSON 文本保存:`causes.messages` 按外层到内层排列本地原因,`causes.truncated` -标记来源链截断,`upstream` 保存上游正文原文,`redacted` 标记敏感上下文移除;没有对应来源时为 `null`。 +标记来源链截断;存在附属清理失败时,`causes.cleanup` 保存各自的原因快照。 +`upstream` 保存上游正文,`redacted` 标记令牌等敏感上下文的移除;没有对应来源时为 `null`。 稳定错误分类、原始上游 code 和该详情分别保留,不以公共错误文案代替原始原因。 详情不出现在 Key 用量接口、普通错误信封和诊断包导出中;记录仍受异步观测写入与保留周期约束 diff --git a/docs/architecture.md b/docs/architecture.md index 3c23e5eb4..821fdf384 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -509,13 +509,22 @@ client,OIDC 的 JWKS 缓存与单飞归属对应出口状态。自动刷新提 普通格式化不展开来源。Store 统一基础设施不可用的转换,Provider 拥有业务分类, 插件 Runtime 集中投影 RPC、fault 与上游适配事件,调用点只附加原始来源与执行上下文 -ProviderError 统一生成受控详情,失败 attempt、探测和最终请求复用该出口。 -原始上游正文与本地原因链分开承载,来源链最多保留 32 层、64 KiB 文本,截断明确标记; +Core 的 `ErrorDetails` 统一投影受控详情,Provider、管理请求和后台操作复用该格式。 +原始上游正文与本地原因链分开承载,主原因与附属清理失败合计最多保留 32 个节点、64 KiB 文本, +截断明确标记;显式回滚仍等待资源释放,回滚失败附加到主失败,不替换其分类与原因。 +本地续写拒绝和 WS Close 派生分类不写入原始上游 code;关闭码与原因保存在关闭事实中。 + 请求 URL 等请求方向的敏感上下文在附加来源前移除并标记。错误快照保留来源与错误详情, 协议透传响应和未交付事件仍由当前请求独占。详情计入观测队列的内存预算; 队列满、关闭超时或写入失败仍可能丢失记录,沿用丢弃与写入失败指标,不自动重放不保证幂等的写入 -推理日志由作出处置的边界记录恢复、放弃或终态;仅传播错误的层不重复打印。 +管理请求在 API 出口按 request ID 记录一次内部失败,后台任务由 Host 监督器记录; +OAuth 刷新和准入释放等内部处理后继续运行的失败,由消费失败结果的 owner 记录。 +这些操作通过 `OperationalDiagnostics` 复用现有有界观测队列与 `ops_events`, +不创建模型请求,也不把基础设施失败冒充成 Provider 上游响应。 +OAuth 响应中的令牌字段和回显的刷新令牌在进入受控详情前脱敏,普通日志不展开正文。 + +日志由作出处置的边界记录恢复、放弃或终态;仅传播错误的层不重复打印。 传输收包使用 debug 摘要,普通日志使用稳定阶段、分类与 request/attempt 关联,不展开原始正文、 任意上游 code 或关闭原因。受控详情无法写入时不能回退到普通日志打印原文 @@ -805,8 +814,10 @@ OpenAI 订阅周期属于按需个人信息,不是额度事实。Admin 账号 ### 请求记录与交付 -请求观测通过有界进程内队列异步投影到 PostgreSQL。队列满、Store 暂不可用或进程退出超时时,允许丢失 -可恢复观测并累计指标,但不允许改变客户端响应。Usage 详情中的 attempt 因此是 best-effort,并通过 +请求观测通过有界进程内队列异步投影到 PostgreSQL。普通额度之外预留四分之一容量给失败与请求生命周期记录, +进度更新不能消耗预留;同一请求仍按入队顺序写入。容量耗尽、Store 暂不可用或进程退出超时时, +观测记录可能丢失并累计指标,拥堵与写入失败告警按累计计数限频,不改变客户端响应。 +Usage 详情中的 attempt 因此是 best-effort,并通过 `attemptsComplete: false` 明示不完整性 `model_requests.deadline_at` 是异常回收租约的到期时间,运行会话持续刷新,且不超过显式执行截止。 @@ -887,8 +898,11 @@ worker 复用连接测试探针执行真实上游调用,并按冻结代次处 与普通文件日志开关分别控制,不输出到普通日志或 stdout。`.runtime/logs` 同样属于敏感数据 - `host.logging.request_dump` 默认关闭;开启后独立请求转储包含原始请求头和正文, 可能包含密钥及用户内容,只能在明确的排障范围内使用,不能作为普通日志公开 -- Host 拥有文件日志写入、轮转、保留和关闭流程,以时间窗口完整性为约束,队列满时背压,正常关闭时排空并同步。 - 写入缺口必须通过健康状态持续报告,压缩或清理失败不能伪装成保留完成。 +- 所有接口的日志与诊断观测接收均不得等待队列空位或输出 I/O,请求侧只承担有界采集与入队开销 +- Host 拥有文件日志写入、轮转、保留和关闭流程。文件日志按条数与字节限制内存,WARN / ERROR + 另有预留额度;单个输出使用 FIFO 和后台批量缓冲,正常关闭时排空并同步。 + 容量耗尽或输出关闭的记录不回退到请求线程同步输出,缺口通过健康状态持续报告; + 压缩或清理失败不能伪装成保留完成。 配置、保留窗口和容量规划见 [日志与持久化](../deploy/README.md#持久化与备份) - Provider 将安全诊断的阶段、原因码与消息独立于错误大类和发送状态传入 Core; `attempt.failed` 保存这些字段,最终错误记录优先持久化诊断消息。重试包装不能覆盖底层诊断,