fix(diagnostics): preserve error chains and avoid logging backpressure

Preserve native failure causes through store, provider, plugin, admin, and
worker boundaries, including secondary cleanup failures. Route operational
failures through the shared diagnostics sink with protected error details.

Use bounded nonblocking file logging with batched writes and reserved
warning/error capacity. Reserve observation capacity for failures and
request lifecycle records, and rate-limit overload warnings.

Validate error propagation and blocked-output HTTP, admin, SSE, and WebSocket
behavior; keep the local migration plan out of version control.

Co-authored-by: Codex <noreply@openai.com>

backend/apps/gateway/src/bootstrap/server.rs

@@ -117,6 +117,7 @@ pub async fn run() -> Result<(), BootstrapError> {
             probes,
             host.worker_health(),
             host.connection_lifecycle(),
+            store.diagnostics(),
         )
     )
     .with_middleware({
@@ -155,7 +156,7 @@ pub async fn run() -> Result<(), BootstrapError> {
     ));
     or_shutdown!(
         plugin_runtime,
-        host.start_workers(plan, store.worker_leader_lease())
+        host.start_workers(plan, store.worker_leader_lease(), store.diagnostics())
     );
     host.report_startup_ready("Workers");
     let served = host.serve(api.router()).await;

backend/crates/gateway-admin/src/backup/task.rs

@@ -28,7 +28,7 @@ use crate::{
 
 impl From<BackupError> for WorkerTaskError {
     fn from(error: BackupError) -> Self {
-        WorkerTaskError::safe(error.message())
+        WorkerTaskError::safe(error.message()).with_source(error)
     }
 }
 
@@ -125,8 +125,9 @@ impl BackupTask {
         if cron.is_empty() {
             return Ok(());
         }
-        let schedule = BackupSchedule::parse(cron, self.timezone)
-            .map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?;
+        let schedule = BackupSchedule::parse(cron, self.timezone).map_err(|source| {
+            WorkerTaskError::safe("backup schedule is invalid").with_source(source)
+        })?;
 
         if settings.schedule_timezone.as_deref() != Some(timezone) {
             // 时区切换只初始化未来游标，不把旧时区计划当作应补跑任务
@@ -685,9 +686,9 @@ fn status_transition(
 }
 
 fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError {
-    WorkerTaskError::safe(format!("backup store failed: {error}"))
+    WorkerTaskError::safe("backup store failed").with_source(error)
 }
 
 fn infra_error(error: BackupError) -> WorkerTaskError {
-    WorkerTaskError::safe(error.message())
+    WorkerTaskError::safe(error.message()).with_source(error)
 }

backend/crates/gateway-admin/src/model/mod.rs

@@ -39,12 +39,31 @@ pub enum AdminErrorKind {
     Internal,
 }
 
-/// 不携带基础设施细节、可安全返回给管理员的管理用例错误
-#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+impl AdminErrorKind {
+    #[must_use]
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Invalid => "invalid",
+            Self::Forbidden => "forbidden",
+            Self::Unauthorized => "unauthorized",
+            Self::NotFound => "not_found",
+            Self::Conflict => "conflict",
+            Self::RateLimited => "rate_limited",
+            Self::BadGateway => "bad_gateway",
+            Self::UpstreamResultUnknown => "upstream_result_unknown",
+            Self::Unavailable => "unavailable",
+            Self::Internal => "internal",
+        }
+    }
+}
+
+/// 管理用例的安全消息与内部原因分离，原始来源不进入响应
+#[derive(Debug, Clone, thiserror::Error)]
 #[error("{message}")]
 pub struct AdminError {
     kind: AdminErrorKind,
     message: String,
+    source: Option<gateway_core::error::ErrorSource>,
 }
 
 impl AdminError {
@@ -53,9 +72,21 @@ impl AdminError {
         Self {
             kind,
             message: message.into(),
+            source: None,
         }
     }
 
+    #[must_use]
+    pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
+        self.source = Some(source.into());
+        self
+    }
+
+    #[must_use]
+    pub fn error_details(&self) -> Option<gateway_core::error::ErrorDetails> {
+        gateway_core::error::ErrorDetails::capture(self.source.as_ref(), None, false)
+    }
+
     #[must_use]
     pub const fn kind(&self) -> AdminErrorKind {
         self.kind

backend/crates/gateway-admin/src/model/provider_credentials.rs

@@ -862,7 +862,7 @@ pub struct ProviderSubscription {
 }
 
 /// 按需汇聚的个人信息；资料查询失败不丢弃可用的订阅结果
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone)]
 pub struct AccountPersonalInfo {
     pub profile: Result<ProviderProfileStatistics, AdminError>,
     pub subscription: Option<ProviderSubscription>,

backend/crates/gateway-admin/src/ports/provider.rs

@@ -46,11 +46,12 @@ pub enum ProviderAdminErrorKind {
 /// `message` 是 Provider 局部诊断，可能包含原始上游正文；通用管理用例不得自动把它作为公开文案
 /// 只有明确拥有原始诊断合同的调用方才能读取，`Debug` 始终只记录是否存在
 /// `public_message` 则是明确标记为可公开的静态提示，不允许携带动态上游材料
-#[derive(Clone, PartialEq, Eq, thiserror::Error)]
+#[derive(Clone, thiserror::Error)]
 #[error("provider admin operation failed: {kind:?}")]
 pub struct ProviderAdminError {
     kind: ProviderAdminErrorKind,
-    message: Option<String>,
+    #[source]
+    diagnostic: Option<Arc<ProviderAdminDiagnostic>>,
     public_message: Option<&'static str>,
 }
 
@@ -59,7 +60,7 @@ impl std::fmt::Debug for ProviderAdminError {
         formatter
             .debug_struct("ProviderAdminError")
             .field("kind", &self.kind)
-            .field("message", &self.message.as_ref().map(|_| "<redacted>"))
+            .field("message", &self.message().map(|_| "<redacted>"))
             .field("public_message", &self.public_message)
             .finish()
     }
@@ -70,14 +71,22 @@ impl ProviderAdminError {
     pub const fn new(kind: ProviderAdminErrorKind) -> Self {
         Self {
             kind,
-            message: None,
+            diagnostic: None,
             public_message: None,
         }
     }
 
+    #[must_use]
+    pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
+        Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).source =
+            Some(source.into());
+        self
+    }
+
     #[must_use]
     pub fn with_message(mut self, message: impl Into<String>) -> Self {
-        self.message = Some(message.into());
+        Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).message =
+            Some(message.into());
         self
     }
 
@@ -100,10 +109,20 @@ impl ProviderAdminError {
 
     #[must_use]
     pub fn message(&self) -> Option<&str> {
-        self.message.as_deref()
+        self.diagnostic
+            .as_ref()
+            .and_then(|diagnostic| diagnostic.message.as_deref())
     }
 }
 
+// Provider 局部消息与 typed 来源共用一个不公开的上下文，避免跨用例转换后丢失
+#[derive(Debug, Clone, Default, thiserror::Error)]
+#[error("{}", message.as_deref().unwrap_or("provider admin dependency failed"))]
+struct ProviderAdminDiagnostic {
+    message: Option<String>,
+    source: Option<gateway_core::error::ErrorSource>,
+}
+
 /// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力
 ///
 /// 数据变更由 Provider 返回 prepared facts；config revision、审计与 PostgreSQL 事务

backend/crates/gateway-admin/src/ports/store.rs

@@ -78,8 +78,18 @@ impl AdminStoreError {
 
     /// 来源留在内部诊断链，公开消息不展开基础设施错误
     #[must_use]
-    pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
-        self.source = Some(gateway_core::error::ErrorSource::new(source));
+    pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
+        self.source = Some(source.into());
+        self
+    }
+
+    /// 清理失败不覆盖最初操作的分类与原因
+    #[must_use]
+    pub fn with_cleanup(mut self, cleanup: impl Into<gateway_core::error::ErrorSource>) -> Self {
+        self.source = Some(gateway_core::error::ErrorSource::cleanup(
+            self.source.take(),
+            cleanup,
+        ));
         self
     }
 

backend/crates/gateway-admin/src/use_case/mod.rs

@@ -51,7 +51,6 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError
         | AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict,
         AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable,
     };
-    tracing::warn!(resource, error_kind = ?error.kind(), "admin store operation failed");
     let message = match kind {
         AdminErrorKind::Invalid => "请求参数不合法",
         AdminErrorKind::NotFound => "请求的资源不存在",
@@ -59,7 +58,10 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError
         AdminErrorKind::Unavailable => "依赖服务暂不可用",
         _ => "服务内部错误",
     };
-    AdminError::new(kind, message)
+    AdminError::new(kind, message).with_source(OperationFailure {
+        operation: resource,
+        source: error.into(),
+    })
 }
 
 fn map_provider_error(
@@ -79,7 +81,6 @@ fn map_provider_error(
         ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
         ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
     };
-    tracing::warn!(resource, error_kind = ?error.kind(), "admin provider operation failed");
     let message = match kind {
         AdminErrorKind::Invalid => "Provider 请求不合法",
         AdminErrorKind::NotFound => "Provider 资源不存在",
@@ -90,7 +91,10 @@ fn map_provider_error(
         AdminErrorKind::Internal => "服务内部错误",
         _ => "Provider 操作失败",
     };
-    AdminError::new(kind, error.public_message().unwrap_or(message))
+    AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure {
+        operation: resource,
+        source: error.into(),
+    })
 }
 
 async fn publish_committed(
@@ -454,3 +458,11 @@ async fn delete_credentials(
         account_ids,
     })
 }
+
+// 用例的操作名补充端口资源上下文，底层分类与原始原因仍由来源持有
+#[derive(Debug, thiserror::Error)]
+#[error("admin operation failed: {operation}")]
+struct OperationFailure {
+    operation: &'static str,
+    source: gateway_core::error::ErrorSource,
+}

backend/crates/gateway-admin/tests/model/provider_credentials.rs

@@ -223,8 +223,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() {
                 assert_eq!(
                     PendingAuthorizationMutation::from_storage_v1(Value::Object(
                         expected.to_storage_v1()
-                    )),
-                    Ok(expected.clone()),
+                    ))
+                    .unwrap(),
+                    expected.clone(),
                 );
                 let with_proxy = expected
                     .with_outbound_proxy(Some(
@@ -237,8 +238,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() {
                 assert_eq!(
                     PendingAuthorizationMutation::from_storage_v1(Value::Object(
                         with_proxy.to_storage_v1()
-                    )),
-                    Ok(with_proxy)
+                    ))
+                    .unwrap(),
+                    with_proxy
                 );
             }
         }

backend/crates/gateway-api/src/admin/diagnostics.rs

@@ -0,0 +1,42 @@
+//! 管理请求在 HTTP 出口记录一次受控详情，响应只保留安全投影
+
+use std::sync::Arc;
+
+use axum::{
+    extract::{Request, State},
+    middleware::Next,
+    response::Response,
+};
+use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure};
+use tower_http::request_id::RequestId;
+
+pub(crate) async fn record_failure(
+    State(diagnostics): State<Arc<dyn OperationalDiagnostics>>,
+    request: Request,
+    next: Next,
+) -> Response {
+    let correlation_id = request
+        .extensions()
+        .get::<RequestId>()
+        .and_then(|id| id.header_value().to_str().ok())
+        .map(str::to_owned);
+    let mut response = next.run(request).await;
+    if let Some(error) = response
+        .extensions_mut()
+        .remove::<gateway_admin::model::AdminError>()
+        && let Some(details) = error.error_details()
+    {
+        let mut failure = OperationalFailure::new(
+            "admin",
+            "http_request",
+            error.kind().as_str(),
+            error.message(),
+        );
+        failure.correlation_id = correlation_id;
+        failure.details = Some(details);
+        if diagnostics.record_failure(failure).await.is_err() {
+            tracing::warn!("admin diagnostic could not be recorded");
+        }
+    }
+    response
+}

backend/crates/gateway-api/src/admin/mod.rs

@@ -15,6 +15,7 @@ pub mod accounts;
 pub mod auth;
 pub mod backups;
 pub mod client_keys;
+pub(crate) mod diagnostics;
 mod extract;
 pub mod observability;
 mod plugins;

backend/crates/gateway-api/src/admin/wire.rs

@@ -170,6 +170,7 @@ pub struct AdminError {
     status: StatusCode,
     body: AdminErrorBody,
     retry_after_seconds: Option<u64>,
+    diagnostic: Option<gateway_admin::model::AdminError>,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -266,6 +267,7 @@ impl AdminError {
             status,
             body: AdminErrorBody::new(code, message),
             retry_after_seconds: None,
+            diagnostic: None,
         }
     }
 
@@ -380,12 +382,16 @@ pub(crate) fn map_admin_service_error(error: gateway_admin::model::AdminError) -
     {
         response.body.message = error.message().to_owned();
     }
+    response.diagnostic = Some(error);
     response
 }
 
 impl IntoResponse for AdminError {
     fn into_response(self) -> Response {
         let mut response = (self.status, Json(self.body)).into_response();
+        if let Some(error) = self.diagnostic {
+            response.extensions_mut().insert(error);
+        }
         if let Some(seconds) = self.retry_after_seconds
             && let Ok(value) = axum::http::HeaderValue::from_str(&seconds.to_string())
         {

backend/crates/gateway-api/src/lib.rs

@@ -170,6 +170,7 @@ pub fn initialize(
     probes: Vec<Arc<dyn HealthProbe>>,
     worker_health: Arc<dyn WorkerHealthSource>,
     lifecycle: Arc<dyn ConnectionLifecycle>,
+    diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
 ) -> Result<ApiBundle, ApiError> {
     // 配置加载已解析环境变量与相对路径，初始化只校验，避免覆盖最终目录
     config.validate().map_err(ApiError::Config)?;
@@ -256,6 +257,10 @@ pub fn initialize(
             headers_ms = latency.as_millis(), "HTTP response headers ready");
     });
     let router = router
+        .layer(axum::middleware::from_fn_with_state(
+            diagnostics,
+            admin::diagnostics::record_failure,
+        ))
         .layer(PropagateRequestIdLayer::new(request_id_header.clone()))
         .layer(trace_layer)
         .layer(SetRequestIdLayer::new(

backend/crates/gateway-api/src/openai/error.rs

@@ -165,8 +165,11 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
                 "no upstream provider is currently available for this request",
             )
         }
-        EngineError::Store(_)
-        | EngineError::ProviderMetadataMismatch
+        EngineError::Store(source) => {
+            GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
+                .with_source(source.clone())
+        }
+        EngineError::ProviderMetadataMismatch
         | EngineError::ContinuationPinMismatch
         | EngineError::RequiredAccountMismatch
         | EngineError::AccountOutsideClientScope

backend/crates/gateway-api/tests/admin/error_diagnostics.rs

@@ -0,0 +1,61 @@
+//! 验证管理端来源经用例和 HTTP 出口到达受控记录，公开响应和扩展不泄漏
+
+use axum::{
+    body::{Body, to_bytes},
+    http::{Request, StatusCode},
+};
+use std::sync::Arc;
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn store_failure_is_recorded_once_with_the_incoming_request_id() {
+    let fixture = super::AdminTestFixture::new().await;
+    fixture.auth.insert_session("valid-session");
+    let diagnostics = Arc::new(crate::support::RecordingDiagnostics::default());
+    let bundle = crate::openai::api_bundle_with_diagnostics(
+        fixture.services,
+        gateway_api::ApiConfig {
+            asset_directory: std::env::temp_dir(),
+            request_timeout_seconds: None,
+            cors_allowed_origins: Vec::new(),
+            request_id_header: "x-request-id".to_owned(),
+        },
+        diagnostics.clone(),
+    );
+    let response = bundle
+        .router()
+        .oneshot(
+            Request::builder()
+                .uri("/api/admin/accounts")
+                .header("cookie", "cpr_session=valid-session")
+                .header("x-request-id", "admin-native-cause-test")
+                .body(Body::empty())
+                .unwrap(),
+        )
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+    assert!(
+        response
+            .extensions()
+            .get::<gateway_admin::model::AdminError>()
+            .is_none()
+    );
+    let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
+    assert!(!String::from_utf8_lossy(&bytes).contains("PRIVATE_ADMIN_NATIVE_CAUSE"));
+    let failures = diagnostics.0.lock().unwrap();
+    assert_eq!(failures.len(), 1);
+    assert_eq!(failures[0].kind, "unavailable");
+    assert_eq!(
+        failures[0].correlation_id.as_deref(),
+        Some("admin-native-cause-test")
+    );
+    assert!(
+        failures[0]
+            .details
+            .as_ref()
+            .unwrap()
+            .as_str()
+            .contains("PRIVATE_ADMIN_NATIVE_CAUSE")
+    );
+}

backend/crates/gateway-api/tests/admin/mod.rs

@@ -86,6 +86,7 @@ mod account_groups;
 mod accounts;
 mod auth;
 mod client_keys;
+mod error_diagnostics;
 mod errors;
 mod observability;
 mod proxies;
@@ -1673,6 +1674,7 @@ fn unavailable(resource: &'static str) -> AdminStoreError {
         resource,
         "unused test port",
     )
+    .with_source(std::io::Error::other("PRIVATE_ADMIN_NATIVE_CAUSE"))
 }
 
 fn not_found(resource: &'static str) -> AdminStoreError {

backend/crates/gateway-api/tests/architecture.rs

@@ -45,6 +45,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
         "src/admin/auth.rs",
         "src/admin/backups.rs",
         "src/admin/client_keys.rs",
+        "src/admin/diagnostics.rs",
         "src/admin/extract.rs",
         "src/admin/mod.rs",
         "src/admin/observability/mod.rs",
@@ -124,6 +125,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
         "tests/admin/auth.rs",
         "tests/admin/client_keys.rs",
         "tests/admin/errors.rs",
+        "tests/admin/error_diagnostics.rs",
         "tests/admin/mod.rs",
         "tests/admin/observability/mod.rs",
         "tests/admin/observability/query.rs",

backend/crates/gateway-api/tests/openai/auth.rs

@@ -479,6 +479,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission(
         Arc::new(super::UnusedAdmissions),
         Arc::new(super::UnusedContinuation),
         Arc::new(super::IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     ));
     let admin = crate::admin::AdminTestFixture::new().await;
     let bundle = gateway_api::initialize(
@@ -493,6 +494,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission(
         vec![],
         Arc::new(super::EmptyWorkerHealth),
         Arc::new(super::TestLifecycle::default()),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .unwrap();
     let baseline = bundle.dispatcher();

backend/crates/gateway-api/tests/openai/error.rs

@@ -304,12 +304,14 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
         UpstreamSendState::NotSent,
     ));
 
+    let projected = gateway_error_from_engine(&error);
     assert_eq!(
-        gateway_error_from_engine(&error),
-        GatewayError::new(
-            GatewayErrorKind::AccountCapacityUnavailable,
-            "all eligible upstream accounts are temporarily busy"
-        )
+        projected.kind(),
+        GatewayErrorKind::AccountCapacityUnavailable
+    );
+    assert_eq!(
+        projected.safe_message(),
+        "all eligible upstream accounts are temporarily busy"
     );
 }
 
@@ -352,12 +354,14 @@ fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
         UpstreamSendState::NotSent,
     ));
 
+    let projected = gateway_error_from_engine(&error);
     assert_eq!(
-        gateway_error_from_engine(&error),
-        GatewayError::new(
-            GatewayErrorKind::ProviderInfrastructureUnavailable,
-            "provider account infrastructure is temporarily unavailable"
-        )
+        projected.kind(),
+        GatewayErrorKind::ProviderInfrastructureUnavailable
+    );
+    assert_eq!(
+        projected.safe_message(),
+        "provider account infrastructure is temporarily unavailable"
     );
 }
 
@@ -368,12 +372,11 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
         UpstreamSendState::NotSent,
     ));
 
+    let projected = gateway_error_from_engine(&error);
+    assert_eq!(projected.kind(), GatewayErrorKind::NoAvailableProvider);
     assert_eq!(
-        gateway_error_from_engine(&error),
-        GatewayError::new(
-            GatewayErrorKind::NoAvailableProvider,
-            "no upstream provider is currently available for this request"
-        )
+        projected.safe_message(),
+        "no upstream provider is currently available for this request"
     );
 }
 
@@ -499,13 +502,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() {
     ] {
         let error = EngineError::Provider(ProviderError::new(kind, UpstreamSendState::Ambiguous));
 
-        assert_eq!(
-            gateway_error_from_engine(&error),
-            GatewayError::new(
-                GatewayErrorKind::UpstreamUnavailable,
-                "upstream service is unavailable"
-            )
-        );
+        let projected = gateway_error_from_engine(&error);
+        assert_eq!(projected.kind(), GatewayErrorKind::UpstreamUnavailable);
+        assert_eq!(projected.safe_message(), "upstream service is unavailable");
     }
 }
 
@@ -513,10 +512,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() {
 fn engine_store_error_should_collapse_to_safe_internal_error() {
     let error = EngineError::Store(StoreError::new(StoreErrorKind::Unavailable));
 
-    assert_eq!(
-        gateway_error_from_engine(&error),
-        GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
-    );
+    let projected = gateway_error_from_engine(&error);
+    assert_eq!(projected.kind(), GatewayErrorKind::Internal);
+    assert_eq!(projected.safe_message(), "gateway execution failed");
 }
 #[test]
 fn openai_error_response_should_preserve_only_safe_contract_fields() {
@@ -669,6 +667,7 @@ mod model_routing {
             Arc::new(UnusedAdmissions),
             Arc::new(UnusedContinuation),
             Arc::new(IgnoredClientApiKeyUsage),
+            Arc::new(crate::support::RecordingDiagnostics::default()),
         );
         let response = crate::openai::api_router(Arc::new(execution))
             .await

backend/crates/gateway-api/tests/openai/live/websocket.rs

@@ -138,6 +138,7 @@ async fn blocked_relay(
         Vec::new(),
         Arc::new(crate::openai::EmptyWorkerHealth),
         Arc::new(Lifecycle(trace.clone())),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .unwrap()
     .router();

backend/crates/gateway-api/tests/openai/mod.rs

@@ -92,6 +92,7 @@ pub(super) fn api_router_with_admin_and_client(
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     ));
     api_router_with_admin_and_execution(admin, execution)
 }
@@ -106,6 +107,18 @@ pub(super) fn api_router_with_config(
 pub(super) fn api_bundle(
     admin: gateway_admin::AdminServices,
     config: gateway_api::ApiConfig,
+) -> gateway_api::ApiBundle {
+    api_bundle_with_diagnostics(
+        admin,
+        config,
+        Arc::new(crate::support::RecordingDiagnostics::default()),
+    )
+}
+
+pub(crate) fn api_bundle_with_diagnostics(
+    admin: gateway_admin::AdminServices,
+    config: gateway_api::ApiConfig,
+    diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
 ) -> gateway_api::ApiBundle {
     let execution = Arc::new(DefaultExecutionService::new(
         RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")),
@@ -114,6 +127,7 @@ pub(super) fn api_bundle(
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     ));
     gateway_api::initialize(
         config,
@@ -122,6 +136,7 @@ pub(super) fn api_bundle(
         Vec::new(),
         Arc::new(EmptyWorkerHealth),
         Arc::new(TestLifecycle::default()),
+        diagnostics,
     )
     .unwrap()
 }
@@ -138,6 +153,7 @@ fn api_router_with_config_and_execution(
         Vec::new(),
         Arc::new(EmptyWorkerHealth),
         Arc::new(TestLifecycle::default()),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .expect("API bundle")
     .router()
@@ -180,6 +196,7 @@ async fn api_router_with_origins_and_worker_health(
         Vec::new(),
         worker_health,
         Arc::new(TestLifecycle::default()),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .expect("API bundle")
     .router()
@@ -214,6 +231,7 @@ pub(super) fn authenticated_client_for_provider_with_limit(
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     );
     source
         .authenticate(plaintext)
@@ -244,6 +262,7 @@ pub(super) fn authenticated_client_with_min_versions(
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     );
     source
         .authenticate(plaintext)

backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs

@@ -165,6 +165,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake
         Arc::new(crate::openai::UnusedAdmissions),
         Arc::new(crate::openai::UnusedContinuation),
         Arc::new(crate::openai::IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     ));
     let admin = crate::admin::AdminTestFixture::new().await;
     let plan = Arc::new(SettingsPlan::default());
@@ -187,6 +188,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake
         vec![],
         Arc::new(crate::openai::EmptyWorkerHealth),
         Arc::new(crate::openai::TestLifecycle::default()),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .unwrap()
     .with_middleware(move |_| Some(frozen.clone()))

backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs

@@ -761,6 +761,7 @@ async fn start_active_response_with_middleware(
             Vec::new(),
             Arc::new(crate::openai::EmptyWorkerHealth),
             Arc::new(crate::openai::TestLifecycle::default()),
+            Arc::new(crate::support::RecordingDiagnostics::default()),
         )
         .unwrap()
         .with_middleware(move |_| Some(plan.clone()))

backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs

@@ -162,6 +162,7 @@ async fn interrupt_is_live_scoped_and_keeps_queued_creates_serial() {
         admissions.clone(),
         Arc::new(UnusedContinuation),
         Arc::new(IgnoredClientApiKeyUsage),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     ));
     let app = api_router(execution).await;
     let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();

backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs

@@ -1108,6 +1108,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele
             ports.clone(),
             Arc::new(UnusedContinuation),
             Arc::new(IgnoredClientApiKeyUsage),
+            Arc::new(crate::support::RecordingDiagnostics::default()),
         )
         .with_budget(ports.clone()),
     );
@@ -1125,6 +1126,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele
         Vec::new(),
         Arc::new(EmptyWorkerHealth),
         lifecycle.clone(),
+        Arc::new(crate::support::RecordingDiagnostics::default()),
     )
     .unwrap()
     .router();

backend/crates/gateway-api/tests/support.rs

@@ -159,3 +159,18 @@ impl ClientKeyVerifier for AcceptingVerifier {
         }
     }
 }
+
+#[derive(Default)]
+pub(crate) struct RecordingDiagnostics(
+    pub std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>,
+);
+#[async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
+    async fn record_failure(
+        &self,
+        failure: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), gateway_core::error::StoreError> {
+        self.0.lock().unwrap().push(failure);
+        Ok(())
+    }
+}

backend/crates/gateway-core/src/diagnostics/failure.rs

@@ -0,0 +1,54 @@
+//! 控制面与后台操作的受控故障记录，不伪造模型请求或上游发送事实
+
+use async_trait::async_trait;
+
+use crate::{
+    account::ProviderAccountId,
+    error::{ErrorDetails, OpaqueUpstreamValue, StoreError},
+    identity::ProviderKind,
+};
+
+#[derive(Debug)]
+pub struct OperationalFailure {
+    pub occurred_at: std::time::SystemTime,
+    pub component: &'static str,
+    pub operation: &'static str,
+    pub kind: &'static str,
+    pub message: String,
+    pub correlation_id: Option<String>,
+    pub provider_kind: Option<ProviderKind>,
+    pub account_id: Option<ProviderAccountId>,
+    pub upstream_status: Option<u16>,
+    pub upstream_code: Option<OpaqueUpstreamValue>,
+    pub details: Option<ErrorDetails>,
+}
+
+impl OperationalFailure {
+    #[must_use]
+    pub fn new(
+        component: &'static str,
+        operation: &'static str,
+        kind: &'static str,
+        message: impl Into<String>,
+    ) -> Self {
+        Self {
+            occurred_at: std::time::SystemTime::now(),
+            component,
+            operation,
+            kind,
+            message: message.into(),
+            correlation_id: None,
+            provider_kind: None,
+            account_id: None,
+            upstream_status: None,
+            upstream_code: None,
+            details: None,
+        }
+    }
+}
+
+/// 记录遵循既有观测队列的 best-effort 合同，失败不应替换业务结果
+#[async_trait]
+pub trait OperationalDiagnostics: Send + Sync {
+    async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError>;
+}

backend/crates/gateway-core/src/diagnostics/mod.rs

@@ -1,10 +1,12 @@
-//! 有界、请求局部的诊断证据；不参与路由、重试或客户端协议决策
+//! 请求诊断与操作故障的观测合同，不参与路由、重试或客户端协议决策
 
 mod capture;
+mod failure;
 mod selection;
 mod stream;
 mod trace;
 
 pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json};
+pub use failure::{OperationalDiagnostics, OperationalFailure};
 pub use stream::{StreamCapture, StreamFormat};
 pub use trace::TraceContext;

backend/crates/gateway-core/src/engine/admission.rs

@@ -57,9 +57,9 @@ pub struct ClientAdmissionRestoreResult {
     pub restored_running_requests: u64,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 #[error("client admission store is unavailable")]
-pub struct ClientAdmissionError;
+pub struct ClientAdmissionError(#[source] pub Option<crate::error::ErrorSource>);
 
 pub trait ClientAdmissionPort: Send + Sync {
     /// 仅续期已经准入的并发槽位，不重复消费 RPM
@@ -118,10 +118,10 @@ pub async fn restore_client_admission_startup(
     let expired = execution
         .recover_expired(now)
         .await
-        .map_err(|_| ClientAdmissionError)?;
+        .map_err(|source| ClientAdmissionError(Some(source.into())))?;
     let since = now
         .checked_sub(Duration::from_secs(61))
-        .ok_or(ClientAdmissionError)?;
+        .ok_or(ClientAdmissionError(None))?;
     let recoveries = recovery.load_recovery(since).await?;
     let restored_clients = u64::try_from(recoveries.len()).unwrap_or(u64::MAX);
     let mut restored_recent_requests = 0_u64;

backend/crates/gateway-core/src/engine/budget.rs

@@ -39,9 +39,9 @@ pub struct ClientBudgetCharge {
     pub completed_at: SystemTime,
 }
 
-#[derive(Debug, Clone, Copy, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 #[error("client budget store is unavailable")]
-pub struct ClientBudgetError;
+pub struct ClientBudgetError(#[source] pub Option<crate::error::ErrorSource>);
 
 pub trait ClientBudgetPort: Send + Sync {
     /// 原子检查当前限额与已用金额，不创建预扣费或待结算记录

backend/crates/gateway-core/src/engine/execution.rs

@@ -739,6 +739,7 @@ pub struct DefaultExecutionService {
     snapshots: RuntimeSnapshotHandle,
     /// probe 自身走 transient store，探测失败仍写入持久 store 的 ops_events
     observations: Arc<dyn ExecutionStore>,
+    diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
     providers: ProviderRegistry,
     admissions: Arc<dyn ClientAdmissionPort>,
     admission_waiting: ConcurrencyWaitQueue<ClientApiKeyId>,
@@ -763,10 +764,12 @@ impl DefaultExecutionService {
         admissions: Arc<dyn ClientAdmissionPort>,
         continuation: Arc<dyn NativeContinuationPort>,
         client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
+        diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
     ) -> Self {
         Self {
             snapshots,
             observations: execution,
+            diagnostics,
             providers,
             admissions,
             admission_waiting: ConcurrencyWaitQueue::default(),
@@ -1325,7 +1328,12 @@ impl DefaultExecutionService {
             && let Some(guard) = start_guard.take()
         {
             guard
-                .release_failed(self.budget.as_deref(), request_id, budget_key_id)
+                .release_failed(
+                    self.budget.as_deref(),
+                    request_id,
+                    budget_key_id,
+                    self.diagnostics.as_ref(),
+                )
                 .await;
         }
         result
@@ -1498,6 +1506,7 @@ impl DefaultExecutionService {
                             amount_usd: crate::metering::Decimal::ZERO,
                             completed_at: SystemTime::now(),
                         },
+                        self.diagnostics.as_ref(),
                     )
                     .await;
                 }
@@ -1518,6 +1527,7 @@ impl DefaultExecutionService {
                 active_request,
                 Arc::clone(&self.continuation),
                 self.budget.clone(),
+                self.diagnostics.clone(),
             )),
         })
     }
@@ -1872,6 +1882,7 @@ impl DefaultExecutionService {
             CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), budget);
         let mut admission = AdmissionLease {
             port: Arc::clone(&self.admissions),
+            diagnostics: self.diagnostics.clone(),
             client_api_key_id: key.clone(),
             model_request_id: request_id.clone(),
             armed: false,
@@ -1902,7 +1913,7 @@ impl DefaultExecutionService {
             pin_mut!(acquire, timeout, cancelled);
             let decision = select_biased! {
                 () = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "request admission was cancelled")),
-                result = acquire => result.map_err(|_| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable"))?,
+                result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable").with_source(source))?,
                 _ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")),
             };
             match decision {
@@ -2477,14 +2488,26 @@ struct AdmissionLease {
     renewal: Option<Box<dyn LeaseGuard>>,
     armed: bool,
     port: Arc<dyn ClientAdmissionPort>,
+    diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
     client_api_key_id: ClientApiKeyId,
     model_request_id: ModelRequestId,
 }
 
-async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) {
+async fn settle_budget(
+    port: &dyn ClientBudgetPort,
+    charge: ClientBudgetCharge,
+    diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
+) {
     let request_id = charge.request_id.clone();
     if let Err(error) = port.settle(charge).await {
-        tracing::error!(request_id = request_id.as_str(), operation = "settle_client_budget", %error, "Client budget settlement failed; storage will retry on the next request");
+        record_resource_failure(
+            diagnostics,
+            &request_id,
+            "settle_client_budget",
+            "Client budget settlement failed; storage will retry on the next request",
+            error,
+        )
+        .await;
     }
 }
 
@@ -2496,7 +2519,14 @@ impl AdmissionLease {
             .release(&self.client_api_key_id, &self.model_request_id)
             .await
         {
-            tracing::warn!(request_id = self.model_request_id.as_str(), key_id = self.client_api_key_id.as_str(), operation = "release_client_admission", %error, "Client admission 释放失败，依赖租约 TTL 收敛");
+            record_resource_failure(
+                self.diagnostics.as_ref(),
+                &self.model_request_id,
+                "release_client_admission",
+                "Client admission release failed; lease TTL remains active",
+                error,
+            )
+            .await;
         }
         self.armed = false;
     }
@@ -2537,6 +2567,7 @@ impl ExecutionStartGuard {
         budget: Option<&dyn ClientBudgetPort>,
         request_id: ModelRequestId,
         key_id: ClientApiKeyId,
+        diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
     ) {
         if let Some(active_request) = self.active_request {
             active_request.release();
@@ -2550,6 +2581,7 @@ impl ExecutionStartGuard {
                     amount_usd: crate::metering::Decimal::ZERO,
                     completed_at: SystemTime::now(),
                 },
+                diagnostics,
             )
             .await;
         }
@@ -2565,6 +2597,7 @@ struct DefaultExecutionSession {
     continuation: Arc<dyn NativeContinuationPort>,
     continuation_recorded: bool,
     budget: Option<Arc<dyn ClientBudgetPort>>,
+    diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
 }
 
 impl DefaultExecutionSession {
@@ -2574,6 +2607,7 @@ impl DefaultExecutionSession {
         active_request: Option<ActiveRequestLease>,
         continuation: Arc<dyn NativeContinuationPort>,
         budget: Option<Arc<dyn ClientBudgetPort>>,
+        diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
     ) -> Self {
         Self {
             core,
@@ -2583,6 +2617,7 @@ impl DefaultExecutionSession {
             continuation,
             continuation_recorded: false,
             budget,
+            diagnostics,
         }
     }
 
@@ -2595,12 +2630,13 @@ impl DefaultExecutionSession {
             }
             let budget = self.budget.take();
             let charge = self.core.budget_charge();
+            let diagnostics = self.diagnostics.clone();
             // 在首次 await 前把完整清理责任留在会话内
             // 事件等待被取消后，后续 poll
             // 或 detach 继续同一个 future，既不丢失费用，也不重启已完成的结算
             self.cleanup = Some(Box::pin(async move {
                 if let Some(budget) = budget {
-                    settle_budget(budget.as_ref(), charge).await;
+                    settle_budget(budget.as_ref(), charge, diagnostics.as_ref()).await;
                 }
                 admission.release().await;
             }));
@@ -2923,6 +2959,34 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
                 "no provider is available",
             )
         }
+        EngineError::Store(source) => {
+            GatewayError::new(GatewayErrorKind::Internal, "request execution failed")
+                .with_source(source.clone())
+        }
         _ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
     }
 }
+
+async fn record_resource_failure(
+    diagnostics: &dyn crate::diagnostics::OperationalDiagnostics,
+    request_id: &ModelRequestId,
+    operation: &'static str,
+    message: &'static str,
+    source: impl Into<crate::error::ErrorSource>,
+) {
+    let mut failure = crate::diagnostics::OperationalFailure::new(
+        "core",
+        operation,
+        "resource_unavailable",
+        message,
+    );
+    failure.correlation_id = Some(request_id.as_str().to_owned());
+    failure.details = crate::error::ErrorDetails::capture(Some(&source.into()), None, false);
+    if diagnostics.record_failure(failure).await.is_err() {
+        tracing::warn!(
+            request_id = request_id.as_str(),
+            operation,
+            "resource diagnostic could not be recorded"
+        );
+    }
+}

backend/crates/gateway-core/src/error/details.rs

@@ -0,0 +1,43 @@
+//! 原因链与上游正文的受控诊断投影，不参与普通错误格式化
+
+use super::{ErrorSource, RawUpstreamError};
+
+#[derive(Clone)]
+pub struct ErrorDetails(String);
+
+impl ErrorDetails {
+    #[must_use]
+    pub fn capture(
+        source: Option<&ErrorSource>,
+        upstream: Option<&RawUpstreamError>,
+        redacted: bool,
+    ) -> Option<Self> {
+        if source.is_none() && upstream.is_none() {
+            return None;
+        }
+        Some(Self(
+            serde_json::json!({
+                "causes": source.map(ErrorSource::snapshot),
+                "upstream": upstream.map(RawUpstreamError::as_str),
+                "redacted": redacted,
+            })
+            .to_string(),
+        ))
+    }
+
+    #[must_use]
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    #[must_use]
+    pub fn into_string(self) -> String {
+        self.0
+    }
+}
+
+impl std::fmt::Debug for ErrorDetails {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str("ErrorDetails(<restricted>)")
+    }
+}

backend/crates/gateway-core/src/error/mod.rs

@@ -1,6 +1,8 @@
 //! 网关核心使用的稳定错误分类
 
+mod details;
 mod source;
+pub use details::ErrorDetails;
 
 pub use source::ErrorSource;
 
@@ -865,25 +867,20 @@ impl ProviderError {
 
     /// 包装底层失败；来源只供显式诊断访问，不改变分类和发送事实
     #[must_use]
-    pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self {
-        self.source = Some(ErrorSource::new(source));
+    pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
+        self.source = Some(source.into());
         self
     }
 
     /// 运维详情的唯一投影入口，来源链与上游正文分别保留，不进入公共响应或诊断包
     #[must_use]
     pub fn error_details(&self) -> Option<String> {
-        if self.source.is_none() && self.raw_upstream_error.is_none() {
-            return None;
-        }
-        Some(
-            serde_json::json!({
-                "causes": self.source.as_ref().map(ErrorSource::snapshot),
-                "upstream": self.raw_upstream_error().map(RawUpstreamError::as_str),
-                "redacted": self.sensitive_context_was_redacted(),
-            })
-            .to_string(),
+        ErrorDetails::capture(
+            self.source.as_ref(),
+            self.raw_upstream_error(),
+            self.sensitive_context_was_redacted(),
         )
+        .map(ErrorDetails::into_string)
     }
 
     /// 返回将由运维错误详情原样展示的上游错误返回
@@ -1094,14 +1091,15 @@ impl GatewayErrorKind {
 }
 
 /// 协议无关、可安全暴露的网关错误
-#[derive(Clone, PartialEq, Eq, Error)]
+#[derive(Clone, Error)]
 #[error("{message}")]
 pub struct GatewayError {
     kind: GatewayErrorKind,
     message: &'static str,
     diagnostic: Option<Box<ProviderDiagnostic>>,
-    client_visible_upstream_error: Option<ClientVisibleUpstreamError>,
+    client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
     client_details: Option<Box<GatewayClientErrorDetails>>,
+    source: Option<ErrorSource>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
@@ -1120,9 +1118,21 @@ impl GatewayError {
             diagnostic: None,
             client_visible_upstream_error: None,
             client_details: None,
+            source: None,
         }
     }
 
+    #[must_use]
+    pub fn with_source(mut self, source: impl Into<ErrorSource>) -> Self {
+        self.source = Some(source.into());
+        self
+    }
+
+    #[must_use]
+    pub fn error_details(&self) -> Option<ErrorDetails> {
+        ErrorDetails::capture(self.source.as_ref(), None, false)
+    }
+
     #[must_use]
     pub fn with_client_code(mut self, code: &'static str) -> Self {
         self.client_details
@@ -1232,7 +1242,7 @@ impl GatewayError {
     /// 附加只供请求方协议展示的结构化上游错误
     #[must_use]
     pub fn with_client_visible_upstream_error(mut self, error: ClientVisibleUpstreamError) -> Self {
-        self.client_visible_upstream_error = Some(error);
+        self.client_visible_upstream_error = Some(Box::new(error));
         self
     }
 
@@ -1259,7 +1269,7 @@ impl GatewayError {
     #[must_use]
     pub fn client_message(&self) -> &str {
         self.client_visible_upstream_error
-            .as_ref()
+            .as_deref()
             .map_or(self.message, ClientVisibleUpstreamError::message)
     }
 
@@ -1267,7 +1277,7 @@ impl GatewayError {
     #[must_use]
     pub fn client_error_type(&self) -> Option<&str> {
         self.client_visible_upstream_error
-            .as_ref()
+            .as_deref()
             .and_then(ClientVisibleUpstreamError::error_type)
     }
 
@@ -1275,7 +1285,7 @@ impl GatewayError {
     #[must_use]
     pub fn client_error_code(&self) -> Option<&str> {
         self.client_visible_upstream_error
-            .as_ref()
+            .as_deref()
             .and_then(ClientVisibleUpstreamError::code)
             .or_else(|| {
                 self.client_details
@@ -1337,13 +1347,10 @@ impl StoreError {
 
     /// 包装已有存储错误并保留来源
     #[must_use]
-    pub fn caused_by(
-        kind: StoreErrorKind,
-        source: impl std::error::Error + Send + Sync + 'static,
-    ) -> Self {
+    pub fn caused_by(kind: StoreErrorKind, source: impl Into<crate::error::ErrorSource>) -> Self {
         Self {
             kind,
-            source: Some(ErrorSource::new(source)),
+            source: Some(source.into()),
         }
     }
 

backend/crates/gateway-core/src/error/source.rs

@@ -12,32 +12,125 @@ impl ErrorSource {
         Self(Arc::new(error))
     }
 
+    /// 清理失败是附属事实，标准来源链仍指向最初失败
+    #[must_use]
+    pub fn with_cleanup(self, cleanup: impl Into<Self>) -> Self {
+        Self::new(CleanupFailure {
+            primary: Some(self),
+            cleanup: cleanup.into(),
+        })
+    }
+
+    /// 附属清理错误与可选的原始底层原因分别保留
+    #[must_use]
+    pub fn cleanup(primary: Option<Self>, cleanup: impl Into<Self>) -> Self {
+        Self::new(CleanupFailure {
+            primary,
+            cleanup: cleanup.into(),
+        })
+    }
+
     /// 仅供受控错误详情落盘，限制链深度和总文本大小并显式标记缺口
     pub(super) fn snapshot(&self) -> serde_json::Value {
-        let mut messages = Vec::new();
-        let mut source: Option<&(dyn Error + 'static)> = Some(self.0.as_ref());
-        let mut remaining = 64 * 1024;
-        let mut truncated = false;
-        for _ in 0..32 {
-            let Some(error) = source else { break };
-            let mut message = BoundedMessage {
-                text: String::new(),
-                remaining,
-            };
-            if fmt::write(&mut message, format_args!("{error}")).is_err() {
-                truncated = true;
-            }
-            remaining = message.remaining;
-            messages.push(message.text);
-            source = error.source();
-            if truncated || remaining == 0 {
-                break;
+        snapshot(self.0.as_ref(), &mut 32, &mut (64 * 1024))
+    }
+
+    /// 观测队列估算来源占用，包含附属清理失败；过深或循环的来源按超限处理
+    #[must_use]
+    pub fn estimated_chain_bytes(source: Option<&(dyn Error + 'static)>) -> usize {
+        struct ByteCount(usize);
+        impl fmt::Write for ByteCount {
+            fn write_str(&mut self, value: &str) -> fmt::Result {
+                self.0 = self.0.saturating_add(value.len());
+                Ok(())
             }
         }
-        serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()})
+        let mut pending = Vec::from_iter(source);
+        let mut bytes = ByteCount(0);
+        for _ in 0..32 {
+            let Some(error) = pending.pop() else {
+                return bytes.0;
+            };
+            bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error));
+            if fmt::write(&mut bytes, format_args!("{error}")).is_err() {
+                return usize::MAX;
+            }
+            if let Some(cleanup) = error.downcast_ref::<CleanupFailure>() {
+                pending.push(cleanup.cleanup.0.as_ref());
+            }
+            if let Some(source) = error.source() {
+                pending.push(source);
+            }
+        }
+        if pending.is_empty() {
+            bytes.0
+        } else {
+            usize::MAX
+        }
     }
 }
 
+impl<E: Error + Send + Sync + 'static> From<E> for ErrorSource {
+    fn from(error: E) -> Self {
+        Self::new(error)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("operation failed and cleanup also failed")]
+struct CleanupFailure {
+    #[source]
+    primary: Option<ErrorSource>,
+    cleanup: ErrorSource,
+}
+
+fn snapshot(
+    error: &(dyn Error + 'static),
+    nodes: &mut usize,
+    bytes: &mut usize,
+) -> serde_json::Value {
+    let mut messages = Vec::new();
+    let mut cleanup_sources = Vec::new();
+    let mut source = Some(error);
+    let mut truncated = false;
+    while *nodes > 0 {
+        let Some(error) = source else { break };
+        *nodes -= 1;
+        if let Some(failure) = error.downcast_ref::<CleanupFailure>() {
+            cleanup_sources.push(failure.cleanup.0.as_ref());
+            source = failure
+                .primary
+                .as_ref()
+                .map(|primary| primary.0.as_ref() as &(dyn Error + 'static));
+            continue;
+        }
+        let mut message = BoundedMessage {
+            text: String::new(),
+            remaining: *bytes,
+        };
+        if fmt::write(&mut message, format_args!("{error}")).is_err() {
+            truncated = true;
+        }
+        *bytes = message.remaining;
+        messages.push(message.text);
+        source = error.source();
+        if truncated || *bytes == 0 {
+            break;
+        }
+    }
+    let mut result =
+        serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()});
+    if !cleanup_sources.is_empty() {
+        result["cleanup"] = serde_json::Value::Array(
+            cleanup_sources
+                .into_iter()
+                .map(|error| snapshot(error, nodes, bytes))
+                .collect(),
+        );
+    }
+    result
+}
+
 struct BoundedMessage {
     text: String,
     remaining: usize,

backend/crates/gateway-core/src/lib.rs

@@ -53,6 +53,7 @@ use task::{WorkerContribution, WorkerDefinitionError};
 #[derive(Clone)]
 pub struct CoreStorePorts {
     execution: Arc<dyn ExecutionStore>,
+    diagnostics: Arc<dyn diagnostics::OperationalDiagnostics>,
     admissions: Arc<dyn ClientAdmissionPort>,
     admission_recovery: Arc<dyn ClientAdmissionRecoveryPort>,
     continuation: Arc<dyn NativeContinuationPort>,
@@ -77,9 +78,11 @@ impl CoreStorePorts {
             Arc<dyn SnapshotSubscriptionPort>,
         ),
         client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
+        diagnostics: Arc<dyn diagnostics::OperationalDiagnostics>,
     ) -> Self {
         Self {
             execution,
+            diagnostics,
             admissions,
             admission_recovery,
             continuation,
@@ -201,6 +204,7 @@ fn build_execution_service(
         Arc::clone(&ports.admissions),
         Arc::clone(&ports.continuation),
         Arc::clone(&ports.client_api_key_usage),
+        Arc::clone(&ports.diagnostics),
     );
     if let Some(budget) = &ports.budget {
         service = service.with_budget(Arc::clone(budget));

backend/crates/gateway-core/src/provider_ports.rs

@@ -51,12 +51,12 @@ impl ProviderStoreError {
     pub fn caused_by(
         kind: ProviderStoreErrorKind,
         operation: &'static str,
-        source: impl std::error::Error + Send + Sync + 'static,
+        source: impl Into<crate::error::ErrorSource>,
     ) -> Self {
         Self {
             kind,
             operation,
-            source: Some(crate::error::ErrorSource::new(source)),
+            source: Some(source.into()),
         }
     }
 
@@ -1494,6 +1494,7 @@ pub struct ProviderStorePorts {
     cooldowns: Arc<dyn ProviderCooldownPort>,
     runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
     oauth_pending: Arc<dyn OAuthPendingFlowPort>,
+    diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
 }
 
 impl ProviderStorePorts {
@@ -1511,6 +1512,7 @@ impl ProviderStorePorts {
         cooldowns: Arc<dyn ProviderCooldownPort>,
         runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
         oauth_pending: Arc<dyn OAuthPendingFlowPort>,
+        diagnostics: Arc<dyn crate::diagnostics::OperationalDiagnostics>,
     ) -> Self {
         Self {
             accounts,
@@ -1524,9 +1526,15 @@ impl ProviderStorePorts {
             cooldowns,
             runtime_policy,
             oauth_pending,
+            diagnostics,
         }
     }
 
+    #[must_use]
+    pub fn diagnostics(&self) -> Arc<dyn crate::diagnostics::OperationalDiagnostics> {
+        self.diagnostics.clone()
+    }
+
     #[must_use]
     pub fn accounts(&self) -> Arc<dyn ProviderAccountStore> {
         Arc::clone(&self.accounts)

backend/crates/gateway-core/src/runtime/mod.rs

@@ -454,7 +454,10 @@ impl ScheduledTask for RuntimeSnapshotReconciliationTask {
                 .refresh_with_mode(RefreshMode::Reconcile)
                 .await
                 .map(|_| ())
-                .map_err(|_| WorkerTaskError::safe("runtime snapshot reconciliation failed"))
+                .map_err(|source| {
+                    WorkerTaskError::safe("runtime snapshot reconciliation failed")
+                        .with_source(source)
+                })
         })
     }
 }

backend/crates/gateway-core/src/task.rs

@@ -485,10 +485,11 @@ pub enum WorkerDefinitionError {
 }
 
 /// 不暴露基础设施原文或 lease resource 的错误
-#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 #[error("{message}")]
 pub struct WorkerLeaseError {
     message: String,
+    source: Option<crate::error::ErrorSource>,
 }
 
 impl WorkerLeaseError {
@@ -496,6 +497,7 @@ impl WorkerLeaseError {
     pub fn safe(message: impl Into<String>) -> Self {
         Self {
             message: message.into(),
+            source: None,
         }
     }
 
@@ -503,13 +505,24 @@ impl WorkerLeaseError {
     pub fn as_safe_str(&self) -> &str {
         &self.message
     }
+    #[must_use]
+    pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
+        self.source = Some(source.into());
+        self
+    }
+
+    #[must_use]
+    pub fn error_details(&self) -> Option<crate::error::ErrorDetails> {
+        crate::error::ErrorDetails::capture(self.source.as_ref(), None, false)
+    }
 }
 
 /// 不暴露 Provider 原文或存储细节的后台任务错误
-#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 #[error("{message}")]
 pub struct WorkerTaskError {
     message: String,
+    source: Option<crate::error::ErrorSource>,
 }
 
 impl WorkerTaskError {
@@ -517,6 +530,7 @@ impl WorkerTaskError {
     pub fn safe(message: impl Into<String>) -> Self {
         Self {
             message: message.into(),
+            source: None,
         }
     }
 
@@ -524,6 +538,25 @@ impl WorkerTaskError {
     pub fn as_safe_str(&self) -> &str {
         &self.message
     }
+    #[must_use]
+    pub fn with_source(mut self, source: impl Into<crate::error::ErrorSource>) -> Self {
+        self.source = Some(source.into());
+        self
+    }
+
+    #[must_use]
+    pub fn error_details(&self) -> Option<crate::error::ErrorDetails> {
+        crate::error::ErrorDetails::capture(self.source.as_ref(), None, false)
+    }
+
+    #[must_use]
+    pub fn with_cleanup(mut self, cleanup: impl Into<crate::error::ErrorSource>) -> Self {
+        self.source = Some(crate::error::ErrorSource::cleanup(
+            self.source.take(),
+            cleanup,
+        ));
+        self
+    }
 }
 
 fn valid_owner(owner: &str) -> bool {

backend/crates/gateway-core/tests/engine/admission.rs

@@ -72,7 +72,7 @@ fn client_admission_startup_recovery_should_fail_closed_at_each_boundary() {
         assert!(
             restore_client_admission_startup(
                 &RecoveryStore::success(0),
-                &ScriptedRecoveries::new(Err(ClientAdmissionError)),
+                &ScriptedRecoveries::new(Err(ClientAdmissionError(None))),
                 &RecordingAdmissions::default(),
                 now,
             )
@@ -212,7 +212,7 @@ impl ClientAdmissionPort for RecordingAdmissions {
     ) -> BoxFuture<'_, Result<ClientAdmissionRestoreResult, ClientAdmissionError>> {
         Box::pin(async move {
             if self.fail_restore {
-                return Err(ClientAdmissionError);
+                return Err(ClientAdmissionError(None));
             }
             self.restored.lock().expect("restored").push(recovery);
             Ok(ClientAdmissionRestoreResult {

backend/crates/gateway-core/tests/engine/execution.rs

@@ -99,7 +99,9 @@ impl ClientBudgetPort for Budget {
             assert!(self.active.load(Ordering::SeqCst));
             self.charges.lock().unwrap().push(charge);
             if self.fail_settlement {
-                Err(ClientBudgetError)
+                Err(ClientBudgetError(Some(
+                    std::io::Error::other("PRIVATE_SETTLEMENT_CAUSE").into(),
+                )))
             } else {
                 Ok(())
             }
@@ -115,6 +117,7 @@ fn service(admissions: Arc<Admissions>, budget: Arc<Budget>) -> DefaultExecution
         admissions,
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     )
     .with_budget(budget)
 }
@@ -324,6 +327,7 @@ fn reused_client_uses_updated_limits_for_each_execution() {
         admissions.clone(),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service.authenticate("sk_start_test").unwrap();
     let limited = RateLimits {
@@ -360,6 +364,7 @@ fn reused_client_cannot_start_after_key_disable_or_snapshot_suspension() {
             admissions.clone(),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let next = request(&service, ClientTransport::WebSocket);
         if suspend {
@@ -463,6 +468,7 @@ fn reused_client_is_reauthenticated_by_the_current_frontend_plan_without_identit
             admissions.clone(),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_frontend_authentication(authentication.clone());
         let client = service
@@ -554,6 +560,7 @@ fn early_failure_service(
         admissions,
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     )
     .with_budget(budget)
 }
@@ -911,6 +918,7 @@ fn retry_policy_can_stop_but_cannot_bypass_replay_safety() {
                 Arc::new(Admissions::default()),
                 Arc::new(UnusedContinuation),
                 Arc::new(RecordingClientApiKeyUsage::default()),
+                Arc::new(crate::RecordingDiagnostics::default()),
             )
             .with_request_policies(policies);
             let mut started = service
@@ -1169,6 +1177,7 @@ fn completed_parent_rejects_new_nested_execution_and_cancels_an_active_child() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index);
 
@@ -1360,6 +1369,7 @@ fn native_response_processing_uses_the_real_translation_boundary() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_observers(observer_index);
 
@@ -1510,6 +1520,7 @@ fn response_translation_zero_output_preserves_canonical_usage_and_finalization()
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
 
         let mut started = service
@@ -1663,6 +1674,7 @@ fn native_response_failure_after_downstream_commit_is_not_replayed() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
 
         let mut started = service
@@ -1735,6 +1747,7 @@ fn model_routing_policy_selects_native_provider() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index);
         let mut started = service
@@ -1788,6 +1801,7 @@ fn model_routing_reject_stops_before_provider_execution() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index);
         assert!(matches!(
@@ -1875,6 +1889,7 @@ fn model_routing_cannot_expand_the_frozen_key_model_scope() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index);
 
@@ -2022,6 +2037,7 @@ fn scheduler_reject_is_a_terminal_policy_rejection_before_upstream_send() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index)
         .with_request_observers(observer_index);
@@ -2292,6 +2308,7 @@ fn observation_service(
         admissions,
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     )
     .with_budget(budget)
     .with_request_observers(observer_index);
@@ -2329,6 +2346,7 @@ fn fallback_exhaustion_observation_uses_the_last_actual_provider_not_the_candida
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_observers(observer_index);
         let mut started = service
@@ -2393,6 +2411,7 @@ fn routing_external_effect_stops_a_not_sent_provider_retry() {
             Arc::new(Admissions::default()),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         )
         .with_request_policies(policy_index)
         .with_request_observers(observer_index);
@@ -2579,6 +2598,20 @@ fn charged_service(
     admissions: Arc<Admissions>,
     budget: Option<Arc<Budget>>,
     fail: bool,
+) -> DefaultExecutionService {
+    charged_service_with_diagnostics(
+        admissions,
+        budget,
+        fail,
+        Arc::new(crate::RecordingDiagnostics::default()),
+    )
+}
+
+fn charged_service_with_diagnostics(
+    admissions: Arc<Admissions>,
+    budget: Option<Arc<Budget>>,
+    fail: bool,
+    diagnostics: Arc<crate::RecordingDiagnostics>,
 ) -> DefaultExecutionService {
     let service = DefaultExecutionService::new(
         RuntimeSnapshotHandle::new(start_snapshot()),
@@ -2591,6 +2624,7 @@ fn charged_service(
         admissions,
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        diagnostics,
     );
     match budget {
         Some(budget) => service.with_budget(budget),
@@ -2826,7 +2860,13 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() {
                 fail_settlement,
                 ..Default::default()
             });
-            let service = charged_service(admissions.clone(), Some(budget.clone()), true);
+            let diagnostics = Arc::new(crate::RecordingDiagnostics::default());
+            let service = charged_service_with_diagnostics(
+                admissions.clone(),
+                Some(budget.clone()),
+                true,
+                diagnostics.clone(),
+            );
             let mut started = service
                 .start(request(&service, ClientTransport::WebSocket))
                 .await
@@ -2840,6 +2880,24 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() {
             started.session.detach_finalize().await;
             // Store 端口已接管精确费用后，结算错误不能改写 Provider 错误或触发第二次结算
             assert_cleanup_completed(&admissions, &budget, &started.request_id);
+            let failures = diagnostics.0.lock().unwrap();
+            assert_eq!(failures.len(), usize::from(fail_settlement));
+            if let Some(failure) = failures.first() {
+                assert_eq!(failure.operation, "settle_client_budget");
+                assert_eq!(
+                    failure.correlation_id.as_deref(),
+                    Some(started.request_id.as_str())
+                );
+                assert!(
+                    failure
+                        .details
+                        .as_ref()
+                        .unwrap()
+                        .as_str()
+                        .contains("PRIVATE_SETTLEMENT_CAUSE")
+                );
+                assert!(!format!("{failure:?}").contains("PRIVATE_SETTLEMENT_CAUSE"));
+            }
         }
     });
 }
@@ -2923,6 +2981,7 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let error = block_on(service.probe(
@@ -2954,6 +3013,7 @@ fn probe_failures_should_be_observable_without_a_model_request_row() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let error = block_on(service.probe(
@@ -2999,6 +3059,7 @@ fn provider_local_probe_failure_should_remain_distinct_from_upstream() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let error = block_on(service.probe(
@@ -3041,6 +3102,7 @@ fn diagnostic_probe_does_not_apply_data_plane_account_model_policy() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let error = block_on(service.probe(
@@ -3072,6 +3134,7 @@ fn probe_observation_store_failure_preserves_the_provider_error() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let error = block_on(service.probe(
@@ -3195,6 +3258,7 @@ fn client_catalog_forwards_scope_maps_whole_objects_and_omits_unroutable_models(
             Arc::new(UnusedAdmissions),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let client = service.authenticate("sk_start_test").expect("authenticate");
         let result = block_on(service.client_model_catalog(&client, "codex", "0.154.0"));
@@ -3350,6 +3414,7 @@ fn successful_authentication_should_record_client_key_usage() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         usage.clone(),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     service
@@ -3369,6 +3434,7 @@ fn client_key_verification_should_not_record_client_key_usage() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         usage.clone(),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
 
     let key_id = service
@@ -3390,6 +3456,7 @@ fn request_verification_should_apply_entry_authentication_without_recording_key_
             Arc::new(UnusedAdmissions),
             Arc::new(UnusedContinuation),
             usage.clone(),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
 
         let client = service
@@ -3423,6 +3490,7 @@ fn assert_provider_endpoint_observation(model: Option<&str>) {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service
         .authenticate("sk_usage_test")
@@ -3476,6 +3544,7 @@ fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service
         .authenticate("sk_start_test")
@@ -3522,6 +3591,7 @@ fn continuation_owned_by_another_client_api_key_should_fail_closed() {
         Arc::new(UnusedAdmissions),
         Arc::new(RejectedContinuation::OwnershipMismatch),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service
         .authenticate("sk_start_test")
@@ -3549,6 +3619,7 @@ fn invalid_continuation_record_should_not_be_forwarded_as_an_external_handle() {
         Arc::new(UnusedAdmissions),
         Arc::new(RejectedContinuation::InvalidData),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service
         .authenticate("sk_start_test")
@@ -4024,6 +4095,7 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() {
             admissions.clone(),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let running = service
             .start(request(&service, ClientTransport::HttpSse))
@@ -4133,6 +4205,7 @@ fn queue_service(
         admissions.clone(),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     (service, admissions)
 }
@@ -4257,6 +4330,7 @@ fn reused_websocket_client_gets_group_fast_policy_from_each_new_request_snapshot
         admissions,
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service.authenticate("sk_start_test").unwrap();
     for (revision, fast_mode) in [
@@ -4306,6 +4380,7 @@ fn request_settings_recompute_routing_and_admission_without_changing_sibling_or_
             admissions.clone(),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let client = service.authenticate("sk_start_test").unwrap();
         let mut modified = service.prepare_verified_execution(client.clone()).unwrap();
@@ -4478,6 +4553,7 @@ fn repeated_connection_failures_never_block_later_requests_for_the_provider() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     for _ in 0..5 {
         let client = service
@@ -4528,6 +4604,7 @@ fn entry_settings_freeze_authentication_and_rebase_only_explicit_overrides() {
             Arc::new(UnusedAdmissions),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let baseline = service.request_settings().unwrap();
         let mut values = serde_json::to_value(baseline.values()).unwrap();
@@ -4670,6 +4747,7 @@ fn unchanged_and_precompiled_request_settings_reuse_the_frozen_snapshot() {
         Arc::new(UnusedAdmissions),
         Arc::new(UnusedContinuation),
         Arc::new(RecordingClientApiKeyUsage::default()),
+        Arc::new(crate::RecordingDiagnostics::default()),
     );
     let client = service.authenticate("sk_start_test").unwrap();
     let mut prepared = service.prepare_verified_execution(client).unwrap();
@@ -4767,6 +4845,7 @@ fn child_settings_recompute_key_scope_without_inheriting_parent_defaults() {
             Arc::new(UnusedAdmissions),
             Arc::new(UnusedContinuation),
             Arc::new(RecordingClientApiKeyUsage::default()),
+            Arc::new(crate::RecordingDiagnostics::default()),
         );
         let parent = service
             .prepare_execution(service.authenticate("sk_parent").unwrap())

backend/crates/gateway-core/tests/error/mod.rs

@@ -351,3 +351,34 @@ fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
     assert_eq!(detail.error_type(), Some(error_type.as_str()));
     assert!(!format!("{detail:?}").contains(&message));
 }
+
+#[test]
+fn cleanup_failure_keeps_the_primary_cause_and_separate_bounded_details() {
+    use gateway_core::error::{ErrorDetails, ErrorSource};
+    let primary = ErrorSource::new(std::io::Error::other("PRIMARY_NATIVE_FAILURE"));
+    let combined = primary.with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE"));
+    assert_eq!(
+        combined.source().unwrap().to_string(),
+        "PRIMARY_NATIVE_FAILURE"
+    );
+    let details = ErrorDetails::capture(Some(&combined), None, false).unwrap();
+    let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap();
+    assert_eq!(
+        value["causes"]["messages"],
+        json!(["PRIMARY_NATIVE_FAILURE"])
+    );
+    assert_eq!(
+        value["causes"]["cleanup"][0]["messages"],
+        json!(["ROLLBACK_NATIVE_FAILURE"])
+    );
+    assert_eq!(value["causes"]["truncated"], false);
+    assert!(!format!("{combined:?} {details:?}").contains("NATIVE_FAILURE"));
+
+    let oversized = ErrorSource::new(std::io::Error::other("界".repeat(24_000)))
+        .with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE"));
+    let details = ErrorDetails::capture(Some(&oversized), None, false).unwrap();
+    let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap();
+    assert_eq!(value["causes"]["truncated"], true);
+    assert_eq!(value["causes"]["cleanup"][0]["truncated"], true);
+    assert!(details.as_str().len() < 66_000);
+}

backend/crates/gateway-core/tests/main.rs

@@ -21,3 +21,16 @@ mod task;
 mod upstream;
 
 mod time;
+
+#[derive(Default)]
+struct RecordingDiagnostics(std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>);
+#[async_trait::async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
+    async fn record_failure(
+        &self,
+        failure: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), gateway_core::error::StoreError> {
+        self.0.lock().unwrap().push(failure);
+        Ok(())
+    }
+}

backend/crates/gateway-host/src/lib.rs

@@ -156,8 +156,9 @@ impl HostBundle {
         &self,
         plan: Vec<WorkerContribution>,
         lease: Arc<dyn WorkerLeaderLeasePort>,
+        diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
     ) -> Result<(), HostError> {
-        self.workers.start(plan, lease)?;
+        self.workers.start(plan, lease, diagnostics)?;
         Ok(())
     }
 

backend/crates/gateway-host/src/logging/sink.rs

@@ -1,7 +1,7 @@
-//! 有界文件日志队列：拥堵时背压，正常退出时排空；写入失败显式影响健康状态
+//! 非阻塞文件日志接收与后台批量写入，错误记录保留独立容量，关闭时排空
 
 use std::io::{self, Write};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, mpsc};
 use std::thread::{self, JoinHandle};
 
@@ -11,10 +11,70 @@ use tracing_subscriber::fmt::MakeWriter;
 
 use super::writer::RotatingLogWriter;
 
-const QUEUE_CAPACITY: usize = 64;
+const QUEUE_CAPACITY: usize = 4_096;
+const QUEUE_BYTES: usize = 64 * 1024 * 1024;
+const ERROR_CAPACITY: usize = 512;
+const ERROR_BYTES: usize = 8 * 1024 * 1024;
+const WRITE_BATCH: usize = 64;
+
+// 通道不另设容量；预留在复制正文之前完成，并随记录释放，包含正在写入的记录
+struct Budget {
+    items: AtomicUsize,
+    bytes: AtomicUsize,
+    maximum_items: usize,
+    maximum_bytes: usize,
+}
+
+impl Budget {
+    fn new(maximum_items: usize, maximum_bytes: usize) -> Arc<Self> {
+        Arc::new(Self {
+            items: AtomicUsize::new(0),
+            bytes: AtomicUsize::new(0),
+            maximum_items,
+            maximum_bytes,
+        })
+    }
+
+    fn reserve(self: &Arc<Self>, bytes: usize) -> Option<Reservation> {
+        self.items
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |items| {
+                items
+                    .checked_add(1)
+                    .filter(|items| *items <= self.maximum_items)
+            })
+            .ok()?;
+        if self
+            .bytes
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| {
+                used.checked_add(bytes)
+                    .filter(|used| *used <= self.maximum_bytes)
+            })
+            .is_err()
+        {
+            self.items.fetch_sub(1, Ordering::AcqRel);
+            return None;
+        }
+        Some(Reservation {
+            budget: Arc::clone(self),
+            bytes,
+        })
+    }
+}
+
+struct Reservation {
+    budget: Arc<Budget>,
+    bytes: usize,
+}
+
+impl Drop for Reservation {
+    fn drop(&mut self) {
+        self.budget.bytes.fetch_sub(self.bytes, Ordering::AcqRel);
+        self.budget.items.fetch_sub(1, Ordering::AcqRel);
+    }
+}
 
 enum Command {
-    Write(Vec<u8>),
+    Write(Vec<u8>, Reservation),
     Shutdown,
 }
 
@@ -22,6 +82,8 @@ enum Command {
 pub(super) struct LogHealth {
     write_failures: AtomicU64,
     maintenance_failures: AtomicU64,
+    rejected_records: AtomicU64,
+    rejected_errors: AtomicU64,
 }
 
 impl LogHealth {
@@ -36,6 +98,14 @@ impl LogHealth {
             eprintln!("file_logging: archive/retention maintenance failed ({kind:?})");
         }
     }
+
+    fn rejected(&self, important: bool) {
+        self.rejected_records.fetch_add(1, Ordering::Relaxed);
+        if important {
+            self.rejected_errors.fetch_add(1, Ordering::Relaxed);
+        }
+        // 接收路径只更新状态，不能回退到 stderr 或递归打印新的拥堵日志
+    }
 }
 
 impl HealthProbe for LogHealth {
@@ -46,9 +116,11 @@ impl HealthProbe for LogHealth {
     fn check(&self) -> BoxFuture<'_, HealthState> {
         Box::pin(async move {
             let failures = self.write_failures.load(Ordering::Relaxed);
-            if failures > 0 {
+            let rejected = self.rejected_records.load(Ordering::Relaxed);
+            let errors = self.rejected_errors.load(Ordering::Relaxed);
+            if failures > 0 || rejected > 0 {
                 return HealthState::Unhealthy(format!(
-                    "log completeness compromised: {failures} write/flush failures since startup"
+                    "log completeness compromised: {failures} write/flush failures, {rejected} rejected records ({errors} warnings/errors) since startup"
                 ));
             }
             let failures = self.maintenance_failures.load(Ordering::Relaxed);
@@ -64,12 +136,15 @@ impl HealthProbe for LogHealth {
 
 #[derive(Clone)]
 pub(super) struct FileLogSink {
-    sender: mpsc::SyncSender<Command>,
+    sender: mpsc::Sender<Command>,
+    regular: Arc<Budget>,
+    errors: Arc<Budget>,
+    important: bool,
     health: Arc<LogHealth>,
 }
 
 pub(super) struct FileLogGuard {
-    sender: mpsc::SyncSender<Command>,
+    sender: mpsc::Sender<Command>,
     worker: Option<JoinHandle<()>>,
     health: Arc<LogHealth>,
 }
@@ -80,20 +155,36 @@ impl FileLogSink {
         name: &'static str,
         health: Arc<LogHealth>,
     ) -> io::Result<(Self, FileLogGuard)> {
-        let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY);
+        let (sender, receiver) = mpsc::channel();
         let worker_health = Arc::clone(&health);
         let worker = thread::Builder::new()
             .name(name.to_owned())
             .spawn(move || {
-                while let Ok(command) = receiver.recv() {
-                    match command {
-                        Command::Write(bytes) => {
-                            if let Err(error) = writer.write_all(&bytes) {
-                                worker_health.write_failed(error.kind());
+                let mut reservations = Vec::with_capacity(WRITE_BATCH);
+                'consume: while let Ok(mut command) = receiver.recv() {
+                    for index in 0..WRITE_BATCH {
+                        match command {
+                            Command::Write(bytes, reservation) => {
+                                if let Err(error) = writer.write_all(&bytes) {
+                                    worker_health.write_failed(error.kind());
+                                }
+                                reservations.push(reservation);
                             }
+                            Command::Shutdown => break 'consume,
+                        }
+                        if index + 1 == WRITE_BATCH {
+                            break;
+                        }
+                        match receiver.try_recv() {
+                            Ok(next) => command = next,
+                            Err(_) => break,
                         }
-                        Command::Shutdown => break,
                     }
+                    if let Err(error) = writer.flush() {
+                        worker_health.write_failed(error.kind());
+                    }
+                    // 缓冲刷入文件后才归还额度，慢输出不能让在途记录逃逸预算
+                    reservations.clear();
                 }
                 if let Err(error) = writer.sync() {
                     worker_health.write_failed(error.kind());
@@ -102,6 +193,9 @@ impl FileLogSink {
         Ok((
             Self {
                 sender: sender.clone(),
+                regular: Budget::new(QUEUE_CAPACITY, QUEUE_BYTES),
+                errors: Budget::new(ERROR_CAPACITY, ERROR_BYTES),
+                important: false,
                 health: Arc::clone(&health),
             },
             FileLogGuard {
@@ -119,29 +213,44 @@ impl<'a> MakeWriter<'a> for FileLogSink {
     fn make_writer(&'a self) -> Self::Writer {
         self.clone()
     }
+
+    fn make_writer_for(&'a self, metadata: &tracing::Metadata<'_>) -> Self::Writer {
+        let mut writer = self.clone();
+        writer.important = *metadata.level() <= tracing::Level::WARN;
+        writer
+    }
 }
 
 impl Write for FileLogSink {
     fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
-        self.sender
-            .send(Command::Write(buffer.to_vec()))
-            .map_err(|_| {
-                self.health.write_failed(io::ErrorKind::BrokenPipe);
-                io::Error::new(io::ErrorKind::BrokenPipe, "file log worker unavailable")
-            })?;
+        let bytes = buffer.len().saturating_add(std::mem::size_of::<Command>());
+        let reservation = self
+            .regular
+            .reserve(bytes)
+            .or_else(|| self.important.then(|| self.errors.reserve(bytes)).flatten());
+        if let Some(reservation) = reservation {
+            if self
+                .sender
+                .send(Command::Write(buffer.to_vec(), reservation))
+                .is_err()
+            {
+                self.health.rejected(self.important);
+            }
+        } else {
+            self.health.rejected(self.important);
+        }
+        // tracing 的 writer 失败回退可能同步写 stderr；缺口由独立健康状态报告
         Ok(buffer.len())
     }
 
     fn flush(&mut self) -> io::Result<()> {
-        // 文件不额外缓冲，由守卫负责队列排空与同步的生命周期
         Ok(())
     }
 }
 
 impl Drop for FileLogGuard {
     fn drop(&mut self) {
-        // 关闭指令排在所有已接收记录之后
-        // 等待线程退出，确保正常关闭时不会因超时而丢弃队列中的日志
+        // 仅进程关闭时等待已接收日志落盘，请求侧不承担排空责任
         let _ = self.sender.send(Command::Shutdown);
         if self
             .worker

backend/crates/gateway-host/src/logging/writer.rs

@@ -2,7 +2,7 @@
 
 use std::collections::BTreeMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::{self, Write};
+use std::io::{self, BufWriter, Write};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -20,7 +20,7 @@ pub(super) struct RotatingLogWriter {
     date: NaiveDate,
     segment: usize,
     bytes_written: u64,
-    file: File,
+    file: BufWriter<File>,
     health: Arc<LogHealth>,
 }
 
@@ -58,7 +58,7 @@ impl RotatingLogWriter {
             date,
             segment,
             bytes_written,
-            file,
+            file: BufWriter::with_capacity(64 * 1024, file),
             health,
         };
         // 重启后恢复未完成归档，正在写入的文件不参与压缩
@@ -84,7 +84,7 @@ impl RotatingLogWriter {
         let previous = self
             .directory
             .join(log_file_name(self.prefix, self.date, self.segment));
-        self.file.sync_all()?;
+        self.sync()?;
         let segment = if day_changed {
             managed_log_files(&self.directory, self.prefix)?
                 .into_iter()
@@ -100,10 +100,10 @@ impl RotatingLogWriter {
             .join(log_file_name(self.prefix, date, segment));
         let file = open_log_segment(&path)?;
         // 新文件打开成功后才能提交轮转状态
-        self.file = file;
+        self.file = BufWriter::with_capacity(64 * 1024, file);
         self.date = date;
         self.segment = segment;
-        self.bytes_written = self.file.metadata()?.len();
+        self.bytes_written = self.file.get_ref().metadata()?.len();
         if let Err(error) = compress_log_file(&previous) {
             self.health.maintenance_failed(error.kind());
         }
@@ -121,7 +121,8 @@ impl RotatingLogWriter {
     }
 
     pub(super) fn sync(&mut self) -> io::Result<()> {
-        self.file.sync_all()
+        self.file.flush()?;
+        self.file.get_ref().sync_all()
     }
 }
 
@@ -129,7 +130,7 @@ impl Write for RotatingLogWriter {
     fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
         self.rotate_if_required(buffer.len())?;
         if let Err(error) = self.file.write_all(buffer) {
-            self.bytes_written = self.file.metadata()?.len();
+            self.bytes_written = self.file.get_ref().metadata()?.len();
             return Err(error);
         }
         self.bytes_written = self.bytes_written.saturating_add(buffer.len() as u64);

backend/crates/gateway-host/src/workers.rs

@@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, SystemTime};
 
 use futures::FutureExt as _;
+use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure};
 use gateway_core::health::{
     WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState,
 };
@@ -48,6 +49,7 @@ impl WorkerSupervisor {
         &self,
         plan: Vec<WorkerContribution>,
         leader_lease: Arc<dyn WorkerLeaderLeasePort>,
+        diagnostics: Arc<dyn OperationalDiagnostics>,
     ) -> Result<(), WorkerStartError> {
         if self
             .started
@@ -56,7 +58,7 @@ impl WorkerSupervisor {
         {
             return Err(WorkerStartError::AlreadyStarted);
         }
-        let result = self.start_inner(plan, leader_lease);
+        let result = self.start_inner(plan, leader_lease, diagnostics);
         if result.is_err() {
             self.started.store(false, Ordering::Release);
         }
@@ -67,6 +69,7 @@ impl WorkerSupervisor {
         &self,
         plan: Vec<WorkerContribution>,
         leader_lease: Arc<dyn WorkerLeaderLeasePort>,
+        diagnostics: Arc<dyn OperationalDiagnostics>,
     ) -> Result<(), WorkerStartError> {
         tokio::runtime::Handle::try_current().map_err(|_| WorkerStartError::RuntimeUnavailable)?;
         let WorkerPlan {
@@ -91,14 +94,14 @@ impl WorkerSupervisor {
                         lease,
                         Arc::clone(&leader_lease),
                         self.cancellation.clone(),
-                        Arc::clone(&self.health),
+                        (Arc::clone(&self.health), Arc::clone(&diagnostics)),
                     )),
                     WorkerRunnable::Daemon { restart, task } => tokio::spawn(supervise_daemon(
                         id.clone(),
                         task,
                         restart,
                         self.cancellation.clone(),
-                        Arc::clone(&self.health),
+                        (Arc::clone(&self.health), Arc::clone(&diagnostics)),
                     )),
                 };
                 (id, handle)
@@ -360,8 +363,9 @@ async fn supervise_scheduled(
     lease: Option<WorkerLeaseRequest>,
     leader_lease: Arc<dyn WorkerLeaderLeasePort>,
     cancellation: CancellationToken,
-    health: Arc<WorkerHealthRegistry>,
+    observation: (Arc<WorkerHealthRegistry>, Arc<dyn OperationalDiagnostics>),
 ) {
+    let (health, diagnostics) = observation;
     let mut backoff = schedule.initial_backoff();
     loop {
         if cancellation.is_cancelled() {
@@ -397,7 +401,8 @@ async fn supervise_scheduled(
                     .min(schedule.maximum_backoff())
             }
             ScheduledOutcome::Failed(error) => {
-                let failures = health.failed(&id, error.clone());
+                record_worker_failure(diagnostics.as_ref(), &id, &error).await;
+                let failures = health.failed(&id, error.as_safe_str().to_owned());
                 let delay = take_backoff(&mut backoff, schedule.maximum_backoff());
                 tracing::warn!(
                     worker = %id,
@@ -438,8 +443,10 @@ async fn acquire_and_run(
             run_leased_cycle(id, task, guard, schedule, cancellation, health).await
         }
         Ok(Ok(WorkerLeaseAcquisition::Busy { retry_after })) => ScheduledOutcome::Busy(retry_after),
-        Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()),
-        Err(_) => ScheduledOutcome::Failed("leader lease port panicked".to_owned()),
+        Ok(Err(error)) => {
+            ScheduledOutcome::Failed(WorkerTaskError::safe(error.as_safe_str()).with_source(error))
+        }
+        Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease port panicked")),
     }
 }
 
@@ -478,7 +485,9 @@ async fn run_leased_cycle(
 ) -> ScheduledOutcome {
     let fencing_token = match std::panic::catch_unwind(AssertUnwindSafe(|| guard.fencing_token())) {
         Ok(token) => token,
-        Err(_) => return ScheduledOutcome::Failed("leader lease guard panicked".to_owned()),
+        Err(_) => {
+            return ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease guard panicked"));
+        }
     };
     health.running(id, Some(fencing_token));
     let cycle_cancel = CancellationToken::new();
@@ -518,38 +527,39 @@ async fn run_leased_cycle(
     };
     match release_guard(guard, schedule.leader_lease_renewal_interval()).await {
         Ok(()) => outcome,
-        Err(error) if matches!(outcome, ScheduledOutcome::ShuttingDown) => {
-            tracing::warn!(worker = %id, error, "leader lease 释放失败");
-            outcome
-        }
-        Err(error) => ScheduledOutcome::Failed(error),
+        Err(cleanup) => match outcome {
+            ScheduledOutcome::Failed(error) => {
+                ScheduledOutcome::Failed(error.with_cleanup(cleanup))
+            }
+            _ => ScheduledOutcome::Failed(cleanup),
+        },
     }
 }
 
 async fn renew_guard(
     guard: &mut dyn WorkerLeaderLeaseGuard,
     timeout: Duration,
-) -> Result<(), String> {
+) -> Result<(), WorkerTaskError> {
     let future = AssertUnwindSafe(guard.renew()).catch_unwind();
     match tokio::time::timeout(timeout, future).await {
         Ok(Ok(Ok(()))) => Ok(()),
-        Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()),
-        Ok(Err(_)) => Err("leader lease renewal panicked".to_owned()),
-        Err(_) => Err("leader lease renewal timed out".to_owned()),
+        Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)),
+        Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease renewal panicked")),
+        Err(_) => Err(WorkerTaskError::safe("leader lease renewal timed out")),
     }
 }
 
 async fn release_guard(
     guard: Box<dyn WorkerLeaderLeaseGuard>,
     timeout: Duration,
-) -> Result<(), String> {
+) -> Result<(), WorkerTaskError> {
     let future = std::panic::catch_unwind(AssertUnwindSafe(|| guard.release()))
-        .map_err(|_| "leader lease release panicked".to_owned())?;
+        .map_err(|_| WorkerTaskError::safe("leader lease release panicked"))?;
     match tokio::time::timeout(timeout, AssertUnwindSafe(future).catch_unwind()).await {
         Ok(Ok(Ok(()))) => Ok(()),
-        Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()),
-        Ok(Err(_)) => Err("leader lease release panicked".to_owned()),
-        Err(_) => Err("leader lease release timed out".to_owned()),
+        Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)),
+        Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease release panicked")),
+        Err(_) => Err(WorkerTaskError::safe("leader lease release timed out")),
     }
 }
 
@@ -558,8 +568,9 @@ async fn supervise_daemon(
     task: Box<dyn DaemonTask>,
     restart: DaemonRestartPolicy,
     cancellation: CancellationToken,
-    health: Arc<WorkerHealthRegistry>,
+    observation: (Arc<WorkerHealthRegistry>, Arc<dyn OperationalDiagnostics>),
 ) {
+    let (health, diagnostics) = observation;
     let mut backoff = restart.initial_backoff();
     loop {
         if cancellation.is_cancelled() {
@@ -579,12 +590,13 @@ async fn supervise_daemon(
         };
         let Some(result) = result else { break };
         let error = match result {
-            Ok(Ok(())) => "daemon exited unexpectedly".to_owned(),
-            Ok(Err(error)) => error.as_safe_str().to_owned(),
-            Err(_) => "daemon panicked".to_owned(),
+            Ok(Ok(())) => WorkerTaskError::safe("daemon exited unexpectedly"),
+            Ok(Err(error)) => error,
+            Err(_) => WorkerTaskError::safe("daemon panicked"),
         };
         let delay = take_backoff(&mut backoff, restart.maximum_backoff());
-        let failures = health.failed(&id, error.clone());
+        record_worker_failure(diagnostics.as_ref(), &id, &error).await;
+        let failures = health.failed(&id, error.as_safe_str().to_owned());
         tracing::warn!(
             worker = %id,
             failures,
@@ -603,7 +615,7 @@ async fn supervise_daemon(
 enum ScheduledOutcome {
     Succeeded,
     Busy(Option<Duration>),
-    Failed(String),
+    Failed(WorkerTaskError),
     ShuttingDown,
 }
 
@@ -612,8 +624,8 @@ fn task_result(
 ) -> ScheduledOutcome {
     match result {
         Ok(Ok(())) => ScheduledOutcome::Succeeded,
-        Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()),
-        Err(_) => ScheduledOutcome::Failed("worker panicked".to_owned()),
+        Ok(Err(error)) => ScheduledOutcome::Failed(error),
+        Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("worker panicked")),
     }
 }
 
@@ -641,3 +653,17 @@ fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner)
 }
+
+async fn record_worker_failure(
+    diagnostics: &dyn OperationalDiagnostics,
+    id: &WorkerId,
+    error: &WorkerTaskError,
+) {
+    let mut failure =
+        OperationalFailure::new("worker", "run", "worker_failed", error.as_safe_str());
+    failure.correlation_id = Some(id.to_string());
+    failure.details = error.error_details();
+    if diagnostics.record_failure(failure).await.is_err() {
+        tracing::warn!(worker = %id, "worker diagnostic could not be recorded");
+    }
+}

backend/crates/gateway-host/tests/logging/sink.rs

@@ -110,3 +110,134 @@ fn write_failure_marks_log_completeness_unhealthy_even_after_writes_recover() {
     assert!(body.contains("before-failure") && body.contains("after-recovery"));
     assert!(!body.contains("cannot-write"));
 }
+
+#[cfg(unix)]
+#[test]
+fn blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors() {
+    const CHILD: &str = "logging::sink::blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors";
+    if env::var_os(CHILD_PROCESS_ENV).is_some() {
+        let directory = PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap());
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+        let mut config = logging_config(directory.clone(), false);
+        config.logging.oauth_recovery = false;
+        config.logging.file.max_file_size_mb = 8;
+        let bundle = runtime.block_on(gateway_host::initialize(config)).unwrap();
+        let health = bundle.logging_health_probe();
+        let date = gateway_core::time::DeploymentTimeZone::default()
+            .local(chrono::Utc::now())
+            .date_naive();
+        let fifo = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.1.log"));
+        assert!(
+            Command::new("mkfifo")
+                .arg(&fifo)
+                .status()
+                .unwrap()
+                .success()
+        );
+        let (release, released) = std::sync::mpsc::channel();
+        let reader = std::thread::spawn(move || {
+            // 超时只用于让旧的同步实现能退出测试，接口必须在解除阻塞之前完成
+            let _ = released.recv_timeout(Duration::from_secs(3));
+            let mut body = String::new();
+            fs::File::open(fifo)
+                .unwrap()
+                .read_to_string(&mut body)
+                .unwrap();
+            body
+        });
+        let payload = "x".repeat(8 * 1024 * 1024);
+        tracing::info!(target: APPLICATION_LOG_TARGET, payload, "rotate before requests");
+        tracing::info!(target: APPLICATION_LOG_TARGET, "block on opening next segment");
+        runtime.block_on(async {
+            let initial = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.log"));
+            tokio::time::timeout(Duration::from_secs(2), async {
+                while fs::metadata(&initial).unwrap().len() < 8 * 1024 * 1024 {
+                    tokio::time::sleep(Duration::from_millis(1)).await;
+                }
+            }).await.unwrap();
+            use futures::{SinkExt as _, StreamExt as _};
+            let app = axum::Router::new()
+                .route("/v1/log-test", axum::routing::get(|| async {
+                    tracing::error!(target: APPLICATION_LOG_TARGET, path="v1", "critical_after_congestion");
+                    "ok"
+                }))
+                .route("/admin/log-test", axum::routing::get(|| async {
+                    tracing::error!(target: APPLICATION_LOG_TARGET, path="admin", "critical_after_congestion");
+                    "ok"
+                }))
+                .route("/v1/stream-test", axum::routing::get(|| async {
+                    tracing::error!(target: APPLICATION_LOG_TARGET, path="stream", "critical_after_congestion");
+                    axum::response::Sse::new(futures::stream::iter([
+                        Ok::<_, std::convert::Infallible>(axum::response::sse::Event::default().data("first")),
+                        Ok(axum::response::sse::Event::default().data("last")),
+                    ]))
+                }));
+            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+            let address = listener.local_addr().unwrap();
+            let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); });
+            let started = std::time::Instant::now();
+            for sequence in 0..5_000 {
+                tracing::info!(target: APPLICATION_LOG_TARGET, sequence, "congested regular log");
+            }
+            let client = reqwest::Client::builder().no_proxy().build().unwrap();
+            for path in ["/v1/log-test", "/admin/log-test", "/v1/stream-test"] {
+                let response = client.get(format!("http://{address}{path}")).send().await.unwrap();
+                assert!(response.status().is_success());
+                let body = response.text().await.unwrap();
+                if path.ends_with("stream-test") {
+                    assert!(body.contains("first") && body.contains("last"));
+                } else {
+                    assert_eq!(body, "ok");
+                }
+            }
+            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+            let address = listener.local_addr().unwrap();
+            let websocket_server = tokio::spawn(async move {
+                let (stream, _) = listener.accept().await.unwrap();
+                let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap();
+                tracing::error!(target: APPLICATION_LOG_TARGET, path="websocket", "critical_after_congestion");
+                socket.send(tokio_tungstenite::tungstenite::Message::Text("frame".into())).await.unwrap();
+            });
+            let stream = tokio::net::TcpStream::connect(address).await.unwrap();
+            let (mut websocket, _) = tokio_tungstenite::client_async(format!("ws://{address}/"), stream).await.unwrap();
+            assert_eq!(websocket.next().await.unwrap().unwrap().into_text().unwrap(), "frame");
+            websocket_server.await.unwrap();
+            let elapsed = started.elapsed();
+            assert!(elapsed < Duration::from_secs(1), "interfaces waited for logging: {elapsed:?}");
+            eprintln!("paused log output: 5000 submissions and HTTP/admin/SSE/WS completed in {elapsed:?}");
+            let HealthState::Unhealthy(message) = health.check().await else { panic!("queue overflow must be visible") };
+            assert!(message.contains("0 warnings/errors"), "{message}");
+            server.abort();
+        });
+        release.send(()).unwrap();
+        drop(bundle);
+        let body = reader.join().unwrap();
+        assert_eq!(
+            body.lines()
+                .filter(|line| line.contains("critical_after_congestion"))
+                .count(),
+            4
+        );
+        for line in body.lines() {
+            serde_json::from_str::<serde_json::Value>(line).unwrap();
+        }
+        return;
+    }
+    let directory = tempfile::tempdir().unwrap();
+    let output = Command::new(env::current_exe().unwrap())
+        .args(["--exact", CHILD, "--nocapture"])
+        .env(CHILD_PROCESS_ENV, "1")
+        .env(LOG_DIRECTORY_ENV, directory.path())
+        .env("RUST_LOG", "off,logging_test_application=info")
+        .output()
+        .unwrap();
+    assert!(
+        output.status.success(),
+        "{}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    eprintln!("{}", String::from_utf8_lossy(&output.stderr));
+}

backend/crates/gateway-host/tests/plugin_distribution/github.rs

@@ -198,13 +198,12 @@ async fn rate_limit_applies_to_other_repositories_using_the_same_identity_and_eg
             .kind(),
         AdminErrorKind::RateLimited
     );
-    assert_eq!(
-        distribution
-            .query_release(query(None), vec![], None)
-            .await
-            .unwrap_err(),
-        failure
-    );
+    let cached_failure = distribution
+        .query_release(query(None), vec![], None)
+        .await
+        .unwrap_err();
+    assert_eq!(cached_failure.kind(), failure.kind());
+    assert_eq!(cached_failure.message(), failure.message());
     assert_eq!(server.received_requests().await.unwrap().len(), 1);
 }
 

backend/crates/gateway-host/tests/workers.rs

@@ -54,7 +54,8 @@ impl ScheduledTask for CountingTask {
         Box::pin(async move {
             let run = self.runs.fetch_add(1, Ordering::SeqCst) + 1;
             if run <= self.fail_until {
-                Err(WorkerTaskError::safe("expected test failure"))
+                Err(WorkerTaskError::safe("expected test failure")
+                    .with_source(std::io::Error::other("PRIVATE_WORKER_NATIVE_CAUSE")))
             } else {
                 Ok(())
             }
@@ -343,9 +344,11 @@ async fn registry_composes_multiple_owners_but_rejects_duplicate_identity() {
     let supervisor = supervisor();
 
     assert!(matches!(
-        supervisor.start(plan, Arc::new(FakeLeasePort::default())),
-        Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai"
-    ));
+            supervisor.start(plan, Arc::new(FakeLeasePort::default()),
+    Arc::new(RecordingDiagnostics::default()),
+    ),
+            Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai"
+        ));
 }
 
 #[tokio::test]
@@ -360,7 +363,11 @@ async fn only_database_fact_reasons_can_disable_nonexistent_workers() {
     )));
 
     assert!(matches!(
-        supervisor().start(plan, Arc::new(FakeLeasePort::default())),
+        supervisor().start(
+            plan,
+            Arc::new(FakeLeasePort::default()),
+            Arc::new(RecordingDiagnostics::default()),
+        ),
         Err(WorkerStartError::KindDisabled(WorkerKind::OpsFlush))
     ));
 }
@@ -373,9 +380,11 @@ async fn registry_refuses_start_when_any_real_owner_kind_is_missing() {
         .collect();
 
     assert!(matches!(
-        supervisor().start(plan, Arc::new(FakeLeasePort::default())),
-        Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention]
-    ));
+            supervisor().start(plan, Arc::new(FakeLeasePort::default()),
+    Arc::new(RecordingDiagnostics::default()),
+    ),
+            Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention]
+        ));
 }
 
 #[tokio::test]
@@ -383,7 +392,11 @@ async fn disabled_database_facts_never_execute_or_acquire_a_lease() {
     let leases = Arc::new(FakeLeasePort::default());
     let supervisor = supervisor();
     supervisor
-        .start(complete_plan(Vec::new()), leases.clone())
+        .start(
+            complete_plan(Vec::new()),
+            leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     wait_until(|| leases.requests().len() >= ACTIVE_KINDS.len()).await;
     let snapshot = supervisor.health_source().snapshot();
@@ -422,7 +435,11 @@ async fn one_kind_runs_all_registered_owner_tasks() {
     ]);
     let supervisor = supervisor();
     supervisor
-        .start(plan, Arc::new(FakeLeasePort::default()))
+        .start(
+            plan,
+            Arc::new(FakeLeasePort::default()),
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     wait_until(|| second.runs.load(Ordering::SeqCst) >= 1).await;
 
@@ -456,6 +473,7 @@ async fn leader_guard_lives_through_cycle_and_fence_reaches_owner() {
         .start(
             target_plan(task.clone(), long_interval_schedule()),
             leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
         )
         .expect("complete plan");
     task.entered.notified().await;
@@ -478,6 +496,7 @@ async fn long_cycle_renews_leader_lease_before_ttl() {
         .start(
             target_plan(task.clone(), renewal_schedule()),
             leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
         )
         .expect("complete plan");
     task.entered.notified().await;
@@ -500,7 +519,11 @@ async fn renewal_failure_cancels_old_cycle_before_another_leader_cycle_starts()
     leases.renew_script(target_worker(), [RenewStep::Error]);
     let supervisor = supervisor();
     supervisor
-        .start(target_plan(task.clone(), renewal_schedule()), leases)
+        .start(
+            target_plan(task.clone(), renewal_schedule()),
+            leases,
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     yield_until(|| task.starts.load(Ordering::SeqCst) == 1).await;
     tokio::time::advance(Duration::from_millis(10)).await;
@@ -520,6 +543,7 @@ async fn supervisor_uses_exponential_backoff_and_recovers_health() {
         .start(
             target_plan(task.clone(), backoff_schedule()),
             Arc::new(FakeLeasePort::default()),
+            Arc::new(RecordingDiagnostics::default()),
         )
         .expect("complete plan");
     yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await;
@@ -548,7 +572,11 @@ async fn task_and_lease_port_panics_are_isolated_and_retried() {
     );
     let supervisor = supervisor();
     supervisor
-        .start(target_plan(task.clone(), backoff_schedule()), leases)
+        .start(
+            target_plan(task.clone(), backoff_schedule()),
+            leases,
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     tokio::task::yield_now().await;
     tokio::time::advance(Duration::from_millis(10)).await;
@@ -578,7 +606,11 @@ async fn lease_busy_is_healthy_standby_and_retries_without_running_owner() {
     );
     let supervisor = supervisor();
     supervisor
-        .start(target_plan(task.clone(), backoff_schedule()), leases)
+        .start(
+            target_plan(task.clone(), backoff_schedule()),
+            leases,
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     yield_until(|| task_health(&supervisor, &target_worker()).state == WorkerRuntimeState::Standby)
         .await;
@@ -598,7 +630,11 @@ async fn cooperative_shutdown_propagates_cancel_before_dropping_all_guards() {
     let leases = Arc::new(FakeLeasePort::default());
     let supervisor = supervisor();
     supervisor
-        .start(all_active_plan(task.clone()), leases.clone())
+        .start(
+            all_active_plan(task.clone()),
+            leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     wait_until(|| task.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len()).await;
     supervisor.shutdown(Duration::from_secs(1)).await;
@@ -619,6 +655,7 @@ async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() {
         .start(
             target_plan(task.clone(), long_interval_schedule()),
             leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
         )
         .expect("complete plan");
     task.notification.notified().await;
@@ -666,7 +703,11 @@ async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() {
     let leases = Arc::new(FakeLeasePort::default());
     let supervisor = supervisor();
     supervisor
-        .start(plan, leases.clone())
+        .start(
+            plan,
+            leases.clone(),
+            Arc::new(RecordingDiagnostics::default()),
+        )
         .expect("complete plan");
     hung.notification.notified().await;
     wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await;
@@ -691,6 +732,7 @@ async fn running_worker_becomes_unhealthy_when_last_success_is_stale() {
         .start(
             target_plan(task.clone(), freshness_schedule()),
             Arc::new(FakeLeasePort::default()),
+            Arc::new(RecordingDiagnostics::default()),
         )
         .expect("complete plan");
     task.second_cycle.notified().await;
@@ -898,3 +940,50 @@ fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner)
 }
+
+#[derive(Default)]
+struct RecordingDiagnostics(std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>);
+#[async_trait::async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
+    async fn record_failure(
+        &self,
+        failure: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), gateway_core::error::StoreError> {
+        self.0.lock().unwrap().push(failure);
+        Ok(())
+    }
+}
+
+#[tokio::test]
+async fn worker_boundary_records_one_native_failure_and_recovers_health() {
+    let supervisor = supervisor();
+    let task = CountingTask::new(1);
+    let diagnostics = Arc::new(RecordingDiagnostics::default());
+    supervisor
+        .start(
+            target_plan(task.clone(), test_schedule()),
+            Arc::new(FakeLeasePort::default()),
+            diagnostics.clone(),
+        )
+        .unwrap();
+    wait_until(|| task.runs.load(Ordering::SeqCst) >= 2).await;
+    supervisor.shutdown(Duration::from_secs(1)).await;
+    let failures = diagnostics.0.lock().unwrap();
+    assert_eq!(failures.len(), 1);
+    assert_eq!(
+        failures[0].correlation_id.as_deref(),
+        Some(target_worker().to_string().as_str())
+    );
+    assert!(
+        failures[0]
+            .details
+            .as_ref()
+            .unwrap()
+            .as_str()
+            .contains("PRIVATE_WORKER_NATIVE_CAUSE")
+    );
+    assert!(
+        !format!("{:?}", supervisor.health_source().snapshot())
+            .contains("PRIVATE_WORKER_NATIVE_CAUSE")
+    );
+}

backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs

@@ -135,10 +135,12 @@ fn rpc_failure(error: RpcError) -> &'static str {
     match error {
         RpcError::Timeout => "插件响应超时",
         RpcError::Cancelled => "调用已取消",
-        RpcError::Closed | RpcError::Start(_) => "插件进程不可用",
-        RpcError::Handshake | RpcError::Protocol | RpcError::InvalidResponse(_) => "插件协议错误",
+        RpcError::Closed(_) | RpcError::Start(_) => "插件进程不可用",
+        RpcError::Handshake(_) | RpcError::Protocol(_) | RpcError::InvalidResponse(..) => {
+            "插件协议错误"
+        }
         RpcError::Capacity => "插件调用容量不足",
-        RpcError::Context => "调用上下文无效",
+        RpcError::Context(_) => "调用上下文无效",
         RpcError::Remote(_) => "插件拒绝命令",
     }
 }

backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs

@@ -202,7 +202,7 @@ async fn invoke_middleware(
     );
     let (protocol, headers, payload) = match invocation.request_parts() {
         Ok(projection) => projection,
-        Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context).await,
+        Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context(None)).await,
     };
     let mut call_context = invocation_ports.session.context(
         match entry.mount {

backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs

@@ -46,7 +46,7 @@ async fn write_loop<W: AsyncWrite + Unpin>(
             frame = data.recv() => frame,
         };
         let Some(frame) = frame else {
-            shared.fail(RpcError::Closed);
+            shared.fail(RpcError::Closed(None));
             return;
         };
         // 撤销尚未发送的调用时丢弃排队帧；已开始写入的 Call 必须先于 Cancel
@@ -64,8 +64,10 @@ async fn write_loop<W: AsyncWrite + Unpin>(
             _ = stopped.changed() => return,
             result = write_frame(&mut writer, &frame) => result,
         };
-        if written.is_err() {
-            shared.fail(RpcError::Closed);
+        if let Err(source) = written {
+            shared.fail(RpcError::Closed(Some(
+                gateway_core::error::ErrorSource::new(source),
+            )));
             return;
         }
     }
@@ -87,9 +89,14 @@ async fn read_loop<R: AsyncRead + Unpin>(
             _ = stopped.changed() => return,
             frame = read_frame(&mut reader) => frame,
         };
-        let Ok(frame) = frame else {
-            shared.fail(RpcError::Closed);
-            return;
+        let frame = match frame {
+            Ok(frame) => frame,
+            Err(source) => {
+                shared.fail(RpcError::Closed(Some(
+                    gateway_core::error::ErrorSource::new(source),
+                )));
+                return;
+            }
         };
         let result = match frame.message {
             Message::Cancelled { id } if frame.payload.is_empty() => {
@@ -114,7 +121,7 @@ async fn read_loop<R: AsyncRead + Unpin>(
                 params,
             } => {
                 if id == 0 || id % 2 != 0 || id <= last_callback {
-                    Err(RpcError::Protocol)
+                    Err(RpcError::Protocol(None))
                 } else if let Some(context) = shared.context(parent_id) {
                     last_callback = id;
                     if let Some(permit) = shared.try_callback_slot() {
@@ -171,10 +178,10 @@ async fn read_loop<R: AsyncRead + Unpin>(
                     }));
                     Ok(())
                 } else {
-                    Err(RpcError::Protocol)
+                    Err(RpcError::Protocol(None))
                 }
             }
-            _ => Err(RpcError::Protocol),
+            _ => Err(RpcError::Protocol(None)),
         };
         if let Err(error) = result {
             shared.fail(error);

backend/crates/gateway-plugin/runtime/src/rpc/session.rs

@@ -45,18 +45,18 @@ impl Default for RpcLimits {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+#[derive(Debug, Clone, thiserror::Error)]
 pub enum RpcError {
     #[error("plugin process could not be started: {0}")]
     Start(#[source] ProcessStartError),
     #[error("plugin handshake does not match the prepared instance")]
-    Handshake,
+    Handshake(#[source] Option<gateway_core::error::ErrorSource>),
     #[error("plugin protocol is invalid")]
-    Protocol,
+    Protocol(#[source] Option<gateway_core::error::ErrorSource>),
     #[error("plugin returned an invalid response at {0:?}")]
-    InvalidResponse(Stage),
+    InvalidResponse(Stage, #[source] Option<gateway_core::error::ErrorSource>),
     #[error("plugin process or transport has stopped")]
-    Closed,
+    Closed(#[source] Option<gateway_core::error::ErrorSource>),
     #[error("plugin call exceeded its deadline")]
     Timeout,
     #[error("plugin session was cancelled")]
@@ -64,7 +64,7 @@ pub enum RpcError {
     #[error("plugin call capacity is exhausted")]
     Capacity,
     #[error("plugin call context is invalid")]
-    Context,
+    Context(#[source] Option<gateway_core::error::ErrorSource>),
     #[error("plugin returned an error")]
     Remote(PluginFault),
 }
@@ -74,9 +74,9 @@ impl RpcError {
     pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) {
         match self {
             RpcError::Start(_) => ("process_start", "插件进程启动失败"),
-            RpcError::Handshake => ("handshake", "插件握手失败"),
-            RpcError::Protocol => ("protocol", "插件协议错误"),
-            RpcError::InvalidResponse(stage) => (
+            RpcError::Handshake(_) => ("handshake", "插件握手失败"),
+            RpcError::Protocol(_) => ("protocol", "插件协议错误"),
+            RpcError::InvalidResponse(stage, _) => (
                 "invalid_response",
                 match stage {
                     Stage::Registration => "注册阶段：插件返回了无效响应",
@@ -97,11 +97,11 @@ impl RpcError {
                     Stage::Maintenance => "维护阶段：插件返回了无效响应",
                 },
             ),
-            RpcError::Closed => ("closed", "插件进程或传输已停止"),
+            RpcError::Closed(_) => ("closed", "插件进程或传输已停止"),
             RpcError::Timeout => ("timeout", "插件调用超时"),
             RpcError::Cancelled => ("cancelled", "插件会话已取消"),
             RpcError::Capacity => ("capacity", "插件调用容量已耗尽"),
-            RpcError::Context => ("context", "插件调用上下文无效"),
+            RpcError::Context(_) => ("context", "插件调用上下文无效"),
             RpcError::Remote(_) => ("remote", "插件返回错误"),
         }
     }
@@ -186,7 +186,7 @@ pub struct RpcSession {
 impl RpcSession {
     /// 只用于已确认来自插件的合同错误，不接收业务错误或原始响应内容
     pub(crate) fn invalid_response(&self, stage: Stage) -> RpcError {
-        let error = RpcError::InvalidResponse(stage);
+        let error = RpcError::InvalidResponse(stage, None);
         self.shared.fail(error.clone());
         error
     }
@@ -196,7 +196,14 @@ impl RpcSession {
         stage: Stage,
         value: serde_json::Value,
     ) -> Result<T, RpcError> {
-        serde_json::from_value(value).map_err(|_| self.invalid_response(stage))
+        serde_json::from_value(value).map_err(|source| {
+            let error = RpcError::InvalidResponse(
+                stage,
+                Some(gateway_core::error::ErrorSource::new(source)),
+            );
+            self.shared.fail(error.clone());
+            error
+        })
     }
 
     pub(crate) fn capability_version(
@@ -292,16 +299,16 @@ impl Shared {
 
     pub fn finish(&self, id: u64, result: Result<RpcReply, RpcError>) -> Result<(), RpcError> {
         let mut state = self.state();
-        let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?;
+        let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
         if call.cancellation.is_some() {
             return Ok(());
         }
         if call.stream.is_some() && result.is_ok() {
-            let initial = call.result.take().ok_or(RpcError::Protocol)?;
+            let initial = call.result.take().ok_or(RpcError::Protocol(None))?;
             call.handler.streaming(&call.context);
             let _ = initial.send(result);
         } else {
-            let pending = state.calls.remove(&id).ok_or(RpcError::Protocol)?;
+            let pending = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
             drop(state);
             pending.finish(result);
         }
@@ -310,16 +317,16 @@ impl Shared {
 
     pub fn stream_chunk(&self, id: u64, sequence: u64, payload: Vec<u8>) -> Result<(), RpcError> {
         let mut state = self.state();
-        let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?;
+        let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?;
         if call.cancellation.is_some() {
             return Ok(());
         }
         if call.result.is_some() {
-            return Err(RpcError::Protocol);
+            return Err(RpcError::Protocol(None));
         }
         call.stream
             .as_mut()
-            .ok_or(RpcError::Protocol)?
+            .ok_or(RpcError::Protocol(None))?
             .receive(sequence, payload)
     }
 
@@ -332,11 +339,11 @@ impl Shared {
         {
             return Ok(());
         }
-        let call = state.calls.remove(&id).ok_or(RpcError::Protocol)?;
+        let call = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?;
         drop(state);
         if call.stream.is_none() || call.result.is_some() {
-            call.finish(Err(RpcError::Protocol));
-            return Err(RpcError::Protocol);
+            call.finish(Err(RpcError::Protocol(None)));
+            return Err(RpcError::Protocol(None));
         }
         let result = error.map_or_else(
             || {
@@ -359,7 +366,7 @@ impl Shared {
         };
         call.stream
             .as_mut()
-            .ok_or(RpcError::Protocol)?
+            .ok_or(RpcError::Protocol(None))?
             .release(bytes)?;
         drop(state);
         self.send_control(Frame::control(Message::Credit {
@@ -441,7 +448,7 @@ impl Shared {
             .get(&id)
             .is_some_and(|call| call.cancellation.is_some())
         {
-            return Err(RpcError::Protocol);
+            return Err(RpcError::Protocol(None));
         }
         state.calls.remove(&id);
         Ok(())
@@ -540,10 +547,12 @@ impl RpcSession {
             || limits.handshake_timeout.is_zero()
             || limits.maximum_call_timeout.is_zero()
         {
-            return Err(RpcError::Context);
+            return Err(RpcError::Context(None));
         }
         let manifest = package.package().manifest();
-        let plugin_id = manifest.plugin_id().map_err(|_| RpcError::Handshake)?;
+        let plugin_id = manifest.plugin_id().map_err(|source| {
+            RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source)))
+        })?;
         if handshake.protocol_version != PROTOCOL_VERSION
             || manifest
                 .package
@@ -557,7 +566,7 @@ impl RpcSession {
             || handshake.incarnation.is_empty()
             || handshake.generation == 0
         {
-            return Err(RpcError::Handshake);
+            return Err(RpcError::Handshake(None));
         }
         let deadline = tokio::time::Instant::now() + limits.handshake_timeout;
         let process = loop {
@@ -597,10 +606,12 @@ impl RpcSession {
                 }),
             )
             .await
-            .map_err(|_| RpcError::Handshake)?;
-            let reply = read_frame(&mut output)
-                .await
-                .map_err(|_| RpcError::Handshake)?;
+            .map_err(|source| {
+                RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source)))
+            })?;
+            let reply = read_frame(&mut output).await.map_err(|source| {
+                RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source)))
+            })?;
             match reply.message {
                 Message::Ready {
                     protocol_version,
@@ -611,7 +622,7 @@ impl RpcSession {
                 {
                     Ok(())
                 }
-                _ => Err(RpcError::Handshake),
+                _ => Err(RpcError::Handshake(None)),
             }
         })
         .await
@@ -661,7 +672,7 @@ impl RpcSession {
             if !monitor.expected_stop.load(Ordering::Acquire) {
                 tracing::warn!(instance_id, exit = ?reason, "插件进程意外停止");
             }
-            monitor.fail(RpcError::Closed);
+            monitor.fail(RpcError::Closed(None));
             // 等进程退出后再回收其文件，避免 Windows 上删除正在执行的制品
             drop(package);
         });
@@ -738,7 +749,7 @@ impl RpcSession {
             || method.is_empty()
             || method.len() > 128
         {
-            return Err(RpcError::Context);
+            return Err(RpcError::Context(None));
         }
         Ok(tokio::time::Instant::now() + Duration::from_millis(context.timeout_ms))
     }
@@ -807,7 +818,9 @@ impl RpcSession {
             payload,
         };
         // 本地元数据错误不能进入写队列并关闭整个会话
-        validate_frame(&frame).map_err(|_| RpcError::Context)?;
+        validate_frame(&frame).map_err(|source| {
+            RpcError::Context(Some(gateway_core::error::ErrorSource::new(source)))
+        })?;
         let started = self.begin_call(context, deadline, stream)?;
         let mut guard = CallGuard {
             shared: Arc::clone(&self.shared),
@@ -815,7 +828,10 @@ impl RpcSession {
             armed: true,
         };
         let sent = tokio::time::timeout_at(started.deadline, async {
-            self.data.send(frame).await.map_err(|_| RpcError::Closed)?;
+            self.data
+                .send(frame)
+                .await
+                .map_err(|_| RpcError::Closed(None))?;
             if let Some((bytes, frames)) = credit {
                 // 初始信用跟在 Call 后走同一有序队列，不能被控制队列提前发送
                 self.data
@@ -825,7 +841,7 @@ impl RpcSession {
                         frames,
                     }))
                     .await
-                    .map_err(|_| RpcError::Closed)?;
+                    .map_err(|_| RpcError::Closed(None))?;
             }
             Ok::<(), RpcError>(())
         })
@@ -851,9 +867,9 @@ impl RpcSession {
                 result?
             }
             Ok(Err(_)) => {
-                self.shared.cancel(started.id, RpcError::Closed);
+                self.shared.cancel(started.id, RpcError::Closed(None));
                 guard.armed = false;
-                return Err(RpcError::Closed);
+                return Err(RpcError::Closed(None));
             }
             Err(_) => {
                 self.shared.cancel(started.id, RpcError::Timeout);
@@ -935,7 +951,7 @@ impl RpcSession {
             .await
             .is_err()
         {
-            self.shared.fail(RpcError::Closed);
+            self.shared.fail(RpcError::Closed(None));
             // 强制停止没有第二段宽限；退出和回调析构是返回前必须确认的事实
             self.settle_shutdown().await;
         }
@@ -944,7 +960,7 @@ impl RpcSession {
     async fn settle_shutdown(&self) {
         self.shared.process.exited().await;
         // 不等待监控任务调度；先撤销父调用并中止其回调，再确认回调 future 已析构
-        self.shared.fail(RpcError::Closed);
+        self.shared.fail(RpcError::Closed(None));
         self.shared
             .wait_for_callbacks(self.limits.maximum_callbacks)
             .await;
@@ -954,7 +970,7 @@ impl RpcSession {
 impl Drop for RpcSession {
     fn drop(&mut self) {
         self.shared.expect_stop();
-        self.shared.fail(RpcError::Closed);
+        self.shared.fail(RpcError::Closed(None));
     }
 }
 

backend/crates/gateway-plugin/runtime/src/stream/bridge.rs

@@ -43,7 +43,7 @@ impl StreamIngress {
         self.window.receive(sequence, payload.len())?;
         self.chunks
             .try_send(payload)
-            .map_err(|_| RpcError::Protocol)
+            .map_err(|_| RpcError::Protocol(None))
     }
 
     pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> {
@@ -86,7 +86,7 @@ impl RpcStream {
                 let Some(terminal) = self.terminal.take() else {
                     return Ok(None);
                 };
-                terminal.await.map_err(|_| RpcError::Closed)??;
+                terminal.await.map_err(|_| RpcError::Closed(None))??;
                 Ok(None)
             }
             Err(_) => {

backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs

@@ -19,22 +19,25 @@ impl ReceiveWindow {
     }
 
     pub fn receive(&mut self, sequence: u64, bytes: usize) -> Result<(), RpcError> {
-        let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol)?;
+        let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol(None))?;
         if sequence != self.next_sequence || bytes == 0 || bytes > self.bytes || self.frames == 0 {
-            return Err(RpcError::Protocol);
+            return Err(RpcError::Protocol(None));
         }
         self.next_sequence = self
             .next_sequence
             .checked_add(1)
-            .ok_or(RpcError::Protocol)?;
+            .ok_or(RpcError::Protocol(None))?;
         self.bytes -= bytes;
         self.frames -= 1;
         Ok(())
     }
 
     pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> {
-        self.bytes = self.bytes.checked_add(bytes).ok_or(RpcError::Protocol)?;
-        self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol)?;
+        self.bytes = self
+            .bytes
+            .checked_add(bytes)
+            .ok_or(RpcError::Protocol(None))?;
+        self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol(None))?;
         Ok(())
     }
 }

backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs

@@ -5,6 +5,7 @@ use gateway_admin::{
     model::{
         account_groups::{AccountGroupColor, CreateAccountGroup},
         plugin_resources::{GroupMembersChange, ManagedKeyConfig, PluginResourceOwner},
+        plugins::instances::PluginInstanceRuntimeStatus,
     },
     ports::plugin_resources::PluginResourceAccess,
 };
@@ -112,6 +113,7 @@ async fn revision(environment: &Environment) -> gateway_admin::model::Revision {
 
 async fn start(
     environment: &Environment,
+    phase: &str,
 ) -> (
     Arc<PluginRuntime>,
     gateway_core::CoreBundle,
@@ -120,6 +122,32 @@ async fn start(
     tokio::task::JoinHandle<()>,
 ) {
     let (runtime, core) = environment.runtime().await;
+    // 集合就绪允许保留隔离的启动失败实例；维护测试必须确认插件本身正在运行
+    let snapshot = core.snapshots().snapshot_for_diagnostics().unwrap();
+    let extensions = snapshot.extensions().unwrap();
+    let instances = environment
+        .store
+        .admin_ports()
+        .plugins()
+        .load_instances()
+        .await
+        .unwrap();
+    let diagnostics = gateway_admin::ports::plugins::PluginRuntimeDiagnostics::runtime_diagnostics(
+        runtime.as_ref(),
+        &instances,
+        Some(snapshot.revision().get()),
+        Some(extensions),
+    )
+    .await
+    .unwrap();
+    assert!(
+        !diagnostics.is_empty()
+            && diagnostics
+                .values()
+                .all(|instance| instance.status == PluginInstanceRuntimeStatus::Running),
+        "plugin was not ready during {phase}: {diagnostics:?}"
+    );
+    drop(snapshot);
     let resources = gateway_admin::initialize_plugin_resources(
         environment.store.admin_ports().plugin_resources(),
         core.snapshot_control(),
@@ -141,7 +169,7 @@ async fn start(
     (runtime, core, resources, stop, task)
 }
 
-async fn wait_done(path: &std::path::Path, after: usize) -> Value {
+async fn wait_done(path: &std::path::Path, after: usize, phase: &str) -> Value {
     tokio::time::timeout(Duration::from_secs(15), async {
         loop {
             let entries: Vec<Value> = std::fs::read_to_string(path)
@@ -162,7 +190,7 @@ async fn wait_done(path: &std::path::Path, after: usize) -> Value {
     .await
     .unwrap_or_else(|error| {
         panic!(
-            "plugin reconciliation did not finish: {error}; observations: {}",
+            "plugin reconciliation did not finish during {phase}: {error}; observations: {}",
             std::fs::read_to_string(path).unwrap_or_default(),
         )
     })
@@ -178,7 +206,7 @@ async fn malformed_maintenance_response_does_not_stop_the_host_worker() {
             json!({"maintenance_fixture":true,"invalid_response_method":"plugin.reconcile"}),
         )
         .await;
-    let (runtime, core, resources, stop, task) = start(&environment).await;
+    let (runtime, core, resources, stop, task) = start(&environment, "malformed response").await;
     tokio::time::timeout(Duration::from_secs(5), async {
         while core
             .snapshots()
@@ -220,8 +248,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou
     let marker = environment.directory.path().join("maintenance.jsonl");
     let failed = environment.directory.path().join("failed-once");
     environment.install_plugin(json!({"maintenance_fixture":true, "maintenance_marker":marker, "maintenance_fail_once":failed})).await;
-    let (runtime, core, resources, stop, task) = start(&environment).await;
-    let first = wait_done(&marker, 0).await;
+    let (runtime, core, resources, stop, task) = start(&environment, "initial startup").await;
+    let first = wait_done(&marker, 0, "initial startup").await;
     assert!(failed.exists());
     let group_id =
         gateway_core::routing::AccountGroupId::new(first["group"].as_str().unwrap()).unwrap();
@@ -284,8 +312,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou
     // 离线新增账号后重新启动真实插件子进程；稳定资源键必须复用原分组与 Key
     let offline = environment.account(None).await;
     std::fs::write(&marker, "").unwrap();
-    let (runtime, core, resources, stop, task) = start(&environment).await;
-    let restored = wait_done(&marker, 0).await;
+    let (runtime, core, resources, stop, task) = start(&environment, "restoration").await;
+    let restored = wait_done(&marker, 0, "restoration").await;
     assert_eq!(restored["group"], first["group"]);
     assert_eq!(restored["key"], first["key"]);
     let members = environment

backend/crates/gateway-plugin/runtime/tests/rpc/session.rs

@@ -325,7 +325,7 @@ async fn shutdown_returns_after_in_flight_callback_future_is_dropped() {
     .await
     .expect("session shutdown");
     assert!(callbacks.dropped.load(Ordering::Acquire));
-    assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed)));
+    assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed(_))));
 }
 
 #[tokio::test]
@@ -416,7 +416,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() {
             .await
             .err()
             .expect("oversized call must fail");
-        assert_eq!(error, RpcError::Context);
+        assert!(matches!(error, RpcError::Context(_)));
         assert!(matches!(
             session
                 .call_stream(
@@ -426,7 +426,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() {
                     payload,
                 )
                 .await,
-            Err(RpcError::Context)
+            Err(RpcError::Context(_))
         ));
         assert_eq!(callbacks.begun.load(Ordering::Relaxed), begun);
         assert!(session.is_ready());
@@ -653,7 +653,7 @@ async fn unknown_parent_cannot_borrow_another_calls_resources() {
             vec![],
         )
         .await;
-    assert!(matches!(reply, Err(RpcError::Protocol)));
+    assert!(matches!(reply, Err(RpcError::Protocol(_))));
     assert_eq!(callbacks.called.load(Ordering::Relaxed), 0);
 }
 
@@ -712,12 +712,16 @@ async fn child_crash_fails_the_call_without_waiting_for_its_deadline() {
     let result = tokio::time::timeout(Duration::from_secs(2), call)
         .await
         .unwrap();
-    assert!(matches!(result, Err(RpcError::Closed)));
+    assert!(matches!(result, Err(RpcError::Closed(_))));
 }
 
 #[tokio::test]
 async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() {
-    for method in ["malformed_truncated_frame", "malformed_frame_length"] {
+    for method in [
+        "malformed_truncated_frame",
+        "malformed_frame_length",
+        "malformed_metadata",
+    ] {
         let callbacks = Arc::new(LifecycleCallbacks::default());
         let (_cache, session) = session(Arc::clone(&callbacks)).await;
         let result = tokio::time::timeout(
@@ -731,7 +735,29 @@ async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() {
         )
         .await
         .expect("malformed worker output must fail within the process boundary");
-        assert!(matches!(result, Err(RpcError::Closed)));
+        let error = result.err().expect("malformed frame failure");
+        assert!(matches!(error, RpcError::Closed(Some(_))));
+        if method == "malformed_metadata" {
+            use std::error::Error as _;
+            let frame = error
+                .source()
+                .unwrap()
+                .downcast_ref::<gateway_plugin_sdk::FrameError>()
+                .unwrap();
+            assert!(frame.source().unwrap().is::<serde_json::Error>());
+            let provider = gateway_core::error::ProviderError::new(
+                gateway_core::error::ProviderErrorKind::Protocol,
+                gateway_core::upstream::UpstreamSendState::Ambiguous,
+            )
+            .with_source(error);
+            assert!(
+                provider
+                    .error_details()
+                    .unwrap()
+                    .contains("PRIVATE_INVALID_FRAME_KIND")
+            );
+            assert!(!format!("{provider:?} {provider}").contains("PRIVATE_INVALID_FRAME_KIND"));
+        }
         assert!(!session.is_ready());
         assert_eq!(callbacks.begun.load(Ordering::Relaxed), 1);
         assert_eq!(callbacks.finished.load(Ordering::Relaxed), 1);
@@ -786,7 +812,7 @@ async fn stream_window_and_sequence_violations_fail_closed() {
             )
             .await
             .unwrap();
-        assert!(matches!(stream.next().await, Err(RpcError::Protocol)));
+        assert!(matches!(stream.next().await, Err(RpcError::Protocol(_))));
         session.shutdown(Duration::from_secs(1)).await;
     }
 }

backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs

@@ -1368,9 +1368,19 @@ async fn main() {
                 last_call = id;
                 if matches!(
                     method.as_str(),
-                    "malformed_truncated_frame" | "malformed_frame_length"
+                    "malformed_truncated_frame" | "malformed_frame_length" | "malformed_metadata"
                 ) {
-                    let bytes: &[u8] = if method == "malformed_truncated_frame" {
+                    let mut malformed_metadata = Vec::new();
+                    if method == "malformed_metadata" {
+                        let metadata = br#"{"type":"PRIVATE_INVALID_FRAME_KIND"}"#;
+                        malformed_metadata
+                            .extend_from_slice(&(metadata.len() as u32).to_be_bytes());
+                        malformed_metadata.extend_from_slice(&0_u64.to_be_bytes());
+                        malformed_metadata.extend_from_slice(metadata);
+                    }
+                    let bytes: &[u8] = if method == "malformed_metadata" {
+                        &malformed_metadata
+                    } else if method == "malformed_truncated_frame" {
                         // 声明 16 字节元数据，却只写入一个字节后退出
                         &[0, 0, 0, 16, 0, 0, 0, 0, b'{']
                     } else {

backend/crates/gateway-plugin/sdk/src/client/frame.rs

@@ -14,7 +14,7 @@ pub async fn read_frame<R: AsyncRead + Unpin>(reader: &mut R) -> Result<Frame, F
     check_metadata_length(metadata_len)?;
     let mut metadata = vec![0; metadata_len];
     reader.read_exact(&mut metadata).await?;
-    let message = serde_json::from_slice(&metadata).map_err(|_| FrameError::Metadata)?;
+    let message = serde_json::from_slice(&metadata).map_err(FrameError::Metadata)?;
     let mut payload = Vec::new();
     // 按实际读取进度分配，避免仅凭对端声明的大长度立即占满内存
     while payload.len() < payload_len {
@@ -33,7 +33,7 @@ pub async fn write_frame<W: AsyncWrite + Unpin>(
     writer: &mut W,
     frame: &Frame,
 ) -> Result<(), FrameError> {
-    let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?;
+    let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?;
     check_metadata_length(metadata.len())?;
     let metadata_len = u32::try_from(metadata.len()).map_err(|_| FrameError::Length)?;
     let payload_len = u64::try_from(frame.payload.len()).map_err(|_| FrameError::Length)?;
@@ -49,7 +49,7 @@ pub async fn write_frame<W: AsyncWrite + Unpin>(
 
 /// 入队前校验本地元数据，避免单次编码失败中断共享传输
 pub fn validate_frame(frame: &Frame) -> Result<(), FrameError> {
-    let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?;
+    let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?;
     check_metadata_length(metadata.len())
 }
 

backend/crates/gateway-plugin/sdk/src/message.rs

@@ -116,7 +116,7 @@ pub enum FrameError {
     #[error("plugin frame exceeds its limit or has invalid lengths")]
     Length,
     #[error("plugin frame metadata is invalid")]
-    Metadata,
+    Metadata(#[source] serde_json::Error),
     #[error("plugin transport is closed or incomplete")]
     Io(#[from] std::io::Error),
 }

backend/crates/gateway-plugin/sdk/tests/message.rs

@@ -22,3 +22,24 @@ fn diagnostics_do_not_render_peer_payloads_or_error_details() {
     };
     assert!(!format!("{frame:?} {fault:?}").contains(secret));
 }
+
+#[tokio::test]
+async fn malformed_frame_metadata_retains_the_native_json_location() {
+    use std::error::Error as _;
+    let metadata = br#"{"type": "PRIVATE_UNKNOWN_MESSAGE_KIND"}"#;
+    let mut wire = Vec::new();
+    wire.extend_from_slice(&(metadata.len() as u32).to_be_bytes());
+    wire.extend_from_slice(&0_u64.to_be_bytes());
+    wire.extend_from_slice(metadata);
+    let error = gateway_plugin_sdk::client::read_frame(&mut wire.as_slice())
+        .await
+        .unwrap_err();
+    let source = error
+        .source()
+        .unwrap()
+        .downcast_ref::<serde_json::Error>()
+        .unwrap();
+    assert_eq!(source.line(), 1);
+    assert!(source.column() > 0);
+    assert!(source.is_data());
+}

backend/crates/gateway-store/src/backup/staging.rs

@@ -74,6 +74,7 @@ impl StagingArea {
             .map_err(|source| unavailable("read staging free space", source))?;
         if free < MIN_STAGING_FREE_BYTES {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "backup staging",
                 message: "staging disk space is below 1 GiB".to_owned(),
             });
@@ -103,6 +104,7 @@ fn unavailable(
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "backup staging",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/bundle.rs

@@ -7,6 +7,7 @@ use super::*;
 
 /// 已完成连接、迁移与 hydration 的 Store 能力集合
 pub struct StoreBundle {
+    diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
     admin_ports: AdminStorePorts,
     core_ports: CoreStorePorts,
     provider_ports: ProviderStorePorts,
@@ -19,6 +20,11 @@ pub struct StoreBundle {
 }
 
 impl StoreBundle {
+    #[must_use]
+    pub fn diagnostics(&self) -> Arc<dyn gateway_core::diagnostics::OperationalDiagnostics> {
+        self.diagnostics.clone()
+    }
+
     #[must_use]
     pub fn admin_ports(&self) -> AdminStorePorts {
         self.admin_ports.clone()
@@ -199,6 +205,7 @@ async fn connect(
     let (execution, execution_writer) =
         postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository));
     let execution = Arc::new(execution);
+    let diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics> = execution.clone();
     let (client_key_usage, client_key_usage_writer) =
         postgres::PgClientApiKeyUsageSink::new(pool.clone());
     let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone()));
@@ -212,7 +219,7 @@ async fn connect(
             REDIS_NAMESPACE,
         )?);
     let (admissions, admission_release_writer) =
-        redis::BufferedClientAdmissionPort::new(admissions);
+        redis::BufferedClientAdmissionPort::new(admissions, diagnostics.clone());
     let core_ports = CoreStorePorts::new(
         execution,
         (
@@ -230,6 +237,7 @@ async fn connect(
             )?),
         ),
         Arc::new(client_key_usage),
+        diagnostics.clone(),
     )
     .with_budget(Arc::new(
         postgres::PgClientBudgetStore::new(pool.clone()).with_timezone(config.timezone),
@@ -250,6 +258,7 @@ async fn connect(
         cooldowns,
         runtime_policy,
         oauth_pending,
+        diagnostics.clone(),
     );
     let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new(
         credential_leases,
@@ -283,6 +292,7 @@ async fn connect(
         ),
     };
     Ok(StoreBundle {
+        diagnostics,
         admin_ports,
         core_ports,
         provider_ports,

backend/crates/gateway-store/src/config.rs

@@ -51,6 +51,7 @@ impl StorePoolConfig {
     pub(crate) fn validate(&self) -> StoreResult<()> {
         if self.max_connections < 2 || self.acquire_timeout_seconds == 0 {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message:
                     "pool.max_connections must be at least 2 and acquire timeout must be positive"
@@ -81,6 +82,7 @@ impl StoreConfig {
     pub fn resolve_and_validate(&mut self, runtime_data_dir: &Path) -> StoreResult<()> {
         if runtime_data_dir.as_os_str().is_empty() {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message: "runtime_data_dir must not be empty".to_owned(),
             });
@@ -107,6 +109,7 @@ impl StoreConfig {
         self.pool.validate()?;
         if self.backup_staging_dir.as_os_str().is_empty() {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message: "runtime_data_dir was not resolved".to_owned(),
             });
@@ -132,12 +135,14 @@ impl StoreConfig {
 pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult<Option<String>> {
     match std::env::var(name) {
         Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData {
+            source: None,
             entity: "store config",
             message: format!("environment variable {name} is empty"),
         }),
         Ok(value) => Ok(Some(value)),
         Err(std::env::VarError::NotPresent) => Ok(None),
         Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData {
+            source: None,
             entity: "store config",
             message: format!("environment variable {name} is not Unicode"),
         }),
@@ -167,6 +172,7 @@ impl StoreConnectionConfig {
         if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit())
         {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message: format!("{field}.password must be exactly 48 hexadecimal characters"),
             });
@@ -176,17 +182,20 @@ impl StoreConnectionConfig {
 
     fn connection_url(&self, field: &'static str) -> StoreResult<String> {
         let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData {
+            source: None,
             entity: "store config",
             message: format!("{field}.url is invalid"),
         })?;
         if url.password().is_some() {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message: format!("{field}.url must not contain a password"),
             });
         }
         url.set_password(Some(&self.password))
             .map_err(|()| StoreError::InvalidData {
+                source: None,
                 entity: "store config",
                 message: format!("{field}.url cannot carry credentials"),
             })?;

backend/crates/gateway-store/src/postgres/account_groups.rs

@@ -93,7 +93,7 @@ impl PgAccountGroupRepository {
         let mut transaction = self.pool.begin().await.map_err(|source| {
             admin_store_error(ENTITY, unavailable("begin account group mutation", source))
         })?;
-        let result = async {
+        let result: StoreResult<_> = async {
             let revision = bump_config_revision_in_transaction(&mut transaction).await?;
             mutation(&mut transaction).await?;
             append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?;
@@ -108,12 +108,10 @@ impl PgAccountGroupRepository {
                 admin_revision(revision)
             }
             Err(error) => {
-                transaction.rollback().await.map_err(|source| {
-                    admin_store_error(
-                        ENTITY,
-                        unavailable("rollback account group mutation", source),
-                    )
-                })?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(admin_store_error(ENTITY, error))
             }
         }
@@ -210,14 +208,18 @@ impl AccountGroupStore for PgAccountGroupRepository {
             .map(|row| {
                 let group_id = AccountGroupId::new(
                     row.try_get::<String, _>("account_group_id")
-                        .map_err(|_| invalid("invalid group ID"))?,
+                        .map_err(|source| invalid("invalid group ID").with_source(source))?,
                 )
-                .map_err(|_| invalid("invalid group ID"))?;
+                .map_err(|source| invalid("invalid group ID").with_source(source))?;
                 let default_slots = u64::try_from(
                     row.try_get::<i64, _>("max_concurrent_per_account")
-                        .map_err(|_| invalid("invalid default account concurrency"))?,
+                        .map_err(|source| {
+                            invalid("invalid default account concurrency").with_source(source)
+                        })?,
                 )
-                .map_err(|_| invalid("invalid default account concurrency"))?;
+                .map_err(|source| {
+                    invalid("invalid default account concurrency").with_source(source)
+                })?;
                 let account = account_summary_from_row(row)?;
                 Ok(AccountGroupMemberFact {
                     group_id,
@@ -459,7 +461,8 @@ async fn count_groups(pool: &PgPool, query: &AccountGroupListQuery) -> StoreResu
         .fetch_one(pool)
         .await
         .map_err(|source| unavailable("count account groups", source))?;
-    u64::try_from(count).map_err(|_| invalid("negative account group count"))
+    u64::try_from(count)
+        .map_err(|source| invalid("negative account group count").with_source(source))
 }
 
 async fn load_record(pool: &PgPool, id: &str) -> StoreResult<Option<AccountGroupRecord>> {
@@ -479,33 +482,35 @@ async fn load_record(pool: &PgPool, id: &str) -> StoreResult<Option<AccountGroup
 fn group_record(row: &sqlx::postgres::PgRow) -> StoreResult<AccountGroupRecord> {
     let provider_counts: serde_json::Value = row
         .try_get("provider_counts")
-        .map_err(|_| invalid("invalid provider counts"))?;
+        .map_err(|source| invalid("invalid provider counts").with_source(source))?;
     let provider_counts = serde_json::from_value::<BTreeMap<String, u64>>(provider_counts)
-        .map_err(|_| invalid("invalid provider counts"))?;
+        .map_err(|source| invalid("invalid provider counts").with_source(source))?;
     Ok(AccountGroupRecord {
         fast_mode: FastMode::parse(
             row.try_get("fast_mode")
-                .map_err(|_| invalid("invalid fast_mode"))?,
+                .map_err(|source| invalid("invalid fast_mode").with_source(source))?,
         )
         .ok_or_else(|| invalid("invalid fast_mode"))?,
         id: AccountGroupId::new(
             row.try_get::<String, _>("id")
-                .map_err(|_| invalid("invalid id"))?,
+                .map_err(|source| invalid("invalid id").with_source(source))?,
         )
-        .map_err(|_| invalid("invalid id"))?,
-        name: row.try_get("name").map_err(|_| invalid("invalid name"))?,
+        .map_err(|source| invalid("invalid id").with_source(source))?,
+        name: row
+            .try_get("name")
+            .map_err(|source| invalid("invalid name").with_source(source))?,
         description: row
             .try_get("description")
-            .map_err(|_| invalid("invalid description"))?,
+            .map_err(|source| invalid("invalid description").with_source(source))?,
         color: AccountGroupColor::parse(
             row.try_get::<String, _>("color")
-                .map_err(|_| invalid("invalid color"))?
+                .map_err(|source| invalid("invalid color").with_source(source))?
                 .as_str(),
         )
         .ok_or_else(|| invalid("invalid color"))?,
         enabled: row
             .try_get("enabled")
-            .map_err(|_| invalid("invalid enabled"))?,
+            .map_err(|source| invalid("invalid enabled").with_source(source))?,
         member_count: count_value(row, "member_count")?,
         provider_counts,
         client_key_count: count_value(row, "client_key_count")?,
@@ -519,16 +524,17 @@ fn group_record(row: &sqlx::postgres::PgRow) -> StoreResult<AccountGroupRecord>
             total_slots: Some(0),
         },
         usage: AccountGroupUsage {
-            today_usd: DecimalAmount::from_str("0").map_err(|_| invalid("invalid zero cost"))?,
+            today_usd: DecimalAmount::from_str("0")
+                .map_err(|source| invalid("invalid zero cost").with_source(source))?,
             retained_total_usd: DecimalAmount::from_str("0")
-                .map_err(|_| invalid("invalid zero cost"))?,
+                .map_err(|source| invalid("invalid zero cost").with_source(source))?,
         },
         created_at: row
             .try_get("created_at")
-            .map_err(|_| invalid("invalid created_at"))?,
+            .map_err(|source| invalid("invalid created_at").with_source(source))?,
         updated_at: row
             .try_get("updated_at")
-            .map_err(|_| invalid("invalid updated_at"))?,
+            .map_err(|source| invalid("invalid updated_at").with_source(source))?,
     })
 }
 
@@ -580,20 +586,21 @@ async fn group_costs(
         .map(|row| {
             let group_id: String = row
                 .try_get("group_id")
-                .map_err(|_| invalid("invalid group ID"))?;
+                .map_err(|source| invalid("invalid group ID").with_source(source))?;
             let today = row
                 .try_get::<String, _>("today_usd")
-                .map_err(|_| invalid("invalid today cost"))?;
+                .map_err(|source| invalid("invalid today cost").with_source(source))?;
             let retained_total = row
                 .try_get::<String, _>("retained_total_usd")
-                .map_err(|_| invalid("invalid retained total cost"))?;
+                .map_err(|source| invalid("invalid retained total cost").with_source(source))?;
             Ok((
                 group_id,
                 AccountGroupUsage {
                     today_usd: DecimalAmount::from_str(&today)
-                        .map_err(|_| invalid("invalid today cost"))?,
-                    retained_total_usd: DecimalAmount::from_str(&retained_total)
-                        .map_err(|_| invalid("invalid retained total cost"))?,
+                        .map_err(|source| invalid("invalid today cost").with_source(source))?,
+                    retained_total_usd: DecimalAmount::from_str(&retained_total).map_err(
+                        |source| invalid("invalid retained total cost").with_source(source),
+                    )?,
                 },
             ))
         })
@@ -603,9 +610,9 @@ async fn group_costs(
 fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult<u64> {
     u64::try_from(
         row.try_get::<i64, _>(field)
-            .map_err(|_| invalid("invalid count"))?,
+            .map_err(|source| invalid("invalid count").with_source(source))?,
     )
-    .map_err(|_| invalid("negative count"))
+    .map_err(|source| invalid("negative count").with_source(source))
 }
 
 fn validate_page_query(query: &AccountGroupListQuery) -> AdminStoreResult<()> {
@@ -658,6 +665,7 @@ fn not_found(id: &str) -> AdminStoreError {
 
 fn not_found_store(id: &str) -> StoreError {
     StoreError::NotFound {
+        source: None,
         entity: ENTITY,
         id: id.to_owned(),
     }
@@ -665,6 +673,7 @@ fn not_found_store(id: &str) -> StoreError {
 
 fn conflict(id: &str) -> StoreError {
     StoreError::Conflict {
+        source: None,
         entity: ENTITY,
         id: id.to_owned(),
         kind: ConflictKind::InvalidTransition,
@@ -677,6 +686,7 @@ fn invalid_admin(message: &str) -> AdminStoreError {
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }
@@ -695,7 +705,7 @@ pub(crate) async fn insert_account_group_in_transaction(
     command: &NewAccountGroup,
 ) -> StoreResult<()> {
     validate_group_fields(&command.name, command.description.as_deref())
-        .map_err(|_| invalid("invalid account group fields"))?;
+        .map_err(|source| invalid("invalid account group fields").with_source(source))?;
     sqlx::query(
         "insert into account_groups
          (id, name, description, color, fast_mode, enabled, created_at, updated_at)

backend/crates/gateway-store/src/postgres/admin_security_audit.rs

@@ -214,8 +214,9 @@ pub(crate) async fn append_admin_audit_event_in_transaction(
     event.config_revision = revision
         .into()
         .map(|revision| {
-            i64::try_from(revision.get())
-                .map_err(|_| invalid("config revision exceeds PostgreSQL bigint"))
+            i64::try_from(revision.get()).map_err(|source| {
+                invalid("config revision exceeds PostgreSQL bigint").with_source(source)
+            })
         })
         .transpose()?;
     event.validate()?;
@@ -244,6 +245,7 @@ pub(crate) async fn append_admin_audit_event_in_transaction(
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/postgres/admission_recovery.rs

@@ -110,18 +110,18 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository {
         Box::pin(async move {
             self.load_client_admission_recovery(DateTime::<Utc>::from(since))
                 .await
-                .map_err(|_| ClientAdmissionError)?
+                .map_err(|source| ClientAdmissionError(Some(source.into())))?
                 .into_iter()
                 .map(|recovery| {
                     let client_api_key_id = ClientApiKeyId::new(recovery.client_api_key_ref)
-                        .map_err(|_| ClientAdmissionError)?;
+                        .map_err(|source| ClientAdmissionError(Some(source.into())))?;
                     let recent_requests = recovery
                         .recent_requests
                         .into_iter()
                         .map(|request| {
                             Ok(RecentAdmissionFact {
                                 model_request_id: ModelRequestId::new(request.model_request_id)
-                                    .map_err(|_| ClientAdmissionError)?,
+                                    .map_err(|source| ClientAdmissionError(Some(source.into())))?,
                                 started_at: request.started_at.into(),
                             })
                         })
@@ -132,7 +132,7 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository {
                         .map(|request| {
                             Ok(RunningAdmissionFact {
                                 model_request_id: ModelRequestId::new(request.model_request_id)
-                                    .map_err(|_| ClientAdmissionError)?,
+                                    .map_err(|source| ClientAdmissionError(Some(source.into())))?,
                                 expires_at: request.deadline_at.into(),
                             })
                         })

backend/crates/gateway-store/src/postgres/backup.rs

@@ -55,12 +55,10 @@ impl BackupRepository for PgBackupRepository {
         command: UpdateBackupStorageCommand,
         context: &MutationContext,
     ) -> AdminStoreResult<(BackupSettings, AdminRevision)> {
-        let mut transaction = self
-            .pool
-            .begin()
-            .await
-            .map_err(|_| store_unavailable("begin backup storage update"))?;
-        let result = async {
+        let mut transaction = self.pool.begin().await.map_err(|source| {
+            store_unavailable("begin backup storage update").with_source(source)
+        })?;
+        let result: AdminStoreResult<_> = async {
             let current = lock_settings_in_transaction(&mut transaction).await?;
             let mut changed = storage_changed_fields(&current, &command);
             if changed.is_empty() {
@@ -68,11 +66,14 @@ impl BackupRepository for PgBackupRepository {
                     sqlx::query_scalar("select config_revision from runtime_settings where id = 1")
                         .fetch_one(&mut *transaction)
                         .await
-                        .map_err(|_| store_unavailable("load unchanged backup revision"))?;
+                        .map_err(|source| {
+                            store_unavailable("load unchanged backup revision").with_source(source)
+                        })?;
                 let revision = AdminRevision::new(
-                    u64::try_from(revision).map_err(|_| store_invalid("config revision"))?,
+                    u64::try_from(revision)
+                        .map_err(|source| store_invalid("config revision").with_source(source))?,
                 )
-                .map_err(|_| store_invalid("config revision"))?;
+                .map_err(|source| store_invalid("config revision").with_source(source))?;
                 return Ok((current, revision));
             }
             ensure_storage_identity_stable(&current, &command, &mut transaction).await?;
@@ -106,7 +107,7 @@ impl BackupRepository for PgBackupRepository {
             .bind(command.force_path_style)
             .execute(&mut *transaction)
             .await
-            .map_err(|_| store_unavailable("update backup storage"))?;
+            .map_err(|source| store_unavailable("update backup storage").with_source(source))?;
 
             let store_revision = bump_config_revision_in_transaction(&mut transaction)
                 .await
@@ -135,17 +136,16 @@ impl BackupRepository for PgBackupRepository {
         .await;
         match result {
             Ok(result) => {
-                transaction
-                    .commit()
-                    .await
-                    .map_err(|_| store_unavailable("commit backup storage update"))?;
+                transaction.commit().await.map_err(|source| {
+                    store_unavailable("commit backup storage update").with_source(source)
+                })?;
                 Ok(result)
             }
             Err(error) => {
-                transaction
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback backup storage update"))?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }
@@ -158,11 +158,9 @@ impl BackupRepository for PgBackupRepository {
         context: &MutationContext,
         timezone: gateway_core::time::DeploymentTimeZone,
     ) -> AdminStoreResult<BackupSettings> {
-        let mut transaction = self
-            .pool
-            .begin()
-            .await
-            .map_err(|_| store_unavailable("begin backup schedule update"))?;
+        let mut transaction = self.pool.begin().await.map_err(|source| {
+            store_unavailable("begin backup schedule update").with_source(source)
+        })?;
         let result = async {
             sqlx::query(
                 "update backup_settings
@@ -183,7 +181,7 @@ impl BackupRepository for PgBackupRepository {
             .bind(next_run_at)
             .execute(&mut *transaction)
             .await
-            .map_err(|_| store_unavailable("update backup schedule"))?;
+            .map_err(|source| store_unavailable("update backup schedule").with_source(source))?;
 
             let audit = crate::mutation_audit(
                 context,
@@ -205,17 +203,16 @@ impl BackupRepository for PgBackupRepository {
         .await;
         match result {
             Ok(settings) => {
-                transaction
-                    .commit()
-                    .await
-                    .map_err(|_| store_unavailable("commit backup schedule update"))?;
+                transaction.commit().await.map_err(|source| {
+                    store_unavailable("commit backup schedule update").with_source(source)
+                })?;
                 Ok(settings)
             }
             Err(error) => {
-                transaction
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback backup schedule update"))?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }
@@ -231,20 +228,23 @@ impl BackupRepository for PgBackupRepository {
                 set last_verified_at = $2, updated_at = now()
               where id = 1 and storage_revision = $1",
         )
-        .bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?)
+        .bind(
+            i64::try_from(storage_revision)
+                .map_err(|source| store_invalid("storage revision").with_source(source))?,
+        )
         .bind(at)
         .execute(&self.pool)
         .await
-        .map_err(|_| store_unavailable("record backup storage verification"))?;
+        .map_err(|source| {
+            store_unavailable("record backup storage verification").with_source(source)
+        })?;
         Ok(result.rows_affected() == 1)
     }
 
     async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
-        let mut transaction = self
-            .pool
-            .begin()
-            .await
-            .map_err(|_| store_unavailable("begin backup record insert"))?;
+        let mut transaction = self.pool.begin().await.map_err(|source| {
+            store_unavailable("begin backup record insert").with_source(source)
+        })?;
         let result = async {
             let _ = lock_settings_in_transaction(&mut transaction).await?;
             insert_queued_in_transaction(&mut transaction, &seed).await
@@ -252,17 +252,16 @@ impl BackupRepository for PgBackupRepository {
         .await;
         match result {
             Ok(record) => {
-                transaction
-                    .commit()
-                    .await
-                    .map_err(|_| store_unavailable("commit backup record insert"))?;
+                transaction.commit().await.map_err(|source| {
+                    store_unavailable("commit backup record insert").with_source(source)
+                })?;
                 Ok(record)
             }
             Err(error) => {
-                transaction
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback backup record insert"))?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }
@@ -276,11 +275,10 @@ impl BackupRepository for PgBackupRepository {
         expected_timezone: &str,
         expected_next_run_at: Option<DateTime<Utc>>,
     ) -> AdminStoreResult<bool> {
-        let mut transaction = self
-            .pool
-            .begin()
-            .await
-            .map_err(|_| store_unavailable("begin scheduled insert"))?;
+        let mut transaction =
+            self.pool.begin().await.map_err(|source| {
+                store_unavailable("begin scheduled insert").with_source(source)
+            })?;
         // 先锁定并核对计划，防止配置变更后仍排入旧计划任务
         let advanced = sqlx::query(
             "update backup_settings set next_run_at = $1
@@ -293,52 +291,47 @@ impl BackupRepository for PgBackupRepository {
         .bind(expected_next_run_at)
         .execute(&mut *transaction)
         .await
-        .map_err(|_| store_unavailable("guard scheduled insert"))?
+        .map_err(|source| store_unavailable("guard scheduled insert").with_source(source))?
         .rows_affected()
             > 0;
         if !advanced {
-            transaction
-                .rollback()
-                .await
-                .map_err(|_| store_unavailable("rollback stale schedule"))?;
+            transaction.rollback().await.map_err(|source| {
+                store_unavailable("rollback stale schedule").with_source(source)
+            })?;
             return Ok(false);
         }
         // 唯一约束冲突只跳过本次任务；保存点避免连带撤销游标推进
         let mut insertion = transaction
             .begin()
             .await
-            .map_err(|_| store_unavailable("begin scheduled savepoint"))?;
+            .map_err(|source| store_unavailable("begin scheduled savepoint").with_source(source))?;
         let inserted = match insert_queued_in_transaction(&mut insertion, &seed).await {
             Ok(_) => {
-                insertion
-                    .commit()
-                    .await
-                    .map_err(|_| store_unavailable("release scheduled savepoint"))?;
+                insertion.commit().await.map_err(|source| {
+                    store_unavailable("release scheduled savepoint").with_source(source)
+                })?;
                 true
             }
             Err(error) if is_active_or_scheduled_conflict(&error) => {
-                insertion
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback scheduled savepoint"))?;
+                insertion.rollback().await.map_err(|source| {
+                    store_unavailable("rollback scheduled savepoint").with_source(source)
+                })?;
                 false
             }
-            Err(error) => {
-                insertion
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback scheduled savepoint"))?;
-                transaction
-                    .rollback()
-                    .await
-                    .map_err(|_| store_unavailable("rollback scheduled insert"))?;
+            Err(mut error) => {
+                if let Err(cleanup) = insertion.rollback().await {
+                    error = error.with_cleanup(cleanup);
+                }
+                if let Err(cleanup) = transaction.rollback().await {
+                    error = error.with_cleanup(cleanup);
+                }
                 return Err(error);
             }
         };
         transaction
             .commit()
             .await
-            .map_err(|_| store_unavailable("commit scheduled insert"))?;
+            .map_err(|source| store_unavailable("commit scheduled insert").with_source(source))?;
         Ok(inserted)
     }
 
@@ -361,7 +354,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(trigger)
         .fetch_one(&self.pool)
         .await
-        .map_err(|_| store_unavailable("count backup records"))?;
+        .map_err(|source| store_unavailable("count backup records").with_source(source))?;
 
         let rows = sqlx::query(
             "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
@@ -376,10 +369,13 @@ impl BackupRepository for PgBackupRepository {
         .bind(status)
         .bind(trigger)
         .bind(i64::from(limit))
-        .bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?)
+        .bind(
+            i64::try_from(offset)
+                .map_err(|source| store_invalid("page offset").with_source(source))?,
+        )
         .fetch_all(&self.pool)
         .await
-        .map_err(|_| store_unavailable("list backup records"))?;
+        .map_err(|source| store_unavailable("list backup records").with_source(source))?;
 
         let items = rows
             .iter()
@@ -387,7 +383,8 @@ impl BackupRepository for PgBackupRepository {
             .collect::<Result<Vec<_>, _>>()?;
         Ok(BackupRecordPage {
             items,
-            total: u64::try_from(total).map_err(|_| store_invalid("record count"))?,
+            total: u64::try_from(total)
+                .map_err(|source| store_invalid("record count").with_source(source))?,
             page: query.page,
             page_size: query.page_size,
         })
@@ -403,7 +400,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(id)
         .fetch_optional(&self.pool)
         .await
-        .map_err(|_| store_unavailable("load backup record"))?
+        .map_err(|source| store_unavailable("load backup record").with_source(source))?
         .map(|row| backup_record_from_row(&row))
         .transpose()
     }
@@ -419,7 +416,9 @@ impl BackupRepository for PgBackupRepository {
         )
         .fetch_all(&self.pool)
         .await
-        .map_err(|_| store_unavailable("list intermediate backup records"))?;
+        .map_err(|source| {
+            store_unavailable("list intermediate backup records").with_source(source)
+        })?;
         rows.iter()
             .map(backup_record_from_row)
             .collect::<Result<Vec<_>, _>>()
@@ -439,7 +438,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(i64::from(limit))
         .fetch_all(&self.pool)
         .await
-        .map_err(|_| store_unavailable("list pending deletions"))?;
+        .map_err(|source| store_unavailable("list pending deletions").with_source(source))?;
         rows.iter()
             .map(backup_record_from_row)
             .collect::<Result<Vec<_>, _>>()
@@ -463,7 +462,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(i64::from(limit))
         .fetch_all(&self.pool)
         .await
-        .map_err(|_| store_unavailable("list expired backups"))?;
+        .map_err(|source| store_unavailable("list expired backups").with_source(source))?;
         rows.iter()
             .map(backup_record_from_row)
             .collect::<Result<Vec<_>, _>>()
@@ -497,7 +496,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(now)
         .fetch_optional(&self.pool)
         .await
-        .map_err(|_| store_unavailable("claim queued backup task"))?
+        .map_err(|source| store_unavailable("claim queued backup task").with_source(source))?
         .map(|row| backup_record_from_row(&row))
         .transpose()
     }
@@ -513,7 +512,7 @@ impl BackupRepository for PgBackupRepository {
             .size_bytes
             .map(i64::try_from)
             .transpose()
-            .map_err(|_| store_invalid("size bytes"))?;
+            .map_err(|source| store_invalid("size bytes").with_source(source))?;
         sqlx::query(
             "update backup_records
                 set status = $1,
@@ -542,7 +541,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(transition.from().as_str())
         .fetch_optional(&self.pool)
         .await
-        .map_err(|_| store_unavailable("transition backup task"))?
+        .map_err(|source| store_unavailable("transition backup task").with_source(source))?
         .map(|row| backup_record_from_row(&row))
         .transpose()
     }
@@ -564,7 +563,7 @@ impl BackupRepository for PgBackupRepository {
         .bind(id)
         .fetch_optional(&self.pool)
         .await
-        .map_err(|_| store_unavailable("transition backup to deleting"))?
+        .map_err(|source| store_unavailable("transition backup to deleting").with_source(source))?
         .map(|row| backup_record_from_row(&row))
         .transpose()
     }
@@ -575,7 +574,7 @@ impl BackupRepository for PgBackupRepository {
             .execute(&self.pool)
             .await
             .map(|_| ())
-            .map_err(|_| store_unavailable("delete backup record"))
+            .map_err(|source| store_unavailable("delete backup record").with_source(source))
     }
 
     async fn advance_schedule_cursor(
@@ -602,7 +601,9 @@ impl BackupRepository for PgBackupRepository {
         .bind(timezone)
         .execute(&self.pool)
         .await
-        .map_err(|_| store_unavailable("advance backup schedule cursor"))?;
+        .map_err(|source| {
+            store_unavailable("advance backup schedule cursor").with_source(source)
+        })?;
         Ok(result.rows_affected() == 1)
     }
 
@@ -623,7 +624,9 @@ impl BackupRepository for PgBackupRepository {
         .bind(i64::from(limit))
         .fetch_all(&self.pool)
         .await
-        .map_err(|_| store_unavailable("list scheduled completed backups"))?;
+        .map_err(|source| {
+            store_unavailable("list scheduled completed backups").with_source(source)
+        })?;
         rows.iter()
             .map(backup_record_from_row)
             .collect::<Result<Vec<_>, _>>()
@@ -649,7 +652,7 @@ async fn lock_settings_in_transaction(
     )
     .fetch_optional(&mut **transaction)
     .await
-    .map_err(|_| store_unavailable("lock backup settings"))?
+    .map_err(|source| store_unavailable("lock backup settings").with_source(source))?
     .map(|row| backup_settings_from_row(&row))
     .transpose()?
     .ok_or_else(|| store_not_found("backup settings"))
@@ -666,7 +669,7 @@ async fn load_settings_in_transaction(
     )
     .fetch_optional(&mut **transaction)
     .await
-    .map_err(|_| store_unavailable("load backup settings in transaction"))?
+    .map_err(|source| store_unavailable("load backup settings in transaction").with_source(source))?
     .map(|row| backup_settings_from_row(&row))
     .transpose()?
     .ok_or_else(|| store_not_found("backup settings"))
@@ -681,7 +684,7 @@ async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult<BackupSettings>
     )
     .fetch_optional(pool)
     .await
-    .map_err(|_| store_unavailable("load backup settings"))?
+    .map_err(|source| store_unavailable("load backup settings").with_source(source))?
     .map(|row| backup_settings_from_row(&row))
     .transpose()?
     .ok_or_else(|| store_not_found("backup settings"))
@@ -761,7 +764,9 @@ async fn ensure_storage_identity_stable(
         sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)")
             .fetch_one(&mut **transaction)
             .await
-            .map_err(|_| store_unavailable("check backup records existence"))?;
+            .map_err(|source| {
+                store_unavailable("check backup records existence").with_source(source)
+            })?;
     if records_exist {
         return Err(AdminStoreError::new(
             AdminStoreErrorKind::Conflict,
@@ -815,7 +820,8 @@ async fn append_audit_in_transaction(
 ) -> StoreResult<()> {
     event.config_revision = revision
         .map(|revision| {
-            i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData {
+            i64::try_from(revision.get()).map_err(|source| StoreError::InvalidData {
+                source: Some(source.into()),
                 entity: "backup",
                 message: "revision overflow".to_owned(),
             })
@@ -849,7 +855,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<Backu
     let size_bytes = decode(row.try_get::<Option<i64>, _>("size_bytes"))?
         .map(u64::try_from)
         .transpose()
-        .map_err(|_| store_invalid("size_bytes"))?;
+        .map_err(|source| store_invalid("size_bytes").with_source(source))?;
     Ok(BackupRecord {
         id: decode(row.try_get("id"))?,
         trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?)
@@ -861,7 +867,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<Backu
         size_bytes,
         sha256: decode(row.try_get("sha256"))?,
         attempt_count: u32::try_from(decode(row.try_get::<i32, _>("attempt_count"))?)
-            .map_err(|_| store_invalid("attempt_count"))?,
+            .map_err(|source| store_invalid("attempt_count").with_source(source))?,
         error_code: decode(row.try_get("error_code"))?,
         error_message: decode(row.try_get("error_message"))?,
         started_at: decode(row.try_get("started_at"))?,
@@ -874,7 +880,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<Backu
 
 fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<BackupSettings> {
     let storage_revision = u64::try_from(decode(row.try_get::<i64, _>("storage_revision"))?)
-        .map_err(|_| store_invalid("storage_revision"))?;
+        .map_err(|source| store_invalid("storage_revision").with_source(source))?;
     let secret = decode(row.try_get::<Option<String>, _>("secret_access_key"))?;
     Ok(BackupSettings {
         storage_revision,
@@ -889,9 +895,9 @@ fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<Bac
         cron_expression: decode(row.try_get("cron_expression"))?,
         schedule_timezone: decode(row.try_get("schedule_timezone"))?,
         retention_days: u32::try_from(decode(row.try_get::<i64, _>("retention_days"))?)
-            .map_err(|_| store_invalid("retention_days"))?,
+            .map_err(|source| store_invalid("retention_days").with_source(source))?,
         retention_count: u32::try_from(decode(row.try_get::<i64, _>("retention_count"))?)
-            .map_err(|_| store_invalid("retention_count"))?,
+            .map_err(|source| store_invalid("retention_count").with_source(source))?,
         next_run_at: decode(row.try_get("next_run_at"))?,
         last_verified_at: decode(row.try_get("last_verified_at"))?,
         updated_at: decode(row.try_get("updated_at"))?,
@@ -911,22 +917,7 @@ fn decode<T>(result: Result<T, sqlx::Error>) -> AdminStoreResult<T> {
 
 /// StoreError → AdminStoreError 的显式映射
 fn map_admin_error(error: StoreError) -> AdminStoreError {
-    match error {
-        StoreError::Unavailable { .. } => AdminStoreError::new(
-            AdminStoreErrorKind::Unavailable,
-            "backup",
-            "backup store unavailable",
-        ),
-        StoreError::NotFound { entity, .. } => {
-            AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found")
-        }
-        StoreError::Conflict { entity, .. } => {
-            AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict")
-        }
-        StoreError::InvalidData { entity, message } => {
-            AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message)
-        }
-    }
+    crate::admin_store_error("backup", error)
 }
 
 fn store_unavailable(operation: &'static str) -> AdminStoreError {

backend/crates/gateway-store/src/postgres/client_budgets.rs

@@ -65,6 +65,7 @@ async fn reset_client_key_budget_in_transaction(
             .map_err(|source| postgres_unavailable("lock budget reset key", source))?;
     if exists.is_none() {
         return Err(StoreError::NotFound {
+            source: None,
             entity: "client API key",
             id: command.id.as_str().to_owned(),
         });
@@ -158,9 +159,15 @@ impl PgClientBudgetStore {
             .cloned()
             .collect::<Vec<_>>();
         for charge in retries {
-            self.settle(charge).await.map_err(|_| unavailable())?;
+            self.settle(charge)
+                .await
+                .map_err(|source| unavailable().with_source(source))?;
         }
-        let mut tx = self.pool.begin().await.map_err(|_| unavailable())?;
+        let mut tx = self
+            .pool
+            .begin()
+            .await
+            .map_err(|source| unavailable().with_source(source))?;
         let row = sqlx::query(
             "select daily_limit_usd::text, weekly_limit_usd::text, enabled
             from client_api_keys where id = $1 for update",
@@ -168,7 +175,7 @@ impl PgClientBudgetStore {
         .bind(key_id.as_str())
         .fetch_optional(&mut *tx)
         .await
-        .map_err(|_| unavailable())?
+        .map_err(|source| unavailable().with_source(source))?
         .ok_or_else(|| {
             GatewayError::new(
                 GatewayErrorKind::Unauthorized,
@@ -185,16 +192,16 @@ impl PgClientBudgetStore {
             daily_usd: row
                 .get::<String, _>("daily_limit_usd")
                 .parse()
-                .map_err(|_| unavailable())?,
+                .map_err(|source| unavailable().with_source(source))?,
             weekly_usd: row
                 .get::<String, _>("weekly_limit_usd")
                 .parse()
-                .map_err(|_| unavailable())?,
+                .map_err(|source| unavailable().with_source(source))?,
         };
         let now = Utc::now();
         advance_windows(&mut tx, key_id.as_str(), now, now, self.timezone)
             .await
-            .map_err(|_| unavailable())?;
+            .map_err(|source| unavailable().with_source(source))?;
         if limits.is_limited() {
             let window = sqlx::query(
                 "select daily_used_usd::text, weekly_used_usd::text, daily_end, weekly_end
@@ -203,15 +210,15 @@ impl PgClientBudgetStore {
             .bind(key_id.as_str())
             .fetch_one(&mut *tx)
             .await
-            .map_err(|_| unavailable())?;
+            .map_err(|source| unavailable().with_source(source))?;
             let daily: Decimal = window
                 .get::<String, _>("daily_used_usd")
                 .parse()
-                .map_err(|_| unavailable())?;
+                .map_err(|source| unavailable().with_source(source))?;
             let weekly: Decimal = window
                 .get::<String, _>("weekly_used_usd")
                 .parse()
-                .map_err(|_| unavailable())?;
+                .map_err(|source| unavailable().with_source(source))?;
             let daily_exceeded = limits.daily_usd != Decimal::ZERO && daily >= limits.daily_usd;
             let weekly_exceeded = limits.weekly_usd != Decimal::ZERO && weekly >= limits.weekly_usd;
             if daily_exceeded || weekly_exceeded {
@@ -235,11 +242,17 @@ impl PgClientBudgetStore {
                 .with_retry_after(retry));
             }
         }
-        tx.commit().await.map_err(|_| unavailable())
+        tx.commit()
+            .await
+            .map_err(|source| unavailable().with_source(source))
     }
 
     async fn settle_inner(&self, charge: &ClientBudgetCharge) -> Result<(), ClientBudgetError> {
-        let mut tx = self.pool.begin().await.map_err(|_| ClientBudgetError)?;
+        let mut tx = self
+            .pool
+            .begin()
+            .await
+            .map_err(|source| ClientBudgetError(Some(source.into())))?;
         // 与准入统一先锁 Key，再写窗口和费用，串行化同一 Key 的并发结算
         let key = sqlx::query_scalar::<_, String>(
             "select id from client_api_keys where id = $1 for update",
@@ -247,12 +260,14 @@ impl PgClientBudgetStore {
         .bind(charge.key_id.as_str())
         .fetch_optional(&mut *tx)
         .await
-        .map_err(|_| ClientBudgetError)?;
+        .map_err(|source| ClientBudgetError(Some(source.into())))?;
         let Some(key) = key else { return Ok(()) }; // 删除 Key 时也会删除其费用记录
         settle_in_transaction(&mut tx, &key, charge, self.timezone)
             .await
-            .map_err(|_| ClientBudgetError)?;
-        tx.commit().await.map_err(|_| ClientBudgetError)
+            .map_err(|source| ClientBudgetError(Some(source.into())))?;
+        tx.commit()
+            .await
+            .map_err(|source| ClientBudgetError(Some(source.into())))
     }
 }
 
@@ -296,7 +311,14 @@ impl ClientBudgetPort for PgClientBudgetStore {
     fn settle(&self, charge: ClientBudgetCharge) -> BoxFuture<'_, Result<(), ClientBudgetError>> {
         Box::pin(async move {
             let result = self.settle_inner(&charge).await;
-            let mut retry = self.retry.lock().map_err(|_| ClientBudgetError)?;
+            // 重试缓存的锁异常不能覆盖已捕获的账本失败
+            let mut retry = self.retry.lock().map_err(|_| {
+                result
+                    .as_ref()
+                    .err()
+                    .cloned()
+                    .unwrap_or(ClientBudgetError(None))
+            })?;
             if result.is_err() {
                 retry.insert(charge.request_id.as_str().to_owned(), charge);
             } else {

backend/crates/gateway-store/src/postgres/client_keys.rs

@@ -74,13 +74,17 @@ impl ClientApiKeySnapshot {
     ) -> StoreResult<Self> {
         Ok(Self {
             request_profiles: std::collections::BTreeMap::new(),
-            id: ClientApiKeyId::new(id).map_err(|_| invalid("persisted key ID is invalid"))?,
-            plaintext_key: PlaintextClientApiKey::new(key)
-                .map_err(|_| invalid("persisted plaintext key is invalid"))?,
+            id: ClientApiKeyId::new(id)
+                .map_err(|source| invalid("persisted key ID is invalid").with_source(source))?,
+            plaintext_key: PlaintextClientApiKey::new(key).map_err(|source| {
+                invalid("persisted plaintext key is invalid").with_source(source)
+            })?,
             group_ids: group_ids
                 .into_iter()
                 .map(|id| {
-                    AccountGroupId::new(id).map_err(|_| invalid("persisted group ID is invalid"))
+                    AccountGroupId::new(id).map_err(|source| {
+                        invalid("persisted group ID is invalid").with_source(source)
+                    })
                 })
                 .collect::<StoreResult<Vec<_>>>()?,
             limits: RateLimits {
@@ -703,6 +707,7 @@ impl PgAdminClientKeyStore {
                 admin_store_error(
                     ENTITY,
                     StoreError::NotFound {
+                        source: None,
                         entity: ENTITY,
                         id: id.as_str().to_owned(),
                     },
@@ -748,6 +753,7 @@ impl ClientKeyStore for PgAdminClientKeyStore {
             .map_err(|source| map_error(postgres_unavailable("lock budget limits key", source)))?;
         let (daily, weekly) = current.ok_or_else(|| {
             map_error(StoreError::NotFound {
+                source: None,
                 entity: ENTITY,
                 id: command.id.as_str().to_owned(),
             })
@@ -1159,6 +1165,7 @@ pub(crate) async fn insert_client_api_key_in_transaction(
                 && database.constraint() == Some("client_api_keys_key_sha256_idx")
         }) {
             StoreError::Conflict {
+        source: None,
                 entity: ENTITY,
                 id: key.id.clone(),
                 kind: crate::ConflictKind::InvalidTransition,
@@ -1238,6 +1245,7 @@ async fn ensure_client_key_name_available(
     .map_err(|source| postgres_unavailable("check client API key name", source))?;
     if duplicate {
         return Err(StoreError::Conflict {
+            source: None,
             entity: ENTITY,
             id: id.to_owned(),
             kind: crate::ConflictKind::DuplicateName,
@@ -1282,6 +1290,7 @@ fn require_changed(rows_affected: u64, id: &str) -> StoreResult<()> {
         Ok(())
     } else {
         Err(StoreError::NotFound {
+            source: None,
             entity: ENTITY,
             id: id.to_owned(),
         })
@@ -1304,6 +1313,7 @@ async fn replace_client_api_key_groups_in_transaction(
         .map_err(|source| postgres_unavailable("validate client API key groups", source))?;
         if usize::try_from(count).ok() != Some(group_ids.len()) {
             return Err(StoreError::NotFound {
+                source: None,
                 entity: "account group",
                 id: "client API key group selection".to_owned(),
             });
@@ -1342,8 +1352,9 @@ fn validate_group_ids(group_ids: &[String]) -> StoreResult<()> {
 }
 
 fn validate_key(key: &str) -> StoreResult<()> {
-    PlaintextClientApiKey::validate(key)
-        .map_err(|_| invalid("key must contain nonempty visible ASCII characters"))
+    PlaintextClientApiKey::validate(key).map_err(|source| {
+        invalid("key must contain nonempty visible ASCII characters").with_source(source)
+    })
 }
 
 fn validate_request_profiles<'a>(
@@ -1381,7 +1392,8 @@ fn decode_request_profiles(
         .into_iter()
         .map(|(provider, profile)| {
             Ok((
-                ProviderKind::new(provider).map_err(|_| invalid("invalid request profile key"))?,
+                ProviderKind::new(provider)
+                    .map_err(|source| invalid("invalid request profile key").with_source(source))?,
                 OpaqueProviderData::new(profile),
             ))
         })
@@ -1404,7 +1416,7 @@ fn client_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<ClientApiK
     use sqlx::Row as _;
     let groups: serde_json::Value = row
         .try_get("groups")
-        .map_err(|_| invalid("invalid groups"))?;
+        .map_err(|source| invalid("invalid groups").with_source(source))?;
     Ok(
         ClientApiKeyRecord {
             request_profile_overrides:
@@ -1412,40 +1424,47 @@ fn client_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<ClientApiK
                     row.try_get::<sqlx::types::Json<
                         BTreeMap<String, serde_json::Map<String, serde_json::Value>>,
                     >, _>("request_profile_overrides")
-                        .map_err(|_| invalid("invalid request profiles"))?
+                        .map_err(|source| invalid("invalid request profiles").with_source(source))?
                         .0,
                 )?,
-            id: row.try_get("id").map_err(|_| invalid("invalid id"))?,
-            name: row.try_get("name").map_err(|_| invalid("invalid name"))?,
-            label: row.try_get("label").map_err(|_| invalid("invalid label"))?,
-            groups: serde_json::from_value(groups).map_err(|_| invalid("invalid groups"))?,
+            id: row
+                .try_get("id")
+                .map_err(|source| invalid("invalid id").with_source(source))?,
+            name: row
+                .try_get("name")
+                .map_err(|source| invalid("invalid name").with_source(source))?,
+            label: row
+                .try_get("label")
+                .map_err(|source| invalid("invalid label").with_source(source))?,
+            groups: serde_json::from_value(groups)
+                .map_err(|source| invalid("invalid groups").with_source(source))?,
             provider_kinds: row
                 .try_get("provider_kinds")
-                .map_err(|_| invalid("invalid provider kinds"))?,
+                .map_err(|source| invalid("invalid provider kinds").with_source(source))?,
             prefix: row
                 .try_get("prefix")
-                .map_err(|_| invalid("invalid prefix"))?,
+                .map_err(|source| invalid("invalid prefix").with_source(source))?,
             enabled: row
                 .try_get("enabled")
-                .map_err(|_| invalid("invalid enabled"))?,
+                .map_err(|source| invalid("invalid enabled").with_source(source))?,
             max_concurrency: to_u64(
                 row.try_get("max_concurrency")
-                    .map_err(|_| invalid("invalid max concurrency"))?,
+                    .map_err(|source| invalid("invalid max concurrency").with_source(source))?,
             )?,
             requests_per_minute: to_u64(
                 row.try_get("requests_per_minute")
-                    .map_err(|_| invalid("invalid requests per minute"))?,
+                    .map_err(|source| invalid("invalid requests per minute").with_source(source))?,
             )?,
             budget: ClientBudgetStatus::default(),
             last_used_at: row
                 .try_get("last_used_at")
-                .map_err(|_| invalid("invalid last used at"))?,
+                .map_err(|source| invalid("invalid last used at").with_source(source))?,
             created_at: row
                 .try_get("created_at")
-                .map_err(|_| invalid("invalid created at"))?,
+                .map_err(|source| invalid("invalid created at").with_source(source))?,
             updated_at: row
                 .try_get("updated_at")
-                .map_err(|_| invalid("invalid updated at"))?,
+                .map_err(|source| invalid("invalid updated at").with_source(source))?,
         },
     )
 }
@@ -1525,34 +1544,38 @@ async fn load_client_key_memberships(
         use sqlx::Row as _;
         let key_id: String = row
             .try_get("key_id")
-            .map_err(|_| invalid("invalid client API key membership"))?;
+            .map_err(|source| invalid("invalid client API key membership").with_source(source))?;
         let group_id: Option<String> = row
             .try_get("group_id")
-            .map_err(|_| invalid("invalid client API key group"))?;
+            .map_err(|source| invalid("invalid client API key group").with_source(source))?;
         if let Some(group_id) = group_id {
             groups
                 .entry(key_id.clone())
                 .or_default()
                 .push(ClientApiKeyGroupRecord {
                     id: group_id,
-                    name: row
-                        .try_get("group_name")
-                        .map_err(|_| invalid("invalid client API key group name"))?,
-                    color: row
-                        .try_get("group_color")
-                        .map_err(|_| invalid("invalid client API key group color"))?,
-                    enabled: row
-                        .try_get("group_enabled")
-                        .map_err(|_| invalid("invalid client API key group state"))?,
+                    name: row.try_get("group_name").map_err(|source| {
+                        invalid("invalid client API key group name").with_source(source)
+                    })?,
+                    color: row.try_get("group_color").map_err(|source| {
+                        invalid("invalid client API key group color").with_source(source)
+                    })?,
+                    enabled: row.try_get("group_enabled").map_err(|source| {
+                        invalid("invalid client API key group state").with_source(source)
+                    })?,
                 });
             providers.entry(key_id).or_default().extend(
                 row.try_get::<Vec<String>, _>("group_provider_kinds")
-                    .map_err(|_| invalid("invalid client API key provider kinds"))?,
+                    .map_err(|source| {
+                        invalid("invalid client API key provider kinds").with_source(source)
+                    })?,
             );
         } else {
             providers.entry(key_id).or_default().extend(
                 row.try_get::<Vec<String>, _>("global_provider_kinds")
-                    .map_err(|_| invalid("invalid global provider kinds"))?,
+                    .map_err(|source| {
+                        invalid("invalid global provider kinds").with_source(source)
+                    })?,
             );
         }
     }
@@ -1649,15 +1672,18 @@ fn push_client_key_order(statement: &mut QueryBuilder<Postgres>, sort: ClientApi
 }
 
 fn to_i64(value: u64) -> StoreResult<i64> {
-    i64::try_from(value).map_err(|_| invalid("numeric limit exceeds PostgreSQL bigint"))
+    i64::try_from(value)
+        .map_err(|source| invalid("numeric limit exceeds PostgreSQL bigint").with_source(source))
 }
 
 fn to_u64(value: i64) -> StoreResult<u64> {
-    u64::try_from(value).map_err(|_| invalid("persisted numeric limit is negative"))
+    u64::try_from(value)
+        .map_err(|source| invalid("persisted numeric limit is negative").with_source(source))
 }
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/postgres/execution.rs

@@ -631,7 +631,7 @@ impl ModelRequestRepository for PgExecutionStore {
         .bind(attempt.http_version)
         .bind(
             i32::try_from(attempt.attempt_count)
-                .map_err(|_| invalid("attempt_count is too large"))?,
+                .map_err(|source| invalid("attempt_count is too large").with_source(source))?,
         )
         .bind(optional_i64(
             attempt.account_selection_wait_ms,
@@ -649,11 +649,13 @@ impl ModelRequestRepository for PgExecutionStore {
         .await
         .map_err(|source| postgres_unavailable("begin model request attempt", source))?
         .ok_or(StoreError::Conflict {
+            source: None,
             entity: ENTITY,
             id: attempt.model_request_id,
             kind: ConflictKind::DownstreamAlreadyCommitted,
         })?;
-        u32::try_from(count).map_err(|_| invalid("attempt_count is invalid"))
+        u32::try_from(count)
+            .map_err(|source| invalid("attempt_count is invalid").with_source(source))
     }
 
     async fn mark_upstream_send_state(
@@ -847,7 +849,7 @@ impl ModelRequestRepository for PgExecutionStore {
         .bind(finalization.upstream_send_state.as_str())
         .bind(
             i32::try_from(finalization.attempt_count)
-                .map_err(|_| invalid("attempt_count is too large"))?,
+                .map_err(|source| invalid("attempt_count is too large").with_source(source))?,
         )
         .bind(finalization.downstream_committed_at)
         .bind(finalization.client_status_code.map(i32::from))
@@ -992,6 +994,7 @@ impl ExecutionStore for PgExecutionStore {
                 Box::pin(async move {
                     let ttl_ms = i64::try_from(ttl.as_millis()).map_err(|_| {
                         crate::StoreError::InvalidData {
+                            source: None,
                             entity: "model request",
                             message: "lease TTL is invalid".to_owned(),
                         }
@@ -1172,7 +1175,9 @@ impl ExecutionStore for PgExecutionStore {
                 upstream_model_id: None,
                 failure_kind: error.kind().as_str().to_owned(),
                 upstream_send_state: None,
-                error_details: None,
+                error_details: error
+                    .error_details()
+                    .map(gateway_core::error::ErrorDetails::into_string),
                 status_code: None,
                 provider_error_code: error.client_error_code().map(str::to_owned),
                 retry_after_ms: error
@@ -1596,12 +1601,55 @@ fn validate_connection_observation(finalization: &ModelRequestFinalization) -> S
 }
 
 fn to_i64(value: u64, field: &'static str) -> StoreResult<i64> {
-    i64::try_from(value).map_err(|_| invalid(field))
+    i64::try_from(value).map_err(|source| invalid(field).with_source(source))
 }
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }
 }
+
+#[async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for PgExecutionStore {
+    async fn record_failure(
+        &self,
+        failure: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), CoreStoreError> {
+        use super::{OpsEvent, OpsEventLevel, OpsEventRepository, PgOpsEventRepository};
+        let event = OpsEvent {
+            id: Uuid::now_v7().to_string(),
+            model_request_id: None,
+            attempt_index: None,
+            level: OpsEventLevel::Warning,
+            component: failure.component.to_owned(),
+            operation: failure.operation.to_owned(),
+            provider_kind: failure.provider_kind.map(|kind| kind.as_str().to_owned()),
+            provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
+            provider_account_ref: failure.account_id.map(|id| id.as_str().to_owned()),
+            upstream_model_id: None,
+            failure_kind: failure.kind.to_owned(),
+            upstream_send_state: None,
+            error_details: failure
+                .details
+                .map(gateway_core::error::ErrorDetails::into_string),
+            status_code: failure.upstream_status,
+            provider_error_code: failure.upstream_code.map(|code| code.as_str().to_owned()),
+            retry_after_ms: None,
+            upstream_request_id: None,
+            latency_ms: None,
+            message: serde_json::json!({
+                "correlationId": failure.correlation_id,
+                "message": failure.message,
+            })
+            .to_string(),
+            created_at: failure.occurred_at.into(),
+        };
+        PgOpsEventRepository::new(self.pool.clone())
+            .append_ops_event(event)
+            .await
+            .map_err(core_store_error)
+    }
+}

backend/crates/gateway-store/src/postgres/execution_buffer.rs

@@ -8,6 +8,7 @@ use std::time::{Duration, Instant, SystemTime};
 
 use async_trait::async_trait;
 use futures::future::try_join_all;
+use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure};
 use gateway_core::engine::{
     AttemptRecord, EntryRejection, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
     ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport,
@@ -41,8 +42,8 @@ pub struct ExecutionBufferStats {
 }
 
 struct ExecutionBufferState {
-    maximum_queued_items: usize,
-    maximum_queued_bytes: usize,
+    regular_item_capacity: usize,
+    regular_byte_capacity: usize,
     queued_items: AtomicUsize,
     queued_bytes: AtomicUsize,
     enqueued_total: AtomicU64,
@@ -53,10 +54,10 @@ struct ExecutionBufferState {
 }
 
 impl ExecutionBufferState {
-    fn new(maximum_queued_items: NonZeroUsize, maximum_queued_bytes: NonZeroUsize) -> Self {
+    fn new(regular_item_capacity: NonZeroUsize, regular_byte_capacity: NonZeroUsize) -> Self {
         Self {
-            maximum_queued_items: maximum_queued_items.get(),
-            maximum_queued_bytes: maximum_queued_bytes.get(),
+            regular_item_capacity: regular_item_capacity.get(),
+            regular_byte_capacity: regular_byte_capacity.get(),
             queued_items: AtomicUsize::new(0),
             queued_bytes: AtomicUsize::new(0),
             enqueued_total: AtomicU64::new(0),
@@ -67,12 +68,21 @@ impl ExecutionBufferState {
         }
     }
 
-    fn reserve(&self, bytes: usize) -> Result<(), ReservationFailure> {
+    fn reserve(&self, bytes: usize, critical: bool) -> Result<(), ReservationFailure> {
+        // 常规额度之外保留四分之一给失败与请求生命周期，普通进度更新不能消耗
+        let item_limit = self.regular_item_capacity.saturating_add(if critical {
+            self.regular_item_capacity / 4
+        } else {
+            0
+        });
+        let byte_limit = self.regular_byte_capacity.saturating_add(if critical {
+            self.regular_byte_capacity / 4
+        } else {
+            0
+        });
         self.queued_items
             .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
-                current
-                    .checked_add(1)
-                    .filter(|next| *next <= self.maximum_queued_items)
+                current.checked_add(1).filter(|next| *next <= item_limit)
             })
             .map_err(|_| ReservationFailure::ItemCapacity)?;
         let bytes_reserved = self
@@ -80,7 +90,7 @@ impl ExecutionBufferState {
             .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
                 current
                     .checked_add(bytes)
-                    .filter(|next| *next <= self.maximum_queued_bytes)
+                    .filter(|next| *next <= byte_limit)
             })
             .is_ok();
         if !bytes_reserved {
@@ -189,16 +199,17 @@ impl<S: ?Sized> BufferedExecutionStore<S> {
         )
     }
 
+    /// 参数限制常规接收额度；失败与生命周期记录可使用额外四分之一预留
     #[must_use]
     pub fn with_limits(
         inner: Arc<S>,
         capacity: NonZeroUsize,
-        maximum_queued_bytes: NonZeroUsize,
+        regular_byte_capacity: NonZeroUsize,
     ) -> (Self, ExecutionObservationWriter<S>) {
         let lane_count = PERSISTENCE_LANES.min(capacity.get());
         let (senders, receivers): (Vec<_>, Vec<_>) =
             (0..lane_count).map(|_| mpsc::unbounded_channel()).unzip();
-        let state = Arc::new(ExecutionBufferState::new(capacity, maximum_queued_bytes));
+        let state = Arc::new(ExecutionBufferState::new(capacity, regular_byte_capacity));
         (
             Self {
                 inner: Arc::clone(&inner),
@@ -224,16 +235,19 @@ impl<S: ?Sized> BufferedExecutionStore<S> {
             .estimated_bytes()
             .saturating_add(write.request_id().map_or(0, str::len))
             .max(1);
-        if let Err(failure) = self.state.reserve(estimated_bytes) {
+        if let Err(failure) = self.state.reserve(estimated_bytes, write.is_critical()) {
             self.state.record_dropped(1);
             let stats = self.state.snapshot();
+            if !stats.dropped_total.is_power_of_two() {
+                return;
+            }
             tracing::warn!(
                 operation = write.operation(),
                 request_id = ?write.request_id(),
                 reason = failure.reason(),
                 estimated_bytes,
-                maximum_queued_items = self.state.maximum_queued_items,
-                maximum_queued_bytes = self.state.maximum_queued_bytes,
+                regular_item_capacity = self.state.regular_item_capacity,
+                regular_byte_capacity = self.state.regular_byte_capacity,
                 queued_items = stats.queued_items,
                 queued_bytes = stats.queued_bytes,
                 dropped_total = stats.dropped_total,
@@ -253,6 +267,9 @@ impl<S: ?Sized> BufferedExecutionStore<S> {
                 drop(queued);
                 self.state.record_dropped(1);
                 let stats = self.state.snapshot();
+                if !stats.dropped_total.is_power_of_two() {
+                    return;
+                }
                 tracing::warn!(
                     operation,
                     request_id = ?request_id,
@@ -285,10 +302,20 @@ fn request_lane(request_id: &str, lane_count: usize) -> usize {
     usize::try_from(hash % u64::try_from(lane_count).unwrap_or(u64::MAX)).unwrap_or(0)
 }
 
+#[async_trait]
+impl<S: Send + Sync + ?Sized> OperationalDiagnostics for BufferedExecutionStore<S> {
+    async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError> {
+        self.enqueue(ExecutionObservationWrite::OperationalFailure(Box::new(
+            failure,
+        )));
+        Ok(())
+    }
+}
+
 #[async_trait]
 impl<S> ExecutionStore for BufferedExecutionStore<S>
 where
-    S: ExecutionStore + ?Sized,
+    S: ExecutionStore + OperationalDiagnostics + ?Sized,
 {
     fn maintain_request(
         &self,
@@ -455,7 +482,7 @@ impl ExecutionBufferIdle {
 
 impl<S> DaemonTask for ExecutionObservationWriter<S>
 where
-    S: ExecutionStore + ?Sized,
+    S: ExecutionStore + OperationalDiagnostics + ?Sized,
 {
     fn run(
         &self,
@@ -488,7 +515,7 @@ async fn run_lane<S>(
     shutdown_deadline: &OnceLock<Instant>,
 ) -> Result<(), WorkerTaskError>
 where
-    S: ExecutionStore + ?Sized,
+    S: ExecutionStore + OperationalDiagnostics + ?Sized,
 {
     let mut receiver = receiver.lock().await;
     loop {
@@ -548,7 +575,7 @@ fn shared_shutdown_deadline(deadline: &OnceLock<Instant>) -> Instant {
 
 async fn persist_queued<S>(mut queued: QueuedExecutionObservation, store: &S)
 where
-    S: ExecutionStore + ?Sized,
+    S: ExecutionStore + OperationalDiagnostics + ?Sized,
 {
     let operation = queued.operation();
     let request_id = queued.request_id().map(ToOwned::to_owned);
@@ -570,6 +597,9 @@ where
         Err(error) => {
             state.record_write_failure();
             let stats = state.snapshot();
+            if !stats.write_failure_total.is_power_of_two() {
+                return;
+            }
             tracing::warn!(
                 operation,
                 request_id = ?request_id,
@@ -588,7 +618,7 @@ async fn drain_on_shutdown<S>(
     state: &ExecutionBufferState,
     deadline: Instant,
 ) where
-    S: ExecutionStore + ?Sized,
+    S: ExecutionStore + OperationalDiagnostics + ?Sized,
 {
     receiver.close();
     let queued_at_shutdown = receiver.len();
@@ -695,10 +725,20 @@ enum ExecutionObservationWrite {
     IntermediateFailure(Box<IntermediateFailure>),
     ProbeFailure(Box<ProbeFailure>),
     EntryRejection(Box<EntryRejection>),
+    OperationalFailure(Box<OperationalFailure>),
     Finalize(Box<ModelRequestFinalization>),
 }
 
 impl ExecutionObservationWrite {
+    const fn is_critical(&self) -> bool {
+        !matches!(
+            self,
+            Self::MarkSendState { .. }
+                | Self::MarkDownstreamCommitted { .. }
+                | Self::RecordClientStatus { .. }
+        )
+    }
+
     const fn operation(&self) -> &'static str {
         match self {
             Self::Create(_) => "create_model_request",
@@ -710,6 +750,7 @@ impl ExecutionObservationWrite {
             Self::IntermediateFailure(_) => "record_intermediate_failure",
             Self::ProbeFailure(_) => "record_probe_failure",
             Self::EntryRejection(_) => "record_entry_rejection",
+            Self::OperationalFailure(_) => "record_operational_failure",
             Self::Finalize(_) => "finalize_model_request",
         }
     }
@@ -724,6 +765,7 @@ impl ExecutionObservationWrite {
             | Self::RecordClientStatus { request_id, .. } => Some(request_id.as_str()),
             Self::IntermediateFailure(failure) => Some(failure.request_id.as_str()),
             Self::ProbeFailure(_) | Self::EntryRejection(_) => None,
+            Self::OperationalFailure(failure) => failure.correlation_id.as_deref(),
             Self::Finalize(finalization) => Some(finalization.request_id.as_str()),
         }
     }
@@ -756,13 +798,26 @@ impl ExecutionObservationWrite {
                 rejection.error.client_error_code(),
                 rejection.error.client_error_type(),
             ])
-            .saturating_add(size_of::<EntryRejection>()),
+            .saturating_add(size_of::<EntryRejection>())
+            .saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes(
+                std::error::Error::source(&rejection.error),
+            )),
             Self::ProbeFailure(failure) => text_bytes([
                 Some(failure.provider_kind.as_str()),
                 Some(failure.account_id.as_str()),
                 Some(failure.upstream_model_id.as_str()),
             ])
             .saturating_add(provider_error_bytes(&failure.error)),
+            Self::OperationalFailure(failure) => {
+                size_of::<OperationalFailure>().saturating_add(text_bytes([
+                    Some(failure.message.as_str()),
+                    failure.correlation_id.as_deref(),
+                    failure.provider_kind.as_ref().map(|kind| kind.as_str()),
+                    failure.account_id.as_ref().map(|id| id.as_str()),
+                    failure.upstream_code.as_ref().map(|code| code.as_str()),
+                    failure.details.as_ref().map(|details| details.as_str()),
+                ]))
+            }
             Self::Finalize(finalization) => {
                 let error_bytes = finalization.error.as_ref().map_or(0, |error| {
                     text_bytes([
@@ -770,6 +825,11 @@ impl ExecutionObservationWrite {
                         error.client_error_code(),
                         error.client_error_type(),
                     ])
+                    .saturating_add(
+                        gateway_core::error::ErrorSource::estimated_chain_bytes(
+                            std::error::Error::source(error),
+                        ),
+                    )
                 });
                 text_bytes([
                     Some(finalization.request_id.as_str()),
@@ -794,9 +854,10 @@ impl ExecutionObservationWrite {
 
     async fn persist<S>(self, store: &S) -> Result<(), StoreError>
     where
-        S: ExecutionStore + ?Sized,
+        S: ExecutionStore + OperationalDiagnostics + ?Sized,
     {
         match self {
+            Self::OperationalFailure(failure) => store.record_failure(*failure).await,
             Self::Create(request) => store.create_model_request(*request).await,
             Self::Attempt(attempt) => store.record_attempt(*attempt).await,
             Self::CreateWithAttempt(write) => {
@@ -900,33 +961,9 @@ fn provider_error_bytes(error: &ProviderError) -> usize {
                 .saturating_add(header.value().len());
         }
     }
-    bytes.saturating_add(error_source_bytes(error.source()))
-}
-
-/// 原始来源也占据队列内存；只计数而不保存格式化内容，避免复制或打印敏感原因
-fn error_source_bytes(mut source: Option<&(dyn std::error::Error + 'static)>) -> usize {
-    struct ByteCount(usize);
-    impl std::fmt::Write for ByteCount {
-        fn write_str(&mut self, value: &str) -> std::fmt::Result {
-            self.0 = self.0.saturating_add(value.len());
-            Ok(())
-        }
-    }
-    let mut bytes = ByteCount(0);
-    for _ in 0..32 {
-        let Some(error) = source else { return bytes.0 };
-        bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error));
-        if std::fmt::write(&mut bytes, format_args!("{error}")).is_err() {
-            return usize::MAX;
-        }
-        source = error.source();
-    }
-    // 过深或循环的第三方来源不能让观测写入无限遍历，交给已有超限丢弃计数
-    if source.is_some() {
-        usize::MAX
-    } else {
-        bytes.0
-    }
+    bytes.saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes(
+        error.source(),
+    ))
 }
 
 fn text_bytes<const N: usize>(values: [Option<&str>; N]) -> usize {

backend/crates/gateway-store/src/postgres/mod.rs

@@ -60,6 +60,7 @@ pub async fn connect_and_migrate(
 ) -> StoreResult<PgPool> {
     if database_url.trim().is_empty() {
         return Err(StoreError::InvalidData {
+            source: None,
             entity: "PostgreSQL configuration",
             message: "database URL is empty".to_owned(),
         });
@@ -225,7 +226,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
             self.pool.begin().await.map_err(|source| {
                 postgres_unavailable("begin control plane replacement", source)
             })?;
-        let result = async {
+        let result: StoreResult<_> = async {
             // 锁住所有配置写入共同使用的行，版本检查与修改不能被其他事务穿插
             let current = sqlx::query_scalar::<_, i64>(
                 "select config_revision from runtime_settings where id = 1 for update",
@@ -235,6 +236,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
             .map_err(|source| postgres_unavailable("lock control plane revision", source))?;
             if u64::try_from(current).ok() != Some(replacement.expected_revision.get()) {
                 return Err(StoreError::Conflict {
+                    source: None,
                     entity: "runtime settings",
                     id: "1".to_owned(),
                     kind: crate::ConflictKind::StaleRevision,
@@ -256,9 +258,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
                 Ok(snapshot)
             }
             Err(error) => {
-                transaction.rollback().await.map_err(|source| {
-                    postgres_unavailable("rollback control plane replacement", source)
-                })?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }
@@ -337,7 +340,7 @@ impl PgControlPlaneRepository {
         let mut transaction = self.pool.begin().await.map_err(|source| {
             postgres_unavailable("begin targeted control plane mutation", source)
         })?;
-        let result = async {
+        let result: StoreResult<_> = async {
             let revision = bump_config_revision_in_transaction(&mut transaction).await?;
             match mutation {
                 ControlPlaneMutation::CreateClientApiKey(key) => {
@@ -394,9 +397,10 @@ impl PgControlPlaneRepository {
                 Ok(revision)
             }
             Err(error) => {
-                transaction.rollback().await.map_err(|source| {
-                    postgres_unavailable("rollback targeted control plane mutation", source)
-                })?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }

backend/crates/gateway-store/src/postgres/observability/mapping.rs

@@ -212,6 +212,7 @@ pub(crate) fn admin_latency_percentiles(
 
 pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError {
     observability_error(StoreError::InvalidData {
+        source: None,
         entity: "observability latency percentile",
         message: "latency percentile is outside the admin contract".to_owned(),
     })
@@ -265,6 +266,7 @@ pub(crate) fn admin_decimal_amount(
 ) -> AdminStoreResult<admin_observability::DecimalAmount> {
     admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| {
         observability_error(StoreError::InvalidData {
+            source: None,
             entity: "observability decimal amount",
             message: "amount is outside the admin numeric contract".to_owned(),
         })
@@ -449,6 +451,7 @@ pub(crate) fn admin_usage_list_record(
         (None, None) => None,
         _ => {
             return Err(observability_error(StoreError::InvalidData {
+                source: None,
                 entity: "observability request billing",
                 message: "cost amount and currency must be present together".to_owned(),
             }));
@@ -522,6 +525,7 @@ pub(crate) fn admin_usage_record(
         (None, None) => None,
         _ => {
             return Err(observability_error(StoreError::InvalidData {
+                source: None,
                 entity: "observability request billing",
                 message: "cost amount and currency must be present together".to_owned(),
             }));
@@ -610,6 +614,7 @@ pub(crate) fn admin_request_outcome(
 ) -> AdminStoreResult<admin_observability::RequestOutcome> {
     admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| {
         observability_error(StoreError::InvalidData {
+            source: None,
             entity: "observability request outcome",
             message: "invalid request outcome".to_owned(),
         })
@@ -1042,7 +1047,7 @@ pub(crate) fn opaque_response_id(
     get::<Option<Vec<u8>>>(row, column)?
         .map(String::from_utf8)
         .transpose()
-        .map_err(|_| invalid(column))
+        .map_err(|source| invalid(column).with_source(source))
 }
 
 pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> {
@@ -1075,7 +1080,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -
 where
     T: sqlx::Decode<'r, Postgres> + sqlx::Type<Postgres>,
 {
-    row.try_get(column).map_err(|_| invalid(column))
+    row.try_get(column)
+        .map_err(|source| invalid(column).with_source(source))
 }
 
 pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult<u64> {
@@ -1104,15 +1110,18 @@ pub(crate) fn optional_status(
 }
 
 pub(crate) fn to_u64(value: i64) -> StoreResult<u64> {
-    u64::try_from(value).map_err(|_| invalid("numeric observation is negative"))
+    u64::try_from(value)
+        .map_err(|source| invalid("numeric observation is negative").with_source(source))
 }
 
 pub(crate) fn to_u32(value: i32) -> StoreResult<u32> {
-    u32::try_from(value).map_err(|_| invalid("integer observation is negative"))
+    u32::try_from(value)
+        .map_err(|source| invalid("integer observation is negative").with_source(source))
 }
 
 pub(crate) fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "observability",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/postgres/observability/queries/usage.rs

@@ -256,6 +256,7 @@ pub(crate) async fn usage_record_detail(
         .await
         .map_err(|source| postgres_unavailable("load usage record detail", source))?
         .ok_or_else(|| StoreError::NotFound {
+            source: None,
             entity: "model request",
             id: request_id.to_owned(),
         })?;

backend/crates/gateway-store/src/postgres/observability/query_budget.rs

@@ -23,11 +23,13 @@ impl ObservabilityQueryBudget {
     pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult<Self> {
         if max_connections == 0 || wait_timeout.is_zero() {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "observability query budget",
                 message: "requires a positive connection count and wait timeout".to_owned(),
             });
         }
         let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData {
+            source: None,
             entity: "observability query budget",
             message: "connection budget does not fit this platform".to_owned(),
         })?;

backend/crates/gateway-store/src/postgres/ops_events.rs

@@ -136,7 +136,7 @@ impl OpsEventRepository for PgOpsEventRepository {
                 .attempt_index
                 .map(i32::try_from)
                 .transpose()
-                .map_err(|_| invalid("attempt index is too large"))?,
+                .map_err(|source| invalid("attempt index is too large").with_source(source))?,
         )
         .bind(event.level.as_str())
         .bind(event.component)
@@ -154,7 +154,7 @@ impl OpsEventRepository for PgOpsEventRepository {
                 .retry_after_ms
                 .map(i64::try_from)
                 .transpose()
-                .map_err(|_| invalid("retry_after_ms is too large"))?,
+                .map_err(|source| invalid("retry_after_ms is too large").with_source(source))?,
         )
         .bind(event.upstream_request_id)
         .bind(
@@ -162,7 +162,7 @@ impl OpsEventRepository for PgOpsEventRepository {
                 .latency_ms
                 .map(i64::try_from)
                 .transpose()
-                .map_err(|_| invalid("latency_ms is too large"))?,
+                .map_err(|source| invalid("latency_ms is too large").with_source(source))?,
         )
         .bind(event.message)
         .bind(event.error_details)
@@ -176,6 +176,7 @@ impl OpsEventRepository for PgOpsEventRepository {
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/postgres/pricing.rs

@@ -132,6 +132,7 @@ pub(crate) fn validate_pricing(pricing: &PricingOverrides) -> StoreResult<()> {
 
 fn invalid_pricing() -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "model pricing",
         message: "invalid model pricing".to_owned(),
     }

backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs

@@ -211,6 +211,7 @@ impl PgAdminAccountStore {
                 admin_store_error(
                     ENTITY,
                     StoreError::NotFound {
+                        source: None,
                         entity: ENTITY,
                         id: account_id.to_owned(),
                     },

backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs

@@ -228,6 +228,7 @@ async fn validate_group_filter(
         Err(admin_store_error(
             ENTITY,
             StoreError::NotFound {
+                source: None,
                 entity: "account group",
                 id: group_id.as_str().to_owned(),
             },

backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs

@@ -34,21 +34,25 @@ impl PgAdminAccountStore {
         {
             return Err(invalid_receipt());
         }
-        let mut transaction = self.pool.begin().await.map_err(|_| unavailable_receipt())?;
+        let mut transaction = self
+            .pool
+            .begin()
+            .await
+            .map_err(|source| unavailable_receipt().with_source(source))?;
         let outcome = commit_authorization_in_transaction(&mut transaction, command, context).await;
         match outcome {
             Ok(outcome) => {
                 transaction
                     .commit()
                     .await
-                    .map_err(|_| unavailable_receipt())?;
+                    .map_err(|source| unavailable_receipt().with_source(source))?;
                 Ok(outcome)
             }
             Err(error) => {
-                transaction
-                    .rollback()
-                    .await
-                    .map_err(|_| unavailable_receipt())?;
+                let error = match transaction.rollback().await {
+                    Ok(()) => error,
+                    Err(cleanup) => error.with_cleanup(cleanup),
+                };
                 Err(error)
             }
         }
@@ -71,7 +75,7 @@ async fn commit_authorization_in_transaction(
         .bind(lock)
         .execute(&mut **transaction)
         .await
-        .map_err(|_| unavailable_receipt())?;
+        .map_err(|source| unavailable_receipt().with_source(source))?;
     if let Some(result) = load_receipt(&mut **transaction, key).await? {
         return Ok(AuthorizationCommitResult {
             result,
@@ -140,12 +144,12 @@ async fn commit_authorization_in_transaction(
     }];
     sqlx::query("insert into authorization_receipts(provider_kind, flow_digest, owner_digest, config_revision, accounts_json) values ($1,$2,$3,$4,$5)")
             .bind(key.provider_kind().as_str()).bind(key.flow_digest()).bind(key.owner_digest())
-            .bind(i64::try_from(result.config_revision.get()).map_err(|_| invalid_receipt())?)
-            .bind(serde_json::to_value(accounts).map_err(|_| invalid_receipt())?)
-            .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?;
+            .bind(i64::try_from(result.config_revision.get()).map_err(|source| invalid_receipt().with_source(source))?)
+            .bind(serde_json::to_value(accounts).map_err(|source| invalid_receipt().with_source(source))?)
+            .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?;
     // 保留期长于 OAuth pending 上限；每次提交有界回收，避免引入第二套维护任务
     sqlx::query("delete from authorization_receipts where ctid in (select ctid from authorization_receipts where expires_at <= now() order by expires_at limit 128 for update skip locked)")
-            .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?;
+            .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?;
     Ok(AuthorizationCommitResult {
         result,
         newly_committed: true,
@@ -157,25 +161,28 @@ async fn load_receipt<'a>(
     key: &AuthorizationReceiptKey,
 ) -> AdminStoreResult<Option<CredentialMutationResult>> {
     let row: Option<(String, i64, serde_json::Value)> = sqlx::query_as("select owner_digest, config_revision, accounts_json from authorization_receipts where provider_kind=$1 and flow_digest=$2 and expires_at>now()")
-        .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|_| unavailable_receipt())?;
+        .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|source| unavailable_receipt().with_source(source))?;
     let Some((owner, revision, accounts)) = row else {
         return Ok(None);
     };
     if owner != key.owner_digest() {
         return Err(invalid_receipt());
     }
-    let [account]: [StoredAccount; 1] =
-        serde_json::from_value(accounts).map_err(|_| unavailable_receipt())?;
+    let [account]: [StoredAccount; 1] = serde_json::from_value(accounts)
+        .map_err(|source| unavailable_receipt().with_source(source))?;
     Ok(Some(CredentialMutationResult {
         config_revision: AdminRevision::new(
-            u64::try_from(revision).map_err(|_| unavailable_receipt())?,
+            u64::try_from(revision).map_err(|source| unavailable_receipt().with_source(source))?,
         )
-        .map_err(|_| unavailable_receipt())?,
+        .map_err(|source| unavailable_receipt().with_source(source))?,
         account_id: CoreProviderAccountId::new(account.account_id)
-            .map_err(|_| unavailable_receipt())?,
+            .map_err(|source| unavailable_receipt().with_source(source))?,
         credential_revision: account
             .credential_revision
-            .map(|revision| AdminRevision::new(revision).map_err(|_| unavailable_receipt()))
+            .map(|revision| {
+                AdminRevision::new(revision)
+                    .map_err(|source| unavailable_receipt().with_source(source))
+            })
             .transpose()?,
     }))
 }

backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs

@@ -213,7 +213,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
         .bind(account.next_refresh_at)
         .bind(account.enabled)
         .bind(account.concurrency_limit.map(|limit| i64::from(limit.get())))
-        .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?)
+        .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?)
         .bind(credential_state.as_str())
         .bind(account.credential_observed_at)
         .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url()))
@@ -281,6 +281,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
         .await
         .map_err(|source| postgres_unavailable("compare and swap provider credentials", source))?
         .ok_or(StoreError::Conflict {
+            source: None,
             entity: ENTITY,
             id: update.account_id,
             kind: ConflictKind::StaleRevision,
@@ -587,6 +588,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
             .await
             .map_err(|source| postgres_unavailable("recover provider account state", source))?
             .ok_or_else(|| StoreError::NotFound {
+                source: None,
                 entity: ENTITY,
                 id: command.account_id.clone(),
             })?;
@@ -659,6 +661,7 @@ async fn replace_account_group_assignments_in_transaction(
         .map_err(|source| postgres_unavailable("validate account group assignment", source))?;
         if usize::try_from(known_group_count).ok() != Some(group_ids.len()) {
             return Err(StoreError::NotFound {
+                source: None,
                 entity: "account group",
                 id: "one or more group IDs".to_owned(),
             });
@@ -757,7 +760,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
     .bind(account.next_refresh_at)
     .bind(account.enabled)
     .bind(account.concurrency_limit.map(|limit| i64::from(limit.get())))
-    .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?)
+    .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?)
     .bind(credential_state.as_str())
     .bind(account.credential_observed_at)
     .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url()))
@@ -771,6 +774,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
             .is_some_and(sqlx::error::DatabaseError::is_unique_violation)
         {
             StoreError::Conflict {
+                source: Some(error.into()),
                 entity: ENTITY,
                 id: account.id.clone(),
                 kind: ConflictKind::InvalidTransition,
@@ -780,6 +784,7 @@ pub(crate) async fn upsert_provider_account_in_transaction(
         }
     })?
     .ok_or_else(|| StoreError::Conflict {
+        source: None,
         entity: ENTITY,
         id: account.id.clone(),
         kind: ConflictKind::InvalidTransition,
@@ -850,6 +855,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
             .is_some_and(sqlx::error::DatabaseError::is_unique_violation)
         {
             StoreError::Conflict {
+                source: Some(error.into()),
                 entity: ENTITY,
                 id: update.account_id.clone(),
                 kind: ConflictKind::InvalidTransition,
@@ -859,6 +865,7 @@ pub(crate) async fn rotate_provider_account_in_transaction(
         }
     })?
     .ok_or_else(|| StoreError::Conflict {
+        source: None,
         entity: ENTITY,
         id: update.account_id.clone(),
         kind: ConflictKind::StaleRevision,
@@ -893,7 +900,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction(
     .bind(account_ids)
     .bind(enabled)
     .bind(concurrency_limit.flatten().map(|limit| i64::from(limit.get())))
-    .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|_| invalid("invalid weight"))?)
+    .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|source| invalid("invalid weight").with_source(source))?)
     .bind(outbound_proxy.is_some())
     .bind(proxy.as_ref().map(gateway_core::account::OutboundProxy::expose_url))
     .bind(proxy_id)
@@ -909,6 +916,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction(
         Ok(())
     } else {
         Err(StoreError::NotFound {
+            source: None,
             entity: ENTITY,
             id: "one or more provider account IDs".to_owned(),
         })
@@ -1018,10 +1026,10 @@ pub(crate) async fn finish_admin_transaction<T>(
             Ok(value)
         }
         Err(error) => {
-            transaction
-                .rollback()
-                .await
-                .map_err(|source| postgres_unavailable(operation, source))?;
+            let error = match transaction.rollback().await {
+                Ok(()) => error,
+                Err(cleanup) => error.with_cleanup(cleanup),
+            };
             Err(error)
         }
     }
@@ -1042,6 +1050,7 @@ pub(super) async fn import_provider_accounts_in_transaction(
         .await?;
         if current.as_ref() != Some(&binding.proxy) {
             return Err(StoreError::Conflict {
+                source: None,
                 entity: "outbound proxy",
                 id: binding.id.clone(),
                 kind: ConflictKind::StaleRevision,

backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs

@@ -446,12 +446,12 @@ pub(crate) fn account_record_from_row(
     let credentials = JsonObject::try_from_value(
         "provider_credentials_json",
         row.try_get("provider_credentials_json")
-            .map_err(|_| invalid("invalid credentials JSON"))?,
+            .map_err(|source| invalid("invalid credentials JSON").with_source(source))?,
         CREDENTIALS_MAX_BYTES,
     )?;
     let quota = row
         .try_get::<Option<serde_json::Value>, _>("provider_quota_json")
-        .map_err(|_| invalid("invalid quota JSON"))?
+        .map_err(|source| invalid("invalid quota JSON").with_source(source))?
         .map(|value| JsonObject::try_from_value("provider_quota_json", value, QUOTA_MAX_BYTES))
         .transpose()?;
     Ok(ProviderAccountRecord {
@@ -514,10 +514,10 @@ pub(crate) fn account_summary_from_row(
 ) -> StoreResult<ProviderAccountSummary> {
     let revision = row
         .try_get::<i64, _>("credential_revision")
-        .map_err(|_| invalid("invalid credential revision"))?;
+        .map_err(|source| invalid("invalid credential revision").with_source(source))?;
     let credential_state = row
         .try_get::<String, _>("credential_state")
-        .map_err(|_| invalid("invalid credential_state"))?;
+        .map_err(|source| invalid("invalid credential_state").with_source(source))?;
     let quota_access_state = parse_quota_access_state(&get::<String>(&row, "quota_access_state")?)?;
     let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?;
     let quota_access_observed_at = get::<Option<DateTime<Utc>>>(&row, "quota_access_observed_at")?;
@@ -546,7 +546,7 @@ pub(crate) fn account_summary_from_row(
         outbound_proxy: get::<Option<String>>(&row, "outbound_proxy_url")?
             .map(|url| {
                 gateway_core::account::OutboundProxy::parse(&url)
-                    .map_err(|_| invalid("invalid outbound proxy"))
+                    .map_err(|source| invalid("invalid outbound proxy").with_source(source))
             })
             .transpose()?,
         id: get(&row, "id")?,
@@ -584,7 +584,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -
 where
     T: sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type<sqlx::Postgres>,
 {
-    row.try_get(column).map_err(|_| invalid(column))
+    row.try_get(column)
+        .map_err(|source| invalid(column).with_source(source))
 }
 
 pub(crate) fn validate_object_size(
@@ -594,8 +595,9 @@ pub(crate) fn validate_object_size(
 ) -> StoreResult<()> {
     let size = serde_json::to_vec(&object.as_value())
         .map_err(|error| StoreError::InvalidData {
+            source: Some(error.into()),
             entity: ENTITY,
-            message: error.to_string(),
+            message: "JSON encoding failed".to_owned(),
         })?
         .len();
     if size > max {
@@ -606,15 +608,16 @@ pub(crate) fn validate_object_size(
 }
 
 pub(crate) fn to_i64(value: u64) -> StoreResult<i64> {
-    i64::try_from(value).map_err(|_| invalid("revision is too large"))
+    i64::try_from(value).map_err(|source| invalid("revision is too large").with_source(source))
 }
 
 pub(crate) fn to_u64(value: i64) -> StoreResult<u64> {
-    u64::try_from(value).map_err(|_| invalid("revision must be positive"))
+    u64::try_from(value).map_err(|source| invalid("revision must be positive").with_source(source))
 }
 
 pub(crate) fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/postgres/proxies.rs

@@ -82,6 +82,7 @@ fn unavailable(source: impl std::error::Error + Send + Sync + 'static) -> StoreE
 }
 fn conflict(id: &str) -> StoreError {
     StoreError::Conflict {
+        source: None,
         entity: ENTITY,
         id: id.to_owned(),
         kind: ConflictKind::InvalidTransition,
@@ -89,39 +90,61 @@ fn conflict(id: &str) -> StoreError {
 }
 fn invalid() -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: ENTITY,
         message: "invalid proxy record".to_owned(),
     }
 }
 
 fn record(row: PgRow) -> StoreResult<ProxyRecord> {
-    let success: Option<bool> = row.try_get("last_test_success").map_err(|_| invalid())?;
-    let ip: Option<String> = row.try_get("last_test_ip").map_err(|_| invalid())?;
-    let ipv4: Option<String> = row.try_get("last_test_ipv4").map_err(|_| invalid())?;
-    let ipv6: Option<String> = row.try_get("last_test_ipv6").map_err(|_| invalid())?;
-    let latency: Option<i64> = row.try_get("last_test_latency_ms").map_err(|_| invalid())?;
+    let success: Option<bool> = row
+        .try_get("last_test_success")
+        .map_err(|source| invalid().with_source(source))?;
+    let ip: Option<String> = row
+        .try_get("last_test_ip")
+        .map_err(|source| invalid().with_source(source))?;
+    let ipv4: Option<String> = row
+        .try_get("last_test_ipv4")
+        .map_err(|source| invalid().with_source(source))?;
+    let ipv6: Option<String> = row
+        .try_get("last_test_ipv6")
+        .map_err(|source| invalid().with_source(source))?;
+    let latency: Option<i64> = row
+        .try_get("last_test_latency_ms")
+        .map_err(|source| invalid().with_source(source))?;
     Ok(ProxyRecord {
-        auto_location: row.try_get("auto_location").map_err(|_| invalid())?,
+        auto_location: row
+            .try_get("auto_location")
+            .map_err(|source| invalid().with_source(source))?,
         detected_location: detected_location_from_row(&row)?,
         location: location_from_row(&row)?,
-        id: row.try_get("id").map_err(|_| invalid())?,
-        name: row.try_get("name").map_err(|_| invalid())?,
+        id: row
+            .try_get("id")
+            .map_err(|source| invalid().with_source(source))?,
+        name: row
+            .try_get("name")
+            .map_err(|source| invalid().with_source(source))?,
         proxy: OutboundProxy::parse(
             &row.try_get::<String, _>("proxy_url")
-                .map_err(|_| invalid())?,
+                .map_err(|source| invalid().with_source(source))?,
         )
-        .map_err(|_| invalid())?,
+        .map_err(|source| invalid().with_source(source))?,
         revision: AdminRevision::new(
-            u64::try_from(row.try_get::<i64, _>("revision").map_err(|_| invalid())?)
-                .map_err(|_| invalid())?,
+            u64::try_from(
+                row.try_get::<i64, _>("revision")
+                    .map_err(|source| invalid().with_source(source))?,
+            )
+            .map_err(|source| invalid().with_source(source))?,
         )
-        .map_err(|_| invalid())?,
+        .map_err(|source| invalid().with_source(source))?,
         account_count: u64::try_from(
             row.try_get::<i64, _>("account_count")
-                .map_err(|_| invalid())?,
+                .map_err(|source| invalid().with_source(source))?,
         )
-        .map_err(|_| invalid())?,
-        last_test_at: row.try_get("last_test_at").map_err(|_| invalid())?,
+        .map_err(|source| invalid().with_source(source))?,
+        last_test_at: row
+            .try_get("last_test_at")
+            .map_err(|source| invalid().with_source(source))?,
         last_test: success
             .map(|success| -> StoreResult<_> {
                 Ok(ProxyTestResult {
@@ -129,48 +152,60 @@ fn record(row: PgRow) -> StoreResult<ProxyRecord> {
                         .try_get::<sqlx::types::Json<ProxyLocationDetection>, _>(
                             "last_location_detection_json",
                         )
-                        .map_err(|_| invalid())?
+                        .map_err(|source| invalid().with_source(source))?
                         .0,
                     success,
                     latency_ms: u64::try_from(latency.ok_or_else(invalid)?)
-                        .map_err(|_| invalid())?,
-                    exit_ip: ip.map(|ip| ip.parse().map_err(|_| invalid())).transpose()?,
+                        .map_err(|source| invalid().with_source(source))?,
+                    exit_ip: ip
+                        .map(|ip| ip.parse().map_err(|source| invalid().with_source(source)))
+                        .transpose()?,
                     exit_ipv4: ipv4
-                        .map(|ip| ip.parse().map_err(|_| invalid()))
+                        .map(|ip| ip.parse().map_err(|source| invalid().with_source(source)))
                         .transpose()?,
                     exit_ipv6: ipv6
-                        .map(|ip| ip.parse().map_err(|_| invalid()))
+                        .map(|ip| ip.parse().map_err(|source| invalid().with_source(source)))
                         .transpose()?,
                     message: row
                         .try_get::<Option<String>, _>("last_test_message")
-                        .map_err(|_| invalid())?
+                        .map_err(|source| invalid().with_source(source))?
                         .unwrap_or_default(),
                 })
             })
             .transpose()?,
-        created_at: row.try_get("created_at").map_err(|_| invalid())?,
-        updated_at: row.try_get("updated_at").map_err(|_| invalid())?,
+        created_at: row
+            .try_get("created_at")
+            .map_err(|source| invalid().with_source(source))?,
+        updated_at: row
+            .try_get("updated_at")
+            .map_err(|source| invalid().with_source(source))?,
     })
 }
 
 pub(crate) fn location_from_row(
     row: &PgRow,
 ) -> StoreResult<Option<gateway_core::account::RequestLocation>> {
-    let country: Option<String> = row.try_get("location_country").map_err(|_| invalid())?;
+    let country: Option<String> = row
+        .try_get("location_country")
+        .map_err(|source| invalid().with_source(source))?;
     country
         .map(|country| {
             gateway_core::account::RequestLocation {
                 country,
-                region: row.try_get("location_region").map_err(|_| invalid())?,
-                city: row.try_get("location_city").map_err(|_| invalid())?,
+                region: row
+                    .try_get("location_region")
+                    .map_err(|source| invalid().with_source(source))?,
+                city: row
+                    .try_get("location_city")
+                    .map_err(|source| invalid().with_source(source))?,
                 timezone: row
                     .try_get::<String, _>("location_timezone")
-                    .map_err(|_| invalid())?
+                    .map_err(|source| invalid().with_source(source))?
                     .parse()
-                    .map_err(|_| invalid())?,
+                    .map_err(|source| invalid().with_source(source))?,
             }
             .normalized()
-            .map_err(|_| invalid())
+            .map_err(|source| invalid().with_source(source))
         })
         .transpose()
 }
@@ -178,10 +213,13 @@ pub(crate) fn location_from_row(
 fn detected_location_from_row(row: &PgRow) -> StoreResult<Option<DetectedProxyLocation>> {
     let detected = row
         .try_get::<Option<sqlx::types::Json<DetectedProxyLocation>>, _>("detected_location_json")
-        .map_err(|_| invalid())?
+        .map_err(|source| invalid().with_source(source))?
         .map(|value| value.0);
     if let Some(value) = &detected {
-        value.location.validate().map_err(|_| invalid())?;
+        value
+            .location
+            .validate()
+            .map_err(|source| invalid().with_source(source))?;
     }
     Ok(detected)
 }
@@ -191,7 +229,7 @@ pub(crate) fn effective_location_from_row(
 ) -> StoreResult<Option<gateway_core::account::RequestLocation>> {
     if row
         .try_get::<Option<bool>, _>("auto_location")
-        .map_err(|_| invalid())?
+        .map_err(|source| invalid().with_source(source))?
         .unwrap_or(false)
     {
         Ok(detected_location_from_row(row)?.map(|value| value.location))
@@ -225,12 +263,15 @@ async fn save_test(
 ) -> StoreResult<()> {
     let detected = current.detected_location_after_test(&result);
     if let Some(value) = &detected {
-        value.location.validate().map_err(|_| invalid())?;
+        value
+            .location
+            .validate()
+            .map_err(|source| invalid().with_source(source))?;
     }
     sqlx::query("update outbound_proxies set last_test_at = now(), last_test_success = $2, last_test_latency_ms = $3,
         last_test_ip = $4, last_test_ipv4 = $5, last_test_ipv6 = $6, last_test_message = $7,
         last_location_detection_json = $8, detected_location_json = $9 where id = $1")
-        .bind(&current.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|_| invalid())?)
+        .bind(&current.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|source| invalid().with_source(source))?)
         .bind(result.exit_ip.map(|ip| ip.to_string())).bind(result.exit_ipv4.map(|ip| ip.to_string()))
         .bind(result.exit_ipv6.map(|ip| ip.to_string())).bind(result.message)
         .bind(sqlx::types::Json(result.location)).bind(detected.map(sqlx::types::Json))
@@ -244,7 +285,9 @@ async fn save_location(
     location: Option<&gateway_core::account::RequestLocation>,
 ) -> StoreResult<()> {
     if let Some(location) = location {
-        location.validate().map_err(|_| invalid())?;
+        location
+            .validate()
+            .map_err(|source| invalid().with_source(source))?;
     }
     sqlx::query("update outbound_proxies set location_country = $2, location_region = $3, location_city = $4, location_timezone = $5 where id = $1")
         .bind(id)
@@ -310,12 +353,13 @@ pub(crate) async fn resolve_proxy_selection(
             .await
             .map_err(unavailable)?
             .ok_or_else(|| StoreError::NotFound {
+                source: None,
                 entity: ENTITY,
                 id: id.clone(),
             })?;
             Ok((
                 Some(id.clone()),
-                Some(OutboundProxy::parse(&value).map_err(|_| invalid())?),
+                Some(OutboundProxy::parse(&value).map_err(|source| invalid().with_source(source))?),
             ))
         }
     }
@@ -423,6 +467,7 @@ impl ProxyStore for PgProxyRepository {
         .map_err(|source| store_error(unavailable(source)))?
         .ok_or_else(|| {
             store_error(StoreError::NotFound {
+                source: None,
                 entity: ENTITY,
                 id: query.proxy_id.clone(),
             })
@@ -443,15 +488,15 @@ impl ProxyStore for PgProxyRepository {
         .into_iter()
         .map(|row| {
             Ok(ProxyAccountRef {
-                id: row.try_get("id").map_err(|_| invalid())?,
-                name: row.try_get("name").map_err(|_| invalid())?,
-                email: row.try_get("email").map_err(|_| invalid())?,
-                provider_kind: row.try_get("provider_kind").map_err(|_| invalid())?,
-                authentication_kind: row.try_get("authentication_kind").map_err(|_| invalid())?,
-                plan_type: row.try_get("plan_type").map_err(|_| invalid())?,
+                id: row.try_get("id").map_err(|source| invalid().with_source(source))?,
+                name: row.try_get("name").map_err(|source| invalid().with_source(source))?,
+                email: row.try_get("email").map_err(|source| invalid().with_source(source))?,
+                provider_kind: row.try_get("provider_kind").map_err(|source| invalid().with_source(source))?,
+                authentication_kind: row.try_get("authentication_kind").map_err(|source| invalid().with_source(source))?,
+                plan_type: row.try_get("plan_type").map_err(|source| invalid().with_source(source))?,
                 plan_type_display: None,
                 groups: Vec::new(),
-                enabled: row.try_get("enabled").map_err(|_| invalid())?,
+                enabled: row.try_get("enabled").map_err(|source| invalid().with_source(source))?,
             })
         })
         .collect::<StoreResult<Vec<_>>>()
@@ -586,6 +631,7 @@ impl ProxyStore for PgProxyRepository {
             .map_err(|source| store_error(unavailable(source)))?
             .ok_or_else(|| {
                 store_error(StoreError::NotFound {
+                    source: None,
                     entity: ENTITY,
                     id: id.to_owned(),
                 })
@@ -815,11 +861,11 @@ impl ProxyStore for PgProxyRepository {
             .map_err(store_error)?;
         if current.revision != revision {
             // Drop 只排队回滚，返回冲突前需释放事务锁，避免误挡紧接着的编辑
-            transaction
-                .rollback()
-                .await
-                .map_err(|source| store_error(unavailable(source)))?;
-            return Err(store_error(conflict(id)));
+            let error = match transaction.rollback().await {
+                Ok(()) => conflict(id),
+                Err(cleanup) => conflict(id).with_cleanup(cleanup),
+            };
+            return Err(store_error(error));
         }
         save_test(&mut transaction, &current, result)
             .await

backend/crates/gateway-store/src/postgres/runtime_settings.rs

@@ -206,6 +206,7 @@ impl RuntimeSettingsUpdate {
                 })
         {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "runtime settings",
                 message: "settings violate the frozen runtime constraints".to_owned(),
             });
@@ -276,6 +277,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
     .await
     .map_err(|source| postgres_unavailable("load runtime settings", source))?
     .ok_or_else(|| StoreError::NotFound {
+        source: None,
         entity: "runtime settings",
         id: "1".to_owned(),
     })?;
@@ -453,6 +455,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
     .await
     .map_err(|source| postgres_unavailable("load runtime settings in transaction", source))?
     .ok_or_else(|| StoreError::NotFound {
+        source: None,
         entity: "runtime settings",
         id: "1".to_owned(),
     })?;
@@ -568,6 +571,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
     .await
     .map_err(|source| postgres_unavailable("update runtime settings in transaction", source))?
     .ok_or_else(|| StoreError::NotFound {
+        source: None,
         entity: "runtime settings",
         id: "1".to_owned(),
     })?;
@@ -587,6 +591,7 @@ pub(crate) async fn bump_config_revision_in_transaction(
     .await
     .map_err(|source| postgres_unavailable("bump config revision in transaction", source))?
     .ok_or_else(|| StoreError::NotFound {
+        source: None,
         entity: "runtime settings",
         id: "1".to_owned(),
     })?;
@@ -715,6 +720,7 @@ fn to_u32(value: i64) -> StoreResult<u32> {
 
 fn invalid_location() -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "runtime settings",
         message: "request location is invalid".to_owned(),
     }
@@ -722,6 +728,7 @@ fn invalid_location() -> StoreError {
 
 fn invalid_request_profile() -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "runtime settings",
         message: "Provider request profile key is invalid".to_owned(),
     }
@@ -729,6 +736,7 @@ fn invalid_request_profile() -> StoreError {
 
 fn invalid_numeric() -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "runtime settings",
         message: "numeric field is outside its supported range".to_owned(),
     }

backend/crates/gateway-store/src/postgres/snapshot.rs

@@ -137,6 +137,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository {
         .await
         .map_err(|source| postgres_unavailable("read current config revision", source))?
         .ok_or_else(|| StoreError::NotFound {
+            source: None,
             entity: "runtime settings",
             id: "1".to_owned(),
         })?;
@@ -288,6 +289,7 @@ async fn load_settings(
     .await
     .map_err(|source| postgres_unavailable("load snapshot settings", source))?
     .ok_or_else(|| StoreError::NotFound {
+        source: None,
         entity: "runtime settings",
         id: "1".to_owned(),
     })?;
@@ -375,7 +377,8 @@ async fn load_account_groups(
             Ok(SnapshotAccountGroupData {
                 fast_mode: FastMode::parse(&fast_mode)
                     .ok_or_else(|| invalid("invalid fast_mode"))?,
-                id: AccountGroupId::new(id).map_err(|_| invalid("invalid account group id"))?,
+                id: AccountGroupId::new(id)
+                    .map_err(|source| invalid("invalid account group id").with_source(source))?,
                 name,
                 enabled,
             })
@@ -424,7 +427,7 @@ async fn load_group_memberships(
         .map(|(group_id, account_id)| {
             Ok(SnapshotGroupMembershipData {
                 group_id: AccountGroupId::new(group_id)
-                    .map_err(|_| invalid("invalid membership group id"))?,
+                    .map_err(|source| invalid("invalid membership group id").with_source(source))?,
                 account_id,
             })
         })
@@ -436,15 +439,18 @@ fn revision_from_i64(value: i64) -> StoreResult<Revision> {
 }
 
 fn to_u64(value: i64) -> StoreResult<u64> {
-    u64::try_from(value).map_err(|_| invalid("numeric snapshot field is negative"))
+    u64::try_from(value)
+        .map_err(|source| invalid("numeric snapshot field is negative").with_source(source))
 }
 
 fn to_u32(value: i64) -> StoreResult<u32> {
-    u32::try_from(value).map_err(|_| invalid("numeric snapshot field is outside u32"))
+    u32::try_from(value)
+        .map_err(|source| invalid("numeric snapshot field is outside u32").with_source(source))
 }
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "runtime snapshot",
         message: message.to_owned(),
     }
@@ -459,8 +465,9 @@ fn decode_request_profiles(
         .into_iter()
         .map(|(kind, document)| {
             Ok((
-                gateway_core::routing::ProviderKind::new(kind)
-                    .map_err(|_| invalid("invalid request profile provider"))?,
+                gateway_core::routing::ProviderKind::new(kind).map_err(|source| {
+                    invalid("invalid request profile provider").with_source(source)
+                })?,
                 gateway_core::account::OpaqueProviderData::new(document),
             ))
         })

backend/crates/gateway-store/src/redis/auth.rs

@@ -284,6 +284,7 @@ fn decode_session(value: &str) -> StoreResult<AuthSessionRecord> {
 
 fn auth_invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "authentication state",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/redis/client_admission.rs

@@ -280,7 +280,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository {
         request.validate()?;
         let keys = self.keys(&request.client_api_key_ref)?;
         let lease_ttl_ms = u64::try_from(request.lease_ttl.as_millis())
-            .map_err(|_| invalid("lease TTL is too large"))?;
+            .map_err(|source| invalid("lease TTL is too large").with_source(source))?;
         let mut connection = self.connection.clone();
         let code = Script::new(ADMIT_SCRIPT)
             .key(&keys[0])
@@ -462,7 +462,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository {
                     CoreAdmissionDecision::Rejected(CoreAdmissionRejection::ConcurrencyLimited)
                 }
             })
-            .map_err(|_| CoreAdmissionError)
+            .map_err(|source| CoreAdmissionError(Some(source.into())))
         })
     }
 
@@ -474,7 +474,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository {
         Box::pin(async move {
             self.release_client_request(client_api_key_id.as_str(), model_request_id.as_str())
                 .await
-                .map_err(|_| CoreAdmissionError)
+                .map_err(|source| CoreAdmissionError(Some(source.into())))
         })
     }
 
@@ -508,13 +508,14 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository {
                 restored_recent_requests: restored.restored_recent_requests,
                 restored_running_requests: restored.restored_running_requests,
             })
-            .map_err(|_| CoreAdmissionError)
+            .map_err(|source| CoreAdmissionError(Some(source.into())))
         })
     }
 }
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "client admission",
         message: message.to_owned(),
     }
@@ -525,18 +526,19 @@ fn validate_recovery_request_id(model_request_id: &str) -> StoreResult<()> {
 }
 
 fn redis_duration_millis(duration: Duration) -> StoreResult<u64> {
-    let milliseconds =
-        u64::try_from(duration.as_millis()).map_err(|_| invalid("lease TTL is too large"))?;
+    let milliseconds = u64::try_from(duration.as_millis())
+        .map_err(|source| invalid("lease TTL is too large").with_source(source))?;
     redis_integer(milliseconds, "lease TTL")
 }
 
 fn redis_timestamp_millis(timestamp: DateTime<Utc>, field: &str) -> StoreResult<u64> {
-    let milliseconds = u64::try_from(timestamp.timestamp_millis()).map_err(|_| invalid(field))?;
+    let milliseconds = u64::try_from(timestamp.timestamp_millis())
+        .map_err(|source| invalid(field).with_source(source))?;
     redis_integer(milliseconds, field)
 }
 
 fn redis_len(length: usize, field: &str) -> StoreResult<u64> {
-    let value = u64::try_from(length).map_err(|_| invalid(field))?;
+    let value = u64::try_from(length).map_err(|source| invalid(field).with_source(source))?;
     redis_integer(value, field)
 }
 

backend/crates/gateway-store/src/redis/coordination_buffer.rs

@@ -32,10 +32,14 @@ pub struct BufferedClientAdmissionPort {
 
 impl BufferedClientAdmissionPort {
     #[must_use]
-    pub fn new(inner: Arc<dyn ClientAdmissionPort>) -> (Self, ClientAdmissionReleaseWriter) {
+    pub fn new(
+        inner: Arc<dyn ClientAdmissionPort>,
+        diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
+    ) -> (Self, ClientAdmissionReleaseWriter) {
         Self::with_capacity(
             inner,
             NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"),
+            diagnostics,
         )
     }
 
@@ -43,6 +47,7 @@ impl BufferedClientAdmissionPort {
     pub fn with_capacity(
         inner: Arc<dyn ClientAdmissionPort>,
         capacity: NonZeroUsize,
+        diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
     ) -> (Self, ClientAdmissionReleaseWriter) {
         let (sender, receiver) = mpsc::channel(capacity.get());
         (
@@ -52,6 +57,7 @@ impl BufferedClientAdmissionPort {
             },
             ClientAdmissionReleaseWriter {
                 inner,
+                diagnostics,
                 receiver: Mutex::new(receiver),
             },
         )
@@ -127,6 +133,7 @@ struct AdmissionRelease {
 }
 
 pub struct ClientAdmissionReleaseWriter {
+    diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
     inner: Arc<dyn ClientAdmissionPort>,
     receiver: Mutex<mpsc::Receiver<AdmissionRelease>>,
 }
@@ -138,7 +145,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter {
             loop {
                 let release = tokio::select! {
                     () = cancellation.cancelled() => {
-                        drain_admission_releases(&mut receiver, self.inner.as_ref()).await;
+                        drain_admission_releases(&mut receiver, self.inner.as_ref(), self.diagnostics.as_ref()).await;
                         return Ok(());
                     },
                     release = receiver.recv() => release,
@@ -148,13 +155,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter {
                         "client admission release queue closed",
                     ));
                 };
-                if let Err(error) = self
-                    .inner
-                    .release(&release.client_api_key_id, &release.model_request_id)
-                    .await
-                {
-                    tracing::warn!(%error, "Client admission 后台释放失败，依赖租约 TTL 收敛");
-                }
+                release_admission(self.inner.as_ref(), self.diagnostics.as_ref(), release).await;
             }
         })
     }
@@ -163,21 +164,46 @@ impl DaemonTask for ClientAdmissionReleaseWriter {
 async fn drain_admission_releases(
     receiver: &mut mpsc::Receiver<AdmissionRelease>,
     port: &dyn ClientAdmissionPort,
+    diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics,
 ) {
     receiver.close();
     let started_at = Instant::now();
     while let Some(release) = receiver.recv().await {
         let remaining = SHUTDOWN_DRAIN_TIMEOUT.saturating_sub(started_at.elapsed());
         if remaining.is_zero()
-            || tokio::time::timeout(
-                remaining,
-                port.release(&release.client_api_key_id, &release.model_request_id),
-            )
-            .await
-            .is_err()
+            || tokio::time::timeout(remaining, release_admission(port, diagnostics, release))
+                .await
+                .is_err()
         {
             tracing::warn!("Client admission 关闭排空超时，剩余租约依赖 TTL 收敛");
             break;
         }
     }
 }
+
+async fn release_admission(
+    port: &dyn ClientAdmissionPort,
+    diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics,
+    release: AdmissionRelease,
+) {
+    if let Err(source) = port
+        .release(&release.client_api_key_id, &release.model_request_id)
+        .await
+    {
+        let mut failure = gateway_core::diagnostics::OperationalFailure::new(
+            "store",
+            "release_client_admission",
+            "admission_unavailable",
+            "Client admission release failed; lease TTL remains active",
+        );
+        failure.correlation_id = Some(release.model_request_id.as_str().to_owned());
+        failure.details =
+            gateway_core::error::ErrorDetails::capture(Some(&source.into()), None, false);
+        if diagnostics.record_failure(failure).await.is_err() {
+            tracing::warn!(
+                request_id = release.model_request_id.as_str(),
+                "admission diagnostic could not be recorded"
+            );
+        }
+    }
+}

backend/crates/gateway-store/src/redis/credential_cooldown.rs

@@ -260,10 +260,10 @@ impl RedisCredentialCooldownRepository {
         }
         let revision = revision
             .parse::<u64>()
-            .map_err(|_| invalid("cached cooldown revision is invalid"))?;
+            .map_err(|source| invalid("cached cooldown revision is invalid").with_source(source))?;
         let until_ms = until_ms
             .parse::<i64>()
-            .map_err(|_| invalid("cached cooldown expiry is invalid"))?;
+            .map_err(|source| invalid("cached cooldown expiry is invalid").with_source(source))?;
         let cooldown_until = DateTime::from_timestamp_millis(until_ms)
             .ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
         let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit);
@@ -350,7 +350,7 @@ impl RedisCredentialCooldownRepository {
                     account_id,
                     gateway_admin::model::accounts::AccountFreeze {
                         credential_revision: gateway_admin::model::Revision::new(revision.get())
-                            .map_err(|_| invalid("freeze revision"))?,
+                            .map_err(|source| invalid("freeze revision").with_source(source))?,
                         until,
                         generation,
                         requires_probe: kind.requires_probe(),
@@ -401,7 +401,7 @@ impl RedisCredentialCooldownRepository {
             .map_err(|source| redis_unavailable("read capacity peak in-flight", source))?;
         peak.map(u32::try_from)
             .transpose()
-            .map_err(|_| invalid("capacity peak in-flight is invalid"))
+            .map_err(|source| invalid("capacity peak in-flight is invalid").with_source(source))
     }
 }
 
@@ -741,6 +741,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "credential cooldown",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/redis/credential_leases.rs

@@ -247,6 +247,7 @@ impl CredentialLeaseGuard {
             .grant
             .clone()
             .ok_or_else(|| crate::StoreError::InvalidData {
+                source: None,
                 entity: "credential lease",
                 message: "lease has already been released".to_owned(),
             })?;
@@ -385,9 +386,9 @@ impl RedisCredentialLeaseRepository {
             .invoke_async::<String>(&mut connection)
             .await
             .map_err(|source| redis_unavailable("advance provider scheduling cursor", source))?;
-        cursor
-            .parse::<u64>()
-            .map_err(|_| invalid("Redis returned an invalid scheduling cursor"))
+        cursor.parse::<u64>().map_err(|source| {
+            invalid("Redis returned an invalid scheduling cursor").with_source(source)
+        })
     }
 
     fn keys(&self, request: &CredentialLeaseRequest) -> StoreResult<[String; 3]> {
@@ -486,9 +487,9 @@ impl RedisCredentialLeaseRepository {
             .invoke_async(&mut connection)
             .await
             .map_err(|source| redis_unavailable("load credential runtime signal", source))?;
-        let in_flight = in_flight
-            .parse::<u32>()
-            .map_err(|_| invalid("Redis returned an invalid in-flight count"))?;
+        let in_flight = in_flight.parse::<u32>().map_err(|source| {
+            invalid("Redis returned an invalid in-flight count").with_source(source)
+        })?;
         let last_started_at = if last_started == "0" {
             None
         } else {
@@ -830,7 +831,7 @@ struct LeaseAttempt {
 fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult<CredentialLeaseGrant> {
     let fence = fence
         .parse::<u64>()
-        .map_err(|_| invalid("Redis returned an invalid fencing token"))?;
+        .map_err(|source| invalid("Redis returned an invalid fencing token").with_source(source))?;
     Ok(CredentialLeaseGrant {
         lease_id,
         fencing_token: Revision::new(fence)?,
@@ -841,7 +842,7 @@ fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult<Credent
 fn timestamp(value: &str) -> StoreResult<DateTime<Utc>> {
     let milliseconds = value
         .parse::<i64>()
-        .map_err(|_| invalid("Redis returned an invalid timestamp"))?;
+        .map_err(|source| invalid("Redis returned an invalid timestamp").with_source(source))?;
     DateTime::from_timestamp_millis(milliseconds)
         .ok_or_else(|| invalid("Redis returned an out-of-range timestamp"))
 }
@@ -850,7 +851,7 @@ fn duration(value: &str) -> StoreResult<Duration> {
     value
         .parse::<u64>()
         .map(Duration::from_millis)
-        .map_err(|_| invalid("Redis returned an invalid retry interval"))
+        .map_err(|source| invalid("Redis returned an invalid retry interval").with_source(source))
 }
 
 fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) -> StoreResult<()> {
@@ -863,11 +864,13 @@ fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) ->
 
 fn duration_millis(value: Duration) -> StoreResult<u64> {
     supported_duration(value, true, "duration")?;
-    u64::try_from(value.as_millis()).map_err(|_| invalid("duration is too large"))
+    u64::try_from(value.as_millis())
+        .map_err(|source| invalid("duration is too large").with_source(source))
 }
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "credential lease",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/redis/credential_state.rs

@@ -446,6 +446,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository {
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "credential state cache",
         message: message.to_owned(),
     }
@@ -453,6 +454,7 @@ fn invalid(message: &str) -> StoreError {
 
 fn catalog_invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "provider catalog cache",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/redis/mod.rs

@@ -48,6 +48,7 @@ pub(crate) fn namespace(value: &str) -> StoreResult<String> {
             .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.'))
     {
         return Err(StoreError::InvalidData {
+            source: None,
             entity: "Redis namespace",
             message: "namespace contains unsupported characters".to_owned(),
         });

backend/crates/gateway-store/src/redis/oauth_pending.rs

@@ -322,6 +322,7 @@ fn ttl_millis(ttl: Duration) -> StoreResult<u64> {
         .ok()
         .filter(|value| *value > 0 && *value <= MAX_REDIS_EXACT_INTEGER)
         .ok_or_else(|| StoreError::InvalidData {
+            source: None,
             entity: "OAuth pending flow",
             message: "TTL is outside the supported range".to_owned(),
         })

backend/crates/gateway-store/src/redis/runtime_change.rs

@@ -73,9 +73,9 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository {
             .await
             .map_err(|source| redis_unavailable("subscribe runtime changes", source))?;
         let stream = pubsub.into_on_message().map(|message| {
-            let payload = message
-                .get_payload::<String>()
-                .map_err(|_| invalid("runtime change payload is not UTF-8"))?;
+            let payload = message.get_payload::<String>().map_err(|source| {
+                invalid("runtime change payload is not UTF-8").with_source(source)
+            })?;
             let wire: RuntimeChangeWire =
                 serde_json::from_str(&payload).map_err(|error| invalid(&error.to_string()))?;
             wire.try_into()
@@ -154,6 +154,7 @@ impl TryFrom<RuntimeChangeWire> for RuntimeChange {
 
 fn invalid(message: &str) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "runtime change",
         message: message.to_owned(),
     }

backend/crates/gateway-store/src/redis/worker_lease.rs

@@ -52,7 +52,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard {
                 current,
             )
             .await
-            .map_err(|_| WorkerLeaseError::safe("worker lease renewal failed"))?
+            .map_err(|source| {
+                WorkerLeaseError::safe("worker lease renewal failed").with_source(source)
+            })?
             .ok_or_else(|| WorkerLeaseError::safe("worker lease was lost"))?;
             self.grant = Some(renewed);
             Ok(())
@@ -72,7 +74,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard {
             CredentialLeaseRepository::release_credential_lease(&repository, &request, &grant)
                 .await
                 .map(|_| ())
-                .map_err(|_| WorkerLeaseError::safe("worker lease release failed"))
+                .map_err(|source| {
+                    WorkerLeaseError::safe("worker lease release failed").with_source(source)
+                })
         })
     }
 }
@@ -94,7 +98,9 @@ impl WorkerLeaderLeasePort for RedisWorkerLeaderLeasePort {
                 &lease_request,
             )
             .await
-            .map_err(|_| WorkerLeaseError::safe("worker lease acquisition failed"))?;
+            .map_err(|source| {
+                WorkerLeaseError::safe("worker lease acquisition failed").with_source(source)
+            })?;
             let Some(grant) = grant else {
                 return Ok(WorkerLeaseAcquisition::Busy { retry_after: None });
             };

backend/crates/gateway-store/src/value.rs

@@ -32,20 +32,87 @@ pub enum StoreError {
         source: Option<gateway_core::error::ErrorSource>,
     },
     #[error("{entity} {id} was not found")]
-    NotFound { entity: &'static str, id: String },
+    NotFound {
+        entity: &'static str,
+        id: String,
+        source: Option<gateway_core::error::ErrorSource>,
+    },
     #[error("store conflict for {entity} {id}: {kind:?}")]
     Conflict {
         entity: &'static str,
         id: String,
         kind: ConflictKind,
+        source: Option<gateway_core::error::ErrorSource>,
     },
     #[error("invalid persisted {entity}: {message}")]
     InvalidData {
         entity: &'static str,
         message: String,
+        source: Option<gateway_core::error::ErrorSource>,
     },
 }
 
+impl StoreError {
+    /// 本地校验可没有来源；转换已有失败时附加真实原因
+    #[must_use]
+    pub(crate) fn with_source(
+        mut self,
+        error: impl Into<gateway_core::error::ErrorSource>,
+    ) -> Self {
+        let error = error.into();
+        match &mut self {
+            Self::Unavailable { source, .. }
+            | Self::InvalidData { source, .. }
+            | Self::Conflict { source, .. }
+            | Self::NotFound { source, .. } => *source = Some(error),
+        }
+        self
+    }
+
+    pub(crate) fn with_cleanup(
+        mut self,
+        cleanup: impl Into<gateway_core::error::ErrorSource>,
+    ) -> Self {
+        let source = match &mut self {
+            Self::Unavailable { source, .. }
+            | Self::InvalidData { source, .. }
+            | Self::Conflict { source, .. }
+            | Self::NotFound { source, .. } => source,
+        };
+        *source = Some(gateway_core::error::ErrorSource::cleanup(
+            source.take(),
+            cleanup,
+        ));
+        self
+    }
+
+    fn admin_kind(&self) -> AdminStoreErrorKind {
+        match self {
+            Self::NotFound { .. } => AdminStoreErrorKind::NotFound,
+            Self::Conflict {
+                kind: ConflictKind::StaleRevision,
+                ..
+            } => AdminStoreErrorKind::StaleRevision,
+            Self::Conflict {
+                kind: ConflictKind::DuplicateName,
+                ..
+            } => AdminStoreErrorKind::DuplicateName,
+            Self::Conflict { .. } => AdminStoreErrorKind::Conflict,
+            Self::InvalidData { .. } => AdminStoreErrorKind::Invalid,
+            Self::Unavailable { .. } => AdminStoreErrorKind::Unavailable,
+        }
+    }
+
+    fn core_kind(&self) -> gateway_core::error::StoreErrorKind {
+        use gateway_core::error::StoreErrorKind;
+        match self {
+            Self::Unavailable { .. } => StoreErrorKind::Unavailable,
+            Self::Conflict { .. } => StoreErrorKind::Conflict,
+            Self::NotFound { .. } | Self::InvalidData { .. } => StoreErrorKind::InvalidData,
+        }
+    }
+}
+
 pub type StoreResult<T> = Result<T, StoreError>;
 pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult<Revision> {
     Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error))
@@ -88,21 +155,7 @@ pub(crate) fn mutation_audit(
 }
 
 pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> AdminStoreError {
-    let kind = match error {
-        StoreError::NotFound { .. } => AdminStoreErrorKind::NotFound,
-        StoreError::Conflict {
-            kind: ConflictKind::StaleRevision,
-            ..
-        } => AdminStoreErrorKind::StaleRevision,
-        StoreError::Conflict {
-            kind: ConflictKind::DuplicateName,
-            ..
-        } => AdminStoreErrorKind::DuplicateName,
-        StoreError::Conflict { .. } => AdminStoreErrorKind::Conflict,
-        StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid,
-        StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable,
-    };
-    AdminStoreError::new(kind, resource, "store operation failed").with_source(error)
+    AdminStoreError::new(error.admin_kind(), resource, "store operation failed").with_source(error)
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -113,6 +166,7 @@ impl Revision {
         NonZeroU64::new(value)
             .map(Self)
             .ok_or_else(|| StoreError::InvalidData {
+                source: None,
                 entity: "revision",
                 message: "must be greater than zero".to_owned(),
             })
@@ -160,6 +214,7 @@ impl FromStr for DecimalAmount {
             });
         if !valid {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity: "decimal amount",
                 message: "expected a non-negative numeric(20,10) value".to_owned(),
             });
@@ -190,18 +245,21 @@ impl JsonObject {
     ) -> StoreResult<Self> {
         let serialized_bytes = serde_json::to_vec(&value)
             .map_err(|error| StoreError::InvalidData {
+                source: Some(error.into()),
                 entity,
-                message: error.to_string(),
+                message: "JSON encoding failed".to_owned(),
             })?
             .len();
         let Value::Object(fields) = value else {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity,
                 message: "top-level JSON value must be an object".to_owned(),
             });
         };
         if serialized_bytes > max_serialized_bytes {
             return Err(StoreError::InvalidData {
+                source: None,
                 entity,
                 message: format!("serialized JSON exceeds {max_serialized_bytes} bytes"),
             });
@@ -233,6 +291,7 @@ pub(crate) fn require_nonempty(
 ) -> StoreResult<()> {
     if value.trim().is_empty() {
         Err(StoreError::InvalidData {
+            source: None,
             entity,
             message: format!("{field} must not be empty"),
         })
@@ -276,11 +335,5 @@ pub(crate) fn provider_unavailable(
 }
 
 pub(crate) fn core_store_error(error: StoreError) -> gateway_core::error::StoreError {
-    use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind};
-    let kind = match error {
-        StoreError::Unavailable { .. } => StoreErrorKind::Unavailable,
-        StoreError::Conflict { .. } => StoreErrorKind::Conflict,
-        StoreError::NotFound { .. } | StoreError::InvalidData { .. } => StoreErrorKind::InvalidData,
-    };
-    CoreStoreError::caused_by(kind, error)
+    gateway_core::error::StoreError::caused_by(error.core_kind(), error)
 }

backend/crates/gateway-store/src/workers.rs

@@ -181,6 +181,7 @@ pub(crate) fn worker_definition_error(
     error: gateway_core::task::WorkerDefinitionError,
 ) -> StoreError {
     StoreError::InvalidData {
+        source: None,
         entity: "store worker plan",
         message: error.to_string(),
     }
@@ -202,7 +203,9 @@ impl ScheduledTask for StaleModelRequestRecoveryTask {
             )
             .await
             .map(|_| ())
-            .map_err(|_| WorkerTaskError::safe("stale request recovery failed"))
+            .map_err(|source| {
+                WorkerTaskError::safe("stale request recovery failed").with_source(source)
+            })
         })
     }
 }

backend/crates/gateway-store/tests/main.rs

@@ -6,3 +6,16 @@ mod config;
 mod postgres;
 mod redis;
 mod support;
+
+#[derive(Default)]
+struct RecordingDiagnostics(std::sync::Mutex<Vec<gateway_core::diagnostics::OperationalFailure>>);
+#[async_trait::async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics {
+    async fn record_failure(
+        &self,
+        failure: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), gateway_core::error::StoreError> {
+        self.0.lock().unwrap().push(failure);
+        Ok(())
+    }
+}

backend/crates/gateway-store/tests/postgres/client_budgets.rs

@@ -1209,3 +1209,45 @@ async fn timezone_cutover_preserves_open_windows_and_resumes_without_overlap() {
     );
     database.close().await;
 }
+
+#[tokio::test]
+async fn failed_settlement_retains_native_database_code_through_the_next_admission() {
+    let Some(database) = TestDatabase::create("budget_native_failure").await else {
+        return;
+    };
+    sqlx::query("alter table client_api_keys rename to hidden_test_keys")
+        .execute(&database.pool)
+        .await
+        .unwrap();
+    let budgets = PgClientBudgetStore::new(database.pool.clone());
+    let error = budgets
+        .settle(charge("missing-key", "native-error", "1"))
+        .await
+        .unwrap_err();
+    fn assert_native_code(error: &(dyn std::error::Error + 'static)) {
+        let mut source = Some(error);
+        while let Some(error) = source {
+            if let Some(sqlx::Error::Database(error)) = error.downcast_ref::<sqlx::Error>() {
+                assert_eq!(error.code().as_deref(), Some("42P01"));
+                return;
+            }
+            source = error.source();
+        }
+        panic!("native database failure was lost");
+    }
+    assert_native_code(&error);
+    let retried = budgets.admit(key_id("missing-key")).await.unwrap_err();
+    assert_eq!(
+        retried.kind(),
+        GatewayErrorKind::ProviderInfrastructureUnavailable
+    );
+    assert_native_code(&retried);
+    assert!(
+        retried
+            .error_details()
+            .unwrap()
+            .as_str()
+            .contains("client_api_keys")
+    );
+    database.close().await;
+}

backend/crates/gateway-store/tests/postgres/execution_buffer.rs

@@ -969,3 +969,159 @@ async fn zero_attempt_postgres_write_failures_are_not_retried_and_do_not_stop_th
     assert_eq!(recovered.trace, None);
     database.close().await;
 }
+
+#[async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for RecordingStore {
+    async fn record_failure(
+        &self,
+        _: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), StoreError> {
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl gateway_core::diagnostics::OperationalDiagnostics for BlockingStore {
+    async fn record_failure(
+        &self,
+        _: gateway_core::diagnostics::OperationalFailure,
+    ) -> Result<(), StoreError> {
+        Ok(())
+    }
+}
+
+#[tokio::test]
+async fn operational_failure_uses_the_bounded_queue_without_creating_a_model_request() {
+    use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure};
+    use gateway_core::error::{ErrorDetails, ErrorSource};
+    let Some(database) = TestDatabase::create("operational_details").await else {
+        return;
+    };
+    let inner = Arc::new(PgExecutionStore::new(database.pool.clone()));
+    let (store, writer) =
+        BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(1).unwrap());
+    let mut failure =
+        OperationalFailure::new("admin", "http_request", "unavailable", "safe summary");
+    failure.correlation_id = Some("admin-request-123".to_owned());
+    failure.details = ErrorDetails::capture(
+        Some(&ErrorSource::new(std::io::Error::other(
+            "PRIVATE_STORE_CAUSE",
+        ))),
+        None,
+        false,
+    );
+    let occurred_at = failure.occurred_at;
+    store.record_failure(failure).await.unwrap();
+    store
+        .record_failure(OperationalFailure::new(
+            "worker",
+            "run",
+            "worker_failed",
+            "safe",
+        ))
+        .await
+        .unwrap();
+    assert_eq!(store.stats().queued_items, 1);
+    assert_eq!(store.stats().dropped_total, 1);
+    let cancellation = CancellationToken::new();
+    cancellation.cancel();
+    writer.run(cancellation).await.unwrap();
+    let row: (
+        Option<String>,
+        String,
+        String,
+        chrono::DateTime<chrono::Utc>,
+    ) = sqlx::query_as(
+        "select model_request_id, message, error_details, created_at
+         from ops_events where component = 'admin'",
+    )
+    .fetch_one(&database.pool)
+    .await
+    .unwrap();
+    assert!(row.0.is_none());
+    let message: serde_json::Value = serde_json::from_str(&row.1).unwrap();
+    assert_eq!(message["correlationId"], "admin-request-123");
+    assert!(!row.1.contains("PRIVATE_STORE_CAUSE"));
+    assert!(row.2.contains("PRIVATE_STORE_CAUSE"));
+    assert_eq!(
+        row.3.timestamp_micros(),
+        chrono::DateTime::<chrono::Utc>::from(occurred_at).timestamp_micros()
+    );
+    let requests: i64 = sqlx::query_scalar("select count(*) from model_requests")
+        .fetch_one(&database.pool)
+        .await
+        .unwrap();
+    assert_eq!(requests, 0);
+    database.close().await;
+}
+
+#[tokio::test]
+async fn progress_congestion_preserves_failure_reserve_without_waiting() {
+    use futures::FutureExt as _;
+    use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure};
+    use gateway_core::error::{ErrorDetails, ErrorSource};
+    let Some(database) = TestDatabase::create("failure_reserve").await else {
+        return;
+    };
+    let inner = Arc::new(PgExecutionStore::new(database.pool.clone()));
+    let mut request_ids = Vec::new();
+    for index in 0..4 {
+        let request = accepted_request(&format!("req_reserved_failure_{index}"));
+        request_ids.push(request.id.clone());
+        inner.create_model_request(request).await.unwrap();
+    }
+    let request_id = &request_ids[0];
+    let (store, writer) =
+        BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(4).unwrap());
+    // 不启动消费者，确定性地填满普通额度；每次调用必须首次 poll 就返回
+    for (request_id, status) in request_ids.iter().zip([200, 201, 202, 203]) {
+        store
+            .record_client_status(request_id, status)
+            .now_or_never()
+            .expect("enqueue waited")
+            .unwrap();
+    }
+    let mut failure =
+        OperationalFailure::new("admin", "http_request", "unavailable", "safe failure");
+    failure.correlation_id = Some(request_id.as_str().to_owned());
+    failure.details = ErrorDetails::capture(
+        Some(&ErrorSource::new(std::io::Error::other(
+            "ORIGINAL_RESERVED_CAUSE",
+        ))),
+        None,
+        false,
+    );
+    store
+        .record_failure(failure)
+        .now_or_never()
+        .expect("failure waited for queue capacity")
+        .unwrap();
+    assert_eq!(store.stats().queued_items, 5);
+    assert_eq!(store.stats().dropped_total, 0);
+    store
+        .record_client_status(request_id, 204)
+        .now_or_never()
+        .expect("overflow waited")
+        .unwrap();
+    assert_eq!(store.stats().queued_items, 5);
+    assert_eq!(store.stats().dropped_total, 1);
+    let cancelled = CancellationToken::new();
+    cancelled.cancel();
+    writer.run(cancelled).await.unwrap();
+    assert_eq!(store.stats().persisted_total, 5);
+    let detail: String =
+        sqlx::query_scalar("select error_details from ops_events where component = 'admin'")
+            .fetch_one(&database.pool)
+            .await
+            .unwrap();
+    assert!(detail.contains("ORIGINAL_RESERVED_CAUSE"));
+    let status: i32 =
+        sqlx::query_scalar("select client_status_code from model_requests where id = $1")
+            .bind(request_id.as_str())
+            .fetch_one(&database.pool)
+            .await
+            .unwrap();
+    assert_eq!(status, 200);
+    assert_eq!(store.stats().queued_bytes, 0);
+    database.close().await;
+}