diff --git a/backend/apps/gateway/src/bootstrap/server.rs b/backend/apps/gateway/src/bootstrap/server.rs index 927f295a4..569f032ec 100644 --- a/backend/apps/gateway/src/bootstrap/server.rs +++ b/backend/apps/gateway/src/bootstrap/server.rs @@ -117,6 +117,7 @@ pub async fn run() -> Result<(), BootstrapError> { probes, host.worker_health(), host.connection_lifecycle(), + store.diagnostics(), ) ) .with_middleware({ @@ -155,7 +156,7 @@ pub async fn run() -> Result<(), BootstrapError> { )); or_shutdown!( plugin_runtime, - host.start_workers(plan, store.worker_leader_lease()) + host.start_workers(plan, store.worker_leader_lease(), store.diagnostics()) ); host.report_startup_ready("Workers"); let served = host.serve(api.router()).await; diff --git a/backend/crates/gateway-admin/src/backup/task.rs b/backend/crates/gateway-admin/src/backup/task.rs index d2656afac..06417bb27 100644 --- a/backend/crates/gateway-admin/src/backup/task.rs +++ b/backend/crates/gateway-admin/src/backup/task.rs @@ -28,7 +28,7 @@ use crate::{ impl From for WorkerTaskError { fn from(error: BackupError) -> Self { - WorkerTaskError::safe(error.message()) + WorkerTaskError::safe(error.message()).with_source(error) } } @@ -125,8 +125,9 @@ impl BackupTask { if cron.is_empty() { return Ok(()); } - let schedule = BackupSchedule::parse(cron, self.timezone) - .map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?; + let schedule = BackupSchedule::parse(cron, self.timezone).map_err(|source| { + WorkerTaskError::safe("backup schedule is invalid").with_source(source) + })?; if settings.schedule_timezone.as_deref() != Some(timezone) { // 时区切换只初始化未来游标，不把旧时区计划当作应补跑任务 @@ -685,9 +686,9 @@ fn status_transition( } fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError { - WorkerTaskError::safe(format!("backup store failed: {error}")) + WorkerTaskError::safe("backup store failed").with_source(error) } fn infra_error(error: BackupError) -> WorkerTaskError { - WorkerTaskError::safe(error.message()) + WorkerTaskError::safe(error.message()).with_source(error) } diff --git a/backend/crates/gateway-admin/src/model/mod.rs b/backend/crates/gateway-admin/src/model/mod.rs index 6d7ce6eb1..ca4f0b8ce 100644 --- a/backend/crates/gateway-admin/src/model/mod.rs +++ b/backend/crates/gateway-admin/src/model/mod.rs @@ -39,12 +39,31 @@ pub enum AdminErrorKind { Internal, } -/// 不携带基础设施细节、可安全返回给管理员的管理用例错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +impl AdminErrorKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Invalid => "invalid", + Self::Forbidden => "forbidden", + Self::Unauthorized => "unauthorized", + Self::NotFound => "not_found", + Self::Conflict => "conflict", + Self::RateLimited => "rate_limited", + Self::BadGateway => "bad_gateway", + Self::UpstreamResultUnknown => "upstream_result_unknown", + Self::Unavailable => "unavailable", + Self::Internal => "internal", + } + } +} + +/// 管理用例的安全消息与内部原因分离，原始来源不进入响应 +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct AdminError { kind: AdminErrorKind, message: String, + source: Option, } impl AdminError { @@ -53,9 +72,21 @@ impl AdminError { Self { kind, message: message.into(), + source: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + gateway_core::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } + #[must_use] pub const fn kind(&self) -> AdminErrorKind { self.kind diff --git a/backend/crates/gateway-admin/src/model/provider_credentials.rs b/backend/crates/gateway-admin/src/model/provider_credentials.rs index 27456c462..a8a04185a 100644 --- a/backend/crates/gateway-admin/src/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/src/model/provider_credentials.rs @@ -862,7 +862,7 @@ pub struct ProviderSubscription { } /// 按需汇聚的个人信息；资料查询失败不丢弃可用的订阅结果 -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub struct AccountPersonalInfo { pub profile: Result, pub subscription: Option, diff --git a/backend/crates/gateway-admin/src/ports/provider.rs b/backend/crates/gateway-admin/src/ports/provider.rs index 182e038e0..0079f2610 100644 --- a/backend/crates/gateway-admin/src/ports/provider.rs +++ b/backend/crates/gateway-admin/src/ports/provider.rs @@ -46,11 +46,12 @@ pub enum ProviderAdminErrorKind { /// `message` 是 Provider 局部诊断，可能包含原始上游正文；通用管理用例不得自动把它作为公开文案 /// 只有明确拥有原始诊断合同的调用方才能读取，`Debug` 始终只记录是否存在 /// `public_message` 则是明确标记为可公开的静态提示，不允许携带动态上游材料 -#[derive(Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Clone, thiserror::Error)] #[error("provider admin operation failed: {kind:?}")] pub struct ProviderAdminError { kind: ProviderAdminErrorKind, - message: Option, + #[source] + diagnostic: Option>, public_message: Option<&'static str>, } @@ -59,7 +60,7 @@ impl std::fmt::Debug for ProviderAdminError { formatter .debug_struct("ProviderAdminError") .field("kind", &self.kind) - .field("message", &self.message.as_ref().map(|_| "")) + .field("message", &self.message().map(|_| "")) .field("public_message", &self.public_message) .finish() } @@ -70,14 +71,22 @@ impl ProviderAdminError { pub const fn new(kind: ProviderAdminErrorKind) -> Self { Self { kind, - message: None, + diagnostic: None, public_message: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).source = + Some(source.into()); + self + } + #[must_use] pub fn with_message(mut self, message: impl Into) -> Self { - self.message = Some(message.into()); + Arc::make_mut(self.diagnostic.get_or_insert_with(Default::default)).message = + Some(message.into()); self } @@ -100,10 +109,20 @@ impl ProviderAdminError { #[must_use] pub fn message(&self) -> Option<&str> { - self.message.as_deref() + self.diagnostic + .as_ref() + .and_then(|diagnostic| diagnostic.message.as_deref()) } } +// Provider 局部消息与 typed 来源共用一个不公开的上下文，避免跨用例转换后丢失 +#[derive(Debug, Clone, Default, thiserror::Error)] +#[error("{}", message.as_deref().unwrap_or("provider admin dependency failed"))] +struct ProviderAdminDiagnostic { + message: Option, + source: Option, +} + /// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力 /// /// 数据变更由 Provider 返回 prepared facts；config revision、审计与 PostgreSQL 事务 diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 00b0b9203..16cad9b77 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -78,8 +78,18 @@ impl AdminStoreError { /// 来源留在内部诊断链，公开消息不展开基础设施错误 #[must_use] - pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { - self.source = Some(gateway_core::error::ErrorSource::new(source)); + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + /// 清理失败不覆盖最初操作的分类与原因 + #[must_use] + pub fn with_cleanup(mut self, cleanup: impl Into) -> Self { + self.source = Some(gateway_core::error::ErrorSource::cleanup( + self.source.take(), + cleanup, + )); self } diff --git a/backend/crates/gateway-admin/src/use_case/mod.rs b/backend/crates/gateway-admin/src/use_case/mod.rs index 5a3d86002..8ed47dccd 100644 --- a/backend/crates/gateway-admin/src/use_case/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/mod.rs @@ -51,7 +51,6 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError | AdminStoreErrorKind::Conflict => AdminErrorKind::Conflict, AdminStoreErrorKind::Unavailable => AdminErrorKind::Unavailable, }; - tracing::warn!(resource, error_kind = ?error.kind(), "admin store operation failed"); let message = match kind { AdminErrorKind::Invalid => "请求参数不合法", AdminErrorKind::NotFound => "请求的资源不存在", @@ -59,7 +58,10 @@ fn map_store_error(error: AdminStoreError, resource: &'static str) -> AdminError AdminErrorKind::Unavailable => "依赖服务暂不可用", _ => "服务内部错误", }; - AdminError::new(kind, message) + AdminError::new(kind, message).with_source(OperationFailure { + operation: resource, + source: error.into(), + }) } fn map_provider_error( @@ -79,7 +81,6 @@ fn map_provider_error( ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway, ProviderAdminErrorKind::Internal => AdminErrorKind::Internal, }; - tracing::warn!(resource, error_kind = ?error.kind(), "admin provider operation failed"); let message = match kind { AdminErrorKind::Invalid => "Provider 请求不合法", AdminErrorKind::NotFound => "Provider 资源不存在", @@ -90,7 +91,10 @@ fn map_provider_error( AdminErrorKind::Internal => "服务内部错误", _ => "Provider 操作失败", }; - AdminError::new(kind, error.public_message().unwrap_or(message)) + AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure { + operation: resource, + source: error.into(), + }) } async fn publish_committed( @@ -454,3 +458,11 @@ async fn delete_credentials( account_ids, }) } + +// 用例的操作名补充端口资源上下文，底层分类与原始原因仍由来源持有 +#[derive(Debug, thiserror::Error)] +#[error("admin operation failed: {operation}")] +struct OperationFailure { + operation: &'static str, + source: gateway_core::error::ErrorSource, +} diff --git a/backend/crates/gateway-admin/tests/model/provider_credentials.rs b/backend/crates/gateway-admin/tests/model/provider_credentials.rs index b2ab6e5ce..2465ba2ff 100644 --- a/backend/crates/gateway-admin/tests/model/provider_credentials.rs +++ b/backend/crates/gateway-admin/tests/model/provider_credentials.rs @@ -223,8 +223,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() { assert_eq!( PendingAuthorizationMutation::from_storage_v1(Value::Object( expected.to_storage_v1() - )), - Ok(expected.clone()), + )) + .unwrap(), + expected.clone(), ); let with_proxy = expected .with_outbound_proxy(Some( @@ -237,8 +238,9 @@ fn pending_mutation_v1_should_round_trip_all_targets_and_owners() { assert_eq!( PendingAuthorizationMutation::from_storage_v1(Value::Object( with_proxy.to_storage_v1() - )), - Ok(with_proxy) + )) + .unwrap(), + with_proxy ); } } diff --git a/backend/crates/gateway-api/src/admin/diagnostics.rs b/backend/crates/gateway-api/src/admin/diagnostics.rs new file mode 100644 index 000000000..aa5e9ccdb --- /dev/null +++ b/backend/crates/gateway-api/src/admin/diagnostics.rs @@ -0,0 +1,42 @@ +//! 管理请求在 HTTP 出口记录一次受控详情，响应只保留安全投影 + +use std::sync::Arc; + +use axum::{ + extract::{Request, State}, + middleware::Next, + response::Response, +}; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; +use tower_http::request_id::RequestId; + +pub(crate) async fn record_failure( + State(diagnostics): State>, + request: Request, + next: Next, +) -> Response { + let correlation_id = request + .extensions() + .get::() + .and_then(|id| id.header_value().to_str().ok()) + .map(str::to_owned); + let mut response = next.run(request).await; + if let Some(error) = response + .extensions_mut() + .remove::() + && let Some(details) = error.error_details() + { + let mut failure = OperationalFailure::new( + "admin", + "http_request", + error.kind().as_str(), + error.message(), + ); + failure.correlation_id = correlation_id; + failure.details = Some(details); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!("admin diagnostic could not be recorded"); + } + } + response +} diff --git a/backend/crates/gateway-api/src/admin/mod.rs b/backend/crates/gateway-api/src/admin/mod.rs index d7f11e980..8fb803cd7 100644 --- a/backend/crates/gateway-api/src/admin/mod.rs +++ b/backend/crates/gateway-api/src/admin/mod.rs @@ -15,6 +15,7 @@ pub mod accounts; pub mod auth; pub mod backups; pub mod client_keys; +pub(crate) mod diagnostics; mod extract; pub mod observability; mod plugins; diff --git a/backend/crates/gateway-api/src/admin/wire.rs b/backend/crates/gateway-api/src/admin/wire.rs index e8f455225..0092b88b5 100644 --- a/backend/crates/gateway-api/src/admin/wire.rs +++ b/backend/crates/gateway-api/src/admin/wire.rs @@ -170,6 +170,7 @@ pub struct AdminError { status: StatusCode, body: AdminErrorBody, retry_after_seconds: Option, + diagnostic: Option, } #[derive(Debug, Clone, Copy)] @@ -266,6 +267,7 @@ impl AdminError { status, body: AdminErrorBody::new(code, message), retry_after_seconds: None, + diagnostic: None, } } @@ -380,12 +382,16 @@ pub(crate) fn map_admin_service_error(error: gateway_admin::model::AdminError) - { response.body.message = error.message().to_owned(); } + response.diagnostic = Some(error); response } impl IntoResponse for AdminError { fn into_response(self) -> Response { let mut response = (self.status, Json(self.body)).into_response(); + if let Some(error) = self.diagnostic { + response.extensions_mut().insert(error); + } if let Some(seconds) = self.retry_after_seconds && let Ok(value) = axum::http::HeaderValue::from_str(&seconds.to_string()) { diff --git a/backend/crates/gateway-api/src/lib.rs b/backend/crates/gateway-api/src/lib.rs index 4ecd8cc2b..0f0c4559f 100644 --- a/backend/crates/gateway-api/src/lib.rs +++ b/backend/crates/gateway-api/src/lib.rs @@ -170,6 +170,7 @@ pub fn initialize( probes: Vec>, worker_health: Arc, lifecycle: Arc, + diagnostics: Arc, ) -> Result { // 配置加载已解析环境变量与相对路径，初始化只校验，避免覆盖最终目录 config.validate().map_err(ApiError::Config)?; @@ -256,6 +257,10 @@ pub fn initialize( headers_ms = latency.as_millis(), "HTTP response headers ready"); }); let router = router + .layer(axum::middleware::from_fn_with_state( + diagnostics, + admin::diagnostics::record_failure, + )) .layer(PropagateRequestIdLayer::new(request_id_header.clone())) .layer(trace_layer) .layer(SetRequestIdLayer::new( diff --git a/backend/crates/gateway-api/src/openai/error.rs b/backend/crates/gateway-api/src/openai/error.rs index de8088f25..d6bf964bd 100644 --- a/backend/crates/gateway-api/src/openai/error.rs +++ b/backend/crates/gateway-api/src/openai/error.rs @@ -165,8 +165,11 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError { "no upstream provider is currently available for this request", ) } - EngineError::Store(_) - | EngineError::ProviderMetadataMismatch + EngineError::Store(source) => { + GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed") + .with_source(source.clone()) + } + EngineError::ProviderMetadataMismatch | EngineError::ContinuationPinMismatch | EngineError::RequiredAccountMismatch | EngineError::AccountOutsideClientScope diff --git a/backend/crates/gateway-api/tests/admin/error_diagnostics.rs b/backend/crates/gateway-api/tests/admin/error_diagnostics.rs new file mode 100644 index 000000000..f2be747e9 --- /dev/null +++ b/backend/crates/gateway-api/tests/admin/error_diagnostics.rs @@ -0,0 +1,61 @@ +//! 验证管理端来源经用例和 HTTP 出口到达受控记录，公开响应和扩展不泄漏 + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use std::sync::Arc; +use tower::ServiceExt; + +#[tokio::test] +async fn store_failure_is_recorded_once_with_the_incoming_request_id() { + let fixture = super::AdminTestFixture::new().await; + fixture.auth.insert_session("valid-session"); + let diagnostics = Arc::new(crate::support::RecordingDiagnostics::default()); + let bundle = crate::openai::api_bundle_with_diagnostics( + fixture.services, + gateway_api::ApiConfig { + asset_directory: std::env::temp_dir(), + request_timeout_seconds: None, + cors_allowed_origins: Vec::new(), + request_id_header: "x-request-id".to_owned(), + }, + diagnostics.clone(), + ); + let response = bundle + .router() + .oneshot( + Request::builder() + .uri("/api/admin/accounts") + .header("cookie", "cpr_session=valid-session") + .header("x-request-id", "admin-native-cause-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!( + response + .extensions() + .get::() + .is_none() + ); + let bytes = to_bytes(response.into_body(), 1024 * 1024).await.unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("PRIVATE_ADMIN_NATIVE_CAUSE")); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!(failures[0].kind, "unavailable"); + assert_eq!( + failures[0].correlation_id.as_deref(), + Some("admin-native-cause-test") + ); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_ADMIN_NATIVE_CAUSE") + ); +} diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index 823d89f12..e454f44d6 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -86,6 +86,7 @@ mod account_groups; mod accounts; mod auth; mod client_keys; +mod error_diagnostics; mod errors; mod observability; mod proxies; @@ -1673,6 +1674,7 @@ fn unavailable(resource: &'static str) -> AdminStoreError { resource, "unused test port", ) + .with_source(std::io::Error::other("PRIVATE_ADMIN_NATIVE_CAUSE")) } fn not_found(resource: &'static str) -> AdminStoreError { diff --git a/backend/crates/gateway-api/tests/architecture.rs b/backend/crates/gateway-api/tests/architecture.rs index 5a08380cf..24ca76d0a 100644 --- a/backend/crates/gateway-api/tests/architecture.rs +++ b/backend/crates/gateway-api/tests/architecture.rs @@ -45,6 +45,7 @@ fn source_tree_should_match_frozen_machine_manifest() { "src/admin/auth.rs", "src/admin/backups.rs", "src/admin/client_keys.rs", + "src/admin/diagnostics.rs", "src/admin/extract.rs", "src/admin/mod.rs", "src/admin/observability/mod.rs", @@ -124,6 +125,7 @@ fn test_tree_should_match_frozen_rust_mirror() { "tests/admin/auth.rs", "tests/admin/client_keys.rs", "tests/admin/errors.rs", + "tests/admin/error_diagnostics.rs", "tests/admin/mod.rs", "tests/admin/observability/mod.rs", "tests/admin/observability/query.rs", diff --git a/backend/crates/gateway-api/tests/openai/auth.rs b/backend/crates/gateway-api/tests/openai/auth.rs index 79ef73c15..72dfbaf20 100644 --- a/backend/crates/gateway-api/tests/openai/auth.rs +++ b/backend/crates/gateway-api/tests/openai/auth.rs @@ -479,6 +479,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission( Arc::new(super::UnusedAdmissions), Arc::new(super::UnusedContinuation), Arc::new(super::IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let admin = crate::admin::AdminTestFixture::new().await; let bundle = gateway_api::initialize( @@ -493,6 +494,7 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission( vec![], Arc::new(super::EmptyWorkerHealth), Arc::new(super::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap(); let baseline = bundle.dispatcher(); diff --git a/backend/crates/gateway-api/tests/openai/error.rs b/backend/crates/gateway-api/tests/openai/error.rs index 467ad58fd..a7cf044f8 100644 --- a/backend/crates/gateway-api/tests/openai/error.rs +++ b/backend/crates/gateway-api/tests/openai/error.rs @@ -304,12 +304,14 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::AccountCapacityUnavailable, - "all eligible upstream accounts are temporarily busy" - ) + projected.kind(), + GatewayErrorKind::AccountCapacityUnavailable + ); + assert_eq!( + projected.safe_message(), + "all eligible upstream accounts are temporarily busy" ); } @@ -352,12 +354,14 @@ fn provider_infrastructure_failure_should_have_a_distinct_client_contract() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::ProviderInfrastructureUnavailable, - "provider account infrastructure is temporarily unavailable" - ) + projected.kind(), + GatewayErrorKind::ProviderInfrastructureUnavailable + ); + assert_eq!( + projected.safe_message(), + "provider account infrastructure is temporarily unavailable" ); } @@ -368,12 +372,11 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() { UpstreamSendState::NotSent, )); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::NoAvailableProvider); assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::NoAvailableProvider, - "no upstream provider is currently available for this request" - ) + projected.safe_message(), + "no upstream provider is currently available for this request" ); } @@ -499,13 +502,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() { ] { let error = EngineError::Provider(ProviderError::new(kind, UpstreamSendState::Ambiguous)); - assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new( - GatewayErrorKind::UpstreamUnavailable, - "upstream service is unavailable" - ) - ); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::UpstreamUnavailable); + assert_eq!(projected.safe_message(), "upstream service is unavailable"); } } @@ -513,10 +512,9 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() { fn engine_store_error_should_collapse_to_safe_internal_error() { let error = EngineError::Store(StoreError::new(StoreErrorKind::Unavailable)); - assert_eq!( - gateway_error_from_engine(&error), - GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed") - ); + let projected = gateway_error_from_engine(&error); + assert_eq!(projected.kind(), GatewayErrorKind::Internal); + assert_eq!(projected.safe_message(), "gateway execution failed"); } #[test] fn openai_error_response_should_preserve_only_safe_contract_fields() { @@ -669,6 +667,7 @@ mod model_routing { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); let response = crate::openai::api_router(Arc::new(execution)) .await diff --git a/backend/crates/gateway-api/tests/openai/live/websocket.rs b/backend/crates/gateway-api/tests/openai/live/websocket.rs index 6e0f1ed65..74e9fe0b8 100644 --- a/backend/crates/gateway-api/tests/openai/live/websocket.rs +++ b/backend/crates/gateway-api/tests/openai/live/websocket.rs @@ -138,6 +138,7 @@ async fn blocked_relay( Vec::new(), Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(Lifecycle(trace.clone())), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .router(); diff --git a/backend/crates/gateway-api/tests/openai/mod.rs b/backend/crates/gateway-api/tests/openai/mod.rs index 4628e8bee..42e34aca5 100644 --- a/backend/crates/gateway-api/tests/openai/mod.rs +++ b/backend/crates/gateway-api/tests/openai/mod.rs @@ -92,6 +92,7 @@ pub(super) fn api_router_with_admin_and_client( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); api_router_with_admin_and_execution(admin, execution) } @@ -106,6 +107,18 @@ pub(super) fn api_router_with_config( pub(super) fn api_bundle( admin: gateway_admin::AdminServices, config: gateway_api::ApiConfig, +) -> gateway_api::ApiBundle { + api_bundle_with_diagnostics( + admin, + config, + Arc::new(crate::support::RecordingDiagnostics::default()), + ) +} + +pub(crate) fn api_bundle_with_diagnostics( + admin: gateway_admin::AdminServices, + config: gateway_api::ApiConfig, + diagnostics: Arc, ) -> gateway_api::ApiBundle { let execution = Arc::new(DefaultExecutionService::new( RuntimeSnapshotHandle::new(snapshot("unused-client-route-key", "openai")), @@ -114,6 +127,7 @@ pub(super) fn api_bundle( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); gateway_api::initialize( config, @@ -122,6 +136,7 @@ pub(super) fn api_bundle( Vec::new(), Arc::new(EmptyWorkerHealth), Arc::new(TestLifecycle::default()), + diagnostics, ) .unwrap() } @@ -138,6 +153,7 @@ fn api_router_with_config_and_execution( Vec::new(), Arc::new(EmptyWorkerHealth), Arc::new(TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .expect("API bundle") .router() @@ -180,6 +196,7 @@ async fn api_router_with_origins_and_worker_health( Vec::new(), worker_health, Arc::new(TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .expect("API bundle") .router() @@ -214,6 +231,7 @@ pub(super) fn authenticated_client_for_provider_with_limit( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); source .authenticate(plaintext) @@ -244,6 +262,7 @@ pub(super) fn authenticated_client_with_min_versions( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ); source .authenticate(plaintext) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs index 92e97b45f..e54596e90 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/middleware.rs @@ -165,6 +165,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake Arc::new(crate::openai::UnusedAdmissions), Arc::new(crate::openai::UnusedContinuation), Arc::new(crate::openai::IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let admin = crate::admin::AdminTestFixture::new().await; let plan = Arc::new(SettingsPlan::default()); @@ -187,6 +188,7 @@ async fn turns_refresh_host_settings_and_key_identity_while_preserving_handshake vec![], Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(crate::openai::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .with_middleware(move |_| Some(frozen.clone())) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs index 385c51972..a2eb4b412 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/connection/mod.rs @@ -761,6 +761,7 @@ async fn start_active_response_with_middleware( Vec::new(), Arc::new(crate::openai::EmptyWorkerHealth), Arc::new(crate::openai::TestLifecycle::default()), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .with_middleware(move |_| Some(plan.clone())) diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs index 9b83dcd4b..03ac069d5 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/interrupt.rs @@ -162,6 +162,7 @@ async fn interrupt_is_live_scoped_and_keeps_queued_creates_serial() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), )); let app = api_router(execution).await; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs b/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs index fd6e5b871..49adf16b0 100644 --- a/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs +++ b/backend/crates/gateway-api/tests/openai/responses/websocket/mod.rs @@ -1108,6 +1108,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele ports.clone(), Arc::new(UnusedContinuation), Arc::new(IgnoredClientApiKeyUsage), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .with_budget(ports.clone()), ); @@ -1125,6 +1126,7 @@ async fn websocket_disconnect_during_core_settlement_finishes_charge_before_rele Vec::new(), Arc::new(EmptyWorkerHealth), lifecycle.clone(), + Arc::new(crate::support::RecordingDiagnostics::default()), ) .unwrap() .router(); diff --git a/backend/crates/gateway-api/tests/support.rs b/backend/crates/gateway-api/tests/support.rs index 67c757589..65aeccdd0 100644 --- a/backend/crates/gateway-api/tests/support.rs +++ b/backend/crates/gateway-api/tests/support.rs @@ -159,3 +159,18 @@ impl ClientKeyVerifier for AcceptingVerifier { } } } + +#[derive(Default)] +pub(crate) struct RecordingDiagnostics( + pub std::sync::Mutex>, +); +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-core/src/diagnostics/failure.rs b/backend/crates/gateway-core/src/diagnostics/failure.rs new file mode 100644 index 000000000..4c55a4ce0 --- /dev/null +++ b/backend/crates/gateway-core/src/diagnostics/failure.rs @@ -0,0 +1,54 @@ +//! 控制面与后台操作的受控故障记录，不伪造模型请求或上游发送事实 + +use async_trait::async_trait; + +use crate::{ + account::ProviderAccountId, + error::{ErrorDetails, OpaqueUpstreamValue, StoreError}, + identity::ProviderKind, +}; + +#[derive(Debug)] +pub struct OperationalFailure { + pub occurred_at: std::time::SystemTime, + pub component: &'static str, + pub operation: &'static str, + pub kind: &'static str, + pub message: String, + pub correlation_id: Option, + pub provider_kind: Option, + pub account_id: Option, + pub upstream_status: Option, + pub upstream_code: Option, + pub details: Option, +} + +impl OperationalFailure { + #[must_use] + pub fn new( + component: &'static str, + operation: &'static str, + kind: &'static str, + message: impl Into, + ) -> Self { + Self { + occurred_at: std::time::SystemTime::now(), + component, + operation, + kind, + message: message.into(), + correlation_id: None, + provider_kind: None, + account_id: None, + upstream_status: None, + upstream_code: None, + details: None, + } + } +} + +/// 记录遵循既有观测队列的 best-effort 合同，失败不应替换业务结果 +#[async_trait] +pub trait OperationalDiagnostics: Send + Sync { + async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError>; +} diff --git a/backend/crates/gateway-core/src/diagnostics/mod.rs b/backend/crates/gateway-core/src/diagnostics/mod.rs index 338f6866a..24bd0287c 100644 --- a/backend/crates/gateway-core/src/diagnostics/mod.rs +++ b/backend/crates/gateway-core/src/diagnostics/mod.rs @@ -1,10 +1,12 @@ -//! 有界、请求局部的诊断证据；不参与路由、重试或客户端协议决策 +//! 请求诊断与操作故障的观测合同，不参与路由、重试或客户端协议决策 mod capture; +mod failure; mod selection; mod stream; mod trace; pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json}; +pub use failure::{OperationalDiagnostics, OperationalFailure}; pub use stream::{StreamCapture, StreamFormat}; pub use trace::TraceContext; diff --git a/backend/crates/gateway-core/src/engine/admission.rs b/backend/crates/gateway-core/src/engine/admission.rs index 72eaa46a4..d0eaa6ec3 100644 --- a/backend/crates/gateway-core/src/engine/admission.rs +++ b/backend/crates/gateway-core/src/engine/admission.rs @@ -57,9 +57,9 @@ pub struct ClientAdmissionRestoreResult { pub restored_running_requests: u64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("client admission store is unavailable")] -pub struct ClientAdmissionError; +pub struct ClientAdmissionError(#[source] pub Option); pub trait ClientAdmissionPort: Send + Sync { /// 仅续期已经准入的并发槽位，不重复消费 RPM @@ -118,10 +118,10 @@ pub async fn restore_client_admission_startup( let expired = execution .recover_expired(now) .await - .map_err(|_| ClientAdmissionError)?; + .map_err(|source| ClientAdmissionError(Some(source.into())))?; let since = now .checked_sub(Duration::from_secs(61)) - .ok_or(ClientAdmissionError)?; + .ok_or(ClientAdmissionError(None))?; let recoveries = recovery.load_recovery(since).await?; let restored_clients = u64::try_from(recoveries.len()).unwrap_or(u64::MAX); let mut restored_recent_requests = 0_u64; diff --git a/backend/crates/gateway-core/src/engine/budget.rs b/backend/crates/gateway-core/src/engine/budget.rs index 101de66e2..e42587073 100644 --- a/backend/crates/gateway-core/src/engine/budget.rs +++ b/backend/crates/gateway-core/src/engine/budget.rs @@ -39,9 +39,9 @@ pub struct ClientBudgetCharge { pub completed_at: SystemTime, } -#[derive(Debug, Clone, Copy, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("client budget store is unavailable")] -pub struct ClientBudgetError; +pub struct ClientBudgetError(#[source] pub Option); pub trait ClientBudgetPort: Send + Sync { /// 原子检查当前限额与已用金额，不创建预扣费或待结算记录 diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index 7610d53d5..29975b7f9 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -739,6 +739,7 @@ pub struct DefaultExecutionService { snapshots: RuntimeSnapshotHandle, /// probe 自身走 transient store，探测失败仍写入持久 store 的 ops_events observations: Arc, + diagnostics: Arc, providers: ProviderRegistry, admissions: Arc, admission_waiting: ConcurrencyWaitQueue, @@ -763,10 +764,12 @@ impl DefaultExecutionService { admissions: Arc, continuation: Arc, client_api_key_usage: Arc, + diagnostics: Arc, ) -> Self { Self { snapshots, observations: execution, + diagnostics, providers, admissions, admission_waiting: ConcurrencyWaitQueue::default(), @@ -1325,7 +1328,12 @@ impl DefaultExecutionService { && let Some(guard) = start_guard.take() { guard - .release_failed(self.budget.as_deref(), request_id, budget_key_id) + .release_failed( + self.budget.as_deref(), + request_id, + budget_key_id, + self.diagnostics.as_ref(), + ) .await; } result @@ -1498,6 +1506,7 @@ impl DefaultExecutionService { amount_usd: crate::metering::Decimal::ZERO, completed_at: SystemTime::now(), }, + self.diagnostics.as_ref(), ) .await; } @@ -1518,6 +1527,7 @@ impl DefaultExecutionService { active_request, Arc::clone(&self.continuation), self.budget.clone(), + self.diagnostics.clone(), )), }) } @@ -1872,6 +1882,7 @@ impl DefaultExecutionService { CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), budget); let mut admission = AdmissionLease { port: Arc::clone(&self.admissions), + diagnostics: self.diagnostics.clone(), client_api_key_id: key.clone(), model_request_id: request_id.clone(), armed: false, @@ -1902,7 +1913,7 @@ impl DefaultExecutionService { pin_mut!(acquire, timeout, cancelled); let decision = select_biased! { () = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "request admission was cancelled")), - result = acquire => result.map_err(|_| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable"))?, + result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable").with_source(source))?, _ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")), }; match decision { @@ -2477,14 +2488,26 @@ struct AdmissionLease { renewal: Option>, armed: bool, port: Arc, + diagnostics: Arc, client_api_key_id: ClientApiKeyId, model_request_id: ModelRequestId, } -async fn settle_budget(port: &dyn ClientBudgetPort, charge: ClientBudgetCharge) { +async fn settle_budget( + port: &dyn ClientBudgetPort, + charge: ClientBudgetCharge, + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, +) { let request_id = charge.request_id.clone(); if let Err(error) = port.settle(charge).await { - tracing::error!(request_id = request_id.as_str(), operation = "settle_client_budget", %error, "Client budget settlement failed; storage will retry on the next request"); + record_resource_failure( + diagnostics, + &request_id, + "settle_client_budget", + "Client budget settlement failed; storage will retry on the next request", + error, + ) + .await; } } @@ -2496,7 +2519,14 @@ impl AdmissionLease { .release(&self.client_api_key_id, &self.model_request_id) .await { - tracing::warn!(request_id = self.model_request_id.as_str(), key_id = self.client_api_key_id.as_str(), operation = "release_client_admission", %error, "Client admission 释放失败，依赖租约 TTL 收敛"); + record_resource_failure( + self.diagnostics.as_ref(), + &self.model_request_id, + "release_client_admission", + "Client admission release failed; lease TTL remains active", + error, + ) + .await; } self.armed = false; } @@ -2537,6 +2567,7 @@ impl ExecutionStartGuard { budget: Option<&dyn ClientBudgetPort>, request_id: ModelRequestId, key_id: ClientApiKeyId, + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, ) { if let Some(active_request) = self.active_request { active_request.release(); @@ -2550,6 +2581,7 @@ impl ExecutionStartGuard { amount_usd: crate::metering::Decimal::ZERO, completed_at: SystemTime::now(), }, + diagnostics, ) .await; } @@ -2565,6 +2597,7 @@ struct DefaultExecutionSession { continuation: Arc, continuation_recorded: bool, budget: Option>, + diagnostics: Arc, } impl DefaultExecutionSession { @@ -2574,6 +2607,7 @@ impl DefaultExecutionSession { active_request: Option, continuation: Arc, budget: Option>, + diagnostics: Arc, ) -> Self { Self { core, @@ -2583,6 +2617,7 @@ impl DefaultExecutionSession { continuation, continuation_recorded: false, budget, + diagnostics, } } @@ -2595,12 +2630,13 @@ impl DefaultExecutionSession { } let budget = self.budget.take(); let charge = self.core.budget_charge(); + let diagnostics = self.diagnostics.clone(); // 在首次 await 前把完整清理责任留在会话内 // 事件等待被取消后，后续 poll // 或 detach 继续同一个 future，既不丢失费用，也不重启已完成的结算 self.cleanup = Some(Box::pin(async move { if let Some(budget) = budget { - settle_budget(budget.as_ref(), charge).await; + settle_budget(budget.as_ref(), charge, diagnostics.as_ref()).await; } admission.release().await; })); @@ -2923,6 +2959,34 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError { "no provider is available", ) } + EngineError::Store(source) => { + GatewayError::new(GatewayErrorKind::Internal, "request execution failed") + .with_source(source.clone()) + } _ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"), } } + +async fn record_resource_failure( + diagnostics: &dyn crate::diagnostics::OperationalDiagnostics, + request_id: &ModelRequestId, + operation: &'static str, + message: &'static str, + source: impl Into, +) { + let mut failure = crate::diagnostics::OperationalFailure::new( + "core", + operation, + "resource_unavailable", + message, + ); + failure.correlation_id = Some(request_id.as_str().to_owned()); + failure.details = crate::error::ErrorDetails::capture(Some(&source.into()), None, false); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!( + request_id = request_id.as_str(), + operation, + "resource diagnostic could not be recorded" + ); + } +} diff --git a/backend/crates/gateway-core/src/error/details.rs b/backend/crates/gateway-core/src/error/details.rs new file mode 100644 index 000000000..6b35ee6b9 --- /dev/null +++ b/backend/crates/gateway-core/src/error/details.rs @@ -0,0 +1,43 @@ +//! 原因链与上游正文的受控诊断投影，不参与普通错误格式化 + +use super::{ErrorSource, RawUpstreamError}; + +#[derive(Clone)] +pub struct ErrorDetails(String); + +impl ErrorDetails { + #[must_use] + pub fn capture( + source: Option<&ErrorSource>, + upstream: Option<&RawUpstreamError>, + redacted: bool, + ) -> Option { + if source.is_none() && upstream.is_none() { + return None; + } + Some(Self( + serde_json::json!({ + "causes": source.map(ErrorSource::snapshot), + "upstream": upstream.map(RawUpstreamError::as_str), + "redacted": redacted, + }) + .to_string(), + )) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + #[must_use] + pub fn into_string(self) -> String { + self.0 + } +} + +impl std::fmt::Debug for ErrorDetails { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("ErrorDetails()") + } +} diff --git a/backend/crates/gateway-core/src/error/mod.rs b/backend/crates/gateway-core/src/error/mod.rs index d860eb504..cc97f3ffa 100644 --- a/backend/crates/gateway-core/src/error/mod.rs +++ b/backend/crates/gateway-core/src/error/mod.rs @@ -1,6 +1,8 @@ //! 网关核心使用的稳定错误分类 +mod details; mod source; +pub use details::ErrorDetails; pub use source::ErrorSource; @@ -865,25 +867,20 @@ impl ProviderError { /// 包装底层失败；来源只供显式诊断访问，不改变分类和发送事实 #[must_use] - pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { - self.source = Some(ErrorSource::new(source)); + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); self } /// 运维详情的唯一投影入口，来源链与上游正文分别保留，不进入公共响应或诊断包 #[must_use] pub fn error_details(&self) -> Option { - if self.source.is_none() && self.raw_upstream_error.is_none() { - return None; - } - Some( - serde_json::json!({ - "causes": self.source.as_ref().map(ErrorSource::snapshot), - "upstream": self.raw_upstream_error().map(RawUpstreamError::as_str), - "redacted": self.sensitive_context_was_redacted(), - }) - .to_string(), + ErrorDetails::capture( + self.source.as_ref(), + self.raw_upstream_error(), + self.sensitive_context_was_redacted(), ) + .map(ErrorDetails::into_string) } /// 返回将由运维错误详情原样展示的上游错误返回 @@ -1094,14 +1091,15 @@ impl GatewayErrorKind { } /// 协议无关、可安全暴露的网关错误 -#[derive(Clone, PartialEq, Eq, Error)] +#[derive(Clone, Error)] #[error("{message}")] pub struct GatewayError { kind: GatewayErrorKind, message: &'static str, diagnostic: Option>, - client_visible_upstream_error: Option, + client_visible_upstream_error: Option>, client_details: Option>, + source: Option, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -1120,9 +1118,21 @@ impl GatewayError { diagnostic: None, client_visible_upstream_error: None, client_details: None, + source: None, } } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + ErrorDetails::capture(self.source.as_ref(), None, false) + } + #[must_use] pub fn with_client_code(mut self, code: &'static str) -> Self { self.client_details @@ -1232,7 +1242,7 @@ impl GatewayError { /// 附加只供请求方协议展示的结构化上游错误 #[must_use] pub fn with_client_visible_upstream_error(mut self, error: ClientVisibleUpstreamError) -> Self { - self.client_visible_upstream_error = Some(error); + self.client_visible_upstream_error = Some(Box::new(error)); self } @@ -1259,7 +1269,7 @@ impl GatewayError { #[must_use] pub fn client_message(&self) -> &str { self.client_visible_upstream_error - .as_ref() + .as_deref() .map_or(self.message, ClientVisibleUpstreamError::message) } @@ -1267,7 +1277,7 @@ impl GatewayError { #[must_use] pub fn client_error_type(&self) -> Option<&str> { self.client_visible_upstream_error - .as_ref() + .as_deref() .and_then(ClientVisibleUpstreamError::error_type) } @@ -1275,7 +1285,7 @@ impl GatewayError { #[must_use] pub fn client_error_code(&self) -> Option<&str> { self.client_visible_upstream_error - .as_ref() + .as_deref() .and_then(ClientVisibleUpstreamError::code) .or_else(|| { self.client_details @@ -1337,13 +1347,10 @@ impl StoreError { /// 包装已有存储错误并保留来源 #[must_use] - pub fn caused_by( - kind: StoreErrorKind, - source: impl std::error::Error + Send + Sync + 'static, - ) -> Self { + pub fn caused_by(kind: StoreErrorKind, source: impl Into) -> Self { Self { kind, - source: Some(ErrorSource::new(source)), + source: Some(source.into()), } } diff --git a/backend/crates/gateway-core/src/error/source.rs b/backend/crates/gateway-core/src/error/source.rs index 54a848c67..974581b2c 100644 --- a/backend/crates/gateway-core/src/error/source.rs +++ b/backend/crates/gateway-core/src/error/source.rs @@ -12,32 +12,125 @@ impl ErrorSource { Self(Arc::new(error)) } + /// 清理失败是附属事实，标准来源链仍指向最初失败 + #[must_use] + pub fn with_cleanup(self, cleanup: impl Into) -> Self { + Self::new(CleanupFailure { + primary: Some(self), + cleanup: cleanup.into(), + }) + } + + /// 附属清理错误与可选的原始底层原因分别保留 + #[must_use] + pub fn cleanup(primary: Option, cleanup: impl Into) -> Self { + Self::new(CleanupFailure { + primary, + cleanup: cleanup.into(), + }) + } + /// 仅供受控错误详情落盘，限制链深度和总文本大小并显式标记缺口 pub(super) fn snapshot(&self) -> serde_json::Value { - let mut messages = Vec::new(); - let mut source: Option<&(dyn Error + 'static)> = Some(self.0.as_ref()); - let mut remaining = 64 * 1024; - let mut truncated = false; - for _ in 0..32 { - let Some(error) = source else { break }; - let mut message = BoundedMessage { - text: String::new(), - remaining, - }; - if fmt::write(&mut message, format_args!("{error}")).is_err() { - truncated = true; - } - remaining = message.remaining; - messages.push(message.text); - source = error.source(); - if truncated || remaining == 0 { - break; + snapshot(self.0.as_ref(), &mut 32, &mut (64 * 1024)) + } + + /// 观测队列估算来源占用，包含附属清理失败；过深或循环的来源按超限处理 + #[must_use] + pub fn estimated_chain_bytes(source: Option<&(dyn Error + 'static)>) -> usize { + struct ByteCount(usize); + impl fmt::Write for ByteCount { + fn write_str(&mut self, value: &str) -> fmt::Result { + self.0 = self.0.saturating_add(value.len()); + Ok(()) } } - serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()}) + let mut pending = Vec::from_iter(source); + let mut bytes = ByteCount(0); + for _ in 0..32 { + let Some(error) = pending.pop() else { + return bytes.0; + }; + bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error)); + if fmt::write(&mut bytes, format_args!("{error}")).is_err() { + return usize::MAX; + } + if let Some(cleanup) = error.downcast_ref::() { + pending.push(cleanup.cleanup.0.as_ref()); + } + if let Some(source) = error.source() { + pending.push(source); + } + } + if pending.is_empty() { + bytes.0 + } else { + usize::MAX + } } } +impl From for ErrorSource { + fn from(error: E) -> Self { + Self::new(error) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("operation failed and cleanup also failed")] +struct CleanupFailure { + #[source] + primary: Option, + cleanup: ErrorSource, +} + +fn snapshot( + error: &(dyn Error + 'static), + nodes: &mut usize, + bytes: &mut usize, +) -> serde_json::Value { + let mut messages = Vec::new(); + let mut cleanup_sources = Vec::new(); + let mut source = Some(error); + let mut truncated = false; + while *nodes > 0 { + let Some(error) = source else { break }; + *nodes -= 1; + if let Some(failure) = error.downcast_ref::() { + cleanup_sources.push(failure.cleanup.0.as_ref()); + source = failure + .primary + .as_ref() + .map(|primary| primary.0.as_ref() as &(dyn Error + 'static)); + continue; + } + let mut message = BoundedMessage { + text: String::new(), + remaining: *bytes, + }; + if fmt::write(&mut message, format_args!("{error}")).is_err() { + truncated = true; + } + *bytes = message.remaining; + messages.push(message.text); + source = error.source(); + if truncated || *bytes == 0 { + break; + } + } + let mut result = + serde_json::json!({"messages": messages, "truncated": truncated || source.is_some()}); + if !cleanup_sources.is_empty() { + result["cleanup"] = serde_json::Value::Array( + cleanup_sources + .into_iter() + .map(|error| snapshot(error, nodes, bytes)) + .collect(), + ); + } + result +} + struct BoundedMessage { text: String, remaining: usize, diff --git a/backend/crates/gateway-core/src/lib.rs b/backend/crates/gateway-core/src/lib.rs index b125510f7..7fd908809 100644 --- a/backend/crates/gateway-core/src/lib.rs +++ b/backend/crates/gateway-core/src/lib.rs @@ -53,6 +53,7 @@ use task::{WorkerContribution, WorkerDefinitionError}; #[derive(Clone)] pub struct CoreStorePorts { execution: Arc, + diagnostics: Arc, admissions: Arc, admission_recovery: Arc, continuation: Arc, @@ -77,9 +78,11 @@ impl CoreStorePorts { Arc, ), client_api_key_usage: Arc, + diagnostics: Arc, ) -> Self { Self { execution, + diagnostics, admissions, admission_recovery, continuation, @@ -201,6 +204,7 @@ fn build_execution_service( Arc::clone(&ports.admissions), Arc::clone(&ports.continuation), Arc::clone(&ports.client_api_key_usage), + Arc::clone(&ports.diagnostics), ); if let Some(budget) = &ports.budget { service = service.with_budget(Arc::clone(budget)); diff --git a/backend/crates/gateway-core/src/provider_ports.rs b/backend/crates/gateway-core/src/provider_ports.rs index 6034824b5..5d8d19421 100644 --- a/backend/crates/gateway-core/src/provider_ports.rs +++ b/backend/crates/gateway-core/src/provider_ports.rs @@ -51,12 +51,12 @@ impl ProviderStoreError { pub fn caused_by( kind: ProviderStoreErrorKind, operation: &'static str, - source: impl std::error::Error + Send + Sync + 'static, + source: impl Into, ) -> Self { Self { kind, operation, - source: Some(crate::error::ErrorSource::new(source)), + source: Some(source.into()), } } @@ -1494,6 +1494,7 @@ pub struct ProviderStorePorts { cooldowns: Arc, runtime_policy: Arc, oauth_pending: Arc, + diagnostics: Arc, } impl ProviderStorePorts { @@ -1511,6 +1512,7 @@ impl ProviderStorePorts { cooldowns: Arc, runtime_policy: Arc, oauth_pending: Arc, + diagnostics: Arc, ) -> Self { Self { accounts, @@ -1524,9 +1526,15 @@ impl ProviderStorePorts { cooldowns, runtime_policy, oauth_pending, + diagnostics, } } + #[must_use] + pub fn diagnostics(&self) -> Arc { + self.diagnostics.clone() + } + #[must_use] pub fn accounts(&self) -> Arc { Arc::clone(&self.accounts) diff --git a/backend/crates/gateway-core/src/runtime/mod.rs b/backend/crates/gateway-core/src/runtime/mod.rs index b60cece46..5391cb6f7 100644 --- a/backend/crates/gateway-core/src/runtime/mod.rs +++ b/backend/crates/gateway-core/src/runtime/mod.rs @@ -454,7 +454,10 @@ impl ScheduledTask for RuntimeSnapshotReconciliationTask { .refresh_with_mode(RefreshMode::Reconcile) .await .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("runtime snapshot reconciliation failed")) + .map_err(|source| { + WorkerTaskError::safe("runtime snapshot reconciliation failed") + .with_source(source) + }) }) } } diff --git a/backend/crates/gateway-core/src/task.rs b/backend/crates/gateway-core/src/task.rs index f45d90ffa..52fe06eac 100644 --- a/backend/crates/gateway-core/src/task.rs +++ b/backend/crates/gateway-core/src/task.rs @@ -485,10 +485,11 @@ pub enum WorkerDefinitionError { } /// 不暴露基础设施原文或 lease resource 的错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct WorkerLeaseError { message: String, + source: Option, } impl WorkerLeaseError { @@ -496,6 +497,7 @@ impl WorkerLeaseError { pub fn safe(message: impl Into) -> Self { Self { message: message.into(), + source: None, } } @@ -503,13 +505,24 @@ impl WorkerLeaseError { pub fn as_safe_str(&self) -> &str { &self.message } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + crate::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } } /// 不暴露 Provider 原文或存储细节的后台任务错误 -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] pub struct WorkerTaskError { message: String, + source: Option, } impl WorkerTaskError { @@ -517,6 +530,7 @@ impl WorkerTaskError { pub fn safe(message: impl Into) -> Self { Self { message: message.into(), + source: None, } } @@ -524,6 +538,25 @@ impl WorkerTaskError { pub fn as_safe_str(&self) -> &str { &self.message } + #[must_use] + pub fn with_source(mut self, source: impl Into) -> Self { + self.source = Some(source.into()); + self + } + + #[must_use] + pub fn error_details(&self) -> Option { + crate::error::ErrorDetails::capture(self.source.as_ref(), None, false) + } + + #[must_use] + pub fn with_cleanup(mut self, cleanup: impl Into) -> Self { + self.source = Some(crate::error::ErrorSource::cleanup( + self.source.take(), + cleanup, + )); + self + } } fn valid_owner(owner: &str) -> bool { diff --git a/backend/crates/gateway-core/tests/engine/admission.rs b/backend/crates/gateway-core/tests/engine/admission.rs index ba8808f49..7a3bbe59c 100644 --- a/backend/crates/gateway-core/tests/engine/admission.rs +++ b/backend/crates/gateway-core/tests/engine/admission.rs @@ -72,7 +72,7 @@ fn client_admission_startup_recovery_should_fail_closed_at_each_boundary() { assert!( restore_client_admission_startup( &RecoveryStore::success(0), - &ScriptedRecoveries::new(Err(ClientAdmissionError)), + &ScriptedRecoveries::new(Err(ClientAdmissionError(None))), &RecordingAdmissions::default(), now, ) @@ -212,7 +212,7 @@ impl ClientAdmissionPort for RecordingAdmissions { ) -> BoxFuture<'_, Result> { Box::pin(async move { if self.fail_restore { - return Err(ClientAdmissionError); + return Err(ClientAdmissionError(None)); } self.restored.lock().expect("restored").push(recovery); Ok(ClientAdmissionRestoreResult { diff --git a/backend/crates/gateway-core/tests/engine/execution.rs b/backend/crates/gateway-core/tests/engine/execution.rs index f13f5eb12..3ae6cce71 100644 --- a/backend/crates/gateway-core/tests/engine/execution.rs +++ b/backend/crates/gateway-core/tests/engine/execution.rs @@ -99,7 +99,9 @@ impl ClientBudgetPort for Budget { assert!(self.active.load(Ordering::SeqCst)); self.charges.lock().unwrap().push(charge); if self.fail_settlement { - Err(ClientBudgetError) + Err(ClientBudgetError(Some( + std::io::Error::other("PRIVATE_SETTLEMENT_CAUSE").into(), + ))) } else { Ok(()) } @@ -115,6 +117,7 @@ fn service(admissions: Arc, budget: Arc) -> DefaultExecution admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) } @@ -324,6 +327,7 @@ fn reused_client_uses_updated_limits_for_each_execution() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let limited = RateLimits { @@ -360,6 +364,7 @@ fn reused_client_cannot_start_after_key_disable_or_snapshot_suspension() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let next = request(&service, ClientTransport::WebSocket); if suspend { @@ -463,6 +468,7 @@ fn reused_client_is_reauthenticated_by_the_current_frontend_plan_without_identit admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_frontend_authentication(authentication.clone()); let client = service @@ -554,6 +560,7 @@ fn early_failure_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) } @@ -911,6 +918,7 @@ fn retry_policy_can_stop_but_cannot_bypass_replay_safety() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policies); let mut started = service @@ -1169,6 +1177,7 @@ fn completed_parent_rejects_new_nested_execution_and_cancels_an_active_child() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); @@ -1360,6 +1369,7 @@ fn native_response_processing_uses_the_real_translation_boundary() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_observers(observer_index); @@ -1510,6 +1520,7 @@ fn response_translation_zero_output_preserves_canonical_usage_and_finalization() Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let mut started = service @@ -1663,6 +1674,7 @@ fn native_response_failure_after_downstream_commit_is_not_replayed() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let mut started = service @@ -1735,6 +1747,7 @@ fn model_routing_policy_selects_native_provider() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); let mut started = service @@ -1788,6 +1801,7 @@ fn model_routing_reject_stops_before_provider_execution() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); assert!(matches!( @@ -1875,6 +1889,7 @@ fn model_routing_cannot_expand_the_frozen_key_model_scope() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index); @@ -2022,6 +2037,7 @@ fn scheduler_reject_is_a_terminal_policy_rejection_before_upstream_send() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index) .with_request_observers(observer_index); @@ -2292,6 +2308,7 @@ fn observation_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_budget(budget) .with_request_observers(observer_index); @@ -2329,6 +2346,7 @@ fn fallback_exhaustion_observation_uses_the_last_actual_provider_not_the_candida Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_observers(observer_index); let mut started = service @@ -2393,6 +2411,7 @@ fn routing_external_effect_stops_a_not_sent_provider_retry() { Arc::new(Admissions::default()), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_request_policies(policy_index) .with_request_observers(observer_index); @@ -2579,6 +2598,20 @@ fn charged_service( admissions: Arc, budget: Option>, fail: bool, +) -> DefaultExecutionService { + charged_service_with_diagnostics( + admissions, + budget, + fail, + Arc::new(crate::RecordingDiagnostics::default()), + ) +} + +fn charged_service_with_diagnostics( + admissions: Arc, + budget: Option>, + fail: bool, + diagnostics: Arc, ) -> DefaultExecutionService { let service = DefaultExecutionService::new( RuntimeSnapshotHandle::new(start_snapshot()), @@ -2591,6 +2624,7 @@ fn charged_service( admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + diagnostics, ); match budget { Some(budget) => service.with_budget(budget), @@ -2826,7 +2860,13 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() { fail_settlement, ..Default::default() }); - let service = charged_service(admissions.clone(), Some(budget.clone()), true); + let diagnostics = Arc::new(crate::RecordingDiagnostics::default()); + let service = charged_service_with_diagnostics( + admissions.clone(), + Some(budget.clone()), + true, + diagnostics.clone(), + ); let mut started = service .start(request(&service, ClientTransport::WebSocket)) .await @@ -2840,6 +2880,24 @@ fn settlement_failure_keeps_provider_error_and_releases_concurrency_once() { started.session.detach_finalize().await; // Store 端口已接管精确费用后，结算错误不能改写 Provider 错误或触发第二次结算 assert_cleanup_completed(&admissions, &budget, &started.request_id); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), usize::from(fail_settlement)); + if let Some(failure) = failures.first() { + assert_eq!(failure.operation, "settle_client_budget"); + assert_eq!( + failure.correlation_id.as_deref(), + Some(started.request_id.as_str()) + ); + assert!( + failure + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_SETTLEMENT_CAUSE") + ); + assert!(!format!("{failure:?}").contains("PRIVATE_SETTLEMENT_CAUSE")); + } } }); } @@ -2923,6 +2981,7 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -2954,6 +3013,7 @@ fn probe_failures_should_be_observable_without_a_model_request_row() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -2999,6 +3059,7 @@ fn provider_local_probe_failure_should_remain_distinct_from_upstream() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3041,6 +3102,7 @@ fn diagnostic_probe_does_not_apply_data_plane_account_model_policy() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3072,6 +3134,7 @@ fn probe_observation_store_failure_preserves_the_provider_error() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = block_on(service.probe( @@ -3195,6 +3258,7 @@ fn client_catalog_forwards_scope_maps_whole_objects_and_omits_unroutable_models( Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").expect("authenticate"); let result = block_on(service.client_model_catalog(&client, "codex", "0.154.0")); @@ -3350,6 +3414,7 @@ fn successful_authentication_should_record_client_key_usage() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); service @@ -3369,6 +3434,7 @@ fn client_key_verification_should_not_record_client_key_usage() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); let key_id = service @@ -3390,6 +3456,7 @@ fn request_verification_should_apply_entry_authentication_without_recording_key_ Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), usage.clone(), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service @@ -3423,6 +3490,7 @@ fn assert_provider_endpoint_observation(model: Option<&str>) { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_usage_test") @@ -3476,6 +3544,7 @@ fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -3522,6 +3591,7 @@ fn continuation_owned_by_another_client_api_key_should_fail_closed() { Arc::new(UnusedAdmissions), Arc::new(RejectedContinuation::OwnershipMismatch), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -3549,6 +3619,7 @@ fn invalid_continuation_record_should_not_be_forwarded_as_an_external_handle() { Arc::new(UnusedAdmissions), Arc::new(RejectedContinuation::InvalidData), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service .authenticate("sk_start_test") @@ -4024,6 +4095,7 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() { admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let running = service .start(request(&service, ClientTransport::HttpSse)) @@ -4133,6 +4205,7 @@ fn queue_service( admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); (service, admissions) } @@ -4257,6 +4330,7 @@ fn reused_websocket_client_gets_group_fast_policy_from_each_new_request_snapshot admissions, Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); for (revision, fast_mode) in [ @@ -4306,6 +4380,7 @@ fn request_settings_recompute_routing_and_admission_without_changing_sibling_or_ admissions.clone(), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let mut modified = service.prepare_verified_execution(client.clone()).unwrap(); @@ -4478,6 +4553,7 @@ fn repeated_connection_failures_never_block_later_requests_for_the_provider() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); for _ in 0..5 { let client = service @@ -4528,6 +4604,7 @@ fn entry_settings_freeze_authentication_and_rebase_only_explicit_overrides() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let baseline = service.request_settings().unwrap(); let mut values = serde_json::to_value(baseline.values()).unwrap(); @@ -4670,6 +4747,7 @@ fn unchanged_and_precompiled_request_settings_reuse_the_frozen_snapshot() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = service.authenticate("sk_start_test").unwrap(); let mut prepared = service.prepare_verified_execution(client).unwrap(); @@ -4767,6 +4845,7 @@ fn child_settings_recompute_key_scope_without_inheriting_parent_defaults() { Arc::new(UnusedAdmissions), Arc::new(UnusedContinuation), Arc::new(RecordingClientApiKeyUsage::default()), + Arc::new(crate::RecordingDiagnostics::default()), ); let parent = service .prepare_execution(service.authenticate("sk_parent").unwrap()) diff --git a/backend/crates/gateway-core/tests/error/mod.rs b/backend/crates/gateway-core/tests/error/mod.rs index d95d24486..a264b4729 100644 --- a/backend/crates/gateway-core/tests/error/mod.rs +++ b/backend/crates/gateway-core/tests/error/mod.rs @@ -351,3 +351,34 @@ fn client_visible_upstream_error_should_preserve_opaque_structured_fields() { assert_eq!(detail.error_type(), Some(error_type.as_str())); assert!(!format!("{detail:?}").contains(&message)); } + +#[test] +fn cleanup_failure_keeps_the_primary_cause_and_separate_bounded_details() { + use gateway_core::error::{ErrorDetails, ErrorSource}; + let primary = ErrorSource::new(std::io::Error::other("PRIMARY_NATIVE_FAILURE")); + let combined = primary.with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE")); + assert_eq!( + combined.source().unwrap().to_string(), + "PRIMARY_NATIVE_FAILURE" + ); + let details = ErrorDetails::capture(Some(&combined), None, false).unwrap(); + let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap(); + assert_eq!( + value["causes"]["messages"], + json!(["PRIMARY_NATIVE_FAILURE"]) + ); + assert_eq!( + value["causes"]["cleanup"][0]["messages"], + json!(["ROLLBACK_NATIVE_FAILURE"]) + ); + assert_eq!(value["causes"]["truncated"], false); + assert!(!format!("{combined:?} {details:?}").contains("NATIVE_FAILURE")); + + let oversized = ErrorSource::new(std::io::Error::other("界".repeat(24_000))) + .with_cleanup(std::io::Error::other("ROLLBACK_NATIVE_FAILURE")); + let details = ErrorDetails::capture(Some(&oversized), None, false).unwrap(); + let value: serde_json::Value = serde_json::from_str(details.as_str()).unwrap(); + assert_eq!(value["causes"]["truncated"], true); + assert_eq!(value["causes"]["cleanup"][0]["truncated"], true); + assert!(details.as_str().len() < 66_000); +} diff --git a/backend/crates/gateway-core/tests/main.rs b/backend/crates/gateway-core/tests/main.rs index 395730830..48d0a3cde 100644 --- a/backend/crates/gateway-core/tests/main.rs +++ b/backend/crates/gateway-core/tests/main.rs @@ -21,3 +21,16 @@ mod task; mod upstream; mod time; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-host/src/lib.rs b/backend/crates/gateway-host/src/lib.rs index b409adf35..91efed550 100644 --- a/backend/crates/gateway-host/src/lib.rs +++ b/backend/crates/gateway-host/src/lib.rs @@ -156,8 +156,9 @@ impl HostBundle { &self, plan: Vec, lease: Arc, + diagnostics: Arc, ) -> Result<(), HostError> { - self.workers.start(plan, lease)?; + self.workers.start(plan, lease, diagnostics)?; Ok(()) } diff --git a/backend/crates/gateway-host/src/logging/sink.rs b/backend/crates/gateway-host/src/logging/sink.rs index fe093b946..5f42b984f 100644 --- a/backend/crates/gateway-host/src/logging/sink.rs +++ b/backend/crates/gateway-host/src/logging/sink.rs @@ -1,7 +1,7 @@ -//! 有界文件日志队列：拥堵时背压，正常退出时排空；写入失败显式影响健康状态 +//! 非阻塞文件日志接收与后台批量写入，错误记录保留独立容量，关闭时排空 use std::io::{self, Write}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, mpsc}; use std::thread::{self, JoinHandle}; @@ -11,10 +11,70 @@ use tracing_subscriber::fmt::MakeWriter; use super::writer::RotatingLogWriter; -const QUEUE_CAPACITY: usize = 64; +const QUEUE_CAPACITY: usize = 4_096; +const QUEUE_BYTES: usize = 64 * 1024 * 1024; +const ERROR_CAPACITY: usize = 512; +const ERROR_BYTES: usize = 8 * 1024 * 1024; +const WRITE_BATCH: usize = 64; + +// 通道不另设容量；预留在复制正文之前完成，并随记录释放，包含正在写入的记录 +struct Budget { + items: AtomicUsize, + bytes: AtomicUsize, + maximum_items: usize, + maximum_bytes: usize, +} + +impl Budget { + fn new(maximum_items: usize, maximum_bytes: usize) -> Arc { + Arc::new(Self { + items: AtomicUsize::new(0), + bytes: AtomicUsize::new(0), + maximum_items, + maximum_bytes, + }) + } + + fn reserve(self: &Arc, bytes: usize) -> Option { + self.items + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |items| { + items + .checked_add(1) + .filter(|items| *items <= self.maximum_items) + }) + .ok()?; + if self + .bytes + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| { + used.checked_add(bytes) + .filter(|used| *used <= self.maximum_bytes) + }) + .is_err() + { + self.items.fetch_sub(1, Ordering::AcqRel); + return None; + } + Some(Reservation { + budget: Arc::clone(self), + bytes, + }) + } +} + +struct Reservation { + budget: Arc, + bytes: usize, +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.bytes.fetch_sub(self.bytes, Ordering::AcqRel); + self.budget.items.fetch_sub(1, Ordering::AcqRel); + } +} enum Command { - Write(Vec), + Write(Vec, Reservation), Shutdown, } @@ -22,6 +82,8 @@ enum Command { pub(super) struct LogHealth { write_failures: AtomicU64, maintenance_failures: AtomicU64, + rejected_records: AtomicU64, + rejected_errors: AtomicU64, } impl LogHealth { @@ -36,6 +98,14 @@ impl LogHealth { eprintln!("file_logging: archive/retention maintenance failed ({kind:?})"); } } + + fn rejected(&self, important: bool) { + self.rejected_records.fetch_add(1, Ordering::Relaxed); + if important { + self.rejected_errors.fetch_add(1, Ordering::Relaxed); + } + // 接收路径只更新状态，不能回退到 stderr 或递归打印新的拥堵日志 + } } impl HealthProbe for LogHealth { @@ -46,9 +116,11 @@ impl HealthProbe for LogHealth { fn check(&self) -> BoxFuture<'_, HealthState> { Box::pin(async move { let failures = self.write_failures.load(Ordering::Relaxed); - if failures > 0 { + let rejected = self.rejected_records.load(Ordering::Relaxed); + let errors = self.rejected_errors.load(Ordering::Relaxed); + if failures > 0 || rejected > 0 { return HealthState::Unhealthy(format!( - "log completeness compromised: {failures} write/flush failures since startup" + "log completeness compromised: {failures} write/flush failures, {rejected} rejected records ({errors} warnings/errors) since startup" )); } let failures = self.maintenance_failures.load(Ordering::Relaxed); @@ -64,12 +136,15 @@ impl HealthProbe for LogHealth { #[derive(Clone)] pub(super) struct FileLogSink { - sender: mpsc::SyncSender, + sender: mpsc::Sender, + regular: Arc, + errors: Arc, + important: bool, health: Arc, } pub(super) struct FileLogGuard { - sender: mpsc::SyncSender, + sender: mpsc::Sender, worker: Option>, health: Arc, } @@ -80,20 +155,36 @@ impl FileLogSink { name: &'static str, health: Arc, ) -> io::Result<(Self, FileLogGuard)> { - let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY); + let (sender, receiver) = mpsc::channel(); let worker_health = Arc::clone(&health); let worker = thread::Builder::new() .name(name.to_owned()) .spawn(move || { - while let Ok(command) = receiver.recv() { - match command { - Command::Write(bytes) => { - if let Err(error) = writer.write_all(&bytes) { - worker_health.write_failed(error.kind()); + let mut reservations = Vec::with_capacity(WRITE_BATCH); + 'consume: while let Ok(mut command) = receiver.recv() { + for index in 0..WRITE_BATCH { + match command { + Command::Write(bytes, reservation) => { + if let Err(error) = writer.write_all(&bytes) { + worker_health.write_failed(error.kind()); + } + reservations.push(reservation); } + Command::Shutdown => break 'consume, + } + if index + 1 == WRITE_BATCH { + break; + } + match receiver.try_recv() { + Ok(next) => command = next, + Err(_) => break, } - Command::Shutdown => break, } + if let Err(error) = writer.flush() { + worker_health.write_failed(error.kind()); + } + // 缓冲刷入文件后才归还额度，慢输出不能让在途记录逃逸预算 + reservations.clear(); } if let Err(error) = writer.sync() { worker_health.write_failed(error.kind()); @@ -102,6 +193,9 @@ impl FileLogSink { Ok(( Self { sender: sender.clone(), + regular: Budget::new(QUEUE_CAPACITY, QUEUE_BYTES), + errors: Budget::new(ERROR_CAPACITY, ERROR_BYTES), + important: false, health: Arc::clone(&health), }, FileLogGuard { @@ -119,29 +213,44 @@ impl<'a> MakeWriter<'a> for FileLogSink { fn make_writer(&'a self) -> Self::Writer { self.clone() } + + fn make_writer_for(&'a self, metadata: &tracing::Metadata<'_>) -> Self::Writer { + let mut writer = self.clone(); + writer.important = *metadata.level() <= tracing::Level::WARN; + writer + } } impl Write for FileLogSink { fn write(&mut self, buffer: &[u8]) -> io::Result { - self.sender - .send(Command::Write(buffer.to_vec())) - .map_err(|_| { - self.health.write_failed(io::ErrorKind::BrokenPipe); - io::Error::new(io::ErrorKind::BrokenPipe, "file log worker unavailable") - })?; + let bytes = buffer.len().saturating_add(std::mem::size_of::()); + let reservation = self + .regular + .reserve(bytes) + .or_else(|| self.important.then(|| self.errors.reserve(bytes)).flatten()); + if let Some(reservation) = reservation { + if self + .sender + .send(Command::Write(buffer.to_vec(), reservation)) + .is_err() + { + self.health.rejected(self.important); + } + } else { + self.health.rejected(self.important); + } + // tracing 的 writer 失败回退可能同步写 stderr；缺口由独立健康状态报告 Ok(buffer.len()) } fn flush(&mut self) -> io::Result<()> { - // 文件不额外缓冲，由守卫负责队列排空与同步的生命周期 Ok(()) } } impl Drop for FileLogGuard { fn drop(&mut self) { - // 关闭指令排在所有已接收记录之后 - // 等待线程退出，确保正常关闭时不会因超时而丢弃队列中的日志 + // 仅进程关闭时等待已接收日志落盘，请求侧不承担排空责任 let _ = self.sender.send(Command::Shutdown); if self .worker diff --git a/backend/crates/gateway-host/src/logging/writer.rs b/backend/crates/gateway-host/src/logging/writer.rs index c0561fc83..70743a727 100644 --- a/backend/crates/gateway-host/src/logging/writer.rs +++ b/backend/crates/gateway-host/src/logging/writer.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; -use std::io::{self, Write}; +use std::io::{self, BufWriter, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -20,7 +20,7 @@ pub(super) struct RotatingLogWriter { date: NaiveDate, segment: usize, bytes_written: u64, - file: File, + file: BufWriter, health: Arc, } @@ -58,7 +58,7 @@ impl RotatingLogWriter { date, segment, bytes_written, - file, + file: BufWriter::with_capacity(64 * 1024, file), health, }; // 重启后恢复未完成归档，正在写入的文件不参与压缩 @@ -84,7 +84,7 @@ impl RotatingLogWriter { let previous = self .directory .join(log_file_name(self.prefix, self.date, self.segment)); - self.file.sync_all()?; + self.sync()?; let segment = if day_changed { managed_log_files(&self.directory, self.prefix)? .into_iter() @@ -100,10 +100,10 @@ impl RotatingLogWriter { .join(log_file_name(self.prefix, date, segment)); let file = open_log_segment(&path)?; // 新文件打开成功后才能提交轮转状态 - self.file = file; + self.file = BufWriter::with_capacity(64 * 1024, file); self.date = date; self.segment = segment; - self.bytes_written = self.file.metadata()?.len(); + self.bytes_written = self.file.get_ref().metadata()?.len(); if let Err(error) = compress_log_file(&previous) { self.health.maintenance_failed(error.kind()); } @@ -121,7 +121,8 @@ impl RotatingLogWriter { } pub(super) fn sync(&mut self) -> io::Result<()> { - self.file.sync_all() + self.file.flush()?; + self.file.get_ref().sync_all() } } @@ -129,7 +130,7 @@ impl Write for RotatingLogWriter { fn write(&mut self, buffer: &[u8]) -> io::Result { self.rotate_if_required(buffer.len())?; if let Err(error) = self.file.write_all(buffer) { - self.bytes_written = self.file.metadata()?.len(); + self.bytes_written = self.file.get_ref().metadata()?.len(); return Err(error); } self.bytes_written = self.bytes_written.saturating_add(buffer.len() as u64); diff --git a/backend/crates/gateway-host/src/workers.rs b/backend/crates/gateway-host/src/workers.rs index 4d2136f70..989f5f785 100644 --- a/backend/crates/gateway-host/src/workers.rs +++ b/backend/crates/gateway-host/src/workers.rs @@ -7,6 +7,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use futures::FutureExt as _; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; use gateway_core::health::{ WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState, }; @@ -48,6 +49,7 @@ impl WorkerSupervisor { &self, plan: Vec, leader_lease: Arc, + diagnostics: Arc, ) -> Result<(), WorkerStartError> { if self .started @@ -56,7 +58,7 @@ impl WorkerSupervisor { { return Err(WorkerStartError::AlreadyStarted); } - let result = self.start_inner(plan, leader_lease); + let result = self.start_inner(plan, leader_lease, diagnostics); if result.is_err() { self.started.store(false, Ordering::Release); } @@ -67,6 +69,7 @@ impl WorkerSupervisor { &self, plan: Vec, leader_lease: Arc, + diagnostics: Arc, ) -> Result<(), WorkerStartError> { tokio::runtime::Handle::try_current().map_err(|_| WorkerStartError::RuntimeUnavailable)?; let WorkerPlan { @@ -91,14 +94,14 @@ impl WorkerSupervisor { lease, Arc::clone(&leader_lease), self.cancellation.clone(), - Arc::clone(&self.health), + (Arc::clone(&self.health), Arc::clone(&diagnostics)), )), WorkerRunnable::Daemon { restart, task } => tokio::spawn(supervise_daemon( id.clone(), task, restart, self.cancellation.clone(), - Arc::clone(&self.health), + (Arc::clone(&self.health), Arc::clone(&diagnostics)), )), }; (id, handle) @@ -360,8 +363,9 @@ async fn supervise_scheduled( lease: Option, leader_lease: Arc, cancellation: CancellationToken, - health: Arc, + observation: (Arc, Arc), ) { + let (health, diagnostics) = observation; let mut backoff = schedule.initial_backoff(); loop { if cancellation.is_cancelled() { @@ -397,7 +401,8 @@ async fn supervise_scheduled( .min(schedule.maximum_backoff()) } ScheduledOutcome::Failed(error) => { - let failures = health.failed(&id, error.clone()); + record_worker_failure(diagnostics.as_ref(), &id, &error).await; + let failures = health.failed(&id, error.as_safe_str().to_owned()); let delay = take_backoff(&mut backoff, schedule.maximum_backoff()); tracing::warn!( worker = %id, @@ -438,8 +443,10 @@ async fn acquire_and_run( run_leased_cycle(id, task, guard, schedule, cancellation, health).await } Ok(Ok(WorkerLeaseAcquisition::Busy { retry_after })) => ScheduledOutcome::Busy(retry_after), - Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()), - Err(_) => ScheduledOutcome::Failed("leader lease port panicked".to_owned()), + Ok(Err(error)) => { + ScheduledOutcome::Failed(WorkerTaskError::safe(error.as_safe_str()).with_source(error)) + } + Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease port panicked")), } } @@ -478,7 +485,9 @@ async fn run_leased_cycle( ) -> ScheduledOutcome { let fencing_token = match std::panic::catch_unwind(AssertUnwindSafe(|| guard.fencing_token())) { Ok(token) => token, - Err(_) => return ScheduledOutcome::Failed("leader lease guard panicked".to_owned()), + Err(_) => { + return ScheduledOutcome::Failed(WorkerTaskError::safe("leader lease guard panicked")); + } }; health.running(id, Some(fencing_token)); let cycle_cancel = CancellationToken::new(); @@ -518,38 +527,39 @@ async fn run_leased_cycle( }; match release_guard(guard, schedule.leader_lease_renewal_interval()).await { Ok(()) => outcome, - Err(error) if matches!(outcome, ScheduledOutcome::ShuttingDown) => { - tracing::warn!(worker = %id, error, "leader lease 释放失败"); - outcome - } - Err(error) => ScheduledOutcome::Failed(error), + Err(cleanup) => match outcome { + ScheduledOutcome::Failed(error) => { + ScheduledOutcome::Failed(error.with_cleanup(cleanup)) + } + _ => ScheduledOutcome::Failed(cleanup), + }, } } async fn renew_guard( guard: &mut dyn WorkerLeaderLeaseGuard, timeout: Duration, -) -> Result<(), String> { +) -> Result<(), WorkerTaskError> { let future = AssertUnwindSafe(guard.renew()).catch_unwind(); match tokio::time::timeout(timeout, future).await { Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()), - Ok(Err(_)) => Err("leader lease renewal panicked".to_owned()), - Err(_) => Err("leader lease renewal timed out".to_owned()), + Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)), + Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease renewal panicked")), + Err(_) => Err(WorkerTaskError::safe("leader lease renewal timed out")), } } async fn release_guard( guard: Box, timeout: Duration, -) -> Result<(), String> { +) -> Result<(), WorkerTaskError> { let future = std::panic::catch_unwind(AssertUnwindSafe(|| guard.release())) - .map_err(|_| "leader lease release panicked".to_owned())?; + .map_err(|_| WorkerTaskError::safe("leader lease release panicked"))?; match tokio::time::timeout(timeout, AssertUnwindSafe(future).catch_unwind()).await { Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => Err(error.as_safe_str().to_owned()), - Ok(Err(_)) => Err("leader lease release panicked".to_owned()), - Err(_) => Err("leader lease release timed out".to_owned()), + Ok(Ok(Err(error))) => Err(WorkerTaskError::safe(error.as_safe_str()).with_source(error)), + Ok(Err(_)) => Err(WorkerTaskError::safe("leader lease release panicked")), + Err(_) => Err(WorkerTaskError::safe("leader lease release timed out")), } } @@ -558,8 +568,9 @@ async fn supervise_daemon( task: Box, restart: DaemonRestartPolicy, cancellation: CancellationToken, - health: Arc, + observation: (Arc, Arc), ) { + let (health, diagnostics) = observation; let mut backoff = restart.initial_backoff(); loop { if cancellation.is_cancelled() { @@ -579,12 +590,13 @@ async fn supervise_daemon( }; let Some(result) = result else { break }; let error = match result { - Ok(Ok(())) => "daemon exited unexpectedly".to_owned(), - Ok(Err(error)) => error.as_safe_str().to_owned(), - Err(_) => "daemon panicked".to_owned(), + Ok(Ok(())) => WorkerTaskError::safe("daemon exited unexpectedly"), + Ok(Err(error)) => error, + Err(_) => WorkerTaskError::safe("daemon panicked"), }; let delay = take_backoff(&mut backoff, restart.maximum_backoff()); - let failures = health.failed(&id, error.clone()); + record_worker_failure(diagnostics.as_ref(), &id, &error).await; + let failures = health.failed(&id, error.as_safe_str().to_owned()); tracing::warn!( worker = %id, failures, @@ -603,7 +615,7 @@ async fn supervise_daemon( enum ScheduledOutcome { Succeeded, Busy(Option), - Failed(String), + Failed(WorkerTaskError), ShuttingDown, } @@ -612,8 +624,8 @@ fn task_result( ) -> ScheduledOutcome { match result { Ok(Ok(())) => ScheduledOutcome::Succeeded, - Ok(Err(error)) => ScheduledOutcome::Failed(error.as_safe_str().to_owned()), - Err(_) => ScheduledOutcome::Failed("worker panicked".to_owned()), + Ok(Err(error)) => ScheduledOutcome::Failed(error), + Err(_) => ScheduledOutcome::Failed(WorkerTaskError::safe("worker panicked")), } } @@ -641,3 +653,17 @@ fn lock_unpoisoned(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } + +async fn record_worker_failure( + diagnostics: &dyn OperationalDiagnostics, + id: &WorkerId, + error: &WorkerTaskError, +) { + let mut failure = + OperationalFailure::new("worker", "run", "worker_failed", error.as_safe_str()); + failure.correlation_id = Some(id.to_string()); + failure.details = error.error_details(); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(worker = %id, "worker diagnostic could not be recorded"); + } +} diff --git a/backend/crates/gateway-host/tests/logging/sink.rs b/backend/crates/gateway-host/tests/logging/sink.rs index edb52cffc..59f8e7f73 100644 --- a/backend/crates/gateway-host/tests/logging/sink.rs +++ b/backend/crates/gateway-host/tests/logging/sink.rs @@ -110,3 +110,134 @@ fn write_failure_marks_log_completeness_unhealthy_even_after_writes_recover() { assert!(body.contains("before-failure") && body.contains("after-recovery")); assert!(!body.contains("cannot-write")); } + +#[cfg(unix)] +#[test] +fn blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors() { + const CHILD: &str = "logging::sink::blocked_file_output_keeps_http_and_streaming_responsive_and_reserves_errors"; + if env::var_os(CHILD_PROCESS_ENV).is_some() { + let directory = PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut config = logging_config(directory.clone(), false); + config.logging.oauth_recovery = false; + config.logging.file.max_file_size_mb = 8; + let bundle = runtime.block_on(gateway_host::initialize(config)).unwrap(); + let health = bundle.logging_health_probe(); + let date = gateway_core::time::DeploymentTimeZone::default() + .local(chrono::Utc::now()) + .date_naive(); + let fifo = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.1.log")); + assert!( + Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success() + ); + let (release, released) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + // 超时只用于让旧的同步实现能退出测试，接口必须在解除阻塞之前完成 + let _ = released.recv_timeout(Duration::from_secs(3)); + let mut body = String::new(); + fs::File::open(fifo) + .unwrap() + .read_to_string(&mut body) + .unwrap(); + body + }); + let payload = "x".repeat(8 * 1024 * 1024); + tracing::info!(target: APPLICATION_LOG_TARGET, payload, "rotate before requests"); + tracing::info!(target: APPLICATION_LOG_TARGET, "block on opening next segment"); + runtime.block_on(async { + let initial = directory.join(format!("{APPLICATION_LOG_FILE_PREFIX}{date}.log")); + tokio::time::timeout(Duration::from_secs(2), async { + while fs::metadata(&initial).unwrap().len() < 8 * 1024 * 1024 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }).await.unwrap(); + use futures::{SinkExt as _, StreamExt as _}; + let app = axum::Router::new() + .route("/v1/log-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="v1", "critical_after_congestion"); + "ok" + })) + .route("/admin/log-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="admin", "critical_after_congestion"); + "ok" + })) + .route("/v1/stream-test", axum::routing::get(|| async { + tracing::error!(target: APPLICATION_LOG_TARGET, path="stream", "critical_after_congestion"); + axum::response::Sse::new(futures::stream::iter([ + Ok::<_, std::convert::Infallible>(axum::response::sse::Event::default().data("first")), + Ok(axum::response::sse::Event::default().data("last")), + ])) + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); + let started = std::time::Instant::now(); + for sequence in 0..5_000 { + tracing::info!(target: APPLICATION_LOG_TARGET, sequence, "congested regular log"); + } + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + for path in ["/v1/log-test", "/admin/log-test", "/v1/stream-test"] { + let response = client.get(format!("http://{address}{path}")).send().await.unwrap(); + assert!(response.status().is_success()); + let body = response.text().await.unwrap(); + if path.ends_with("stream-test") { + assert!(body.contains("first") && body.contains("last")); + } else { + assert_eq!(body, "ok"); + } + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let websocket_server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(stream).await.unwrap(); + tracing::error!(target: APPLICATION_LOG_TARGET, path="websocket", "critical_after_congestion"); + socket.send(tokio_tungstenite::tungstenite::Message::Text("frame".into())).await.unwrap(); + }); + let stream = tokio::net::TcpStream::connect(address).await.unwrap(); + let (mut websocket, _) = tokio_tungstenite::client_async(format!("ws://{address}/"), stream).await.unwrap(); + assert_eq!(websocket.next().await.unwrap().unwrap().into_text().unwrap(), "frame"); + websocket_server.await.unwrap(); + let elapsed = started.elapsed(); + assert!(elapsed < Duration::from_secs(1), "interfaces waited for logging: {elapsed:?}"); + eprintln!("paused log output: 5000 submissions and HTTP/admin/SSE/WS completed in {elapsed:?}"); + let HealthState::Unhealthy(message) = health.check().await else { panic!("queue overflow must be visible") }; + assert!(message.contains("0 warnings/errors"), "{message}"); + server.abort(); + }); + release.send(()).unwrap(); + drop(bundle); + let body = reader.join().unwrap(); + assert_eq!( + body.lines() + .filter(|line| line.contains("critical_after_congestion")) + .count(), + 4 + ); + for line in body.lines() { + serde_json::from_str::(line).unwrap(); + } + return; + } + let directory = tempfile::tempdir().unwrap(); + let output = Command::new(env::current_exe().unwrap()) + .args(["--exact", CHILD, "--nocapture"]) + .env(CHILD_PROCESS_ENV, "1") + .env(LOG_DIRECTORY_ENV, directory.path()) + .env("RUST_LOG", "off,logging_test_application=info") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); +} diff --git a/backend/crates/gateway-host/tests/plugin_distribution/github.rs b/backend/crates/gateway-host/tests/plugin_distribution/github.rs index d71c65d41..1fd014f27 100644 --- a/backend/crates/gateway-host/tests/plugin_distribution/github.rs +++ b/backend/crates/gateway-host/tests/plugin_distribution/github.rs @@ -198,13 +198,12 @@ async fn rate_limit_applies_to_other_repositories_using_the_same_identity_and_eg .kind(), AdminErrorKind::RateLimited ); - assert_eq!( - distribution - .query_release(query(None), vec![], None) - .await - .unwrap_err(), - failure - ); + let cached_failure = distribution + .query_release(query(None), vec![], None) + .await + .unwrap_err(); + assert_eq!(cached_failure.kind(), failure.kind()); + assert_eq!(cached_failure.message(), failure.message()); assert_eq!(server.received_requests().await.unwrap().len(), 1); } diff --git a/backend/crates/gateway-host/tests/workers.rs b/backend/crates/gateway-host/tests/workers.rs index 91988de7c..9ff6ecb3b 100644 --- a/backend/crates/gateway-host/tests/workers.rs +++ b/backend/crates/gateway-host/tests/workers.rs @@ -54,7 +54,8 @@ impl ScheduledTask for CountingTask { Box::pin(async move { let run = self.runs.fetch_add(1, Ordering::SeqCst) + 1; if run <= self.fail_until { - Err(WorkerTaskError::safe("expected test failure")) + Err(WorkerTaskError::safe("expected test failure") + .with_source(std::io::Error::other("PRIVATE_WORKER_NATIVE_CAUSE"))) } else { Ok(()) } @@ -343,9 +344,11 @@ async fn registry_composes_multiple_owners_but_rejects_duplicate_identity() { let supervisor = supervisor(); assert!(matches!( - supervisor.start(plan, Arc::new(FakeLeasePort::default())), - Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai" - )); + supervisor.start(plan, Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), + Err(WorkerStartError::Duplicate(id)) if id.owner() == "openai" + )); } #[tokio::test] @@ -360,7 +363,11 @@ async fn only_database_fact_reasons_can_disable_nonexistent_workers() { ))); assert!(matches!( - supervisor().start(plan, Arc::new(FakeLeasePort::default())), + supervisor().start( + plan, + Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), Err(WorkerStartError::KindDisabled(WorkerKind::OpsFlush)) )); } @@ -373,9 +380,11 @@ async fn registry_refuses_start_when_any_real_owner_kind_is_missing() { .collect(); assert!(matches!( - supervisor().start(plan, Arc::new(FakeLeasePort::default())), - Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention] - )); + supervisor().start(plan, Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ), + Err(WorkerStartError::Missing(missing)) if missing == vec![WorkerKind::Retention] + )); } #[tokio::test] @@ -383,7 +392,11 @@ async fn disabled_database_facts_never_execute_or_acquire_a_lease() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(complete_plan(Vec::new()), leases.clone()) + .start( + complete_plan(Vec::new()), + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| leases.requests().len() >= ACTIVE_KINDS.len()).await; let snapshot = supervisor.health_source().snapshot(); @@ -422,7 +435,11 @@ async fn one_kind_runs_all_registered_owner_tasks() { ]); let supervisor = supervisor(); supervisor - .start(plan, Arc::new(FakeLeasePort::default())) + .start( + plan, + Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| second.runs.load(Ordering::SeqCst) >= 1).await; @@ -456,6 +473,7 @@ async fn leader_guard_lives_through_cycle_and_fence_reaches_owner() { .start( target_plan(task.clone(), long_interval_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.entered.notified().await; @@ -478,6 +496,7 @@ async fn long_cycle_renews_leader_lease_before_ttl() { .start( target_plan(task.clone(), renewal_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.entered.notified().await; @@ -500,7 +519,11 @@ async fn renewal_failure_cancels_old_cycle_before_another_leader_cycle_starts() leases.renew_script(target_worker(), [RenewStep::Error]); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), renewal_schedule()), leases) + .start( + target_plan(task.clone(), renewal_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); yield_until(|| task.starts.load(Ordering::SeqCst) == 1).await; tokio::time::advance(Duration::from_millis(10)).await; @@ -520,6 +543,7 @@ async fn supervisor_uses_exponential_backoff_and_recovers_health() { .start( target_plan(task.clone(), backoff_schedule()), Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); yield_until(|| task.runs.load(Ordering::SeqCst) == 1).await; @@ -548,7 +572,11 @@ async fn task_and_lease_port_panics_are_isolated_and_retried() { ); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), backoff_schedule()), leases) + .start( + target_plan(task.clone(), backoff_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); tokio::task::yield_now().await; tokio::time::advance(Duration::from_millis(10)).await; @@ -578,7 +606,11 @@ async fn lease_busy_is_healthy_standby_and_retries_without_running_owner() { ); let supervisor = supervisor(); supervisor - .start(target_plan(task.clone(), backoff_schedule()), leases) + .start( + target_plan(task.clone(), backoff_schedule()), + leases, + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); yield_until(|| task_health(&supervisor, &target_worker()).state == WorkerRuntimeState::Standby) .await; @@ -598,7 +630,11 @@ async fn cooperative_shutdown_propagates_cancel_before_dropping_all_guards() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(all_active_plan(task.clone()), leases.clone()) + .start( + all_active_plan(task.clone()), + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); wait_until(|| task.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len()).await; supervisor.shutdown(Duration::from_secs(1)).await; @@ -619,6 +655,7 @@ async fn shutdown_timeout_aborts_uncooperative_task_and_drops_guard() { .start( target_plan(task.clone(), long_interval_schedule()), leases.clone(), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.notification.notified().await; @@ -666,7 +703,11 @@ async fn shutdown_timeout_must_not_rejoin_workers_that_already_exited() { let leases = Arc::new(FakeLeasePort::default()); let supervisor = supervisor(); supervisor - .start(plan, leases.clone()) + .start( + plan, + leases.clone(), + Arc::new(RecordingDiagnostics::default()), + ) .expect("complete plan"); hung.notification.notified().await; wait_until(|| cooperative.entered.load(Ordering::SeqCst) == ACTIVE_KINDS.len() - 1).await; @@ -691,6 +732,7 @@ async fn running_worker_becomes_unhealthy_when_last_success_is_stale() { .start( target_plan(task.clone(), freshness_schedule()), Arc::new(FakeLeasePort::default()), + Arc::new(RecordingDiagnostics::default()), ) .expect("complete plan"); task.second_cycle.notified().await; @@ -898,3 +940,50 @@ fn lock_unpoisoned(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} + +#[tokio::test] +async fn worker_boundary_records_one_native_failure_and_recovers_health() { + let supervisor = supervisor(); + let task = CountingTask::new(1); + let diagnostics = Arc::new(RecordingDiagnostics::default()); + supervisor + .start( + target_plan(task.clone(), test_schedule()), + Arc::new(FakeLeasePort::default()), + diagnostics.clone(), + ) + .unwrap(); + wait_until(|| task.runs.load(Ordering::SeqCst) >= 2).await; + supervisor.shutdown(Duration::from_secs(1)).await; + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].correlation_id.as_deref(), + Some(target_worker().to_string().as_str()) + ); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_WORKER_NATIVE_CAUSE") + ); + assert!( + !format!("{:?}", supervisor.health_source().snapshot()) + .contains("PRIVATE_WORKER_NATIVE_CAUSE") + ); +} diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs index f89a4cd25..fb224f993 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/command_line/execution.rs @@ -135,10 +135,12 @@ fn rpc_failure(error: RpcError) -> &'static str { match error { RpcError::Timeout => "插件响应超时", RpcError::Cancelled => "调用已取消", - RpcError::Closed | RpcError::Start(_) => "插件进程不可用", - RpcError::Handshake | RpcError::Protocol | RpcError::InvalidResponse(_) => "插件协议错误", + RpcError::Closed(_) | RpcError::Start(_) => "插件进程不可用", + RpcError::Handshake(_) | RpcError::Protocol(_) | RpcError::InvalidResponse(..) => { + "插件协议错误" + } RpcError::Capacity => "插件调用容量不足", - RpcError::Context => "调用上下文无效", + RpcError::Context(_) => "调用上下文无效", RpcError::Remote(_) => "插件拒绝命令", } } diff --git a/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs b/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs index 2aeed2a12..121f16d97 100644 --- a/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs +++ b/backend/crates/gateway-plugin/runtime/src/adapter/policy/middleware.rs @@ -202,7 +202,7 @@ async fn invoke_middleware( ); let (protocol, headers, payload) = match invocation.request_parts() { Ok(projection) => projection, - Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context).await, + Err(_) => return recover_rpc(&entry, &invocation, RpcError::Context(None)).await, }; let mut call_context = invocation_ports.session.context( match entry.mount { diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs b/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs index 9a6388bf8..b5b30217e 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/dispatch.rs @@ -46,7 +46,7 @@ async fn write_loop( frame = data.recv() => frame, }; let Some(frame) = frame else { - shared.fail(RpcError::Closed); + shared.fail(RpcError::Closed(None)); return; }; // 撤销尚未发送的调用时丢弃排队帧；已开始写入的 Call 必须先于 Cancel @@ -64,8 +64,10 @@ async fn write_loop( _ = stopped.changed() => return, result = write_frame(&mut writer, &frame) => result, }; - if written.is_err() { - shared.fail(RpcError::Closed); + if let Err(source) = written { + shared.fail(RpcError::Closed(Some( + gateway_core::error::ErrorSource::new(source), + ))); return; } } @@ -87,9 +89,14 @@ async fn read_loop( _ = stopped.changed() => return, frame = read_frame(&mut reader) => frame, }; - let Ok(frame) = frame else { - shared.fail(RpcError::Closed); - return; + let frame = match frame { + Ok(frame) => frame, + Err(source) => { + shared.fail(RpcError::Closed(Some( + gateway_core::error::ErrorSource::new(source), + ))); + return; + } }; let result = match frame.message { Message::Cancelled { id } if frame.payload.is_empty() => { @@ -114,7 +121,7 @@ async fn read_loop( params, } => { if id == 0 || id % 2 != 0 || id <= last_callback { - Err(RpcError::Protocol) + Err(RpcError::Protocol(None)) } else if let Some(context) = shared.context(parent_id) { last_callback = id; if let Some(permit) = shared.try_callback_slot() { @@ -171,10 +178,10 @@ async fn read_loop( })); Ok(()) } else { - Err(RpcError::Protocol) + Err(RpcError::Protocol(None)) } } - _ => Err(RpcError::Protocol), + _ => Err(RpcError::Protocol(None)), }; if let Err(error) = result { shared.fail(error); diff --git a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs index 2bd0af2c9..70c401968 100644 --- a/backend/crates/gateway-plugin/runtime/src/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/src/rpc/session.rs @@ -45,18 +45,18 @@ impl Default for RpcLimits { } } -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, thiserror::Error)] pub enum RpcError { #[error("plugin process could not be started: {0}")] Start(#[source] ProcessStartError), #[error("plugin handshake does not match the prepared instance")] - Handshake, + Handshake(#[source] Option), #[error("plugin protocol is invalid")] - Protocol, + Protocol(#[source] Option), #[error("plugin returned an invalid response at {0:?}")] - InvalidResponse(Stage), + InvalidResponse(Stage, #[source] Option), #[error("plugin process or transport has stopped")] - Closed, + Closed(#[source] Option), #[error("plugin call exceeded its deadline")] Timeout, #[error("plugin session was cancelled")] @@ -64,7 +64,7 @@ pub enum RpcError { #[error("plugin call capacity is exhausted")] Capacity, #[error("plugin call context is invalid")] - Context, + Context(#[source] Option), #[error("plugin returned an error")] Remote(PluginFault), } @@ -74,9 +74,9 @@ impl RpcError { pub(crate) fn diagnostic(&self) -> (&'static str, &'static str) { match self { RpcError::Start(_) => ("process_start", "插件进程启动失败"), - RpcError::Handshake => ("handshake", "插件握手失败"), - RpcError::Protocol => ("protocol", "插件协议错误"), - RpcError::InvalidResponse(stage) => ( + RpcError::Handshake(_) => ("handshake", "插件握手失败"), + RpcError::Protocol(_) => ("protocol", "插件协议错误"), + RpcError::InvalidResponse(stage, _) => ( "invalid_response", match stage { Stage::Registration => "注册阶段：插件返回了无效响应", @@ -97,11 +97,11 @@ impl RpcError { Stage::Maintenance => "维护阶段：插件返回了无效响应", }, ), - RpcError::Closed => ("closed", "插件进程或传输已停止"), + RpcError::Closed(_) => ("closed", "插件进程或传输已停止"), RpcError::Timeout => ("timeout", "插件调用超时"), RpcError::Cancelled => ("cancelled", "插件会话已取消"), RpcError::Capacity => ("capacity", "插件调用容量已耗尽"), - RpcError::Context => ("context", "插件调用上下文无效"), + RpcError::Context(_) => ("context", "插件调用上下文无效"), RpcError::Remote(_) => ("remote", "插件返回错误"), } } @@ -186,7 +186,7 @@ pub struct RpcSession { impl RpcSession { /// 只用于已确认来自插件的合同错误，不接收业务错误或原始响应内容 pub(crate) fn invalid_response(&self, stage: Stage) -> RpcError { - let error = RpcError::InvalidResponse(stage); + let error = RpcError::InvalidResponse(stage, None); self.shared.fail(error.clone()); error } @@ -196,7 +196,14 @@ impl RpcSession { stage: Stage, value: serde_json::Value, ) -> Result { - serde_json::from_value(value).map_err(|_| self.invalid_response(stage)) + serde_json::from_value(value).map_err(|source| { + let error = RpcError::InvalidResponse( + stage, + Some(gateway_core::error::ErrorSource::new(source)), + ); + self.shared.fail(error.clone()); + error + }) } pub(crate) fn capability_version( @@ -292,16 +299,16 @@ impl Shared { pub fn finish(&self, id: u64, result: Result) -> Result<(), RpcError> { let mut state = self.state(); - let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?; if call.cancellation.is_some() { return Ok(()); } if call.stream.is_some() && result.is_ok() { - let initial = call.result.take().ok_or(RpcError::Protocol)?; + let initial = call.result.take().ok_or(RpcError::Protocol(None))?; call.handler.streaming(&call.context); let _ = initial.send(result); } else { - let pending = state.calls.remove(&id).ok_or(RpcError::Protocol)?; + let pending = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?; drop(state); pending.finish(result); } @@ -310,16 +317,16 @@ impl Shared { pub fn stream_chunk(&self, id: u64, sequence: u64, payload: Vec) -> Result<(), RpcError> { let mut state = self.state(); - let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.get_mut(&id).ok_or(RpcError::Protocol(None))?; if call.cancellation.is_some() { return Ok(()); } if call.result.is_some() { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } call.stream .as_mut() - .ok_or(RpcError::Protocol)? + .ok_or(RpcError::Protocol(None))? .receive(sequence, payload) } @@ -332,11 +339,11 @@ impl Shared { { return Ok(()); } - let call = state.calls.remove(&id).ok_or(RpcError::Protocol)?; + let call = state.calls.remove(&id).ok_or(RpcError::Protocol(None))?; drop(state); if call.stream.is_none() || call.result.is_some() { - call.finish(Err(RpcError::Protocol)); - return Err(RpcError::Protocol); + call.finish(Err(RpcError::Protocol(None))); + return Err(RpcError::Protocol(None)); } let result = error.map_or_else( || { @@ -359,7 +366,7 @@ impl Shared { }; call.stream .as_mut() - .ok_or(RpcError::Protocol)? + .ok_or(RpcError::Protocol(None))? .release(bytes)?; drop(state); self.send_control(Frame::control(Message::Credit { @@ -441,7 +448,7 @@ impl Shared { .get(&id) .is_some_and(|call| call.cancellation.is_some()) { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } state.calls.remove(&id); Ok(()) @@ -540,10 +547,12 @@ impl RpcSession { || limits.handshake_timeout.is_zero() || limits.maximum_call_timeout.is_zero() { - return Err(RpcError::Context); + return Err(RpcError::Context(None)); } let manifest = package.package().manifest(); - let plugin_id = manifest.plugin_id().map_err(|_| RpcError::Handshake)?; + let plugin_id = manifest.plugin_id().map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; if handshake.protocol_version != PROTOCOL_VERSION || manifest .package @@ -557,7 +566,7 @@ impl RpcSession { || handshake.incarnation.is_empty() || handshake.generation == 0 { - return Err(RpcError::Handshake); + return Err(RpcError::Handshake(None)); } let deadline = tokio::time::Instant::now() + limits.handshake_timeout; let process = loop { @@ -597,10 +606,12 @@ impl RpcSession { }), ) .await - .map_err(|_| RpcError::Handshake)?; - let reply = read_frame(&mut output) - .await - .map_err(|_| RpcError::Handshake)?; + .map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; + let reply = read_frame(&mut output).await.map_err(|source| { + RpcError::Handshake(Some(gateway_core::error::ErrorSource::new(source))) + })?; match reply.message { Message::Ready { protocol_version, @@ -611,7 +622,7 @@ impl RpcSession { { Ok(()) } - _ => Err(RpcError::Handshake), + _ => Err(RpcError::Handshake(None)), } }) .await @@ -661,7 +672,7 @@ impl RpcSession { if !monitor.expected_stop.load(Ordering::Acquire) { tracing::warn!(instance_id, exit = ?reason, "插件进程意外停止"); } - monitor.fail(RpcError::Closed); + monitor.fail(RpcError::Closed(None)); // 等进程退出后再回收其文件，避免 Windows 上删除正在执行的制品 drop(package); }); @@ -738,7 +749,7 @@ impl RpcSession { || method.is_empty() || method.len() > 128 { - return Err(RpcError::Context); + return Err(RpcError::Context(None)); } Ok(tokio::time::Instant::now() + Duration::from_millis(context.timeout_ms)) } @@ -807,7 +818,9 @@ impl RpcSession { payload, }; // 本地元数据错误不能进入写队列并关闭整个会话 - validate_frame(&frame).map_err(|_| RpcError::Context)?; + validate_frame(&frame).map_err(|source| { + RpcError::Context(Some(gateway_core::error::ErrorSource::new(source))) + })?; let started = self.begin_call(context, deadline, stream)?; let mut guard = CallGuard { shared: Arc::clone(&self.shared), @@ -815,7 +828,10 @@ impl RpcSession { armed: true, }; let sent = tokio::time::timeout_at(started.deadline, async { - self.data.send(frame).await.map_err(|_| RpcError::Closed)?; + self.data + .send(frame) + .await + .map_err(|_| RpcError::Closed(None))?; if let Some((bytes, frames)) = credit { // 初始信用跟在 Call 后走同一有序队列，不能被控制队列提前发送 self.data @@ -825,7 +841,7 @@ impl RpcSession { frames, })) .await - .map_err(|_| RpcError::Closed)?; + .map_err(|_| RpcError::Closed(None))?; } Ok::<(), RpcError>(()) }) @@ -851,9 +867,9 @@ impl RpcSession { result? } Ok(Err(_)) => { - self.shared.cancel(started.id, RpcError::Closed); + self.shared.cancel(started.id, RpcError::Closed(None)); guard.armed = false; - return Err(RpcError::Closed); + return Err(RpcError::Closed(None)); } Err(_) => { self.shared.cancel(started.id, RpcError::Timeout); @@ -935,7 +951,7 @@ impl RpcSession { .await .is_err() { - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); // 强制停止没有第二段宽限；退出和回调析构是返回前必须确认的事实 self.settle_shutdown().await; } @@ -944,7 +960,7 @@ impl RpcSession { async fn settle_shutdown(&self) { self.shared.process.exited().await; // 不等待监控任务调度；先撤销父调用并中止其回调，再确认回调 future 已析构 - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); self.shared .wait_for_callbacks(self.limits.maximum_callbacks) .await; @@ -954,7 +970,7 @@ impl RpcSession { impl Drop for RpcSession { fn drop(&mut self) { self.shared.expect_stop(); - self.shared.fail(RpcError::Closed); + self.shared.fail(RpcError::Closed(None)); } } diff --git a/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs b/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs index b942b7bb7..500431236 100644 --- a/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs +++ b/backend/crates/gateway-plugin/runtime/src/stream/bridge.rs @@ -43,7 +43,7 @@ impl StreamIngress { self.window.receive(sequence, payload.len())?; self.chunks .try_send(payload) - .map_err(|_| RpcError::Protocol) + .map_err(|_| RpcError::Protocol(None)) } pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> { @@ -86,7 +86,7 @@ impl RpcStream { let Some(terminal) = self.terminal.take() else { return Ok(None); }; - terminal.await.map_err(|_| RpcError::Closed)??; + terminal.await.map_err(|_| RpcError::Closed(None))??; Ok(None) } Err(_) => { diff --git a/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs b/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs index 79d274d46..042b9fe19 100644 --- a/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs +++ b/backend/crates/gateway-plugin/runtime/src/stream/flow_control.rs @@ -19,22 +19,25 @@ impl ReceiveWindow { } pub fn receive(&mut self, sequence: u64, bytes: usize) -> Result<(), RpcError> { - let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol)?; + let bytes = u32::try_from(bytes).map_err(|_| RpcError::Protocol(None))?; if sequence != self.next_sequence || bytes == 0 || bytes > self.bytes || self.frames == 0 { - return Err(RpcError::Protocol); + return Err(RpcError::Protocol(None)); } self.next_sequence = self .next_sequence .checked_add(1) - .ok_or(RpcError::Protocol)?; + .ok_or(RpcError::Protocol(None))?; self.bytes -= bytes; self.frames -= 1; Ok(()) } pub fn release(&mut self, bytes: u32) -> Result<(), RpcError> { - self.bytes = self.bytes.checked_add(bytes).ok_or(RpcError::Protocol)?; - self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol)?; + self.bytes = self + .bytes + .checked_add(bytes) + .ok_or(RpcError::Protocol(None))?; + self.frames = self.frames.checked_add(1).ok_or(RpcError::Protocol(None))?; Ok(()) } } diff --git a/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs b/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs index 20e5846ed..2bf95b3a6 100644 --- a/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs +++ b/backend/crates/gateway-plugin/runtime/tests/generation/maintenance.rs @@ -5,6 +5,7 @@ use gateway_admin::{ model::{ account_groups::{AccountGroupColor, CreateAccountGroup}, plugin_resources::{GroupMembersChange, ManagedKeyConfig, PluginResourceOwner}, + plugins::instances::PluginInstanceRuntimeStatus, }, ports::plugin_resources::PluginResourceAccess, }; @@ -112,6 +113,7 @@ async fn revision(environment: &Environment) -> gateway_admin::model::Revision { async fn start( environment: &Environment, + phase: &str, ) -> ( Arc, gateway_core::CoreBundle, @@ -120,6 +122,32 @@ async fn start( tokio::task::JoinHandle<()>, ) { let (runtime, core) = environment.runtime().await; + // 集合就绪允许保留隔离的启动失败实例；维护测试必须确认插件本身正在运行 + let snapshot = core.snapshots().snapshot_for_diagnostics().unwrap(); + let extensions = snapshot.extensions().unwrap(); + let instances = environment + .store + .admin_ports() + .plugins() + .load_instances() + .await + .unwrap(); + let diagnostics = gateway_admin::ports::plugins::PluginRuntimeDiagnostics::runtime_diagnostics( + runtime.as_ref(), + &instances, + Some(snapshot.revision().get()), + Some(extensions), + ) + .await + .unwrap(); + assert!( + !diagnostics.is_empty() + && diagnostics + .values() + .all(|instance| instance.status == PluginInstanceRuntimeStatus::Running), + "plugin was not ready during {phase}: {diagnostics:?}" + ); + drop(snapshot); let resources = gateway_admin::initialize_plugin_resources( environment.store.admin_ports().plugin_resources(), core.snapshot_control(), @@ -141,7 +169,7 @@ async fn start( (runtime, core, resources, stop, task) } -async fn wait_done(path: &std::path::Path, after: usize) -> Value { +async fn wait_done(path: &std::path::Path, after: usize, phase: &str) -> Value { tokio::time::timeout(Duration::from_secs(15), async { loop { let entries: Vec = std::fs::read_to_string(path) @@ -162,7 +190,7 @@ async fn wait_done(path: &std::path::Path, after: usize) -> Value { .await .unwrap_or_else(|error| { panic!( - "plugin reconciliation did not finish: {error}; observations: {}", + "plugin reconciliation did not finish during {phase}: {error}; observations: {}", std::fs::read_to_string(path).unwrap_or_default(), ) }) @@ -178,7 +206,7 @@ async fn malformed_maintenance_response_does_not_stop_the_host_worker() { json!({"maintenance_fixture":true,"invalid_response_method":"plugin.reconcile"}), ) .await; - let (runtime, core, resources, stop, task) = start(&environment).await; + let (runtime, core, resources, stop, task) = start(&environment, "malformed response").await; tokio::time::timeout(Duration::from_secs(5), async { while core .snapshots() @@ -220,8 +248,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou let marker = environment.directory.path().join("maintenance.jsonl"); let failed = environment.directory.path().join("failed-once"); environment.install_plugin(json!({"maintenance_fixture":true, "maintenance_marker":marker, "maintenance_fail_once":failed})).await; - let (runtime, core, resources, stop, task) = start(&environment).await; - let first = wait_done(&marker, 0).await; + let (runtime, core, resources, stop, task) = start(&environment, "initial startup").await; + let first = wait_done(&marker, 0, "initial startup").await; assert!(failed.exists()); let group_id = gateway_core::routing::AccountGroupId::new(first["group"].as_str().unwrap()).unwrap(); @@ -284,8 +312,8 @@ async fn published_reconciliation_provisions_retries_imports_and_restores_withou // 离线新增账号后重新启动真实插件子进程；稳定资源键必须复用原分组与 Key let offline = environment.account(None).await; std::fs::write(&marker, "").unwrap(); - let (runtime, core, resources, stop, task) = start(&environment).await; - let restored = wait_done(&marker, 0).await; + let (runtime, core, resources, stop, task) = start(&environment, "restoration").await; + let restored = wait_done(&marker, 0, "restoration").await; assert_eq!(restored["group"], first["group"]); assert_eq!(restored["key"], first["key"]); let members = environment diff --git a/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs b/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs index 2292a82fa..2ca74a003 100644 --- a/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs +++ b/backend/crates/gateway-plugin/runtime/tests/rpc/session.rs @@ -325,7 +325,7 @@ async fn shutdown_returns_after_in_flight_callback_future_is_dropped() { .await .expect("session shutdown"); assert!(callbacks.dropped.load(Ordering::Acquire)); - assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed))); + assert!(matches!(calling.await.unwrap(), Err(RpcError::Closed(_)))); } #[tokio::test] @@ -416,7 +416,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() { .await .err() .expect("oversized call must fail"); - assert_eq!(error, RpcError::Context); + assert!(matches!(error, RpcError::Context(_))); assert!(matches!( session .call_stream( @@ -426,7 +426,7 @@ async fn oversized_call_frames_do_not_close_the_session_or_begin_callbacks() { payload, ) .await, - Err(RpcError::Context) + Err(RpcError::Context(_)) )); assert_eq!(callbacks.begun.load(Ordering::Relaxed), begun); assert!(session.is_ready()); @@ -653,7 +653,7 @@ async fn unknown_parent_cannot_borrow_another_calls_resources() { vec![], ) .await; - assert!(matches!(reply, Err(RpcError::Protocol))); + assert!(matches!(reply, Err(RpcError::Protocol(_)))); assert_eq!(callbacks.called.load(Ordering::Relaxed), 0); } @@ -712,12 +712,16 @@ async fn child_crash_fails_the_call_without_waiting_for_its_deadline() { let result = tokio::time::timeout(Duration::from_secs(2), call) .await .unwrap(); - assert!(matches!(result, Err(RpcError::Closed))); + assert!(matches!(result, Err(RpcError::Closed(_)))); } #[tokio::test] async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() { - for method in ["malformed_truncated_frame", "malformed_frame_length"] { + for method in [ + "malformed_truncated_frame", + "malformed_frame_length", + "malformed_metadata", + ] { let callbacks = Arc::new(LifecycleCallbacks::default()); let (_cache, session) = session(Arc::clone(&callbacks)).await; let result = tokio::time::timeout( @@ -731,7 +735,29 @@ async fn malformed_worker_frames_fail_boundedly_and_release_call_resources() { ) .await .expect("malformed worker output must fail within the process boundary"); - assert!(matches!(result, Err(RpcError::Closed))); + let error = result.err().expect("malformed frame failure"); + assert!(matches!(error, RpcError::Closed(Some(_)))); + if method == "malformed_metadata" { + use std::error::Error as _; + let frame = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(frame.source().unwrap().is::()); + let provider = gateway_core::error::ProviderError::new( + gateway_core::error::ProviderErrorKind::Protocol, + gateway_core::upstream::UpstreamSendState::Ambiguous, + ) + .with_source(error); + assert!( + provider + .error_details() + .unwrap() + .contains("PRIVATE_INVALID_FRAME_KIND") + ); + assert!(!format!("{provider:?} {provider}").contains("PRIVATE_INVALID_FRAME_KIND")); + } assert!(!session.is_ready()); assert_eq!(callbacks.begun.load(Ordering::Relaxed), 1); assert_eq!(callbacks.finished.load(Ordering::Relaxed), 1); @@ -786,7 +812,7 @@ async fn stream_window_and_sequence_violations_fail_closed() { ) .await .unwrap(); - assert!(matches!(stream.next().await, Err(RpcError::Protocol))); + assert!(matches!(stream.next().await, Err(RpcError::Protocol(_)))); session.shutdown(Duration::from_secs(1)).await; } } diff --git a/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs b/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs index f3fbb3ade..fce20bbbc 100644 --- a/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs +++ b/backend/crates/gateway-plugin/runtime/tests/support/worker/mod.rs @@ -1368,9 +1368,19 @@ async fn main() { last_call = id; if matches!( method.as_str(), - "malformed_truncated_frame" | "malformed_frame_length" + "malformed_truncated_frame" | "malformed_frame_length" | "malformed_metadata" ) { - let bytes: &[u8] = if method == "malformed_truncated_frame" { + let mut malformed_metadata = Vec::new(); + if method == "malformed_metadata" { + let metadata = br#"{"type":"PRIVATE_INVALID_FRAME_KIND"}"#; + malformed_metadata + .extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + malformed_metadata.extend_from_slice(&0_u64.to_be_bytes()); + malformed_metadata.extend_from_slice(metadata); + } + let bytes: &[u8] = if method == "malformed_metadata" { + &malformed_metadata + } else if method == "malformed_truncated_frame" { // 声明 16 字节元数据，却只写入一个字节后退出 &[0, 0, 0, 16, 0, 0, 0, 0, b'{'] } else { diff --git a/backend/crates/gateway-plugin/sdk/src/client/frame.rs b/backend/crates/gateway-plugin/sdk/src/client/frame.rs index e7e666216..68707cf3d 100644 --- a/backend/crates/gateway-plugin/sdk/src/client/frame.rs +++ b/backend/crates/gateway-plugin/sdk/src/client/frame.rs @@ -14,7 +14,7 @@ pub async fn read_frame(reader: &mut R) -> Result( writer: &mut W, frame: &Frame, ) -> Result<(), FrameError> { - let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?; + let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?; check_metadata_length(metadata.len())?; let metadata_len = u32::try_from(metadata.len()).map_err(|_| FrameError::Length)?; let payload_len = u64::try_from(frame.payload.len()).map_err(|_| FrameError::Length)?; @@ -49,7 +49,7 @@ pub async fn write_frame( /// 入队前校验本地元数据，避免单次编码失败中断共享传输 pub fn validate_frame(frame: &Frame) -> Result<(), FrameError> { - let metadata = serde_json::to_vec(&frame.message).map_err(|_| FrameError::Metadata)?; + let metadata = serde_json::to_vec(&frame.message).map_err(FrameError::Metadata)?; check_metadata_length(metadata.len()) } diff --git a/backend/crates/gateway-plugin/sdk/src/message.rs b/backend/crates/gateway-plugin/sdk/src/message.rs index dc1693ae6..8c4ddedfa 100644 --- a/backend/crates/gateway-plugin/sdk/src/message.rs +++ b/backend/crates/gateway-plugin/sdk/src/message.rs @@ -116,7 +116,7 @@ pub enum FrameError { #[error("plugin frame exceeds its limit or has invalid lengths")] Length, #[error("plugin frame metadata is invalid")] - Metadata, + Metadata(#[source] serde_json::Error), #[error("plugin transport is closed or incomplete")] Io(#[from] std::io::Error), } diff --git a/backend/crates/gateway-plugin/sdk/tests/message.rs b/backend/crates/gateway-plugin/sdk/tests/message.rs index 6265d0028..b82f97715 100644 --- a/backend/crates/gateway-plugin/sdk/tests/message.rs +++ b/backend/crates/gateway-plugin/sdk/tests/message.rs @@ -22,3 +22,24 @@ fn diagnostics_do_not_render_peer_payloads_or_error_details() { }; assert!(!format!("{frame:?} {fault:?}").contains(secret)); } + +#[tokio::test] +async fn malformed_frame_metadata_retains_the_native_json_location() { + use std::error::Error as _; + let metadata = br#"{"type": "PRIVATE_UNKNOWN_MESSAGE_KIND"}"#; + let mut wire = Vec::new(); + wire.extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + wire.extend_from_slice(&0_u64.to_be_bytes()); + wire.extend_from_slice(metadata); + let error = gateway_plugin_sdk::client::read_frame(&mut wire.as_slice()) + .await + .unwrap_err(); + let source = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert_eq!(source.line(), 1); + assert!(source.column() > 0); + assert!(source.is_data()); +} diff --git a/backend/crates/gateway-store/src/backup/staging.rs b/backend/crates/gateway-store/src/backup/staging.rs index b20f83611..3e5326722 100644 --- a/backend/crates/gateway-store/src/backup/staging.rs +++ b/backend/crates/gateway-store/src/backup/staging.rs @@ -74,6 +74,7 @@ impl StagingArea { .map_err(|source| unavailable("read staging free space", source))?; if free < MIN_STAGING_FREE_BYTES { return Err(StoreError::InvalidData { + source: None, entity: "backup staging", message: "staging disk space is below 1 GiB".to_owned(), }); @@ -103,6 +104,7 @@ fn unavailable( fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "backup staging", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/bundle.rs b/backend/crates/gateway-store/src/bundle.rs index 4a6406a21..9cbf61a4b 100644 --- a/backend/crates/gateway-store/src/bundle.rs +++ b/backend/crates/gateway-store/src/bundle.rs @@ -7,6 +7,7 @@ use super::*; /// 已完成连接、迁移与 hydration 的 Store 能力集合 pub struct StoreBundle { + diagnostics: Arc, admin_ports: AdminStorePorts, core_ports: CoreStorePorts, provider_ports: ProviderStorePorts, @@ -19,6 +20,11 @@ pub struct StoreBundle { } impl StoreBundle { + #[must_use] + pub fn diagnostics(&self) -> Arc { + self.diagnostics.clone() + } + #[must_use] pub fn admin_ports(&self) -> AdminStorePorts { self.admin_ports.clone() @@ -199,6 +205,7 @@ async fn connect( let (execution, execution_writer) = postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository)); let execution = Arc::new(execution); + let diagnostics: Arc = execution.clone(); let (client_key_usage, client_key_usage_writer) = postgres::PgClientApiKeyUsageSink::new(pool.clone()); let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone())); @@ -212,7 +219,7 @@ async fn connect( REDIS_NAMESPACE, )?); let (admissions, admission_release_writer) = - redis::BufferedClientAdmissionPort::new(admissions); + redis::BufferedClientAdmissionPort::new(admissions, diagnostics.clone()); let core_ports = CoreStorePorts::new( execution, ( @@ -230,6 +237,7 @@ async fn connect( )?), ), Arc::new(client_key_usage), + diagnostics.clone(), ) .with_budget(Arc::new( postgres::PgClientBudgetStore::new(pool.clone()).with_timezone(config.timezone), @@ -250,6 +258,7 @@ async fn connect( cooldowns, runtime_policy, oauth_pending, + diagnostics.clone(), ); let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new( credential_leases, @@ -283,6 +292,7 @@ async fn connect( ), }; Ok(StoreBundle { + diagnostics, admin_ports, core_ports, provider_ports, diff --git a/backend/crates/gateway-store/src/config.rs b/backend/crates/gateway-store/src/config.rs index f772b3739..fffb15284 100644 --- a/backend/crates/gateway-store/src/config.rs +++ b/backend/crates/gateway-store/src/config.rs @@ -51,6 +51,7 @@ impl StorePoolConfig { pub(crate) fn validate(&self) -> StoreResult<()> { if self.max_connections < 2 || self.acquire_timeout_seconds == 0 { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "pool.max_connections must be at least 2 and acquire timeout must be positive" @@ -81,6 +82,7 @@ impl StoreConfig { pub fn resolve_and_validate(&mut self, runtime_data_dir: &Path) -> StoreResult<()> { if runtime_data_dir.as_os_str().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "runtime_data_dir must not be empty".to_owned(), }); @@ -107,6 +109,7 @@ impl StoreConfig { self.pool.validate()?; if self.backup_staging_dir.as_os_str().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: "runtime_data_dir was not resolved".to_owned(), }); @@ -132,12 +135,14 @@ impl StoreConfig { pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult> { match std::env::var(name) { Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("environment variable {name} is empty"), }), Ok(value) => Ok(Some(value)), Err(std::env::VarError::NotPresent) => Ok(None), Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("environment variable {name} is not Unicode"), }), @@ -167,6 +172,7 @@ impl StoreConnectionConfig { if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.password must be exactly 48 hexadecimal characters"), }); @@ -176,17 +182,20 @@ impl StoreConnectionConfig { fn connection_url(&self, field: &'static str) -> StoreResult { let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url is invalid"), })?; if url.password().is_some() { return Err(StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url must not contain a password"), }); } url.set_password(Some(&self.password)) .map_err(|()| StoreError::InvalidData { + source: None, entity: "store config", message: format!("{field}.url cannot carry credentials"), })?; diff --git a/backend/crates/gateway-store/src/postgres/account_groups.rs b/backend/crates/gateway-store/src/postgres/account_groups.rs index fcae8fe90..04022aa5b 100644 --- a/backend/crates/gateway-store/src/postgres/account_groups.rs +++ b/backend/crates/gateway-store/src/postgres/account_groups.rs @@ -93,7 +93,7 @@ impl PgAccountGroupRepository { let mut transaction = self.pool.begin().await.map_err(|source| { admin_store_error(ENTITY, unavailable("begin account group mutation", source)) })?; - let result = async { + let result: StoreResult<_> = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; mutation(&mut transaction).await?; append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?; @@ -108,12 +108,10 @@ impl PgAccountGroupRepository { admin_revision(revision) } Err(error) => { - transaction.rollback().await.map_err(|source| { - admin_store_error( - ENTITY, - unavailable("rollback account group mutation", source), - ) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(admin_store_error(ENTITY, error)) } } @@ -210,14 +208,18 @@ impl AccountGroupStore for PgAccountGroupRepository { .map(|row| { let group_id = AccountGroupId::new( row.try_get::("account_group_id") - .map_err(|_| invalid("invalid group ID"))?, + .map_err(|source| invalid("invalid group ID").with_source(source))?, ) - .map_err(|_| invalid("invalid group ID"))?; + .map_err(|source| invalid("invalid group ID").with_source(source))?; let default_slots = u64::try_from( row.try_get::("max_concurrent_per_account") - .map_err(|_| invalid("invalid default account concurrency"))?, + .map_err(|source| { + invalid("invalid default account concurrency").with_source(source) + })?, ) - .map_err(|_| invalid("invalid default account concurrency"))?; + .map_err(|source| { + invalid("invalid default account concurrency").with_source(source) + })?; let account = account_summary_from_row(row)?; Ok(AccountGroupMemberFact { group_id, @@ -459,7 +461,8 @@ async fn count_groups(pool: &PgPool, query: &AccountGroupListQuery) -> StoreResu .fetch_one(pool) .await .map_err(|source| unavailable("count account groups", source))?; - u64::try_from(count).map_err(|_| invalid("negative account group count")) + u64::try_from(count) + .map_err(|source| invalid("negative account group count").with_source(source)) } async fn load_record(pool: &PgPool, id: &str) -> StoreResult> { @@ -479,33 +482,35 @@ async fn load_record(pool: &PgPool, id: &str) -> StoreResult StoreResult { let provider_counts: serde_json::Value = row .try_get("provider_counts") - .map_err(|_| invalid("invalid provider counts"))?; + .map_err(|source| invalid("invalid provider counts").with_source(source))?; let provider_counts = serde_json::from_value::>(provider_counts) - .map_err(|_| invalid("invalid provider counts"))?; + .map_err(|source| invalid("invalid provider counts").with_source(source))?; Ok(AccountGroupRecord { fast_mode: FastMode::parse( row.try_get("fast_mode") - .map_err(|_| invalid("invalid fast_mode"))?, + .map_err(|source| invalid("invalid fast_mode").with_source(source))?, ) .ok_or_else(|| invalid("invalid fast_mode"))?, id: AccountGroupId::new( row.try_get::("id") - .map_err(|_| invalid("invalid id"))?, + .map_err(|source| invalid("invalid id").with_source(source))?, ) - .map_err(|_| invalid("invalid id"))?, - name: row.try_get("name").map_err(|_| invalid("invalid name"))?, + .map_err(|source| invalid("invalid id").with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid("invalid name").with_source(source))?, description: row .try_get("description") - .map_err(|_| invalid("invalid description"))?, + .map_err(|source| invalid("invalid description").with_source(source))?, color: AccountGroupColor::parse( row.try_get::("color") - .map_err(|_| invalid("invalid color"))? + .map_err(|source| invalid("invalid color").with_source(source))? .as_str(), ) .ok_or_else(|| invalid("invalid color"))?, enabled: row .try_get("enabled") - .map_err(|_| invalid("invalid enabled"))?, + .map_err(|source| invalid("invalid enabled").with_source(source))?, member_count: count_value(row, "member_count")?, provider_counts, client_key_count: count_value(row, "client_key_count")?, @@ -519,16 +524,17 @@ fn group_record(row: &sqlx::postgres::PgRow) -> StoreResult total_slots: Some(0), }, usage: AccountGroupUsage { - today_usd: DecimalAmount::from_str("0").map_err(|_| invalid("invalid zero cost"))?, + today_usd: DecimalAmount::from_str("0") + .map_err(|source| invalid("invalid zero cost").with_source(source))?, retained_total_usd: DecimalAmount::from_str("0") - .map_err(|_| invalid("invalid zero cost"))?, + .map_err(|source| invalid("invalid zero cost").with_source(source))?, }, created_at: row .try_get("created_at") - .map_err(|_| invalid("invalid created_at"))?, + .map_err(|source| invalid("invalid created_at").with_source(source))?, updated_at: row .try_get("updated_at") - .map_err(|_| invalid("invalid updated_at"))?, + .map_err(|source| invalid("invalid updated_at").with_source(source))?, }) } @@ -580,20 +586,21 @@ async fn group_costs( .map(|row| { let group_id: String = row .try_get("group_id") - .map_err(|_| invalid("invalid group ID"))?; + .map_err(|source| invalid("invalid group ID").with_source(source))?; let today = row .try_get::("today_usd") - .map_err(|_| invalid("invalid today cost"))?; + .map_err(|source| invalid("invalid today cost").with_source(source))?; let retained_total = row .try_get::("retained_total_usd") - .map_err(|_| invalid("invalid retained total cost"))?; + .map_err(|source| invalid("invalid retained total cost").with_source(source))?; Ok(( group_id, AccountGroupUsage { today_usd: DecimalAmount::from_str(&today) - .map_err(|_| invalid("invalid today cost"))?, - retained_total_usd: DecimalAmount::from_str(&retained_total) - .map_err(|_| invalid("invalid retained total cost"))?, + .map_err(|source| invalid("invalid today cost").with_source(source))?, + retained_total_usd: DecimalAmount::from_str(&retained_total).map_err( + |source| invalid("invalid retained total cost").with_source(source), + )?, }, )) }) @@ -603,9 +610,9 @@ async fn group_costs( fn count_value(row: &sqlx::postgres::PgRow, field: &str) -> StoreResult { u64::try_from( row.try_get::(field) - .map_err(|_| invalid("invalid count"))?, + .map_err(|source| invalid("invalid count").with_source(source))?, ) - .map_err(|_| invalid("negative count")) + .map_err(|source| invalid("negative count").with_source(source)) } fn validate_page_query(query: &AccountGroupListQuery) -> AdminStoreResult<()> { @@ -658,6 +665,7 @@ fn not_found(id: &str) -> AdminStoreError { fn not_found_store(id: &str) -> StoreError { StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), } @@ -665,6 +673,7 @@ fn not_found_store(id: &str) -> StoreError { fn conflict(id: &str) -> StoreError { StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: ConflictKind::InvalidTransition, @@ -677,6 +686,7 @@ fn invalid_admin(message: &str) -> AdminStoreError { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } @@ -695,7 +705,7 @@ pub(crate) async fn insert_account_group_in_transaction( command: &NewAccountGroup, ) -> StoreResult<()> { validate_group_fields(&command.name, command.description.as_deref()) - .map_err(|_| invalid("invalid account group fields"))?; + .map_err(|source| invalid("invalid account group fields").with_source(source))?; sqlx::query( "insert into account_groups (id, name, description, color, fast_mode, enabled, created_at, updated_at) diff --git a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs index ad8d5f2eb..814589c66 100644 --- a/backend/crates/gateway-store/src/postgres/admin_security_audit.rs +++ b/backend/crates/gateway-store/src/postgres/admin_security_audit.rs @@ -214,8 +214,9 @@ pub(crate) async fn append_admin_audit_event_in_transaction( event.config_revision = revision .into() .map(|revision| { - i64::try_from(revision.get()) - .map_err(|_| invalid("config revision exceeds PostgreSQL bigint")) + i64::try_from(revision.get()).map_err(|source| { + invalid("config revision exceeds PostgreSQL bigint").with_source(source) + }) }) .transpose()?; event.validate()?; @@ -244,6 +245,7 @@ pub(crate) async fn append_admin_audit_event_in_transaction( fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/admission_recovery.rs b/backend/crates/gateway-store/src/postgres/admission_recovery.rs index 217bf4c96..98ceaedea 100644 --- a/backend/crates/gateway-store/src/postgres/admission_recovery.rs +++ b/backend/crates/gateway-store/src/postgres/admission_recovery.rs @@ -110,18 +110,18 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository { Box::pin(async move { self.load_client_admission_recovery(DateTime::::from(since)) .await - .map_err(|_| ClientAdmissionError)? + .map_err(|source| ClientAdmissionError(Some(source.into())))? .into_iter() .map(|recovery| { let client_api_key_id = ClientApiKeyId::new(recovery.client_api_key_ref) - .map_err(|_| ClientAdmissionError)?; + .map_err(|source| ClientAdmissionError(Some(source.into())))?; let recent_requests = recovery .recent_requests .into_iter() .map(|request| { Ok(RecentAdmissionFact { model_request_id: ModelRequestId::new(request.model_request_id) - .map_err(|_| ClientAdmissionError)?, + .map_err(|source| ClientAdmissionError(Some(source.into())))?, started_at: request.started_at.into(), }) }) @@ -132,7 +132,7 @@ impl ClientAdmissionRecoveryPort for PgClientAdmissionRecoveryRepository { .map(|request| { Ok(RunningAdmissionFact { model_request_id: ModelRequestId::new(request.model_request_id) - .map_err(|_| ClientAdmissionError)?, + .map_err(|source| ClientAdmissionError(Some(source.into())))?, expires_at: request.deadline_at.into(), }) }) diff --git a/backend/crates/gateway-store/src/postgres/backup.rs b/backend/crates/gateway-store/src/postgres/backup.rs index 243cf4b3d..8f4a0a83d 100644 --- a/backend/crates/gateway-store/src/postgres/backup.rs +++ b/backend/crates/gateway-store/src/postgres/backup.rs @@ -55,12 +55,10 @@ impl BackupRepository for PgBackupRepository { command: UpdateBackupStorageCommand, context: &MutationContext, ) -> AdminStoreResult<(BackupSettings, AdminRevision)> { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup storage update"))?; - let result = async { + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup storage update").with_source(source) + })?; + let result: AdminStoreResult<_> = async { let current = lock_settings_in_transaction(&mut transaction).await?; let mut changed = storage_changed_fields(¤t, &command); if changed.is_empty() { @@ -68,11 +66,14 @@ impl BackupRepository for PgBackupRepository { sqlx::query_scalar("select config_revision from runtime_settings where id = 1") .fetch_one(&mut *transaction) .await - .map_err(|_| store_unavailable("load unchanged backup revision"))?; + .map_err(|source| { + store_unavailable("load unchanged backup revision").with_source(source) + })?; let revision = AdminRevision::new( - u64::try_from(revision).map_err(|_| store_invalid("config revision"))?, + u64::try_from(revision) + .map_err(|source| store_invalid("config revision").with_source(source))?, ) - .map_err(|_| store_invalid("config revision"))?; + .map_err(|source| store_invalid("config revision").with_source(source))?; return Ok((current, revision)); } ensure_storage_identity_stable(¤t, &command, &mut transaction).await?; @@ -106,7 +107,7 @@ impl BackupRepository for PgBackupRepository { .bind(command.force_path_style) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("update backup storage"))?; + .map_err(|source| store_unavailable("update backup storage").with_source(source))?; let store_revision = bump_config_revision_in_transaction(&mut transaction) .await @@ -135,17 +136,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(result) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup storage update"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup storage update").with_source(source) + })?; Ok(result) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup storage update"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -158,11 +158,9 @@ impl BackupRepository for PgBackupRepository { context: &MutationContext, timezone: gateway_core::time::DeploymentTimeZone, ) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup schedule update"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup schedule update").with_source(source) + })?; let result = async { sqlx::query( "update backup_settings @@ -183,7 +181,7 @@ impl BackupRepository for PgBackupRepository { .bind(next_run_at) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("update backup schedule"))?; + .map_err(|source| store_unavailable("update backup schedule").with_source(source))?; let audit = crate::mutation_audit( context, @@ -205,17 +203,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(settings) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup schedule update"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup schedule update").with_source(source) + })?; Ok(settings) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup schedule update"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -231,20 +228,23 @@ impl BackupRepository for PgBackupRepository { set last_verified_at = $2, updated_at = now() where id = 1 and storage_revision = $1", ) - .bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?) + .bind( + i64::try_from(storage_revision) + .map_err(|source| store_invalid("storage revision").with_source(source))?, + ) .bind(at) .execute(&self.pool) .await - .map_err(|_| store_unavailable("record backup storage verification"))?; + .map_err(|source| { + store_unavailable("record backup storage verification").with_source(source) + })?; Ok(result.rows_affected() == 1) } async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin backup record insert"))?; + let mut transaction = self.pool.begin().await.map_err(|source| { + store_unavailable("begin backup record insert").with_source(source) + })?; let result = async { let _ = lock_settings_in_transaction(&mut transaction).await?; insert_queued_in_transaction(&mut transaction, &seed).await @@ -252,17 +252,16 @@ impl BackupRepository for PgBackupRepository { .await; match result { Ok(record) => { - transaction - .commit() - .await - .map_err(|_| store_unavailable("commit backup record insert"))?; + transaction.commit().await.map_err(|source| { + store_unavailable("commit backup record insert").with_source(source) + })?; Ok(record) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback backup record insert"))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -276,11 +275,10 @@ impl BackupRepository for PgBackupRepository { expected_timezone: &str, expected_next_run_at: Option>, ) -> AdminStoreResult { - let mut transaction = self - .pool - .begin() - .await - .map_err(|_| store_unavailable("begin scheduled insert"))?; + let mut transaction = + self.pool.begin().await.map_err(|source| { + store_unavailable("begin scheduled insert").with_source(source) + })?; // 先锁定并核对计划，防止配置变更后仍排入旧计划任务 let advanced = sqlx::query( "update backup_settings set next_run_at = $1 @@ -293,52 +291,47 @@ impl BackupRepository for PgBackupRepository { .bind(expected_next_run_at) .execute(&mut *transaction) .await - .map_err(|_| store_unavailable("guard scheduled insert"))? + .map_err(|source| store_unavailable("guard scheduled insert").with_source(source))? .rows_affected() > 0; if !advanced { - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback stale schedule"))?; + transaction.rollback().await.map_err(|source| { + store_unavailable("rollback stale schedule").with_source(source) + })?; return Ok(false); } // 唯一约束冲突只跳过本次任务；保存点避免连带撤销游标推进 let mut insertion = transaction .begin() .await - .map_err(|_| store_unavailable("begin scheduled savepoint"))?; + .map_err(|source| store_unavailable("begin scheduled savepoint").with_source(source))?; let inserted = match insert_queued_in_transaction(&mut insertion, &seed).await { Ok(_) => { - insertion - .commit() - .await - .map_err(|_| store_unavailable("release scheduled savepoint"))?; + insertion.commit().await.map_err(|source| { + store_unavailable("release scheduled savepoint").with_source(source) + })?; true } Err(error) if is_active_or_scheduled_conflict(&error) => { - insertion - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled savepoint"))?; + insertion.rollback().await.map_err(|source| { + store_unavailable("rollback scheduled savepoint").with_source(source) + })?; false } - Err(error) => { - insertion - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled savepoint"))?; - transaction - .rollback() - .await - .map_err(|_| store_unavailable("rollback scheduled insert"))?; + Err(mut error) => { + if let Err(cleanup) = insertion.rollback().await { + error = error.with_cleanup(cleanup); + } + if let Err(cleanup) = transaction.rollback().await { + error = error.with_cleanup(cleanup); + } return Err(error); } }; transaction .commit() .await - .map_err(|_| store_unavailable("commit scheduled insert"))?; + .map_err(|source| store_unavailable("commit scheduled insert").with_source(source))?; Ok(inserted) } @@ -361,7 +354,7 @@ impl BackupRepository for PgBackupRepository { .bind(trigger) .fetch_one(&self.pool) .await - .map_err(|_| store_unavailable("count backup records"))?; + .map_err(|source| store_unavailable("count backup records").with_source(source))?; let rows = sqlx::query( "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, @@ -376,10 +369,13 @@ impl BackupRepository for PgBackupRepository { .bind(status) .bind(trigger) .bind(i64::from(limit)) - .bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?) + .bind( + i64::try_from(offset) + .map_err(|source| store_invalid("page offset").with_source(source))?, + ) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list backup records"))?; + .map_err(|source| store_unavailable("list backup records").with_source(source))?; let items = rows .iter() @@ -387,7 +383,8 @@ impl BackupRepository for PgBackupRepository { .collect::, _>>()?; Ok(BackupRecordPage { items, - total: u64::try_from(total).map_err(|_| store_invalid("record count"))?, + total: u64::try_from(total) + .map_err(|source| store_invalid("record count").with_source(source))?, page: query.page, page_size: query.page_size, }) @@ -403,7 +400,7 @@ impl BackupRepository for PgBackupRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("load backup record"))? + .map_err(|source| store_unavailable("load backup record").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -419,7 +416,9 @@ impl BackupRepository for PgBackupRepository { ) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list intermediate backup records"))?; + .map_err(|source| { + store_unavailable("list intermediate backup records").with_source(source) + })?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -439,7 +438,7 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list pending deletions"))?; + .map_err(|source| store_unavailable("list pending deletions").with_source(source))?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -463,7 +462,7 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list expired backups"))?; + .map_err(|source| store_unavailable("list expired backups").with_source(source))?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -497,7 +496,7 @@ impl BackupRepository for PgBackupRepository { .bind(now) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("claim queued backup task"))? + .map_err(|source| store_unavailable("claim queued backup task").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -513,7 +512,7 @@ impl BackupRepository for PgBackupRepository { .size_bytes .map(i64::try_from) .transpose() - .map_err(|_| store_invalid("size bytes"))?; + .map_err(|source| store_invalid("size bytes").with_source(source))?; sqlx::query( "update backup_records set status = $1, @@ -542,7 +541,7 @@ impl BackupRepository for PgBackupRepository { .bind(transition.from().as_str()) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("transition backup task"))? + .map_err(|source| store_unavailable("transition backup task").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -564,7 +563,7 @@ impl BackupRepository for PgBackupRepository { .bind(id) .fetch_optional(&self.pool) .await - .map_err(|_| store_unavailable("transition backup to deleting"))? + .map_err(|source| store_unavailable("transition backup to deleting").with_source(source))? .map(|row| backup_record_from_row(&row)) .transpose() } @@ -575,7 +574,7 @@ impl BackupRepository for PgBackupRepository { .execute(&self.pool) .await .map(|_| ()) - .map_err(|_| store_unavailable("delete backup record")) + .map_err(|source| store_unavailable("delete backup record").with_source(source)) } async fn advance_schedule_cursor( @@ -602,7 +601,9 @@ impl BackupRepository for PgBackupRepository { .bind(timezone) .execute(&self.pool) .await - .map_err(|_| store_unavailable("advance backup schedule cursor"))?; + .map_err(|source| { + store_unavailable("advance backup schedule cursor").with_source(source) + })?; Ok(result.rows_affected() == 1) } @@ -623,7 +624,9 @@ impl BackupRepository for PgBackupRepository { .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|_| store_unavailable("list scheduled completed backups"))?; + .map_err(|source| { + store_unavailable("list scheduled completed backups").with_source(source) + })?; rows.iter() .map(backup_record_from_row) .collect::, _>>() @@ -649,7 +652,7 @@ async fn lock_settings_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| store_unavailable("lock backup settings"))? + .map_err(|source| store_unavailable("lock backup settings").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -666,7 +669,7 @@ async fn load_settings_in_transaction( ) .fetch_optional(&mut **transaction) .await - .map_err(|_| store_unavailable("load backup settings in transaction"))? + .map_err(|source| store_unavailable("load backup settings in transaction").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -681,7 +684,7 @@ async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult ) .fetch_optional(pool) .await - .map_err(|_| store_unavailable("load backup settings"))? + .map_err(|source| store_unavailable("load backup settings").with_source(source))? .map(|row| backup_settings_from_row(&row)) .transpose()? .ok_or_else(|| store_not_found("backup settings")) @@ -761,7 +764,9 @@ async fn ensure_storage_identity_stable( sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)") .fetch_one(&mut **transaction) .await - .map_err(|_| store_unavailable("check backup records existence"))?; + .map_err(|source| { + store_unavailable("check backup records existence").with_source(source) + })?; if records_exist { return Err(AdminStoreError::new( AdminStoreErrorKind::Conflict, @@ -815,7 +820,8 @@ async fn append_audit_in_transaction( ) -> StoreResult<()> { event.config_revision = revision .map(|revision| { - i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData { + i64::try_from(revision.get()).map_err(|source| StoreError::InvalidData { + source: Some(source.into()), entity: "backup", message: "revision overflow".to_owned(), }) @@ -849,7 +855,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult, _>("size_bytes"))? .map(u64::try_from) .transpose() - .map_err(|_| store_invalid("size_bytes"))?; + .map_err(|source| store_invalid("size_bytes").with_source(source))?; Ok(BackupRecord { id: decode(row.try_get("id"))?, trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?) @@ -861,7 +867,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult("attempt_count"))?) - .map_err(|_| store_invalid("attempt_count"))?, + .map_err(|source| store_invalid("attempt_count").with_source(source))?, error_code: decode(row.try_get("error_code"))?, error_message: decode(row.try_get("error_message"))?, started_at: decode(row.try_get("started_at"))?, @@ -874,7 +880,7 @@ fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult AdminStoreResult { let storage_revision = u64::try_from(decode(row.try_get::("storage_revision"))?) - .map_err(|_| store_invalid("storage_revision"))?; + .map_err(|source| store_invalid("storage_revision").with_source(source))?; let secret = decode(row.try_get::, _>("secret_access_key"))?; Ok(BackupSettings { storage_revision, @@ -889,9 +895,9 @@ fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult("retention_days"))?) - .map_err(|_| store_invalid("retention_days"))?, + .map_err(|source| store_invalid("retention_days").with_source(source))?, retention_count: u32::try_from(decode(row.try_get::("retention_count"))?) - .map_err(|_| store_invalid("retention_count"))?, + .map_err(|source| store_invalid("retention_count").with_source(source))?, next_run_at: decode(row.try_get("next_run_at"))?, last_verified_at: decode(row.try_get("last_verified_at"))?, updated_at: decode(row.try_get("updated_at"))?, @@ -911,22 +917,7 @@ fn decode(result: Result) -> AdminStoreResult { /// StoreError → AdminStoreError 的显式映射 fn map_admin_error(error: StoreError) -> AdminStoreError { - match error { - StoreError::Unavailable { .. } => AdminStoreError::new( - AdminStoreErrorKind::Unavailable, - "backup", - "backup store unavailable", - ), - StoreError::NotFound { entity, .. } => { - AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found") - } - StoreError::Conflict { entity, .. } => { - AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict") - } - StoreError::InvalidData { entity, message } => { - AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message) - } - } + crate::admin_store_error("backup", error) } fn store_unavailable(operation: &'static str) -> AdminStoreError { diff --git a/backend/crates/gateway-store/src/postgres/client_budgets.rs b/backend/crates/gateway-store/src/postgres/client_budgets.rs index d1e7dad32..190880430 100644 --- a/backend/crates/gateway-store/src/postgres/client_budgets.rs +++ b/backend/crates/gateway-store/src/postgres/client_budgets.rs @@ -65,6 +65,7 @@ async fn reset_client_key_budget_in_transaction( .map_err(|source| postgres_unavailable("lock budget reset key", source))?; if exists.is_none() { return Err(StoreError::NotFound { + source: None, entity: "client API key", id: command.id.as_str().to_owned(), }); @@ -158,9 +159,15 @@ impl PgClientBudgetStore { .cloned() .collect::>(); for charge in retries { - self.settle(charge).await.map_err(|_| unavailable())?; + self.settle(charge) + .await + .map_err(|source| unavailable().with_source(source))?; } - let mut tx = self.pool.begin().await.map_err(|_| unavailable())?; + let mut tx = self + .pool + .begin() + .await + .map_err(|source| unavailable().with_source(source))?; let row = sqlx::query( "select daily_limit_usd::text, weekly_limit_usd::text, enabled from client_api_keys where id = $1 for update", @@ -168,7 +175,7 @@ impl PgClientBudgetStore { .bind(key_id.as_str()) .fetch_optional(&mut *tx) .await - .map_err(|_| unavailable())? + .map_err(|source| unavailable().with_source(source))? .ok_or_else(|| { GatewayError::new( GatewayErrorKind::Unauthorized, @@ -185,16 +192,16 @@ impl PgClientBudgetStore { daily_usd: row .get::("daily_limit_usd") .parse() - .map_err(|_| unavailable())?, + .map_err(|source| unavailable().with_source(source))?, weekly_usd: row .get::("weekly_limit_usd") .parse() - .map_err(|_| unavailable())?, + .map_err(|source| unavailable().with_source(source))?, }; let now = Utc::now(); advance_windows(&mut tx, key_id.as_str(), now, now, self.timezone) .await - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; if limits.is_limited() { let window = sqlx::query( "select daily_used_usd::text, weekly_used_usd::text, daily_end, weekly_end @@ -203,15 +210,15 @@ impl PgClientBudgetStore { .bind(key_id.as_str()) .fetch_one(&mut *tx) .await - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let daily: Decimal = window .get::("daily_used_usd") .parse() - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let weekly: Decimal = window .get::("weekly_used_usd") .parse() - .map_err(|_| unavailable())?; + .map_err(|source| unavailable().with_source(source))?; let daily_exceeded = limits.daily_usd != Decimal::ZERO && daily >= limits.daily_usd; let weekly_exceeded = limits.weekly_usd != Decimal::ZERO && weekly >= limits.weekly_usd; if daily_exceeded || weekly_exceeded { @@ -235,11 +242,17 @@ impl PgClientBudgetStore { .with_retry_after(retry)); } } - tx.commit().await.map_err(|_| unavailable()) + tx.commit() + .await + .map_err(|source| unavailable().with_source(source)) } async fn settle_inner(&self, charge: &ClientBudgetCharge) -> Result<(), ClientBudgetError> { - let mut tx = self.pool.begin().await.map_err(|_| ClientBudgetError)?; + let mut tx = self + .pool + .begin() + .await + .map_err(|source| ClientBudgetError(Some(source.into())))?; // 与准入统一先锁 Key，再写窗口和费用，串行化同一 Key 的并发结算 let key = sqlx::query_scalar::<_, String>( "select id from client_api_keys where id = $1 for update", @@ -247,12 +260,14 @@ impl PgClientBudgetStore { .bind(charge.key_id.as_str()) .fetch_optional(&mut *tx) .await - .map_err(|_| ClientBudgetError)?; + .map_err(|source| ClientBudgetError(Some(source.into())))?; let Some(key) = key else { return Ok(()) }; // 删除 Key 时也会删除其费用记录 settle_in_transaction(&mut tx, &key, charge, self.timezone) .await - .map_err(|_| ClientBudgetError)?; - tx.commit().await.map_err(|_| ClientBudgetError) + .map_err(|source| ClientBudgetError(Some(source.into())))?; + tx.commit() + .await + .map_err(|source| ClientBudgetError(Some(source.into()))) } } @@ -296,7 +311,14 @@ impl ClientBudgetPort for PgClientBudgetStore { fn settle(&self, charge: ClientBudgetCharge) -> BoxFuture<'_, Result<(), ClientBudgetError>> { Box::pin(async move { let result = self.settle_inner(&charge).await; - let mut retry = self.retry.lock().map_err(|_| ClientBudgetError)?; + // 重试缓存的锁异常不能覆盖已捕获的账本失败 + let mut retry = self.retry.lock().map_err(|_| { + result + .as_ref() + .err() + .cloned() + .unwrap_or(ClientBudgetError(None)) + })?; if result.is_err() { retry.insert(charge.request_id.as_str().to_owned(), charge); } else { diff --git a/backend/crates/gateway-store/src/postgres/client_keys.rs b/backend/crates/gateway-store/src/postgres/client_keys.rs index 86dd97045..fbe4baeb9 100644 --- a/backend/crates/gateway-store/src/postgres/client_keys.rs +++ b/backend/crates/gateway-store/src/postgres/client_keys.rs @@ -74,13 +74,17 @@ impl ClientApiKeySnapshot { ) -> StoreResult { Ok(Self { request_profiles: std::collections::BTreeMap::new(), - id: ClientApiKeyId::new(id).map_err(|_| invalid("persisted key ID is invalid"))?, - plaintext_key: PlaintextClientApiKey::new(key) - .map_err(|_| invalid("persisted plaintext key is invalid"))?, + id: ClientApiKeyId::new(id) + .map_err(|source| invalid("persisted key ID is invalid").with_source(source))?, + plaintext_key: PlaintextClientApiKey::new(key).map_err(|source| { + invalid("persisted plaintext key is invalid").with_source(source) + })?, group_ids: group_ids .into_iter() .map(|id| { - AccountGroupId::new(id).map_err(|_| invalid("persisted group ID is invalid")) + AccountGroupId::new(id).map_err(|source| { + invalid("persisted group ID is invalid").with_source(source) + }) }) .collect::>>()?, limits: RateLimits { @@ -703,6 +707,7 @@ impl PgAdminClientKeyStore { admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: ENTITY, id: id.as_str().to_owned(), }, @@ -748,6 +753,7 @@ impl ClientKeyStore for PgAdminClientKeyStore { .map_err(|source| map_error(postgres_unavailable("lock budget limits key", source)))?; let (daily, weekly) = current.ok_or_else(|| { map_error(StoreError::NotFound { + source: None, entity: ENTITY, id: command.id.as_str().to_owned(), }) @@ -1159,6 +1165,7 @@ pub(crate) async fn insert_client_api_key_in_transaction( && database.constraint() == Some("client_api_keys_key_sha256_idx") }) { StoreError::Conflict { + source: None, entity: ENTITY, id: key.id.clone(), kind: crate::ConflictKind::InvalidTransition, @@ -1238,6 +1245,7 @@ async fn ensure_client_key_name_available( .map_err(|source| postgres_unavailable("check client API key name", source))?; if duplicate { return Err(StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: crate::ConflictKind::DuplicateName, @@ -1282,6 +1290,7 @@ fn require_changed(rows_affected: u64, id: &str) -> StoreResult<()> { Ok(()) } else { Err(StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), }) @@ -1304,6 +1313,7 @@ async fn replace_client_api_key_groups_in_transaction( .map_err(|source| postgres_unavailable("validate client API key groups", source))?; if usize::try_from(count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { + source: None, entity: "account group", id: "client API key group selection".to_owned(), }); @@ -1342,8 +1352,9 @@ fn validate_group_ids(group_ids: &[String]) -> StoreResult<()> { } fn validate_key(key: &str) -> StoreResult<()> { - PlaintextClientApiKey::validate(key) - .map_err(|_| invalid("key must contain nonempty visible ASCII characters")) + PlaintextClientApiKey::validate(key).map_err(|source| { + invalid("key must contain nonempty visible ASCII characters").with_source(source) + }) } fn validate_request_profiles<'a>( @@ -1381,7 +1392,8 @@ fn decode_request_profiles( .into_iter() .map(|(provider, profile)| { Ok(( - ProviderKind::new(provider).map_err(|_| invalid("invalid request profile key"))?, + ProviderKind::new(provider) + .map_err(|source| invalid("invalid request profile key").with_source(source))?, OpaqueProviderData::new(profile), )) }) @@ -1404,7 +1416,7 @@ fn client_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult StoreResult>, >, _>("request_profile_overrides") - .map_err(|_| invalid("invalid request profiles"))? + .map_err(|source| invalid("invalid request profiles").with_source(source))? .0, )?, - id: row.try_get("id").map_err(|_| invalid("invalid id"))?, - name: row.try_get("name").map_err(|_| invalid("invalid name"))?, - label: row.try_get("label").map_err(|_| invalid("invalid label"))?, - groups: serde_json::from_value(groups).map_err(|_| invalid("invalid groups"))?, + id: row + .try_get("id") + .map_err(|source| invalid("invalid id").with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid("invalid name").with_source(source))?, + label: row + .try_get("label") + .map_err(|source| invalid("invalid label").with_source(source))?, + groups: serde_json::from_value(groups) + .map_err(|source| invalid("invalid groups").with_source(source))?, provider_kinds: row .try_get("provider_kinds") - .map_err(|_| invalid("invalid provider kinds"))?, + .map_err(|source| invalid("invalid provider kinds").with_source(source))?, prefix: row .try_get("prefix") - .map_err(|_| invalid("invalid prefix"))?, + .map_err(|source| invalid("invalid prefix").with_source(source))?, enabled: row .try_get("enabled") - .map_err(|_| invalid("invalid enabled"))?, + .map_err(|source| invalid("invalid enabled").with_source(source))?, max_concurrency: to_u64( row.try_get("max_concurrency") - .map_err(|_| invalid("invalid max concurrency"))?, + .map_err(|source| invalid("invalid max concurrency").with_source(source))?, )?, requests_per_minute: to_u64( row.try_get("requests_per_minute") - .map_err(|_| invalid("invalid requests per minute"))?, + .map_err(|source| invalid("invalid requests per minute").with_source(source))?, )?, budget: ClientBudgetStatus::default(), last_used_at: row .try_get("last_used_at") - .map_err(|_| invalid("invalid last used at"))?, + .map_err(|source| invalid("invalid last used at").with_source(source))?, created_at: row .try_get("created_at") - .map_err(|_| invalid("invalid created at"))?, + .map_err(|source| invalid("invalid created at").with_source(source))?, updated_at: row .try_get("updated_at") - .map_err(|_| invalid("invalid updated at"))?, + .map_err(|source| invalid("invalid updated at").with_source(source))?, }, ) } @@ -1525,34 +1544,38 @@ async fn load_client_key_memberships( use sqlx::Row as _; let key_id: String = row .try_get("key_id") - .map_err(|_| invalid("invalid client API key membership"))?; + .map_err(|source| invalid("invalid client API key membership").with_source(source))?; let group_id: Option = row .try_get("group_id") - .map_err(|_| invalid("invalid client API key group"))?; + .map_err(|source| invalid("invalid client API key group").with_source(source))?; if let Some(group_id) = group_id { groups .entry(key_id.clone()) .or_default() .push(ClientApiKeyGroupRecord { id: group_id, - name: row - .try_get("group_name") - .map_err(|_| invalid("invalid client API key group name"))?, - color: row - .try_get("group_color") - .map_err(|_| invalid("invalid client API key group color"))?, - enabled: row - .try_get("group_enabled") - .map_err(|_| invalid("invalid client API key group state"))?, + name: row.try_get("group_name").map_err(|source| { + invalid("invalid client API key group name").with_source(source) + })?, + color: row.try_get("group_color").map_err(|source| { + invalid("invalid client API key group color").with_source(source) + })?, + enabled: row.try_get("group_enabled").map_err(|source| { + invalid("invalid client API key group state").with_source(source) + })?, }); providers.entry(key_id).or_default().extend( row.try_get::, _>("group_provider_kinds") - .map_err(|_| invalid("invalid client API key provider kinds"))?, + .map_err(|source| { + invalid("invalid client API key provider kinds").with_source(source) + })?, ); } else { providers.entry(key_id).or_default().extend( row.try_get::, _>("global_provider_kinds") - .map_err(|_| invalid("invalid global provider kinds"))?, + .map_err(|source| { + invalid("invalid global provider kinds").with_source(source) + })?, ); } } @@ -1649,15 +1672,18 @@ fn push_client_key_order(statement: &mut QueryBuilder, sort: ClientApi } fn to_i64(value: u64) -> StoreResult { - i64::try_from(value).map_err(|_| invalid("numeric limit exceeds PostgreSQL bigint")) + i64::try_from(value) + .map_err(|source| invalid("numeric limit exceeds PostgreSQL bigint").with_source(source)) } fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("persisted numeric limit is negative")) + u64::try_from(value) + .map_err(|source| invalid("persisted numeric limit is negative").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index d77577ce6..9ea7c305f 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -631,7 +631,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(attempt.http_version) .bind( i32::try_from(attempt.attempt_count) - .map_err(|_| invalid("attempt_count is too large"))?, + .map_err(|source| invalid("attempt_count is too large").with_source(source))?, ) .bind(optional_i64( attempt.account_selection_wait_ms, @@ -649,11 +649,13 @@ impl ModelRequestRepository for PgExecutionStore { .await .map_err(|source| postgres_unavailable("begin model request attempt", source))? .ok_or(StoreError::Conflict { + source: None, entity: ENTITY, id: attempt.model_request_id, kind: ConflictKind::DownstreamAlreadyCommitted, })?; - u32::try_from(count).map_err(|_| invalid("attempt_count is invalid")) + u32::try_from(count) + .map_err(|source| invalid("attempt_count is invalid").with_source(source)) } async fn mark_upstream_send_state( @@ -847,7 +849,7 @@ impl ModelRequestRepository for PgExecutionStore { .bind(finalization.upstream_send_state.as_str()) .bind( i32::try_from(finalization.attempt_count) - .map_err(|_| invalid("attempt_count is too large"))?, + .map_err(|source| invalid("attempt_count is too large").with_source(source))?, ) .bind(finalization.downstream_committed_at) .bind(finalization.client_status_code.map(i32::from)) @@ -992,6 +994,7 @@ impl ExecutionStore for PgExecutionStore { Box::pin(async move { let ttl_ms = i64::try_from(ttl.as_millis()).map_err(|_| { crate::StoreError::InvalidData { + source: None, entity: "model request", message: "lease TTL is invalid".to_owned(), } @@ -1172,7 +1175,9 @@ impl ExecutionStore for PgExecutionStore { upstream_model_id: None, failure_kind: error.kind().as_str().to_owned(), upstream_send_state: None, - error_details: None, + error_details: error + .error_details() + .map(gateway_core::error::ErrorDetails::into_string), status_code: None, provider_error_code: error.client_error_code().map(str::to_owned), retry_after_ms: error @@ -1596,12 +1601,55 @@ fn validate_connection_observation(finalization: &ModelRequestFinalization) -> S } fn to_i64(value: u64, field: &'static str) -> StoreResult { - i64::try_from(value).map_err(|_| invalid(field)) + i64::try_from(value).map_err(|source| invalid(field).with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } } + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for PgExecutionStore { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), CoreStoreError> { + use super::{OpsEvent, OpsEventLevel, OpsEventRepository, PgOpsEventRepository}; + let event = OpsEvent { + id: Uuid::now_v7().to_string(), + model_request_id: None, + attempt_index: None, + level: OpsEventLevel::Warning, + component: failure.component.to_owned(), + operation: failure.operation.to_owned(), + provider_kind: failure.provider_kind.map(|kind| kind.as_str().to_owned()), + provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()), + provider_account_ref: failure.account_id.map(|id| id.as_str().to_owned()), + upstream_model_id: None, + failure_kind: failure.kind.to_owned(), + upstream_send_state: None, + error_details: failure + .details + .map(gateway_core::error::ErrorDetails::into_string), + status_code: failure.upstream_status, + provider_error_code: failure.upstream_code.map(|code| code.as_str().to_owned()), + retry_after_ms: None, + upstream_request_id: None, + latency_ms: None, + message: serde_json::json!({ + "correlationId": failure.correlation_id, + "message": failure.message, + }) + .to_string(), + created_at: failure.occurred_at.into(), + }; + PgOpsEventRepository::new(self.pool.clone()) + .append_ops_event(event) + .await + .map_err(core_store_error) + } +} diff --git a/backend/crates/gateway-store/src/postgres/execution_buffer.rs b/backend/crates/gateway-store/src/postgres/execution_buffer.rs index faac9b2a2..61990b64c 100644 --- a/backend/crates/gateway-store/src/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/src/postgres/execution_buffer.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant, SystemTime}; use async_trait::async_trait; use futures::future::try_join_all; +use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure}; use gateway_core::engine::{ AttemptRecord, EntryRejection, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, @@ -41,8 +42,8 @@ pub struct ExecutionBufferStats { } struct ExecutionBufferState { - maximum_queued_items: usize, - maximum_queued_bytes: usize, + regular_item_capacity: usize, + regular_byte_capacity: usize, queued_items: AtomicUsize, queued_bytes: AtomicUsize, enqueued_total: AtomicU64, @@ -53,10 +54,10 @@ struct ExecutionBufferState { } impl ExecutionBufferState { - fn new(maximum_queued_items: NonZeroUsize, maximum_queued_bytes: NonZeroUsize) -> Self { + fn new(regular_item_capacity: NonZeroUsize, regular_byte_capacity: NonZeroUsize) -> Self { Self { - maximum_queued_items: maximum_queued_items.get(), - maximum_queued_bytes: maximum_queued_bytes.get(), + regular_item_capacity: regular_item_capacity.get(), + regular_byte_capacity: regular_byte_capacity.get(), queued_items: AtomicUsize::new(0), queued_bytes: AtomicUsize::new(0), enqueued_total: AtomicU64::new(0), @@ -67,12 +68,21 @@ impl ExecutionBufferState { } } - fn reserve(&self, bytes: usize) -> Result<(), ReservationFailure> { + fn reserve(&self, bytes: usize, critical: bool) -> Result<(), ReservationFailure> { + // 常规额度之外保留四分之一给失败与请求生命周期，普通进度更新不能消耗 + let item_limit = self.regular_item_capacity.saturating_add(if critical { + self.regular_item_capacity / 4 + } else { + 0 + }); + let byte_limit = self.regular_byte_capacity.saturating_add(if critical { + self.regular_byte_capacity / 4 + } else { + 0 + }); self.queued_items .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { - current - .checked_add(1) - .filter(|next| *next <= self.maximum_queued_items) + current.checked_add(1).filter(|next| *next <= item_limit) }) .map_err(|_| ReservationFailure::ItemCapacity)?; let bytes_reserved = self @@ -80,7 +90,7 @@ impl ExecutionBufferState { .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { current .checked_add(bytes) - .filter(|next| *next <= self.maximum_queued_bytes) + .filter(|next| *next <= byte_limit) }) .is_ok(); if !bytes_reserved { @@ -189,16 +199,17 @@ impl BufferedExecutionStore { ) } + /// 参数限制常规接收额度；失败与生命周期记录可使用额外四分之一预留 #[must_use] pub fn with_limits( inner: Arc, capacity: NonZeroUsize, - maximum_queued_bytes: NonZeroUsize, + regular_byte_capacity: NonZeroUsize, ) -> (Self, ExecutionObservationWriter) { let lane_count = PERSISTENCE_LANES.min(capacity.get()); let (senders, receivers): (Vec<_>, Vec<_>) = (0..lane_count).map(|_| mpsc::unbounded_channel()).unzip(); - let state = Arc::new(ExecutionBufferState::new(capacity, maximum_queued_bytes)); + let state = Arc::new(ExecutionBufferState::new(capacity, regular_byte_capacity)); ( Self { inner: Arc::clone(&inner), @@ -224,16 +235,19 @@ impl BufferedExecutionStore { .estimated_bytes() .saturating_add(write.request_id().map_or(0, str::len)) .max(1); - if let Err(failure) = self.state.reserve(estimated_bytes) { + if let Err(failure) = self.state.reserve(estimated_bytes, write.is_critical()) { self.state.record_dropped(1); let stats = self.state.snapshot(); + if !stats.dropped_total.is_power_of_two() { + return; + } tracing::warn!( operation = write.operation(), request_id = ?write.request_id(), reason = failure.reason(), estimated_bytes, - maximum_queued_items = self.state.maximum_queued_items, - maximum_queued_bytes = self.state.maximum_queued_bytes, + regular_item_capacity = self.state.regular_item_capacity, + regular_byte_capacity = self.state.regular_byte_capacity, queued_items = stats.queued_items, queued_bytes = stats.queued_bytes, dropped_total = stats.dropped_total, @@ -253,6 +267,9 @@ impl BufferedExecutionStore { drop(queued); self.state.record_dropped(1); let stats = self.state.snapshot(); + if !stats.dropped_total.is_power_of_two() { + return; + } tracing::warn!( operation, request_id = ?request_id, @@ -285,10 +302,20 @@ fn request_lane(request_id: &str, lane_count: usize) -> usize { usize::try_from(hash % u64::try_from(lane_count).unwrap_or(u64::MAX)).unwrap_or(0) } +#[async_trait] +impl OperationalDiagnostics for BufferedExecutionStore { + async fn record_failure(&self, failure: OperationalFailure) -> Result<(), StoreError> { + self.enqueue(ExecutionObservationWrite::OperationalFailure(Box::new( + failure, + ))); + Ok(()) + } +} + #[async_trait] impl ExecutionStore for BufferedExecutionStore where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { fn maintain_request( &self, @@ -455,7 +482,7 @@ impl ExecutionBufferIdle { impl DaemonTask for ExecutionObservationWriter where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { fn run( &self, @@ -488,7 +515,7 @@ async fn run_lane( shutdown_deadline: &OnceLock, ) -> Result<(), WorkerTaskError> where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { let mut receiver = receiver.lock().await; loop { @@ -548,7 +575,7 @@ fn shared_shutdown_deadline(deadline: &OnceLock) -> Instant { async fn persist_queued(mut queued: QueuedExecutionObservation, store: &S) where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { let operation = queued.operation(); let request_id = queued.request_id().map(ToOwned::to_owned); @@ -570,6 +597,9 @@ where Err(error) => { state.record_write_failure(); let stats = state.snapshot(); + if !stats.write_failure_total.is_power_of_two() { + return; + } tracing::warn!( operation, request_id = ?request_id, @@ -588,7 +618,7 @@ async fn drain_on_shutdown( state: &ExecutionBufferState, deadline: Instant, ) where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { receiver.close(); let queued_at_shutdown = receiver.len(); @@ -695,10 +725,20 @@ enum ExecutionObservationWrite { IntermediateFailure(Box), ProbeFailure(Box), EntryRejection(Box), + OperationalFailure(Box), Finalize(Box), } impl ExecutionObservationWrite { + const fn is_critical(&self) -> bool { + !matches!( + self, + Self::MarkSendState { .. } + | Self::MarkDownstreamCommitted { .. } + | Self::RecordClientStatus { .. } + ) + } + const fn operation(&self) -> &'static str { match self { Self::Create(_) => "create_model_request", @@ -710,6 +750,7 @@ impl ExecutionObservationWrite { Self::IntermediateFailure(_) => "record_intermediate_failure", Self::ProbeFailure(_) => "record_probe_failure", Self::EntryRejection(_) => "record_entry_rejection", + Self::OperationalFailure(_) => "record_operational_failure", Self::Finalize(_) => "finalize_model_request", } } @@ -724,6 +765,7 @@ impl ExecutionObservationWrite { | Self::RecordClientStatus { request_id, .. } => Some(request_id.as_str()), Self::IntermediateFailure(failure) => Some(failure.request_id.as_str()), Self::ProbeFailure(_) | Self::EntryRejection(_) => None, + Self::OperationalFailure(failure) => failure.correlation_id.as_deref(), Self::Finalize(finalization) => Some(finalization.request_id.as_str()), } } @@ -756,13 +798,26 @@ impl ExecutionObservationWrite { rejection.error.client_error_code(), rejection.error.client_error_type(), ]) - .saturating_add(size_of::()), + .saturating_add(size_of::()) + .saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes( + std::error::Error::source(&rejection.error), + )), Self::ProbeFailure(failure) => text_bytes([ Some(failure.provider_kind.as_str()), Some(failure.account_id.as_str()), Some(failure.upstream_model_id.as_str()), ]) .saturating_add(provider_error_bytes(&failure.error)), + Self::OperationalFailure(failure) => { + size_of::().saturating_add(text_bytes([ + Some(failure.message.as_str()), + failure.correlation_id.as_deref(), + failure.provider_kind.as_ref().map(|kind| kind.as_str()), + failure.account_id.as_ref().map(|id| id.as_str()), + failure.upstream_code.as_ref().map(|code| code.as_str()), + failure.details.as_ref().map(|details| details.as_str()), + ])) + } Self::Finalize(finalization) => { let error_bytes = finalization.error.as_ref().map_or(0, |error| { text_bytes([ @@ -770,6 +825,11 @@ impl ExecutionObservationWrite { error.client_error_code(), error.client_error_type(), ]) + .saturating_add( + gateway_core::error::ErrorSource::estimated_chain_bytes( + std::error::Error::source(error), + ), + ) }); text_bytes([ Some(finalization.request_id.as_str()), @@ -794,9 +854,10 @@ impl ExecutionObservationWrite { async fn persist(self, store: &S) -> Result<(), StoreError> where - S: ExecutionStore + ?Sized, + S: ExecutionStore + OperationalDiagnostics + ?Sized, { match self { + Self::OperationalFailure(failure) => store.record_failure(*failure).await, Self::Create(request) => store.create_model_request(*request).await, Self::Attempt(attempt) => store.record_attempt(*attempt).await, Self::CreateWithAttempt(write) => { @@ -900,33 +961,9 @@ fn provider_error_bytes(error: &ProviderError) -> usize { .saturating_add(header.value().len()); } } - bytes.saturating_add(error_source_bytes(error.source())) -} - -/// 原始来源也占据队列内存；只计数而不保存格式化内容，避免复制或打印敏感原因 -fn error_source_bytes(mut source: Option<&(dyn std::error::Error + 'static)>) -> usize { - struct ByteCount(usize); - impl std::fmt::Write for ByteCount { - fn write_str(&mut self, value: &str) -> std::fmt::Result { - self.0 = self.0.saturating_add(value.len()); - Ok(()) - } - } - let mut bytes = ByteCount(0); - for _ in 0..32 { - let Some(error) = source else { return bytes.0 }; - bytes.0 = bytes.0.saturating_add(std::mem::size_of_val(error)); - if std::fmt::write(&mut bytes, format_args!("{error}")).is_err() { - return usize::MAX; - } - source = error.source(); - } - // 过深或循环的第三方来源不能让观测写入无限遍历，交给已有超限丢弃计数 - if source.is_some() { - usize::MAX - } else { - bytes.0 - } + bytes.saturating_add(gateway_core::error::ErrorSource::estimated_chain_bytes( + error.source(), + )) } fn text_bytes(values: [Option<&str>; N]) -> usize { diff --git a/backend/crates/gateway-store/src/postgres/mod.rs b/backend/crates/gateway-store/src/postgres/mod.rs index d83a67eda..36e5773c1 100644 --- a/backend/crates/gateway-store/src/postgres/mod.rs +++ b/backend/crates/gateway-store/src/postgres/mod.rs @@ -60,6 +60,7 @@ pub async fn connect_and_migrate( ) -> StoreResult { if database_url.trim().is_empty() { return Err(StoreError::InvalidData { + source: None, entity: "PostgreSQL configuration", message: "database URL is empty".to_owned(), }); @@ -225,7 +226,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository { self.pool.begin().await.map_err(|source| { postgres_unavailable("begin control plane replacement", source) })?; - let result = async { + let result: StoreResult<_> = async { // 锁住所有配置写入共同使用的行，版本检查与修改不能被其他事务穿插 let current = sqlx::query_scalar::<_, i64>( "select config_revision from runtime_settings where id = 1 for update", @@ -235,6 +236,7 @@ impl ControlPlaneRepository for PgControlPlaneRepository { .map_err(|source| postgres_unavailable("lock control plane revision", source))?; if u64::try_from(current).ok() != Some(replacement.expected_revision.get()) { return Err(StoreError::Conflict { + source: None, entity: "runtime settings", id: "1".to_owned(), kind: crate::ConflictKind::StaleRevision, @@ -256,9 +258,10 @@ impl ControlPlaneRepository for PgControlPlaneRepository { Ok(snapshot) } Err(error) => { - transaction.rollback().await.map_err(|source| { - postgres_unavailable("rollback control plane replacement", source) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -337,7 +340,7 @@ impl PgControlPlaneRepository { let mut transaction = self.pool.begin().await.map_err(|source| { postgres_unavailable("begin targeted control plane mutation", source) })?; - let result = async { + let result: StoreResult<_> = async { let revision = bump_config_revision_in_transaction(&mut transaction).await?; match mutation { ControlPlaneMutation::CreateClientApiKey(key) => { @@ -394,9 +397,10 @@ impl PgControlPlaneRepository { Ok(revision) } Err(error) => { - transaction.rollback().await.map_err(|source| { - postgres_unavailable("rollback targeted control plane mutation", source) - })?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } diff --git a/backend/crates/gateway-store/src/postgres/observability/mapping.rs b/backend/crates/gateway-store/src/postgres/observability/mapping.rs index 427c3c6b3..ce36cb383 100644 --- a/backend/crates/gateway-store/src/postgres/observability/mapping.rs +++ b/backend/crates/gateway-store/src/postgres/observability/mapping.rs @@ -212,6 +212,7 @@ pub(crate) fn admin_latency_percentiles( pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError { observability_error(StoreError::InvalidData { + source: None, entity: "observability latency percentile", message: "latency percentile is outside the admin contract".to_owned(), }) @@ -265,6 +266,7 @@ pub(crate) fn admin_decimal_amount( ) -> AdminStoreResult { admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| { observability_error(StoreError::InvalidData { + source: None, entity: "observability decimal amount", message: "amount is outside the admin numeric contract".to_owned(), }) @@ -449,6 +451,7 @@ pub(crate) fn admin_usage_list_record( (None, None) => None, _ => { return Err(observability_error(StoreError::InvalidData { + source: None, entity: "observability request billing", message: "cost amount and currency must be present together".to_owned(), })); @@ -522,6 +525,7 @@ pub(crate) fn admin_usage_record( (None, None) => None, _ => { return Err(observability_error(StoreError::InvalidData { + source: None, entity: "observability request billing", message: "cost amount and currency must be present together".to_owned(), })); @@ -610,6 +614,7 @@ pub(crate) fn admin_request_outcome( ) -> AdminStoreResult { admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| { observability_error(StoreError::InvalidData { + source: None, entity: "observability request outcome", message: "invalid request outcome".to_owned(), }) @@ -1042,7 +1047,7 @@ pub(crate) fn opaque_response_id( get::>>(row, column)? .map(String::from_utf8) .transpose() - .map_err(|_| invalid(column)) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> { @@ -1075,7 +1080,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) - where T: sqlx::Decode<'r, Postgres> + sqlx::Type, { - row.try_get(column).map_err(|_| invalid(column)) + row.try_get(column) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult { @@ -1104,15 +1110,18 @@ pub(crate) fn optional_status( } pub(crate) fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("numeric observation is negative")) + u64::try_from(value) + .map_err(|source| invalid("numeric observation is negative").with_source(source)) } pub(crate) fn to_u32(value: i32) -> StoreResult { - u32::try_from(value).map_err(|_| invalid("integer observation is negative")) + u32::try_from(value) + .map_err(|source| invalid("integer observation is negative").with_source(source)) } pub(crate) fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "observability", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs index 6c3aa6829..3fa1402bd 100644 --- a/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs +++ b/backend/crates/gateway-store/src/postgres/observability/queries/usage.rs @@ -256,6 +256,7 @@ pub(crate) async fn usage_record_detail( .await .map_err(|source| postgres_unavailable("load usage record detail", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "model request", id: request_id.to_owned(), })?; diff --git a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs index 7ed466534..1c2f24c0a 100644 --- a/backend/crates/gateway-store/src/postgres/observability/query_budget.rs +++ b/backend/crates/gateway-store/src/postgres/observability/query_budget.rs @@ -23,11 +23,13 @@ impl ObservabilityQueryBudget { pub fn try_new(max_connections: u32, wait_timeout: Duration) -> StoreResult { if max_connections == 0 || wait_timeout.is_zero() { return Err(StoreError::InvalidData { + source: None, entity: "observability query budget", message: "requires a positive connection count and wait timeout".to_owned(), }); } let permits = usize::try_from(max_connections).map_err(|_| StoreError::InvalidData { + source: None, entity: "observability query budget", message: "connection budget does not fit this platform".to_owned(), })?; diff --git a/backend/crates/gateway-store/src/postgres/ops_events.rs b/backend/crates/gateway-store/src/postgres/ops_events.rs index d16548b05..c6d3fb7c5 100644 --- a/backend/crates/gateway-store/src/postgres/ops_events.rs +++ b/backend/crates/gateway-store/src/postgres/ops_events.rs @@ -136,7 +136,7 @@ impl OpsEventRepository for PgOpsEventRepository { .attempt_index .map(i32::try_from) .transpose() - .map_err(|_| invalid("attempt index is too large"))?, + .map_err(|source| invalid("attempt index is too large").with_source(source))?, ) .bind(event.level.as_str()) .bind(event.component) @@ -154,7 +154,7 @@ impl OpsEventRepository for PgOpsEventRepository { .retry_after_ms .map(i64::try_from) .transpose() - .map_err(|_| invalid("retry_after_ms is too large"))?, + .map_err(|source| invalid("retry_after_ms is too large").with_source(source))?, ) .bind(event.upstream_request_id) .bind( @@ -162,7 +162,7 @@ impl OpsEventRepository for PgOpsEventRepository { .latency_ms .map(i64::try_from) .transpose() - .map_err(|_| invalid("latency_ms is too large"))?, + .map_err(|source| invalid("latency_ms is too large").with_source(source))?, ) .bind(event.message) .bind(event.error_details) @@ -176,6 +176,7 @@ impl OpsEventRepository for PgOpsEventRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/pricing.rs b/backend/crates/gateway-store/src/postgres/pricing.rs index 2c3eb3e8b..a8ed63cfa 100644 --- a/backend/crates/gateway-store/src/postgres/pricing.rs +++ b/backend/crates/gateway-store/src/postgres/pricing.rs @@ -132,6 +132,7 @@ pub(crate) fn validate_pricing(pricing: &PricingOverrides) -> StoreResult<()> { fn invalid_pricing() -> StoreError { StoreError::InvalidData { + source: None, entity: "model pricing", message: "invalid model pricing".to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs index 2e80e3c4d..36cf5a6d5 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_adapter.rs @@ -211,6 +211,7 @@ impl PgAdminAccountStore { admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: ENTITY, id: account_id.to_owned(), }, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs index be1135b11..b2c9c03d2 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/admin_queries.rs @@ -228,6 +228,7 @@ async fn validate_group_filter( Err(admin_store_error( ENTITY, StoreError::NotFound { + source: None, entity: "account group", id: group_id.as_str().to_owned(), }, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs index bd93fc841..f52ea4aae 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/authorization.rs @@ -34,21 +34,25 @@ impl PgAdminAccountStore { { return Err(invalid_receipt()); } - let mut transaction = self.pool.begin().await.map_err(|_| unavailable_receipt())?; + let mut transaction = self + .pool + .begin() + .await + .map_err(|source| unavailable_receipt().with_source(source))?; let outcome = commit_authorization_in_transaction(&mut transaction, command, context).await; match outcome { Ok(outcome) => { transaction .commit() .await - .map_err(|_| unavailable_receipt())?; + .map_err(|source| unavailable_receipt().with_source(source))?; Ok(outcome) } Err(error) => { - transaction - .rollback() - .await - .map_err(|_| unavailable_receipt())?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -71,7 +75,7 @@ async fn commit_authorization_in_transaction( .bind(lock) .execute(&mut **transaction) .await - .map_err(|_| unavailable_receipt())?; + .map_err(|source| unavailable_receipt().with_source(source))?; if let Some(result) = load_receipt(&mut **transaction, key).await? { return Ok(AuthorizationCommitResult { result, @@ -140,12 +144,12 @@ async fn commit_authorization_in_transaction( }]; sqlx::query("insert into authorization_receipts(provider_kind, flow_digest, owner_digest, config_revision, accounts_json) values ($1,$2,$3,$4,$5)") .bind(key.provider_kind().as_str()).bind(key.flow_digest()).bind(key.owner_digest()) - .bind(i64::try_from(result.config_revision.get()).map_err(|_| invalid_receipt())?) - .bind(serde_json::to_value(accounts).map_err(|_| invalid_receipt())?) - .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?; + .bind(i64::try_from(result.config_revision.get()).map_err(|source| invalid_receipt().with_source(source))?) + .bind(serde_json::to_value(accounts).map_err(|source| invalid_receipt().with_source(source))?) + .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?; // 保留期长于 OAuth pending 上限；每次提交有界回收，避免引入第二套维护任务 sqlx::query("delete from authorization_receipts where ctid in (select ctid from authorization_receipts where expires_at <= now() order by expires_at limit 128 for update skip locked)") - .execute(&mut **transaction).await.map_err(|_| unavailable_receipt())?; + .execute(&mut **transaction).await.map_err(|source| unavailable_receipt().with_source(source))?; Ok(AuthorizationCommitResult { result, newly_committed: true, @@ -157,25 +161,28 @@ async fn load_receipt<'a>( key: &AuthorizationReceiptKey, ) -> AdminStoreResult> { let row: Option<(String, i64, serde_json::Value)> = sqlx::query_as("select owner_digest, config_revision, accounts_json from authorization_receipts where provider_kind=$1 and flow_digest=$2 and expires_at>now()") - .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|_| unavailable_receipt())?; + .bind(key.provider_kind().as_str()).bind(key.flow_digest()).fetch_optional(executor).await.map_err(|source| unavailable_receipt().with_source(source))?; let Some((owner, revision, accounts)) = row else { return Ok(None); }; if owner != key.owner_digest() { return Err(invalid_receipt()); } - let [account]: [StoredAccount; 1] = - serde_json::from_value(accounts).map_err(|_| unavailable_receipt())?; + let [account]: [StoredAccount; 1] = serde_json::from_value(accounts) + .map_err(|source| unavailable_receipt().with_source(source))?; Ok(Some(CredentialMutationResult { config_revision: AdminRevision::new( - u64::try_from(revision).map_err(|_| unavailable_receipt())?, + u64::try_from(revision).map_err(|source| unavailable_receipt().with_source(source))?, ) - .map_err(|_| unavailable_receipt())?, + .map_err(|source| unavailable_receipt().with_source(source))?, account_id: CoreProviderAccountId::new(account.account_id) - .map_err(|_| unavailable_receipt())?, + .map_err(|source| unavailable_receipt().with_source(source))?, credential_revision: account .credential_revision - .map(|revision| AdminRevision::new(revision).map_err(|_| unavailable_receipt())) + .map(|revision| { + AdminRevision::new(revision) + .map_err(|source| unavailable_receipt().with_source(source)) + }) .transpose()?, })) } diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs index 4090f40f8..00fadcb22 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/repository.rs @@ -213,7 +213,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.concurrency_limit.map(|limit| i64::from(limit.get()))) - .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?) + .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?) .bind(credential_state.as_str()) .bind(account.credential_observed_at) .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url())) @@ -281,6 +281,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository { .await .map_err(|source| postgres_unavailable("compare and swap provider credentials", source))? .ok_or(StoreError::Conflict { + source: None, entity: ENTITY, id: update.account_id, kind: ConflictKind::StaleRevision, @@ -587,6 +588,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository { .await .map_err(|source| postgres_unavailable("recover provider account state", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: ENTITY, id: command.account_id.clone(), })?; @@ -659,6 +661,7 @@ async fn replace_account_group_assignments_in_transaction( .map_err(|source| postgres_unavailable("validate account group assignment", source))?; if usize::try_from(known_group_count).ok() != Some(group_ids.len()) { return Err(StoreError::NotFound { + source: None, entity: "account group", id: "one or more group IDs".to_owned(), }); @@ -757,7 +760,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( .bind(account.next_refresh_at) .bind(account.enabled) .bind(account.concurrency_limit.map(|limit| i64::from(limit.get()))) - .bind(i16::try_from(account.weight.get()).map_err(|_| invalid("invalid weight"))?) + .bind(i16::try_from(account.weight.get()).map_err(|source| invalid("invalid weight").with_source(source))?) .bind(credential_state.as_str()) .bind(account.credential_observed_at) .bind(account.outbound_proxy.as_ref().map(|proxy| proxy.expose_url())) @@ -771,6 +774,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( .is_some_and(sqlx::error::DatabaseError::is_unique_violation) { StoreError::Conflict { + source: Some(error.into()), entity: ENTITY, id: account.id.clone(), kind: ConflictKind::InvalidTransition, @@ -780,6 +784,7 @@ pub(crate) async fn upsert_provider_account_in_transaction( } })? .ok_or_else(|| StoreError::Conflict { + source: None, entity: ENTITY, id: account.id.clone(), kind: ConflictKind::InvalidTransition, @@ -850,6 +855,7 @@ pub(crate) async fn rotate_provider_account_in_transaction( .is_some_and(sqlx::error::DatabaseError::is_unique_violation) { StoreError::Conflict { + source: Some(error.into()), entity: ENTITY, id: update.account_id.clone(), kind: ConflictKind::InvalidTransition, @@ -859,6 +865,7 @@ pub(crate) async fn rotate_provider_account_in_transaction( } })? .ok_or_else(|| StoreError::Conflict { + source: None, entity: ENTITY, id: update.account_id.clone(), kind: ConflictKind::StaleRevision, @@ -893,7 +900,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction( .bind(account_ids) .bind(enabled) .bind(concurrency_limit.flatten().map(|limit| i64::from(limit.get()))) - .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|_| invalid("invalid weight"))?) + .bind(weight.map(|weight| i16::try_from(weight.get())).transpose().map_err(|source| invalid("invalid weight").with_source(source))?) .bind(outbound_proxy.is_some()) .bind(proxy.as_ref().map(gateway_core::account::OutboundProxy::expose_url)) .bind(proxy_id) @@ -909,6 +916,7 @@ pub(crate) async fn update_provider_accounts_scheduling_in_transaction( Ok(()) } else { Err(StoreError::NotFound { + source: None, entity: ENTITY, id: "one or more provider account IDs".to_owned(), }) @@ -1018,10 +1026,10 @@ pub(crate) async fn finish_admin_transaction( Ok(value) } Err(error) => { - transaction - .rollback() - .await - .map_err(|source| postgres_unavailable(operation, source))?; + let error = match transaction.rollback().await { + Ok(()) => error, + Err(cleanup) => error.with_cleanup(cleanup), + }; Err(error) } } @@ -1042,6 +1050,7 @@ pub(super) async fn import_provider_accounts_in_transaction( .await?; if current.as_ref() != Some(&binding.proxy) { return Err(StoreError::Conflict { + source: None, entity: "outbound proxy", id: binding.id.clone(), kind: ConflictKind::StaleRevision, diff --git a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs index 7d73af863..85ebcec68 100644 --- a/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs +++ b/backend/crates/gateway-store/src/postgres/provider_accounts/rows.rs @@ -446,12 +446,12 @@ pub(crate) fn account_record_from_row( let credentials = JsonObject::try_from_value( "provider_credentials_json", row.try_get("provider_credentials_json") - .map_err(|_| invalid("invalid credentials JSON"))?, + .map_err(|source| invalid("invalid credentials JSON").with_source(source))?, CREDENTIALS_MAX_BYTES, )?; let quota = row .try_get::, _>("provider_quota_json") - .map_err(|_| invalid("invalid quota JSON"))? + .map_err(|source| invalid("invalid quota JSON").with_source(source))? .map(|value| JsonObject::try_from_value("provider_quota_json", value, QUOTA_MAX_BYTES)) .transpose()?; Ok(ProviderAccountRecord { @@ -514,10 +514,10 @@ pub(crate) fn account_summary_from_row( ) -> StoreResult { let revision = row .try_get::("credential_revision") - .map_err(|_| invalid("invalid credential revision"))?; + .map_err(|source| invalid("invalid credential revision").with_source(source))?; let credential_state = row .try_get::("credential_state") - .map_err(|_| invalid("invalid credential_state"))?; + .map_err(|source| invalid("invalid credential_state").with_source(source))?; let quota_access_state = parse_quota_access_state(&get::(&row, "quota_access_state")?)?; let quota_evidence = parse_quota_evidence(get(&row, "quota_evidence")?)?; let quota_access_observed_at = get::>>(&row, "quota_access_observed_at")?; @@ -546,7 +546,7 @@ pub(crate) fn account_summary_from_row( outbound_proxy: get::>(&row, "outbound_proxy_url")? .map(|url| { gateway_core::account::OutboundProxy::parse(&url) - .map_err(|_| invalid("invalid outbound proxy")) + .map_err(|source| invalid("invalid outbound proxy").with_source(source)) }) .transpose()?, id: get(&row, "id")?, @@ -584,7 +584,8 @@ pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) - where T: sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, { - row.try_get(column).map_err(|_| invalid(column)) + row.try_get(column) + .map_err(|source| invalid(column).with_source(source)) } pub(crate) fn validate_object_size( @@ -594,8 +595,9 @@ pub(crate) fn validate_object_size( ) -> StoreResult<()> { let size = serde_json::to_vec(&object.as_value()) .map_err(|error| StoreError::InvalidData { + source: Some(error.into()), entity: ENTITY, - message: error.to_string(), + message: "JSON encoding failed".to_owned(), })? .len(); if size > max { @@ -606,15 +608,16 @@ pub(crate) fn validate_object_size( } pub(crate) fn to_i64(value: u64) -> StoreResult { - i64::try_from(value).map_err(|_| invalid("revision is too large")) + i64::try_from(value).map_err(|source| invalid("revision is too large").with_source(source)) } pub(crate) fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("revision must be positive")) + u64::try_from(value).map_err(|source| invalid("revision must be positive").with_source(source)) } pub(crate) fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/proxies.rs b/backend/crates/gateway-store/src/postgres/proxies.rs index 938e8b884..e7cf5267b 100644 --- a/backend/crates/gateway-store/src/postgres/proxies.rs +++ b/backend/crates/gateway-store/src/postgres/proxies.rs @@ -82,6 +82,7 @@ fn unavailable(source: impl std::error::Error + Send + Sync + 'static) -> StoreE } fn conflict(id: &str) -> StoreError { StoreError::Conflict { + source: None, entity: ENTITY, id: id.to_owned(), kind: ConflictKind::InvalidTransition, @@ -89,39 +90,61 @@ fn conflict(id: &str) -> StoreError { } fn invalid() -> StoreError { StoreError::InvalidData { + source: None, entity: ENTITY, message: "invalid proxy record".to_owned(), } } fn record(row: PgRow) -> StoreResult { - let success: Option = row.try_get("last_test_success").map_err(|_| invalid())?; - let ip: Option = row.try_get("last_test_ip").map_err(|_| invalid())?; - let ipv4: Option = row.try_get("last_test_ipv4").map_err(|_| invalid())?; - let ipv6: Option = row.try_get("last_test_ipv6").map_err(|_| invalid())?; - let latency: Option = row.try_get("last_test_latency_ms").map_err(|_| invalid())?; + let success: Option = row + .try_get("last_test_success") + .map_err(|source| invalid().with_source(source))?; + let ip: Option = row + .try_get("last_test_ip") + .map_err(|source| invalid().with_source(source))?; + let ipv4: Option = row + .try_get("last_test_ipv4") + .map_err(|source| invalid().with_source(source))?; + let ipv6: Option = row + .try_get("last_test_ipv6") + .map_err(|source| invalid().with_source(source))?; + let latency: Option = row + .try_get("last_test_latency_ms") + .map_err(|source| invalid().with_source(source))?; Ok(ProxyRecord { - auto_location: row.try_get("auto_location").map_err(|_| invalid())?, + auto_location: row + .try_get("auto_location") + .map_err(|source| invalid().with_source(source))?, detected_location: detected_location_from_row(&row)?, location: location_from_row(&row)?, - id: row.try_get("id").map_err(|_| invalid())?, - name: row.try_get("name").map_err(|_| invalid())?, + id: row + .try_get("id") + .map_err(|source| invalid().with_source(source))?, + name: row + .try_get("name") + .map_err(|source| invalid().with_source(source))?, proxy: OutboundProxy::parse( &row.try_get::("proxy_url") - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, revision: AdminRevision::new( - u64::try_from(row.try_get::("revision").map_err(|_| invalid())?) - .map_err(|_| invalid())?, + u64::try_from( + row.try_get::("revision") + .map_err(|source| invalid().with_source(source))?, + ) + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, account_count: u64::try_from( row.try_get::("account_count") - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, ) - .map_err(|_| invalid())?, - last_test_at: row.try_get("last_test_at").map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, + last_test_at: row + .try_get("last_test_at") + .map_err(|source| invalid().with_source(source))?, last_test: success .map(|success| -> StoreResult<_> { Ok(ProxyTestResult { @@ -129,48 +152,60 @@ fn record(row: PgRow) -> StoreResult { .try_get::, _>( "last_location_detection_json", ) - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .0, success, latency_ms: u64::try_from(latency.ok_or_else(invalid)?) - .map_err(|_| invalid())?, - exit_ip: ip.map(|ip| ip.parse().map_err(|_| invalid())).transpose()?, + .map_err(|source| invalid().with_source(source))?, + exit_ip: ip + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) + .transpose()?, exit_ipv4: ipv4 - .map(|ip| ip.parse().map_err(|_| invalid())) + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) .transpose()?, exit_ipv6: ipv6 - .map(|ip| ip.parse().map_err(|_| invalid())) + .map(|ip| ip.parse().map_err(|source| invalid().with_source(source))) .transpose()?, message: row .try_get::, _>("last_test_message") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .unwrap_or_default(), }) }) .transpose()?, - created_at: row.try_get("created_at").map_err(|_| invalid())?, - updated_at: row.try_get("updated_at").map_err(|_| invalid())?, + created_at: row + .try_get("created_at") + .map_err(|source| invalid().with_source(source))?, + updated_at: row + .try_get("updated_at") + .map_err(|source| invalid().with_source(source))?, }) } pub(crate) fn location_from_row( row: &PgRow, ) -> StoreResult> { - let country: Option = row.try_get("location_country").map_err(|_| invalid())?; + let country: Option = row + .try_get("location_country") + .map_err(|source| invalid().with_source(source))?; country .map(|country| { gateway_core::account::RequestLocation { country, - region: row.try_get("location_region").map_err(|_| invalid())?, - city: row.try_get("location_city").map_err(|_| invalid())?, + region: row + .try_get("location_region") + .map_err(|source| invalid().with_source(source))?, + city: row + .try_get("location_city") + .map_err(|source| invalid().with_source(source))?, timezone: row .try_get::("location_timezone") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .parse() - .map_err(|_| invalid())?, + .map_err(|source| invalid().with_source(source))?, } .normalized() - .map_err(|_| invalid()) + .map_err(|source| invalid().with_source(source)) }) .transpose() } @@ -178,10 +213,13 @@ pub(crate) fn location_from_row( fn detected_location_from_row(row: &PgRow) -> StoreResult> { let detected = row .try_get::>, _>("detected_location_json") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .map(|value| value.0); if let Some(value) = &detected { - value.location.validate().map_err(|_| invalid())?; + value + .location + .validate() + .map_err(|source| invalid().with_source(source))?; } Ok(detected) } @@ -191,7 +229,7 @@ pub(crate) fn effective_location_from_row( ) -> StoreResult> { if row .try_get::, _>("auto_location") - .map_err(|_| invalid())? + .map_err(|source| invalid().with_source(source))? .unwrap_or(false) { Ok(detected_location_from_row(row)?.map(|value| value.location)) @@ -225,12 +263,15 @@ async fn save_test( ) -> StoreResult<()> { let detected = current.detected_location_after_test(&result); if let Some(value) = &detected { - value.location.validate().map_err(|_| invalid())?; + value + .location + .validate() + .map_err(|source| invalid().with_source(source))?; } sqlx::query("update outbound_proxies set last_test_at = now(), last_test_success = $2, last_test_latency_ms = $3, last_test_ip = $4, last_test_ipv4 = $5, last_test_ipv6 = $6, last_test_message = $7, last_location_detection_json = $8, detected_location_json = $9 where id = $1") - .bind(¤t.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|_| invalid())?) + .bind(¤t.id).bind(result.success).bind(i64::try_from(result.latency_ms).map_err(|source| invalid().with_source(source))?) .bind(result.exit_ip.map(|ip| ip.to_string())).bind(result.exit_ipv4.map(|ip| ip.to_string())) .bind(result.exit_ipv6.map(|ip| ip.to_string())).bind(result.message) .bind(sqlx::types::Json(result.location)).bind(detected.map(sqlx::types::Json)) @@ -244,7 +285,9 @@ async fn save_location( location: Option<&gateway_core::account::RequestLocation>, ) -> StoreResult<()> { if let Some(location) = location { - location.validate().map_err(|_| invalid())?; + location + .validate() + .map_err(|source| invalid().with_source(source))?; } sqlx::query("update outbound_proxies set location_country = $2, location_region = $3, location_city = $4, location_timezone = $5 where id = $1") .bind(id) @@ -310,12 +353,13 @@ pub(crate) async fn resolve_proxy_selection( .await .map_err(unavailable)? .ok_or_else(|| StoreError::NotFound { + source: None, entity: ENTITY, id: id.clone(), })?; Ok(( Some(id.clone()), - Some(OutboundProxy::parse(&value).map_err(|_| invalid())?), + Some(OutboundProxy::parse(&value).map_err(|source| invalid().with_source(source))?), )) } } @@ -423,6 +467,7 @@ impl ProxyStore for PgProxyRepository { .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { + source: None, entity: ENTITY, id: query.proxy_id.clone(), }) @@ -443,15 +488,15 @@ impl ProxyStore for PgProxyRepository { .into_iter() .map(|row| { Ok(ProxyAccountRef { - id: row.try_get("id").map_err(|_| invalid())?, - name: row.try_get("name").map_err(|_| invalid())?, - email: row.try_get("email").map_err(|_| invalid())?, - provider_kind: row.try_get("provider_kind").map_err(|_| invalid())?, - authentication_kind: row.try_get("authentication_kind").map_err(|_| invalid())?, - plan_type: row.try_get("plan_type").map_err(|_| invalid())?, + id: row.try_get("id").map_err(|source| invalid().with_source(source))?, + name: row.try_get("name").map_err(|source| invalid().with_source(source))?, + email: row.try_get("email").map_err(|source| invalid().with_source(source))?, + provider_kind: row.try_get("provider_kind").map_err(|source| invalid().with_source(source))?, + authentication_kind: row.try_get("authentication_kind").map_err(|source| invalid().with_source(source))?, + plan_type: row.try_get("plan_type").map_err(|source| invalid().with_source(source))?, plan_type_display: None, groups: Vec::new(), - enabled: row.try_get("enabled").map_err(|_| invalid())?, + enabled: row.try_get("enabled").map_err(|source| invalid().with_source(source))?, }) }) .collect::>>() @@ -586,6 +631,7 @@ impl ProxyStore for PgProxyRepository { .map_err(|source| store_error(unavailable(source)))? .ok_or_else(|| { store_error(StoreError::NotFound { + source: None, entity: ENTITY, id: id.to_owned(), }) @@ -815,11 +861,11 @@ impl ProxyStore for PgProxyRepository { .map_err(store_error)?; if current.revision != revision { // Drop 只排队回滚，返回冲突前需释放事务锁，避免误挡紧接着的编辑 - transaction - .rollback() - .await - .map_err(|source| store_error(unavailable(source)))?; - return Err(store_error(conflict(id))); + let error = match transaction.rollback().await { + Ok(()) => conflict(id), + Err(cleanup) => conflict(id).with_cleanup(cleanup), + }; + return Err(store_error(error)); } save_test(&mut transaction, ¤t, result) .await diff --git a/backend/crates/gateway-store/src/postgres/runtime_settings.rs b/backend/crates/gateway-store/src/postgres/runtime_settings.rs index 7f024b464..2b0b27707 100644 --- a/backend/crates/gateway-store/src/postgres/runtime_settings.rs +++ b/backend/crates/gateway-store/src/postgres/runtime_settings.rs @@ -206,6 +206,7 @@ impl RuntimeSettingsUpdate { }) { return Err(StoreError::InvalidData { + source: None, entity: "runtime settings", message: "settings violate the frozen runtime constraints".to_owned(), }); @@ -276,6 +277,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul .await .map_err(|source| postgres_unavailable("load runtime settings", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -453,6 +455,7 @@ pub(crate) async fn load_runtime_settings_in_transaction( .await .map_err(|source| postgres_unavailable("load runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -568,6 +571,7 @@ pub(crate) async fn update_runtime_settings_in_transaction( .await .map_err(|source| postgres_unavailable("update runtime settings in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -587,6 +591,7 @@ pub(crate) async fn bump_config_revision_in_transaction( .await .map_err(|source| postgres_unavailable("bump config revision in transaction", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -715,6 +720,7 @@ fn to_u32(value: i64) -> StoreResult { fn invalid_location() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "request location is invalid".to_owned(), } @@ -722,6 +728,7 @@ fn invalid_location() -> StoreError { fn invalid_request_profile() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "Provider request profile key is invalid".to_owned(), } @@ -729,6 +736,7 @@ fn invalid_request_profile() -> StoreError { fn invalid_numeric() -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime settings", message: "numeric field is outside its supported range".to_owned(), } diff --git a/backend/crates/gateway-store/src/postgres/snapshot.rs b/backend/crates/gateway-store/src/postgres/snapshot.rs index b6afead3a..23b6c84e7 100644 --- a/backend/crates/gateway-store/src/postgres/snapshot.rs +++ b/backend/crates/gateway-store/src/postgres/snapshot.rs @@ -137,6 +137,7 @@ impl RuntimeSnapshotRepository for PgRuntimeSnapshotRepository { .await .map_err(|source| postgres_unavailable("read current config revision", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -288,6 +289,7 @@ async fn load_settings( .await .map_err(|source| postgres_unavailable("load snapshot settings", source))? .ok_or_else(|| StoreError::NotFound { + source: None, entity: "runtime settings", id: "1".to_owned(), })?; @@ -375,7 +377,8 @@ async fn load_account_groups( Ok(SnapshotAccountGroupData { fast_mode: FastMode::parse(&fast_mode) .ok_or_else(|| invalid("invalid fast_mode"))?, - id: AccountGroupId::new(id).map_err(|_| invalid("invalid account group id"))?, + id: AccountGroupId::new(id) + .map_err(|source| invalid("invalid account group id").with_source(source))?, name, enabled, }) @@ -424,7 +427,7 @@ async fn load_group_memberships( .map(|(group_id, account_id)| { Ok(SnapshotGroupMembershipData { group_id: AccountGroupId::new(group_id) - .map_err(|_| invalid("invalid membership group id"))?, + .map_err(|source| invalid("invalid membership group id").with_source(source))?, account_id, }) }) @@ -436,15 +439,18 @@ fn revision_from_i64(value: i64) -> StoreResult { } fn to_u64(value: i64) -> StoreResult { - u64::try_from(value).map_err(|_| invalid("numeric snapshot field is negative")) + u64::try_from(value) + .map_err(|source| invalid("numeric snapshot field is negative").with_source(source)) } fn to_u32(value: i64) -> StoreResult { - u32::try_from(value).map_err(|_| invalid("numeric snapshot field is outside u32")) + u32::try_from(value) + .map_err(|source| invalid("numeric snapshot field is outside u32").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime snapshot", message: message.to_owned(), } @@ -459,8 +465,9 @@ fn decode_request_profiles( .into_iter() .map(|(kind, document)| { Ok(( - gateway_core::routing::ProviderKind::new(kind) - .map_err(|_| invalid("invalid request profile provider"))?, + gateway_core::routing::ProviderKind::new(kind).map_err(|source| { + invalid("invalid request profile provider").with_source(source) + })?, gateway_core::account::OpaqueProviderData::new(document), )) }) diff --git a/backend/crates/gateway-store/src/redis/auth.rs b/backend/crates/gateway-store/src/redis/auth.rs index d2422d094..d4dff2c57 100644 --- a/backend/crates/gateway-store/src/redis/auth.rs +++ b/backend/crates/gateway-store/src/redis/auth.rs @@ -284,6 +284,7 @@ fn decode_session(value: &str) -> StoreResult { fn auth_invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "authentication state", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/client_admission.rs b/backend/crates/gateway-store/src/redis/client_admission.rs index df9039046..889b62bd5 100644 --- a/backend/crates/gateway-store/src/redis/client_admission.rs +++ b/backend/crates/gateway-store/src/redis/client_admission.rs @@ -280,7 +280,7 @@ impl ClientAdmissionRepository for RedisClientAdmissionRepository { request.validate()?; let keys = self.keys(&request.client_api_key_ref)?; let lease_ttl_ms = u64::try_from(request.lease_ttl.as_millis()) - .map_err(|_| invalid("lease TTL is too large"))?; + .map_err(|source| invalid("lease TTL is too large").with_source(source))?; let mut connection = self.connection.clone(); let code = Script::new(ADMIT_SCRIPT) .key(&keys[0]) @@ -462,7 +462,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { CoreAdmissionDecision::Rejected(CoreAdmissionRejection::ConcurrencyLimited) } }) - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } @@ -474,7 +474,7 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { Box::pin(async move { self.release_client_request(client_api_key_id.as_str(), model_request_id.as_str()) .await - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } @@ -508,13 +508,14 @@ impl ClientAdmissionPort for RedisClientAdmissionRepository { restored_recent_requests: restored.restored_recent_requests, restored_running_requests: restored.restored_running_requests, }) - .map_err(|_| CoreAdmissionError) + .map_err(|source| CoreAdmissionError(Some(source.into()))) }) } } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "client admission", message: message.to_owned(), } @@ -525,18 +526,19 @@ fn validate_recovery_request_id(model_request_id: &str) -> StoreResult<()> { } fn redis_duration_millis(duration: Duration) -> StoreResult { - let milliseconds = - u64::try_from(duration.as_millis()).map_err(|_| invalid("lease TTL is too large"))?; + let milliseconds = u64::try_from(duration.as_millis()) + .map_err(|source| invalid("lease TTL is too large").with_source(source))?; redis_integer(milliseconds, "lease TTL") } fn redis_timestamp_millis(timestamp: DateTime, field: &str) -> StoreResult { - let milliseconds = u64::try_from(timestamp.timestamp_millis()).map_err(|_| invalid(field))?; + let milliseconds = u64::try_from(timestamp.timestamp_millis()) + .map_err(|source| invalid(field).with_source(source))?; redis_integer(milliseconds, field) } fn redis_len(length: usize, field: &str) -> StoreResult { - let value = u64::try_from(length).map_err(|_| invalid(field))?; + let value = u64::try_from(length).map_err(|source| invalid(field).with_source(source))?; redis_integer(value, field) } diff --git a/backend/crates/gateway-store/src/redis/coordination_buffer.rs b/backend/crates/gateway-store/src/redis/coordination_buffer.rs index aa138d672..18f818342 100644 --- a/backend/crates/gateway-store/src/redis/coordination_buffer.rs +++ b/backend/crates/gateway-store/src/redis/coordination_buffer.rs @@ -32,10 +32,14 @@ pub struct BufferedClientAdmissionPort { impl BufferedClientAdmissionPort { #[must_use] - pub fn new(inner: Arc) -> (Self, ClientAdmissionReleaseWriter) { + pub fn new( + inner: Arc, + diagnostics: Arc, + ) -> (Self, ClientAdmissionReleaseWriter) { Self::with_capacity( inner, NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"), + diagnostics, ) } @@ -43,6 +47,7 @@ impl BufferedClientAdmissionPort { pub fn with_capacity( inner: Arc, capacity: NonZeroUsize, + diagnostics: Arc, ) -> (Self, ClientAdmissionReleaseWriter) { let (sender, receiver) = mpsc::channel(capacity.get()); ( @@ -52,6 +57,7 @@ impl BufferedClientAdmissionPort { }, ClientAdmissionReleaseWriter { inner, + diagnostics, receiver: Mutex::new(receiver), }, ) @@ -127,6 +133,7 @@ struct AdmissionRelease { } pub struct ClientAdmissionReleaseWriter { + diagnostics: Arc, inner: Arc, receiver: Mutex>, } @@ -138,7 +145,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter { loop { let release = tokio::select! { () = cancellation.cancelled() => { - drain_admission_releases(&mut receiver, self.inner.as_ref()).await; + drain_admission_releases(&mut receiver, self.inner.as_ref(), self.diagnostics.as_ref()).await; return Ok(()); }, release = receiver.recv() => release, @@ -148,13 +155,7 @@ impl DaemonTask for ClientAdmissionReleaseWriter { "client admission release queue closed", )); }; - if let Err(error) = self - .inner - .release(&release.client_api_key_id, &release.model_request_id) - .await - { - tracing::warn!(%error, "Client admission 后台释放失败，依赖租约 TTL 收敛"); - } + release_admission(self.inner.as_ref(), self.diagnostics.as_ref(), release).await; } }) } @@ -163,21 +164,46 @@ impl DaemonTask for ClientAdmissionReleaseWriter { async fn drain_admission_releases( receiver: &mut mpsc::Receiver, port: &dyn ClientAdmissionPort, + diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics, ) { receiver.close(); let started_at = Instant::now(); while let Some(release) = receiver.recv().await { let remaining = SHUTDOWN_DRAIN_TIMEOUT.saturating_sub(started_at.elapsed()); if remaining.is_zero() - || tokio::time::timeout( - remaining, - port.release(&release.client_api_key_id, &release.model_request_id), - ) - .await - .is_err() + || tokio::time::timeout(remaining, release_admission(port, diagnostics, release)) + .await + .is_err() { tracing::warn!("Client admission 关闭排空超时，剩余租约依赖 TTL 收敛"); break; } } } + +async fn release_admission( + port: &dyn ClientAdmissionPort, + diagnostics: &dyn gateway_core::diagnostics::OperationalDiagnostics, + release: AdmissionRelease, +) { + if let Err(source) = port + .release(&release.client_api_key_id, &release.model_request_id) + .await + { + let mut failure = gateway_core::diagnostics::OperationalFailure::new( + "store", + "release_client_admission", + "admission_unavailable", + "Client admission release failed; lease TTL remains active", + ); + failure.correlation_id = Some(release.model_request_id.as_str().to_owned()); + failure.details = + gateway_core::error::ErrorDetails::capture(Some(&source.into()), None, false); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!( + request_id = release.model_request_id.as_str(), + "admission diagnostic could not be recorded" + ); + } + } +} diff --git a/backend/crates/gateway-store/src/redis/credential_cooldown.rs b/backend/crates/gateway-store/src/redis/credential_cooldown.rs index d37eb50b9..ba412f6f1 100644 --- a/backend/crates/gateway-store/src/redis/credential_cooldown.rs +++ b/backend/crates/gateway-store/src/redis/credential_cooldown.rs @@ -260,10 +260,10 @@ impl RedisCredentialCooldownRepository { } let revision = revision .parse::() - .map_err(|_| invalid("cached cooldown revision is invalid"))?; + .map_err(|source| invalid("cached cooldown revision is invalid").with_source(source))?; let until_ms = until_ms .parse::() - .map_err(|_| invalid("cached cooldown expiry is invalid"))?; + .map_err(|source| invalid("cached cooldown expiry is invalid").with_source(source))?; let cooldown_until = DateTime::from_timestamp_millis(until_ms) .ok_or_else(|| invalid("cached cooldown expiry is invalid"))?; let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit); @@ -350,7 +350,7 @@ impl RedisCredentialCooldownRepository { account_id, gateway_admin::model::accounts::AccountFreeze { credential_revision: gateway_admin::model::Revision::new(revision.get()) - .map_err(|_| invalid("freeze revision"))?, + .map_err(|source| invalid("freeze revision").with_source(source))?, until, generation, requires_probe: kind.requires_probe(), @@ -401,7 +401,7 @@ impl RedisCredentialCooldownRepository { .map_err(|source| redis_unavailable("read capacity peak in-flight", source))?; peak.map(u32::try_from) .transpose() - .map_err(|_| invalid("capacity peak in-flight is invalid")) + .map_err(|source| invalid("capacity peak in-flight is invalid").with_source(source)) } } @@ -741,6 +741,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential cooldown", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/credential_leases.rs b/backend/crates/gateway-store/src/redis/credential_leases.rs index e44b87f08..1887f1b35 100644 --- a/backend/crates/gateway-store/src/redis/credential_leases.rs +++ b/backend/crates/gateway-store/src/redis/credential_leases.rs @@ -247,6 +247,7 @@ impl CredentialLeaseGuard { .grant .clone() .ok_or_else(|| crate::StoreError::InvalidData { + source: None, entity: "credential lease", message: "lease has already been released".to_owned(), })?; @@ -385,9 +386,9 @@ impl RedisCredentialLeaseRepository { .invoke_async::(&mut connection) .await .map_err(|source| redis_unavailable("advance provider scheduling cursor", source))?; - cursor - .parse::() - .map_err(|_| invalid("Redis returned an invalid scheduling cursor")) + cursor.parse::().map_err(|source| { + invalid("Redis returned an invalid scheduling cursor").with_source(source) + }) } fn keys(&self, request: &CredentialLeaseRequest) -> StoreResult<[String; 3]> { @@ -486,9 +487,9 @@ impl RedisCredentialLeaseRepository { .invoke_async(&mut connection) .await .map_err(|source| redis_unavailable("load credential runtime signal", source))?; - let in_flight = in_flight - .parse::() - .map_err(|_| invalid("Redis returned an invalid in-flight count"))?; + let in_flight = in_flight.parse::().map_err(|source| { + invalid("Redis returned an invalid in-flight count").with_source(source) + })?; let last_started_at = if last_started == "0" { None } else { @@ -830,7 +831,7 @@ struct LeaseAttempt { fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult { let fence = fence .parse::() - .map_err(|_| invalid("Redis returned an invalid fencing token"))?; + .map_err(|source| invalid("Redis returned an invalid fencing token").with_source(source))?; Ok(CredentialLeaseGrant { lease_id, fencing_token: Revision::new(fence)?, @@ -841,7 +842,7 @@ fn grant(lease_id: String, fence: &str, expires_at: &str) -> StoreResult StoreResult> { let milliseconds = value .parse::() - .map_err(|_| invalid("Redis returned an invalid timestamp"))?; + .map_err(|source| invalid("Redis returned an invalid timestamp").with_source(source))?; DateTime::from_timestamp_millis(milliseconds) .ok_or_else(|| invalid("Redis returned an out-of-range timestamp")) } @@ -850,7 +851,7 @@ fn duration(value: &str) -> StoreResult { value .parse::() .map(Duration::from_millis) - .map_err(|_| invalid("Redis returned an invalid retry interval")) + .map_err(|source| invalid("Redis returned an invalid retry interval").with_source(source)) } fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) -> StoreResult<()> { @@ -863,11 +864,13 @@ fn supported_duration(value: Duration, allow_zero: bool, field: &'static str) -> fn duration_millis(value: Duration) -> StoreResult { supported_duration(value, true, "duration")?; - u64::try_from(value.as_millis()).map_err(|_| invalid("duration is too large")) + u64::try_from(value.as_millis()) + .map_err(|source| invalid("duration is too large").with_source(source)) } fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential lease", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/credential_state.rs b/backend/crates/gateway-store/src/redis/credential_state.rs index 42b781169..23149b242 100644 --- a/backend/crates/gateway-store/src/redis/credential_state.rs +++ b/backend/crates/gateway-store/src/redis/credential_state.rs @@ -446,6 +446,7 @@ impl ProviderCatalogCachePort for RedisCredentialStateRepository { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "credential state cache", message: message.to_owned(), } @@ -453,6 +454,7 @@ fn invalid(message: &str) -> StoreError { fn catalog_invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "provider catalog cache", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/mod.rs b/backend/crates/gateway-store/src/redis/mod.rs index 9e09ab0fa..721654506 100644 --- a/backend/crates/gateway-store/src/redis/mod.rs +++ b/backend/crates/gateway-store/src/redis/mod.rs @@ -48,6 +48,7 @@ pub(crate) fn namespace(value: &str) -> StoreResult { .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) { return Err(StoreError::InvalidData { + source: None, entity: "Redis namespace", message: "namespace contains unsupported characters".to_owned(), }); diff --git a/backend/crates/gateway-store/src/redis/oauth_pending.rs b/backend/crates/gateway-store/src/redis/oauth_pending.rs index 1bcbaab6d..60e8b95b5 100644 --- a/backend/crates/gateway-store/src/redis/oauth_pending.rs +++ b/backend/crates/gateway-store/src/redis/oauth_pending.rs @@ -322,6 +322,7 @@ fn ttl_millis(ttl: Duration) -> StoreResult { .ok() .filter(|value| *value > 0 && *value <= MAX_REDIS_EXACT_INTEGER) .ok_or_else(|| StoreError::InvalidData { + source: None, entity: "OAuth pending flow", message: "TTL is outside the supported range".to_owned(), }) diff --git a/backend/crates/gateway-store/src/redis/runtime_change.rs b/backend/crates/gateway-store/src/redis/runtime_change.rs index 0a743cd91..72c500d4d 100644 --- a/backend/crates/gateway-store/src/redis/runtime_change.rs +++ b/backend/crates/gateway-store/src/redis/runtime_change.rs @@ -73,9 +73,9 @@ impl RuntimeChangeRepository for RedisRuntimeChangeRepository { .await .map_err(|source| redis_unavailable("subscribe runtime changes", source))?; let stream = pubsub.into_on_message().map(|message| { - let payload = message - .get_payload::() - .map_err(|_| invalid("runtime change payload is not UTF-8"))?; + let payload = message.get_payload::().map_err(|source| { + invalid("runtime change payload is not UTF-8").with_source(source) + })?; let wire: RuntimeChangeWire = serde_json::from_str(&payload).map_err(|error| invalid(&error.to_string()))?; wire.try_into() @@ -154,6 +154,7 @@ impl TryFrom for RuntimeChange { fn invalid(message: &str) -> StoreError { StoreError::InvalidData { + source: None, entity: "runtime change", message: message.to_owned(), } diff --git a/backend/crates/gateway-store/src/redis/worker_lease.rs b/backend/crates/gateway-store/src/redis/worker_lease.rs index d37db1beb..4ebd50714 100644 --- a/backend/crates/gateway-store/src/redis/worker_lease.rs +++ b/backend/crates/gateway-store/src/redis/worker_lease.rs @@ -52,7 +52,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard { current, ) .await - .map_err(|_| WorkerLeaseError::safe("worker lease renewal failed"))? + .map_err(|source| { + WorkerLeaseError::safe("worker lease renewal failed").with_source(source) + })? .ok_or_else(|| WorkerLeaseError::safe("worker lease was lost"))?; self.grant = Some(renewed); Ok(()) @@ -72,7 +74,9 @@ impl WorkerLeaderLeaseGuard for RedisWorkerLeaderLeaseGuard { CredentialLeaseRepository::release_credential_lease(&repository, &request, &grant) .await .map(|_| ()) - .map_err(|_| WorkerLeaseError::safe("worker lease release failed")) + .map_err(|source| { + WorkerLeaseError::safe("worker lease release failed").with_source(source) + }) }) } } @@ -94,7 +98,9 @@ impl WorkerLeaderLeasePort for RedisWorkerLeaderLeasePort { &lease_request, ) .await - .map_err(|_| WorkerLeaseError::safe("worker lease acquisition failed"))?; + .map_err(|source| { + WorkerLeaseError::safe("worker lease acquisition failed").with_source(source) + })?; let Some(grant) = grant else { return Ok(WorkerLeaseAcquisition::Busy { retry_after: None }); }; diff --git a/backend/crates/gateway-store/src/value.rs b/backend/crates/gateway-store/src/value.rs index 6e201acfd..4b794ddc3 100644 --- a/backend/crates/gateway-store/src/value.rs +++ b/backend/crates/gateway-store/src/value.rs @@ -32,20 +32,87 @@ pub enum StoreError { source: Option, }, #[error("{entity} {id} was not found")] - NotFound { entity: &'static str, id: String }, + NotFound { + entity: &'static str, + id: String, + source: Option, + }, #[error("store conflict for {entity} {id}: {kind:?}")] Conflict { entity: &'static str, id: String, kind: ConflictKind, + source: Option, }, #[error("invalid persisted {entity}: {message}")] InvalidData { entity: &'static str, message: String, + source: Option, }, } +impl StoreError { + /// 本地校验可没有来源；转换已有失败时附加真实原因 + #[must_use] + pub(crate) fn with_source( + mut self, + error: impl Into, + ) -> Self { + let error = error.into(); + match &mut self { + Self::Unavailable { source, .. } + | Self::InvalidData { source, .. } + | Self::Conflict { source, .. } + | Self::NotFound { source, .. } => *source = Some(error), + } + self + } + + pub(crate) fn with_cleanup( + mut self, + cleanup: impl Into, + ) -> Self { + let source = match &mut self { + Self::Unavailable { source, .. } + | Self::InvalidData { source, .. } + | Self::Conflict { source, .. } + | Self::NotFound { source, .. } => source, + }; + *source = Some(gateway_core::error::ErrorSource::cleanup( + source.take(), + cleanup, + )); + self + } + + fn admin_kind(&self) -> AdminStoreErrorKind { + match self { + Self::NotFound { .. } => AdminStoreErrorKind::NotFound, + Self::Conflict { + kind: ConflictKind::StaleRevision, + .. + } => AdminStoreErrorKind::StaleRevision, + Self::Conflict { + kind: ConflictKind::DuplicateName, + .. + } => AdminStoreErrorKind::DuplicateName, + Self::Conflict { .. } => AdminStoreErrorKind::Conflict, + Self::InvalidData { .. } => AdminStoreErrorKind::Invalid, + Self::Unavailable { .. } => AdminStoreErrorKind::Unavailable, + } + } + + fn core_kind(&self) -> gateway_core::error::StoreErrorKind { + use gateway_core::error::StoreErrorKind; + match self { + Self::Unavailable { .. } => StoreErrorKind::Unavailable, + Self::Conflict { .. } => StoreErrorKind::Conflict, + Self::NotFound { .. } | Self::InvalidData { .. } => StoreErrorKind::InvalidData, + } + } +} + pub type StoreResult = Result; pub(crate) fn store_revision(revision: AdminRevision) -> AdminStoreResult { Revision::new(revision.get()).map_err(|error| admin_store_error("config revision", error)) @@ -88,21 +155,7 @@ pub(crate) fn mutation_audit( } pub(crate) fn admin_store_error(resource: &'static str, error: StoreError) -> AdminStoreError { - let kind = match error { - StoreError::NotFound { .. } => AdminStoreErrorKind::NotFound, - StoreError::Conflict { - kind: ConflictKind::StaleRevision, - .. - } => AdminStoreErrorKind::StaleRevision, - StoreError::Conflict { - kind: ConflictKind::DuplicateName, - .. - } => AdminStoreErrorKind::DuplicateName, - StoreError::Conflict { .. } => AdminStoreErrorKind::Conflict, - StoreError::InvalidData { .. } => AdminStoreErrorKind::Invalid, - StoreError::Unavailable { .. } => AdminStoreErrorKind::Unavailable, - }; - AdminStoreError::new(kind, resource, "store operation failed").with_source(error) + AdminStoreError::new(error.admin_kind(), resource, "store operation failed").with_source(error) } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -113,6 +166,7 @@ impl Revision { NonZeroU64::new(value) .map(Self) .ok_or_else(|| StoreError::InvalidData { + source: None, entity: "revision", message: "must be greater than zero".to_owned(), }) @@ -160,6 +214,7 @@ impl FromStr for DecimalAmount { }); if !valid { return Err(StoreError::InvalidData { + source: None, entity: "decimal amount", message: "expected a non-negative numeric(20,10) value".to_owned(), }); @@ -190,18 +245,21 @@ impl JsonObject { ) -> StoreResult { let serialized_bytes = serde_json::to_vec(&value) .map_err(|error| StoreError::InvalidData { + source: Some(error.into()), entity, - message: error.to_string(), + message: "JSON encoding failed".to_owned(), })? .len(); let Value::Object(fields) = value else { return Err(StoreError::InvalidData { + source: None, entity, message: "top-level JSON value must be an object".to_owned(), }); }; if serialized_bytes > max_serialized_bytes { return Err(StoreError::InvalidData { + source: None, entity, message: format!("serialized JSON exceeds {max_serialized_bytes} bytes"), }); @@ -233,6 +291,7 @@ pub(crate) fn require_nonempty( ) -> StoreResult<()> { if value.trim().is_empty() { Err(StoreError::InvalidData { + source: None, entity, message: format!("{field} must not be empty"), }) @@ -276,11 +335,5 @@ pub(crate) fn provider_unavailable( } pub(crate) fn core_store_error(error: StoreError) -> gateway_core::error::StoreError { - use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind}; - let kind = match error { - StoreError::Unavailable { .. } => StoreErrorKind::Unavailable, - StoreError::Conflict { .. } => StoreErrorKind::Conflict, - StoreError::NotFound { .. } | StoreError::InvalidData { .. } => StoreErrorKind::InvalidData, - }; - CoreStoreError::caused_by(kind, error) + gateway_core::error::StoreError::caused_by(error.core_kind(), error) } diff --git a/backend/crates/gateway-store/src/workers.rs b/backend/crates/gateway-store/src/workers.rs index df7463b58..d3fc6bb24 100644 --- a/backend/crates/gateway-store/src/workers.rs +++ b/backend/crates/gateway-store/src/workers.rs @@ -181,6 +181,7 @@ pub(crate) fn worker_definition_error( error: gateway_core::task::WorkerDefinitionError, ) -> StoreError { StoreError::InvalidData { + source: None, entity: "store worker plan", message: error.to_string(), } @@ -202,7 +203,9 @@ impl ScheduledTask for StaleModelRequestRecoveryTask { ) .await .map(|_| ()) - .map_err(|_| WorkerTaskError::safe("stale request recovery failed")) + .map_err(|source| { + WorkerTaskError::safe("stale request recovery failed").with_source(source) + }) }) } } diff --git a/backend/crates/gateway-store/tests/main.rs b/backend/crates/gateway-store/tests/main.rs index ce7b14283..965a96667 100644 --- a/backend/crates/gateway-store/tests/main.rs +++ b/backend/crates/gateway-store/tests/main.rs @@ -6,3 +6,16 @@ mod config; mod postgres; mod redis; mod support; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/gateway-store/tests/postgres/client_budgets.rs b/backend/crates/gateway-store/tests/postgres/client_budgets.rs index 03334f105..2704c998a 100644 --- a/backend/crates/gateway-store/tests/postgres/client_budgets.rs +++ b/backend/crates/gateway-store/tests/postgres/client_budgets.rs @@ -1209,3 +1209,45 @@ async fn timezone_cutover_preserves_open_windows_and_resumes_without_overlap() { ); database.close().await; } + +#[tokio::test] +async fn failed_settlement_retains_native_database_code_through_the_next_admission() { + let Some(database) = TestDatabase::create("budget_native_failure").await else { + return; + }; + sqlx::query("alter table client_api_keys rename to hidden_test_keys") + .execute(&database.pool) + .await + .unwrap(); + let budgets = PgClientBudgetStore::new(database.pool.clone()); + let error = budgets + .settle(charge("missing-key", "native-error", "1")) + .await + .unwrap_err(); + fn assert_native_code(error: &(dyn std::error::Error + 'static)) { + let mut source = Some(error); + while let Some(error) = source { + if let Some(sqlx::Error::Database(error)) = error.downcast_ref::() { + assert_eq!(error.code().as_deref(), Some("42P01")); + return; + } + source = error.source(); + } + panic!("native database failure was lost"); + } + assert_native_code(&error); + let retried = budgets.admit(key_id("missing-key")).await.unwrap_err(); + assert_eq!( + retried.kind(), + GatewayErrorKind::ProviderInfrastructureUnavailable + ); + assert_native_code(&retried); + assert!( + retried + .error_details() + .unwrap() + .as_str() + .contains("client_api_keys") + ); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs index def00d8ea..d42496cfd 100644 --- a/backend/crates/gateway-store/tests/postgres/execution_buffer.rs +++ b/backend/crates/gateway-store/tests/postgres/execution_buffer.rs @@ -969,3 +969,159 @@ async fn zero_attempt_postgres_write_failures_are_not_retried_and_do_not_stop_th assert_eq!(recovered.trace, None); database.close().await; } + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingStore { + async fn record_failure( + &self, + _: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), StoreError> { + Ok(()) + } +} + +#[async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for BlockingStore { + async fn record_failure( + &self, + _: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), StoreError> { + Ok(()) + } +} + +#[tokio::test] +async fn operational_failure_uses_the_bounded_queue_without_creating_a_model_request() { + use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure}; + use gateway_core::error::{ErrorDetails, ErrorSource}; + let Some(database) = TestDatabase::create("operational_details").await else { + return; + }; + let inner = Arc::new(PgExecutionStore::new(database.pool.clone())); + let (store, writer) = + BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(1).unwrap()); + let mut failure = + OperationalFailure::new("admin", "http_request", "unavailable", "safe summary"); + failure.correlation_id = Some("admin-request-123".to_owned()); + failure.details = ErrorDetails::capture( + Some(&ErrorSource::new(std::io::Error::other( + "PRIVATE_STORE_CAUSE", + ))), + None, + false, + ); + let occurred_at = failure.occurred_at; + store.record_failure(failure).await.unwrap(); + store + .record_failure(OperationalFailure::new( + "worker", + "run", + "worker_failed", + "safe", + )) + .await + .unwrap(); + assert_eq!(store.stats().queued_items, 1); + assert_eq!(store.stats().dropped_total, 1); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + writer.run(cancellation).await.unwrap(); + let row: ( + Option, + String, + String, + chrono::DateTime, + ) = sqlx::query_as( + "select model_request_id, message, error_details, created_at + from ops_events where component = 'admin'", + ) + .fetch_one(&database.pool) + .await + .unwrap(); + assert!(row.0.is_none()); + let message: serde_json::Value = serde_json::from_str(&row.1).unwrap(); + assert_eq!(message["correlationId"], "admin-request-123"); + assert!(!row.1.contains("PRIVATE_STORE_CAUSE")); + assert!(row.2.contains("PRIVATE_STORE_CAUSE")); + assert_eq!( + row.3.timestamp_micros(), + chrono::DateTime::::from(occurred_at).timestamp_micros() + ); + let requests: i64 = sqlx::query_scalar("select count(*) from model_requests") + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(requests, 0); + database.close().await; +} + +#[tokio::test] +async fn progress_congestion_preserves_failure_reserve_without_waiting() { + use futures::FutureExt as _; + use gateway_core::diagnostics::{OperationalDiagnostics as _, OperationalFailure}; + use gateway_core::error::{ErrorDetails, ErrorSource}; + let Some(database) = TestDatabase::create("failure_reserve").await else { + return; + }; + let inner = Arc::new(PgExecutionStore::new(database.pool.clone())); + let mut request_ids = Vec::new(); + for index in 0..4 { + let request = accepted_request(&format!("req_reserved_failure_{index}")); + request_ids.push(request.id.clone()); + inner.create_model_request(request).await.unwrap(); + } + let request_id = &request_ids[0]; + let (store, writer) = + BufferedExecutionStore::with_capacity(inner, NonZeroUsize::new(4).unwrap()); + // 不启动消费者，确定性地填满普通额度；每次调用必须首次 poll 就返回 + for (request_id, status) in request_ids.iter().zip([200, 201, 202, 203]) { + store + .record_client_status(request_id, status) + .now_or_never() + .expect("enqueue waited") + .unwrap(); + } + let mut failure = + OperationalFailure::new("admin", "http_request", "unavailable", "safe failure"); + failure.correlation_id = Some(request_id.as_str().to_owned()); + failure.details = ErrorDetails::capture( + Some(&ErrorSource::new(std::io::Error::other( + "ORIGINAL_RESERVED_CAUSE", + ))), + None, + false, + ); + store + .record_failure(failure) + .now_or_never() + .expect("failure waited for queue capacity") + .unwrap(); + assert_eq!(store.stats().queued_items, 5); + assert_eq!(store.stats().dropped_total, 0); + store + .record_client_status(request_id, 204) + .now_or_never() + .expect("overflow waited") + .unwrap(); + assert_eq!(store.stats().queued_items, 5); + assert_eq!(store.stats().dropped_total, 1); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + writer.run(cancelled).await.unwrap(); + assert_eq!(store.stats().persisted_total, 5); + let detail: String = + sqlx::query_scalar("select error_details from ops_events where component = 'admin'") + .fetch_one(&database.pool) + .await + .unwrap(); + assert!(detail.contains("ORIGINAL_RESERVED_CAUSE")); + let status: i32 = + sqlx::query_scalar("select client_status_code from model_requests where id = $1") + .bind(request_id.as_str()) + .fetch_one(&database.pool) + .await + .unwrap(); + assert_eq!(status, 200); + assert_eq!(store.stats().queued_bytes, 0); + database.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/query_budget.rs b/backend/crates/gateway-store/tests/postgres/query_budget.rs index 4c18a90fa..8459a25d9 100644 --- a/backend/crates/gateway-store/tests/postgres/query_budget.rs +++ b/backend/crates/gateway-store/tests/postgres/query_budget.rs @@ -19,6 +19,7 @@ async fn query_stream_holds_budget_until_completion_error_or_drop() { None, Some(Ok(2)), Some(Err(StoreError::InvalidData { + source: None, entity: "test stream", message: "decode failed".to_owned(), })), diff --git a/backend/crates/gateway-store/tests/redis/coordination_buffer.rs b/backend/crates/gateway-store/tests/redis/coordination_buffer.rs index d65ccd38a..121a3b351 100644 --- a/backend/crates/gateway-store/tests/redis/coordination_buffer.rs +++ b/backend/crates/gateway-store/tests/redis/coordination_buffer.rs @@ -141,6 +141,7 @@ async fn full_recoverable_coordination_queues_should_drop_writes_without_waiting let (admissions, _admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(1).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_buffer_test").expect("client key"); let request = ModelRequestId::new("req_buffer_test").expect("request ID"); @@ -167,6 +168,7 @@ async fn redis_coordination_writers_should_flush_each_side_effect() { let (admissions, admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(8).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_writer_test").expect("client key"); let request = ModelRequestId::new("req_writer_test").expect("request ID"); @@ -207,6 +209,7 @@ async fn awaited_buffered_release_should_not_publish_capacity_before_writer_runs let (admissions, admission_writer) = BufferedClientAdmissionPort::with_capacity( inner.clone(), NonZeroUsize::new(8).expect("capacity"), + Arc::new(crate::RecordingDiagnostics::default()), ); let client = ClientApiKeyId::new("key_capacity_handoff").expect("client key"); let request = |index| ClientAdmissionRequest { diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index 2a3deccf3..3291c843e 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -1632,6 +1632,7 @@ fn map_store_error(error: gateway_core::error::StoreError) -> ProviderAdminError StoreErrorKind::Unavailable => ProviderAdminErrorKind::Unavailable, _ => ProviderAdminErrorKind::Internal, }) + .with_source(error) } fn map_credential_admin_error(error: CodexCredentialAdminError) -> ProviderAdminError { diff --git a/backend/crates/providers/openai/src/credential/admin.rs b/backend/crates/providers/openai/src/credential/admin.rs index 0353eec8b..335763569 100644 --- a/backend/crates/providers/openai/src/credential/admin.rs +++ b/backend/crates/providers/openai/src/credential/admin.rs @@ -864,6 +864,7 @@ pub struct CodexCredentialAdminService { personal_access_token_client: Option>, leases: Arc, runtime_policy: Arc, + diagnostics: Arc, } impl fmt::Debug for CodexCredentialAdminService { @@ -886,12 +887,14 @@ impl CodexCredentialAdminService { refresher: Arc, leases: Arc, runtime_policy: Arc, + diagnostics: Arc, ) -> Self { Self { refresher, personal_access_token_client: None, leases, runtime_policy, + diagnostics, } } @@ -959,9 +962,20 @@ impl CodexCredentialAdminService { refresh_token.expose_secret(), current.account.outbound_proxy(), ) - .await - .inspect_err(|error| log_manual_refresh_failure(&account_id, error)) - .map_err(map_refresh_failure)?; + .await; + let tokens = match tokens { + Ok(tokens) => tokens, + Err(error) => { + super::diagnostics::record_refresh_failure( + self.diagnostics.as_ref(), + &account_id, + "manual_refresh", + &error, + ) + .await; + return Err(map_refresh_failure(error)); + } + }; let access_token_expires_at = match tokens.access_token.as_deref() { Some(access_token) => parse_access_token_expiration(access_token), None => current @@ -1236,7 +1250,9 @@ fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError { RefreshFailure::RetryableTransport { .. } => CodexCredentialAdminError::RefreshUnavailable, // Worker 的 Transport 分类还承担 401 退避；管理提示只按已收到的响应事实细分， // 不改变后台刷新策略，也不把明确失败响应误报为租约冲突或执行结果未知 - RefreshFailure::Transport { message, upstream } => match upstream { + RefreshFailure::Transport { + message, upstream, .. + } => match upstream { Some(upstream) => CodexCredentialAdminError::RefreshUpstream { status: upstream.status(), code: upstream.code().map(str::to_owned), @@ -1247,20 +1263,6 @@ fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError { } } -fn log_manual_refresh_failure(account_id: &ProviderAccountId, error: &RefreshFailure) { - let upstream = error.upstream(); - tracing::warn!( - account_id = %account_id, - failure_class = error.classification(), - upstream_message = ?error.message(), - upstream_status = ?upstream.map(super::token_client::RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(super::token_client::RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(super::token_client::RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(super::token_client::RefreshUpstreamFailure::body), - "OpenAI OAuth manual refresh failed" - ); -} - fn parse_import_document( payload: &Value, default_proxy: Option<&gateway_core::account::OutboundProxy>, diff --git a/backend/crates/providers/openai/src/credential/diagnostics.rs b/backend/crates/providers/openai/src/credential/diagnostics.rs new file mode 100644 index 000000000..b848ef7da --- /dev/null +++ b/backend/crates/providers/openai/src/credential/diagnostics.rs @@ -0,0 +1,52 @@ +//! OAuth 刷新失败的受控观测投影，普通日志只记录分类和关联 + +use super::token_client::RefreshFailure; +use gateway_core::{ + account::ProviderAccountId, + diagnostics::{OperationalDiagnostics, OperationalFailure}, + error::{ErrorDetails, ErrorSource, OpaqueUpstreamValue, RawUpstreamError}, + identity::ProviderKind, +}; + +pub(crate) async fn record_refresh_failure( + diagnostics: &dyn OperationalDiagnostics, + account_id: &ProviderAccountId, + operation: &'static str, + error: &RefreshFailure, +) { + let upstream = error.upstream(); + let mut failure = OperationalFailure::new( + "oauth", + operation, + error.classification(), + error.to_string(), + ); + failure.provider_kind = Some(ProviderKind::new("openai").expect("static provider")); + failure.account_id = Some(account_id.clone()); + failure.upstream_status = upstream.map(|value| value.status()); + failure.upstream_code = upstream + .and_then(|value| value.code()) + .map(|code| OpaqueUpstreamValue::new(code.to_owned())); + let source = ErrorSource::new(RefreshDiagnostic(error.clone())); + let body = upstream.map(|value| RawUpstreamError::new(value.body())); + failure.details = ErrorDetails::capture(Some(&source), body.as_ref(), error.redacted()); + if diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(account_id = %account_id, operation, "OAuth diagnostic could not be recorded"); + } +} + +#[derive(Debug)] +struct RefreshDiagnostic(RefreshFailure); +impl std::fmt::Display for RefreshDiagnostic { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0.message() { + Some(message) => formatter.write_str(message), + None => std::fmt::Display::fmt(&self.0, formatter), + } + } +} +impl std::error::Error for RefreshDiagnostic { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} diff --git a/backend/crates/providers/openai/src/credential/mod.rs b/backend/crates/providers/openai/src/credential/mod.rs index a1fb5bbb7..435044416 100644 --- a/backend/crates/providers/openai/src/credential/mod.rs +++ b/backend/crates/providers/openai/src/credential/mod.rs @@ -76,3 +76,5 @@ pub use types::{ CodexCredentialData, CodexCredentialPrincipal, CodexOAuthCredentialData, CodexOAuthSecret, ResponsesTransport, RuntimeCodexCookie, UpsertCodexCookie, }; + +mod diagnostics; diff --git a/backend/crates/providers/openai/src/credential/refresh.rs b/backend/crates/providers/openai/src/credential/refresh.rs index eededdd56..40d91b741 100644 --- a/backend/crates/providers/openai/src/credential/refresh.rs +++ b/backend/crates/providers/openai/src/credential/refresh.rs @@ -95,7 +95,6 @@ fn log_refresh_deferred( access_token_expires_at: Option, attempt: u32, reason: &'static str, - upstream_message: Option<&str>, upstream: Option<&RefreshUpstreamFailure>, retry_at: SystemTime, ) { @@ -103,11 +102,11 @@ fn log_refresh_deferred( account_id = %account_id, attempt, reason, - upstream_message = ?upstream_message, + upstream_status = ?upstream.map(RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(RefreshUpstreamFailure::body), + + + retry_at = %DateTime::::from(retry_at), access_token_expires_at = ?access_token_expires_at.map(DateTime::::from), recovery_deadline = ?refresh_recovery_deadline(access_token_expires_at) @@ -193,6 +192,7 @@ pub struct CodexCredentialRefreshService { leases: Arc, credential_state: Arc, runtime_policy: Arc, + diagnostics: Arc, } impl CodexCredentialRefreshService { @@ -202,6 +202,7 @@ impl CodexCredentialRefreshService { leases: Arc, credential_state: Arc, runtime_policy: Arc, + diagnostics: Arc, ) -> Self { Self { repository, @@ -209,6 +210,7 @@ impl CodexCredentialRefreshService { leases, credential_state, runtime_policy, + diagnostics, } } @@ -239,11 +241,21 @@ impl CodexCredentialRefreshService { match result { Ok(outcome) => outcomes.push(outcome), Err(error) => { - tracing::warn!( - account_id = %account_id, - error = %error, - "OpenAI OAuth refresh attempt failed" + let mut failure = gateway_core::diagnostics::OperationalFailure::new( + "oauth", + "scheduled_refresh", + "refresh_storage", + "OpenAI OAuth refresh attempt failed", ); + failure.account_id = ProviderAccountId::new(account_id.clone()).ok(); + failure.details = gateway_core::error::ErrorDetails::capture( + Some(&ErrorSource::new(error)), + None, + false, + ); + if self.diagnostics.record_failure(failure).await.is_err() { + tracing::warn!(account_id = %account_id, "OAuth diagnostic could not be recorded"); + } outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id }); } } @@ -294,6 +306,15 @@ impl CodexCredentialRefreshService { .refresher .refresh_with_proxy(refresh_token.expose_secret(), due.account.outbound_proxy()) .await; + if let Err(failure) = &refresh_result { + super::diagnostics::record_refresh_failure( + self.diagnostics.as_ref(), + due.account.id(), + "scheduled_refresh", + failure, + ) + .await; + } if recovery_window_exhausted && let Err(failure) = &refresh_result { let message = failure.message().map(str::to_owned); let upstream = failure.upstream(); @@ -356,7 +377,7 @@ impl CodexCredentialRefreshService { ) .await } - Err(RefreshFailure::RetryableTransport { message }) => { + Err(RefreshFailure::RetryableTransport { message, .. }) => { if self .defer_refresh(&due.account, "transport-not-sent", Some(&message), None) .await? @@ -366,7 +387,9 @@ impl CodexCredentialRefreshService { Ok(CodexCredentialRefreshOutcome::Stale { account_id }) } } - Err(RefreshFailure::Transport { message, upstream }) => { + Err(RefreshFailure::Transport { + message, upstream, .. + }) => { // 上游瞬态（401/429/5xx/超时/畸形响应等）保留现有凭据、 // 记录最近一次失败并推进有界退避 if self @@ -576,11 +599,11 @@ impl CodexCredentialRefreshService { account_id = %account.id(), ?credential_state, reason, - upstream_message = ?message.as_deref(), + upstream_status = ?upstream.map(RefreshUpstreamFailure::status), - upstream_code = ?upstream.and_then(RefreshUpstreamFailure::code), - upstream_type = ?upstream.and_then(RefreshUpstreamFailure::error_type), - upstream_body = ?upstream.map(RefreshUpstreamFailure::body), + + + access_token_expires_at = ?account.access_token_expires_at() .map(DateTime::::from), recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at()) @@ -637,7 +660,6 @@ impl CodexCredentialRefreshService { account.access_token_expires_at(), attempt, reason, - upstream_message, upstream, retry_at, ); diff --git a/backend/crates/providers/openai/src/credential/token_client.rs b/backend/crates/providers/openai/src/credential/token_client.rs index 0560e415d..177b77da9 100644 --- a/backend/crates/providers/openai/src/credential/token_client.rs +++ b/backend/crates/providers/openai/src/credential/token_client.rs @@ -81,7 +81,7 @@ impl fmt::Debug for TokenPair { } /// Codex token 刷新的稳定失败分类 -#[derive(Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Clone, thiserror::Error)] pub enum RefreshFailure { #[error("refresh token is invalid or expired")] InvalidGrant { @@ -94,11 +94,19 @@ pub enum RefreshFailure { upstream: Option>, }, #[error("refresh transport failed before server processing")] - RetryableTransport { message: String }, + RetryableTransport { + message: String, + #[source] + source: Option, + redacted: bool, + }, #[error("refresh transport failed after possible server processing")] Transport { message: Option, upstream: Option>, + #[source] + source: Option, + redacted: bool, }, } @@ -120,7 +128,7 @@ impl RefreshFailure { Self::InvalidGrant { message, .. } | Self::Banned { message, .. } | Self::Transport { message, .. } => message.as_deref(), - Self::RetryableTransport { message } => Some(message), + Self::RetryableTransport { message, .. } => Some(message), } } @@ -134,6 +142,47 @@ impl RefreshFailure { } } + pub(crate) fn redacted(&self) -> bool { + self.upstream() + .is_some_and(RefreshUpstreamFailure::redacted) + || match self { + Self::RetryableTransport { redacted, .. } | Self::Transport { redacted, .. } => { + *redacted + } + _ => false, + } + } + + fn redact_refresh_token(&mut self, secret: &str) { + if secret.is_empty() { + return; + } + match self { + Self::InvalidGrant { message, upstream } + | Self::Banned { message, upstream } + | Self::Transport { + message, upstream, .. + } => { + if let Some(message) = message { + *message = message.replace(secret, "[REDACTED]"); + } + if let Some(upstream) = upstream { + for value in [&mut upstream.body] + .into_iter() + .chain(upstream.code.iter_mut()) + .chain(upstream.error_type.iter_mut()) + { + if value.contains(secret) { + *value = value.replace(secret, "[REDACTED]"); + upstream.redacted = true; + } + } + } + } + Self::RetryableTransport { .. } => {} + } + } + #[must_use] pub const fn classification(&self) -> &'static str { match self { @@ -147,18 +196,28 @@ impl RefreshFailure { /// 当前 OAuth 刷新请求收到的完整非成功响应 /// -/// 该值不持久化；`Debug` 不输出正文 -/// 调用方仅在受控刷新失败日志中显式记录正文 +/// 原文只进入受控诊断详情；普通日志不展开正文或任意上游值 #[derive(Clone, PartialEq, Eq)] pub struct RefreshUpstreamFailure { status: u16, code: Option, error_type: Option, body: String, + redacted: bool, } impl RefreshUpstreamFailure { fn new(status: StatusCode, body: &[u8], error: Option<&RefreshErrorResponse>) -> Self { + let mut text = String::from_utf8_lossy(body).into_owned(); + let redacted = if let Ok(mut value) = serde_json::from_slice::(body) { + let changed = redact_oauth_tokens(&mut value); + if changed { + text = value.to_string(); + } + changed + } else { + false + }; Self { status: status.as_u16(), code: error @@ -167,10 +226,16 @@ impl RefreshUpstreamFailure { error_type: error .and_then(RefreshErrorResponse::error_type) .map(str::to_owned), - body: String::from_utf8_lossy(body).into_owned(), + body: text, + redacted, } } + #[must_use] + pub const fn redacted(&self) -> bool { + self.redacted + } + #[must_use] pub const fn status(&self) -> u16 { self.status @@ -197,8 +262,11 @@ impl fmt::Debug for RefreshUpstreamFailure { formatter .debug_struct("RefreshUpstreamFailure") .field("status", &self.status) - .field("code", &self.code) - .field("error_type", &self.error_type) + .field("code", &self.code.as_ref().map(|_| "")) + .field( + "error_type", + &self.error_type.as_ref().map(|_| ""), + ) .field("body", &"") .finish() } @@ -222,6 +290,8 @@ pub trait TokenRefresher: Send + Sync + 'static { fn proxy_refresh_failure() -> RefreshFailure { RefreshFailure::RetryableTransport { + redacted: false, + source: None, message: "account OAuth egress unavailable".to_owned(), } } @@ -503,13 +573,19 @@ impl TokenRefresher for OpenAiTokenClient { proxy: Option<&gateway_core::account::OutboundProxy>, ) -> Result { self.with_proxy(proxy) - .map_err(|_| proxy_refresh_failure())? + .map_err(|source| RefreshFailure::RetryableTransport { + redacted: false, + message: "account OAuth egress unavailable".to_owned(), + source: Some(source.into()), + })? .refresh(refresh_token) .await } async fn refresh(&self, refresh_token: &str) -> Result { - let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|_| { + let headers = build_codex_profile_headers(&self.profile.snapshot()).map_err(|source| { RefreshFailure::Transport { + redacted: false, + source: Some(source.into()), message: Some("OpenAI OAuth refresh profile is invalid".to_owned()), upstream: None, } @@ -525,12 +601,16 @@ impl TokenRefresher for OpenAiTokenClient { }) .send() .await - .map_err(|error| refresh_transport_failure(&error))?; + .map_err(refresh_transport_failure)?; let (status, body) = read_bounded_response(response).await?; if !status.is_success() { - return Err(classify_refresh_failure(status, &body)); + let mut failure = classify_refresh_failure(status, &body); + failure.redact_refresh_token(refresh_token); + return Err(failure); } - parse_token_pair(&body).map_err(|()| RefreshFailure::Transport { + parse_token_pair(&body).map_err(|source| RefreshFailure::Transport { + redacted: true, + source: Some(source), message: Some("OpenAI OAuth refresh returned an invalid success response".to_owned()), upstream: None, }) @@ -604,8 +684,10 @@ async fn read_bounded_response( .chunk() .await .map_err(|error| RefreshFailure::Transport { - message: Some(error.to_string()), + redacted: error.url().is_some(), + message: Some("OpenAI OAuth response body read failed".to_owned()), upstream: None, + source: Some(error.without_url().into()), })? { let next_len = body @@ -617,6 +699,8 @@ async fn read_bounded_response( "OpenAI OAuth response exceeded {MAX_OAUTH_RESPONSE_BYTES} bytes" )), upstream: None, + source: None, + redacted: true, })?; body.reserve(next_len.saturating_sub(body.len())); body.extend_from_slice(&chunk); @@ -624,14 +708,19 @@ async fn read_bounded_response( Ok((status, body)) } -fn parse_token_pair(body: &[u8]) -> Result { - let tokens = serde_json::from_slice::(body).map_err(|_| ())?; +#[derive(Debug, thiserror::Error)] +#[error("OpenAI OAuth ID token claims are invalid")] +struct InvalidRefreshClaims; + +fn parse_token_pair(body: &[u8]) -> Result { + let tokens = serde_json::from_slice::(body) + .map_err(gateway_core::error::ErrorSource::new)?; if tokens .id_token .as_deref() .is_some_and(|token| super::types::parse_chatgpt_jwt_claims(token).is_err()) { - return Err(()); + return Err(InvalidRefreshClaims.into()); } Ok(TokenPair { access_token: tokens.access_token, @@ -662,6 +751,8 @@ fn classify_refresh_failure(status: StatusCode, body: &[u8]) -> RefreshFailure { // 显式 401 先进入有界恢复退避，避免瞬时授权故障直接失效账号 if status == StatusCode::UNAUTHORIZED { return RefreshFailure::Transport { + redacted: false, + source: None, message, upstream: upstream(), }; @@ -687,19 +778,30 @@ fn classify_refresh_failure(status: StatusCode, body: &[u8]) -> RefreshFailure { }; } RefreshFailure::Transport { + redacted: false, + source: None, message, upstream: upstream(), } } -fn refresh_transport_failure(error: &reqwest::Error) -> RefreshFailure { - let message = error.to_string(); - if is_safe_to_retry_refresh_transport(error) { - RefreshFailure::RetryableTransport { message } +fn refresh_transport_failure(error: reqwest::Error) -> RefreshFailure { + let retryable = is_safe_to_retry_refresh_transport(&error); + let redacted = error.url().is_some(); + let source = Some(error.without_url().into()); + let message = "OpenAI OAuth transport failed".to_owned(); + if retryable { + RefreshFailure::RetryableTransport { + message, + source, + redacted, + } } else { RefreshFailure::Transport { message: Some(message), upstream: None, + source, + redacted, } } } @@ -718,3 +820,36 @@ fn is_safe_to_retry_refresh_transport(error: &reqwest::Error) -> bool { || message.contains("network is unreachable") || message.contains("tls handshake") } + +fn redact_oauth_tokens(value: &mut serde_json::Value) -> bool { + let mut redacted = false; + match value { + serde_json::Value::Object(fields) => { + for (key, value) in fields { + let sensitive = [ + "access_token", + "refresh_token", + "id_token", + "client_secret", + "authorization", + "cookie", + ] + .iter() + .any(|field| key.eq_ignore_ascii_case(field)); + if sensitive && !value.is_null() { + *value = serde_json::Value::String("[REDACTED]".to_owned()); + redacted = true; + } else { + redacted |= redact_oauth_tokens(value); + } + } + } + serde_json::Value::Array(values) => { + for value in values { + redacted |= redact_oauth_tokens(value); + } + } + _ => {} + } + redacted +} diff --git a/backend/crates/providers/openai/src/lib.rs b/backend/crates/providers/openai/src/lib.rs index a48c02f27..d1506bbf0 100644 --- a/backend/crates/providers/openai/src/lib.rs +++ b/backend/crates/providers/openai/src/lib.rs @@ -199,6 +199,7 @@ pub async fn initialize( Arc::clone(&refresher), Arc::clone(&leases), Arc::clone(&runtime_policy), + ports.diagnostics(), ) .with_personal_access_token_client(token_client), ); @@ -208,6 +209,7 @@ pub async fn initialize( Arc::clone(&leases), credential_state, Arc::clone(&runtime_policy), + ports.diagnostics(), )); let pending = Arc::new(OpenAiOAuthPendingStore::new( ports.oauth_pending(), diff --git a/backend/crates/providers/openai/src/provider/failure.rs b/backend/crates/providers/openai/src/provider/failure.rs index bc2bd85f5..2fb6226ab 100644 --- a/backend/crates/providers/openai/src/provider/failure.rs +++ b/backend/crates/providers/openai/src/provider/failure.rs @@ -710,9 +710,6 @@ pub(super) fn continuation_replay_required_error(reason: &'static str) -> Provid ContinuationRecoveryDisposition::ClientReplayRequired, ) .with_continuation_unavailable_reason(reason) - .with_upstream_code(OpaqueUpstreamValue::new( - PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned(), - )) .with_diagnostic(ProviderDiagnostic::new( "previous response is unavailable in the selected upstream scope; client replay is required", )) @@ -974,30 +971,20 @@ pub(super) fn map_client_error( failure } CodexClientError::WebSocket(error) => { - let close_code = error.close_before_terminal().and_then(|close| close.code()); let client_visible_error = websocket_client_visible_error(error); let mut failure = MappedProviderFailure::plain(provider_error( websocket_error_kind(error), websocket_send_state(error), )); - if let Some(close_code) = close_code { - failure.error = - failure - .error - .with_upstream_code(OpaqueUpstreamValue::new(format!( - "websocket_close_{close_code}" - ))); - } if let CodexWebSocketExchangeError::ConnectionLimitReached(upstream) = error.classified() { - failure.error = - failure + failure.error = failure.error.with_replay_safe(); + if let Some(code) = upstream.upstream_code.as_ref() { + failure.error = failure .error - .with_replay_safe() - .with_upstream_code(OpaqueUpstreamValue::new( - WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(), - )); + .with_upstream_code(OpaqueUpstreamValue::new(code.clone())); + } if let Some(status) = upstream.explicit_status_code { failure.error = failure.error.with_status(status); } diff --git a/backend/crates/providers/openai/src/provider/mod.rs b/backend/crates/providers/openai/src/provider/mod.rs index 5fe833688..bb34992e6 100644 --- a/backend/crates/providers/openai/src/provider/mod.rs +++ b/backend/crates/providers/openai/src/provider/mod.rs @@ -82,9 +82,7 @@ use crate::transport::protocol::responses::{ CodexResponsesRequest, PREVIOUS_RESPONSE_NOT_FOUND_CODE, PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE, PreviousResponseScope, ResponseEventSignals, TransportRequirement, transport_requirement, }; -use crate::transport::protocol::websocket::{ - WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE, websocket_response_create_payload_len, -}; +use crate::transport::protocol::websocket::websocket_response_create_payload_len; use crate::transport::request::{ CodexRequestEncodeError, RequestAccountScope, align_structured_location_fields, clear_request_turn_state, encode_generate_request, scope_request_to_account, diff --git a/backend/crates/providers/openai/src/provider/workers.rs b/backend/crates/providers/openai/src/provider/workers.rs index 96c4c07ea..c48342aec 100644 --- a/backend/crates/providers/openai/src/provider/workers.rs +++ b/backend/crates/providers/openai/src/provider/workers.rs @@ -141,8 +141,7 @@ impl ScheduledTask for OpenAiOAuthRefreshTask { return Ok(()); } let outcomes = self.service.refresh_due().await.map_err(|error| { - tracing::error!(error = %error, "OpenAI OAuth refresh cycle failed"); - WorkerTaskError::safe("OpenAI OAuth refresh failed") + WorkerTaskError::safe("OpenAI OAuth refresh failed").with_source(error) })?; let mut refreshed = 0_u64; let mut invalidated = 0_u64; @@ -259,11 +258,8 @@ impl ScheduledTask for OpenAiQuotaTask { } Ok(_) => {} Err(error) => { - tracing::warn!( - error = %error, - "OpenAI quota synchronization failed" - ); - return Err(WorkerTaskError::safe("OpenAI quota synchronization failed")); + return Err(WorkerTaskError::safe("OpenAI quota synchronization failed") + .with_source(error)); } } Ok(()) @@ -279,12 +275,10 @@ impl ScheduledTask for OpenAiCatalogTask { } let result = match self.catalog.refresh_catalogs().await { Ok(_) | Err(CodexCredentialCatalogError::NoEligibleCredential) => Ok(()), - Err(error) => { - tracing::warn!(error = %error, "OpenAI model catalog refresh failed"); - Err(WorkerTaskError::safe( - "OpenAI model catalog synchronization failed", - )) - } + Err(error) => Err(WorkerTaskError::safe( + "OpenAI model catalog synchronization failed", + ) + .with_source(error)), }; // 成功周期尾部追加 [0, 20%) 单侧随机抖动，打破固定周期轮询特征； // 放在周期尾部避免延迟冷启动首轮刷新，失败路径交给宿主退避不叠加。 @@ -385,8 +379,7 @@ impl ScheduledTask for OpenAiWarmupTask { .load_warmup_policy() .await .map_err(|error| { - tracing::warn!(error = %error, "OpenAI warmup policy load failed"); - WorkerTaskError::safe("OpenAI warmup policy load failed") + WorkerTaskError::safe("OpenAI warmup policy load failed").with_source(error) })?; if !policy.enabled() { return Ok(()); @@ -424,7 +417,9 @@ impl ScheduledTask for OpenAiWarmupTask { .runtime_policy() .claim_warmup_slot(self.timezone, slot) .await - .map_err(|_| WorkerTaskError::safe("warmup slot is unavailable"))? + .map_err(|source| { + WorkerTaskError::safe("warmup slot is unavailable").with_source(source) + })? { return Ok(()); } diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index 7fad73770..9946579d9 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -1729,6 +1729,7 @@ fn provider_ports_with_catalog( Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), pending, + Arc::new(RecordingDiagnostics::default()), ) } @@ -2530,6 +2531,7 @@ mod errors { Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ) } @@ -2736,3 +2738,16 @@ async fn oauth_transport_settings_preserve_tokens_refresh_schedule_and_health() json!({"transport":"http"}).as_object().unwrap() ); } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/openai/tests/credential/admin.rs b/backend/crates/providers/openai/tests/credential/admin.rs index 008c0c356..6ad1aad92 100644 --- a/backend/crates/providers/openai/tests/credential/admin.rs +++ b/backend/crates/providers/openai/tests/credential/admin.rs @@ -30,6 +30,7 @@ async fn sub2api_import_resolves_distinct_proxy_bindings_and_encodes_credentials Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service.prepare_import_document(serde_json::json!({"data": { "accounts": [ @@ -64,6 +65,7 @@ async fn import_default_proxy_preserves_explicit_account_exits() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let default_proxy = gateway_core::account::OutboundProxy::parse("http://default.example:8080").unwrap(); @@ -112,6 +114,7 @@ async fn sub2api_invalid_proxy_bindings_are_rejected_before_any_token_refresh() Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let proxy = serde_json::json!({"proxy_key": "bound", "protocol": "http", "host": "127.0.0.1", "port": 18080, "status": "active"}); let mut cases = vec![ @@ -152,6 +155,7 @@ async fn direct_import_persists_opaque_tokens_without_profile_or_token_validatio Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -193,6 +197,7 @@ async fn direct_import_accepts_snake_case_oauth_token_aliases() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -246,6 +251,7 @@ async fn direct_import_projects_access_token_jwt_expiry_without_persisting_refre Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -286,6 +292,7 @@ fn pat_service(server: &MockServer) -> CodexCredentialAdminService { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_personal_access_token_client(Arc::new(client)) } @@ -397,6 +404,7 @@ async fn pat_import_times_out_without_falling_back_to_document_identity() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ) .with_personal_access_token_client(Arc::new(client)); let error = service @@ -419,6 +427,7 @@ async fn pat_import_requires_a_validation_client() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = service .prepare_import_document(serde_json::json!({ @@ -679,6 +688,7 @@ async fn oauth_import_uses_official_chatgpt_access_token_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -707,6 +717,7 @@ async fn oauth_import_uses_official_plan_alias_projection_and_user_id_fallback() Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -726,6 +737,7 @@ async fn oauth_import_should_preserve_pro_max_and_future_plan_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for plan in ["prolite", "pro", "promax", "future_plan"] { let access_token = test_jwt(serde_json::json!({"https://api.openai.com/auth":{ @@ -757,6 +769,7 @@ async fn oauth_import_uses_id_token_then_access_token_for_missing_claims() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -780,6 +793,7 @@ async fn oauth_import_does_not_use_top_level_identity_fields() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let prepared = service .prepare_import_document(serde_json::json!({ @@ -805,6 +819,7 @@ async fn oauth_import_rejects_legacy_bare_token_field() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let error = service .prepare_import_document(serde_json::json!({ @@ -869,6 +884,7 @@ async fn api_key_import_export_preserves_target_without_oauth_exchange() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); let imported = service.prepare_import_document(serde_json::json!({ "provider": "openai", "authentication_kind": "api_key", "name": "relay", "base_url": "https://relay.example/custom/v2", "api_key": "sk-test-only" @@ -922,6 +938,7 @@ async fn api_key_import_rejects_unsafe_urls_and_empty_keys() { Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for (url, key) in [ ("http://remote.example/v1", "sk-test"), @@ -947,6 +964,7 @@ async fn sub2api_api_key_import_preserves_versioned_and_explicit_responses_paths Arc::new(UnusedRefresher), Arc::new(TestLeaseCoordinator::default()), runtime_policy(), + Arc::new(crate::RecordingDiagnostics::default()), ); for (base, expected) in [ ("https://example.com", "https://example.com/v1"), diff --git a/backend/crates/providers/openai/tests/credential/refresh.rs b/backend/crates/providers/openai/tests/credential/refresh.rs index 17f5e1dc8..1ae8824ff 100644 --- a/backend/crates/providers/openai/tests/credential/refresh.rs +++ b/backend/crates/providers/openai/tests/credential/refresh.rs @@ -97,6 +97,8 @@ impl TokenRefresher for SingleUseRefresher { .expect("refresh response lock") .take() .ok_or_else(|| RefreshFailure::Transport { + redacted: false, + source: None, message: Some("test refresh response is exhausted".to_owned()), upstream: None, }) @@ -220,6 +222,7 @@ fn refresh_service( Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), runtime_policy, + Arc::new(crate::RecordingDiagnostics::default()), ) } @@ -302,6 +305,7 @@ async fn scheduled_refresh_stops_retrying_rejected_disabled_credentials() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), MutableRuntimePolicy::new(Duration::from_secs(5 * 60)), + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_disabled_rejected"; seed_refreshable_account(&store, account_id, SystemTime::now(), None).await; @@ -507,6 +511,7 @@ async fn scheduled_refresh_persists_the_original_upstream_error_message() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_invalid_refresh_token"; seed_refreshable_account( @@ -570,6 +575,7 @@ async fn scheduled_refresh_persists_retryable_message_inside_the_two_hour_window Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_retryable_unauthorized"; let expires_at = SystemTime::now() @@ -614,6 +620,8 @@ async fn scheduled_refresh_uses_the_final_message_after_the_two_hour_window() { store.repository(), Arc::new(FailingRefresher { failure: RefreshFailure::Transport { + redacted: false, + source: None, message: Some(upstream_message.to_owned()), upstream: None, }, @@ -621,6 +629,7 @@ async fn scheduled_refresh_uses_the_final_message_after_the_two_hour_window() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), policy, + Arc::new(crate::RecordingDiagnostics::default()), ); let account_id = "acct_exhausted_unauthorized"; seed_refreshable_account( @@ -935,6 +944,7 @@ async fn quota_rejection_during_oauth_refresh_preserves_terminal_result() { Arc::new(RefreshLeases), Arc::new(RefreshCredentialState), MutableRuntimePolicy::new(Duration::from_secs(5 * 60)), + Arc::new(crate::RecordingDiagnostics::default()), ); let outcomes = service.refresh_due().await.expect("refresh cycle"); assert!( @@ -952,3 +962,46 @@ async fn quota_rejection_during_oauth_refresh_preserves_terminal_result() { ); } } + +#[tokio::test] +async fn scheduled_refresh_records_native_failure_once_and_keeps_retry_state() { + let store = Arc::new(MemoryAccountStore::default()); + let diagnostics = Arc::new(crate::RecordingDiagnostics::default()); + let service = CodexCredentialRefreshService::new( + store.repository(), + Arc::new(FailingRefresher { + failure: RefreshFailure::Transport { + redacted: false, + message: Some("safe transport summary".to_owned()), + upstream: None, + source: Some(std::io::Error::other("PRIVATE_OAUTH_NATIVE_CAUSE").into()), + }, + }), + Arc::new(RefreshLeases), + Arc::new(RefreshCredentialState), + MutableRuntimePolicy::new(Duration::from_secs(300)), + diagnostics.clone(), + ); + seed_refreshable_account(&store, "acct_diagnostic", SystemTime::now(), None).await; + let outcomes = service.refresh_due().await.unwrap(); + assert!(matches!( + outcomes.as_slice(), + [CodexCredentialRefreshOutcome::Transient { .. }] + )); + let failures = diagnostics.0.lock().unwrap(); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].account_id.as_ref().unwrap().as_str(), + "acct_diagnostic" + ); + assert_eq!(failures[0].operation, "scheduled_refresh"); + assert!( + failures[0] + .details + .as_ref() + .unwrap() + .as_str() + .contains("PRIVATE_OAUTH_NATIVE_CAUSE") + ); + assert!(!format!("{:?}", failures[0]).contains("PRIVATE_OAUTH_NATIVE_CAUSE")); +} diff --git a/backend/crates/providers/openai/tests/credential/token_client.rs b/backend/crates/providers/openai/tests/credential/token_client.rs index fe23fb365..f0581546d 100644 --- a/backend/crates/providers/openai/tests/credential/token_client.rs +++ b/backend/crates/providers/openai/tests/credential/token_client.rs @@ -423,7 +423,10 @@ async fn generic_invalid_grant_should_remain_transient_like_official_codex() { let failure = refresh_failure(400, body).await; let diagnostic = format!("{failure:?} {failure}"); - let RefreshFailure::Transport { message, upstream } = failure else { + let RefreshFailure::Transport { + message, upstream, .. + } = failure + else { panic!("unknown official refresh code must remain transient"); }; assert_eq!(message, None); @@ -445,7 +448,10 @@ async fn unauthorized_should_preserve_upstream_details_and_remain_retryable() { }"#; let failure = refresh_failure(401, body).await; - let RefreshFailure::Transport { message, upstream } = failure else { + let RefreshFailure::Transport { + message, upstream, .. + } = failure + else { panic!("production policy gives every 401 a bounded recovery window"); }; assert_eq!(message.as_deref(), Some("Invalid refresh token.")); @@ -548,6 +554,7 @@ fn assert_transport_failure( let RefreshFailure::Transport { message: actual_message, upstream, + .. } = failure else { panic!("status {status} must classify as transient"); @@ -572,3 +579,47 @@ async fn refresh_failure(status: u16, body: &str) -> RefreshFailure { .await .expect_err("refresh must fail") } + +#[tokio::test] +async fn oauth_failure_retains_unknown_code_and_redacts_token_material() { + let server = MockServer::start().await; + Mock::given(method("POST")).and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(503).set_body_json(serde_json::json!({ + "error": {"code":"FUTURE_OAUTH_FAILURE","message":"request PRIVATE_REFRESH_INPUT failed"}, + "access_token":"PRIVATE_ACCESS_OUTPUT", "nested":[{"id_token":"PRIVATE_ID_OUTPUT"}, {"refresh_token":"PRIVATE_SECOND_OUTPUT"}] + }))).mount(&server).await; + let error = client(&server) + .refresh("PRIVATE_REFRESH_INPUT") + .await + .unwrap_err(); + assert_eq!(error.classification(), "transport-ambiguous"); + let upstream = error.upstream().unwrap(); + assert_eq!(upstream.status(), 503); + assert_eq!(upstream.code(), Some("FUTURE_OAUTH_FAILURE")); + assert!(upstream.redacted()); + assert!(upstream.body().contains("FUTURE_OAUTH_FAILURE")); + assert!(!upstream.body().contains("PRIVATE_")); + assert!(!format!("{error:?} {error}").contains("FUTURE_OAUTH_FAILURE")); +} + +#[tokio::test] +async fn malformed_success_retains_json_source_without_retaining_token_body() { + use std::error::Error as _; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("{\"access_token\":\"PRIVATE_ACCESS\", invalid"), + ) + .mount(&server) + .await; + let error = client(&server).refresh("refresh").await.unwrap_err(); + let source = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(source.column() > 0); + assert!(error.upstream().is_none()); + assert!(!format!("{error:?} {error}").contains("PRIVATE_ACCESS")); +} diff --git a/backend/crates/providers/openai/tests/main.rs b/backend/crates/providers/openai/tests/main.rs index 4d5a35f07..1c5432eb6 100644 --- a/backend/crates/providers/openai/tests/main.rs +++ b/backend/crates/providers/openai/tests/main.rs @@ -7,3 +7,16 @@ mod jitter; mod provider; mod support; mod transport; + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/openai/tests/provider/contract/mod.rs b/backend/crates/providers/openai/tests/provider/contract/mod.rs index 38a3087cd..de7ec2c1a 100644 --- a/backend/crates/providers/openai/tests/provider/contract/mod.rs +++ b/backend/crates/providers/openai/tests/provider/contract/mod.rs @@ -3846,10 +3846,11 @@ async fn repeated_message_too_big_closes_keep_session_on_websocket() { assert_eq!(detail.message(), "message too big"); assert_eq!(detail.code(), Some("message_too_big")); assert_eq!(detail.error_type(), Some("invalid_request_error")); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1009") - ); + assert!(error.upstream_code().is_none()); + let close: Value = + serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1009); } let second_operation = Operation::Generate(generate_with_persisted_session_context( @@ -4653,10 +4654,10 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation() assert!(saw_websocket_observation); assert_eq!(error.send_state(), UpstreamSendState::Ambiguous); assert_eq!(error.pre_delivery_retry(), None); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1000") - ); + assert!(error.upstream_code().is_none()); + let close: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1000); assert_eq!( error.diagnostic().and_then(|diagnostic| diagnostic.code()), Some("upstream_close") @@ -5956,10 +5957,10 @@ async fn websocket_turn_state_metadata_close_does_not_authorize_replay() { assert_eq!(error.send_state(), UpstreamSendState::Ambiguous); assert!(!error.replay_is_safe()); assert_eq!(error.pre_delivery_retry(), None); - assert_eq!( - error.upstream_code().map(|code| code.as_str()), - Some("websocket_close_1000") - ); + assert!(error.upstream_code().is_none()); + let close: Value = serde_json::from_str(error.raw_upstream_error().unwrap().as_str()).unwrap(); + assert_eq!(close["type"], "websocket.close"); + assert_eq!(close["code"], 1000); assert_eq!( error.diagnostic().map(|diagnostic| diagnostic.as_str()), Some( diff --git a/backend/crates/providers/xai/src/admin.rs b/backend/crates/providers/xai/src/admin.rs index 39415b468..448d3bc66 100644 --- a/backend/crates/providers/xai/src/admin.rs +++ b/backend/crates/providers/xai/src/admin.rs @@ -1491,21 +1491,23 @@ fn map_store_error(error: gateway_core::error::StoreError) -> ProviderAdminError StoreErrorKind::Unavailable => ProviderAdminErrorKind::Unavailable, _ => ProviderAdminErrorKind::Internal, }) + .with_source(error) } fn map_repository_error(error: GrokCredentialRepositoryError) -> ProviderAdminError { use GrokCredentialRepositoryError as Error; - provider_error(match error { + provider_error(match &error { Error::InvalidInput(_) | Error::WrongProviderKind | Error::IdentityRebind - | Error::InvalidCredentialData => ProviderAdminErrorKind::Invalid, + | Error::InvalidCredentialData(_) => ProviderAdminErrorKind::Invalid, Error::CredentialNotFound => ProviderAdminErrorKind::NotFound, - Error::StaleCredentialRevision | Error::Conflict | Error::RevisionOverflow => { + Error::StaleCredentialRevision(_) | Error::Conflict(_) | Error::RevisionOverflow => { ProviderAdminErrorKind::Conflict } - Error::Store => ProviderAdminErrorKind::Unavailable, + Error::Store(_) => ProviderAdminErrorKind::Unavailable, }) + .with_source(error) } fn map_refresh_error(error: GrokCredentialRefreshError) -> ProviderAdminError { diff --git a/backend/crates/providers/xai/src/credential/catalog.rs b/backend/crates/providers/xai/src/credential/catalog.rs index 92bd13cc8..35dfb578f 100644 --- a/backend/crates/providers/xai/src/credential/catalog.rs +++ b/backend/crates/providers/xai/src/credential/catalog.rs @@ -1394,13 +1394,13 @@ fn map_quota_repository_error( RepositoryError::CredentialNotFound | RepositoryError::WrongProviderKind => { GrokQuotaError::AccountUnavailable } - RepositoryError::StaleCredentialRevision | RepositoryError::Conflict => { + RepositoryError::StaleCredentialRevision(_) | RepositoryError::Conflict(_) => { GrokQuotaError::StaleCredentialSnapshot } - RepositoryError::Store => GrokQuotaError::Store, + RepositoryError::Store(_) => GrokQuotaError::Store, RepositoryError::InvalidInput(_) | RepositoryError::IdentityRebind - | RepositoryError::InvalidCredentialData + | RepositoryError::InvalidCredentialData(_) | RepositoryError::RevisionOverflow => GrokQuotaError::InvalidData, } } diff --git a/backend/crates/providers/xai/src/credential/refresh.rs b/backend/crates/providers/xai/src/credential/refresh.rs index 9f84ef1fd..fde6a45ee 100644 --- a/backend/crates/providers/xai/src/credential/refresh.rs +++ b/backend/crates/providers/xai/src/credential/refresh.rs @@ -401,7 +401,7 @@ impl GrokCredentialRefreshService { let subject = loaded .account .upstream_user_id() - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_owned(); let credential = DueGrokCredential { account_id: account_id.clone(), @@ -487,7 +487,7 @@ impl GrokCredentialRefreshService { let subject = loaded .account .upstream_user_id() - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_owned(); let prepared = GrokCredentialAdmin .prepare_rotation(&RotateManagedGrokCredential { @@ -918,10 +918,13 @@ impl GrokCredentialRepository { .cloned() .collect::>(); let now = SystemTime::now(); - let provider = ProviderKind::new(PROVIDER_NAME) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(PROVIDER_NAME).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let limit = NonZeroU32::new(MAX_REFRESH_BATCH) - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)?; + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))?; let query = ProviderRefreshQuery::new( provider, now.checked_add(policy.margin()).unwrap_or(now), @@ -1015,7 +1018,7 @@ fn stale_repository_error(error: &GrokCredentialRepositoryError) -> bool { matches!( error, GrokCredentialRepositoryError::CredentialNotFound - | GrokCredentialRepositoryError::Conflict - | GrokCredentialRepositoryError::StaleCredentialRevision + | GrokCredentialRepositoryError::Conflict(_) + | GrokCredentialRepositoryError::StaleCredentialRevision(_) ) } diff --git a/backend/crates/providers/xai/src/credential/repository.rs b/backend/crates/providers/xai/src/credential/repository.rs index 9a82aa20f..3e4446961 100644 --- a/backend/crates/providers/xai/src/credential/repository.rs +++ b/backend/crates/providers/xai/src/credential/repository.rs @@ -256,7 +256,7 @@ impl GrokCredentialAdmin { .account .access_token_expires_at() .map(DateTime::::from) - .ok_or(GrokCredentialRepositoryError::InvalidCredentialData)? + .ok_or(GrokCredentialRepositoryError::InvalidCredentialData(None))? .to_rfc3339(), ), ); @@ -398,7 +398,7 @@ impl GrokCredentialRepository { .await .map_err(map_store_error)?; let CredentialCasOutcome::Updated(revision) = outcome else { - return Err(GrokCredentialRepositoryError::StaleCredentialRevision); + return Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)); }; Ok(GrokCredentialRecord { @@ -435,7 +435,11 @@ impl GrokCredentialRepository { current.account.access_token_expires_at(), Some(next_refresh_at), ) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; match self .store .compare_and_swap_credential(update) @@ -444,7 +448,7 @@ impl GrokCredentialRepository { { CredentialCasOutcome::Updated(_) => Ok(()), CredentialCasOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -501,8 +505,11 @@ impl GrokCredentialRepository { pub(crate) async fn list_loaded_for_provider( &self, ) -> Result, GrokCredentialRepositoryError> { - let provider = ProviderKind::new(XAI_PROVIDER_KIND) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(XAI_PROVIDER_KIND).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let accounts = self .store .list_for_provider(&provider) @@ -520,8 +527,11 @@ impl GrokCredentialRepository { pub(crate) async fn list_accounts_for_provider( &self, ) -> Result, GrokCredentialRepositoryError> { - let provider = ProviderKind::new(XAI_PROVIDER_KIND) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let provider = ProviderKind::new(XAI_PROVIDER_KIND).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let accounts = self .store .list_for_provider(&provider) @@ -568,7 +578,7 @@ impl GrokCredentialRepository { match outcome { QuotaWriteOutcome::Updated => Ok(()), QuotaWriteOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -591,7 +601,7 @@ impl GrokCredentialRepository { match outcome { QuotaWriteOutcome::Updated => Ok(()), QuotaWriteOutcome::Conflict => { - Err(GrokCredentialRepositoryError::StaleCredentialRevision) + Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)) } } } @@ -673,12 +683,20 @@ fn encode_secret( scope: &secret.scope, refresh_token_expires_at: account.refresh_token_expires_at, }) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let Value::Object(object) = value else { - return Err(GrokCredentialRepositoryError::InvalidCredentialData); + return Err(GrokCredentialRepositoryError::InvalidCredentialData(None)); }; if serde_json::to_vec(&object) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)? + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .len() > MAX_SECRET_BYTES { @@ -698,7 +716,7 @@ fn prepare_rotation( validate_secret(secret)?; ensure_xai(current)?; if current.revision() != expected_revision { - return Err(GrokCredentialRepositoryError::StaleCredentialRevision); + return Err(GrokCredentialRepositoryError::StaleCredentialRevision(None)); } if current.upstream_user_id() != Some(verified_account.subject.as_str()) || current.upstream_account_id() != verified_account.upstream_account_id.as_deref() @@ -730,7 +748,11 @@ fn prepare_rotation( Some(to_system_time(verified_account.access_token_expires_at)), None, ) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + .map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; Ok(PreparedGrokCredentialRotation::new(profile, credential)) } @@ -738,15 +760,18 @@ fn decode_secret( credential: &PlaintextCredential, ) -> Result { let value = Value::Object(credential.expose_to_provider().clone()); - let wire: OwnedGrokOAuthSecretWire = serde_json::from_value(value) - .map_err(|_| GrokCredentialRepositoryError::InvalidCredentialData)?; + let wire: OwnedGrokOAuthSecretWire = serde_json::from_value(value).map_err(|source| { + GrokCredentialRepositoryError::InvalidCredentialData(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if wire.schema_version != CREDENTIAL_SCHEMA_VERSION || wire.auth_method != OAUTH_AUTH_METHOD || wire.access_token.is_empty() || wire.refresh_token.is_empty() || validate_scope(&wire.scope).is_err() { - return Err(GrokCredentialRepositoryError::InvalidCredentialData); + return Err(GrokCredentialRepositoryError::InvalidCredentialData(None)); } Ok(wire) } @@ -861,13 +886,15 @@ fn ensure_xai(account: &ProviderAccount) -> Result<(), GrokCredentialRepositoryE } fn map_store_error(error: gateway_core::error::StoreError) -> GrokCredentialRepositoryError { + use gateway_core::error::ErrorSource; match error.kind() { - StoreErrorKind::Conflict => GrokCredentialRepositoryError::Conflict, - StoreErrorKind::InvalidData | StoreErrorKind::InvalidState => { - GrokCredentialRepositoryError::InvalidCredentialData + StoreErrorKind::Conflict => { + GrokCredentialRepositoryError::Conflict(Some(ErrorSource::new(error))) } - StoreErrorKind::Unavailable => GrokCredentialRepositoryError::Store, - _ => GrokCredentialRepositoryError::Store, + StoreErrorKind::InvalidData | StoreErrorKind::InvalidState => { + GrokCredentialRepositoryError::InvalidCredentialData(Some(ErrorSource::new(error))) + } + _ => GrokCredentialRepositoryError::Store(error), } } @@ -875,7 +902,9 @@ fn map_rotation_load_error( error: gateway_core::error::StoreError, ) -> GrokCredentialRepositoryError { if error.kind() == StoreErrorKind::Conflict { - GrokCredentialRepositoryError::StaleCredentialRevision + GrokCredentialRepositoryError::StaleCredentialRevision(Some( + gateway_core::error::ErrorSource::new(error), + )) } else { map_store_error(error) } @@ -886,7 +915,7 @@ fn to_system_time(value: DateTime) -> SystemTime { } /// xAI Provider account adapter 的脱敏错误 -#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[derive(Debug, Error, Clone)] pub enum GrokCredentialRepositoryError { #[error("invalid xAI credential input: {0}")] InvalidInput(&'static str), @@ -895,15 +924,15 @@ pub enum GrokCredentialRepositoryError { #[error("provider account belongs to another Provider")] WrongProviderKind, #[error("credential revision is stale")] - StaleCredentialRevision, + StaleCredentialRevision(#[source] Option), #[error("verified xAI identity cannot be rebound")] IdentityRebind, #[error("xAI credential data is invalid")] - InvalidCredentialData, + InvalidCredentialData(#[source] Option), #[error("xAI credential mutation conflicts with current state")] - Conflict, + Conflict(#[source] Option), #[error("credential revision overflow")] RevisionOverflow, #[error("provider account store is unavailable")] - Store, + Store(#[source] gateway_core::error::StoreError), } diff --git a/backend/crates/providers/xai/src/credential/selector.rs b/backend/crates/providers/xai/src/credential/selector.rs index 375eaf7db..f089575a0 100644 --- a/backend/crates/providers/xai/src/credential/selector.rs +++ b/backend/crates/providers/xai/src/credential/selector.rs @@ -103,19 +103,27 @@ impl GrokAccountSessionSelector { let diagnostic = request.eligibility() == AccountEligibilityPolicy::BypassForDiagnostic; // store 侧常规调度列表不包含停用账号；管理端诊断要对固定账号执行真实上游 // 验证，这里把不在列表里的 required 账号显式补回候选 - let mut accounts = self - .repository - .list_accounts_for_provider() - .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + let mut accounts = + self.repository + .list_accounts_for_provider() + .await + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if diagnostic && let Some(required) = request.required_account() && !accounts.iter().any(|account| account.id() == required) - && let Some(account) = self - .repository - .account_by_id(required) - .await - .map_err(|_| GrokSessionSelectorError::Unavailable)? + && let Some(account) = + self.repository + .account_by_id(required) + .await + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? && account.provider() == &self.provider_kind { accounts.push(account); @@ -184,7 +192,11 @@ impl GrokAccountSessionSelector { &account_ids, ) .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let runtime_cooldowns = if diagnostic { std::collections::BTreeMap::new() } else { @@ -202,7 +214,7 @@ impl GrokAccountSessionSelector { .signals() .get(account.id()) .cloned() - .ok_or(GrokSessionSelectorError::Unavailable)? + .ok_or(GrokSessionSelectorError::Unavailable(None))? .with_provider_quota(self.quota.scheduling_signals(&account)) .with_runtime_health(health.0, health.1); Ok(AccountCandidate { account, signals }) @@ -293,7 +305,11 @@ impl GrokAccountSessionSelector { .with_cancellation(request.cancellation().clone()), )) .await - .map_err(|_| GrokSessionSelectorError::Unavailable)?; + .map_err(|source| { + GrokSessionSelectorError::Unavailable(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; let guard = match lease { ProviderLeaseAcquisition::Acquired(guard) => guard, ProviderLeaseAcquisition::Busy { @@ -310,21 +326,29 @@ impl GrokAccountSessionSelector { .repository .load(&selected_id, selected_revision) .await - .map_err(|_| GrokSessionSelectorError::InvalidSession)?; + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })?; if !diagnostic && loaded .refresh_token_expires_at .is_some_and(|expires_at| expires_at <= Utc::now()) { - return Err(GrokSessionSelectorError::InvalidSession); + return Err(GrokSessionSelectorError::InvalidSession(None)); } let binding = GrokSessionBinding::new(selected_id.as_str()) - .map_err(|_| GrokSessionSelectorError::InvalidSession)? + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .with_outbound_proxy(loaded.account.outbound_proxy().cloned()); let upstream_user_id = loaded .account .upstream_user_id() - .ok_or(GrokSessionSelectorError::InvalidSession)?; + .ok_or(GrokSessionSelectorError::InvalidSession(None))?; let session = SelectedGrokSession::new( selected_id, selected_revision, @@ -337,7 +361,11 @@ impl GrokAccountSessionSelector { binding, guard, ) - .map_err(|_| GrokSessionSelectorError::InvalidSession)? + .map_err(|source| { + GrokSessionSelectorError::InvalidSession(Some( + gateway_core::error::ErrorSource::new(source), + )) + })? .with_capacity_snapshot( capacity.map(AccountCapacitySnapshot::with_acquired_request), ); @@ -498,8 +526,8 @@ impl GrokAccountSessionSelector { { Ok(()) => return, Err( - GrokCredentialRepositoryError::Conflict - | GrokCredentialRepositoryError::StaleCredentialRevision, + GrokCredentialRepositoryError::Conflict(_) + | GrokCredentialRepositoryError::StaleCredentialRevision(_), ) => {} Err(error) => { tracing::warn!( diff --git a/backend/crates/providers/xai/src/provider/failure.rs b/backend/crates/providers/xai/src/provider/failure.rs index b6a9cf707..8a038c0c0 100644 --- a/backend/crates/providers/xai/src/provider/failure.rs +++ b/backend/crates/providers/xai/src/provider/failure.rs @@ -398,22 +398,31 @@ pub(super) fn map_request_error(error: GrokRequestEncodeError) -> ProviderError /// 将选择阶段失败映射为带结构化 code 与 retry_after 的 Provider 错误 pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderError { + let mapped = selection_failure(&error); + if std::error::Error::source(&error).is_some() { + mapped.with_source(error) + } else { + mapped + } +} + +fn selection_failure(error: &GrokSessionSelectorError) -> ProviderError { let (retry_after, message, code) = match error { GrokSessionSelectorError::QueueRejected(error) => { return provider_error(error.provider_kind(), UpstreamSendState::NotSent); } GrokSessionSelectorError::AccountCoolingDown { retry_after } => ( - retry_after, - cooling_down_message(retry_after), + *retry_after, + cooling_down_message(*retry_after), "account_cooling_down", ), GrokSessionSelectorError::ModelCoolingDown { retry_after } => ( - retry_after, - model_cooling_down_message(retry_after), + *retry_after, + model_cooling_down_message(*retry_after), "model_cooling_down", ), GrokSessionSelectorError::CapacityUnavailable { retry_after } => ( - retry_after, + *retry_after, "account is at its concurrency or request-interval limit".to_owned(), "account_capacity_busy", ), @@ -422,7 +431,7 @@ pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderEr "no account is eligible for the requested model".to_owned(), "no_eligible_account", ), - GrokSessionSelectorError::Unavailable => ( + GrokSessionSelectorError::Unavailable(_) => ( None, "account scheduling state is temporarily unreadable".to_owned(), "account_selector_unavailable", @@ -436,7 +445,7 @@ pub(super) fn map_selection_error(error: GrokSessionSelectorError) -> ProviderEr GrokSessionSelectorError::PolicyUnavailable => { return provider_error(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent); } - GrokSessionSelectorError::InvalidSession => { + GrokSessionSelectorError::InvalidSession(_) => { return provider_error(ProviderErrorKind::Protocol, UpstreamSendState::NotSent); } }; diff --git a/backend/crates/providers/xai/src/provider/workers.rs b/backend/crates/providers/xai/src/provider/workers.rs index 48233005e..0ea175b4d 100644 --- a/backend/crates/providers/xai/src/provider/workers.rs +++ b/backend/crates/providers/xai/src/provider/workers.rs @@ -107,8 +107,7 @@ impl ScheduledTask for XaiOAuthRefreshTask { return Ok(()); } let outcomes = self.service.refresh_due().await.map_err(|error| { - tracing::error!(error = %error, "xAI OAuth refresh cycle failed"); - WorkerTaskError::safe("xAI OAuth refresh failed") + WorkerTaskError::safe("xAI OAuth refresh failed").with_source(error) })?; let failures = outcomes .iter() @@ -151,7 +150,9 @@ impl ScheduledTask for XaiQuotaCatalogTask { .accounts .list_for_provider(&self.provider_kind) .await - .map_err(|_| WorkerTaskError::safe("xAI Provider accounts unavailable"))?; + .map_err(|source| { + WorkerTaskError::safe("xAI Provider accounts unavailable").with_source(source) + })?; let mut failures = 0_u64; let now = SystemTime::now(); let accounts = self.reserve_periodic_refreshes(accounts, now); diff --git a/backend/crates/providers/xai/src/transport/session.rs b/backend/crates/providers/xai/src/transport/session.rs index 4859a9cf9..90ce8bd93 100644 --- a/backend/crates/providers/xai/src/transport/session.rs +++ b/backend/crates/providers/xai/src/transport/session.rs @@ -468,7 +468,7 @@ pub trait GrokSessionSelector: Send + Sync { } /// 不含密钥的选择器失败 -#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[derive(Debug, thiserror::Error)] pub enum GrokSessionSelectorError { #[error(transparent)] QueueRejected(#[from] gateway_core::concurrency::QueueRejection), @@ -495,10 +495,10 @@ pub enum GrokSessionSelectorError { }, /// 会话元数据或 Provider 持有的明文密钥非法 #[error("Grok Build session data is invalid")] - InvalidSession, + InvalidSession(#[source] Option), /// 选择器依赖的后端服务不可用 #[error("Grok Build session selector is unavailable")] - Unavailable, + Unavailable(#[source] Option), /// 插件调度策略明确拒绝本次请求 #[error("account scheduling policy rejected the request")] PolicyRejected, diff --git a/backend/crates/providers/xai/tests/admin.rs b/backend/crates/providers/xai/tests/admin.rs index de667cb1b..c0620025a 100644 --- a/backend/crates/providers/xai/tests/admin.rs +++ b/backend/crates/providers/xai/tests/admin.rs @@ -87,6 +87,7 @@ async fn account_unavailable_clears_real_selector_cooldowns_only_for_deleted_acc cooldowns.clone(), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ); let repository = GrokCredentialRepository::new(store.clone()); let selector = GrokAccountSessionSelector::new( @@ -191,7 +192,9 @@ async fn xai_quota_catalog_worker_treats_empty_account_pool_as_idle() { .await .expect("xAI bundle"); - assert_eq!(run_quota_catalog_cycle(&mut bundle).await, Ok(())); + run_quota_catalog_cycle(&mut bundle) + .await + .expect("idle quota cycle"); } #[tokio::test] @@ -650,6 +653,7 @@ fn provider_ports_with_catalog( Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), pending, + Arc::new(RecordingDiagnostics::default()), ) } @@ -1174,6 +1178,7 @@ mod errors { Arc::new(TestCooldown), Arc::new(TestRuntimePolicy), Arc::new(TestOAuthPending::default()), + Arc::new(RecordingDiagnostics::default()), ); let bundle = provider_xai::initialize(ports).await.unwrap(); let error = bundle @@ -1285,3 +1290,16 @@ async fn client_profile_preview_and_dashboard_use_saved_configuration() { assert!(dashboard.release.is_none()); assert!(dashboard.verified_at.is_none()); } + +#[derive(Default)] +struct RecordingDiagnostics(std::sync::Mutex>); +#[async_trait::async_trait] +impl gateway_core::diagnostics::OperationalDiagnostics for RecordingDiagnostics { + async fn record_failure( + &self, + failure: gateway_core::diagnostics::OperationalFailure, + ) -> Result<(), gateway_core::error::StoreError> { + self.0.lock().unwrap().push(failure); + Ok(()) + } +} diff --git a/backend/crates/providers/xai/tests/credential/contract.rs b/backend/crates/providers/xai/tests/credential/contract.rs index 179629c61..916bd545a 100644 --- a/backend/crates/providers/xai/tests/credential/contract.rs +++ b/backend/crates/providers/xai/tests/credential/contract.rs @@ -317,12 +317,12 @@ async fn invalid_token_lifetime_is_rejected_before_store_write() { let (store, _) = repository(); let mut input = create_input("lifetime", "subject-lifetime"); input.account.refresh_token_expires_at = Some(input.account.access_token_expires_at); - assert_eq!( + assert!(matches!( GrokCredentialAdmin.prepare_import(&input), Err(GrokCredentialRepositoryError::InvalidInput( "token_lifetime" )) - ); + )); assert_eq!(store.len(), 0); } diff --git a/backend/crates/providers/xai/tests/credential/repository.rs b/backend/crates/providers/xai/tests/credential/repository.rs index c1119c108..ecf740dd3 100644 --- a/backend/crates/providers/xai/tests/credential/repository.rs +++ b/backend/crates/providers/xai/tests/credential/repository.rs @@ -12,10 +12,10 @@ fn repository_rejects_identity_that_cannot_be_sent_as_official_header() { let mut input = create_input("invalid", "subject"); input.account.subject = "subject-with-非-ascii".to_owned(); - assert_eq!( + assert!(matches!( GrokCredentialAdmin.prepare_import(&input), Err(GrokCredentialRepositoryError::InvalidInput("subject")) - ); + )); } #[tokio::test] diff --git a/backend/crates/providers/xai/tests/provider/contract.rs b/backend/crates/providers/xai/tests/provider/contract.rs index ce002f5ad..90fe1b11e 100644 --- a/backend/crates/providers/xai/tests/provider/contract.rs +++ b/backend/crates/providers/xai/tests/provider/contract.rs @@ -888,7 +888,7 @@ impl GrokSessionSelector for StubSelector { GrokSessionBinding::new("acct_provider").expect("binding"), (), ) - .map_err(|_| GrokSessionSelectorError::InvalidSession) + .map_err(|_| GrokSessionSelectorError::InvalidSession(None)) }) } @@ -938,7 +938,7 @@ impl GrokSessionSelector for SequencedAccountSelector { GrokSessionBinding::new("acct_provider").expect("binding"), (), ) - .map_err(|_| GrokSessionSelectorError::InvalidSession) + .map_err(|_| GrokSessionSelectorError::InvalidSession(None)) }) } @@ -3140,7 +3140,7 @@ async fn selection_failures_carry_a_distinguishable_client_error_code() { "no_eligible_account", ), ( - GrokSessionSelectorError::Unavailable, + GrokSessionSelectorError::Unavailable(None), "account_selector_unavailable", ), ]; @@ -3174,6 +3174,57 @@ async fn selection_failures_carry_a_distinguishable_client_error_code() { } } +#[tokio::test] +async fn selection_failure_preserves_native_cause_without_exposing_it_to_clients() { + use gateway_core::error::{ErrorSource, StoreError, StoreErrorKind}; + let source = StoreError::caused_by( + StoreErrorKind::Unavailable, + std::io::Error::other("PRIVATE_XAI_DATABASE_CAUSE"), + ); + let transport = StubInferenceTransport::success(); + let provider = provider( + StubSelector::failing(GrokSessionSelectorError::Unavailable(Some( + ErrorSource::new(source), + ))), + transport.clone(), + ) + .await; + let error = match provider + .execute( + provider_request("xai"), + context(CancellationToken::new(), None), + ) + .await + { + Ok(_) => panic!("selection must fail"), + Err(error) => error, + }; + assert_eq!(error.kind(), ProviderErrorKind::Unavailable); + assert_eq!(error.send_state(), UpstreamSendState::NotSent); + assert!( + error + .error_details() + .expect("protected details") + .contains("PRIVATE_XAI_DATABASE_CAUSE") + ); + let mut cause = std::error::Error::source(&error); + let mut native = false; + while let Some(error) = cause { + native |= error.downcast_ref::().is_some(); + cause = error.source(); + } + assert!(native); + assert!(!format!("{error:?} {error}").contains("PRIVATE_XAI_DATABASE_CAUSE")); + assert!( + !error + .client_visible_upstream_error() + .expect("safe detail") + .message() + .contains("PRIVATE_XAI_DATABASE_CAUSE") + ); + assert_eq!(transport.calls.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn native_previous_response_is_rejected_before_selection() { let selector = StubSelector::success(); diff --git a/deploy/README.md b/deploy/README.md index 4a2a220d3..b237db3f5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -439,18 +439,22 @@ Provider schema 以明文 JSON 保存于 PostgreSQL。Redis 只保存可重建 关闭的分片压缩为 `.log.gz`；成功压缩、同步并发布归档后才删除原文件，保留原修改时间。 清理发生在启动和轮转时；空闲期间过期文件可能暂时多保留。检索时须同时读取 `.log` 与 `.log.gz` -报文开关为 `host.logging.request_dump`，默认关闭；开启时原始报文按块完整写入独立文件， +报文开关为 `host.logging.request_dump`，默认关闭；开启时原始报文按块写入独立文件， +缓冲拒收或写入失败可能造成缺块，需结合 `file_logging` 健康状态判断完整性。 数据库请求诊断仍是有界摘要，不能用摘要事件数代替全量报文完整性 OAuth 恢复开关为 `host.logging.oauth_recovery`，默认关闭，与普通文件日志开关分别控制： - 开启后，OpenAI 每次成功取得 AT 和可用 RT 时，在资料补全及数据库写入前写入独立的 `oauth-recovery` 文件集 -- 该通道不受普通日志级别过滤，关闭时也不会改写到普通日志或 stdout；有界队列满时等待写入 +- 该通道不受普通日志级别过滤，关闭时也不会改写到普通日志或 stdout；接收端不等待后台写入 恢复记录和请求转储含原始凭据，`.runtime/logs` 须按敏感数据控制访问和备份 -文件写入使用有界队列背压，正常退出会排空队列并同步文件。`file_logging` 健康探针在写入/同步失败后 -报告 `Unhealthy`，本次进程内恢复写入也不会清除已有缺口；压缩或清理失败报告 `Degraded`。 +每个文件输出使用非阻塞队列，普通额度为 4096 条 / 64 MiB，WARN / ERROR 另有 512 条 / 8 MiB 预留， +额度包含正在写入的记录。后台批量缓冲写文件，正常退出会排空已接收的队列并同步。 +持续拥堵、超大记录或输出关闭导致的拒收会损失整条记录，所有接口均不会为日志等待； +`file_logging` 健康探针在拒收或写入/同步失败后报告 `Unhealthy`， +本次进程内恢复写入也不会清除已有缺口；压缩或清理失败报告 `Degraded`。 断电、强杀或磁盘故障可能造成日志丢失；需要恢复已删除文件时，应使用独立备份。 容量不足时不会提前删除保留窗口内日志，必须根据完整日期的压缩后实际用量规划空间，并监控磁盘余量和 健康探针。Docker stdout 的独立轮转不承担应用文件日志的完整保留承诺 diff --git a/docs/api.md b/docs/api.md index 6f0cce983..9a7bf0b61 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1704,7 +1704,8 @@ Key 已删除或未关联时为 `null`，不影响记录返回，不包含密钥 运维错误记录的 `errorDetails` 为受控诊断文本，无详情时为 `null`。 错误来源快照以 JSON 文本保存：`causes.messages` 按外层到内层排列本地原因，`causes.truncated` -标记来源链截断，`upstream` 保存上游正文原文，`redacted` 标记敏感上下文移除；没有对应来源时为 `null`。 +标记来源链截断；存在附属清理失败时，`causes.cleanup` 保存各自的原因快照。 +`upstream` 保存上游正文，`redacted` 标记令牌等敏感上下文的移除；没有对应来源时为 `null`。 稳定错误分类、原始上游 code 和该详情分别保留，不以公共错误文案代替原始原因。 详情不出现在 Key 用量接口、普通错误信封和诊断包导出中；记录仍受异步观测写入与保留周期约束 diff --git a/docs/architecture.md b/docs/architecture.md index 3c23e5eb4..821fdf384 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -509,13 +509,22 @@ client，OIDC 的 JWKS 缓存与单飞归属对应出口状态。自动刷新提 普通格式化不展开来源。Store 统一基础设施不可用的转换，Provider 拥有业务分类， 插件 Runtime 集中投影 RPC、fault 与上游适配事件，调用点只附加原始来源与执行上下文 -ProviderError 统一生成受控详情，失败 attempt、探测和最终请求复用该出口。 -原始上游正文与本地原因链分开承载，来源链最多保留 32 层、64 KiB 文本，截断明确标记； +Core 的 `ErrorDetails` 统一投影受控详情，Provider、管理请求和后台操作复用该格式。 +原始上游正文与本地原因链分开承载，主原因与附属清理失败合计最多保留 32 个节点、64 KiB 文本， +截断明确标记；显式回滚仍等待资源释放，回滚失败附加到主失败，不替换其分类与原因。 +本地续写拒绝和 WS Close 派生分类不写入原始上游 code；关闭码与原因保存在关闭事实中。 + 请求 URL 等请求方向的敏感上下文在附加来源前移除并标记。错误快照保留来源与错误详情， 协议透传响应和未交付事件仍由当前请求独占。详情计入观测队列的内存预算； 队列满、关闭超时或写入失败仍可能丢失记录，沿用丢弃与写入失败指标，不自动重放不保证幂等的写入 -推理日志由作出处置的边界记录恢复、放弃或终态；仅传播错误的层不重复打印。 +管理请求在 API 出口按 request ID 记录一次内部失败，后台任务由 Host 监督器记录； +OAuth 刷新和准入释放等内部处理后继续运行的失败，由消费失败结果的 owner 记录。 +这些操作通过 `OperationalDiagnostics` 复用现有有界观测队列与 `ops_events`， +不创建模型请求，也不把基础设施失败冒充成 Provider 上游响应。 +OAuth 响应中的令牌字段和回显的刷新令牌在进入受控详情前脱敏，普通日志不展开正文。 + +日志由作出处置的边界记录恢复、放弃或终态；仅传播错误的层不重复打印。 传输收包使用 debug 摘要，普通日志使用稳定阶段、分类与 request/attempt 关联，不展开原始正文、 任意上游 code 或关闭原因。受控详情无法写入时不能回退到普通日志打印原文 @@ -805,8 +814,10 @@ OpenAI 订阅周期属于按需个人信息，不是额度事实。Admin 账号 ### 请求记录与交付 -请求观测通过有界进程内队列异步投影到 PostgreSQL。队列满、Store 暂不可用或进程退出超时时，允许丢失 -可恢复观测并累计指标，但不允许改变客户端响应。Usage 详情中的 attempt 因此是 best-effort，并通过 +请求观测通过有界进程内队列异步投影到 PostgreSQL。普通额度之外预留四分之一容量给失败与请求生命周期记录， +进度更新不能消耗预留；同一请求仍按入队顺序写入。容量耗尽、Store 暂不可用或进程退出超时时， +观测记录可能丢失并累计指标，拥堵与写入失败告警按累计计数限频，不改变客户端响应。 +Usage 详情中的 attempt 因此是 best-effort，并通过 `attemptsComplete: false` 明示不完整性 `model_requests.deadline_at` 是异常回收租约的到期时间，运行会话持续刷新，且不超过显式执行截止。 @@ -887,8 +898,11 @@ worker 复用连接测试探针执行真实上游调用，并按冻结代次处 与普通文件日志开关分别控制，不输出到普通日志或 stdout。`.runtime/logs` 同样属于敏感数据 - `host.logging.request_dump` 默认关闭；开启后独立请求转储包含原始请求头和正文， 可能包含密钥及用户内容，只能在明确的排障范围内使用，不能作为普通日志公开 -- Host 拥有文件日志写入、轮转、保留和关闭流程，以时间窗口完整性为约束，队列满时背压，正常关闭时排空并同步。 - 写入缺口必须通过健康状态持续报告，压缩或清理失败不能伪装成保留完成。 +- 所有接口的日志与诊断观测接收均不得等待队列空位或输出 I/O，请求侧只承担有界采集与入队开销 +- Host 拥有文件日志写入、轮转、保留和关闭流程。文件日志按条数与字节限制内存，WARN / ERROR + 另有预留额度；单个输出使用 FIFO 和后台批量缓冲，正常关闭时排空并同步。 + 容量耗尽或输出关闭的记录不回退到请求线程同步输出，缺口通过健康状态持续报告； + 压缩或清理失败不能伪装成保留完成。 配置、保留窗口和容量规划见 [日志与持久化](../deploy/README.md#持久化与备份) - Provider 将安全诊断的阶段、原因码与消息独立于错误大类和发送状态传入 Core； `attempt.failed` 保存这些字段，最终错误记录优先持久化诊断消息。重试包装不能覆盖底层诊断，