diff --git a/backend/crates/gateway-core/src/engine/execution.rs b/backend/crates/gateway-core/src/engine/execution.rs index 23183cbd9..f9fd749de 100644 --- a/backend/crates/gateway-core/src/engine/execution.rs +++ b/backend/crates/gateway-core/src/engine/execution.rs @@ -22,7 +22,8 @@ use crate::engine::provider::ProviderRegistry; use crate::engine::{ AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement, CoordinatedEvent, EngineError, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, - ModelRequestId, NewModelRequest, ProviderAttemptOutcome, RecoveryReport, UpstreamSendState, + ModelRequestId, NewModelRequest, ProbeFailure, ProviderAccountId, ProviderAttemptOutcome, + RecoveryReport, UpstreamSendState, }; use crate::error::{GatewayError, GatewayErrorKind, ProviderErrorKind, StoreError}; use crate::event::{GatewayEvent, ProviderEvent, ProviderResponseHeader}; @@ -31,6 +32,7 @@ use crate::policy::{ClientApiKeyId, ClientPolicy}; use crate::routing::snapshot::RuntimeSnapshotHandle; use crate::routing::{ ProviderKind, PublicModelId, PublicModelProfile, RoutingContext, RuntimeSnapshot, + UpstreamModelId, }; const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60); @@ -187,6 +189,8 @@ pub struct DefaultExecutionService { snapshots: RuntimeSnapshotHandle, coordinator: Arc>, probe_coordinator: Arc>, + /// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events。 + observations: Arc, providers: ProviderRegistry, admissions: Arc, circuits: Arc, @@ -205,6 +209,7 @@ impl DefaultExecutionService { continuation: Arc, client_api_key_usage: Arc, ) -> Self { + let observations = Arc::clone(&execution); let engine = GatewayEngine::::new(execution, providers.clone()); let transient: Arc = Arc::new(TransientExecutionStore); let probe_engine = GatewayEngine::::new(transient, providers.clone()); @@ -212,6 +217,7 @@ impl DefaultExecutionService { snapshots, coordinator: Arc::new(AttemptCoordinator::new(engine)), probe_coordinator: Arc::new(AttemptCoordinator::new(probe_engine)), + observations, providers, admissions, circuits, @@ -417,6 +423,11 @@ impl DefaultExecutionService { upstream_model, operation, } = request; + let observed = ProbeObservation { + provider_kind: provider_kind.clone(), + account_id: account_id.clone(), + upstream_model: upstream_model.clone(), + }; let snapshot = self.snapshots.acquire().map_err(|_| { GatewayError::new( GatewayErrorKind::Internal, @@ -464,7 +475,7 @@ impl DefaultExecutionService { started_at, deadline_at, }; - let mut session = self + let mut session = match self .probe_coordinator .start( new_request, @@ -475,18 +486,33 @@ impl DefaultExecutionService { CancellationToken::new(), ) .await - .map_err(|error| gateway_error_from_engine(&error))?; + { + Ok(session) => session, + Err(error) => { + return Err(self + .observe_probe_failure(&observed, started_at, &error) + .await); + } + }; let events = session.collect_uncommitted().await; publish_provider_attempt_outcomes( self.circuits.as_ref(), session.provider_attempt_outcomes(), ) .await; - let events = events.map_err(|error| gateway_error_from_engine(&error))?; - session - .commit_downstream(Some(200)) - .await - .map_err(|error| gateway_error_from_engine(&error))?; + let events = match events { + Ok(events) => events, + Err(error) => { + return Err(self + .observe_probe_failure(&observed, started_at, &error) + .await); + } + }; + if let Err(error) = session.commit_downstream(Some(200)).await { + return Err(self + .observe_probe_failure(&observed, started_at, &error) + .await); + } Ok(AccountProbeResult { text: events .into_iter() @@ -498,6 +524,34 @@ impl DefaultExecutionService { .collect(), }) } + + /// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。 + async fn observe_probe_failure( + &self, + observed: &ProbeObservation, + started_at: SystemTime, + error: &EngineError, + ) -> GatewayError { + if let EngineError::Provider(provider_error) = error { + let _ = self + .observations + .record_probe_failure(ProbeFailure { + provider_kind: observed.provider_kind.clone(), + account_id: observed.account_id.clone(), + upstream_model_id: observed.upstream_model.clone(), + error: provider_error.clone(), + latency: started_at.elapsed().unwrap_or_default(), + }) + .await; + } + gateway_error_from_engine(error) + } +} + +struct ProbeObservation { + provider_kind: ProviderKind, + account_id: ProviderAccountId, + upstream_model: UpstreamModelId, } struct TransientExecutionStore; diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index 9430bfa59..9e679d75a 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -543,6 +543,16 @@ pub struct IntermediateFailure { pub latency: Duration, } +/// 不属于任何 `model_requests` 行的管理端账号探测失败。 +#[derive(Debug)] +pub struct ProbeFailure { + pub provider_kind: ProviderKind, + pub account_id: ProviderAccountId, + pub upstream_model_id: UpstreamModelId, + pub error: ProviderError, + pub latency: Duration, +} + /// `model_requests` 可用的毫秒级阶段耗时。 #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ModelRequestTimings { @@ -629,6 +639,10 @@ pub trait ExecutionStore: Send + Sync { &self, failure: IntermediateFailure, ) -> Result<(), StoreError>; + /// 记录不挂在 `model_requests` 上的账号探测失败;默认丢弃。 + async fn record_probe_failure(&self, _failure: ProbeFailure) -> Result<(), StoreError> { + Ok(()) + } async fn finalize_model_request( &self, finalization: ModelRequestFinalization, diff --git a/backend/crates/gateway-core/tests/engine/execution.rs b/backend/crates/gateway-core/tests/engine/execution.rs index 04cd93cf3..093d8bea8 100644 --- a/backend/crates/gateway-core/tests/engine/execution.rs +++ b/backend/crates/gateway-core/tests/engine/execution.rs @@ -24,12 +24,15 @@ use gateway_core::engine::execution::{ ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit, }; use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest}; -use gateway_core::engine::provider::ProviderRegistry; -use gateway_core::engine::{ - AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId, - NewModelRequest, RecoveryReport, UpstreamSendState, +use gateway_core::engine::provider::{ + Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry, + ProviderRequest, ProviderStream, }; -use gateway_core::error::{GatewayErrorKind, ProviderErrorKind, StoreError}; +use gateway_core::engine::{ + AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, + ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState, +}; +use gateway_core::error::{GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError}; use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload}; use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits}; use gateway_core::routing::snapshot::RuntimeSnapshotHandle; @@ -78,6 +81,64 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() { assert!(!store.touched.load(Ordering::SeqCst)); } +#[test] +fn probe_failures_should_be_observable_without_a_model_request_row() { + let store = Arc::new(TrackingExecutionStore::default()); + let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc]) + .expect("provider registry"); + let service = DefaultExecutionService::new( + RuntimeSnapshotHandle::new(probe_snapshot()), + store.clone(), + providers, + Arc::new(UnusedAdmissions), + Arc::new(UnusedCircuits), + Arc::new(UnusedContinuation), + Arc::new(RecordingClientApiKeyUsage::default()), + ); + + let error = block_on(service.probe(AccountProbeRequest { + account_id: ProviderAccountId::new("acct_probe").expect("account ID"), + provider_kind: ProviderKind::new("openai").expect("provider kind"), + upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"), + operation: probe_operation(), + })) + .expect_err("the provider rejects every probe"); + + assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable); + assert!(!store.touched.load(Ordering::SeqCst)); + assert_eq!(store.probe_failures(), vec!["transport".to_owned()]); +} + +struct FailingProvider; + +#[async_trait] +impl Provider for FailingProvider { + fn name(&self) -> &'static str { + "openai" + } + + fn catalog_generation(&self) -> ProviderCatalogGeneration { + ProviderCatalogGeneration::default() + } + + async fn query_model_capabilities( + &self, + ) -> Result, ProviderError> { + Ok(Vec::new()) + } + + async fn execute( + &self, + _request: ProviderRequest, + _context: AttemptContext, + ) -> Result { + Err(ProviderError::new( + ProviderErrorKind::Transport, + UpstreamSendState::NotSent, + )) + } +} + #[test] fn successful_authentication_should_record_client_key_usage() { let usage = Arc::new(RecordingClientApiKeyUsage::default()); @@ -121,12 +182,20 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage { #[derive(Default)] struct TrackingExecutionStore { touched: AtomicBool, + probe_failures: Mutex>, } impl TrackingExecutionStore { fn touch(&self) { self.touched.store(true, Ordering::SeqCst); } + + fn probe_failures(&self) -> Vec { + self.probe_failures + .lock() + .expect("probe failures lock") + .clone() + } } #[async_trait] @@ -170,6 +239,14 @@ impl ExecutionStore for TrackingExecutionStore { Ok(()) } + async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> { + self.probe_failures + .lock() + .expect("probe failures lock") + .push(failure.error.kind().as_str().to_owned()); + Ok(()) + } + async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> { self.touch(); Ok(()) diff --git a/backend/crates/gateway-store/src/postgres/execution.rs b/backend/crates/gateway-store/src/postgres/execution.rs index 0b25d115a..acca053b2 100644 --- a/backend/crates/gateway-store/src/postgres/execution.rs +++ b/backend/crates/gateway-store/src/postgres/execution.rs @@ -10,7 +10,7 @@ use gateway_core::accounting::{CostSource as CoreCostSource, Usage as CoreUsage} use gateway_core::engine::{ AttemptRecord as CoreAttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId, - NewModelRequest as CoreNewModelRequest, RecoveryReport as CoreRecoveryReport, + NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport, UpstreamSendState as CoreUpstreamSendState, }; use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind}; @@ -819,6 +819,56 @@ impl ExecutionStore for PgExecutionStore { .map_err(core_store_error) } + async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), CoreStoreError> { + let error = failure.error; + let retry_after_ms = error + .retry_after() + .map(|duration| u64::try_from(duration.as_millis())) + .transpose() + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let latency_ms = u64::try_from(failure.latency.as_millis()) + .map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?; + let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned()); + tracing::warn!( + target: "gateway_probe", + provider_kind = failure.provider_kind.as_str(), + account_id = failure.account_id.as_str(), + upstream_model = failure.upstream_model_id.as_str(), + failure_kind = error.kind().as_str(), + send_state = ?error.send_state(), + upstream_status = ?error.upstream_status(), + provider_error_code = ?provider_error_code, + latency_ms, + "账号连接测试失败" + ); + super::OpsEventRepository::append_ops_event( + &super::PgOpsEventRepository::new(self.pool.clone()), + super::OpsEvent { + id: Uuid::now_v7().to_string(), + model_request_id: None, + attempt_index: None, + level: super::OpsEventLevel::Warning, + component: "account_probe".to_owned(), + operation: "connection_test".to_owned(), + provider_kind: Some(failure.provider_kind.as_str().to_owned()), + provider_account_id: Some(failure.account_id.as_str().to_owned()), + provider_account_ref: Some(failure.account_id.as_str().to_owned()), + upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()), + failure_kind: error.kind().as_str().to_owned(), + status_code: error.upstream_status(), + provider_error_code, + retry_after_ms, + upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()), + latency_ms: Some(latency_ms), + message: "account connection test failed".to_owned(), + occurrence_count: 1, + created_at: Utc::now(), + }, + ) + .await + .map_err(core_store_error) + } + async fn finalize_model_request( &self, finalization: CoreModelRequestFinalization,