feat(core,store): record account probe failures as ops events

Connection tests run through a transient execution store, so a failed probe
left no trace at all: nothing in model_requests, nothing in ops_events, and
nothing in the log at info level. The reason a probe failed — upstream status,
provider error code, send state — was only reachable by reproducing it.

Add a ProbeFailure port on ExecutionStore, defaulting to a no-op so existing
implementations are unaffected, and have the execution service report probe
failures to the persistent store while the probe itself keeps using the
transient one. The Postgres store logs a warning and appends an ops event with
no model_request_id.

Such events already fit the surrounding contracts: the ops error list left
joins model_requests, health metrics inner join it so probes stay out of
business failure rates, and retention already targets rows with a null
model_request_id.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
zyycn
2026-07-26 22:44:25 +08:00
co-authored by Claude Opus 5
parent ae46d89de6
commit 237ba96233
4 changed files with 209 additions and 14 deletions
@@ -22,7 +22,8 @@ use crate::engine::provider::ProviderRegistry;
use crate::engine::{
AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement, CoordinatedEvent,
EngineError, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization,
ModelRequestId, NewModelRequest, ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
ModelRequestId, NewModelRequest, ProbeFailure, ProviderAccountId, ProviderAttemptOutcome,
RecoveryReport, UpstreamSendState,
};
use crate::error::{GatewayError, GatewayErrorKind, ProviderErrorKind, StoreError};
use crate::event::{GatewayEvent, ProviderEvent, ProviderResponseHeader};
@@ -31,6 +32,7 @@ use crate::policy::{ClientApiKeyId, ClientPolicy};
use crate::routing::snapshot::RuntimeSnapshotHandle;
use crate::routing::{
ProviderKind, PublicModelId, PublicModelProfile, RoutingContext, RuntimeSnapshot,
UpstreamModelId,
};
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
@@ -187,6 +189,8 @@ pub struct DefaultExecutionService {
snapshots: RuntimeSnapshotHandle,
coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
probe_coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
/// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events。
observations: Arc<dyn ExecutionStore>,
providers: ProviderRegistry,
admissions: Arc<dyn ClientAdmissionPort>,
circuits: Arc<dyn ProviderCircuitPort>,
@@ -205,6 +209,7 @@ impl DefaultExecutionService {
continuation: Arc<dyn NativeContinuationPort>,
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
) -> Self {
let observations = Arc::clone(&execution);
let engine = GatewayEngine::<dyn ExecutionStore>::new(execution, providers.clone());
let transient: Arc<dyn ExecutionStore> = Arc::new(TransientExecutionStore);
let probe_engine = GatewayEngine::<dyn ExecutionStore>::new(transient, providers.clone());
@@ -212,6 +217,7 @@ impl DefaultExecutionService {
snapshots,
coordinator: Arc::new(AttemptCoordinator::new(engine)),
probe_coordinator: Arc::new(AttemptCoordinator::new(probe_engine)),
observations,
providers,
admissions,
circuits,
@@ -417,6 +423,11 @@ impl DefaultExecutionService {
upstream_model,
operation,
} = request;
let observed = ProbeObservation {
provider_kind: provider_kind.clone(),
account_id: account_id.clone(),
upstream_model: upstream_model.clone(),
};
let snapshot = self.snapshots.acquire().map_err(|_| {
GatewayError::new(
GatewayErrorKind::Internal,
@@ -464,7 +475,7 @@ impl DefaultExecutionService {
started_at,
deadline_at,
};
let mut session = self
let mut session = match self
.probe_coordinator
.start(
new_request,
@@ -475,18 +486,33 @@ impl DefaultExecutionService {
CancellationToken::new(),
)
.await
.map_err(|error| gateway_error_from_engine(&error))?;
{
Ok(session) => session,
Err(error) => {
return Err(self
.observe_probe_failure(&observed, started_at, &error)
.await);
}
};
let events = session.collect_uncommitted().await;
publish_provider_attempt_outcomes(
self.circuits.as_ref(),
session.provider_attempt_outcomes(),
)
.await;
let events = events.map_err(|error| gateway_error_from_engine(&error))?;
session
.commit_downstream(Some(200))
.await
.map_err(|error| gateway_error_from_engine(&error))?;
let events = match events {
Ok(events) => events,
Err(error) => {
return Err(self
.observe_probe_failure(&observed, started_at, &error)
.await);
}
};
if let Err(error) = session.commit_downstream(Some(200)).await {
return Err(self
.observe_probe_failure(&observed, started_at, &error)
.await);
}
Ok(AccountProbeResult {
text: events
.into_iter()
@@ -498,6 +524,34 @@ impl DefaultExecutionService {
.collect(),
})
}
/// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。
async fn observe_probe_failure(
&self,
observed: &ProbeObservation,
started_at: SystemTime,
error: &EngineError,
) -> GatewayError {
if let EngineError::Provider(provider_error) = error {
let _ = self
.observations
.record_probe_failure(ProbeFailure {
provider_kind: observed.provider_kind.clone(),
account_id: observed.account_id.clone(),
upstream_model_id: observed.upstream_model.clone(),
error: provider_error.clone(),
latency: started_at.elapsed().unwrap_or_default(),
})
.await;
}
gateway_error_from_engine(error)
}
}
struct ProbeObservation {
provider_kind: ProviderKind,
account_id: ProviderAccountId,
upstream_model: UpstreamModelId,
}
struct TransientExecutionStore;
@@ -543,6 +543,16 @@ pub struct IntermediateFailure {
pub latency: Duration,
}
/// 不属于任何 `model_requests` 行的管理端账号探测失败。
#[derive(Debug)]
pub struct ProbeFailure {
pub provider_kind: ProviderKind,
pub account_id: ProviderAccountId,
pub upstream_model_id: UpstreamModelId,
pub error: ProviderError,
pub latency: Duration,
}
/// `model_requests` 可用的毫秒级阶段耗时。
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ModelRequestTimings {
@@ -629,6 +639,10 @@ pub trait ExecutionStore: Send + Sync {
&self,
failure: IntermediateFailure,
) -> Result<(), StoreError>;
/// 记录不挂在 `model_requests` 上的账号探测失败;默认丢弃。
async fn record_probe_failure(&self, _failure: ProbeFailure) -> Result<(), StoreError> {
Ok(())
}
async fn finalize_model_request(
&self,
finalization: ModelRequestFinalization,
@@ -24,12 +24,15 @@ use gateway_core::engine::execution::{
ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit,
};
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
use gateway_core::engine::provider::ProviderRegistry;
use gateway_core::engine::{
AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
NewModelRequest, RecoveryReport, UpstreamSendState,
use gateway_core::engine::provider::{
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
ProviderRequest, ProviderStream,
};
use gateway_core::error::{GatewayErrorKind, ProviderErrorKind, StoreError};
use gateway_core::engine::{
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
};
use gateway_core::error::{GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
@@ -78,6 +81,64 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() {
assert!(!store.touched.load(Ordering::SeqCst));
}
#[test]
fn probe_failures_should_be_observable_without_a_model_request_row() {
let store = Arc::new(TrackingExecutionStore::default());
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
.expect("provider registry");
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(probe_snapshot()),
store.clone(),
providers,
Arc::new(UnusedAdmissions),
Arc::new(UnusedCircuits),
Arc::new(UnusedContinuation),
Arc::new(RecordingClientApiKeyUsage::default()),
);
let error = block_on(service.probe(AccountProbeRequest {
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
provider_kind: ProviderKind::new("openai").expect("provider kind"),
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
operation: probe_operation(),
}))
.expect_err("the provider rejects every probe");
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
assert!(!store.touched.load(Ordering::SeqCst));
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
}
struct FailingProvider;
#[async_trait]
impl Provider for FailingProvider {
fn name(&self) -> &'static str {
"openai"
}
fn catalog_generation(&self) -> ProviderCatalogGeneration {
ProviderCatalogGeneration::default()
}
async fn query_model_capabilities(
&self,
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
Ok(Vec::new())
}
async fn execute(
&self,
_request: ProviderRequest,
_context: AttemptContext,
) -> Result<ProviderStream, ProviderError> {
Err(ProviderError::new(
ProviderErrorKind::Transport,
UpstreamSendState::NotSent,
))
}
}
#[test]
fn successful_authentication_should_record_client_key_usage() {
let usage = Arc::new(RecordingClientApiKeyUsage::default());
@@ -121,12 +182,20 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
#[derive(Default)]
struct TrackingExecutionStore {
touched: AtomicBool,
probe_failures: Mutex<Vec<String>>,
}
impl TrackingExecutionStore {
fn touch(&self) {
self.touched.store(true, Ordering::SeqCst);
}
fn probe_failures(&self) -> Vec<String> {
self.probe_failures
.lock()
.expect("probe failures lock")
.clone()
}
}
#[async_trait]
@@ -170,6 +239,14 @@ impl ExecutionStore for TrackingExecutionStore {
Ok(())
}
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
self.probe_failures
.lock()
.expect("probe failures lock")
.push(failure.error.kind().as_str().to_owned());
Ok(())
}
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
self.touch();
Ok(())
@@ -10,7 +10,7 @@ use gateway_core::accounting::{CostSource as CoreCostSource, Usage as CoreUsage}
use gateway_core::engine::{
AttemptRecord as CoreAttemptRecord, ExecutionStore, IntermediateFailure,
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
NewModelRequest as CoreNewModelRequest, RecoveryReport as CoreRecoveryReport,
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
UpstreamSendState as CoreUpstreamSendState,
};
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
@@ -819,6 +819,56 @@ impl ExecutionStore for PgExecutionStore {
.map_err(core_store_error)
}
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), CoreStoreError> {
let error = failure.error;
let retry_after_ms = error
.retry_after()
.map(|duration| u64::try_from(duration.as_millis()))
.transpose()
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let latency_ms = u64::try_from(failure.latency.as_millis())
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
tracing::warn!(
target: "gateway_probe",
provider_kind = failure.provider_kind.as_str(),
account_id = failure.account_id.as_str(),
upstream_model = failure.upstream_model_id.as_str(),
failure_kind = error.kind().as_str(),
send_state = ?error.send_state(),
upstream_status = ?error.upstream_status(),
provider_error_code = ?provider_error_code,
latency_ms,
"账号连接测试失败"
);
super::OpsEventRepository::append_ops_event(
&super::PgOpsEventRepository::new(self.pool.clone()),
super::OpsEvent {
id: Uuid::now_v7().to_string(),
model_request_id: None,
attempt_index: None,
level: super::OpsEventLevel::Warning,
component: "account_probe".to_owned(),
operation: "connection_test".to_owned(),
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
provider_account_id: Some(failure.account_id.as_str().to_owned()),
provider_account_ref: Some(failure.account_id.as_str().to_owned()),
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
failure_kind: error.kind().as_str().to_owned(),
status_code: error.upstream_status(),
provider_error_code,
retry_after_ms,
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
latency_ms: Some(latency_ms),
message: "account connection test failed".to_owned(),
occurrence_count: 1,
created_at: Utc::now(),
},
)
.await
.map_err(core_store_error)
}
async fn finalize_model_request(
&self,
finalization: CoreModelRequestFinalization,