mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-21 08:01:41 +00:00
feat(core,store): record account probe failures as ops events
Connection tests run through a transient execution store, so a failed probe left no trace at all: nothing in model_requests, nothing in ops_events, and nothing in the log at info level. The reason a probe failed — upstream status, provider error code, send state — was only reachable by reproducing it. Add a ProbeFailure port on ExecutionStore, defaulting to a no-op so existing implementations are unaffected, and have the execution service report probe failures to the persistent store while the probe itself keeps using the transient one. The Postgres store logs a warning and appends an ops event with no model_request_id. Such events already fit the surrounding contracts: the ops error list left joins model_requests, health metrics inner join it so probes stay out of business failure rates, and retention already targets rows with a null model_request_id. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,8 @@ use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement, CoordinatedEvent,
|
||||
EngineError, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, ProviderAccountId, ProviderAttemptOutcome,
|
||||
RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use crate::event::{GatewayEvent, ProviderEvent, ProviderResponseHeader};
|
||||
@@ -31,6 +32,7 @@ use crate::policy::{ClientApiKeyId, ClientPolicy};
|
||||
use crate::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use crate::routing::{
|
||||
ProviderKind, PublicModelId, PublicModelProfile, RoutingContext, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
@@ -187,6 +189,8 @@ pub struct DefaultExecutionService {
|
||||
snapshots: RuntimeSnapshotHandle,
|
||||
coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
probe_coordinator: Arc<AttemptCoordinator<dyn ExecutionStore>>,
|
||||
/// probe 自身走 transient store,探测失败仍写入持久 store 的 ops_events。
|
||||
observations: Arc<dyn ExecutionStore>,
|
||||
providers: ProviderRegistry,
|
||||
admissions: Arc<dyn ClientAdmissionPort>,
|
||||
circuits: Arc<dyn ProviderCircuitPort>,
|
||||
@@ -205,6 +209,7 @@ impl DefaultExecutionService {
|
||||
continuation: Arc<dyn NativeContinuationPort>,
|
||||
client_api_key_usage: Arc<dyn ClientApiKeyUsageSink>,
|
||||
) -> Self {
|
||||
let observations = Arc::clone(&execution);
|
||||
let engine = GatewayEngine::<dyn ExecutionStore>::new(execution, providers.clone());
|
||||
let transient: Arc<dyn ExecutionStore> = Arc::new(TransientExecutionStore);
|
||||
let probe_engine = GatewayEngine::<dyn ExecutionStore>::new(transient, providers.clone());
|
||||
@@ -212,6 +217,7 @@ impl DefaultExecutionService {
|
||||
snapshots,
|
||||
coordinator: Arc::new(AttemptCoordinator::new(engine)),
|
||||
probe_coordinator: Arc::new(AttemptCoordinator::new(probe_engine)),
|
||||
observations,
|
||||
providers,
|
||||
admissions,
|
||||
circuits,
|
||||
@@ -417,6 +423,11 @@ impl DefaultExecutionService {
|
||||
upstream_model,
|
||||
operation,
|
||||
} = request;
|
||||
let observed = ProbeObservation {
|
||||
provider_kind: provider_kind.clone(),
|
||||
account_id: account_id.clone(),
|
||||
upstream_model: upstream_model.clone(),
|
||||
};
|
||||
let snapshot = self.snapshots.acquire().map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -464,7 +475,7 @@ impl DefaultExecutionService {
|
||||
started_at,
|
||||
deadline_at,
|
||||
};
|
||||
let mut session = self
|
||||
let mut session = match self
|
||||
.probe_coordinator
|
||||
.start(
|
||||
new_request,
|
||||
@@ -475,18 +486,33 @@ impl DefaultExecutionService {
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
{
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
let events = session.collect_uncommitted().await;
|
||||
publish_provider_attempt_outcomes(
|
||||
self.circuits.as_ref(),
|
||||
session.provider_attempt_outcomes(),
|
||||
)
|
||||
.await;
|
||||
let events = events.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
session
|
||||
.commit_downstream(Some(200))
|
||||
.await
|
||||
.map_err(|error| gateway_error_from_engine(&error))?;
|
||||
let events = match events {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(200)).await {
|
||||
return Err(self
|
||||
.observe_probe_failure(&observed, started_at, &error)
|
||||
.await);
|
||||
}
|
||||
Ok(AccountProbeResult {
|
||||
text: events
|
||||
.into_iter()
|
||||
@@ -498,6 +524,34 @@ impl DefaultExecutionService {
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> GatewayError {
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let _ = self
|
||||
.observations
|
||||
.record_probe_failure(ProbeFailure {
|
||||
provider_kind: observed.provider_kind.clone(),
|
||||
account_id: observed.account_id.clone(),
|
||||
upstream_model_id: observed.upstream_model.clone(),
|
||||
error: provider_error.clone(),
|
||||
latency: started_at.elapsed().unwrap_or_default(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
gateway_error_from_engine(error)
|
||||
}
|
||||
}
|
||||
|
||||
struct ProbeObservation {
|
||||
provider_kind: ProviderKind,
|
||||
account_id: ProviderAccountId,
|
||||
upstream_model: UpstreamModelId,
|
||||
}
|
||||
|
||||
struct TransientExecutionStore;
|
||||
|
||||
@@ -543,6 +543,16 @@ pub struct IntermediateFailure {
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// 不属于任何 `model_requests` 行的管理端账号探测失败。
|
||||
#[derive(Debug)]
|
||||
pub struct ProbeFailure {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub account_id: ProviderAccountId,
|
||||
pub upstream_model_id: UpstreamModelId,
|
||||
pub error: ProviderError,
|
||||
pub latency: Duration,
|
||||
}
|
||||
|
||||
/// `model_requests` 可用的毫秒级阶段耗时。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ModelRequestTimings {
|
||||
@@ -629,6 +639,10 @@ pub trait ExecutionStore: Send + Sync {
|
||||
&self,
|
||||
failure: IntermediateFailure,
|
||||
) -> Result<(), StoreError>;
|
||||
/// 记录不挂在 `model_requests` 上的账号探测失败;默认丢弃。
|
||||
async fn record_probe_failure(&self, _failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: ModelRequestFinalization,
|
||||
|
||||
@@ -24,12 +24,15 @@ use gateway_core::engine::execution::{
|
||||
ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit,
|
||||
};
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::ProviderRegistry;
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest, RecoveryReport, UpstreamSendState,
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
|
||||
ProviderRequest, ProviderStream,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderErrorKind, StoreError};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
@@ -78,6 +81,64 @@ fn account_probe_should_not_write_to_the_persistent_execution_store() {
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_failures_should_be_observable_without_a_model_request_row() {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry");
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(probe_snapshot()),
|
||||
store.clone(),
|
||||
providers,
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
|
||||
let error = block_on(service.probe(AccountProbeRequest {
|
||||
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
|
||||
operation: probe_operation(),
|
||||
}))
|
||||
.expect_err("the provider rejects every probe");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
|
||||
}
|
||||
|
||||
struct FailingProvider;
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for FailingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
_request: ProviderRequest,
|
||||
_context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_authentication_should_record_client_key_usage() {
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
@@ -121,12 +182,20 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
|
||||
#[derive(Default)]
|
||||
struct TrackingExecutionStore {
|
||||
touched: AtomicBool,
|
||||
probe_failures: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl TrackingExecutionStore {
|
||||
fn touch(&self) {
|
||||
self.touched.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn probe_failures(&self) -> Vec<String> {
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -170,6 +239,14 @@ impl ExecutionStore for TrackingExecutionStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
.push(failure.error.kind().as_str().to_owned());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
|
||||
self.touch();
|
||||
Ok(())
|
||||
|
||||
@@ -10,7 +10,7 @@ use gateway_core::accounting::{CostSource as CoreCostSource, Usage as CoreUsage}
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord as CoreAttemptRecord, ExecutionStore, IntermediateFailure,
|
||||
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
|
||||
NewModelRequest as CoreNewModelRequest, RecoveryReport as CoreRecoveryReport,
|
||||
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
|
||||
UpstreamSendState as CoreUpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
@@ -819,6 +819,56 @@ impl ExecutionStore for PgExecutionStore {
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), CoreStoreError> {
|
||||
let error = failure.error;
|
||||
let retry_after_ms = error
|
||||
.retry_after()
|
||||
.map(|duration| u64::try_from(duration.as_millis()))
|
||||
.transpose()
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let latency_ms = u64::try_from(failure.latency.as_millis())
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = failure.provider_kind.as_str(),
|
||||
account_id = failure.account_id.as_str(),
|
||||
upstream_model = failure.upstream_model_id.as_str(),
|
||||
failure_kind = error.kind().as_str(),
|
||||
send_state = ?error.send_state(),
|
||||
upstream_status = ?error.upstream_status(),
|
||||
provider_error_code = ?provider_error_code,
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
super::OpsEventRepository::append_ops_event(
|
||||
&super::PgOpsEventRepository::new(self.pool.clone()),
|
||||
super::OpsEvent {
|
||||
id: Uuid::now_v7().to_string(),
|
||||
model_request_id: None,
|
||||
attempt_index: None,
|
||||
level: super::OpsEventLevel::Warning,
|
||||
component: "account_probe".to_owned(),
|
||||
operation: "connection_test".to_owned(),
|
||||
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
||||
provider_account_id: Some(failure.account_id.as_str().to_owned()),
|
||||
provider_account_ref: Some(failure.account_id.as_str().to_owned()),
|
||||
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
|
||||
failure_kind: error.kind().as_str().to_owned(),
|
||||
status_code: error.upstream_status(),
|
||||
provider_error_code,
|
||||
retry_after_ms,
|
||||
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
|
||||
latency_ms: Some(latency_ms),
|
||||
message: "account connection test failed".to_owned(),
|
||||
occurrence_count: 1,
|
||||
created_at: Utc::now(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(core_store_error)
|
||||
}
|
||||
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: CoreModelRequestFinalization,
|
||||
|
||||
Reference in New Issue
Block a user