From 521201c9264949cb19f8281bb3ed1a0908377663 Mon Sep 17 00:00:00 2001 From: Zyy Date: Fri, 17 Jul 2026 14:52:19 +0800 Subject: [PATCH] fix(telemetry): preserve timed-out record writes --- backend/src/telemetry/recorder.rs | 9 +++++++-- .../tests/dispatch/service/usage_logging.rs | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/src/telemetry/recorder.rs b/backend/src/telemetry/recorder.rs index 59ce7e70b..c1fb4ec30 100644 --- a/backend/src/telemetry/recorder.rs +++ b/backend/src/telemetry/recorder.rs @@ -25,6 +25,8 @@ pub struct Recorder { pub enum RecorderError { #[error("invalid success usage fact")] InvalidUsageFact, + #[error("telemetry persistence task failed: {0}")] + Task(#[from] tokio::task::JoinError), #[error(transparent)] Usage(#[from] PgUsageRecordStoreError), #[error(transparent)] @@ -62,13 +64,16 @@ impl Recorder { return Err(RecorderError::InvalidUsageFact); } apply_capture_body_policy(&mut event.metadata, self.capture_body); - self.usage_records.append(&event).await?; + let usage_records = self.usage_records.clone(); + // Controller timeouts must stop waiting without cancelling an in-flight fact write. + tokio::spawn(async move { usage_records.append(&event).await }).await??; Ok(()) } pub async fn record_error(&self, mut event: OpsErrorLog) -> Result<(), RecorderError> { apply_capture_body_policy(&mut event.metadata, self.capture_body); - self.ops_errors.append(&event).await?; + let ops_errors = self.ops_errors.clone(); + tokio::spawn(async move { ops_errors.append(&event).await }).await??; Ok(()) } diff --git a/backend/tests/dispatch/service/usage_logging.rs b/backend/tests/dispatch/service/usage_logging.rs index fcb137587..8d2b4b614 100644 --- a/backend/tests/dispatch/service/usage_logging.rs +++ b/backend/tests/dispatch/service/usage_logging.rs @@ -730,6 +730,24 @@ async fn responses_stream_should_preserve_body_metadata_when_capture_body_enable let (app, api_key, pool, _dir) = test_app_with_account_pool_and_telemetry_capture_body(server.uri()).await; + // Force telemetry past the controller wait budget; persistence must outlive that timeout. + sqlx::raw_sql( + r#" + create function delay_usage_record_insert() returns trigger as $$ + begin + perform pg_sleep(0.2); + return new; + end; + $$ language plpgsql; + + create trigger delay_usage_record_insert + before insert on usage_records + for each row execute function delay_usage_record_insert(); + "#, + ) + .execute(&pool) + .await + .unwrap(); let response = app .oneshot( Request::builder()