From 304b3d4772dbb5e17f779cc24429b25b9658d655 Mon Sep 17 00:00:00 2001 From: zyycn Date: Sun, 26 Jul 2026 21:05:00 +0800 Subject: [PATCH] fix(store,openai): bound dashboard concurrency and restore quota state re-read Split the dashboard summary into two three-way query batches so one render holds at most three pool connections and cannot starve data-plane writes under the five-second acquire timeout. Restore the fresh account read before the passive-quota availability write-back: the revision fence only covers credential rotation, so deciding transitions from the selection-time snapshot could overwrite availability changes made by other paths and delay exhausted-account recovery. Drop the redundant admin_audit_events retention index (the initial created_at index already serves the delete predicate) and document the rolling-deploy indexing procedure in the migration. Co-authored-by: Claude --- backend/crates/gateway-core/src/engine/mod.rs | 3 +++ .../src/postgres/observability.rs | 7 +++++-- .../providers/openai/src/credential/quota.rs | 19 ++++++++++++------- .../0002_retention_delete_indexes.sql | 10 ++++++---- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/backend/crates/gateway-core/src/engine/mod.rs b/backend/crates/gateway-core/src/engine/mod.rs index 497fe53d0..9430bfa59 100644 --- a/backend/crates/gateway-core/src/engine/mod.rs +++ b/backend/crates/gateway-core/src/engine/mod.rs @@ -598,6 +598,9 @@ pub trait ExecutionStore: Send + Sync { async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError>; /// 请求插入与首次 attempt 合并持久化；两者写同一行，支持合并写的 /// store 可覆写为单次往返，缩短首 token 前的关键路径。 + /// + /// 覆写实现应保证原子性：默认的两步实现里第二步失败会留下一条 + /// `running` 请求行，由 deadline 回收器收敛。 async fn create_model_request_with_attempt( &self, request: NewModelRequest, diff --git a/backend/crates/gateway-store/src/postgres/observability.rs b/backend/crates/gateway-store/src/postgres/observability.rs index 2bc5479f5..f0a14789d 100644 --- a/backend/crates/gateway-store/src/postgres/observability.rs +++ b/backend/crates/gateway-store/src/postgres/observability.rs @@ -776,11 +776,14 @@ impl ObservabilityRepository for PgObservabilityRepository { page: ObservabilityPageNumber::new(1)?, page_size: ObservabilityPageSize::new(10)?, }; - // 六路独立查询并发执行；页面延迟由最慢一路决定而非各路之和。 - let (requests, attempts, provider_accounts, trend, account_usage, recent_requests) = futures::try_join!( + // 分两批并发：页面延迟从六路之和降为两批各自最慢者之和，同时单次 + // 渲染最多占用 3 个池连接，不与数据面写路径争抢整个连接池。 + let (requests, attempts, provider_accounts) = futures::try_join!( request_metrics(&self.pool, range, &filter), attempt_metrics(&self.pool, range, &filter), provider_account_metrics(&self.pool, range.end), + )?; + let (trend, account_usage, recent_requests) = futures::try_join!( request_metric_series(&self.pool, range, &filter), provider_account_usage(&self.pool, account_usage_query), async { Ok(list_usage_records(&self.pool, recent_query).await?.items) }, diff --git a/backend/crates/providers/openai/src/credential/quota.rs b/backend/crates/providers/openai/src/credential/quota.rs index 425f8379e..ed0378fa6 100644 --- a/backend/crates/providers/openai/src/credential/quota.rs +++ b/backend/crates/providers/openai/src/credential/quota.rs @@ -658,20 +658,25 @@ impl CodexCredentialQuotaService { if outcome == QuotaWriteOutcome::Conflict { return Ok(false); } - // quota CAS 以 account.revision 为 fence，成功即证明该 revision 在写入时 - // 仍是当前凭据；后续状态写回带同一 fence，轮换竞态下安全失败。 - // 可用性判断使用选择时的账号快照，偏差在下一次响应或 quota 轮询收敛。 + // 可用性写回必须基于最新账号状态：revision fence 只覆盖凭据轮换， + // 不覆盖其他路径的仅状态变化（Cooldown/QuotaExhausted 转换）。 + let Some(current) = self.store.get_account(account.id()).await? else { + return Ok(false); + }; + if current.revision() != account.revision() { + return Ok(false); + } self.scheduling - .observe(account.id().clone(), account.revision(), observed_at, fact); + .observe(current.id().clone(), current.revision(), observed_at, fact); if let Some(availability) = quota_success_availability( - account.availability(), - account.cooldown_until(), + current.availability(), + current.cooldown_until(), fact.exhausted(), observed_at, ) { self.repository .apply_state( - account, + ¤t, availability, fact.exhausted().then_some("quota_exhausted".to_owned()), None, diff --git a/backend/migrations/0002_retention_delete_indexes.sql b/backend/migrations/0002_retention_delete_indexes.sql index b214e2cfd..7b9bbbccc 100644 --- a/backend/migrations/0002_retention_delete_indexes.sql +++ b/backend/migrations/0002_retention_delete_indexes.sql @@ -1,5 +1,10 @@ -- 保留期清理的删除谓词索引：清理任务按终态时间批量删除， --- 无索引时每轮都会全表扫描三张审计/用量表。 +-- 无索引时每轮都会全表扫描。admin_audit_events 已有 (created_at desc, id desc) +-- 索引覆盖其删除谓词，不再重复建。 +-- +-- 索引在启动迁移事务内构建，期间对目标表持 SHARE 锁（挡写不挡读）。 +-- 单实例部署在 serve 之前执行，无并发写；多实例滚动部署应在升级前 +-- 手动 `create index concurrently` 预建同名索引，本迁移随后幂等跳过。 create index if not exists idx_model_requests_retention on model_requests (completed_at) @@ -8,6 +13,3 @@ create index if not exists idx_model_requests_retention create index if not exists idx_ops_events_retention on ops_events (created_at) where model_request_id is null; - -create index if not exists idx_admin_audit_events_retention - on admin_audit_events (created_at);