From fe5494a614e80acedc68483e09dd79cf72c364d5 Mon Sep 17 00:00:00 2001 From: zyycn Date: Fri, 10 Jul 2026 22:53:27 +0800 Subject: [PATCH] refactor: align implementation with authoritative architecture Co-authored-by: Codex --- .github/workflows/ci.yml | 3 - backend/scripts/check_architecture.sh | 91 -- backend/src/accounts/account.rs | 42 - backend/src/accounts/manage/cookies.rs | 11 - backend/src/accounts/manage/lifecycle.rs | 15 +- backend/src/accounts/manage/mod.rs | 1 - backend/src/accounts/manage/probe.rs | 110 +- backend/src/accounts/manage/quota.rs | 17 + backend/src/accounts/manage/service.rs | 4 - backend/src/accounts/manage/types.rs | 1 + backend/src/accounts/mod.rs | 1 - backend/src/accounts/pool.rs | 11 +- backend/src/accounts/pool/runtime.rs | 148 ++- backend/src/accounts/quota/runtime.rs | 18 +- backend/src/accounts/refresh/mod.rs | 4 +- backend/src/accounts/refresh/policy.rs | 485 ++++++++ backend/src/accounts/refresh/runtime.rs | 777 ------------ backend/src/accounts/refresh/runtime/types.rs | 109 -- backend/src/accounts/refresh/service.rs | 1076 ++++++++++++----- backend/src/accounts/scheduler/candidates.rs | 30 +- backend/src/accounts/service.rs | 32 - backend/src/accounts/store.rs | 113 +- backend/src/accounts/store/queries.rs | 231 +--- backend/src/accounts/store/rows.rs | 107 +- backend/src/accounts/store/write.rs | 141 +-- backend/src/accounts/window.rs | 32 +- backend/src/api/admin/accounts_routes.rs | 8 +- .../src/api/admin/accounts_routes/actions.rs | 23 +- .../src/api/admin/accounts_routes/stats.rs | 5 +- backend/src/api/admin/dashboard_routes.rs | 240 ++-- .../src/api/admin/dashboard_routes/data.rs | 428 +------ backend/src/api/admin/keys_routes.rs | 6 +- backend/src/api/admin/mod.rs | 1 - backend/src/api/admin/ops_routes.rs | 2 +- backend/src/api/admin/response.rs | 59 +- backend/src/api/admin/router.rs | 2 +- backend/src/api/admin/session.rs | 2 +- backend/src/api/admin/settings_routes.rs | 7 +- backend/src/api/admin/system_routes.rs | 19 +- backend/src/api/admin/update_payload.rs | 62 - backend/src/api/admin/usage_routes.rs | 2 +- backend/src/api/client/auth.rs | 2 +- backend/src/api/client/errors.rs | 2 +- backend/src/api/client/models.rs | 3 +- backend/src/api/client/responses.rs | 4 +- backend/src/api/client/router.rs | 2 +- backend/src/api/mod.rs | 2 + backend/src/api/router.rs | 70 +- backend/src/bootstrap/application.rs | 168 --- backend/src/bootstrap/mod.rs | 1 - backend/src/bootstrap/services.rs | 298 +++-- backend/src/bootstrap/shutdown.rs | 17 + backend/src/bootstrap/state.rs | 37 +- backend/src/bootstrap/tasks/coordinator.rs | 47 +- backend/src/bootstrap/tasks/model_refresh.rs | 3 +- backend/src/bootstrap/tasks/quota_refresh.rs | 3 + backend/src/dispatch/affinity/resolve.rs | 2 +- backend/src/dispatch/affinity/service.rs | 8 + backend/src/dispatch/errors.rs | 351 +++++- backend/src/dispatch/mod.rs | 11 +- .../event_recording.rs => recording.rs} | 8 +- backend/src/dispatch/recovery/auth.rs | 22 + .../src/dispatch/{ => recovery}/cloudflare.rs | 0 .../src/dispatch/{ => recovery}/exhaustion.rs | 0 .../{ => recovery}/implicit_resume.rs | 0 backend/src/dispatch/recovery/mod.rs | 7 + .../{ => recovery}/reasoning_replay.rs | 0 backend/src/dispatch/responses/errors.rs | 342 ------ backend/src/dispatch/responses/mod.rs | 10 - .../src/dispatch/{responses => }/service.rs | 53 +- .../{responses => }/service/compact.rs | 0 .../{responses => }/service/complete.rs | 0 .../{responses => }/service/stream.rs | 0 .../lifecycle.rs} | 70 +- .../live_stream.rs => stream/live.rs} | 6 +- backend/src/dispatch/stream/mod.rs | 7 + .../{responses => stream}/prefetch.rs | 4 +- .../{responses => stream}/sse_failure.rs | 38 +- .../dispatch/{responses => stream}/trace.rs | 15 +- .../{upstream.rs => upstream_call.rs} | 4 +- backend/src/main.rs | 2 +- backend/src/models/catalog.rs | 23 +- backend/src/models/config.rs | 12 - backend/src/models/info.rs | 53 - backend/src/models/mod.rs | 4 +- backend/src/models/service.rs | 35 +- backend/src/models/snapshot.rs | 15 - backend/src/models/store.rs | 2 +- backend/src/models/types.rs | 119 ++ backend/src/settings/service.rs | 32 +- backend/src/telemetry/account_usage/store.rs | 503 +++++++- backend/src/telemetry/dashboard.rs | 539 +++++++++ backend/src/telemetry/diagnostics.rs | 90 -- backend/src/telemetry/mod.rs | 2 +- backend/src/upstream/openai/protocol/mod.rs | 2 - .../upstream/openai/protocol/model_catalog.rs | 73 -- .../src/upstream/openai/transport/client.rs | 30 +- .../openai/transport/client/requests.rs | 4 +- .../{cloudflare.rs => cookies/cooldown.rs} | 0 backend/tests/accounts/cookies/mod.rs | 2 + .../accounts/{cookies.rs => cookies/store.rs} | 0 backend/tests/accounts/mod.rs | 10 +- .../accounts/{account_pool => pool}/mod.rs | 1 + .../accounts/{account_pool => pool}/quota.rs | 0 .../mod.rs => accounts/pool/runtime.rs} | 7 +- .../{account_pool => pool}/selection.rs | 6 + .../{account_pool => pool}/usage_window.rs | 0 .../accounts/{quota_refresh.rs => quota.rs} | 53 +- .../{token_refresh => refresh}/failures.rs | 0 .../{refresh_leases.rs => refresh/lease.rs} | 0 .../{token_refresh => refresh}/mod.rs | 1 + .../{token_refresh => refresh}/scheduling.rs | 0 .../{token_refresh => refresh}/success.rs | 0 .../accounts/{account_store.rs => store.rs} | 45 +- .../export.rs} | 0 .../import.rs} | 12 +- .../lifecycle.rs | 26 +- .../{accounts => accounts_routes}/list.rs | 18 +- .../{accounts => accounts_routes}/mod.rs | 75 +- .../{accounts => accounts_routes}/oauth.rs | 0 .../testing.rs => accounts_routes/probe.rs} | 32 +- .../{accounts => accounts_routes}/quota.rs | 0 .../api/admin/{auth => auth_routes}/mod.rs | 1 - .../admin/{auth => auth_routes}/password.rs | 0 .../admin/{auth => auth_routes}/session.rs | 4 +- .../dashboard.rs => dashboard_routes.rs} | 7 +- .../{keys => keys_routes}/authorization.rs | 0 .../admin/{keys => keys_routes}/lifecycle.rs | 0 .../api/admin/{keys => keys_routes}/mod.rs | 7 +- backend/tests/api/admin/mod.rs | 13 +- backend/tests/api/admin/monitoring/mod.rs | 4 - .../admin/{response/mod.rs => response.rs} | 0 .../{settings => settings_routes}/mod.rs | 0 .../{settings => settings_routes}/routes.rs | 44 +- .../admin/{system => system_routes}/mod.rs | 6 +- ...usage_record_routes.rs => usage_routes.rs} | 5 +- .../api/{http/web_assets/mod.rs => assets.rs} | 4 +- backend/tests/api/client/mod.rs | 3 +- .../{openai/models_routes.rs => models.rs} | 6 +- backend/tests/api/client/openai/mod.rs | 2 - .../responses_routes.rs => responses.rs} | 6 +- backend/tests/api/http/mod.rs | 2 - backend/tests/api/middleware/mod.rs | 1 + .../mod.rs => middleware/trace.rs} | 0 backend/tests/api/mod.rs | 3 +- backend/tests/auth/mod.rs | 1 + .../auth/session_store.rs => auth/store.rs} | 0 backend/tests/bootstrap/mod.rs | 1 - .../{fingerprint.rs => fingerprint_update.rs} | 0 backend/tests/bootstrap/tasks/mod.rs | 73 +- .../tests/bootstrap/tasks/model_refresh.rs | 107 +- .../dispatch/{affinity.rs => affinity/mod.rs} | 0 backend/tests/dispatch/mod.rs | 2 +- .../compact_routes.rs | 0 .../{chat_upstream => service}/mod.rs | 38 +- .../responses_http.rs | 12 +- .../responses_recovery.rs | 0 .../responses_websocket.rs | 14 +- .../usage_logging.rs | 0 .../{storage_schema/mod.rs => database.rs} | 0 .../infra/{log_rotation/mod.rs => logging.rs} | 0 backend/tests/infra/mod.rs | 4 +- backend/tests/keys/mod.rs | 1 + backend/tests/{api/admin => }/keys/store.rs | 0 backend/tests/main.rs | 2 + backend/tests/models/catalog.rs | 17 +- backend/tests/models/store.rs | 9 +- backend/tests/telemetry/mod.rs | 2 + backend/tests/telemetry/ops/mod.rs | 1 + .../ops/store.rs} | 0 backend/tests/telemetry/usage/mod.rs | 1 + .../usage/store.rs} | 0 .../{usage_rate_limits.rs => events.rs} | 0 backend/tests/upstream/openai/protocol/mod.rs | 8 +- .../{openai_responses.rs => responses.rs} | 0 .../{response_first_event.rs => sse.rs} | 0 .../{codex_websocket.rs => websocket.rs} | 0 .../upstream/openai/transport/headers.rs | 2 +- deploy/Dockerfile | 2 +- docs/architecture.md | 254 ++-- docs/database.md | 2 +- 181 files changed, 4398 insertions(+), 4194 deletions(-) delete mode 100755 backend/scripts/check_architecture.sh delete mode 100644 backend/src/accounts/manage/cookies.rs create mode 100644 backend/src/accounts/refresh/policy.rs delete mode 100644 backend/src/accounts/refresh/runtime.rs delete mode 100644 backend/src/accounts/refresh/runtime/types.rs delete mode 100644 backend/src/accounts/service.rs delete mode 100644 backend/src/api/admin/update_payload.rs delete mode 100644 backend/src/bootstrap/application.rs rename backend/src/dispatch/{responses/event_recording.rs => recording.rs} (99%) create mode 100644 backend/src/dispatch/recovery/auth.rs rename backend/src/dispatch/{ => recovery}/cloudflare.rs (100%) rename backend/src/dispatch/{ => recovery}/exhaustion.rs (100%) rename backend/src/dispatch/{ => recovery}/implicit_resume.rs (100%) create mode 100644 backend/src/dispatch/recovery/mod.rs rename backend/src/dispatch/{ => recovery}/reasoning_replay.rs (100%) delete mode 100644 backend/src/dispatch/responses/errors.rs delete mode 100644 backend/src/dispatch/responses/mod.rs rename backend/src/dispatch/{responses => }/service.rs (90%) rename backend/src/dispatch/{responses => }/service/compact.rs (100%) rename backend/src/dispatch/{responses => }/service/complete.rs (100%) rename backend/src/dispatch/{responses => }/service/stream.rs (100%) rename backend/src/dispatch/{responses/stream_lifecycle.rs => stream/lifecycle.rs} (79%) rename backend/src/dispatch/{responses/live_stream.rs => stream/live.rs} (98%) create mode 100644 backend/src/dispatch/stream/mod.rs rename backend/src/dispatch/{responses => stream}/prefetch.rs (90%) rename backend/src/dispatch/{responses => stream}/sse_failure.rs (82%) rename backend/src/dispatch/{responses => stream}/trace.rs (60%) rename backend/src/dispatch/{upstream.rs => upstream_call.rs} (99%) delete mode 100644 backend/src/models/config.rs delete mode 100644 backend/src/models/info.rs delete mode 100644 backend/src/models/snapshot.rs create mode 100644 backend/src/models/types.rs create mode 100644 backend/src/telemetry/dashboard.rs delete mode 100644 backend/src/telemetry/diagnostics.rs delete mode 100644 backend/src/upstream/openai/protocol/model_catalog.rs rename backend/tests/accounts/{cloudflare.rs => cookies/cooldown.rs} (100%) create mode 100644 backend/tests/accounts/cookies/mod.rs rename backend/tests/accounts/{cookies.rs => cookies/store.rs} (100%) rename backend/tests/accounts/{account_pool => pool}/mod.rs (98%) rename backend/tests/accounts/{account_pool => pool}/quota.rs (100%) rename backend/tests/{bootstrap/account_pool_restore/mod.rs => accounts/pool/runtime.rs} (92%) rename backend/tests/accounts/{account_pool => pool}/selection.rs (99%) rename backend/tests/accounts/{account_pool => pool}/usage_window.rs (100%) rename backend/tests/accounts/{quota_refresh.rs => quota.rs} (94%) rename backend/tests/accounts/{token_refresh => refresh}/failures.rs (100%) rename backend/tests/accounts/{refresh_leases.rs => refresh/lease.rs} (100%) rename backend/tests/accounts/{token_refresh => refresh}/mod.rs (99%) rename backend/tests/accounts/{token_refresh => refresh}/scheduling.rs (100%) rename backend/tests/accounts/{token_refresh => refresh}/success.rs (100%) rename backend/tests/accounts/{account_store.rs => store.rs} (94%) rename backend/tests/api/admin/{accounts/exporting.rs => accounts_routes/export.rs} (100%) rename backend/tests/api/admin/{accounts/importing.rs => accounts_routes/import.rs} (99%) rename backend/tests/api/admin/{accounts => accounts_routes}/lifecycle.rs (98%) rename backend/tests/api/admin/{accounts => accounts_routes}/list.rs (98%) rename backend/tests/api/admin/{accounts => accounts_routes}/mod.rs (84%) rename backend/tests/api/admin/{accounts => accounts_routes}/oauth.rs (100%) rename backend/tests/api/admin/{accounts/testing.rs => accounts_routes/probe.rs} (96%) rename backend/tests/api/admin/{accounts => accounts_routes}/quota.rs (100%) rename backend/tests/api/admin/{auth => auth_routes}/mod.rs (62%) rename backend/tests/api/admin/{auth => auth_routes}/password.rs (100%) rename backend/tests/api/admin/{auth => auth_routes}/session.rs (98%) rename backend/tests/api/admin/{monitoring/dashboard.rs => dashboard_routes.rs} (99%) rename backend/tests/api/admin/{keys => keys_routes}/authorization.rs (100%) rename backend/tests/api/admin/{keys => keys_routes}/lifecycle.rs (100%) rename backend/tests/api/admin/{keys => keys_routes}/mod.rs (91%) delete mode 100644 backend/tests/api/admin/monitoring/mod.rs rename backend/tests/api/admin/{response/mod.rs => response.rs} (100%) rename backend/tests/api/admin/{settings => settings_routes}/mod.rs (100%) rename backend/tests/api/admin/{settings => settings_routes}/routes.rs (93%) rename backend/tests/api/admin/{system => system_routes}/mod.rs (99%) rename backend/tests/api/admin/{monitoring/usage_record_routes.rs => usage_routes.rs} (98%) rename backend/tests/api/{http/web_assets/mod.rs => assets.rs} (98%) rename backend/tests/api/client/{openai/models_routes.rs => models.rs} (97%) delete mode 100644 backend/tests/api/client/openai/mod.rs rename backend/tests/api/client/{openai/responses_routes.rs => responses.rs} (97%) delete mode 100644 backend/tests/api/http/mod.rs create mode 100644 backend/tests/api/middleware/mod.rs rename backend/tests/api/{http/trace_middleware/mod.rs => middleware/trace.rs} (100%) create mode 100644 backend/tests/auth/mod.rs rename backend/tests/{api/admin/auth/session_store.rs => auth/store.rs} (100%) rename backend/tests/bootstrap/tasks/{fingerprint.rs => fingerprint_update.rs} (100%) rename backend/tests/dispatch/{affinity.rs => affinity/mod.rs} (100%) rename backend/tests/dispatch/{chat_upstream => service}/compact_routes.rs (100%) rename backend/tests/dispatch/{chat_upstream => service}/mod.rs (98%) rename backend/tests/dispatch/{chat_upstream => service}/responses_http.rs (99%) rename backend/tests/dispatch/{chat_upstream => service}/responses_recovery.rs (100%) rename backend/tests/dispatch/{chat_upstream => service}/responses_websocket.rs (99%) rename backend/tests/dispatch/{chat_upstream => service}/usage_logging.rs (100%) rename backend/tests/infra/{storage_schema/mod.rs => database.rs} (100%) rename backend/tests/infra/{log_rotation/mod.rs => logging.rs} (100%) create mode 100644 backend/tests/keys/mod.rs rename backend/tests/{api/admin => }/keys/store.rs (100%) create mode 100644 backend/tests/telemetry/ops/mod.rs rename backend/tests/{api/admin/monitoring/ops_error_store.rs => telemetry/ops/store.rs} (100%) create mode 100644 backend/tests/telemetry/usage/mod.rs rename backend/tests/{api/admin/monitoring/usage_record_store.rs => telemetry/usage/store.rs} (100%) rename backend/tests/upstream/openai/protocol/{usage_rate_limits.rs => events.rs} (100%) rename backend/tests/upstream/openai/protocol/{openai_responses.rs => responses.rs} (100%) rename backend/tests/upstream/openai/protocol/{response_first_event.rs => sse.rs} (100%) rename backend/tests/upstream/openai/protocol/{codex_websocket.rs => websocket.rs} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1dea14e58..a42f1e1f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,9 +61,6 @@ jobs: working-directory: backend run: cargo clippy --all-targets --all-features --locked -- -D warnings - - name: Architecture boundaries - run: backend/scripts/check_architecture.sh - - name: Rust tests working-directory: backend run: cargo test --test main --locked diff --git a/backend/scripts/check_architecture.sh b/backend/scripts/check_architecture.sh deleted file mode 100755 index cbcfcdc1d..000000000 --- a/backend/scripts/check_architecture.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -cd "$(dirname "$0")/.." - -fail=0 - -deny() { - local description=$1 - local pattern=$2 - shift 2 - if grep -rn --include='*.rs' -E "$pattern" "$@"; then - echo "ARCH VIOLATION: $description" >&2 - fail=1 - fi -} - -domain_paths=( - src/infra src/upstream src/telemetry src/keys src/auth src/settings src/update - src/accounts src/models src/dispatch -) - -deny "domain -> api/bootstrap" '(^|[^[:alnum:]_])(api|bootstrap)::' "${domain_paths[@]}" -deny "upstream -> domain" '(^|[^[:alnum:]_])(accounts|dispatch|telemetry|keys|auth|settings|update|models)::' src/upstream -deny "axum outside api/bootstrap" '\baxum(::|\b)' "${domain_paths[@]}" -deny "sqlx in api" '\bsqlx(::|\b)' src/api -deny "redis in api" '\bredis::' src/api -deny "legacy module path" 'crate::(admin|proxy|runtime|web|http|config)::' src -deny "Admin-prefixed domain type" '\bAdmin[A-Z][A-Za-z0-9]*\b' "${domain_paths[@]}" -deny "Repository naming" '\b[A-Za-z0-9]*Repository\b' src -deny "persistence repository naming" '\b[A-Za-z0-9_]*repository[A-Za-z0-9_]*\b' \ - src/accounts src/auth src/dispatch src/keys src/models src/settings src/telemetry \ - src/upstream/openai/fingerprint -deny "Arc alias" '\bArc[[:space:]]+as[[:space:]]+' src - -storage_command_check() { - local kind=$1 - local pattern=$2 - local file - while IFS= read -r file; do - case "$file" in - */store.rs | */store/*) - continue - ;; - esac - if [ "$kind" = sql ]; then - case "$file" in - src/infra/database.rs | src/bootstrap/import_sqlite.rs | src/bootstrap/import_sqlite/* | src/telemetry/rebuild.rs | */query.rs) - continue - ;; - esac - else - case "$file" in - src/infra/redis.rs | src/accounts/refresh/lease.rs) - continue - ;; - esac - fi - echo "$file" - echo "ARCH VIOLATION: $kind command outside an allowed storage adapter" >&2 - fail=1 - done < <(grep -rl --include='*.rs' -E "$pattern" src || true) -} - -storage_command_check sql 'sqlx::(query|query_as|query_scalar|raw_sql)' -storage_command_check redis 'redis::(cmd|pipe)|[.]query_async[[:space:]]*[(]|use redis::.*AsyncCommands' - -banned=$(find src \( -name 'importing.rs' -o -name 'exporting.rs' -o -name 'testing.rs' \ - -o -name 'util.rs' -o -name 'helper.rs' -o -name 'common.rs' -o -name 'misc.rs' \)) -if [ -n "$banned" ]; then - echo "ARCH VIOLATION: banned file names: $banned" >&2 - fail=1 -fi - -while read -r file; do - lines=$(wc -l < "$file") - if [ "$lines" -gt 50 ]; then - echo "ARCH VIOLATION: $file has $lines lines (mod.rs <= 50)" >&2 - fail=1 - fi -done < <(find src -name mod.rs) - -while read -r file; do - lines=$(wc -l < "$file") - if [ "$lines" -gt 800 ]; then - echo "ARCH VIOLATION: $file has $lines lines (> 800)" >&2 - fail=1 - fi -done < <(find src -name '*.rs' ! -name mod.rs) - -exit "$fail" diff --git a/backend/src/accounts/account.rs b/backend/src/accounts/account.rs index fead7291c..b4a1a72f6 100644 --- a/backend/src/accounts/account.rs +++ b/backend/src/accounts/account.rs @@ -109,45 +109,3 @@ pub struct Account { /// 最近一次使用时间。 pub last_used_at: Option, } - -/// 账号用量增量。 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct AccountUsageDelta { - /// 请求数增量。 - pub requests: u64, - /// 输入 token 增量。 - pub input_tokens: u64, - /// 输出 token 增量。 - pub output_tokens: u64, - /// 缓存 token 增量。 - pub cached_tokens: u64, - /// reasoning token 增量。 - pub reasoning_tokens: u64, - /// 上游返回的总 token 增量。 - pub total_tokens: u64, - /// 空响应数增量。 - pub empty_responses: u64, - /// 图片工具输入 token 增量。 - pub image_input_tokens: u64, - /// 图片工具输出 token 增量。 - pub image_output_tokens: u64, - /// 图片工具成功请求数增量。 - pub image_requests: u64, - /// 图片工具失败请求数增量。 - pub image_request_failures: u64, -} - -/// 账号模型维度用量增量。 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct AccountModelUsageDelta { - /// 请求数增量。 - pub requests: u64, - /// 错误数增量。 - pub errors: u64, - /// 输入 token 增量。 - pub input_tokens: u64, - /// 输出 token 增量。 - pub output_tokens: u64, - /// 缓存 token 增量。 - pub cached_tokens: u64, -} diff --git a/backend/src/accounts/manage/cookies.rs b/backend/src/accounts/manage/cookies.rs deleted file mode 100644 index f301f37a1..000000000 --- a/backend/src/accounts/manage/cookies.rs +++ /dev/null @@ -1,11 +0,0 @@ -use super::AccountManageService; - -impl AccountManageService { - pub(crate) async fn usage_cookie_header(&self, account_id: &str) -> Option { - self.cookies - .cookie_header_for_request(account_id, "chatgpt.com", "/codex/usage") - .await - .ok() - .flatten() - } -} diff --git a/backend/src/accounts/manage/lifecycle.rs b/backend/src/accounts/manage/lifecycle.rs index d14d916fa..f084fe7da 100644 --- a/backend/src/accounts/manage/lifecycle.rs +++ b/backend/src/accounts/manage/lifecycle.rs @@ -238,25 +238,24 @@ impl AccountManageService { return Err(AccountManageError::EmptyIds); } let mut deleted = 0u32; + let mut deleted_ids = Vec::new(); let mut not_found = Vec::new(); for id in ids { match self.store.delete(&id).await { Ok(true) => { deleted += 1; self.account_pool.remove_account(&id).await; - if let Err(error) = self.session_affinity.forget_account(&id).await { - tracing::warn!( - account_id = id, - error = %error, - "failed to remove Redis affinities for deleted account" - ); - } + deleted_ids.push(id); } Ok(false) => not_found.push(id), Err(_) => return Err(AccountManageError::Delete), } } - Ok(BatchDeleteAccounts { deleted, not_found }) + Ok(BatchDeleteAccounts { + deleted, + not_found, + deleted_ids, + }) } pub async fn refresh_account( &self, diff --git a/backend/src/accounts/manage/mod.rs b/backend/src/accounts/manage/mod.rs index d41046aae..864c5337d 100644 --- a/backend/src/accounts/manage/mod.rs +++ b/backend/src/accounts/manage/mod.rs @@ -1,6 +1,5 @@ //! 账号管理用例。 -mod cookies; mod export; mod import; mod lifecycle; diff --git a/backend/src/accounts/manage/probe.rs b/backend/src/accounts/manage/probe.rs index 6a31123cf..66126489c 100644 --- a/backend/src/accounts/manage/probe.rs +++ b/backend/src/accounts/manage/probe.rs @@ -7,23 +7,16 @@ use serde_json::{json, Value}; use tokio::sync::mpsc; use crate::{ - accounts::{ - account::{Account, AccountStatus}, - store::StoredAccount, - }, - dispatch::{ - errors::{ - auth_failure_account_status, is_auth_upstream_error, is_quota_exhausted_upstream_error, - }, - responses::sse_failure::sse_failure_account_status, - }, - models::{info::CodexModelInfo, service::ModelRefreshPlanAccount}, + accounts::{account::AccountStatus, store::StoredAccount}, + models::{service::ModelRefreshPlanAccount, types::CodexModelInfo}, upstream::openai::{ protocol::{ responses::{CodexResponsesRequest, ResponsesSseFailure}, sse::{encode_sse_event, parse_sse_events, SseEvent}, }, - transport::{is_banned_upstream_error, CodexClientError, CodexRequestContext}, + transport::{ + is_banned_auth_signal, is_banned_upstream_error, CodexClientError, CodexRequestContext, + }, }, }; @@ -87,7 +80,8 @@ impl AccountManageService { let request_id = uuid::Uuid::new_v4().to_string(); let plan_account = ModelRefreshPlanAccount { plan_type: plan_type.to_string(), - account: stored_account_to_model_refresh_account(account.clone()), + access_token: account.access_token.expose_secret().to_string(), + account_id: account.account_id.clone(), }; if let Err(error) = self .models @@ -257,47 +251,6 @@ fn account_plan_type(account: &StoredAccount) -> String { .to_string() } -fn stored_account_to_model_refresh_account(stored: StoredAccount) -> Account { - Account { - id: stored.id, - email: stored.email, - account_id: stored.account_id, - user_id: stored.user_id, - label: stored.label, - plan_type: stored.plan_type, - access_token: stored.access_token.expose_secret().to_string(), - refresh_token: stored - .refresh_token - .map(|token| token.expose_secret().to_string()), - access_token_expires_at: stored.access_token_expires_at, - next_refresh_at: stored.next_refresh_at, - status: stored.status, - quota_limit_reached: false, - quota_verify_required: false, - quota_cooldown_until: None, - cloudflare_cooldown_until: None, - request_count: 0, - empty_response_count: 0, - image_input_tokens: 0, - image_output_tokens: 0, - image_request_count: 0, - image_request_failed_count: 0, - window_request_count: 0, - window_input_tokens: 0, - window_output_tokens: 0, - window_cached_tokens: 0, - window_image_input_tokens: 0, - window_image_output_tokens: 0, - window_image_request_count: 0, - window_image_request_failed_count: 0, - window_started_at: None, - window_reset_at: None, - limit_window_seconds: None, - added_at: stored.added_at, - last_used_at: None, - } -} - fn test_responses_request(model: String) -> CodexResponsesRequest { let mut request = CodexResponsesRequest::new_http_sse( model, @@ -396,15 +349,54 @@ async fn process_sse_event( fn account_status_from_client_error(error: &CodexClientError) -> Option { if is_banned_upstream_error(error) { Some(AccountStatus::Banned) - } else if is_quota_exhausted_upstream_error(error) { - Some(AccountStatus::QuotaExhausted) - } else if is_auth_upstream_error(error) { - Some(auth_failure_account_status(error)) } else { - None + match error { + CodexClientError::Upstream { status, .. } if status.as_u16() == 402 => { + Some(AccountStatus::QuotaExhausted) + } + CodexClientError::Upstream { status, body, .. } if status.as_u16() == 401 => { + Some(if is_banned_auth_signal(body) { + AccountStatus::Banned + } else { + AccountStatus::Expired + }) + } + _ => None, + } } } +fn sse_failure_account_status(failure: &ResponsesSseFailure) -> Option { + let code = failure + .upstream_code + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase(); + let message = failure.message.to_ascii_lowercase(); + if matches!(code.as_str(), "quota_exceeded" | "insufficient_quota") || message.contains("quota") + { + return Some(AccountStatus::QuotaExhausted); + } + let authentication_failed = matches!( + code.as_str(), + "token_invalid" + | "token_expired" + | "token_revoked" + | "account_deactivated" + | "unauthorized" + | "invalid_api_key" + ) || message.contains("token revoked") + || message.contains("token invalid") + || message.contains("token expired"); + authentication_failed.then(|| { + if is_banned_auth_signal(&failure.message) { + AccountStatus::Banned + } else { + AccountStatus::Expired + } + }) +} + fn take_sse_frame(buffer: &mut String) -> Option { let index = buffer.find("\n\n").or_else(|| buffer.find("\r\n\r\n"))?; let delimiter_len = if buffer[index..].starts_with("\r\n\r\n") { diff --git a/backend/src/accounts/manage/quota.rs b/backend/src/accounts/manage/quota.rs index 6dfeab9cc..6bc9320a9 100644 --- a/backend/src/accounts/manage/quota.rs +++ b/backend/src/accounts/manage/quota.rs @@ -3,6 +3,23 @@ use secrecy::ExposeSecret; use super::{types::AccountManageError, AccountManageService}; impl AccountManageService { + async fn usage_cookie_header(&self, account_id: &str) -> Option { + self.cookies + .cookie_header_for_request(account_id, "chatgpt.com", "/codex/usage") + .await + .ok() + .flatten() + } + + pub async fn quota_snapshots( + &self, + ) -> Result, AccountManageError> { + self.store + .list_quota_snapshots() + .await + .map_err(|_| AccountManageError::Inspect) + } + pub async fn account_quota( &self, account_id: &str, diff --git a/backend/src/accounts/manage/service.rs b/backend/src/accounts/manage/service.rs index 28c2879e3..707810c89 100644 --- a/backend/src/accounts/manage/service.rs +++ b/backend/src/accounts/manage/service.rs @@ -14,7 +14,6 @@ use crate::{ refresh::{RedisRefreshLeaseStore, RuntimeRefreshPolicy}, store::PgAccountStore, }, - dispatch::affinity::RedisSessionAffinityStore, models::service::ModelService, upstream::openai::{token_client::TokenRefresher, transport::CodexBackendClient}, }; @@ -30,7 +29,6 @@ pub struct AccountManageService { pub(crate) account_pool: Arc, pub(crate) token_refresher: Arc, pub(crate) refresh_leases: RedisRefreshLeaseStore, - pub(crate) session_affinity: RedisSessionAffinityStore, pub(crate) refresh_lease_owner_prefix: String, pub(crate) oauth: oauth::AccountOAuthService, pub(crate) refresh_policy: RuntimeRefreshPolicy, @@ -45,7 +43,6 @@ pub struct AccountManageServiceParts { pub account_pool: Arc, pub token_refresher: Arc, pub refresh_leases: RedisRefreshLeaseStore, - pub session_affinity: RedisSessionAffinityStore, pub oauth: oauth::AccountOAuthService, pub refresh_policy: RuntimeRefreshPolicy, pub installation_id: Option, @@ -61,7 +58,6 @@ impl AccountManageService { account_pool: parts.account_pool, token_refresher: parts.token_refresher, refresh_leases: parts.refresh_leases, - session_affinity: parts.session_affinity, refresh_lease_owner_prefix: format!( "admin-account-refresh:{}", Uuid::new_v4().simple() diff --git a/backend/src/accounts/manage/types.rs b/backend/src/accounts/manage/types.rs index c8f230f97..cc5c1a94b 100644 --- a/backend/src/accounts/manage/types.rs +++ b/backend/src/accounts/manage/types.rs @@ -176,6 +176,7 @@ impl AccountUpdate { pub struct BatchDeleteAccounts { pub deleted: u32, pub not_found: Vec, + pub(crate) deleted_ids: Vec, } #[derive(Debug, Clone)] diff --git a/backend/src/accounts/mod.rs b/backend/src/accounts/mod.rs index e72bb1666..79b535847 100644 --- a/backend/src/accounts/mod.rs +++ b/backend/src/accounts/mod.rs @@ -8,6 +8,5 @@ pub mod pool; pub mod quota; pub mod refresh; pub mod scheduler; -pub mod service; pub mod store; mod window; diff --git a/backend/src/accounts/pool.rs b/backend/src/accounts/pool.rs index 6703b7d64..f9da2c973 100644 --- a/backend/src/accounts/pool.rs +++ b/backend/src/accounts/pool.rs @@ -17,10 +17,13 @@ use crate::accounts::scheduler::{ AccountScheduler, ScoreWeights, }; use crate::accounts::store::AccountStore; -use crate::accounts::window::should_reset_usage_window; use crate::accounts::{ - account::{Account, AccountModelUsageDelta, AccountStatus, AccountUsageDelta}, - service::AccountService, + account::{Account, AccountStatus}, + window::should_reset_usage_window, +}; +use crate::telemetry::account_usage::store::{ + AccountModelUsageDelta, AccountUsageDelta, AccountUsageSnapshot, AccountUsageStore, + AccountUsageWindow, }; use crate::upstream::openai::protocol::events::{ parse_rate_limit_headers, rate_limit_quota, TokenUsage as CodexTokenUsage, @@ -528,7 +531,7 @@ impl AccountPool { let refreshed_accounts = self.refresh_account_statuses(now); let is_capacity_account = |account: &Account| { account.status == AccountStatus::Active - && AccountService::quota_available_at(account, now, self.options.skip_quota_limited) + && candidates::quota_available_at(account, now, self.options.skip_quota_limited) }; let active_accounts = self .accounts diff --git a/backend/src/accounts/pool/runtime.rs b/backend/src/accounts/pool/runtime.rs index f50532501..987f563b2 100644 --- a/backend/src/accounts/pool/runtime.rs +++ b/backend/src/accounts/pool/runtime.rs @@ -1,9 +1,36 @@ use super::*; +/// 账号运行参数中不支持热更新的部分。 +#[derive(Debug, Clone)] +pub struct AccountPoolStaticSettings { + pub skip_quota_limited: bool, + pub tier_priority: Vec, +} + +impl AccountPoolStaticSettings { + /// 将设置快照转换为账号池选项。 + pub fn pool_options(&self, settings: &crate::settings::SettingsSnapshot) -> AccountPoolOptions { + AccountPoolOptions { + max_concurrent_per_account: settings.max_concurrent_per_account, + rotation_strategy: match settings.rotation_strategy.as_str() { + "smart" => RotationStrategy::Smart, + "quota_reset_priority" => RotationStrategy::QuotaResetPriority, + "round_robin" => RotationStrategy::RoundRobin, + "sticky" => RotationStrategy::Sticky, + _ => RotationStrategy::Smart, + }, + skip_quota_limited: self.skip_quota_limited, + tier_priority: self.tier_priority.clone(), + ..AccountPoolOptions::default() + } + } +} + #[derive(Clone)] pub struct RuntimeAccountPoolService { pool: Arc>, store: Arc, + usage_store: Arc, request_interval: Arc>, } @@ -11,12 +38,14 @@ impl RuntimeAccountPoolService { /// 构造运行时账号池服务。 pub fn new( store: Arc, + usage_store: Arc, options: AccountPoolOptions, request_interval_ms: u64, ) -> Self { Self { pool: Arc::new(tokio::sync::Mutex::new(AccountPool::with_options(options))), store, + usage_store, request_interval: Arc::new(std::sync::RwLock::new(StdDuration::from_millis( request_interval_ms, ))), @@ -36,6 +65,22 @@ impl RuntimeAccountPoolService { StdDuration::from_millis(request_interval_ms); } + /// 持续接收运行时设置并更新账号池参数。 + pub async fn subscribe_settings( + self: Arc, + mut receiver: tokio::sync::watch::Receiver, + static_settings: AccountPoolStaticSettings, + ) { + while receiver.changed().await.is_ok() { + let settings = receiver.borrow_and_update().clone(); + self.apply_options( + static_settings.pool_options(&settings), + settings.request_interval_ms, + ) + .await; + } + } + /// 更新模型到可用账号计划的调度约束。 pub(crate) async fn apply_model_plan_routing( &self, @@ -49,11 +94,25 @@ impl RuntimeAccountPoolService { /// 从持久化账号恢复运行时账号池。 pub async fn restore_from_store(&self) -> Result { - let accounts = self + let mut accounts = self .store .list_pool_accounts() .await .map_err(|_| RuntimeAccountPoolError::Generic)?; + let account_ids = accounts + .iter() + .map(|account| account.id.clone()) + .collect::>(); + let mut usage = self + .usage_store + .snapshots(&account_ids) + .await + .map_err(|_| RuntimeAccountPoolError::Generic)?; + for account in &mut accounts { + if let Some(snapshot) = usage.remove(&account.id) { + apply_usage_snapshot(account, snapshot); + } + } let count = accounts.len(); let mut pool = self.pool.lock().await; pool.clear(); @@ -105,7 +164,7 @@ impl RuntimeAccountPoolService { if !record_request_usage { return; } - if let Err(error) = self.store.record_request(account_id).await { + if let Err(error) = self.usage_store.record_request(account_id).await { tracing::warn!( account_id, error = %error, @@ -120,7 +179,7 @@ impl RuntimeAccountPoolService { return; }; if let Err(error) = self - .store + .usage_store .record_model_usage_delta( account_id, &model, @@ -202,7 +261,7 @@ impl RuntimeAccountPoolService { for refreshed in refreshed_states { if let Err(error) = self .store - .sync_runtime_account_state(&refreshed.account, refreshed.sync_usage_window) + .sync_runtime_account_state(&refreshed.account) .await { tracing::warn!( @@ -211,6 +270,22 @@ impl RuntimeAccountPoolService { "failed to persist refreshed runtime account state" ); } + if refreshed.sync_usage_window { + if let Err(error) = self + .usage_store + .sync_runtime_window( + &refreshed.account.id, + account_usage_window(&refreshed.account), + ) + .await + { + tracing::warn!( + account_id = %refreshed.account.id, + error = %error, + "failed to persist refreshed runtime account usage window" + ); + } + } } } @@ -264,7 +339,7 @@ impl RuntimeAccountPoolService { ..AccountUsageDelta::default() }; if let Err(error) = self - .store + .usage_store .record_usage_delta(account_id, persisted_usage) .await { @@ -275,7 +350,7 @@ impl RuntimeAccountPoolService { ); } if let Err(error) = self - .store + .usage_store .record_model_usage_delta( account_id, model, @@ -321,7 +396,7 @@ impl RuntimeAccountPoolService { image_request_failures: u64::from(image_generation_requested), ..AccountUsageDelta::default() }; - if let Err(error) = self.store.record_usage_delta(account_id, usage).await { + if let Err(error) = self.usage_store.record_usage_delta(account_id, usage).await { tracing::warn!( account_id, error = %error, @@ -329,7 +404,7 @@ impl RuntimeAccountPoolService { ); } if let Err(error) = self - .store + .usage_store .record_model_usage_delta( account_id, model, @@ -429,7 +504,7 @@ impl RuntimeAccountPoolService { if let Some(reset_at) = reset_at { if let Err(error) = self - .store + .usage_store .sync_rate_limit_window(account_id, reset_at, limit_window_seconds) .await { @@ -468,12 +543,20 @@ impl RuntimeAccountPoolService { .get_pool_account(account_id) .await .map_err(|_| RuntimeAccountPoolError::Generic)?; - let mut pool = self.pool.lock().await; - if let Some(account) = account { - pool.insert(account); + if let Some(mut account) = account { + let account_ids = vec![account.id.clone()]; + let mut usage = self + .usage_store + .snapshots(&account_ids) + .await + .map_err(|_| RuntimeAccountPoolError::Generic)?; + if let Some(snapshot) = usage.remove(&account.id) { + apply_usage_snapshot(&mut account, snapshot); + } + self.pool.lock().await.insert(account); return Ok(true); } - Ok(pool.remove(account_id)) + Ok(self.pool.lock().await.remove(account_id)) } /// 设置 Cloudflare 冷却状态。 @@ -506,6 +589,45 @@ impl RuntimeAccountPoolService { } } +fn apply_usage_snapshot(account: &mut Account, snapshot: AccountUsageSnapshot) { + account.request_count = snapshot.request_count; + account.empty_response_count = snapshot.empty_response_count; + account.image_input_tokens = snapshot.image_input_tokens; + account.image_output_tokens = snapshot.image_output_tokens; + account.image_request_count = snapshot.image_request_count; + account.image_request_failed_count = snapshot.image_request_failed_count; + account.window_request_count = snapshot.window_request_count; + account.window_input_tokens = snapshot.window_input_tokens; + account.window_output_tokens = snapshot.window_output_tokens; + account.window_cached_tokens = snapshot.window_cached_tokens; + account.window_image_input_tokens = snapshot.window_image_input_tokens; + account.window_image_output_tokens = snapshot.window_image_output_tokens; + account.window_image_request_count = snapshot.window_image_request_count; + account.window_image_request_failed_count = snapshot.window_image_request_failed_count; + account.window_started_at = snapshot.window_started_at; + account.window_reset_at = snapshot.window_reset_at.or(account.window_reset_at); + account.limit_window_seconds = snapshot + .limit_window_seconds + .or(account.limit_window_seconds); + account.last_used_at = snapshot.last_used_at.map(|value| value.to_rfc3339()); +} + +fn account_usage_window(account: &Account) -> AccountUsageWindow { + AccountUsageWindow { + request_count: account.window_request_count, + input_tokens: account.window_input_tokens, + output_tokens: account.window_output_tokens, + cached_tokens: account.window_cached_tokens, + image_input_tokens: account.window_image_input_tokens, + image_output_tokens: account.window_image_output_tokens, + image_request_count: account.window_image_request_count, + image_request_failed_count: account.window_image_request_failed_count, + started_at: account.window_started_at, + reset_at: account.window_reset_at, + limit_window_seconds: account.limit_window_seconds, + } +} + /// 运行时账号池错误。 #[derive(Debug, Error)] pub enum RuntimeAccountPoolError { diff --git a/backend/src/accounts/quota/runtime.rs b/backend/src/accounts/quota/runtime.rs index 5843150e4..4b77681af 100644 --- a/backend/src/accounts/quota/runtime.rs +++ b/backend/src/accounts/quota/runtime.rs @@ -18,6 +18,9 @@ use crate::accounts::{ }; use crate::{ accounts::cookies::PgCookieStore, + telemetry::account_usage::store::{ + AccountUsageStore, AccountUsageStoreError, PgAccountUsageStore, + }, upstream::openai::transport::{CodexBackendClient, CodexClientError, CodexRequestContext}, }; @@ -28,6 +31,7 @@ const COOLDOWN_REFRESH_GRACE_SECS: i64 = 30; /// 运行时 quota refresh 服务。 pub struct RuntimeQuotaRefreshService { store: PgAccountStore, + usage_store: PgAccountUsageStore, codex: Arc, min_refresh_interval_secs: u64, request_spacing: Duration, @@ -38,9 +42,14 @@ pub struct RuntimeQuotaRefreshService { impl RuntimeQuotaRefreshService { /// 构造默认 quota refresh 服务。 - pub fn new(store: PgAccountStore, codex: Arc) -> Self { + pub fn new( + store: PgAccountStore, + usage_store: PgAccountUsageStore, + codex: Arc, + ) -> Self { Self { store, + usage_store, codex, min_refresh_interval_secs: MIN_REFRESH_INTERVAL_SECS, request_spacing: Self::default_request_spacing(), @@ -57,11 +66,13 @@ impl RuntimeQuotaRefreshService { /// 使用自定义最小刷新间隔构造 quota refresh 服务。 pub fn with_min_refresh_interval_secs( store: PgAccountStore, + usage_store: PgAccountUsageStore, codex: Arc, min_refresh_interval_secs: u64, ) -> Self { Self { store, + usage_store, codex, min_refresh_interval_secs, request_spacing: Self::default_request_spacing(), @@ -182,7 +193,7 @@ impl RuntimeQuotaRefreshService { summary.missing += 1; } if let Some(reset_at) = reset_at { - self.store + self.usage_store .sync_rate_limit_window( &account.id, reset_at, @@ -267,6 +278,9 @@ pub enum QuotaRefreshServiceError { /// 账号写入失败。 #[error("failed to persist quota refresh result: {0}")] Store(#[from] PgAccountStoreError), + /// 用量窗口写入失败。 + #[error("failed to persist quota usage window: {0}")] + UsageStore(#[from] AccountUsageStoreError), /// 上游 usage 请求失败。 #[error("failed to fetch quota usage: {0}")] Codex(#[from] CodexClientError), diff --git a/backend/src/accounts/refresh/mod.rs b/backend/src/accounts/refresh/mod.rs index 2cdc80f73..efd6a1c6e 100644 --- a/backend/src/accounts/refresh/mod.rs +++ b/backend/src/accounts/refresh/mod.rs @@ -1,9 +1,9 @@ //! Token 刷新策略、租约与运行时服务。 mod lease; -mod runtime; +mod policy; mod service; pub use lease::*; -pub use runtime::*; +pub use policy::*; pub use service::*; diff --git a/backend/src/accounts/refresh/policy.rs b/backend/src/accounts/refresh/policy.rs new file mode 100644 index 000000000..a0f9337a6 --- /dev/null +++ b/backend/src/accounts/refresh/policy.rs @@ -0,0 +1,485 @@ +//! 刷新租约存储、JWT 解码与账号 claims 验证。 + +use std::{ + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, + sync::{Arc, RwLock}, + time::Duration as StdDuration, +}; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use chrono::{DateTime, Duration, TimeZone, Utc}; +use serde_json::{Map, Value}; +use tokio::sync::{watch, Semaphore}; + +use crate::accounts::account::{Account, AccountStatus}; +use crate::upstream::openai::token_client::{RefreshFailure, TokenPair, TokenRefresher}; + +pub(crate) const PERMANENT_FAILURE_CONFIRMATION_THRESHOLD: usize = 2; +pub(crate) const RECOVERY_DELAY_SECONDS: i64 = 10 * 60; +pub(crate) const RETRY_DELAY_JITTER: f64 = 0.30; +pub(crate) const RECOVERY_DELAY_JITTER: f64 = 0.20; + +const MAX_REFRESH_ATTEMPTS: usize = 5; +const REFRESH_RETRY_BASE_DELAY_MILLIS: u64 = 5_000; +const REFRESH_RETRY_MAX_DELAY_MILLIS: u64 = 300_000; + +// --------------------------------------------------------------------------- +// JWT 解码 +// --------------------------------------------------------------------------- + +/// JWT 过期状态。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JwtExpiry { + /// token 已过期。 + Expired, + /// token 仍然有效。 + Valid, + /// token 缺失、格式错误或不包含可解析的 exp。 + MissingOrInvalid, +} + +/// 按给定时间点判断 JWT 的 `exp` 是否已过期。 +pub fn jwt_expiry(token: &str, now: DateTime) -> JwtExpiry { + let Some(exp) = jwt_exp(token) else { + return JwtExpiry::MissingOrInvalid; + }; + if now.timestamp() >= exp { + JwtExpiry::Expired + } else { + JwtExpiry::Valid + } +} + +/// 读取 JWT `exp` 并转换成 UTC 时间。 +pub fn jwt_expiration(token: &str) -> Option> { + Utc.timestamp_opt(jwt_exp(token)?, 0).single() +} + +fn jwt_exp(token: &str) -> Option { + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + let _signature = parts.next()?; + if parts.next().is_some() { + return None; + } + + let decoded = URL_SAFE_NO_PAD.decode(payload).ok()?; + let value = serde_json::from_slice::(&decoded).ok()?; + value.get("exp")?.as_i64() +} + +// --------------------------------------------------------------------------- +// 账号 claims 解码（手动创建/导入时验证 JWT） +// --------------------------------------------------------------------------- + +/// 手动创建账号时从 JWT 提取的 claims。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManualAccountClaims { + /// ChatGPT 账号 ID。 + pub account_id: Option, + /// ChatGPT 用户 ID。 + pub user_id: Option, + /// 邮箱。 + pub email: Option, + /// 订阅计划类型。 + pub plan_type: Option, + /// access token 过期时间。 + pub expires_at: DateTime, +} + +/// 从 JWT 中解码 payload 部分。 +pub fn decode_jwt_payload(token: &str) -> Option> { + let mut parts = token.split('.'); + let _header = parts.next()?; + let payload = parts.next()?; + if payload.is_empty() { + return None; + } + let bytes = URL_SAFE_NO_PAD.decode(payload).ok()?; + serde_json::from_slice::(&bytes) + .ok()? + .as_object() + .cloned() +} + +/// 从 JWT 中提取手动创建账号所需的 claims。 +pub fn manual_account_claims( + token: &str, + now: DateTime, +) -> Result { + let payload = decode_jwt_payload(token).ok_or("Invalid JWT format")?; + let exp = payload + .get("exp") + .and_then(Value::as_i64) + .ok_or("Token is expired")?; + if now.timestamp() >= exp { + return Err("Token is expired"); + } + let expires_at = DateTime::::from_timestamp(exp, 0).ok_or("Invalid JWT exp claim")?; + let auth = payload + .get("https://api.openai.com/auth") + .and_then(Value::as_object) + .ok_or("Token missing OpenAI auth claim")?; + let account_id = string_claim(auth, "chatgpt_account_id"); + let profile = payload + .get("https://api.openai.com/profile") + .and_then(Value::as_object); + let user_id = string_claim(auth, "chatgpt_user_id") + .or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_user_id"))) + .or_else(|| string_claim(auth, "user_id")) + .or_else(|| profile.and_then(|profile| string_claim(profile, "user_id"))); + let plan_type = string_claim(auth, "chatgpt_plan_type") + .or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_plan_type"))); + let email = profile.and_then(|profile| string_claim(profile, "email")); + + Ok(ManualAccountClaims { + account_id, + user_id, + email, + plan_type, + expires_at, + }) +} + +fn string_claim(map: &Map, key: &str) -> Option { + map.get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + +// --------------------------------------------------------------------------- +// 刷新策略和上游端口 +// --------------------------------------------------------------------------- + +/// 刷新任务的调度策略。 +#[derive(Debug, Clone, Copy)] +pub struct RefreshPolicy { + /// 提前多久开始刷新访问令牌。 + pub refresh_margin_seconds: u64, + /// 允许并发执行的刷新任务数。 + pub refresh_concurrency: u32, +} + +/// 单次刷新扫描摘要。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TokenRefreshSummary { + /// 扫描账号数。 + pub scanned: usize, + /// 成功刷新 token 的账号数。 + pub refreshed: usize, + /// 仅更新状态的账号数。 + pub status_updated: usize, + /// 无需刷新的账号数。 + pub skipped: usize, + /// 传输失败账号数。 + pub failed: usize, +} + +/// 账号级刷新定时器调度摘要。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TokenTimerSummary { + /// 扫描账号数。 + pub scanned: usize, + /// 已调度的未来刷新定时器数。 + pub scheduled: usize, + /// 已调度的立即刷新定时器数。 + pub immediate: usize, + /// 无需调度的账号数。 + pub skipped: usize, + /// 被替换的既有定时器数。 + pub replaced: usize, +} + +impl TokenTimerSummary { + /// 返回本轮调度发生变化的定时器数量。 + pub fn changed(&self) -> usize { + self.scheduled + self.immediate + } +} + +/// 运行时共享刷新策略。 +#[derive(Clone)] +pub struct RuntimeRefreshPolicy { + policy: Arc>, + semaphore: Arc>>, +} + +impl RuntimeRefreshPolicy { + /// 构造共享刷新策略。 + pub fn new(policy: RefreshPolicy) -> Self { + Self { + policy: Arc::new(RwLock::new(policy)), + semaphore: Arc::new(RwLock::new(Arc::new(Semaphore::new( + policy.refresh_concurrency.max(1) as usize, + )))), + } + } + + /// 返回当前策略快照。 + pub fn snapshot(&self) -> RefreshPolicy { + *self + .policy + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// 更新当前策略。 + pub fn update(&self, policy: RefreshPolicy) { + *self + .policy + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = policy; + *self + .semaphore + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Arc::new(Semaphore::new(policy.refresh_concurrency.max(1) as usize)); + } + + /// 返回当前提前刷新秒数。 + pub fn refresh_margin_seconds(&self) -> u64 { + self.snapshot().refresh_margin_seconds + } + + /// 持续接收运行时设置并更新刷新策略。 + pub async fn subscribe_settings( + self, + mut receiver: watch::Receiver, + ) { + while receiver.changed().await.is_ok() { + let settings = receiver.borrow_and_update(); + self.update(RefreshPolicy { + refresh_margin_seconds: settings.refresh_margin_seconds, + refresh_concurrency: settings.refresh_concurrency, + }); + } + } + + fn semaphore(&self) -> Arc { + self.semaphore + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } +} + +impl From for RuntimeRefreshPolicy { + fn from(policy: RefreshPolicy) -> Self { + Self::new(policy) + } +} + +/// 按账号稳定扰动刷新提前量，范围与 TS 版 `jitter(margin, 0.15)` 一致。 +pub(crate) fn jittered_refresh_at( + account_id: &str, + expires_at: DateTime, + margin_seconds: u64, +) -> DateTime { + expires_at - Duration::seconds(jittered_refresh_margin_seconds(account_id, margin_seconds)) +} + +fn jittered_refresh_margin_seconds(account_id: &str, margin_seconds: u64) -> i64 { + if margin_seconds == 0 { + return 0; + } + + let mut hasher = DefaultHasher::new(); + account_id.hash(&mut hasher); + let unit = hasher.finish() as f64 / u64::MAX as f64; + let factor = 0.85 + unit * 0.30; + (margin_seconds as f64 * factor) + .round() + .clamp(0.0, i64::MAX as f64) as i64 +} + +/// 调度器自身的执行错误。 +#[derive(Debug, thiserror::Error)] +pub enum RefreshError { + /// 并发限制信号量已关闭。 + #[error("refresh task semaphore closed")] + ConcurrencyClosed, + /// 刷新请求在到达服务端前失败，可安全复用当前 refresh token 重试。 + #[error("refresh transport failed before server processing")] + RetryableTransport, + /// 刷新请求在传输层失败，refresh token 可能已经被服务端消费。 + #[error("refresh transport failed after possible server processing")] + Transport, +} + +/// 负责执行单账号刷新策略的调度器。 +#[derive(Clone)] +pub struct RefreshScheduler { + policy: RuntimeRefreshPolicy, + client: Arc, +} + +impl RefreshScheduler +where + C: TokenRefresher, +{ + /// 使用策略和上游刷新端口构造调度器。 + pub fn new(policy: impl Into, client: C) -> Self { + Self { + policy: policy.into(), + client: Arc::new(client), + } + } + + /// 更新刷新策略。 + pub fn update_policy(&self, policy: RefreshPolicy) { + self.policy.update(policy); + } + + /// 在给定时间点按到期策略刷新账号。 + /// + /// 当账号不需要刷新时返回原账号快照；当刷新失败但属于可映射的领域错误时， + /// 返回更新过状态的账号。 + /// + /// # Errors + /// + /// 当并发控制已关闭或刷新传输失败时返回 [`RefreshError`]。 + pub async fn refresh_account_at( + &self, + account: &Account, + now: DateTime, + ) -> Result { + if !self.should_refresh_account_at(account, now) { + return Ok(account.clone()); + } + + let Some(refresh_token) = account.refresh_token.as_deref() else { + let mut expired = account.clone(); + expired.status = AccountStatus::Expired; + return Ok(expired); + }; + + let semaphore = self.policy.semaphore(); + let _permit = semaphore + .acquire() + .await + .map_err(|_| RefreshError::ConcurrencyClosed)?; + + match self.client.refresh(refresh_token).await { + Ok(token_pair) => Ok(apply_token_pair(account, token_pair)), + Err(RefreshFailure::RetryableTransport) => Err(RefreshError::RetryableTransport), + Err(RefreshFailure::Transport) => Err(RefreshError::Transport), + Err(error) => Ok(apply_refresh_failure(account, error)), + } + } + + /// 判断账号在给定时间点是否需要刷新。 + pub fn should_refresh_account_at(&self, account: &Account, now: DateTime) -> bool { + token_refresh_status_eligible(account.status) + && self.should_refresh_before_expiry(account, now) + } + + fn should_refresh_before_expiry(&self, account: &Account, now: DateTime) -> bool { + if let Some(next_refresh_at) = account.next_refresh_at { + return next_refresh_at <= now; + } + + account + .access_token_expires_at + .is_some_and(|expires_at| self.refresh_at(account, expires_at) <= now) + } + + fn refresh_at(&self, account: &Account, expires_at: DateTime) -> DateTime { + let policy = self.policy.snapshot(); + jittered_refresh_at( + account.id.as_str(), + expires_at, + policy.refresh_margin_seconds, + ) + } +} + +pub(crate) fn token_refresh_status_eligible(status: AccountStatus) -> bool { + matches!( + status, + AccountStatus::Active | AccountStatus::QuotaExhausted + ) +} + +/// 将新的 token 对应用到账号快照上。 +pub fn apply_token_pair(account: &Account, token_pair: TokenPair) -> Account { + let mut refreshed = account.clone(); + refreshed.access_token = token_pair.access_token; + + // 刷新响应不返回 refresh_token 时，继续保留已有值，避免永久失去刷新能力。 + if let Some(refresh_token) = token_pair.refresh_token { + refreshed.refresh_token = Some(refresh_token); + } + + refreshed +} + +/// 将刷新失败映射为账号状态变更。 +pub fn apply_refresh_failure(account: &Account, failure: RefreshFailure) -> Account { + let mut updated = account.clone(); + updated.status = match failure { + RefreshFailure::InvalidGrant => AccountStatus::Expired, + RefreshFailure::Banned => AccountStatus::Banned, + RefreshFailure::RetryableTransport | RefreshFailure::Transport => account.status, + }; + updated +} + +pub(crate) fn is_permanent_refresh_failure_status(status: AccountStatus) -> bool { + matches!(status, AccountStatus::Expired | AccountStatus::Banned) +} + +pub(crate) fn default_refresh_retry_delays() -> Vec { + (0..MAX_REFRESH_ATTEMPTS.saturating_sub(1)) + .map(|attempt_index| { + let multiplier = 3_u64.saturating_pow(attempt_index as u32); + let millis = REFRESH_RETRY_BASE_DELAY_MILLIS + .saturating_mul(multiplier) + .min(REFRESH_RETRY_MAX_DELAY_MILLIS); + StdDuration::from_millis(millis) + }) + .collect() +} + +pub(crate) fn stable_jittered_duration( + account_id: &str, + base: StdDuration, + variance: f64, + salt: &str, +) -> StdDuration { + if base.is_zero() { + return StdDuration::ZERO; + } + + let mut hasher = DefaultHasher::new(); + account_id.hash(&mut hasher); + salt.hash(&mut hasher); + let unit = hasher.finish() as f64 / u64::MAX as f64; + let factor = (1.0 - variance) + unit * variance * 2.0; + let millis = (base.as_millis() as f64 * factor) + .round() + .clamp(0.0, u64::MAX as f64) as u64; + StdDuration::from_millis(millis) +} + +pub(crate) fn scheduled_at_from_delay(now: DateTime, delay: StdDuration) -> DateTime { + match chrono::Duration::from_std(delay) { + Ok(delay) => now + delay, + Err(_) => now, + } +} + +pub(crate) fn duration_until(now: DateTime, scheduled_at: DateTime) -> StdDuration { + (scheduled_at - now).to_std().unwrap_or(StdDuration::ZERO) +} + +pub(crate) fn same_refresh_time(left: Option>, right: Option>) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.timestamp() == right.timestamp(), + (None, None) => true, + _ => false, + } +} diff --git a/backend/src/accounts/refresh/runtime.rs b/backend/src/accounts/refresh/runtime.rs deleted file mode 100644 index 5508d4bba..000000000 --- a/backend/src/accounts/refresh/runtime.rs +++ /dev/null @@ -1,777 +0,0 @@ -//! 运行时 token refresh 业务服务。 - -use std::{ - collections::{hash_map::DefaultHasher, HashMap, HashSet}, - future::Future, - hash::{Hash, Hasher}, - pin::Pin, - sync::Arc, - time::Duration, -}; - -use chrono::{DateTime, Utc}; -use secrecy::{ExposeSecret, SecretString}; -use thiserror::Error; -use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; -use tracing::{debug, info, warn}; -use uuid::Uuid; - -use crate::accounts::{ - account::{Account, AccountStatus}, - refresh::{ - jittered_refresh_at, jwt_expiration, token_refresh_status_eligible, RedisRefreshLeaseStore, - RedisRefreshLeaseStoreError, RefreshError, RefreshPolicy, RefreshScheduler, - RuntimeRefreshPolicy, - }, - store::{ - AccountClaimsUpdate, AccountStore, PgAccountStore, PgAccountStoreError, StoredAccount, - }, -}; -use crate::upstream::openai::token_client::TokenRefresher; - -const REFRESH_LEASE_TTL_SECONDS: i64 = 5 * 60; -const MAX_REFRESH_ATTEMPTS: usize = 5; -const REFRESH_RETRY_BASE_DELAY_MILLIS: u64 = 5_000; -const REFRESH_RETRY_MAX_DELAY_MILLIS: u64 = 300_000; -const PERMANENT_FAILURE_CONFIRMATION_THRESHOLD: usize = 2; -const RECOVERY_DELAY_SECONDS: i64 = 10 * 60; -const RETRY_DELAY_JITTER: f64 = 0.30; -const RECOVERY_DELAY_JITTER: f64 = 0.20; - -/// 运行时 token refresh 服务。 -pub struct RuntimeTokenRefreshService -where - C: TokenRefresher, -{ - store: PgAccountStore, - scheduler: Arc>, - policy: RuntimeRefreshPolicy, - refresh_leases: Option, - lease_owner: String, - retry_delays: Vec, - in_flight: Arc>>, - timers: Arc>>, -} - -struct ScheduledRefreshTimer { - scheduled_at: DateTime, - handle: JoinHandle<()>, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ScheduleTimerResult { - Scheduled, - Replaced, - Unchanged, -} - -impl Clone for RuntimeTokenRefreshService -where - C: TokenRefresher, -{ - fn clone(&self) -> Self { - Self { - store: self.store.clone(), - scheduler: self.scheduler.clone(), - policy: self.policy.clone(), - refresh_leases: self.refresh_leases.clone(), - lease_owner: self.lease_owner.clone(), - retry_delays: self.retry_delays.clone(), - in_flight: self.in_flight.clone(), - timers: self.timers.clone(), - } - } -} - -impl RuntimeTokenRefreshService -where - C: TokenRefresher, -{ - /// 构造运行时 token refresh 服务。 - pub fn new(store: PgAccountStore, policy: impl Into, client: C) -> Self { - let policy = policy.into(); - Self { - store, - scheduler: Arc::new(RefreshScheduler::new(policy.clone(), client)), - policy, - refresh_leases: None, - lease_owner: refresh_lease_owner(), - retry_delays: default_refresh_retry_delays(), - in_flight: Arc::new(Mutex::new(HashSet::new())), - timers: Arc::new(Mutex::new(HashMap::new())), - } - } - - /// 使用刷新租约存储保护账号刷新。 - pub fn with_refresh_lease_store(mut self, refresh_leases: RedisRefreshLeaseStore) -> Self { - self.refresh_leases = Some(refresh_leases); - self - } - - /// 更新刷新策略。 - pub fn update_policy(&self, policy: RefreshPolicy) { - self.policy.update(policy); - } - - /// 使用自定义刷新重试延迟。 - pub fn with_retry_delays(mut self, retry_delays: I) -> Self - where - I: IntoIterator, - { - self.retry_delays = retry_delays.into_iter().collect(); - self - } - - /// 返回给定账号集合中当前正在执行 token 刷新的账号 ID。 - pub async fn refreshing_account_ids( - &self, - account_ids: &[String], - now: DateTime, - ) -> TokenRefreshServiceResult> { - if account_ids.is_empty() { - return Ok(HashSet::new()); - } - - let requested_ids = account_ids - .iter() - .map(String::as_str) - .collect::>(); - let mut refreshing = self - .in_flight - .lock() - .await - .iter() - .filter(|account_id| requested_ids.contains(account_id.as_str())) - .cloned() - .collect::>(); - - if let Some(refresh_leases) = self.refresh_leases.as_ref() { - refreshing.extend( - refresh_leases - .active_account_ids(account_ids, now) - .await - .map_err(TokenRefreshServiceError::Lease)?, - ); - } - - Ok(refreshing) - } - - /// 执行一次账号级刷新定时器调度。 - pub async fn schedule_account_timers_once( - &self, - ) -> TokenRefreshServiceResult { - self.schedule_account_timers_once_at(Utc::now()).await - } - - /// 在指定时间点执行一次账号级刷新定时器调度。 - pub async fn schedule_account_timers_once_at( - &self, - now: DateTime, - ) -> TokenRefreshServiceResult { - let accounts = self - .store - .list_pool_accounts() - .await - .map_err(TokenRefreshServiceError::AccountStore)?; - let mut summary = TokenTimerSummary { - scanned: accounts.len(), - ..TokenTimerSummary::default() - }; - for account in accounts { - if self.clear_refresh_schedule_if_ineligible(&account).await? { - summary.skipped += 1; - continue; - } - - let Some(delay) = self.timer_delay_for_account(&account, now) else { - summary.skipped += 1; - continue; - }; - if delay.is_zero() { - if self.clear_scheduled_timer(&account.id).await { - summary.replaced += 1; - } - let outcome = self.refresh_scheduled_account(&account.id, now).await?; - if matches!(outcome, TokenRefreshOutcome::Skipped) { - summary.skipped += 1; - } else { - summary.immediate += 1; - } - continue; - } - - let scheduled_at = scheduled_at_from_delay(now, delay); - self.persist_next_refresh_at_if_changed(&account, Some(scheduled_at)) - .await?; - let schedule_result = self - .schedule_account_timer(account.id.clone(), delay, scheduled_at) - .await; - match schedule_result { - ScheduleTimerResult::Unchanged => { - summary.skipped += 1; - continue; - } - ScheduleTimerResult::Replaced => summary.replaced += 1, - ScheduleTimerResult::Scheduled => {} - } - - if delay.is_zero() { - summary.immediate += 1; - } else { - summary.scheduled += 1; - } - } - - Ok(summary) - } - - /// 返回当前账号级刷新定时器数量。 - pub async fn scheduled_timer_count(&self) -> usize { - self.timers.lock().await.len() - } - - /// 在指定时间点执行一次到期刷新扫描。 - pub async fn refresh_due_accounts_once_at( - &self, - now: DateTime, - ) -> TokenRefreshServiceResult { - let accounts = self - .store - .list_pool_accounts() - .await - .map_err(TokenRefreshServiceError::AccountStore)?; - let mut summary = TokenRefreshSummary { - scanned: accounts.len(), - ..TokenRefreshSummary::default() - }; - - for account in accounts { - if self.clear_refresh_schedule_if_ineligible(&account).await? { - summary.skipped += 1; - continue; - } - - if account.next_refresh_at.is_some_and(|value| value > now) { - summary.skipped += 1; - continue; - } - - if !self.scheduler.should_refresh_account_at(&account, now) { - summary.skipped += 1; - continue; - } - - if !self.try_mark_in_flight(&account.id).await { - summary.skipped += 1; - continue; - } - - let lease_acquired = match self.try_acquire_refresh_lease(&account.id, now).await { - Ok(acquired) => acquired, - Err(error) => { - self.release_in_flight(&account.id).await; - return Err(error); - } - }; - if !lease_acquired { - self.release_in_flight(&account.id).await; - summary.skipped += 1; - continue; - } - - let outcome = self - .refresh_account_with_status_transitions(&account, now) - .await; - - self.release_in_flight(&account.id).await; - self.release_refresh_lease(&account.id).await?; - - let outcome = outcome?; - self.schedule_next_refresh_after_outcome(&outcome, Utc::now()) - .await; - - match outcome { - TokenRefreshOutcome::Refreshed(_) => summary.refreshed += 1, - TokenRefreshOutcome::StatusUpdated => summary.status_updated += 1, - TokenRefreshOutcome::Skipped => summary.skipped += 1, - TokenRefreshOutcome::Failed => summary.failed += 1, - }; - } - - Ok(summary) - } - - fn timer_delay_for_account(&self, account: &Account, now: DateTime) -> Option { - if let Some(next_refresh_at) = account.next_refresh_at.filter(|value| *value > now) { - return Some(duration_until(now, next_refresh_at)); - } - - let refresh_at = self.computed_refresh_at(account)?; - - if refresh_at <= now { - return Some(Duration::ZERO); - } - - (refresh_at - now).to_std().ok() - } - - async fn clear_refresh_schedule_if_ineligible( - &self, - account: &Account, - ) -> TokenRefreshServiceResult { - if account.refresh_token.is_some() && token_refresh_status_eligible(account.status) { - return Ok(false); - } - - self.clear_scheduled_timer(&account.id).await; - if account.next_refresh_at.is_some() { - self.persist_next_refresh_at_if_changed(account, None) - .await?; - } - Ok(true) - } - - fn computed_refresh_at(&self, account: &Account) -> Option> { - let expires_at = account - .access_token_expires_at - .or_else(|| jwt_expiration(&account.access_token))?; - Some(jittered_refresh_at( - &account.id, - expires_at, - self.policy.refresh_margin_seconds(), - )) - } - - async fn persist_next_refresh_at_if_changed( - &self, - account: &Account, - next_refresh_at: Option>, - ) -> TokenRefreshServiceResult<()> { - if same_refresh_time(account.next_refresh_at, next_refresh_at) { - return Ok(()); - } - persist_next_refresh_at(&self.store, &account.id, next_refresh_at).await - } - - fn schedule_account_timer( - &self, - account_id: String, - delay: Duration, - scheduled_at: DateTime, - ) -> Pin + Send + '_>> { - Box::pin(async move { - { - let timers = self.timers.lock().await; - if timers - .get(&account_id) - .is_some_and(|timer| timer.scheduled_at == scheduled_at) - { - return ScheduleTimerResult::Unchanged; - } - } - - let replaced = self.clear_scheduled_timer(&account_id).await; - let task = self.clone(); - let timer_account_id = account_id.clone(); - let handle = tokio::spawn(async move { - if !delay.is_zero() { - sleep(delay).await; - } - task.timers.lock().await.remove(&timer_account_id); - if let Err(error) = task - .refresh_scheduled_account(&timer_account_id, scheduled_at) - .await - { - warn!( - account_id = %timer_account_id, - error = %error, - "scheduled token refresh failed" - ); - } - }); - - self.timers.lock().await.insert( - account_id, - ScheduledRefreshTimer { - scheduled_at, - handle, - }, - ); - if replaced { - ScheduleTimerResult::Replaced - } else { - ScheduleTimerResult::Scheduled - } - }) - } - - async fn clear_scheduled_timer(&self, account_id: &str) -> bool { - if let Some(timer) = self.timers.lock().await.remove(account_id) { - timer.handle.abort(); - true - } else { - false - } - } - - /// 清理全部已调度的账号级刷新定时器。 - pub async fn clear_scheduled_timers(&self) { - let mut timers = self.timers.lock().await; - for (_, timer) in timers.drain() { - timer.handle.abort(); - } - } - - async fn refresh_scheduled_account( - &self, - account_id: &str, - now: DateTime, - ) -> TokenRefreshServiceResult { - let Some(account) = self.store.get(account_id).await? else { - return Ok(TokenRefreshOutcome::Skipped); - }; - let account = stored_account_to_refresh_account(account); - if self.clear_refresh_schedule_if_ineligible(&account).await? { - return Ok(TokenRefreshOutcome::Skipped); - } - - if account.next_refresh_at.is_some_and(|value| value > now) { - return Ok(TokenRefreshOutcome::Skipped); - } - - if !self.try_mark_in_flight(&account.id).await { - return Ok(TokenRefreshOutcome::Skipped); - } - - let lease_acquired = match self.try_acquire_refresh_lease(&account.id, now).await { - Ok(acquired) => acquired, - Err(error) => { - self.release_in_flight(&account.id).await; - return Err(error); - } - }; - if !lease_acquired { - self.release_in_flight(&account.id).await; - return Ok(TokenRefreshOutcome::Skipped); - } - - let outcome = self - .refresh_account_with_status_transitions(&account, now) - .await; - - self.release_in_flight(&account.id).await; - self.release_refresh_lease(&account.id).await?; - - let outcome = outcome?; - self.schedule_next_refresh_after_outcome(&outcome, Utc::now()) - .await; - Ok(outcome) - } - - async fn schedule_next_refresh_after_outcome( - &self, - outcome: &TokenRefreshOutcome, - now: DateTime, - ) { - let TokenRefreshOutcome::Refreshed(account) = outcome else { - return; - }; - - let Some(delay) = self.timer_delay_for_account(account, now) else { - return; - }; - - if delay.is_zero() { - debug!( - account_id = %account.id, - "refreshed token is already within refresh margin; skipping next timer" - ); - return; - } - - let scheduled_at = scheduled_at_from_delay(now, delay); - self.schedule_account_timer(account.id.clone(), delay, scheduled_at) - .await; - } - - async fn refresh_account_with_status_transitions( - &self, - account: &Account, - now: DateTime, - ) -> TokenRefreshServiceResult { - let max_attempts = self.retry_delays.len() + 1; - let mut permanent_failures = 0; - - for attempt_index in 0..max_attempts { - let Some(attempt) = self.prepare_refresh_attempt(account, now).await? else { - return Ok(TokenRefreshOutcome::Skipped); - }; - - if attempt.refresh_token.is_none() { - persist_status(&self.store, &attempt.id, AccountStatus::Expired).await?; - persist_next_refresh_at(&self.store, &attempt.id, None).await?; - return Ok(TokenRefreshOutcome::StatusUpdated); - } - - match self.scheduler.refresh_account_at(&attempt, now).await { - Ok(updated) if is_permanent_refresh_failure_status(updated.status) => { - permanent_failures += 1; - if permanent_failures < PERMANENT_FAILURE_CONFIRMATION_THRESHOLD - && self.sleep_before_retry(&attempt.id, attempt_index).await - { - continue; - } - - persist_status(&self.store, &attempt.id, updated.status).await?; - persist_next_refresh_at(&self.store, &attempt.id, None).await?; - return Ok(TokenRefreshOutcome::StatusUpdated); - } - Ok(mut updated) => { - updated.access_token_expires_at = - jwt_expiration(&updated.access_token).or(updated.access_token_expires_at); - updated.next_refresh_at = self.computed_refresh_at(&updated); - let refresh_token_rotated = updated.refresh_token != attempt.refresh_token; - persist_token_update(&self.store, &updated).await?; - info!( - account_id = %updated.id, - attempt = attempt_index + 1, - status = %updated.status, - access_token_expires_at = ?updated.access_token_expires_at, - next_refresh_at = ?updated.next_refresh_at, - refresh_token_present = updated.refresh_token.is_some(), - refresh_token_rotated, - "token refresh succeeded" - ); - return Ok(TokenRefreshOutcome::Refreshed(Box::new(updated))); - } - Err(RefreshError::RetryableTransport | RefreshError::Transport) - if self.sleep_before_retry(&attempt.id, attempt_index).await => {} - Err(RefreshError::RetryableTransport | RefreshError::Transport) => { - persist_status(&self.store, &attempt.id, attempt.status).await?; - let recovery_delay = stable_jittered_duration( - &attempt.id, - Duration::from_secs(RECOVERY_DELAY_SECONDS as u64), - RECOVERY_DELAY_JITTER, - "recovery", - ); - let next_refresh_at = scheduled_at_from_delay(now, recovery_delay); - persist_next_refresh_at(&self.store, &attempt.id, Some(next_refresh_at)) - .await?; - return Ok(TokenRefreshOutcome::Failed); - } - Err(error) => return Err(TokenRefreshServiceError::Refresh(error)), - } - } - - Ok(TokenRefreshOutcome::Skipped) - } - - async fn prepare_refresh_attempt( - &self, - account: &Account, - now: DateTime, - ) -> TokenRefreshServiceResult> { - let Some(stored) = self.store.get(&account.id).await? else { - return Err(TokenRefreshServiceError::AccountNotFound( - account.id.clone(), - )); - }; - let attempt = stored_account_to_refresh_account(stored); - if !token_refresh_status_eligible(attempt.status) - || attempt.next_refresh_at.is_some_and(|value| value > now) - || attempt.refresh_token != account.refresh_token - { - return Ok(None); - } - - if !self.scheduler.should_refresh_account_at(&attempt, now) { - return Ok(None); - } - - Ok(Some(attempt)) - } - - async fn sleep_before_retry(&self, account_id: &str, attempt_index: usize) -> bool { - let Some(delay) = self.retry_delays.get(attempt_index).copied() else { - return false; - }; - - let delay = stable_jittered_duration( - account_id, - delay, - RETRY_DELAY_JITTER, - &format!("retry:{attempt_index}"), - ); - if !delay.is_zero() { - sleep(delay).await; - } - true - } - - async fn try_mark_in_flight(&self, account_id: &str) -> bool { - self.in_flight.lock().await.insert(account_id.to_string()) - } - - async fn release_in_flight(&self, account_id: &str) { - self.in_flight.lock().await.remove(account_id); - } - - async fn try_acquire_refresh_lease( - &self, - account_id: &str, - now: DateTime, - ) -> TokenRefreshServiceResult { - let Some(refresh_leases) = self.refresh_leases.as_ref() else { - return Ok(true); - }; - refresh_leases - .try_acquire( - account_id, - &self.lease_owner, - now + chrono::Duration::seconds(REFRESH_LEASE_TTL_SECONDS), - now, - ) - .await - .map_err(TokenRefreshServiceError::Lease) - } - - async fn release_refresh_lease(&self, account_id: &str) -> TokenRefreshServiceResult<()> { - let Some(refresh_leases) = self.refresh_leases.as_ref() else { - return Ok(()); - }; - refresh_leases - .release(account_id, &self.lease_owner) - .await - .map(|_| ()) - .map_err(TokenRefreshServiceError::Lease) - } -} - -mod types; -use types::*; - -async fn persist_token_update( - store: &PgAccountStore, - account: &Account, -) -> TokenRefreshServiceResult<()> { - let updated = store - .update_from_claims( - &account.id, - AccountClaimsUpdate { - email: account.email.clone(), - account_id: account.account_id.clone(), - user_id: account.user_id.clone(), - plan_type: account.plan_type.clone(), - access_token: SecretString::new(account.access_token.clone().into()), - refresh_token: account - .refresh_token - .clone() - .map(|token| SecretString::new(token.into())), - access_token_expires_at: jwt_expiration(&account.access_token) - .or(account.access_token_expires_at), - next_refresh_at: account.next_refresh_at, - status: account.status, - }, - ) - .await?; - if updated { - Ok(()) - } else { - Err(TokenRefreshServiceError::AccountNotFound( - account.id.clone(), - )) - } -} - -async fn persist_status( - store: &PgAccountStore, - account_id: &str, - status: AccountStatus, -) -> TokenRefreshServiceResult<()> { - let updated = store.set_status(account_id, status).await?; - if updated { - Ok(()) - } else { - Err(TokenRefreshServiceError::AccountNotFound( - account_id.to_string(), - )) - } -} - -async fn persist_next_refresh_at( - store: &PgAccountStore, - account_id: &str, - next_refresh_at: Option>, -) -> TokenRefreshServiceResult<()> { - let updated = store - .set_next_refresh_at(account_id, next_refresh_at) - .await?; - if updated { - Ok(()) - } else { - Err(TokenRefreshServiceError::AccountNotFound( - account_id.to_string(), - )) - } -} - -fn is_permanent_refresh_failure_status(status: AccountStatus) -> bool { - matches!(status, AccountStatus::Expired | AccountStatus::Banned) -} - -fn refresh_lease_owner() -> String { - format!("runtime-token-refresh:{}", Uuid::new_v4().simple()) -} - -fn default_refresh_retry_delays() -> Vec { - (0..MAX_REFRESH_ATTEMPTS.saturating_sub(1)) - .map(|attempt_index| { - let multiplier = 3_u64.saturating_pow(attempt_index as u32); - let millis = REFRESH_RETRY_BASE_DELAY_MILLIS - .saturating_mul(multiplier) - .min(REFRESH_RETRY_MAX_DELAY_MILLIS); - Duration::from_millis(millis) - }) - .collect() -} - -fn stable_jittered_duration( - account_id: &str, - base: Duration, - variance: f64, - salt: &str, -) -> Duration { - if base.is_zero() { - return Duration::ZERO; - } - - let mut hasher = DefaultHasher::new(); - account_id.hash(&mut hasher); - salt.hash(&mut hasher); - let unit = hasher.finish() as f64 / u64::MAX as f64; - let factor = (1.0 - variance) + unit * variance * 2.0; - let millis = (base.as_millis() as f64 * factor) - .round() - .clamp(0.0, u64::MAX as f64) as u64; - Duration::from_millis(millis) -} - -fn scheduled_at_from_delay(now: DateTime, delay: Duration) -> DateTime { - match chrono::Duration::from_std(delay) { - Ok(delay) => now + delay, - Err(_) => now, - } -} - -fn duration_until(now: DateTime, scheduled_at: DateTime) -> Duration { - (scheduled_at - now).to_std().unwrap_or(Duration::ZERO) -} - -fn same_refresh_time(left: Option>, right: Option>) -> bool { - match (left, right) { - (Some(left), Some(right)) => left.timestamp() == right.timestamp(), - (None, None) => true, - _ => false, - } -} diff --git a/backend/src/accounts/refresh/runtime/types.rs b/backend/src/accounts/refresh/runtime/types.rs deleted file mode 100644 index 8607e0446..000000000 --- a/backend/src/accounts/refresh/runtime/types.rs +++ /dev/null @@ -1,109 +0,0 @@ -use super::*; - -pub(super) enum TokenRefreshOutcome { - Refreshed(Box), - StatusUpdated, - Skipped, - Failed, -} - -/// 单次刷新扫描摘要。 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct TokenRefreshSummary { - /// 扫描账号数。 - pub scanned: usize, - /// 成功刷新 token 的账号数。 - pub refreshed: usize, - /// 仅更新状态的账号数。 - pub status_updated: usize, - /// 无需刷新的账号数。 - pub skipped: usize, - /// 传输失败账号数。 - pub failed: usize, -} - -/// 账号级刷新定时器调度摘要。 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct TokenTimerSummary { - /// 扫描账号数。 - pub scanned: usize, - /// 已调度的未来刷新定时器数。 - pub scheduled: usize, - /// 已调度的立即刷新定时器数。 - pub immediate: usize, - /// 无需调度的账号数。 - pub skipped: usize, - /// 被替换的既有定时器数。 - pub replaced: usize, -} - -impl TokenTimerSummary { - /// 返回本轮调度发生变化的定时器数量。 - pub fn changed(&self) -> usize { - self.scheduled + self.immediate - } -} - -/// 令牌刷新任务错误。 -#[derive(Debug, Error)] -pub enum TokenRefreshServiceError { - /// 账号存储读取失败。 - #[error("failed to list accounts for token refresh: {0}")] - AccountStore(#[from] crate::accounts::store::AccountStoreError), - /// 账号存储写入失败。 - #[error("failed to persist token refresh result: {0}")] - Store(#[from] PgAccountStoreError), - /// 刷新租约存储失败。 - #[error("failed to coordinate token refresh lease: {0}")] - Lease(#[from] RedisRefreshLeaseStoreError), - /// core 刷新调度失败。 - #[error("token refresh scheduler failed: {0}")] - Refresh(#[from] RefreshError), - /// 刷新期间账号不存在。 - #[error("account disappeared during token refresh: {0}")] - AccountNotFound(String), -} - -/// 令牌刷新任务结果。 -pub type TokenRefreshServiceResult = Result; - -pub(super) fn stored_account_to_refresh_account(stored: StoredAccount) -> Account { - Account { - id: stored.id, - email: stored.email, - account_id: stored.account_id, - user_id: stored.user_id, - label: stored.label, - plan_type: stored.plan_type, - access_token: stored.access_token.expose_secret().to_string(), - refresh_token: stored - .refresh_token - .map(|token| token.expose_secret().to_string()), - access_token_expires_at: stored.access_token_expires_at, - next_refresh_at: stored.next_refresh_at, - status: stored.status, - quota_limit_reached: false, - quota_verify_required: false, - quota_cooldown_until: None, - cloudflare_cooldown_until: None, - request_count: 0, - empty_response_count: 0, - image_input_tokens: 0, - image_output_tokens: 0, - image_request_count: 0, - image_request_failed_count: 0, - window_request_count: 0, - window_input_tokens: 0, - window_output_tokens: 0, - window_cached_tokens: 0, - window_image_input_tokens: 0, - window_image_output_tokens: 0, - window_image_request_count: 0, - window_image_request_failed_count: 0, - window_started_at: None, - window_reset_at: None, - limit_window_seconds: None, - added_at: stored.added_at, - last_used_at: None, - } -} diff --git a/backend/src/accounts/refresh/service.rs b/backend/src/accounts/refresh/service.rs index 9fea61628..e43dffec7 100644 --- a/backend/src/accounts/refresh/service.rs +++ b/backend/src/accounts/refresh/service.rs @@ -1,368 +1,788 @@ -//! 刷新租约存储、JWT 解码与账号 claims 验证。 +//! 运行时 token refresh 业务服务。 use std::{ - collections::hash_map::DefaultHasher, - hash::{Hash, Hasher}, - sync::{Arc, RwLock}, + collections::{HashMap, HashSet}, + future::Future, + pin::Pin, + sync::Arc, + time::Duration, }; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; -use chrono::{DateTime, Duration, TimeZone, Utc}; -use serde_json::{Map, Value}; -use tokio::sync::Semaphore; +use chrono::{DateTime, Utc}; +use secrecy::{ExposeSecret, SecretString}; +use thiserror::Error; +use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; +use tracing::{debug, info, warn}; +use uuid::Uuid; -use crate::accounts::account::{Account, AccountStatus}; -use crate::upstream::openai::token_client::{RefreshFailure, TokenPair, TokenRefresher}; +use crate::accounts::{ + account::{Account, AccountStatus}, + store::{ + AccountClaimsUpdate, AccountStore, PgAccountStore, PgAccountStoreError, StoredAccount, + }, +}; +use crate::upstream::openai::token_client::TokenRefresher; -// --------------------------------------------------------------------------- -// JWT 解码 -// --------------------------------------------------------------------------- +use super::{ + lease::{RedisRefreshLeaseStore, RedisRefreshLeaseStoreError}, + policy::{ + default_refresh_retry_delays, duration_until, is_permanent_refresh_failure_status, + jittered_refresh_at, jwt_expiration, same_refresh_time, scheduled_at_from_delay, + stable_jittered_duration, token_refresh_status_eligible, RefreshError, RefreshPolicy, + RefreshScheduler, RuntimeRefreshPolicy, TokenRefreshSummary, TokenTimerSummary, + PERMANENT_FAILURE_CONFIRMATION_THRESHOLD, RECOVERY_DELAY_JITTER, RECOVERY_DELAY_SECONDS, + RETRY_DELAY_JITTER, + }, +}; -/// JWT 过期状态。 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum JwtExpiry { - /// token 已过期。 - Expired, - /// token 仍然有效。 - Valid, - /// token 缺失、格式错误或不包含可解析的 exp。 - MissingOrInvalid, -} +const REFRESH_LEASE_TTL_SECONDS: i64 = 5 * 60; -/// 按给定时间点判断 JWT 的 `exp` 是否已过期。 -pub fn jwt_expiry(token: &str, now: DateTime) -> JwtExpiry { - let Some(exp) = jwt_exp(token) else { - return JwtExpiry::MissingOrInvalid; - }; - if now.timestamp() >= exp { - JwtExpiry::Expired - } else { - JwtExpiry::Valid - } -} - -/// 读取 JWT `exp` 并转换成 UTC 时间。 -pub fn jwt_expiration(token: &str) -> Option> { - Utc.timestamp_opt(jwt_exp(token)?, 0).single() -} - -fn jwt_exp(token: &str) -> Option { - let mut parts = token.split('.'); - let _header = parts.next()?; - let payload = parts.next()?; - let _signature = parts.next()?; - if parts.next().is_some() { - return None; - } - - let decoded = URL_SAFE_NO_PAD.decode(payload).ok()?; - let value = serde_json::from_slice::(&decoded).ok()?; - value.get("exp")?.as_i64() -} - -// --------------------------------------------------------------------------- -// 账号 claims 解码（手动创建/导入时验证 JWT） -// --------------------------------------------------------------------------- - -/// 手动创建账号时从 JWT 提取的 claims。 -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManualAccountClaims { - /// ChatGPT 账号 ID。 - pub account_id: Option, - /// ChatGPT 用户 ID。 - pub user_id: Option, - /// 邮箱。 - pub email: Option, - /// 订阅计划类型。 - pub plan_type: Option, - /// access token 过期时间。 - pub expires_at: DateTime, -} - -/// 从 JWT 中解码 payload 部分。 -pub fn decode_jwt_payload(token: &str) -> Option> { - let mut parts = token.split('.'); - let _header = parts.next()?; - let payload = parts.next()?; - if payload.is_empty() { - return None; - } - let bytes = URL_SAFE_NO_PAD.decode(payload).ok()?; - serde_json::from_slice::(&bytes) - .ok()? - .as_object() - .cloned() -} - -/// 从 JWT 中提取手动创建账号所需的 claims。 -pub fn manual_account_claims( - token: &str, - now: DateTime, -) -> Result { - let payload = decode_jwt_payload(token).ok_or("Invalid JWT format")?; - let exp = payload - .get("exp") - .and_then(Value::as_i64) - .ok_or("Token is expired")?; - if now.timestamp() >= exp { - return Err("Token is expired"); - } - let expires_at = DateTime::::from_timestamp(exp, 0).ok_or("Invalid JWT exp claim")?; - let auth = payload - .get("https://api.openai.com/auth") - .and_then(Value::as_object) - .ok_or("Token missing OpenAI auth claim")?; - let account_id = string_claim(auth, "chatgpt_account_id"); - let profile = payload - .get("https://api.openai.com/profile") - .and_then(Value::as_object); - let user_id = string_claim(auth, "chatgpt_user_id") - .or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_user_id"))) - .or_else(|| string_claim(auth, "user_id")) - .or_else(|| profile.and_then(|profile| string_claim(profile, "user_id"))); - let plan_type = string_claim(auth, "chatgpt_plan_type") - .or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_plan_type"))); - let email = profile.and_then(|profile| string_claim(profile, "email")); - - Ok(ManualAccountClaims { - account_id, - user_id, - email, - plan_type, - expires_at, - }) -} - -fn string_claim(map: &Map, key: &str) -> Option { - map.get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToString::to_string) -} - -// --------------------------------------------------------------------------- -// 刷新策略和上游端口 -// --------------------------------------------------------------------------- - -/// 刷新任务的调度策略。 -#[derive(Debug, Clone, Copy)] -pub struct RefreshPolicy { - /// 提前多久开始刷新访问令牌。 - pub refresh_margin_seconds: u64, - /// 允许并发执行的刷新任务数。 - pub refresh_concurrency: u32, -} - -/// 运行时共享刷新策略。 -#[derive(Clone)] -pub struct RuntimeRefreshPolicy { - policy: Arc>, - semaphore: Arc>>, -} - -impl RuntimeRefreshPolicy { - /// 构造共享刷新策略。 - pub fn new(policy: RefreshPolicy) -> Self { - Self { - policy: Arc::new(RwLock::new(policy)), - semaphore: Arc::new(RwLock::new(Arc::new(Semaphore::new( - policy.refresh_concurrency.max(1) as usize, - )))), - } - } - - /// 返回当前策略快照。 - pub fn snapshot(&self) -> RefreshPolicy { - *self - .policy - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - /// 更新当前策略。 - pub fn update(&self, policy: RefreshPolicy) { - *self - .policy - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = policy; - *self - .semaphore - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Arc::new(Semaphore::new(policy.refresh_concurrency.max(1) as usize)); - } - - /// 返回当前提前刷新秒数。 - pub fn refresh_margin_seconds(&self) -> u64 { - self.snapshot().refresh_margin_seconds - } - - fn semaphore(&self) -> Arc { - self.semaphore - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone() - } -} - -impl From for RuntimeRefreshPolicy { - fn from(policy: RefreshPolicy) -> Self { - Self::new(policy) - } -} - -/// 按账号稳定扰动刷新提前量，范围与 TS 版 `jitter(margin, 0.15)` 一致。 -pub(crate) fn jittered_refresh_at( - account_id: &str, - expires_at: DateTime, - margin_seconds: u64, -) -> DateTime { - expires_at - Duration::seconds(jittered_refresh_margin_seconds(account_id, margin_seconds)) -} - -fn jittered_refresh_margin_seconds(account_id: &str, margin_seconds: u64) -> i64 { - if margin_seconds == 0 { - return 0; - } - - let mut hasher = DefaultHasher::new(); - account_id.hash(&mut hasher); - let unit = hasher.finish() as f64 / u64::MAX as f64; - let factor = 0.85 + unit * 0.30; - (margin_seconds as f64 * factor) - .round() - .clamp(0.0, i64::MAX as f64) as i64 -} - -/// 调度器自身的执行错误。 -#[derive(Debug, thiserror::Error)] -pub enum RefreshError { - /// 并发限制信号量已关闭。 - #[error("refresh task semaphore closed")] - ConcurrencyClosed, - /// 刷新请求在到达服务端前失败，可安全复用当前 refresh token 重试。 - #[error("refresh transport failed before server processing")] - RetryableTransport, - /// 刷新请求在传输层失败，refresh token 可能已经被服务端消费。 - #[error("refresh transport failed after possible server processing")] - Transport, -} - -/// 负责执行单账号刷新策略的调度器。 -#[derive(Clone)] -pub struct RefreshScheduler { - policy: RuntimeRefreshPolicy, - client: Arc, -} - -impl RefreshScheduler +/// 运行时 token refresh 服务。 +pub struct RuntimeTokenRefreshService where C: TokenRefresher, { - /// 使用策略和上游刷新端口构造调度器。 - pub fn new(policy: impl Into, client: C) -> Self { + store: PgAccountStore, + scheduler: Arc>, + policy: RuntimeRefreshPolicy, + refresh_leases: Option, + lease_owner: String, + retry_delays: Vec, + in_flight: Arc>>, + timers: Arc>>, +} + +struct ScheduledRefreshTimer { + scheduled_at: DateTime, + handle: JoinHandle<()>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScheduleTimerResult { + Scheduled, + Replaced, + Unchanged, +} + +enum TokenRefreshOutcome { + Refreshed(Box), + StatusUpdated, + Skipped, + Failed, +} + +/// 令牌刷新任务错误。 +#[derive(Debug, Error)] +pub enum TokenRefreshServiceError { + /// 账号存储读取失败。 + #[error("failed to list accounts for token refresh: {0}")] + AccountStore(#[from] crate::accounts::store::AccountStoreError), + /// 账号存储写入失败。 + #[error("failed to persist token refresh result: {0}")] + Store(#[from] PgAccountStoreError), + /// 刷新租约存储失败。 + #[error("failed to coordinate token refresh lease: {0}")] + Lease(#[from] RedisRefreshLeaseStoreError), + /// core 刷新调度失败。 + #[error("token refresh scheduler failed: {0}")] + Refresh(#[from] RefreshError), + /// 刷新期间账号不存在。 + #[error("account disappeared during token refresh: {0}")] + AccountNotFound(String), +} + +/// 令牌刷新任务结果。 +pub type TokenRefreshServiceResult = Result; + +impl Clone for RuntimeTokenRefreshService +where + C: TokenRefresher, +{ + fn clone(&self) -> Self { Self { - policy: policy.into(), - client: Arc::new(client), + store: self.store.clone(), + scheduler: self.scheduler.clone(), + policy: self.policy.clone(), + refresh_leases: self.refresh_leases.clone(), + lease_owner: self.lease_owner.clone(), + retry_delays: self.retry_delays.clone(), + in_flight: self.in_flight.clone(), + timers: self.timers.clone(), } } +} + +impl RuntimeTokenRefreshService +where + C: TokenRefresher, +{ + /// 构造运行时 token refresh 服务。 + pub fn new(store: PgAccountStore, policy: impl Into, client: C) -> Self { + let policy = policy.into(); + Self { + store, + scheduler: Arc::new(RefreshScheduler::new(policy.clone(), client)), + policy, + refresh_leases: None, + lease_owner: refresh_lease_owner(), + retry_delays: default_refresh_retry_delays(), + in_flight: Arc::new(Mutex::new(HashSet::new())), + timers: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// 使用刷新租约存储保护账号刷新。 + pub fn with_refresh_lease_store(mut self, refresh_leases: RedisRefreshLeaseStore) -> Self { + self.refresh_leases = Some(refresh_leases); + self + } /// 更新刷新策略。 pub fn update_policy(&self, policy: RefreshPolicy) { self.policy.update(policy); } - /// 在给定时间点按到期策略刷新账号。 - /// - /// 当账号不需要刷新时返回原账号快照；当刷新失败但属于可映射的领域错误时， - /// 返回更新过状态的账号。 - /// - /// # Errors - /// - /// 当并发控制已关闭或刷新传输失败时返回 [`RefreshError`]。 - pub async fn refresh_account_at( + /// 使用自定义刷新重试延迟。 + pub fn with_retry_delays(mut self, retry_delays: I) -> Self + where + I: IntoIterator, + { + self.retry_delays = retry_delays.into_iter().collect(); + self + } + + /// 返回给定账号集合中当前正在执行 token 刷新的账号 ID。 + pub async fn refreshing_account_ids( + &self, + account_ids: &[String], + now: DateTime, + ) -> TokenRefreshServiceResult> { + if account_ids.is_empty() { + return Ok(HashSet::new()); + } + + let requested_ids = account_ids + .iter() + .map(String::as_str) + .collect::>(); + let mut refreshing = self + .in_flight + .lock() + .await + .iter() + .filter(|account_id| requested_ids.contains(account_id.as_str())) + .cloned() + .collect::>(); + + if let Some(refresh_leases) = self.refresh_leases.as_ref() { + refreshing.extend( + refresh_leases + .active_account_ids(account_ids, now) + .await + .map_err(TokenRefreshServiceError::Lease)?, + ); + } + + Ok(refreshing) + } + + /// 执行一次账号级刷新定时器调度。 + pub async fn schedule_account_timers_once( + &self, + ) -> TokenRefreshServiceResult { + self.schedule_account_timers_once_at(Utc::now()).await + } + + /// 在指定时间点执行一次账号级刷新定时器调度。 + pub async fn schedule_account_timers_once_at( + &self, + now: DateTime, + ) -> TokenRefreshServiceResult { + let accounts = self + .store + .list_pool_accounts() + .await + .map_err(TokenRefreshServiceError::AccountStore)?; + let mut summary = TokenTimerSummary { + scanned: accounts.len(), + ..TokenTimerSummary::default() + }; + for account in accounts { + if self.clear_refresh_schedule_if_ineligible(&account).await? { + summary.skipped += 1; + continue; + } + + let Some(delay) = self.timer_delay_for_account(&account, now) else { + summary.skipped += 1; + continue; + }; + if delay.is_zero() { + if self.clear_scheduled_timer(&account.id).await { + summary.replaced += 1; + } + let outcome = self.refresh_scheduled_account(&account.id, now).await?; + if matches!(outcome, TokenRefreshOutcome::Skipped) { + summary.skipped += 1; + } else { + summary.immediate += 1; + } + continue; + } + + let scheduled_at = scheduled_at_from_delay(now, delay); + self.persist_next_refresh_at_if_changed(&account, Some(scheduled_at)) + .await?; + let schedule_result = self + .schedule_account_timer(account.id.clone(), delay, scheduled_at) + .await; + match schedule_result { + ScheduleTimerResult::Unchanged => { + summary.skipped += 1; + continue; + } + ScheduleTimerResult::Replaced => summary.replaced += 1, + ScheduleTimerResult::Scheduled => {} + } + + if delay.is_zero() { + summary.immediate += 1; + } else { + summary.scheduled += 1; + } + } + + Ok(summary) + } + + /// 返回当前账号级刷新定时器数量。 + pub async fn scheduled_timer_count(&self) -> usize { + self.timers.lock().await.len() + } + + /// 在指定时间点执行一次到期刷新扫描。 + pub async fn refresh_due_accounts_once_at( + &self, + now: DateTime, + ) -> TokenRefreshServiceResult { + let accounts = self + .store + .list_pool_accounts() + .await + .map_err(TokenRefreshServiceError::AccountStore)?; + let mut summary = TokenRefreshSummary { + scanned: accounts.len(), + ..TokenRefreshSummary::default() + }; + + for account in accounts { + if self.clear_refresh_schedule_if_ineligible(&account).await? { + summary.skipped += 1; + continue; + } + + if account.next_refresh_at.is_some_and(|value| value > now) { + summary.skipped += 1; + continue; + } + + if !self.scheduler.should_refresh_account_at(&account, now) { + summary.skipped += 1; + continue; + } + + if !self.try_mark_in_flight(&account.id).await { + summary.skipped += 1; + continue; + } + + let lease_acquired = match self.try_acquire_refresh_lease(&account.id, now).await { + Ok(acquired) => acquired, + Err(error) => { + self.release_in_flight(&account.id).await; + return Err(error); + } + }; + if !lease_acquired { + self.release_in_flight(&account.id).await; + summary.skipped += 1; + continue; + } + + let outcome = self + .refresh_account_with_status_transitions(&account, now) + .await; + + self.release_in_flight(&account.id).await; + self.release_refresh_lease(&account.id).await?; + + let outcome = outcome?; + self.schedule_next_refresh_after_outcome(&outcome, Utc::now()) + .await; + + match outcome { + TokenRefreshOutcome::Refreshed(_) => summary.refreshed += 1, + TokenRefreshOutcome::StatusUpdated => summary.status_updated += 1, + TokenRefreshOutcome::Skipped => summary.skipped += 1, + TokenRefreshOutcome::Failed => summary.failed += 1, + }; + } + + Ok(summary) + } + + fn timer_delay_for_account(&self, account: &Account, now: DateTime) -> Option { + if let Some(next_refresh_at) = account.next_refresh_at.filter(|value| *value > now) { + return Some(duration_until(now, next_refresh_at)); + } + + let refresh_at = self.computed_refresh_at(account)?; + + if refresh_at <= now { + return Some(Duration::ZERO); + } + + (refresh_at - now).to_std().ok() + } + + async fn clear_refresh_schedule_if_ineligible( + &self, + account: &Account, + ) -> TokenRefreshServiceResult { + if account.refresh_token.is_some() && token_refresh_status_eligible(account.status) { + return Ok(false); + } + + self.clear_scheduled_timer(&account.id).await; + if account.next_refresh_at.is_some() { + self.persist_next_refresh_at_if_changed(account, None) + .await?; + } + Ok(true) + } + + fn computed_refresh_at(&self, account: &Account) -> Option> { + let expires_at = account + .access_token_expires_at + .or_else(|| jwt_expiration(&account.access_token))?; + Some(jittered_refresh_at( + &account.id, + expires_at, + self.policy.refresh_margin_seconds(), + )) + } + + async fn persist_next_refresh_at_if_changed( + &self, + account: &Account, + next_refresh_at: Option>, + ) -> TokenRefreshServiceResult<()> { + if same_refresh_time(account.next_refresh_at, next_refresh_at) { + return Ok(()); + } + persist_next_refresh_at(&self.store, &account.id, next_refresh_at).await + } + + fn schedule_account_timer( + &self, + account_id: String, + delay: Duration, + scheduled_at: DateTime, + ) -> Pin + Send + '_>> { + Box::pin(async move { + { + let timers = self.timers.lock().await; + if timers + .get(&account_id) + .is_some_and(|timer| timer.scheduled_at == scheduled_at) + { + return ScheduleTimerResult::Unchanged; + } + } + + let replaced = self.clear_scheduled_timer(&account_id).await; + let task = self.clone(); + let timer_account_id = account_id.clone(); + let handle = tokio::spawn(async move { + if !delay.is_zero() { + sleep(delay).await; + } + task.timers.lock().await.remove(&timer_account_id); + if let Err(error) = task + .refresh_scheduled_account(&timer_account_id, scheduled_at) + .await + { + warn!( + account_id = %timer_account_id, + error = %error, + "scheduled token refresh failed" + ); + } + }); + + self.timers.lock().await.insert( + account_id, + ScheduledRefreshTimer { + scheduled_at, + handle, + }, + ); + if replaced { + ScheduleTimerResult::Replaced + } else { + ScheduleTimerResult::Scheduled + } + }) + } + + async fn clear_scheduled_timer(&self, account_id: &str) -> bool { + if let Some(timer) = self.timers.lock().await.remove(account_id) { + timer.handle.abort(); + true + } else { + false + } + } + + /// 清理全部已调度的账号级刷新定时器。 + pub async fn clear_scheduled_timers(&self) { + let mut timers = self.timers.lock().await; + for (_, timer) in timers.drain() { + timer.handle.abort(); + } + } + + async fn refresh_scheduled_account( + &self, + account_id: &str, + now: DateTime, + ) -> TokenRefreshServiceResult { + let Some(account) = self.store.get(account_id).await? else { + return Ok(TokenRefreshOutcome::Skipped); + }; + let account = stored_account_to_refresh_account(account); + if self.clear_refresh_schedule_if_ineligible(&account).await? { + return Ok(TokenRefreshOutcome::Skipped); + } + + if account.next_refresh_at.is_some_and(|value| value > now) { + return Ok(TokenRefreshOutcome::Skipped); + } + + if !self.try_mark_in_flight(&account.id).await { + return Ok(TokenRefreshOutcome::Skipped); + } + + let lease_acquired = match self.try_acquire_refresh_lease(&account.id, now).await { + Ok(acquired) => acquired, + Err(error) => { + self.release_in_flight(&account.id).await; + return Err(error); + } + }; + if !lease_acquired { + self.release_in_flight(&account.id).await; + return Ok(TokenRefreshOutcome::Skipped); + } + + let outcome = self + .refresh_account_with_status_transitions(&account, now) + .await; + + self.release_in_flight(&account.id).await; + self.release_refresh_lease(&account.id).await?; + + let outcome = outcome?; + self.schedule_next_refresh_after_outcome(&outcome, Utc::now()) + .await; + Ok(outcome) + } + + async fn schedule_next_refresh_after_outcome( + &self, + outcome: &TokenRefreshOutcome, + now: DateTime, + ) { + let TokenRefreshOutcome::Refreshed(account) = outcome else { + return; + }; + + let Some(delay) = self.timer_delay_for_account(account, now) else { + return; + }; + + if delay.is_zero() { + debug!( + account_id = %account.id, + "refreshed token is already within refresh margin; skipping next timer" + ); + return; + } + + let scheduled_at = scheduled_at_from_delay(now, delay); + self.schedule_account_timer(account.id.clone(), delay, scheduled_at) + .await; + } + + async fn refresh_account_with_status_transitions( &self, account: &Account, now: DateTime, - ) -> Result { - if !self.should_refresh_account_at(account, now) { - return Ok(account.clone()); + ) -> TokenRefreshServiceResult { + let max_attempts = self.retry_delays.len() + 1; + let mut permanent_failures = 0; + + for attempt_index in 0..max_attempts { + let Some(attempt) = self.prepare_refresh_attempt(account, now).await? else { + return Ok(TokenRefreshOutcome::Skipped); + }; + + if attempt.refresh_token.is_none() { + persist_status(&self.store, &attempt.id, AccountStatus::Expired).await?; + persist_next_refresh_at(&self.store, &attempt.id, None).await?; + return Ok(TokenRefreshOutcome::StatusUpdated); + } + + match self.scheduler.refresh_account_at(&attempt, now).await { + Ok(updated) if is_permanent_refresh_failure_status(updated.status) => { + permanent_failures += 1; + if permanent_failures < PERMANENT_FAILURE_CONFIRMATION_THRESHOLD + && self.sleep_before_retry(&attempt.id, attempt_index).await + { + continue; + } + + persist_status(&self.store, &attempt.id, updated.status).await?; + persist_next_refresh_at(&self.store, &attempt.id, None).await?; + return Ok(TokenRefreshOutcome::StatusUpdated); + } + Ok(mut updated) => { + updated.access_token_expires_at = + jwt_expiration(&updated.access_token).or(updated.access_token_expires_at); + updated.next_refresh_at = self.computed_refresh_at(&updated); + let refresh_token_rotated = updated.refresh_token != attempt.refresh_token; + persist_token_update(&self.store, &updated).await?; + info!( + account_id = %updated.id, + attempt = attempt_index + 1, + status = %updated.status, + access_token_expires_at = ?updated.access_token_expires_at, + next_refresh_at = ?updated.next_refresh_at, + refresh_token_present = updated.refresh_token.is_some(), + refresh_token_rotated, + "token refresh succeeded" + ); + return Ok(TokenRefreshOutcome::Refreshed(Box::new(updated))); + } + Err(RefreshError::RetryableTransport | RefreshError::Transport) + if self.sleep_before_retry(&attempt.id, attempt_index).await => {} + Err(RefreshError::RetryableTransport | RefreshError::Transport) => { + persist_status(&self.store, &attempt.id, attempt.status).await?; + let recovery_delay = stable_jittered_duration( + &attempt.id, + Duration::from_secs(RECOVERY_DELAY_SECONDS as u64), + RECOVERY_DELAY_JITTER, + "recovery", + ); + let next_refresh_at = scheduled_at_from_delay(now, recovery_delay); + persist_next_refresh_at(&self.store, &attempt.id, Some(next_refresh_at)) + .await?; + return Ok(TokenRefreshOutcome::Failed); + } + Err(error) => return Err(TokenRefreshServiceError::Refresh(error)), + } } - let Some(refresh_token) = account.refresh_token.as_deref() else { - let mut expired = account.clone(); - expired.status = AccountStatus::Expired; - return Ok(expired); + Ok(TokenRefreshOutcome::Skipped) + } + + async fn prepare_refresh_attempt( + &self, + account: &Account, + now: DateTime, + ) -> TokenRefreshServiceResult> { + let Some(stored) = self.store.get(&account.id).await? else { + return Err(TokenRefreshServiceError::AccountNotFound( + account.id.clone(), + )); + }; + let attempt = stored_account_to_refresh_account(stored); + if !token_refresh_status_eligible(attempt.status) + || attempt.next_refresh_at.is_some_and(|value| value > now) + || attempt.refresh_token != account.refresh_token + { + return Ok(None); + } + + if !self.scheduler.should_refresh_account_at(&attempt, now) { + return Ok(None); + } + + Ok(Some(attempt)) + } + + async fn sleep_before_retry(&self, account_id: &str, attempt_index: usize) -> bool { + let Some(delay) = self.retry_delays.get(attempt_index).copied() else { + return false; }; - let semaphore = self.policy.semaphore(); - let _permit = semaphore - .acquire() + let delay = stable_jittered_duration( + account_id, + delay, + RETRY_DELAY_JITTER, + &format!("retry:{attempt_index}"), + ); + if !delay.is_zero() { + sleep(delay).await; + } + true + } + + async fn try_mark_in_flight(&self, account_id: &str) -> bool { + self.in_flight.lock().await.insert(account_id.to_string()) + } + + async fn release_in_flight(&self, account_id: &str) { + self.in_flight.lock().await.remove(account_id); + } + + async fn try_acquire_refresh_lease( + &self, + account_id: &str, + now: DateTime, + ) -> TokenRefreshServiceResult { + let Some(refresh_leases) = self.refresh_leases.as_ref() else { + return Ok(true); + }; + refresh_leases + .try_acquire( + account_id, + &self.lease_owner, + now + chrono::Duration::seconds(REFRESH_LEASE_TTL_SECONDS), + now, + ) .await - .map_err(|_| RefreshError::ConcurrencyClosed)?; - - match self.client.refresh(refresh_token).await { - Ok(token_pair) => Ok(apply_token_pair(account, token_pair)), - Err(RefreshFailure::RetryableTransport) => Err(RefreshError::RetryableTransport), - Err(RefreshFailure::Transport) => Err(RefreshError::Transport), - Err(error) => Ok(apply_refresh_failure(account, error)), - } + .map_err(TokenRefreshServiceError::Lease) } - /// 判断账号在给定时间点是否需要刷新。 - pub fn should_refresh_account_at(&self, account: &Account, now: DateTime) -> bool { - token_refresh_status_eligible(account.status) - && self.should_refresh_before_expiry(account, now) + async fn release_refresh_lease(&self, account_id: &str) -> TokenRefreshServiceResult<()> { + let Some(refresh_leases) = self.refresh_leases.as_ref() else { + return Ok(()); + }; + refresh_leases + .release(account_id, &self.lease_owner) + .await + .map(|_| ()) + .map_err(TokenRefreshServiceError::Lease) } +} - fn should_refresh_before_expiry(&self, account: &Account, now: DateTime) -> bool { - if let Some(next_refresh_at) = account.next_refresh_at { - return next_refresh_at <= now; - } - - account - .access_token_expires_at - .is_some_and(|expires_at| self.refresh_at(account, expires_at) <= now) - } - - fn refresh_at(&self, account: &Account, expires_at: DateTime) -> DateTime { - let policy = self.policy.snapshot(); - jittered_refresh_at( - account.id.as_str(), - expires_at, - policy.refresh_margin_seconds, +async fn persist_token_update( + store: &PgAccountStore, + account: &Account, +) -> TokenRefreshServiceResult<()> { + let updated = store + .update_from_claims( + &account.id, + AccountClaimsUpdate { + email: account.email.clone(), + account_id: account.account_id.clone(), + user_id: account.user_id.clone(), + plan_type: account.plan_type.clone(), + access_token: SecretString::new(account.access_token.clone().into()), + refresh_token: account + .refresh_token + .clone() + .map(|token| SecretString::new(token.into())), + access_token_expires_at: jwt_expiration(&account.access_token) + .or(account.access_token_expires_at), + next_refresh_at: account.next_refresh_at, + status: account.status, + }, ) + .await?; + if updated { + Ok(()) + } else { + Err(TokenRefreshServiceError::AccountNotFound( + account.id.clone(), + )) } } -pub(crate) fn token_refresh_status_eligible(status: AccountStatus) -> bool { - matches!( - status, - AccountStatus::Active | AccountStatus::QuotaExhausted - ) -} - -/// 将新的 token 对应用到账号快照上。 -pub fn apply_token_pair(account: &Account, token_pair: TokenPair) -> Account { - let mut refreshed = account.clone(); - refreshed.access_token = token_pair.access_token; - - // 刷新响应不返回 refresh_token 时，继续保留已有值，避免永久失去刷新能力。 - if let Some(refresh_token) = token_pair.refresh_token { - refreshed.refresh_token = Some(refresh_token); +async fn persist_status( + store: &PgAccountStore, + account_id: &str, + status: AccountStatus, +) -> TokenRefreshServiceResult<()> { + let updated = store.set_status(account_id, status).await?; + if updated { + Ok(()) + } else { + Err(TokenRefreshServiceError::AccountNotFound( + account_id.to_string(), + )) } - - refreshed } -/// 将刷新失败映射为账号状态变更。 -pub fn apply_refresh_failure(account: &Account, failure: RefreshFailure) -> Account { - let mut updated = account.clone(); - updated.status = match failure { - RefreshFailure::InvalidGrant => AccountStatus::Expired, - RefreshFailure::Banned => AccountStatus::Banned, - RefreshFailure::RetryableTransport | RefreshFailure::Transport => account.status, - }; - updated +async fn persist_next_refresh_at( + store: &PgAccountStore, + account_id: &str, + next_refresh_at: Option>, +) -> TokenRefreshServiceResult<()> { + let updated = store + .set_next_refresh_at(account_id, next_refresh_at) + .await?; + if updated { + Ok(()) + } else { + Err(TokenRefreshServiceError::AccountNotFound( + account_id.to_string(), + )) + } +} + +fn refresh_lease_owner() -> String { + format!("runtime-token-refresh:{}", Uuid::new_v4().simple()) +} + +fn stored_account_to_refresh_account(stored: StoredAccount) -> Account { + Account { + id: stored.id, + email: stored.email, + account_id: stored.account_id, + user_id: stored.user_id, + label: stored.label, + plan_type: stored.plan_type, + access_token: stored.access_token.expose_secret().to_string(), + refresh_token: stored + .refresh_token + .map(|token| token.expose_secret().to_string()), + access_token_expires_at: stored.access_token_expires_at, + next_refresh_at: stored.next_refresh_at, + status: stored.status, + quota_limit_reached: false, + quota_verify_required: false, + quota_cooldown_until: None, + cloudflare_cooldown_until: None, + request_count: 0, + empty_response_count: 0, + image_input_tokens: 0, + image_output_tokens: 0, + image_request_count: 0, + image_request_failed_count: 0, + window_request_count: 0, + window_input_tokens: 0, + window_output_tokens: 0, + window_cached_tokens: 0, + window_image_input_tokens: 0, + window_image_output_tokens: 0, + window_image_request_count: 0, + window_image_request_failed_count: 0, + window_started_at: None, + window_reset_at: None, + limit_window_seconds: None, + added_at: stored.added_at, + last_used_at: None, + } } diff --git a/backend/src/accounts/scheduler/candidates.rs b/backend/src/accounts/scheduler/candidates.rs index 81b3c27b3..ab823759f 100644 --- a/backend/src/accounts/scheduler/candidates.rs +++ b/backend/src/accounts/scheduler/candidates.rs @@ -7,7 +7,7 @@ //! //! 1. 账号状态为 [`AccountStatus::Active`]； //! 2. 在途槽位数未达单账号并发上限； -//! 3. 配额可用（[`AccountService::quota_available_at`]，受 `skip_quota_limited` 控制）； +//! 3. 配额可用（受 `skip_quota_limited` 控制）； //! 4. 模型在账号订阅计划的允许列表内； //! 5. 不在本次请求的排除列表内； //! 6. 不处于 Cloudflare 冷却期。 @@ -18,7 +18,6 @@ use chrono::{DateTime, Utc}; use crate::accounts::account::{Account, AccountStatus}; -use crate::accounts::service::AccountService; /// 候选过滤所需的账号池策略参数（只读视图）。 pub struct CandidateFilter<'a> { @@ -78,7 +77,7 @@ fn is_base_available( ) -> bool { account.status == AccountStatus::Active && (filter.slot_count)(&account.id) < filter.max_concurrent_per_account - && AccountService::quota_available_at(account, request.now, filter.skip_quota_limited) + && quota_available_at(account, request.now, filter.skip_quota_limited) && is_model_allowed( account, request.model, @@ -89,7 +88,7 @@ fn is_base_available( .exclude_account_ids .iter() .any(|account_id| account_id == &account.id) - && AccountService::cloudflare_available_at(account, request.now) + && cloudflare_available_at(account, request.now) } /// 判断模型是否允许该账号的订阅计划使用。 @@ -132,6 +131,25 @@ pub fn is_model_refresh_available( ) -> bool { account.status == AccountStatus::Active && slot_count(&account.id) < max_concurrent_per_account - && AccountService::quota_available_at(account, now, skip_quota_limited) - && AccountService::cloudflare_available_at(account, now) + && quota_available_at(account, now, skip_quota_limited) + && cloudflare_available_at(account, now) +} + +pub(crate) fn quota_available_at( + account: &Account, + now: DateTime, + skip_quota_limited: bool, +) -> bool { + if !skip_quota_limited || !account.quota_limit_reached { + return true; + } + account + .quota_cooldown_until + .is_some_and(|cooldown_until| now >= cooldown_until) +} + +fn cloudflare_available_at(account: &Account, now: DateTime) -> bool { + account + .cloudflare_cooldown_until + .is_none_or(|cooldown_until| now >= cooldown_until) } diff --git a/backend/src/accounts/service.rs b/backend/src/accounts/service.rs deleted file mode 100644 index 0cc64956d..000000000 --- a/backend/src/accounts/service.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! 账号管理用例辅助。 - -use chrono::{DateTime, Utc}; - -use crate::accounts::account::Account; - -/// 账号服务。 -#[derive(Debug, Clone, Default)] -pub struct AccountService; - -impl AccountService { - /// 判断账号配额锁定是否已经释放。 - pub fn quota_available_at( - account: &Account, - now: DateTime, - skip_quota_limited: bool, - ) -> bool { - if !skip_quota_limited || !account.quota_limit_reached { - return true; - } - account - .quota_cooldown_until - .is_some_and(|cooldown_until| now >= cooldown_until) - } - - /// 判断 Cloudflare 冷却是否已经释放。 - pub fn cloudflare_available_at(account: &Account, now: DateTime) -> bool { - account - .cloudflare_cooldown_until - .is_none_or(|cooldown_until| now >= cooldown_until) - } -} diff --git a/backend/src/accounts/store.rs b/backend/src/accounts/store.rs index 3e3a60630..08ccd714d 100644 --- a/backend/src/accounts/store.rs +++ b/backend/src/accounts/store.rs @@ -7,8 +7,7 @@ use serde_json::Value; use sqlx::{PgPool, Postgres, QueryBuilder, Row}; use thiserror::Error; -use crate::accounts::account::{Account, AccountModelUsageDelta, AccountStatus, AccountUsageDelta}; -use crate::accounts::window::should_reset_usage_window; +use crate::accounts::account::{Account, AccountStatus}; use crate::infra::{ json::{decode_cursor, page_offset, NumberedPage, Page}, time::parse_rfc3339_utc, @@ -20,9 +19,8 @@ mod rows; use queries::*; use rows::{ count_account_metadata, get_pool_account, list_pool_accounts, map_account_store_error, - metadata_from_row, optional_positive_i64_to_u64, optional_update_value, pg_usage_delta, - push_account_metadata_search, quota_plan_type, quota_snapshot_from_row, status_to_db, - stored_account_from_row, to_page, u64_to_i64_saturating, + metadata_from_row, optional_update_value, push_account_metadata_search, quota_plan_type, + quota_snapshot_from_row, status_to_db, stored_account_from_row, to_page, }; // ============================================================================ @@ -210,30 +208,6 @@ pub struct AccountQuotaSnapshot { pub quota_fetched_at: Option>, } -/// 用量增量（PostgreSQL 内部表示）。 -#[derive(Debug, Clone, Copy, Default)] -pub struct UsageDelta { - request_count: i64, - empty_response_count: i64, - input_tokens: i64, - output_tokens: i64, - cached_tokens: i64, - reasoning_tokens: i64, - total_tokens: i64, - image_input_tokens: i64, - image_output_tokens: i64, - image_request_count: i64, - image_request_failed_count: i64, - window_request_count: i64, - window_input_tokens: i64, - window_output_tokens: i64, - window_cached_tokens: i64, - window_image_input_tokens: i64, - window_image_output_tokens: i64, - window_image_request_count: i64, - window_image_request_failed_count: i64, -} - // ============================================================================ // AccountStore 端口 trait // ============================================================================ @@ -284,21 +258,6 @@ pub trait AccountStore: Send + Sync + 'static { async fn set_status(&self, account_id: &str, status: AccountStatus) -> AccountStoreResult; - /// 记录账号用量增量。 - async fn record_usage_delta( - &self, - account_id: &str, - usage: AccountUsageDelta, - ) -> AccountStoreResult<()>; - - /// 记录账号模型维度用量增量。 - async fn record_model_usage_delta( - &self, - account_id: &str, - model: &str, - usage: AccountModelUsageDelta, - ) -> AccountStoreResult<()>; - /// 读取账号当前配额 JSON。 async fn get_quota_json(&self, account_id: &str) -> AccountStoreResult>; @@ -312,31 +271,7 @@ pub trait AccountStore: Send + Sync + 'static { ) -> AccountStoreResult; /// 同步运行时自然刷新出来的账号状态。 - async fn sync_runtime_account_state( - &self, - account: &Account, - sync_usage_window: bool, - ) -> AccountStoreResult; - - /// 同步账号当前 rate-limit 统计窗口。 - async fn sync_rate_limit_window( - &self, - account_id: &str, - reset_at: DateTime, - limit_window_seconds: Option, - ) -> AccountStoreResult<()>; - - /// 记录账号被用于一次外部请求。 - async fn record_request(&self, account_id: &str) -> AccountStoreResult<()> { - self.record_usage_delta( - account_id, - AccountUsageDelta { - requests: 1, - ..AccountUsageDelta::default() - }, - ) - .await - } + async fn sync_runtime_account_state(&self, account: &Account) -> AccountStoreResult; } #[derive(Clone)] @@ -409,27 +344,6 @@ impl AccountStore for PgAccountStore { .map_err(|error| map_account_store_error(&error)) } - async fn record_usage_delta( - &self, - account_id: &str, - usage: AccountUsageDelta, - ) -> AccountStoreResult<()> { - self.record_usage(account_id, pg_usage_delta(usage)) - .await - .map_err(|error| map_account_store_error(&error)) - } - - async fn record_model_usage_delta( - &self, - account_id: &str, - model: &str, - usage: AccountModelUsageDelta, - ) -> AccountStoreResult<()> { - self.record_model_usage(account_id, model, usage) - .await - .map_err(|error| map_account_store_error(&error)) - } - async fn get_quota_json(&self, account_id: &str) -> AccountStoreResult> { PgAccountStore::get_quota_json(self, account_id) .await @@ -454,23 +368,8 @@ impl AccountStore for PgAccountStore { .map_err(|error| map_account_store_error(&error)) } - async fn sync_runtime_account_state( - &self, - account: &Account, - sync_usage_window: bool, - ) -> AccountStoreResult { - PgAccountStore::sync_runtime_account_state(self, account, sync_usage_window) - .await - .map_err(|error| map_account_store_error(&error)) - } - - async fn sync_rate_limit_window( - &self, - account_id: &str, - reset_at: DateTime, - limit_window_seconds: Option, - ) -> AccountStoreResult<()> { - PgAccountStore::sync_rate_limit_window(self, account_id, reset_at, limit_window_seconds) + async fn sync_runtime_account_state(&self, account: &Account) -> AccountStoreResult { + PgAccountStore::sync_runtime_account_state(self, account) .await .map_err(|error| map_account_store_error(&error)) } diff --git a/backend/src/accounts/store/queries.rs b/backend/src/accounts/store/queries.rs index a669bb6a2..a00a5f324 100644 --- a/backend/src/accounts/store/queries.rs +++ b/backend/src/accounts/store/queries.rs @@ -6,85 +6,21 @@ pub(super) const LIST_POOL_ACCOUNTS_SQL: &str = r" select - a.id, - a.email, - a.chatgpt_account_id as account_id, - a.chatgpt_user_id as user_id, - a.label, - a.plan_type, - a.access_token, - a.refresh_token, - a.access_token_expires_at, - a.next_refresh_at, - a.status, - a.quota_limit_reached, - a.quota_verify_required, - a.quota_cooldown_until, - a.quota_json, - a.cloudflare_cooldown_until, - a.added_at, - coalesce(au.request_count, 0) as usage_request_count, - coalesce(au.empty_response_count, 0) as usage_empty_response_count, - coalesce(au.image_input_tokens, 0) as usage_image_input_tokens, - coalesce(au.image_output_tokens, 0) as usage_image_output_tokens, - coalesce(au.image_request_count, 0) as usage_image_request_count, - coalesce(au.image_request_failed_count, 0) as usage_image_request_failed_count, - coalesce(au.window_request_count, 0) as usage_window_request_count, - coalesce(au.window_input_tokens, 0) as usage_window_input_tokens, - coalesce(au.window_output_tokens, 0) as usage_window_output_tokens, - coalesce(au.window_cached_tokens, 0) as usage_window_cached_tokens, - coalesce(au.window_image_input_tokens, 0) as usage_window_image_input_tokens, - coalesce(au.window_image_output_tokens, 0) as usage_window_image_output_tokens, - coalesce(au.window_image_request_count, 0) as usage_window_image_request_count, - coalesce(au.window_image_request_failed_count, 0) as usage_window_image_request_failed_count, - au.window_started_at as usage_window_started_at, - au.window_reset_at as usage_window_reset_at, - au.limit_window_seconds as usage_limit_window_seconds, - au.last_used_at as usage_last_used_at -from accounts a -left join account_usage au on au.account_id = a.id -order by a.added_at asc, a.id asc"; + id, email, chatgpt_account_id as account_id, chatgpt_user_id as user_id, + label, plan_type, access_token, refresh_token, access_token_expires_at, + next_refresh_at, status, quota_limit_reached, quota_verify_required, + quota_cooldown_until, quota_json, cloudflare_cooldown_until, added_at +from accounts +order by added_at asc, id asc"; pub(super) const GET_POOL_ACCOUNT_SQL: &str = r" select - a.id, - a.email, - a.chatgpt_account_id as account_id, - a.chatgpt_user_id as user_id, - a.label, - a.plan_type, - a.access_token, - a.refresh_token, - a.access_token_expires_at, - a.next_refresh_at, - a.status, - a.quota_limit_reached, - a.quota_verify_required, - a.quota_cooldown_until, - a.quota_json, - a.cloudflare_cooldown_until, - a.added_at, - coalesce(au.request_count, 0) as usage_request_count, - coalesce(au.empty_response_count, 0) as usage_empty_response_count, - coalesce(au.image_input_tokens, 0) as usage_image_input_tokens, - coalesce(au.image_output_tokens, 0) as usage_image_output_tokens, - coalesce(au.image_request_count, 0) as usage_image_request_count, - coalesce(au.image_request_failed_count, 0) as usage_image_request_failed_count, - coalesce(au.window_request_count, 0) as usage_window_request_count, - coalesce(au.window_input_tokens, 0) as usage_window_input_tokens, - coalesce(au.window_output_tokens, 0) as usage_window_output_tokens, - coalesce(au.window_cached_tokens, 0) as usage_window_cached_tokens, - coalesce(au.window_image_input_tokens, 0) as usage_window_image_input_tokens, - coalesce(au.window_image_output_tokens, 0) as usage_window_image_output_tokens, - coalesce(au.window_image_request_count, 0) as usage_window_image_request_count, - coalesce(au.window_image_request_failed_count, 0) as usage_window_image_request_failed_count, - au.window_started_at as usage_window_started_at, - au.window_reset_at as usage_window_reset_at, - au.limit_window_seconds as usage_limit_window_seconds, - au.last_used_at as usage_last_used_at -from accounts a -left join account_usage au on au.account_id = a.id -where a.id = $1"; + id, email, chatgpt_account_id as account_id, chatgpt_user_id as user_id, + label, plan_type, access_token, refresh_token, access_token_expires_at, + next_refresh_at, status, quota_limit_reached, quota_verify_required, + quota_cooldown_until, quota_json, cloudflare_cooldown_until, added_at +from accounts +where id = $1"; pub(super) const INSERT_ACCOUNT_SQL: &str = r" insert into accounts ( @@ -266,73 +202,6 @@ select updated_at from accounts"; -pub(super) const RECORD_USAGE_SQL: &str = r" -insert into account_usage ( - account_id, - request_count, - empty_response_count, - input_tokens, - output_tokens, - cached_tokens, - reasoning_tokens, - total_tokens, - image_input_tokens, - image_output_tokens, - image_request_count, - image_request_failed_count, - window_request_count, - window_input_tokens, - window_output_tokens, - window_cached_tokens, - window_image_input_tokens, - window_image_output_tokens, - window_image_request_count, - window_image_request_failed_count, - window_started_at, - last_used_at -) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) -on conflict(account_id) do update set - request_count = account_usage.request_count + excluded.request_count, - empty_response_count = account_usage.empty_response_count + excluded.empty_response_count, - input_tokens = account_usage.input_tokens + excluded.input_tokens, - output_tokens = account_usage.output_tokens + excluded.output_tokens, - cached_tokens = account_usage.cached_tokens + excluded.cached_tokens, - reasoning_tokens = account_usage.reasoning_tokens + excluded.reasoning_tokens, - total_tokens = account_usage.total_tokens + excluded.total_tokens, - image_input_tokens = account_usage.image_input_tokens + excluded.image_input_tokens, - image_output_tokens = account_usage.image_output_tokens + excluded.image_output_tokens, - image_request_count = account_usage.image_request_count + excluded.image_request_count, - image_request_failed_count = account_usage.image_request_failed_count + excluded.image_request_failed_count, - window_request_count = account_usage.window_request_count + excluded.window_request_count, - window_input_tokens = account_usage.window_input_tokens + excluded.window_input_tokens, - window_output_tokens = account_usage.window_output_tokens + excluded.window_output_tokens, - window_cached_tokens = account_usage.window_cached_tokens + excluded.window_cached_tokens, - window_image_input_tokens = account_usage.window_image_input_tokens + excluded.window_image_input_tokens, - window_image_output_tokens = account_usage.window_image_output_tokens + excluded.window_image_output_tokens, - window_image_request_count = account_usage.window_image_request_count + excluded.window_image_request_count, - window_image_request_failed_count = account_usage.window_image_request_failed_count + excluded.window_image_request_failed_count, - window_started_at = coalesce(account_usage.window_started_at, excluded.window_started_at), - last_used_at = excluded.last_used_at"; - -pub(super) const RECORD_MODEL_USAGE_SQL: &str = r" -insert into account_model_usage ( - account_id, - model, - request_count, - error_count, - input_tokens, - output_tokens, - cached_tokens, - last_used_at -) values ($1, $2, $3, $4, $5, $6, $7, $8) -on conflict(account_id, model) do update set - request_count = account_model_usage.request_count + excluded.request_count, - error_count = account_model_usage.error_count + excluded.error_count, - input_tokens = account_model_usage.input_tokens + excluded.input_tokens, - output_tokens = account_model_usage.output_tokens + excluded.output_tokens, - cached_tokens = account_model_usage.cached_tokens + excluded.cached_tokens, - last_used_at = excluded.last_used_at"; - pub(super) const LIST_QUOTA_SNAPSHOTS_SQL: &str = r" select id, @@ -370,51 +239,6 @@ set updated_at = $7 where id = $8"; -pub(super) const SELECT_RATE_LIMIT_WINDOW_SQL: &str = r" -select - window_reset_at, - limit_window_seconds -from account_usage -where account_id = $1"; - -pub(super) const SYNC_RATE_LIMIT_WINDOW_RESET_SQL: &str = r" -insert into account_usage ( - account_id, - window_request_count, - window_input_tokens, - window_output_tokens, - window_cached_tokens, - window_image_input_tokens, - window_image_output_tokens, - window_image_request_count, - window_image_request_failed_count, - window_started_at, - window_reset_at, - limit_window_seconds -) values ($1, 0, 0, 0, 0, 0, 0, 0, 0, $2, $3, $4) -on conflict(account_id) do update set - window_request_count = 0, - window_input_tokens = 0, - window_output_tokens = 0, - window_cached_tokens = 0, - window_image_input_tokens = 0, - window_image_output_tokens = 0, - window_image_request_count = 0, - window_image_request_failed_count = 0, - window_started_at = excluded.window_started_at, - window_reset_at = excluded.window_reset_at, - limit_window_seconds = coalesce(excluded.limit_window_seconds, account_usage.limit_window_seconds)"; - -pub(super) const SYNC_RATE_LIMIT_WINDOW_SQL: &str = r" -insert into account_usage ( - account_id, - window_reset_at, - limit_window_seconds -) values ($1, $2, $3) -on conflict(account_id) do update set - window_reset_at = excluded.window_reset_at, - limit_window_seconds = coalesce(excluded.limit_window_seconds, account_usage.limit_window_seconds)"; - pub(super) const MARK_QUOTA_LIMITED_UNTIL_SQL: &str = r" update accounts set @@ -460,37 +284,6 @@ set updated_at = $17 where id = $18"; -pub(super) const SYNC_RUNTIME_ACCOUNT_USAGE_WINDOW_SQL: &str = r" -insert into account_usage ( - account_id, - window_request_count, - window_input_tokens, - window_output_tokens, - window_cached_tokens, - window_image_input_tokens, - window_image_output_tokens, - window_image_request_count, - window_image_request_failed_count, - window_started_at, - window_reset_at, - limit_window_seconds -) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) -on conflict(account_id) do update set - window_request_count = excluded.window_request_count, - window_input_tokens = excluded.window_input_tokens, - window_output_tokens = excluded.window_output_tokens, - window_cached_tokens = excluded.window_cached_tokens, - window_image_input_tokens = excluded.window_image_input_tokens, - window_image_output_tokens = excluded.window_image_output_tokens, - window_image_request_count = excluded.window_image_request_count, - window_image_request_failed_count = excluded.window_image_request_failed_count, - window_started_at = excluded.window_started_at, - window_reset_at = excluded.window_reset_at, - limit_window_seconds = excluded.limit_window_seconds -where account_usage.window_reset_at is null - or account_usage.window_reset_at <= excluded.window_reset_at - or account_usage.window_reset_at <= $13"; - pub(super) const SET_CLOUDFLARE_COOLDOWN_UNTIL_SQL: &str = r" update accounts set diff --git a/backend/src/accounts/store/rows.rs b/backend/src/accounts/store/rows.rs index 6e2466324..1ba7c06e5 100644 --- a/backend/src/accounts/store/rows.rs +++ b/backend/src/accounts/store/rows.rs @@ -6,7 +6,7 @@ use sqlx::{postgres::PgRow, PgPool, Postgres, QueryBuilder, Row}; use crate::{ accounts::{ - account::{Account, AccountStatus, AccountUsageDelta}, + account::{Account, AccountStatus}, quota::{quota_snapshot_limit_window_seconds, quota_snapshot_reset_at}, }, infra::{format::nonnegative_i64_to_u64, json::Page}, @@ -15,7 +15,7 @@ use crate::{ use super::{ queries::{GET_POOL_ACCOUNT_SQL, LIST_POOL_ACCOUNTS_SQL}, AccountQuotaSnapshot, AccountStoreError, PgAccountStore, PgAccountStoreError, - PgAccountStoreResult, StoredAccount, StoredAccountMetadata, UsageDelta, + PgAccountStoreResult, StoredAccount, StoredAccountMetadata, }; pub(super) async fn list_pool_accounts( @@ -69,52 +69,27 @@ pub(super) fn pool_account_from_row(row: &PgRow) -> PgAccountStoreResult("usage_request_count")), - empty_response_count: nonnegative_i64_to_u64( - row.get::("usage_empty_response_count"), - ), - image_input_tokens: nonnegative_i64_to_u64(row.get::("usage_image_input_tokens")), - image_output_tokens: nonnegative_i64_to_u64(row.get::("usage_image_output_tokens")), - image_request_count: nonnegative_i64_to_u64(row.get::("usage_image_request_count")), - image_request_failed_count: nonnegative_i64_to_u64( - row.get::("usage_image_request_failed_count"), - ), - window_request_count: nonnegative_i64_to_u64( - row.get::("usage_window_request_count"), - ), - window_input_tokens: nonnegative_i64_to_u64(row.get::("usage_window_input_tokens")), - window_output_tokens: nonnegative_i64_to_u64( - row.get::("usage_window_output_tokens"), - ), - window_cached_tokens: nonnegative_i64_to_u64( - row.get::("usage_window_cached_tokens"), - ), - window_image_input_tokens: nonnegative_i64_to_u64( - row.get::("usage_window_image_input_tokens"), - ), - window_image_output_tokens: nonnegative_i64_to_u64( - row.get::("usage_window_image_output_tokens"), - ), - window_image_request_count: nonnegative_i64_to_u64( - row.get::("usage_window_image_request_count"), - ), - window_image_request_failed_count: nonnegative_i64_to_u64( - row.get::("usage_window_image_request_failed_count"), - ), - window_started_at: row.get("usage_window_started_at"), - window_reset_at: row - .get::>, _>("usage_window_reset_at") - .or(quota_window_reset_at), - limit_window_seconds: optional_positive_i64_to_u64( - row.get::, _>("usage_limit_window_seconds"), - ) - .or(quota_limit_window_seconds), + request_count: 0, + empty_response_count: 0, + image_input_tokens: 0, + image_output_tokens: 0, + image_request_count: 0, + image_request_failed_count: 0, + window_request_count: 0, + window_input_tokens: 0, + window_output_tokens: 0, + window_cached_tokens: 0, + window_image_input_tokens: 0, + window_image_output_tokens: 0, + window_image_request_count: 0, + window_image_request_failed_count: 0, + window_started_at: None, + window_reset_at: quota_window_reset_at, + limit_window_seconds: quota_limit_window_seconds, added_at: row .get::, _>("added_at") .to_rfc3339(), - last_used_at: row - .get::>, _>("usage_last_used_at") - .map(|value| value.to_rfc3339()), + last_used_at: None, }) } @@ -168,38 +143,6 @@ pub(super) fn map_account_store_error(error: &impl ToString) -> AccountStoreErro } } -pub(super) fn pg_usage_delta(usage: AccountUsageDelta) -> UsageDelta { - let request_count = u64_to_i64_saturating(usage.requests); - let input_tokens = u64_to_i64_saturating(usage.input_tokens); - let output_tokens = u64_to_i64_saturating(usage.output_tokens); - let cached_tokens = u64_to_i64_saturating(usage.cached_tokens); - let image_input_tokens = u64_to_i64_saturating(usage.image_input_tokens); - let image_output_tokens = u64_to_i64_saturating(usage.image_output_tokens); - let image_request_count = u64_to_i64_saturating(usage.image_requests); - let image_request_failed_count = u64_to_i64_saturating(usage.image_request_failures); - UsageDelta { - request_count, - empty_response_count: u64_to_i64_saturating(usage.empty_responses), - input_tokens, - output_tokens, - cached_tokens, - reasoning_tokens: u64_to_i64_saturating(usage.reasoning_tokens), - total_tokens: u64_to_i64_saturating(usage.total_tokens), - image_input_tokens, - image_output_tokens, - image_request_count, - image_request_failed_count, - window_request_count: request_count, - window_input_tokens: input_tokens, - window_output_tokens: output_tokens, - window_cached_tokens: cached_tokens, - window_image_input_tokens: image_input_tokens, - window_image_output_tokens: image_output_tokens, - window_image_request_count: image_request_count, - window_image_request_failed_count: image_request_failed_count, - } -} - pub(super) fn quota_plan_type(quota_json: &str) -> Option { serde_json::from_str::(quota_json) .ok()? @@ -245,16 +188,6 @@ pub(super) fn status_from_db(value: &str) -> PgAccountStoreResult } } -pub(super) fn u64_to_i64_saturating(value: u64) -> i64 { - value.min(i64::MAX as u64) as i64 -} - -pub(super) fn optional_positive_i64_to_u64(value: Option) -> Option { - value - .and_then(|value| u64::try_from(value).ok()) - .filter(|value| *value > 0) -} - pub(super) async fn count_account_metadata( pool: &PgPool, search: Option<&str>, diff --git a/backend/src/accounts/store/write.rs b/backend/src/accounts/store/write.rs index a31d6da0c..0a4eced55 100644 --- a/backend/src/accounts/store/write.rs +++ b/backend/src/accounts/store/write.rs @@ -345,14 +345,12 @@ impl PgAccountStore { pub async fn sync_runtime_account_state( &self, account: &Account, - sync_usage_window: bool, ) -> PgAccountStoreResult { let now = Utc::now(); let quota_limit_reached = account.quota_limit_reached; let quota_verify_required = account.quota_verify_required; let quota_cooldown_until = account.quota_cooldown_until; let cloudflare_cooldown_until = account.cloudflare_cooldown_until; - let mut tx = self.pool.begin().await?; let result = sqlx::query(SYNC_RUNTIME_ACCOUNT_STATE_SQL) .bind(quota_limit_reached) .bind(now) @@ -372,36 +370,9 @@ impl PgAccountStore { .bind(cloudflare_cooldown_until) .bind(now) .bind(&account.id) - .execute(&mut *tx) + .execute(&self.pool) .await?; - if result.rows_affected() == 0 { - tx.commit().await?; - return Ok(false); - } - - if sync_usage_window { - sqlx::query(SYNC_RUNTIME_ACCOUNT_USAGE_WINDOW_SQL) - .bind(&account.id) - .bind(u64_to_i64_saturating(account.window_request_count)) - .bind(u64_to_i64_saturating(account.window_input_tokens)) - .bind(u64_to_i64_saturating(account.window_output_tokens)) - .bind(u64_to_i64_saturating(account.window_cached_tokens)) - .bind(u64_to_i64_saturating(account.window_image_input_tokens)) - .bind(u64_to_i64_saturating(account.window_image_output_tokens)) - .bind(u64_to_i64_saturating(account.window_image_request_count)) - .bind(u64_to_i64_saturating( - account.window_image_request_failed_count, - )) - .bind(account.window_started_at) - .bind(account.window_reset_at) - .bind(account.limit_window_seconds.map(u64_to_i64_saturating)) - .bind(now) - .execute(&mut *tx) - .await?; - } - - tx.commit().await?; - Ok(true) + Ok(result.rows_affected() > 0) } /// 标记账号 Cloudflare 冷却期。 @@ -448,67 +419,6 @@ where id = $4", Ok(result.rows_affected() > 0) } - /// 记录用量。 - pub async fn record_usage( - &self, - account_id: &str, - delta: UsageDelta, - ) -> PgAccountStoreResult<()> { - let last_used_at = Utc::now(); - sqlx::query(RECORD_USAGE_SQL) - .bind(account_id) - .bind(delta.request_count) - .bind(delta.empty_response_count) - .bind(delta.input_tokens) - .bind(delta.output_tokens) - .bind(delta.cached_tokens) - .bind(delta.reasoning_tokens) - .bind(delta.total_tokens) - .bind(delta.image_input_tokens) - .bind(delta.image_output_tokens) - .bind(delta.image_request_count) - .bind(delta.image_request_failed_count) - .bind(delta.window_request_count) - .bind(delta.window_input_tokens) - .bind(delta.window_output_tokens) - .bind(delta.window_cached_tokens) - .bind(delta.window_image_input_tokens) - .bind(delta.window_image_output_tokens) - .bind(delta.window_image_request_count) - .bind(delta.window_image_request_failed_count) - .bind(last_used_at) - .bind(last_used_at) - .execute(&self.pool) - .await?; - Ok(()) - } - - /// 记录模型维度用量。 - pub async fn record_model_usage( - &self, - account_id: &str, - model: &str, - delta: AccountModelUsageDelta, - ) -> PgAccountStoreResult<()> { - let model = model.trim(); - if model.is_empty() { - return Ok(()); - } - let last_used_at = Utc::now(); - sqlx::query(RECORD_MODEL_USAGE_SQL) - .bind(account_id) - .bind(model) - .bind(u64_to_i64_saturating(delta.requests)) - .bind(u64_to_i64_saturating(delta.errors)) - .bind(u64_to_i64_saturating(delta.input_tokens)) - .bind(u64_to_i64_saturating(delta.output_tokens)) - .bind(u64_to_i64_saturating(delta.cached_tokens)) - .bind(last_used_at) - .execute(&self.pool) - .await?; - Ok(()) - } - /// 读取配额快照列表。 pub async fn list_quota_snapshots(&self) -> PgAccountStoreResult> { let rows = sqlx::query(LIST_QUOTA_SNAPSHOTS_SQL) @@ -602,53 +512,6 @@ where id = $4", Ok(result.rows_affected() > 0) } - /// 同步 rate-limit 窗口（含重置）。 - pub async fn sync_rate_limit_window( - &self, - account_id: &str, - reset_at: DateTime, - limit_window_seconds: Option, - ) -> PgAccountStoreResult<()> { - let existing = sqlx::query(SELECT_RATE_LIMIT_WINDOW_SQL) - .bind(account_id) - .fetch_optional(&self.pool) - .await?; - - let should_reset = existing - .as_ref() - .map(|row| { - let existing_reset_at = row.get::>, _>("window_reset_at"); - let existing_limit_window_seconds = - optional_positive_i64_to_u64(row.get::, _>("limit_window_seconds")); - should_reset_usage_window( - existing_reset_at, - existing_limit_window_seconds, - reset_at, - limit_window_seconds, - ) - }) - .unwrap_or_default(); - - if should_reset { - let window_started_at = Utc::now(); - sqlx::query(SYNC_RATE_LIMIT_WINDOW_RESET_SQL) - .bind(account_id) - .bind(window_started_at) - .bind(reset_at) - .bind(limit_window_seconds.map(u64_to_i64_saturating)) - .execute(&self.pool) - .await?; - } else { - sqlx::query(SYNC_RATE_LIMIT_WINDOW_SQL) - .bind(account_id) - .bind(reset_at) - .bind(limit_window_seconds.map(u64_to_i64_saturating)) - .execute(&self.pool) - .await?; - } - Ok(()) - } - /// 删除单个账号。 pub async fn delete(&self, account_id: &str) -> PgAccountStoreResult { let result = sqlx::query(DELETE_ACCOUNT_SQL) diff --git a/backend/src/accounts/window.rs b/backend/src/accounts/window.rs index 2546fef50..3591f8241 100644 --- a/backend/src/accounts/window.rs +++ b/backend/src/accounts/window.rs @@ -1,31 +1,3 @@ -//! 账号用量窗口规则。 +//! 账号配额窗口规则。 -use chrono::{DateTime, Utc}; - -pub(crate) fn should_reset_usage_window( - existing_reset_at: Option>, - existing_limit_window_seconds: Option, - new_reset_at: DateTime, - new_limit_window_seconds: Option, -) -> bool { - let Some(existing_reset_at) = existing_reset_at else { - return false; - }; - if existing_reset_at == new_reset_at { - return false; - } - - let drift = existing_reset_at - .signed_duration_since(new_reset_at) - .num_seconds() - .unsigned_abs(); - let window_seconds = new_limit_window_seconds - .or(existing_limit_window_seconds) - .unwrap_or(0); - let threshold = if window_seconds > 0 { - window_seconds / 2 - } else { - 3_600 - }; - drift >= threshold -} +pub(crate) use crate::telemetry::account_usage::store::should_reset_usage_window; diff --git a/backend/src/api/admin/accounts_routes.rs b/backend/src/api/admin/accounts_routes.rs index d6ee60b85..4485d4c71 100644 --- a/backend/src/api/admin/accounts_routes.rs +++ b/backend/src/api/admin/accounts_routes.rs @@ -30,13 +30,13 @@ use crate::{ }, api::admin::{ response::{ - AdminEnvelope, AdminError, AdminResponse, BatchDeleteData, CursorPageMeta, - NumberedPageMeta, PageMeta, ADMIN_OK_CODE, ADMIN_OK_MESSAGE, + parse_editable_update, AdminEnvelope, AdminError, AdminResponse, BatchDeleteData, + CursorPageMeta, EditableUpdateMessages, NumberedPageMeta, PageMeta, ADMIN_OK_CODE, + ADMIN_OK_MESSAGE, }, session::AdminAuth, - update_payload::{parse_editable_update, EditableUpdateMessages}, }, - bootstrap::state::AppState, + api::AppState, infra::{ format::{ format_cost, format_percent, format_plain_number, format_tokens, nonnegative_i64_to_u64, diff --git a/backend/src/api/admin/accounts_routes/actions.rs b/backend/src/api/admin/accounts_routes/actions.rs index 4dfd8114e..813c7426d 100644 --- a/backend/src/api/admin/accounts_routes/actions.rs +++ b/backend/src/api/admin/accounts_routes/actions.rs @@ -277,13 +277,22 @@ pub(crate) async fn batch_delete_accounts( .batch_delete(payload.ids) .await { - Ok(result) => Ok(AdminResponse::new( - StatusCode::OK, - AdminEnvelope::ok(BatchDeleteData { - deleted: result.deleted, - not_found: result.not_found, - }), - )), + Ok(result) => { + for account_id in &result.deleted_ids { + state + .services + .session_affinity + .forget_account(account_id) + .await; + } + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(BatchDeleteData { + deleted: result.deleted, + not_found: result.not_found, + }), + )) + } Err(error) => Err(account_error(&error)), } } diff --git a/backend/src/api/admin/accounts_routes/stats.rs b/backend/src/api/admin/accounts_routes/stats.rs index 63a04f923..ec6a6ac53 100644 --- a/backend/src/api/admin/accounts_routes/stats.rs +++ b/backend/src/api/admin/accounts_routes/stats.rs @@ -238,9 +238,8 @@ pub(super) async fn quota_snapshots_by_account( ) -> HashMap { state .services - .background_tasks - .accounts - .list_quota_snapshots() + .admin_accounts + .quota_snapshots() .await .unwrap_or_default() .into_iter() diff --git a/backend/src/api/admin/dashboard_routes.rs b/backend/src/api/admin/dashboard_routes.rs index 09e7a6c95..365d1831d 100644 --- a/backend/src/api/admin/dashboard_routes.rs +++ b/backend/src/api/admin/dashboard_routes.rs @@ -14,6 +14,7 @@ use serde_json::Value; use crate::{ accounts::{ account::{Account, AccountStatus}, + pool::AccountCapacitySummary, refresh::token_refresh_status_eligible, }, api::admin::{ @@ -24,21 +25,20 @@ use crate::{ UsageRecordCostDetailsData, UsageRecordTokenDetailsData, }, }, - bootstrap::state::AppState, + api::AppState, infra::{ - format::{ - format_compact_number, format_cost, format_duration_ms, format_percent, format_rate, - format_tokens, nonnegative_i64_to_u64, - }, + format::{format_duration_ms, format_tokens}, time::{ - china_datetime, china_datetime_rfc3339_str, china_day_start, china_hour, - china_hour_start, china_quarter_hour_start, china_relative_time, + china_datetime, china_datetime_rfc3339_str, china_day_start, china_quarter_hour_start, + china_relative_time, }, }, telemetry::{ - account_usage::query::{AccountUsageSummary, AccountUsageTimeBucket}, - diagnostics::{ - fingerprint_diagnostics, AccountCapacityDiagnostics, AccountPoolDiagnostics, + account_usage::query::AccountUsageTimeBucket, + dashboard::{ + dashboard_cards, dashboard_health_timeline_data, dashboard_trend_data, + DashboardAccountCounts, DashboardCardsData, DashboardHealthTimelineData, + DashboardTrendData, DashboardTrendKind, }, usage::{ query::UsageQueryFilter, @@ -46,14 +46,13 @@ use crate::{ types::{metadata_string, UsageRecord}, }, }, + upstream::openai::fingerprint::Fingerprint, }; const DASHBOARD_ACCOUNT_USAGE_LIMIT: u32 = 4; const DASHBOARD_USAGE_RECORD_LIMIT: u32 = 10; -const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15; +const DASHBOARD_TIME_BUCKET_MINUTES: i64 = 15; const DASHBOARD_TIME_BUCKET_SLOTS: i64 = 7 * 24 * 4; -const HEALTH_TIMELINE_SLOTS: i64 = 24 * 4; -const HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD: u64 = 3; const FIVE_HOUR_WINDOW_SECONDS: u64 = 18_000; const WEEK_WINDOW_SECONDS: u64 = 604_800; const MONTH_WINDOW_SECONDS: u64 = 2_592_000; @@ -64,15 +63,6 @@ pub(crate) struct DashboardTrendQuery { kind: Option, } -#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -enum DashboardTrendKind { - #[default] - Usage, - Latency, - Errors, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct DashboardSummaryData { @@ -87,107 +77,6 @@ struct DashboardSummaryData { rotation_strategy: Option, } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardHealthTimelineData { - title: String, - description: String, - reliability_display: String, - points: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardCardsData { - accounts: DashboardAccountsCardData, - traffic: DashboardTrafficCardData, - tokens: DashboardTokenCardData, - cache: DashboardCacheCardData, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardAccountsCardData { - total: String, - total_value: u64, - enabled: String, - enabled_value: u64, - abnormal: String, - abnormal_value: u64, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardTrafficCardData { - today_requests: String, - today_requests_value: u64, - yesterday_requests_value: u64, - total_requests: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardTokenCardData { - today_tokens: String, - today_tokens_value: u64, - yesterday_tokens_value: u64, - total_tokens: String, - today_cost_usd: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardCacheCardData { - today_hit_rate: String, - today_hit_rate_value: Option, - yesterday_hit_rate_value: Option, - total_hit_rate: String, - total_cached_tokens: String, - average_first_token_latency_ms: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardTrendData { - kind: DashboardTrendKind, - points: Vec, - summary: Vec, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardTrendPointData { - time: String, - requests: String, - requests_value: u64, - input_tokens: String, - input_tokens_value: u64, - output_tokens: String, - output_tokens_value: u64, - cached_tokens: String, - cached_tokens_value: u64, - cache_hit_rate_value: f64, - tokens_value: u64, - errors: String, - errors_value: u64, - latency: String, - latency_value: u64, - max_latency: String, - max_latency_value: u64, - min_latency: String, - min_latency_value: u64, - success_rate: String, - success_rate_value: f64, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct DashboardTrendSummaryData { - label: String, - value: String, - ratio: Option, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct DashboardAccountUsageData { @@ -229,17 +118,65 @@ struct DashboardServiceStatusData { tone: String, } -#[derive(Debug, Clone, Copy, Default)] -struct UsageWindow { - requests: u64, - input_tokens: u64, - output_tokens: u64, - cached_tokens: u64, - errors: u64, - first_token_latency_sum: u64, - first_token_latency_count: u64, - max_first_token_bucket_latency: u64, - min_first_token_bucket_latency: u64, +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct AccountPoolDiagnostics { + total: usize, + active: usize, + expired: usize, + quota_exhausted: usize, + refreshing: usize, + disabled: usize, + banned: usize, +} + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct AccountCapacityDiagnostics { + max_concurrent_per_account: usize, + total_slots: usize, + used_slots: usize, + available_slots: usize, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct FingerprintDiagnostics { + source: &'static str, + originator: String, + app_version: String, + build_number: String, + platform: String, + arch: String, + chromium_version: String, + user_agent: String, + #[serde(skip_serializing_if = "Option::is_none")] + updated_at: Option, +} + +fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics { + FingerprintDiagnostics { + source: "runtime", + originator: fingerprint.originator.clone(), + app_version: fingerprint.app_version.clone(), + build_number: fingerprint.build_number.clone(), + platform: fingerprint.platform.clone(), + arch: fingerprint.arch.clone(), + chromium_version: fingerprint.chromium_version.clone(), + user_agent: fingerprint.user_agent(), + updated_at: fingerprint.updated_at.clone(), + } +} + +impl From for AccountCapacityDiagnostics { + fn from(summary: AccountCapacitySummary) -> Self { + Self { + max_concurrent_per_account: summary.max_concurrent_per_account, + total_slots: summary.total_slots, + used_slots: summary.used_slots, + available_slots: summary.available_slots, + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -250,33 +187,6 @@ enum QuotaWindowPriority { Other, } -impl UsageWindow { - fn tokens(self) -> u64 { - self.input_tokens + self.output_tokens - } - - fn cache_hit_rate(self) -> Option { - if self.input_tokens > 0 { - Some(self.cached_tokens as f64 / self.input_tokens as f64) - } else { - None - } - } - - fn avg_first_token_latency(self) -> Option { - self.first_token_latency_sum - .checked_div(self.first_token_latency_count) - } - - fn max_first_token_bucket_latency(self) -> Option { - (self.max_first_token_bucket_latency > 0).then_some(self.max_first_token_bucket_latency) - } - - fn min_first_token_bucket_latency(self) -> Option { - (self.min_first_token_bucket_latency > 0).then_some(self.min_first_token_bucket_latency) - } -} - /// `GET /api/admin/dashboard/summary` pub(crate) async fn dashboard_summary( State(state): State, @@ -322,7 +232,7 @@ pub(crate) async fn dashboard_summary( let trend = dashboard_trend_data(&time_buckets, query.kind.unwrap_or_default()); let quota_used_by_account = account_quota_used_percent_by_id(&state, &account_usage_records).await; - let settings = state.services.settings.current().await; + let settings = state.services.settings.current(); let recent_usage_records = recent_events; let account_emails = state .services @@ -336,7 +246,11 @@ pub(crate) async fn dashboard_summary( Ok(AdminResponse::new( StatusCode::OK, AdminEnvelope::ok(DashboardSummaryData { - cards: dashboard_cards(&accounts, &time_buckets, &lifetime_usage), + cards: dashboard_cards( + dashboard_account_counts(&accounts), + &time_buckets, + &lifetime_usage, + ), trend, health_timeline: dashboard_health_timeline_data(&time_buckets), account_usage: account_usage_data( diff --git a/backend/src/api/admin/dashboard_routes/data.rs b/backend/src/api/admin/dashboard_routes/data.rs index 02e898b1d..eb113891b 100644 --- a/backend/src/api/admin/dashboard_routes/data.rs +++ b/backend/src/api/admin/dashboard_routes/data.rs @@ -1,102 +1,12 @@ use super::*; -pub(super) fn dashboard_cards( - accounts: &[Account], - buckets: &[AccountUsageTimeBucket], - lifetime_usage: &AccountUsageSummary, -) -> DashboardCardsData { - dashboard_cards_at(accounts, buckets, lifetime_usage, Utc::now()) -} - -pub(super) fn dashboard_cards_at( - accounts: &[Account], - buckets: &[AccountUsageTimeBucket], - lifetime_usage: &AccountUsageSummary, - now: DateTime, -) -> DashboardCardsData { - let today_start = china_day_start(now); - let yesterday_start = today_start - Duration::days(1); - let today = usage_window(buckets, today_start, now); - let yesterday = usage_window(buckets, yesterday_start, today_start); - let today_cost = cost_window(buckets, today_start, now).unwrap_or(0.0); - let total_requests = nonnegative_i64_to_u64(lifetime_usage.request_count); - let total_tokens = nonnegative_i64_to_u64(lifetime_usage.total_tokens); - let total_cached_tokens = nonnegative_i64_to_u64(lifetime_usage.cached_tokens); - let total_hit_rate = summary_cache_hit_rate(lifetime_usage); - - let total_accounts = accounts.len() as u64; - let enabled_accounts = accounts - .iter() - .filter(|account| account.status == AccountStatus::Active) - .count() as u64; - let abnormal_accounts = accounts - .iter() - .filter(|account| { - matches!( - account.status, - AccountStatus::Expired - | AccountStatus::QuotaExhausted - | AccountStatus::Disabled - | AccountStatus::Banned - ) - }) - .count() as u64; - - DashboardCardsData { - accounts: DashboardAccountsCardData { - total: format_compact_number(total_accounts), - total_value: total_accounts, - enabled: format_compact_number(enabled_accounts), - enabled_value: enabled_accounts, - abnormal: format_compact_number(abnormal_accounts), - abnormal_value: abnormal_accounts, - }, - traffic: DashboardTrafficCardData { - today_requests: format_compact_number(today.requests), - today_requests_value: today.requests, - yesterday_requests_value: yesterday.requests, - total_requests: format_compact_number(total_requests), - }, - tokens: DashboardTokenCardData { - today_tokens: format_tokens(today.tokens()), - today_tokens_value: today.tokens(), - yesterday_tokens_value: yesterday.tokens(), - total_tokens: format_tokens(total_tokens), - today_cost_usd: format_cost(today_cost), - }, - cache: DashboardCacheCardData { - today_hit_rate: format_rate(today.cache_hit_rate()), - today_hit_rate_value: today.cache_hit_rate(), - yesterday_hit_rate_value: yesterday.cache_hit_rate(), - total_hit_rate: format_rate(total_hit_rate), - total_cached_tokens: format_tokens(total_cached_tokens), - average_first_token_latency_ms: format_optional_duration_ms( - today.avg_first_token_latency(), - ), - }, - } -} - -pub(super) fn summary_cache_hit_rate(summary: &AccountUsageSummary) -> Option { - let input_tokens = nonnegative_i64_to_u64(summary.input_tokens); - if input_tokens > 0 { - Some(nonnegative_i64_to_u64(summary.cached_tokens) as f64 / input_tokens as f64) - } else { - None - } -} - -pub(super) fn format_optional_duration_ms(value: Option) -> String { - format_duration_ms(value.and_then(|value| i64::try_from(value).ok())) -} - pub(super) async fn dashboard_time_buckets( state: &AppState, now: DateTime, ) -> Vec { let current_slot = china_quarter_hour_start(now); let start = current_slot - - Duration::minutes(HEALTH_TIMELINE_SLOT_MINUTES * (DASHBOARD_TIME_BUCKET_SLOTS - 1)); + - Duration::minutes(DASHBOARD_TIME_BUCKET_MINUTES * (DASHBOARD_TIME_BUCKET_SLOTS - 1)); state .services .usage @@ -105,332 +15,18 @@ pub(super) async fn dashboard_time_buckets( .unwrap_or_default() } -pub(super) fn dashboard_trend_data( - buckets: &[AccountUsageTimeBucket], - kind: DashboardTrendKind, -) -> DashboardTrendData { - dashboard_trend_data_at(buckets, kind, Utc::now()) -} - -pub(super) fn dashboard_trend_data_at( - records: &[AccountUsageTimeBucket], - kind: DashboardTrendKind, - now: DateTime, -) -> DashboardTrendData { - let current_hour = china_hour_start(now); - let start = china_day_start(now); - let elapsed_hours = current_hour.signed_duration_since(start).num_hours(); - let mut buckets = (0..=elapsed_hours) - .map(|index| { - let bucket_start = start + Duration::hours(index); - (bucket_start, UsageWindow::default()) - }) - .collect::>(); - - for record in records { - if record.bucket_start < start || record.bucket_start > now { - continue; - } - let record_hour = china_hour_start(record.bucket_start); - if let Some((_, bucket)) = buckets - .iter_mut() - .find(|(bucket_start, _)| *bucket_start == record_hour) - { - apply_bucket(bucket, record); - } +pub(super) fn dashboard_account_counts(accounts: &[Account]) -> DashboardAccountCounts { + DashboardAccountCounts { + total: accounts.len() as u64, + enabled: accounts + .iter() + .filter(|account| account.status == AccountStatus::Active) + .count() as u64, + abnormal: accounts + .iter() + .filter(|account| account.status != AccountStatus::Active) + .count() as u64, } - - let points = buckets - .iter() - .map(|(bucket_start, bucket)| { - let latency = bucket.avg_first_token_latency().unwrap_or(0); - let max_latency = bucket.max_first_token_bucket_latency().unwrap_or(0); - let min_latency = bucket.min_first_token_bucket_latency().unwrap_or(0); - let tokens = bucket.tokens(); - let cache_hit_rate = bucket.cache_hit_rate().unwrap_or(0.0); - let success_rate = if bucket.requests > 0 { - ((bucket.requests - bucket.errors) as f64 / bucket.requests as f64 * 1000.0).round() - / 10.0 - } else { - 0.0 - }; - DashboardTrendPointData { - time: format!("{:02}", china_hour(bucket_start)), - requests: format_compact_number(bucket.requests), - requests_value: bucket.requests, - input_tokens: format_tokens(bucket.input_tokens), - input_tokens_value: bucket.input_tokens, - output_tokens: format_tokens(bucket.output_tokens), - output_tokens_value: bucket.output_tokens, - cached_tokens: format_tokens(bucket.cached_tokens), - cached_tokens_value: bucket.cached_tokens, - cache_hit_rate_value: cache_hit_rate, - tokens_value: tokens, - errors: format_compact_number(bucket.errors), - errors_value: bucket.errors, - latency: format_optional_duration_ms(Some(latency)), - latency_value: latency, - max_latency: format_optional_duration_ms(Some(max_latency)), - max_latency_value: max_latency, - min_latency: format_optional_duration_ms(Some(min_latency)), - min_latency_value: min_latency, - success_rate: format_percent(success_rate), - success_rate_value: success_rate, - } - }) - .collect::>(); - - DashboardTrendData { - kind, - summary: trend_summary(kind, &points), - points, - } -} - -pub(super) fn dashboard_health_timeline_data( - buckets: &[AccountUsageTimeBucket], -) -> DashboardHealthTimelineData { - dashboard_health_timeline_data_at(buckets, Utc::now()) -} - -pub(super) fn dashboard_health_timeline_data_at( - records: &[AccountUsageTimeBucket], - now: DateTime, -) -> DashboardHealthTimelineData { - let current_slot = china_quarter_hour_start(now); - let start = china_day_start(now); - let mut buckets = (0..HEALTH_TIMELINE_SLOTS) - .map(|index| { - let bucket_start = start + Duration::minutes(HEALTH_TIMELINE_SLOT_MINUTES * index); - (bucket_start, UsageWindow::default()) - }) - .collect::>(); - - for record in records { - if record.bucket_start < start || record.bucket_start > now { - continue; - } - let record_slot = china_quarter_hour_start(record.bucket_start); - if let Some((_, bucket)) = buckets - .iter_mut() - .find(|(bucket_start, _)| *bucket_start == record_slot) - { - apply_bucket(bucket, record); - } - } - - let requests = buckets - .iter() - .filter(|(bucket_start, _)| *bucket_start <= current_slot) - .map(|(_, bucket)| bucket.requests) - .sum::(); - let errors = buckets - .iter() - .filter(|(bucket_start, _)| *bucket_start <= current_slot) - .map(|(_, bucket)| bucket.errors) - .sum::(); - let successes = requests.saturating_sub(errors); - let reliability = if requests > 0 { - (successes as f64 / requests as f64 * 1000.0).round() / 10.0 - } else { - 0.0 - }; - DashboardHealthTimelineData { - title: "请求健康时间线".to_string(), - description: "请求可靠性".to_string(), - reliability_display: if requests > 0 { - format!("{reliability:.1}%") - } else { - "-".to_string() - }, - points: buckets - .into_iter() - .map(|(bucket_start, bucket)| { - let success_rate = if bucket.requests > 0 { - (bucket.requests.saturating_sub(bucket.errors) as f64 / bucket.requests as f64 - * 1000.0) - .round() - / 10.0 - } else { - 0.0 - }; - health_tone(bucket, success_rate, bucket_start > current_slot) - }) - .collect(), - } -} - -pub(super) fn health_tone(bucket: UsageWindow, success_rate: f64, is_future: bool) -> char { - let successes = bucket.requests.saturating_sub(bucket.errors); - if is_future { - '0' - } else if bucket.requests == 0 { - '1' - } else if successes == 0 { - '2' - } else if success_rate < 90.0 { - '3' - } else if successes < HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD { - '4' - } else { - '5' - } -} - -pub(super) fn trend_summary( - kind: DashboardTrendKind, - points: &[DashboardTrendPointData], -) -> Vec { - match kind { - DashboardTrendKind::Usage => vec![ - trend_summary_item( - kind, - "输入", - points.iter().map(|point| point.input_tokens_value).sum(), - None, - ), - trend_summary_item( - kind, - "输出", - points.iter().map(|point| point.output_tokens_value).sum(), - None, - ), - trend_summary_item( - kind, - "缓存", - points.iter().map(|point| point.cached_tokens_value).sum(), - None, - ), - ], - DashboardTrendKind::Latency => { - let samples = points - .iter() - .filter(|point| point.latency_value > 0) - .collect::>(); - let avg = if samples.is_empty() { - 0 - } else { - samples.iter().map(|point| point.latency_value).sum::() / samples.len() as u64 - }; - vec![ - trend_summary_item(kind, "平均", avg, None), - trend_summary_item( - kind, - "最高", - samples - .iter() - .map(|point| point.max_latency_value) - .max() - .unwrap_or(0), - None, - ), - trend_summary_item( - kind, - "最低", - samples - .iter() - .filter_map(|point| { - (point.min_latency_value > 0).then_some(point.min_latency_value) - }) - .min() - .unwrap_or(0), - None, - ), - ] - } - DashboardTrendKind::Errors => { - let errors = points.iter().map(|point| point.errors_value).sum::(); - let requests = points.iter().map(|point| point.requests_value).sum::(); - let success_rate = if requests > 0 { - Some(((requests - errors) as f64 / requests as f64 * 1000.0).round() / 10.0) - } else { - None - }; - vec![ - trend_summary_item(kind, "错误数", errors, None), - trend_summary_item(kind, "成功率", 0, success_rate), - trend_summary_item(kind, "总请求", requests, None), - ] - } - } -} - -pub(super) fn trend_summary_item( - kind: DashboardTrendKind, - label: &str, - value: u64, - ratio: Option, -) -> DashboardTrendSummaryData { - DashboardTrendSummaryData { - label: label.to_string(), - value: trend_summary_value_display(kind, value), - ratio: ratio.map(format_percent), - } -} - -pub(super) fn trend_summary_value_display(kind: DashboardTrendKind, value: u64) -> String { - match kind { - DashboardTrendKind::Usage => format_tokens(value), - DashboardTrendKind::Latency => format_optional_duration_ms(Some(value)), - DashboardTrendKind::Errors => format_compact_number(value), - } -} - -pub(super) fn usage_window( - records: &[AccountUsageTimeBucket], - start: DateTime, - end: DateTime, -) -> UsageWindow { - let mut window = UsageWindow::default(); - for record in records { - if record.bucket_start >= start && record.bucket_start < end { - apply_bucket(&mut window, record); - } - } - window -} - -pub(super) fn cost_window( - records: &[AccountUsageTimeBucket], - start: DateTime, - end: DateTime, -) -> Option { - let mut total = 0.0; - let mut has_usage = false; - for record in records { - if record.bucket_start >= start && record.bucket_start < end { - if let Some(cost) = record.cost_usd { - total += cost; - has_usage = true; - } - } - } - has_usage.then_some(total) -} - -pub(super) fn apply_bucket(window: &mut UsageWindow, record: &AccountUsageTimeBucket) { - window.requests += nonnegative_i64_to_u64(record.request_count); - window.input_tokens += nonnegative_i64_to_u64(record.input_tokens); - window.output_tokens += nonnegative_i64_to_u64(record.output_tokens); - window.cached_tokens += nonnegative_i64_to_u64(record.cached_tokens); - window.errors += nonnegative_i64_to_u64(record.error_count); - window.first_token_latency_sum += nonnegative_i64_to_u64(record.first_token_latency_sum); - window.first_token_latency_count += nonnegative_i64_to_u64(record.first_token_latency_count); - let bucket_first_token_latency = first_token_bucket_latency(record); - if let Some(latency) = bucket_first_token_latency { - window.max_first_token_bucket_latency = window.max_first_token_bucket_latency.max(latency); - window.min_first_token_bucket_latency = if window.min_first_token_bucket_latency == 0 { - latency - } else { - window.min_first_token_bucket_latency.min(latency) - }; - } -} - -pub(super) fn first_token_bucket_latency(record: &AccountUsageTimeBucket) -> Option { - let sum = nonnegative_i64_to_u64(record.first_token_latency_sum); - let count = nonnegative_i64_to_u64(record.first_token_latency_count); - sum.checked_div(count).filter(|latency| *latency > 0) } pub(super) fn account_pool_summary( diff --git a/backend/src/api/admin/keys_routes.rs b/backend/src/api/admin/keys_routes.rs index 7e0728910..5a3f39c40 100644 --- a/backend/src/api/admin/keys_routes.rs +++ b/backend/src/api/admin/keys_routes.rs @@ -11,11 +11,11 @@ use serde_json::Value; use crate::{ api::admin::response::{ - AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse, BatchDeleteData, + parse_editable_update, AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse, + BatchDeleteData, EditableUpdateMessages, }, api::admin::session::AdminAuth, - api::admin::update_payload::{parse_editable_update, EditableUpdateMessages}, - bootstrap::state::AppState, + api::AppState, infra::{ json::{clamp_limit, Page}, time::{china_relative_time_str, china_rfc3339_str}, diff --git a/backend/src/api/admin/mod.rs b/backend/src/api/admin/mod.rs index dc908655d..e657c55f6 100644 --- a/backend/src/api/admin/mod.rs +++ b/backend/src/api/admin/mod.rs @@ -9,5 +9,4 @@ pub mod router; pub mod session; pub mod settings_routes; pub mod system_routes; -pub mod update_payload; pub mod usage_routes; diff --git a/backend/src/api/admin/ops_routes.rs b/backend/src/api/admin/ops_routes.rs index d6aeb6846..75792648d 100644 --- a/backend/src/api/admin/ops_routes.rs +++ b/backend/src/api/admin/ops_routes.rs @@ -12,7 +12,7 @@ use crate::{ response::{AdminError, AdminPageEnvelope, AdminResponse}, session::AdminAuth, }, - bootstrap::state::AppState, + api::AppState, infra::{ json::{clamp_limit, clamp_page, NumberedPage, Page}, time::china_datetime, diff --git a/backend/src/api/admin/response.rs b/backend/src/api/admin/response.rs index 94eab9486..86cb13b30 100644 --- a/backend/src/api/admin/response.rs +++ b/backend/src/api/admin/response.rs @@ -6,7 +6,8 @@ use axum::{ response::{IntoResponse, Response}, Json, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::fmt; pub(crate) const ADMIN_OK_CODE: u32 = 200; @@ -291,3 +292,59 @@ impl IntoResponse for AdminResponse { (self.status, Json(self.body)).into_response() } } + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct EditableUpdateRequest { + id: String, + label: Option>, + status: Option, +} + +pub(crate) struct ParsedEditableUpdate { + pub id: String, + pub label: Option>, + pub status: Option, +} + +pub(crate) struct EditableUpdateMessages<'a> { + pub object_required: &'a str, + pub invalid: &'a str, + pub empty_update: &'a str, + pub unknown_field_editable: bool, +} + +pub(crate) fn parse_editable_update( + payload: &Value, + messages: EditableUpdateMessages<'_>, +) -> Result { + let object = payload + .as_object() + .ok_or_else(|| AdminError::bad_request(messages.object_required))?; + if messages.unknown_field_editable { + for field in object.keys() { + if !matches!(field.as_str(), "id" | "label" | "status") { + return Err(AdminError::bad_request(format!("{field} is not editable"))); + } + } + } + + let update = serde_json::from_value::(payload.clone()) + .map_err(|_| AdminError::bad_request(messages.invalid))?; + if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) { + return Err(AdminError::bad_request(messages.invalid)); + } + if update.label.is_none() && update.status.is_none() { + return Err(AdminError::bad_request(messages.empty_update)); + } + + Ok(ParsedEditableUpdate { + id: update.id, + label: update.label, + status: update.status, + }) +} + +fn is_blank(value: &str) -> bool { + value.trim().is_empty() +} diff --git a/backend/src/api/admin/router.rs b/backend/src/api/admin/router.rs index eb9329e2a..9f0ccf0ed 100644 --- a/backend/src/api/admin/router.rs +++ b/backend/src/api/admin/router.rs @@ -5,7 +5,7 @@ use axum::{ Router, }; -use crate::bootstrap::state::AppState; +use crate::api::AppState; use super::{ accounts_routes::{ diff --git a/backend/src/api/admin/session.rs b/backend/src/api/admin/session.rs index 8e9e9f3ca..37376c395 100644 --- a/backend/src/api/admin/session.rs +++ b/backend/src/api/admin/session.rs @@ -9,12 +9,12 @@ use axum::{ use serde::{Deserialize, Serialize}; use crate::{ + api::AppState, api::{ admin::response::{AdminEnvelope, AdminError, AdminResponse}, middleware::request_id::ClientIp, }, auth::types::SessionError, - bootstrap::state::AppState, infra::time::china_rfc3339, }; diff --git a/backend/src/api/admin/settings_routes.rs b/backend/src/api/admin/settings_routes.rs index eb653f020..ffff58b07 100644 --- a/backend/src/api/admin/settings_routes.rs +++ b/backend/src/api/admin/settings_routes.rs @@ -10,7 +10,7 @@ use crate::{ response::{AdminEnvelope, AdminError, AdminResponse}, session::AdminAuth, }, - bootstrap::state::AppState, + api::AppState, settings::{SettingsError, SettingsPatch, SettingsSnapshot, SettingsValidationError}, }; @@ -52,7 +52,7 @@ pub(crate) async fn settings( State(state): State, _auth: AdminAuth, ) -> Result { - let config = state.services.settings.current().await; + let config = state.services.settings.current(); Ok(AdminResponse::new( StatusCode::OK, AdminEnvelope::ok(AdminSettingsData::from_settings(&config)), @@ -68,7 +68,8 @@ pub(crate) async fn update_settings( let patch = parse_settings_patch(&body)?; let config = state .services - .update_settings(patch) + .settings + .update(patch) .await .map_err(settings_service_error)?; diff --git a/backend/src/api/admin/system_routes.rs b/backend/src/api/admin/system_routes.rs index 1d306ba3a..bf062c0e2 100644 --- a/backend/src/api/admin/system_routes.rs +++ b/backend/src/api/admin/system_routes.rs @@ -3,7 +3,7 @@ use std::{convert::Infallible, time::Duration}; use axum::{ - extract::{Json, Query}, + extract::{Json, Query, State}, http::StatusCode, response::{ sse::{Event, KeepAlive, Sse}, @@ -18,7 +18,7 @@ use crate::{ response::{AdminEnvelope, AdminError, AdminResponse}, session::AdminAuth, }, - bootstrap::shutdown::{request_process_restart, request_shutdown, shutdown_subscription}, + api::AppState, update::{ service::{self, RestartAction}, state::operation_id, @@ -56,10 +56,11 @@ pub(crate) async fn update_detail( } pub(crate) async fn update_event_stream( + State(state): State, _auth: AdminAuth, ) -> Result>>, AdminError> { let receiver = service::subscribe_update_events(); - let shutdown = shutdown_subscription(); + let shutdown = state.services.process_control.subscribe_shutdown(); let stream = futures::stream::unfold( (receiver, shutdown, false), |(mut receiver, mut shutdown, close_after_send)| async move { @@ -122,14 +123,20 @@ pub(crate) async fn rollback(_auth: AdminAuth) -> Result Result { +pub(crate) async fn restart( + State(state): State, + _auth: AdminAuth, +) -> Result { let plan = service::restart_plan().map_err(update_error)?; let message = plan.message; + let process_control = state.services.process_control.clone(); tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(500)).await; match plan.action { - RestartAction::Exec(executable_path) => request_process_restart(executable_path), - RestartAction::Shutdown => request_shutdown(), + RestartAction::Exec(executable_path) => { + process_control.request_restart(executable_path); + } + RestartAction::Shutdown => process_control.request_shutdown(), } }); Ok(AdminResponse::new( diff --git a/backend/src/api/admin/update_payload.rs b/backend/src/api/admin/update_payload.rs deleted file mode 100644 index 9a52804a0..000000000 --- a/backend/src/api/admin/update_payload.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! 管理端通用更新请求解析。 - -use serde::Deserialize; -use serde_json::Value; - -use crate::api::admin::response::AdminError; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct EditableUpdateRequest { - id: String, - label: Option>, - status: Option, -} - -pub(crate) struct ParsedEditableUpdate { - pub id: String, - pub label: Option>, - pub status: Option, -} - -pub(crate) struct EditableUpdateMessages<'a> { - pub object_required: &'a str, - pub invalid: &'a str, - pub empty_update: &'a str, - pub unknown_field_editable: bool, -} - -pub(crate) fn parse_editable_update( - payload: &Value, - messages: EditableUpdateMessages<'_>, -) -> Result { - let object = payload - .as_object() - .ok_or_else(|| AdminError::bad_request(messages.object_required))?; - if messages.unknown_field_editable { - for field in object.keys() { - if !matches!(field.as_str(), "id" | "label" | "status") { - return Err(AdminError::bad_request(format!("{field} is not editable"))); - } - } - } - - let update = serde_json::from_value::(payload.clone()) - .map_err(|_| AdminError::bad_request(messages.invalid))?; - if is_blank(&update.id) || update.status.as_deref().is_some_and(is_blank) { - return Err(AdminError::bad_request(messages.invalid)); - } - if update.label.is_none() && update.status.is_none() { - return Err(AdminError::bad_request(messages.empty_update)); - } - - Ok(ParsedEditableUpdate { - id: update.id, - label: update.label, - status: update.status, - }) -} - -fn is_blank(value: &str) -> bool { - value.trim().is_empty() -} diff --git a/backend/src/api/admin/usage_routes.rs b/backend/src/api/admin/usage_routes.rs index 0123c47fd..aff3fe0c9 100644 --- a/backend/src/api/admin/usage_routes.rs +++ b/backend/src/api/admin/usage_routes.rs @@ -14,7 +14,7 @@ use crate::{ response::{AdminEnvelope, AdminError, AdminPageEnvelope, AdminResponse}, session::AdminAuth, }, - bootstrap::state::AppState, + api::AppState, infra::{ format::{ format_compact_number, format_cost, format_duration_ms, format_duration_ms_f64, diff --git a/backend/src/api/client/auth.rs b/backend/src/api/client/auth.rs index 1c895b6f0..d3edcd2dd 100644 --- a/backend/src/api/client/auth.rs +++ b/backend/src/api/client/auth.rs @@ -4,7 +4,7 @@ use axum::http::{header::AUTHORIZATION, HeaderMap}; -use crate::bootstrap::state::AppState; +use crate::api::AppState; /// client API key 鉴权失败原因。 #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/backend/src/api/client/errors.rs b/backend/src/api/client/errors.rs index b1c461ea0..fe49fcbf7 100644 --- a/backend/src/api/client/errors.rs +++ b/backend/src/api/client/errors.rs @@ -8,7 +8,7 @@ use axum::{ use serde_json::{json, Value}; use crate::{ - dispatch::{errors::DispatchFailureClass, responses::errors::ResponseDispatchError}, + dispatch::errors::{DispatchFailureClass, ResponseDispatchError}, upstream::openai::protocol::sse::response_failed_sse_event as encode_response_failed_sse_event, }; diff --git a/backend/src/api/client/models.rs b/backend/src/api/client/models.rs index 3d33d5adc..951ed2d3e 100644 --- a/backend/src/api/client/models.rs +++ b/backend/src/api/client/models.rs @@ -9,8 +9,7 @@ use axum::{ use serde_json::{json, Value}; use crate::{ - api::client::auth::authorize_client_api_key, bootstrap::state::AppState, - models::catalog::ModelCatalog, + api::client::auth::authorize_client_api_key, api::AppState, models::catalog::ModelCatalog, }; use super::errors::{missing_client_api_key_response, model_not_found_response}; diff --git a/backend/src/api/client/responses.rs b/backend/src/api/client/responses.rs index 1c490d4a8..ac6e58b38 100644 --- a/backend/src/api/client/responses.rs +++ b/backend/src/api/client/responses.rs @@ -12,8 +12,8 @@ use serde_json::{Map, Value}; use crate::{ api::middleware::request_id::{ClientIp, RequestId}, - bootstrap::state::AppState, - dispatch::responses::{errors::ResponseDispatchError, service::ResponseDispatchStream}, + api::AppState, + dispatch::{errors::ResponseDispatchError, service::ResponseDispatchStream}, upstream::openai::protocol::{ responses::{CodexCompactRequest, CodexResponsesRequest}, sse::sse_body_has_done, diff --git a/backend/src/api/client/router.rs b/backend/src/api/client/router.rs index 34f08f073..78e7e8704 100644 --- a/backend/src/api/client/router.rs +++ b/backend/src/api/client/router.rs @@ -6,7 +6,7 @@ use axum::{ Router, }; -use crate::bootstrap::state::AppState; +use crate::api::AppState; use super::{ models::{model_catalog, model_detail, model_info, models}, diff --git a/backend/src/api/mod.rs b/backend/src/api/mod.rs index 1f4d7da64..9b18c43c1 100644 --- a/backend/src/api/mod.rs +++ b/backend/src/api/mod.rs @@ -5,3 +5,5 @@ pub mod assets; pub mod client; pub mod middleware; pub mod router; + +pub use router::AppState; diff --git a/backend/src/api/router.rs b/backend/src/api/router.rs index 9338c986d..5a78fa4fa 100644 --- a/backend/src/api/router.rs +++ b/backend/src/api/router.rs @@ -1,17 +1,75 @@ //! 顶层 HTTP 路由 —— 组合 OpenAI API、管理端 API 和静态资源服务。 -use std::path::Path; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use async_trait::async_trait; use axum::{extract::State, http::StatusCode, middleware, routing::get, Router}; use crate::{ + accounts::{ + manage::AccountManageService, pool::RuntimeAccountPoolService, + refresh::RuntimeTokenRefreshService, store::AccountStore, + }, api::{ admin, assets, client, middleware::{request_id::attach_request_id, trace::http_trace_layer}, }, - bootstrap::state::AppState, + auth::service::SessionService, + dispatch::{affinity::RuntimeSessionAffinityService, service::ResponseDispatchService}, + keys::{manage::KeyManageService, service::KeyVerifier}, + models::service::ModelService, + settings::service::RuntimeSettingsService, + telemetry::{ + account_usage::query::AccountUsageQueryService, ops::query::OpsQueryService, + usage::query::UsageQueryService, + }, + upstream::openai::{fingerprint::RuntimeFingerprint, token_client::OpenAiTokenClient}, }; +/// HTTP API 所需的领域服务集合。 +#[derive(Clone)] +pub struct ApiServices { + pub health_probe: Arc, + pub models: Arc, + pub accounts: Arc, + pub client_keys: Arc, + pub admin_client_keys: Arc, + pub admin_sessions: Arc, + pub settings: Arc, + pub admin_accounts: Arc, + pub usage_records: Arc, + pub ops_errors: Arc, + pub usage: Arc, + pub account_pool: Arc, + pub token_refresh: Arc>, + pub responses: Arc, + pub session_affinity: Arc, + pub fingerprint: RuntimeFingerprint, + pub process_control: Arc, +} + +/// 由 bootstrap 实现的进程生命周期控制端口。 +pub trait ProcessControl: Send + Sync + 'static { + fn request_shutdown(&self); + fn request_restart(&self, executable_path: PathBuf); + fn subscribe_shutdown(&self) -> tokio::sync::broadcast::Receiver<()>; +} + +/// 由 bootstrap 实现的基础设施健康检查端口。 +#[async_trait] +pub trait HealthProbe: Send + Sync + 'static { + async fn check(&self) -> Result<(), String>; +} + +/// HTTP handler 共享状态。 +#[derive(Clone)] +pub struct AppState { + pub services: ApiServices, +} + /// 默认前端构建产物目录。 pub const DEFAULT_ASSET_DIST_DIR: &str = "web/dist"; @@ -32,12 +90,8 @@ pub fn router_with_assets(dist_dir: impl AsRef) -> Router { } async fn healthz(State(state): State) -> StatusCode { - if let Err(error) = crate::infra::database::ping(&state.services.database).await { - tracing::warn!(error = %error, "PostgreSQL health check failed"); - return StatusCode::SERVICE_UNAVAILABLE; - } - if let Err(error) = state.services.redis.ping().await { - tracing::warn!(error = %error, "Redis health check failed"); + if let Err(error) = state.services.health_probe.check().await { + tracing::warn!(error, "health check failed"); return StatusCode::SERVICE_UNAVAILABLE; } StatusCode::NO_CONTENT diff --git a/backend/src/bootstrap/application.rs b/backend/src/bootstrap/application.rs deleted file mode 100644 index db114b972..000000000 --- a/backend/src/bootstrap/application.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::{env, error::Error, time::Duration}; - -use axum::Router; - -use crate::bootstrap::config::AppConfig; -use crate::bootstrap::services::{ - apply_settings_to_config, fingerprint_from_config, settings_snapshot_from_config, - BackgroundTaskStores, Services, -}; -use crate::bootstrap::shutdown::{restart_executable_path, shutdown_signal}; -use crate::bootstrap::state::{AppState, RuntimeConfig}; -use crate::bootstrap::tasks::coordinator::TaskCoordinator; -use crate::infra::database::connect; -use crate::infra::logging::{init_tracing, LogGuard, RotationConfig}; -use crate::infra::paths::{ensure_data_dir, load_or_create_installation_id}; -use crate::infra::redis::RedisConnection; -use crate::settings::service::RuntimeSettingsService; -use crate::upstream::openai::fingerprint::{PgFingerprintStore, RuntimeFingerprint}; - -const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS"; - -pub async fn run() -> Result<(), Box> { - wait_for_scheduled_restart().await; - - let config = AppConfig::load()?; - let host = config.server.host.clone(); - let port = config.server.port; - let log_guard = init_logging(&config)?; - let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; - let (app, task_coordinator) = build_application(config).await?; - - let serve_result = axum::serve( - listener, - app.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(shutdown_signal()) - .await; - - task_coordinator.shutdown().await; - serve_result?; - - if let Some(executable_path) = restart_executable_path() { - tracing::info!( - executable = %executable_path.display(), - "正在以更新后的二进制替换当前进程" - ); - drop(log_guard); - exec_replacement_process(executable_path)?; - } - - Ok(()) -} - -async fn build_application( - mut config: AppConfig, -) -> Result<(Router, TaskCoordinator), Box> { - let pool = connect(&config.database.url).await?; - let redis = RedisConnection::connect(&config.redis.url, "cpr").await?; - let settings = - RuntimeSettingsService::load_or_initialize(settings_snapshot_from_config(&config), &pool) - .await?; - apply_settings_to_config(&mut config, &settings); - - let fingerprint_store = PgFingerprintStore::new(pool.clone()); - let stores = BackgroundTaskStores { - redis: redis.clone(), - accounts: crate::accounts::store::PgAccountStore::new(pool.clone()), - admin_users: crate::auth::store::PgAdminUserStore::new(pool.clone()), - admin_sessions: crate::auth::store::RedisAdminSessionStore::new(redis.clone()), - cookies: crate::accounts::cookies::PgCookieStore::new(pool.clone()), - fingerprints: fingerprint_store.clone(), - session_affinity: crate::dispatch::affinity::RedisSessionAffinityStore::new(redis.clone()), - refresh_leases: crate::accounts::refresh::RedisRefreshLeaseStore::new(redis.clone()), - model_snapshots: crate::models::store::RedisModelSnapshotStore::new(redis.clone()), - client_keys: crate::keys::store::PgClientKeyStore::new(pool.clone()), - usage_records: crate::telemetry::usage::store::PgUsageRecordStore::new(pool.clone()), - ops_errors: crate::telemetry::ops::store::PgOpsErrorLogStore::new(pool.clone()), - account_usage: crate::telemetry::account_usage::store::PgAccountUsageStore::new( - pool.clone(), - ), - request_buckets: crate::telemetry::buckets::store::PgRequestBucketStore::new(pool), - }; - - config.admin.validate_default_password()?; - let default_admin_password = config.admin.default_password.clone(); - let data_dir = ensure_data_dir()?; - let installation_id = load_or_create_installation_id(Some(&data_dir))?; - let default_fingerprint = fingerprint_from_config(&config.fingerprint); - let runtime_fingerprint = fingerprint_store - .ensure_current_seed(&default_fingerprint) - .await?; - let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint); - let runtime_config = RuntimeConfig::from(config.clone()); - let services = Services::try_with_installation_id( - &config, - stores, - runtime_fingerprint, - Some(installation_id), - )?; - services.initialize_hot_path_state().await?; - - let created_default_admin = services - .admin_sessions - .ensure_default_admin(&default_admin_password) - .await?; - tracing::info!( - created = created_default_admin, - "默认管理员账号已完成初始化检查" - ); - - let restored_accounts = services.account_pool.restore_from_store().await?; - tracing::info!( - count = restored_accounts, - "运行时账号池已从 PostgreSQL 恢复" - ); - - let task_coordinator = TaskCoordinator::start(&runtime_config, &services); - let app_state = AppState { services }; - - Ok(( - crate::api::router::router().with_state(app_state), - task_coordinator, - )) -} - -fn init_logging(config: &AppConfig) -> Result, crate::infra::logging::LogError> { - if !config.logging.enabled { - return Ok(None); - } - - let rotation = RotationConfig::new(&config.logging.directory, config.logging.retention_days); - Ok(Some(init_tracing(&rotation)?)) -} - -async fn wait_for_scheduled_restart() { - let Some(delay_ms) = env::var(RESTART_DELAY_ENV) - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|value| *value > 0) - else { - return; - }; - - tokio::time::sleep(Duration::from_millis(delay_ms)).await; -} - -#[cfg(unix)] -fn exec_replacement_process( - executable_path: std::path::PathBuf, -) -> Result<(), Box> { - use std::os::unix::process::CommandExt; - use std::process::Command; - - let error = Command::new(executable_path) - .args(env::args_os().skip(1)) - .exec(); - Err(Box::new(error)) -} - -#[cfg(not(unix))] -fn exec_replacement_process( - _executable_path: std::path::PathBuf, -) -> Result<(), Box> { - Err(Box::new(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "process exec restart is only supported on Unix", - ))) -} diff --git a/backend/src/bootstrap/mod.rs b/backend/src/bootstrap/mod.rs index 360d32a56..d33502480 100644 --- a/backend/src/bootstrap/mod.rs +++ b/backend/src/bootstrap/mod.rs @@ -1,6 +1,5 @@ //! 进程装配、共享状态与后台任务。 -pub mod application; pub mod config; pub mod import_sqlite; pub mod services; diff --git a/backend/src/bootstrap/services.rs b/backend/src/bootstrap/services.rs index 19186f975..7b38e01b9 100644 --- a/backend/src/bootstrap/services.rs +++ b/backend/src/bootstrap/services.rs @@ -1,17 +1,18 @@ //! 运行时服务集合。 -use std::sync::Arc; +use std::{env, error::Error, sync::Arc, time::Duration}; + +use axum::Router; use crate::{ accounts::{ cookies::PgCookieStore, manage::{AccountManageService, AccountManageServiceParts}, - pool::RuntimeAccountPoolService, - pool::{AccountPoolOptions, RotationStrategy}, + pool::{AccountPoolOptions, AccountPoolStaticSettings, RuntimeAccountPoolService}, refresh::{ RedisRefreshLeaseStore, RefreshPolicy, RuntimeRefreshPolicy, RuntimeTokenRefreshService, }, - store::{AccountStore as AccountStoreTrait, PgAccountStore}, + store::{AccountStore, PgAccountStore}, }, auth::{ service::SessionService, @@ -20,19 +21,24 @@ use crate::{ bootstrap::config::AppConfig, dispatch::{ affinity::{RedisSessionAffinityStore, RuntimeSessionAffinityService}, - responses::service::{ResponseDispatchService, ResponseDispatchServiceParts}, + service::{ResponseDispatchService, ResponseDispatchServiceParts}, + }, + infra::{ + database::connect, + logging::{init_tracing, LogGuard, RotationConfig}, + paths::{ensure_data_dir, load_or_create_installation_id}, + redis::RedisConnection, }, - infra::redis::RedisConnection, keys::{manage::KeyManageService, service::KeyVerifier, store::PgClientKeyStore}, models::{ - config::ModelConfig, service::{ModelService, ModelServiceError}, store::{ModelSnapshotStore, RedisModelSnapshotStore}, + types::ModelConfig, }, - settings::{service::RuntimeSettingsService, SettingsError, SettingsPatch, SettingsSnapshot}, + settings::{service::RuntimeSettingsService, SettingsSnapshot}, telemetry::{ account_usage::query::AccountUsageQueryService, - account_usage::store::PgAccountUsageStore, + account_usage::store::{AccountUsageStore, PgAccountUsageStore}, buckets::store::PgRequestBucketStore, ops::query::OpsQueryService, ops::store::PgOpsErrorLogStore, @@ -50,6 +56,8 @@ use crate::{ }, }; +const RESTART_DELAY_ENV: &str = "CPR_RESTART_DELAY_MS"; + /// 运行时热路径内存状态初始化错误。 #[derive(Debug, thiserror::Error)] pub enum RuntimeServiceInitializationError { @@ -90,7 +98,7 @@ pub struct Services { pub database: sqlx::PgPool, pub redis: RedisConnection, pub models: Arc, - pub accounts: Arc, + pub accounts: Arc, pub client_keys: Arc, pub admin_client_keys: Arc, pub admin_sessions: Arc, @@ -109,14 +117,7 @@ pub struct Services { pub fingerprint: RuntimeFingerprint, pub installation_id: Option, pub background_tasks: BackgroundTaskStores, - account_pool_static: AccountPoolStaticSettings, - settings_update_lock: Arc>, -} - -#[derive(Debug, Clone)] -struct AccountPoolStaticSettings { - skip_quota_limited: bool, - tier_priority: Vec, + pub(crate) account_pool_static: AccountPoolStaticSettings, } /// 使用记录运行选项。 @@ -178,7 +179,9 @@ impl Services { let installation_id = installation_id.filter(|id| !id.trim().is_empty()); let database = stores.accounts.pool().clone(); let redis = stores.redis.clone(); - let account_store_trait = Arc::new(stores.accounts.clone()) as Arc; + let account_store = Arc::new(stores.accounts.clone()) as Arc; + let account_usage_store_trait = + Arc::new(stores.account_usage.clone()) as Arc; let websocket_pool = config.ws_pool.enabled.then(|| { Arc::new(CodexWebSocketPool::with_config(CodexWebSocketPoolConfig { enabled: true, @@ -213,24 +216,23 @@ impl Services { usage_record_options.enabled, usage_record_options.capture_body, )); + let settings_snapshot = settings_snapshot_from_config(config); let account_pool_static = AccountPoolStaticSettings { skip_quota_limited: config.quota.skip_exhausted, tier_priority: config.auth.tier_priority.clone(), }; let account_pool = Arc::new(RuntimeAccountPoolService::new( - account_store_trait.clone(), - account_pool_options_from_settings( - &settings_snapshot_from_config(config), - &account_pool_static, - ), - config.auth.request_interval_ms, + account_store.clone(), + account_usage_store_trait, + account_pool_static.pool_options(&settings_snapshot), + settings_snapshot.request_interval_ms, )); let refresh_policy = RuntimeRefreshPolicy::new(RefreshPolicy { refresh_margin_seconds: config.auth.refresh_margin_seconds, refresh_concurrency: config.auth.refresh_concurrency, }); let settings = Arc::new(RuntimeSettingsService::new( - settings_snapshot_from_config(config), + settings_snapshot, stores.accounts.pool().clone(), )); let admin_sessions = Arc::new(SessionService::new( @@ -267,7 +269,6 @@ impl Services { account_pool: account_pool.clone(), token_refresher: Arc::new(token_client), refresh_leases: stores.refresh_leases.clone(), - session_affinity: stores.session_affinity.clone(), oauth: crate::accounts::manage::oauth::AccountOAuthService::new( reqwest::Client::new(), config.auth.oauth_client_id.clone(), @@ -282,7 +283,7 @@ impl Services { )); let cloudflare_recovery = - crate::dispatch::cloudflare::CloudflareRecovery::new(stores.cookies.clone()); + crate::dispatch::recovery::cloudflare::CloudflareRecovery::new(stores.cookies.clone()); let responses = Arc::new(ResponseDispatchService::new(ResponseDispatchServiceParts { account_pool: account_pool.clone(), models: models.clone(), @@ -297,7 +298,7 @@ impl Services { database, redis, models, - accounts: account_store_trait, + accounts: account_store, client_keys, admin_client_keys, admin_sessions, @@ -317,7 +318,6 @@ impl Services { installation_id, background_tasks: stores, account_pool_static, - settings_update_lock: Arc::new(tokio::sync::Mutex::new(())), }) } @@ -326,32 +326,46 @@ impl Services { self.models.reload_from_store().await?; Ok(()) } +} - /// 持久化设置并同步到所有运行时消费者。 - pub async fn update_settings( - &self, - patch: SettingsPatch, - ) -> Result, SettingsError> { - let _guard = self.settings_update_lock.lock().await; - let settings = self.settings.update(patch).await?; - self.apply_settings(&settings).await; - Ok(settings) +impl From<&Services> for crate::api::router::ApiServices { + fn from(services: &Services) -> Self { + Self { + health_probe: Arc::new(crate::bootstrap::state::RuntimeHealthProbe::new( + services.database.clone(), + services.redis.clone(), + )), + models: services.models.clone(), + accounts: services.accounts.clone(), + client_keys: services.client_keys.clone(), + admin_client_keys: services.admin_client_keys.clone(), + admin_sessions: services.admin_sessions.clone(), + settings: services.settings.clone(), + admin_accounts: services.admin_accounts.clone(), + usage_records: services.usage_records.clone(), + ops_errors: services.ops_errors.clone(), + usage: services.usage.clone(), + account_pool: services.account_pool.clone(), + token_refresh: services.token_refresh.clone(), + responses: services.responses.clone(), + session_affinity: services.session_affinity.clone(), + fingerprint: services.fingerprint.clone(), + process_control: Arc::new(crate::bootstrap::shutdown::RuntimeProcessControl), + } } +} - async fn apply_settings(&self, settings: &SettingsSnapshot) { - self.models.update_config(ModelConfig { - model_aliases: settings.model_aliases.clone(), - }); - self.account_pool - .apply_options( - account_pool_options_from_settings(settings, &self.account_pool_static), - settings.request_interval_ms, - ) - .await; - self.refresh_policy.update(RefreshPolicy { - refresh_margin_seconds: settings.refresh_margin_seconds, - refresh_concurrency: settings.refresh_concurrency, - }); +impl From<&Services> for crate::api::router::AppState { + fn from(services: &Services) -> Self { + Self { + services: crate::api::router::ApiServices::from(services), + } + } +} + +impl From for crate::api::router::AppState { + fn from(services: Services) -> Self { + Self::from(&services) } } @@ -403,31 +417,163 @@ pub fn fingerprint_from_config( } } -fn account_pool_options_from_settings( - settings: &SettingsSnapshot, - static_settings: &AccountPoolStaticSettings, -) -> AccountPoolOptions { - AccountPoolOptions { - max_concurrent_per_account: settings.max_concurrent_per_account, - rotation_strategy: match settings.rotation_strategy.as_str() { - "smart" => RotationStrategy::Smart, - "quota_reset_priority" => RotationStrategy::QuotaResetPriority, - "round_robin" => RotationStrategy::RoundRobin, - "sticky" => RotationStrategy::Sticky, - _ => RotationStrategy::Smart, - }, - skip_quota_limited: static_settings.skip_quota_limited, - tier_priority: static_settings.tier_priority.clone(), - ..AccountPoolOptions::default() +pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions { + AccountPoolStaticSettings { + skip_quota_limited: config.quota.skip_exhausted, + tier_priority: config.auth.tier_priority.clone(), } + .pool_options(&settings_snapshot_from_config(config)) } -pub fn account_pool_options_from_config(config: &AppConfig) -> AccountPoolOptions { - account_pool_options_from_settings( - &settings_snapshot_from_config(config), - &AccountPoolStaticSettings { - skip_quota_limited: config.quota.skip_exhausted, - tier_priority: config.auth.tier_priority.clone(), - }, +pub async fn run() -> Result<(), Box> { + wait_for_scheduled_restart().await; + + let config = AppConfig::load()?; + let host = config.server.host.clone(); + let port = config.server.port; + let log_guard = init_logging(&config)?; + let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; + let (router, task_coordinator) = build_router(config).await?; + + let serve_result = axum::serve( + listener, + router.into_make_service_with_connect_info::(), ) + .with_graceful_shutdown(crate::bootstrap::shutdown::shutdown_signal()) + .await; + + task_coordinator.shutdown().await; + serve_result?; + + if let Some(executable_path) = crate::bootstrap::shutdown::restart_executable_path() { + tracing::info!( + executable = %executable_path.display(), + "正在以更新后的二进制替换当前进程" + ); + drop(log_guard); + exec_replacement_process(executable_path)?; + } + + Ok(()) +} + +async fn build_router( + mut config: AppConfig, +) -> Result< + ( + Router, + crate::bootstrap::tasks::coordinator::TaskCoordinator, + ), + Box, +> { + let pool = connect(&config.database.url).await?; + let redis = RedisConnection::connect(&config.redis.url, "cpr").await?; + let settings = + RuntimeSettingsService::load_or_initialize(settings_snapshot_from_config(&config), &pool) + .await?; + apply_settings_to_config(&mut config, &settings); + + let fingerprint_store = PgFingerprintStore::new(pool.clone()); + let stores = BackgroundTaskStores { + redis: redis.clone(), + accounts: PgAccountStore::new(pool.clone()), + admin_users: PgAdminUserStore::new(pool.clone()), + admin_sessions: RedisAdminSessionStore::new(redis.clone()), + cookies: PgCookieStore::new(pool.clone()), + fingerprints: fingerprint_store.clone(), + session_affinity: RedisSessionAffinityStore::new(redis.clone()), + refresh_leases: RedisRefreshLeaseStore::new(redis.clone()), + model_snapshots: RedisModelSnapshotStore::new(redis.clone()), + client_keys: PgClientKeyStore::new(pool.clone()), + usage_records: PgUsageRecordStore::new(pool.clone()), + ops_errors: PgOpsErrorLogStore::new(pool.clone()), + account_usage: PgAccountUsageStore::new(pool.clone()), + request_buckets: PgRequestBucketStore::new(pool), + }; + + config.admin.validate_default_password()?; + let default_admin_password = config.admin.default_password.clone(); + let data_dir = ensure_data_dir()?; + let installation_id = load_or_create_installation_id(Some(&data_dir))?; + let default_fingerprint = fingerprint_from_config(&config.fingerprint); + let runtime_fingerprint = fingerprint_store + .ensure_current_seed(&default_fingerprint) + .await?; + let runtime_fingerprint = RuntimeFingerprint::new(runtime_fingerprint); + let runtime_config = crate::bootstrap::state::RuntimeConfig::from(config.clone()); + let services = Services::try_with_installation_id( + &config, + stores, + runtime_fingerprint, + Some(installation_id), + )?; + services.initialize_hot_path_state().await?; + + let created_default_admin = services + .admin_sessions + .ensure_default_admin(&default_admin_password) + .await?; + tracing::info!( + created = created_default_admin, + "默认管理员账号已完成初始化检查" + ); + + let restored_accounts = services.account_pool.restore_from_store().await?; + tracing::info!( + count = restored_accounts, + "运行时账号池已从 PostgreSQL 恢复" + ); + + let task_coordinator = + crate::bootstrap::tasks::coordinator::TaskCoordinator::start(&runtime_config, &services); + let app_state = crate::api::AppState::from(&services); + + Ok(( + crate::api::router::router().with_state(app_state), + task_coordinator, + )) +} + +fn init_logging(config: &AppConfig) -> Result, crate::infra::logging::LogError> { + if !config.logging.enabled { + return Ok(None); + } + + let rotation = RotationConfig::new(&config.logging.directory, config.logging.retention_days); + Ok(Some(init_tracing(&rotation)?)) +} + +async fn wait_for_scheduled_restart() { + let Some(delay_ms) = env::var(RESTART_DELAY_ENV) + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + else { + return; + }; + + tokio::time::sleep(Duration::from_millis(delay_ms)).await; +} + +#[cfg(unix)] +fn exec_replacement_process( + executable_path: std::path::PathBuf, +) -> Result<(), Box> { + use std::os::unix::process::CommandExt; + use std::process::Command; + + let error = Command::new(executable_path) + .args(env::args_os().skip(1)) + .exec(); + Err(Box::new(error)) +} + +#[cfg(not(unix))] +fn exec_replacement_process( + _executable_path: std::path::PathBuf, +) -> Result<(), Box> { + Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "process exec restart is only supported on Unix", + ))) } diff --git a/backend/src/bootstrap/shutdown.rs b/backend/src/bootstrap/shutdown.rs index 0e86d3fb3..df18d9890 100644 --- a/backend/src/bootstrap/shutdown.rs +++ b/backend/src/bootstrap/shutdown.rs @@ -15,6 +15,23 @@ static SHUTDOWN_REQUESTS: OnceLock> = OnceLock::new(); static RESTART_REQUESTED: AtomicBool = AtomicBool::new(false); static RESTART_EXECUTABLE_PATH: OnceLock = OnceLock::new(); +#[derive(Debug, Clone, Copy, Default)] +pub struct RuntimeProcessControl; + +impl crate::api::router::ProcessControl for RuntimeProcessControl { + fn request_shutdown(&self) { + request_shutdown(); + } + + fn request_restart(&self, executable_path: PathBuf) { + request_process_restart(executable_path); + } + + fn subscribe_shutdown(&self) -> Receiver<()> { + shutdown_subscription() + } +} + /// 等待进程关闭信号（Ctrl+C 或 SIGTERM）。 pub async fn shutdown_signal() { let mut requested = shutdown_sender().subscribe(); diff --git a/backend/src/bootstrap/state.rs b/backend/src/bootstrap/state.rs index b35de5a83..5e3d33b4b 100644 --- a/backend/src/bootstrap/state.rs +++ b/backend/src/bootstrap/state.rs @@ -1,13 +1,10 @@ //! 运行时共享状态。 -use crate::bootstrap::config::AppConfig; -use crate::bootstrap::services::Services; +use async_trait::async_trait; -/// HTTP handler 共享的运行时状态。 -#[derive(Clone)] -pub struct AppState { - pub services: Services, -} +use crate::{ + api::router::HealthProbe, bootstrap::config::AppConfig, infra::redis::RedisConnection, +}; /// 运行时配置镜像（从 AppConfig 衍生）。 #[derive(Debug, Clone)] @@ -17,6 +14,32 @@ pub struct RuntimeConfig { pub quota: crate::bootstrap::config::QuotaConfig, } +/// PostgreSQL 与 Redis 的进程健康检查实现。 +#[derive(Clone)] +pub struct RuntimeHealthProbe { + database: sqlx::PgPool, + redis: RedisConnection, +} + +impl RuntimeHealthProbe { + pub fn new(database: sqlx::PgPool, redis: RedisConnection) -> Self { + Self { database, redis } + } +} + +#[async_trait] +impl HealthProbe for RuntimeHealthProbe { + async fn check(&self) -> Result<(), String> { + crate::infra::database::ping(&self.database) + .await + .map_err(|error| format!("PostgreSQL: {error}"))?; + self.redis + .ping() + .await + .map_err(|error| format!("Redis: {error}")) + } +} + impl From for RuntimeConfig { fn from(config: AppConfig) -> Self { Self { diff --git a/backend/src/bootstrap/tasks/coordinator.rs b/backend/src/bootstrap/tasks/coordinator.rs index 2299585e0..12aafec06 100644 --- a/backend/src/bootstrap/tasks/coordinator.rs +++ b/backend/src/bootstrap/tasks/coordinator.rs @@ -1,6 +1,6 @@ //! 后台任务协调器。 -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{future::Future, path::PathBuf, sync::Arc, time::Duration}; use tokio::task::JoinHandle; @@ -33,7 +33,7 @@ impl TaskCoordinator { config: &crate::bootstrap::state::RuntimeConfig, services: &crate::bootstrap::services::Services, ) -> Self { - let mut coordinator = TaskCoordinator::default(); + let mut coordinator = Self::start_settings_subscriptions(services); let stores = &services.background_tasks; coordinator.push( @@ -65,6 +65,7 @@ impl TaskCoordinator { "quota_refresh", QuotaRefreshTask::with_intervals( stores.accounts.clone(), + stores.account_usage.clone(), services.codex.clone(), config .quota @@ -99,6 +100,37 @@ impl TaskCoordinator { coordinator } + /// 启动各领域的设置订阅任务。 + pub fn start_settings_subscriptions(services: &crate::bootstrap::services::Services) -> Self { + let mut coordinator = TaskCoordinator::default(); + coordinator.push( + "model_settings", + SchedulerHandle::spawn( + services + .models + .clone() + .subscribe_settings(services.settings.subscribe()), + ), + ); + coordinator.push( + "account_pool_settings", + SchedulerHandle::spawn(services.account_pool.clone().subscribe_settings( + services.settings.subscribe(), + services.account_pool_static.clone(), + )), + ); + coordinator.push( + "refresh_policy_settings", + SchedulerHandle::spawn( + services + .refresh_policy + .clone() + .subscribe_settings(services.settings.subscribe()), + ), + ); + coordinator + } + /// 关闭所有后台任务。 pub async fn shutdown(self) { tracing::info!("正在关闭后台任务"); @@ -122,6 +154,17 @@ pub enum SchedulerHandle { } impl SchedulerHandle { + fn spawn(future: impl Future + Send + 'static) -> Self { + let (shutdown_tx, mut shutdown_rx) = tokio::sync::mpsc::channel(1); + let handle = tokio::spawn(async move { + tokio::select! { + () = future => {} + _ = shutdown_rx.recv() => {} + } + }); + Self::new(shutdown_tx, handle) + } + /// 使用关闭 channel 构造句柄。 pub(crate) fn new(shutdown_tx: tokio::sync::mpsc::Sender<()>, handle: JoinHandle<()>) -> Self { Self::Channel { diff --git a/backend/src/bootstrap/tasks/model_refresh.rs b/backend/src/bootstrap/tasks/model_refresh.rs index 68ada6f1e..2deb92334 100644 --- a/backend/src/bootstrap/tasks/model_refresh.rs +++ b/backend/src/bootstrap/tasks/model_refresh.rs @@ -75,7 +75,8 @@ impl ModelRefreshTask { .iter() .map(|selected| ModelRefreshPlanAccount { plan_type: selected.plan_type.clone(), - account: selected.account.clone(), + access_token: selected.account.access_token.clone(), + account_id: selected.account.account_id.clone(), }) .collect::>(); diff --git a/backend/src/bootstrap/tasks/quota_refresh.rs b/backend/src/bootstrap/tasks/quota_refresh.rs index 2944c4373..699a2034e 100644 --- a/backend/src/bootstrap/tasks/quota_refresh.rs +++ b/backend/src/bootstrap/tasks/quota_refresh.rs @@ -12,6 +12,7 @@ use crate::{ quota::{QuotaRefreshSummary, RuntimeQuotaRefreshService}, store::PgAccountStore, }, + telemetry::account_usage::store::PgAccountUsageStore, upstream::openai::transport::CodexBackendClient, }; @@ -31,6 +32,7 @@ impl QuotaRefreshTask { /// 使用自定义间隔构造配额刷新后台任务。 pub fn with_intervals( store: PgAccountStore, + usage_store: PgAccountUsageStore, codex: Arc, interval_secs: u64, min_refresh_interval_secs: u64, @@ -38,6 +40,7 @@ impl QuotaRefreshTask { Self { service: RuntimeQuotaRefreshService::with_min_refresh_interval_secs( store, + usage_store, codex, min_refresh_interval_secs, ), diff --git a/backend/src/dispatch/affinity/resolve.rs b/backend/src/dispatch/affinity/resolve.rs index 1d11afbd1..2d3615cad 100644 --- a/backend/src/dispatch/affinity/resolve.rs +++ b/backend/src/dispatch/affinity/resolve.rs @@ -6,7 +6,7 @@ use tokio::sync::Mutex; use crate::{ dispatch::{ affinity::{compute_variant_hash, hash_instructions, RuntimeSessionAffinityService}, - reasoning_replay::ReasoningReplayCache, + recovery::reasoning_replay::ReasoningReplayCache, }, upstream::openai::protocol::{ events::TokenUsage, diff --git a/backend/src/dispatch/affinity/service.rs b/backend/src/dispatch/affinity/service.rs index 254015f4c..5103f9b48 100644 --- a/backend/src/dispatch/affinity/service.rs +++ b/backend/src/dispatch/affinity/service.rs @@ -60,6 +60,14 @@ impl RuntimeSessionAffinityService { .and_then(|entry| entry.turn_state) } + pub async fn lookup( + &self, + response_id: &str, + now: DateTime, + ) -> Option { + self.entry(response_id, now).await + } + pub async fn lookup_instructions_hash( &self, response_id: &str, diff --git a/backend/src/dispatch/errors.rs b/backend/src/dispatch/errors.rs index 221b65d0f..eef5a1758 100644 --- a/backend/src/dispatch/errors.rs +++ b/backend/src/dispatch/errors.rs @@ -2,12 +2,16 @@ use chrono::{DateTime, Duration, Utc}; use serde_json::{json, Map, Value}; +use thiserror::Error; use crate::{ - accounts::account::AccountStatus, - dispatch::exhaustion::{ExhaustedAccountKind, ExhaustedAccountRef}, - upstream::openai::transport::{ - is_banned_auth_signal, CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics, + dispatch::{ + recovery::exhaustion::{ExhaustedAccount, ExhaustedAccountKind, ExhaustedAccountRef}, + stream::sse_failure::{sse_failure_error_body, stream_failure_http_status}, + }, + upstream::openai::{ + protocol::{responses::ResponsesSseFailure, sse::SseError}, + transport::{CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics}, }, }; @@ -202,13 +206,6 @@ pub(crate) fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> boo ) } -pub(crate) fn is_auth_upstream_error(error: &CodexClientError) -> bool { - matches!( - error, - CodexClientError::Upstream { status, .. } if status.as_u16() == 401 - ) -} - pub(crate) fn is_model_unsupported_upstream_error(error: &CodexClientError) -> bool { matches!( error, @@ -226,15 +223,6 @@ pub(crate) fn is_history_recovery_upstream_error(error: &CodexClientError) -> bo ) } -pub(crate) fn auth_failure_account_status(error: &CodexClientError) -> AccountStatus { - match error { - CodexClientError::Upstream { body, .. } if is_banned_auth_signal(body) => { - AccountStatus::Banned - } - _ => AccountStatus::Expired, - } -} - pub(crate) fn upstream_error_body(error: &CodexClientError) -> String { match error { CodexClientError::Upstream { body, .. } => body.clone(), @@ -315,3 +303,326 @@ fn status_code_is_quota_exhausted(status_code: u16) -> bool { fn status_code_is_transient_upstream(status_code: u16) -> bool { matches!(status_code, 500..=599) } +/// Responses 调度错误。 +#[derive(Debug, Error)] +pub enum ResponseDispatchError { + #[error("no active account is available")] + NoActiveAccount, + #[error("all accounts exhausted by quota")] + QuotaExhausted { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by rate limit")] + RateLimited { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by expired auth")] + Expired { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by disabled auth")] + Disabled { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by banned auth")] + Banned { + count: usize, + upstream_error: String, + status_code: u16, + }, + #[error("all accounts exhausted by Cloudflare challenge")] + CloudflareChallenge { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by Cloudflare path-block")] + CloudflarePathBlocked { + count: usize, + upstream_error: String, + }, + #[error("all accounts exhausted by unsupported model")] + ModelUnsupported { + count: usize, + upstream_error: String, + }, + #[error("previous response context is unavailable: {upstream_error}")] + HistoryUnavailable { upstream_error: String }, + #[error("upstream request failed: {0}")] + Upstream(#[from] CodexClientError), + #[error("invalid upstream SSE response: {0}")] + InvalidSse(#[from] SseError), + #[error("upstream response did not include response.completed")] + MissingCompleted, + #[error("upstream response did not include visible output")] + EmptyUpstreamResponse, + #[error("upstream response failed: {0:?}")] + Failed(ResponsesSseFailure), +} + +impl ResponseDispatchError { + pub(crate) fn from_exhausted_account(exhausted: ExhaustedAccount) -> Self { + match exhausted.kind { + ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::RateLimited => Self::RateLimited { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::Expired => Self::Expired { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::Disabled => Self::Disabled { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::Banned => Self::Banned { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + status_code: exhausted.status_code.unwrap_or(403), + }, + ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported { + count: exhausted.count, + upstream_error: exhausted.upstream_error, + }, + } + } + + pub fn http_status_code(&self) -> u16 { + match self { + Self::NoActiveAccount => 503, + Self::QuotaExhausted { .. } => 429, + Self::RateLimited { .. } => 429, + Self::Expired { .. } | Self::Disabled { .. } => 401, + Self::Banned { status_code, .. } => *status_code, + Self::CloudflareChallenge { .. } + | Self::CloudflarePathBlocked { .. } + | Self::InvalidSse(_) + | Self::MissingCompleted + | Self::EmptyUpstreamResponse => 502, + Self::Failed(failure) => stream_failure_http_status(failure), + Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400, + Self::Upstream(error) => upstream_error_http_status(error), + } + } + + pub(crate) fn metadata(&self) -> DispatchErrorMetadata { + match self { + Self::NoActiveAccount => DispatchErrorMetadata::no_available_accounts(), + Self::QuotaExhausted { + count, + upstream_error, + } => Self::exhausted_metadata( + ExhaustedAccountKind::QuotaExhausted, + *count, + upstream_error, + ), + Self::RateLimited { + count, + upstream_error, + } => { + Self::exhausted_metadata(ExhaustedAccountKind::RateLimited, *count, upstream_error) + } + Self::Expired { + count, + upstream_error, + } => Self::exhausted_metadata(ExhaustedAccountKind::Expired, *count, upstream_error), + Self::Disabled { + count, + upstream_error, + } => Self::exhausted_metadata(ExhaustedAccountKind::Disabled, *count, upstream_error), + Self::Banned { + count, + upstream_error, + .. + } => Self::exhausted_metadata(ExhaustedAccountKind::Banned, *count, upstream_error), + Self::CloudflareChallenge { + count, + upstream_error, + } => Self::exhausted_metadata( + ExhaustedAccountKind::CloudflareChallenge, + *count, + upstream_error, + ), + Self::CloudflarePathBlocked { + count, + upstream_error, + } => Self::exhausted_metadata( + ExhaustedAccountKind::CloudflarePathBlocked, + *count, + upstream_error, + ), + Self::ModelUnsupported { + count, + upstream_error, + } => Self::exhausted_metadata( + ExhaustedAccountKind::ModelUnsupported, + *count, + upstream_error, + ), + Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata { + failure_class: DispatchFailureClass::HistoryUnavailable, + exhausted_count: None, + upstream_error: Some(upstream_error.clone()), + upstream_status: Some(400), + diagnostics: None, + }, + Self::Upstream(error) => DispatchErrorMetadata::upstream(error), + Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse), + Self::MissingCompleted => { + DispatchErrorMetadata::simple(DispatchFailureClass::MissingCompleted) + } + Self::EmptyUpstreamResponse => { + DispatchErrorMetadata::simple(DispatchFailureClass::EmptyUpstreamResponse) + } + Self::Failed(failure) => DispatchErrorMetadata { + failure_class: DispatchFailureClass::ResponseFailed, + exhausted_count: None, + upstream_error: Some(sse_failure_error_body(failure)), + upstream_status: None, + diagnostics: None, + }, + } + } + + pub(crate) fn exhausted_account(&self) -> Option> { + match self { + Self::QuotaExhausted { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::QuotaExhausted, + count: *count, + upstream_error, + }), + Self::RateLimited { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::RateLimited, + count: *count, + upstream_error, + }), + Self::Expired { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::Expired, + count: *count, + upstream_error, + }), + Self::Disabled { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::Disabled, + count: *count, + upstream_error, + }), + Self::Banned { + count, + upstream_error, + .. + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::Banned, + count: *count, + upstream_error, + }), + Self::CloudflareChallenge { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::CloudflareChallenge, + count: *count, + upstream_error, + }), + Self::CloudflarePathBlocked { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::CloudflarePathBlocked, + count: *count, + upstream_error, + }), + Self::ModelUnsupported { + count, + upstream_error, + } => Some(ExhaustedAccountRef { + kind: ExhaustedAccountKind::ModelUnsupported, + count: *count, + upstream_error, + }), + _ => None, + } + } + + fn exhausted_metadata( + kind: ExhaustedAccountKind, + count: usize, + upstream_error: &str, + ) -> DispatchErrorMetadata { + DispatchErrorMetadata::exhausted_ref(ExhaustedAccountRef { + kind, + count, + upstream_error, + }) + } +} + +/// Responses live SSE body stream error. +#[derive(Debug, Error)] +pub enum ResponseDispatchStreamError { + #[error("upstream stream failed: {0}")] + Upstream(#[from] CodexClientError), +} + +pub(super) fn dispatch_error_metadata( + error: impl std::fmt::Display, + stream: bool, + compact: bool, + transport: Option<&str>, +) -> Value { + let mut metadata = serde_json::json!({ + "stream": stream, + "failed": true, + "errorKind": "dispatch", + "error": error.to_string(), + }); + let Some(object) = metadata.as_object_mut() else { + return metadata; + }; + if compact { + object.insert("compact".to_string(), Value::Bool(true)); + } + if let Some(transport) = transport { + object.insert( + "transport".to_string(), + Value::String(transport.to_string()), + ); + } + metadata +} + +pub(super) fn enrich_response_dispatch_error_metadata( + metadata: &mut Value, + error: &ResponseDispatchError, +) { + let Some(object) = metadata.as_object_mut() else { + return; + }; + insert_dispatch_error_metadata(object, error.metadata()); +} diff --git a/backend/src/dispatch/mod.rs b/backend/src/dispatch/mod.rs index 67404e836..5979d966e 100644 --- a/backend/src/dispatch/mod.rs +++ b/backend/src/dispatch/mod.rs @@ -1,10 +1,9 @@ //! 请求调度模块（账号选择、回退、恢复、会话亲和性）。 pub mod affinity; -pub mod cloudflare; pub mod errors; -mod exhaustion; -pub mod implicit_resume; -pub mod reasoning_replay; -pub mod responses; -pub mod upstream; +mod recording; +pub mod recovery; +pub(crate) mod service; +pub(crate) mod stream; +pub mod upstream_call; diff --git a/backend/src/dispatch/responses/event_recording.rs b/backend/src/dispatch/recording.rs similarity index 99% rename from backend/src/dispatch/responses/event_recording.rs rename to backend/src/dispatch/recording.rs index a2a95b81c..c83f3c150 100644 --- a/backend/src/dispatch/responses/event_recording.rs +++ b/backend/src/dispatch/recording.rs @@ -30,9 +30,11 @@ use super::{ errors::{ dispatch_error_metadata, enrich_response_dispatch_error_metadata, ResponseDispatchError, }, - sse_failure::{status_code_for_stream_failure, stream_failure_metadata}, - stream_lifecycle::{latest_response_id, LiveResponseStreamContext}, - trace::{ResponseDispatchAttempt, ResponseDispatchTrace}, + stream::{ + lifecycle::{latest_response_id, LiveResponseStreamContext}, + sse_failure::{status_code_for_stream_failure, stream_failure_metadata}, + trace::{ResponseDispatchAttempt, ResponseDispatchTrace}, + }, }; pub(super) struct ResponseUpstreamErrorEventRecord<'a> { diff --git a/backend/src/dispatch/recovery/auth.rs b/backend/src/dispatch/recovery/auth.rs new file mode 100644 index 000000000..4ea664f9a --- /dev/null +++ b/backend/src/dispatch/recovery/auth.rs @@ -0,0 +1,22 @@ +//! 上游认证失败分类。 + +use crate::{ + accounts::account::AccountStatus, + upstream::openai::transport::{is_banned_auth_signal, CodexClientError}, +}; + +pub(crate) fn is_auth_upstream_error(error: &CodexClientError) -> bool { + matches!( + error, + CodexClientError::Upstream { status, .. } if status.as_u16() == 401 + ) +} + +pub(crate) fn auth_failure_account_status(error: &CodexClientError) -> AccountStatus { + match error { + CodexClientError::Upstream { body, .. } if is_banned_auth_signal(body) => { + AccountStatus::Banned + } + _ => AccountStatus::Expired, + } +} diff --git a/backend/src/dispatch/cloudflare.rs b/backend/src/dispatch/recovery/cloudflare.rs similarity index 100% rename from backend/src/dispatch/cloudflare.rs rename to backend/src/dispatch/recovery/cloudflare.rs diff --git a/backend/src/dispatch/exhaustion.rs b/backend/src/dispatch/recovery/exhaustion.rs similarity index 100% rename from backend/src/dispatch/exhaustion.rs rename to backend/src/dispatch/recovery/exhaustion.rs diff --git a/backend/src/dispatch/implicit_resume.rs b/backend/src/dispatch/recovery/implicit_resume.rs similarity index 100% rename from backend/src/dispatch/implicit_resume.rs rename to backend/src/dispatch/recovery/implicit_resume.rs diff --git a/backend/src/dispatch/recovery/mod.rs b/backend/src/dispatch/recovery/mod.rs new file mode 100644 index 000000000..0d9d76bc0 --- /dev/null +++ b/backend/src/dispatch/recovery/mod.rs @@ -0,0 +1,7 @@ +//! 调度失败恢复策略。 + +pub(crate) mod auth; +pub mod cloudflare; +pub(crate) mod exhaustion; +pub mod implicit_resume; +pub mod reasoning_replay; diff --git a/backend/src/dispatch/reasoning_replay.rs b/backend/src/dispatch/recovery/reasoning_replay.rs similarity index 100% rename from backend/src/dispatch/reasoning_replay.rs rename to backend/src/dispatch/recovery/reasoning_replay.rs diff --git a/backend/src/dispatch/responses/errors.rs b/backend/src/dispatch/responses/errors.rs deleted file mode 100644 index aefac7b77..000000000 --- a/backend/src/dispatch/responses/errors.rs +++ /dev/null @@ -1,342 +0,0 @@ -use serde_json::Value; -use thiserror::Error; - -use crate::{ - dispatch::{ - errors::{ - insert_dispatch_error_metadata, upstream_error_http_status, DispatchErrorMetadata, - DispatchFailureClass, - }, - exhaustion::{ExhaustedAccount, ExhaustedAccountKind, ExhaustedAccountRef}, - }, - upstream::openai::{ - protocol::{responses::ResponsesSseFailure, sse::SseError}, - transport::CodexClientError, - }, -}; - -use super::sse_failure::{sse_failure_error_body, stream_failure_http_status}; - -/// Responses 调度错误。 -#[derive(Debug, Error)] -pub enum ResponseDispatchError { - #[error("no active account is available")] - NoActiveAccount, - #[error("all accounts exhausted by quota")] - QuotaExhausted { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by rate limit")] - RateLimited { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by expired auth")] - Expired { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by disabled auth")] - Disabled { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by banned auth")] - Banned { - count: usize, - upstream_error: String, - status_code: u16, - }, - #[error("all accounts exhausted by Cloudflare challenge")] - CloudflareChallenge { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by Cloudflare path-block")] - CloudflarePathBlocked { - count: usize, - upstream_error: String, - }, - #[error("all accounts exhausted by unsupported model")] - ModelUnsupported { - count: usize, - upstream_error: String, - }, - #[error("previous response context is unavailable: {upstream_error}")] - HistoryUnavailable { upstream_error: String }, - #[error("upstream request failed: {0}")] - Upstream(#[from] CodexClientError), - #[error("invalid upstream SSE response: {0}")] - InvalidSse(#[from] SseError), - #[error("upstream response did not include response.completed")] - MissingCompleted, - #[error("upstream response did not include visible output")] - EmptyUpstreamResponse, - #[error("upstream response failed: {0:?}")] - Failed(ResponsesSseFailure), -} - -impl ResponseDispatchError { - pub(crate) fn from_exhausted_account(exhausted: ExhaustedAccount) -> Self { - match exhausted.kind { - ExhaustedAccountKind::QuotaExhausted => Self::QuotaExhausted { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::RateLimited => Self::RateLimited { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::Expired => Self::Expired { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::Disabled => Self::Disabled { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::Banned => Self::Banned { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - status_code: exhausted.status_code.unwrap_or(403), - }, - ExhaustedAccountKind::CloudflareChallenge => Self::CloudflareChallenge { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::CloudflarePathBlocked => Self::CloudflarePathBlocked { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - ExhaustedAccountKind::ModelUnsupported => Self::ModelUnsupported { - count: exhausted.count, - upstream_error: exhausted.upstream_error, - }, - } - } - - pub fn http_status_code(&self) -> u16 { - match self { - Self::NoActiveAccount => 503, - Self::QuotaExhausted { .. } => 429, - Self::RateLimited { .. } => 429, - Self::Expired { .. } | Self::Disabled { .. } => 401, - Self::Banned { status_code, .. } => *status_code, - Self::CloudflareChallenge { .. } - | Self::CloudflarePathBlocked { .. } - | Self::InvalidSse(_) - | Self::MissingCompleted - | Self::EmptyUpstreamResponse => 502, - Self::Failed(failure) => stream_failure_http_status(failure), - Self::ModelUnsupported { .. } | Self::HistoryUnavailable { .. } => 400, - Self::Upstream(error) => upstream_error_http_status(error), - } - } - - pub(crate) fn metadata(&self) -> DispatchErrorMetadata { - match self { - Self::NoActiveAccount => DispatchErrorMetadata::no_available_accounts(), - Self::QuotaExhausted { - count, - upstream_error, - } => Self::exhausted_metadata( - ExhaustedAccountKind::QuotaExhausted, - *count, - upstream_error, - ), - Self::RateLimited { - count, - upstream_error, - } => { - Self::exhausted_metadata(ExhaustedAccountKind::RateLimited, *count, upstream_error) - } - Self::Expired { - count, - upstream_error, - } => Self::exhausted_metadata(ExhaustedAccountKind::Expired, *count, upstream_error), - Self::Disabled { - count, - upstream_error, - } => Self::exhausted_metadata(ExhaustedAccountKind::Disabled, *count, upstream_error), - Self::Banned { - count, - upstream_error, - .. - } => Self::exhausted_metadata(ExhaustedAccountKind::Banned, *count, upstream_error), - Self::CloudflareChallenge { - count, - upstream_error, - } => Self::exhausted_metadata( - ExhaustedAccountKind::CloudflareChallenge, - *count, - upstream_error, - ), - Self::CloudflarePathBlocked { - count, - upstream_error, - } => Self::exhausted_metadata( - ExhaustedAccountKind::CloudflarePathBlocked, - *count, - upstream_error, - ), - Self::ModelUnsupported { - count, - upstream_error, - } => Self::exhausted_metadata( - ExhaustedAccountKind::ModelUnsupported, - *count, - upstream_error, - ), - Self::HistoryUnavailable { upstream_error } => DispatchErrorMetadata { - failure_class: DispatchFailureClass::HistoryUnavailable, - exhausted_count: None, - upstream_error: Some(upstream_error.clone()), - upstream_status: Some(400), - diagnostics: None, - }, - Self::Upstream(error) => DispatchErrorMetadata::upstream(error), - Self::InvalidSse(_) => DispatchErrorMetadata::simple(DispatchFailureClass::InvalidSse), - Self::MissingCompleted => { - DispatchErrorMetadata::simple(DispatchFailureClass::MissingCompleted) - } - Self::EmptyUpstreamResponse => { - DispatchErrorMetadata::simple(DispatchFailureClass::EmptyUpstreamResponse) - } - Self::Failed(failure) => DispatchErrorMetadata { - failure_class: DispatchFailureClass::ResponseFailed, - exhausted_count: None, - upstream_error: Some(sse_failure_error_body(failure)), - upstream_status: None, - diagnostics: None, - }, - } - } - - pub(crate) fn exhausted_account(&self) -> Option> { - match self { - Self::QuotaExhausted { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::QuotaExhausted, - count: *count, - upstream_error, - }), - Self::RateLimited { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::RateLimited, - count: *count, - upstream_error, - }), - Self::Expired { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::Expired, - count: *count, - upstream_error, - }), - Self::Disabled { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::Disabled, - count: *count, - upstream_error, - }), - Self::Banned { - count, - upstream_error, - .. - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::Banned, - count: *count, - upstream_error, - }), - Self::CloudflareChallenge { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::CloudflareChallenge, - count: *count, - upstream_error, - }), - Self::CloudflarePathBlocked { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::CloudflarePathBlocked, - count: *count, - upstream_error, - }), - Self::ModelUnsupported { - count, - upstream_error, - } => Some(ExhaustedAccountRef { - kind: ExhaustedAccountKind::ModelUnsupported, - count: *count, - upstream_error, - }), - _ => None, - } - } - - fn exhausted_metadata( - kind: ExhaustedAccountKind, - count: usize, - upstream_error: &str, - ) -> DispatchErrorMetadata { - DispatchErrorMetadata::exhausted_ref(ExhaustedAccountRef { - kind, - count, - upstream_error, - }) - } -} - -/// Responses live SSE body stream error. -#[derive(Debug, Error)] -pub enum ResponseDispatchStreamError { - #[error("upstream stream failed: {0}")] - Upstream(#[from] CodexClientError), -} - -pub(super) fn dispatch_error_metadata( - error: impl std::fmt::Display, - stream: bool, - compact: bool, - transport: Option<&str>, -) -> Value { - let mut metadata = serde_json::json!({ - "stream": stream, - "failed": true, - "errorKind": "dispatch", - "error": error.to_string(), - }); - let Some(object) = metadata.as_object_mut() else { - return metadata; - }; - if compact { - object.insert("compact".to_string(), Value::Bool(true)); - } - if let Some(transport) = transport { - object.insert( - "transport".to_string(), - Value::String(transport.to_string()), - ); - } - metadata -} - -pub(super) fn enrich_response_dispatch_error_metadata( - metadata: &mut Value, - error: &ResponseDispatchError, -) { - let Some(object) = metadata.as_object_mut() else { - return; - }; - insert_dispatch_error_metadata(object, error.metadata()); -} diff --git a/backend/src/dispatch/responses/mod.rs b/backend/src/dispatch/responses/mod.rs deleted file mode 100644 index ac71b5f2e..000000000 --- a/backend/src/dispatch/responses/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Responses 调度领域。 - -pub(crate) mod errors; -mod event_recording; -mod live_stream; -mod prefetch; -pub(crate) mod service; -pub(crate) mod sse_failure; -mod stream_lifecycle; -mod trace; diff --git a/backend/src/dispatch/responses/service.rs b/backend/src/dispatch/service.rs similarity index 90% rename from backend/src/dispatch/responses/service.rs rename to backend/src/dispatch/service.rs index 060f3cd2e..41ea9fa17 100644 --- a/backend/src/dispatch/responses/service.rs +++ b/backend/src/dispatch/service.rs @@ -26,21 +26,23 @@ use crate::{ compute_variant_hash, ensure_prompt_cache_key, hash_instructions, prepare_variant_identity, RuntimeSessionAffinityService, }, - cloudflare::{ - cloudflare_challenge_error_message, cloudflare_path_block_error_message, - is_cloudflare_challenge_upstream_error, is_cloudflare_path_block_upstream_error, - CloudflareRecovery, - }, errors::{ - auth_failure_account_status, backend_transport_name, is_auth_upstream_error, - is_history_recovery_upstream_error, is_model_unsupported_upstream_error, - is_quota_exhausted_upstream_error, is_rate_limit_upstream_error, - rate_limit_cooldown_until, upstream_error_body, upstream_error_http_status, - upstream_error_set_cookie_headers, + backend_transport_name, is_history_recovery_upstream_error, + is_model_unsupported_upstream_error, is_quota_exhausted_upstream_error, + is_rate_limit_upstream_error, rate_limit_cooldown_until, upstream_error_body, + upstream_error_http_status, upstream_error_set_cookie_headers, }, - exhaustion::AccountExhaustionTracker, - reasoning_replay::ReasoningReplayCache, - upstream::{ + recovery::{ + auth::{auth_failure_account_status, is_auth_upstream_error}, + cloudflare::{ + cloudflare_challenge_error_message, cloudflare_path_block_error_message, + is_cloudflare_challenge_upstream_error, is_cloudflare_path_block_upstream_error, + CloudflareRecovery, + }, + exhaustion::AccountExhaustionTracker, + reasoning_replay::ReasoningReplayCache, + }, + upstream_call::{ create_compact_response_with_account_retrying_5xx, create_response_stream_with_account_retrying_5xx, create_response_with_account_retrying_5xx, verify_acquired_quota_if_required, @@ -71,13 +73,13 @@ use crate::{ }, }; -use crate::dispatch::implicit_resume::{ +use crate::dispatch::recovery::implicit_resume::{ continuation_input_start, implicit_resume_allowed, ImplicitResumeSnapshot, }; use super::{ errors::{ResponseDispatchError, ResponseDispatchStreamError}, - event_recording::{ + recording::{ insert_response_status_metadata, insert_response_trace_metadata, insert_response_upstream_diagnostics, insert_websocket_pool_decision, record_prefetched_response_stream_failure_event, record_response_dispatch_error_event, @@ -85,16 +87,19 @@ use super::{ ResponseDispatchErrorEventRecord, ResponseStreamFailureEventRecord, ResponseUpstreamErrorEventRecord, }, - live_stream::spawn_live_response_stream, - prefetch::prefetch_first_sse_chunk, - sse_failure::{ - auth_sse_failure_account_status, client_error_invalid_reasoning_replay, first_sse_failure, - is_auth_sse_failure, is_history_recovery_sse_failure, is_model_unsupported_sse_failure, - is_quota_exhausted_sse_failure, sse_failure_error_body, - sse_failure_invalid_reasoning_replay, stream_failure_http_status, + stream::{ + lifecycle::LiveResponseStreamContext, + live::spawn_live_response_stream, + prefetch::prefetch_first_sse_chunk, + sse_failure::{ + auth_sse_failure_account_status, client_error_invalid_reasoning_replay, + first_sse_failure, is_auth_sse_failure, is_history_recovery_sse_failure, + is_model_unsupported_sse_failure, is_quota_exhausted_sse_failure, + sse_failure_error_body, sse_failure_invalid_reasoning_replay, + stream_failure_http_status, + }, + trace::{ResponseDispatchAttempt, ResponseDispatchTrace}, }, - stream_lifecycle::LiveResponseStreamContext, - trace::{ResponseDispatchAttempt, ResponseDispatchTrace}, }; /// OpenAI Responses 调度服务。 diff --git a/backend/src/dispatch/responses/service/compact.rs b/backend/src/dispatch/service/compact.rs similarity index 100% rename from backend/src/dispatch/responses/service/compact.rs rename to backend/src/dispatch/service/compact.rs diff --git a/backend/src/dispatch/responses/service/complete.rs b/backend/src/dispatch/service/complete.rs similarity index 100% rename from backend/src/dispatch/responses/service/complete.rs rename to backend/src/dispatch/service/complete.rs diff --git a/backend/src/dispatch/responses/service/stream.rs b/backend/src/dispatch/service/stream.rs similarity index 100% rename from backend/src/dispatch/responses/service/stream.rs rename to backend/src/dispatch/service/stream.rs diff --git a/backend/src/dispatch/responses/stream_lifecycle.rs b/backend/src/dispatch/stream/lifecycle.rs similarity index 79% rename from backend/src/dispatch/responses/stream_lifecycle.rs rename to backend/src/dispatch/stream/lifecycle.rs index 1119e101f..5abdb0cd6 100644 --- a/backend/src/dispatch/responses/stream_lifecycle.rs +++ b/backend/src/dispatch/stream/lifecycle.rs @@ -11,7 +11,11 @@ use crate::{ RuntimeSessionAffinityService, }, errors::backend_transport_name, - reasoning_replay::ReasoningReplayCache, + recording::{ + insert_first_token_ms, live_response_rate_limit_headers, live_response_turn_state, + record_live_response_stream_event, + }, + recovery::reasoning_replay::ReasoningReplayCache, }, infra::time::elapsed_millis_i64, telemetry::{recorder::Recorder, usage::types::UsageRecordLevel}, @@ -29,10 +33,6 @@ use crate::{ }; use super::{ - event_recording::{ - insert_first_token_ms, live_response_rate_limit_headers, live_response_turn_state, - record_live_response_stream_event, - }, sse_failure::{ sse_failure_invalid_reasoning_replay, status_code_for_stream_failure, stream_failure_metadata, stream_failure_source, synthetic_stream_disconnected_detail, @@ -40,37 +40,37 @@ use super::{ }, trace::ResponseDispatchAttempt, }; -use crate::dispatch::cloudflare::CloudflareRecovery; +use crate::dispatch::recovery::cloudflare::CloudflareRecovery; -pub(super) struct LiveResponseStreamContext { - pub(super) account_pool: Arc, - pub(super) session_affinity: Arc, - pub(super) reasoning_replay: Arc>, - pub(super) recorder: Arc, - pub(super) cloudflare: CloudflareRecovery, - pub(super) account_id: String, - pub(super) account_plan_type: Option, - pub(super) request_id: String, - pub(super) route: String, - pub(super) model: String, - pub(super) display_model: String, - pub(super) requested_model: String, - pub(super) client_ip: Option, - pub(super) request: CodexResponsesRequest, - pub(super) tuple_schema: Option, - pub(super) transport: CodexBackendTransport, - pub(super) rate_limit_headers: Vec<(String, String)>, - pub(super) rate_limit_header_updates: Option, - pub(super) turn_state_update: Option, - pub(super) websocket_pool_decision: Option, - pub(super) turn_state: Option, - pub(super) diagnostics: CodexUpstreamDiagnostics, - pub(super) attempt: ResponseDispatchAttempt, - pub(super) attempts: Vec, - pub(super) started_at: Instant, +pub(in crate::dispatch) struct LiveResponseStreamContext { + pub(in crate::dispatch) account_pool: Arc, + pub(in crate::dispatch) session_affinity: Arc, + pub(in crate::dispatch) reasoning_replay: Arc>, + pub(in crate::dispatch) recorder: Arc, + pub(in crate::dispatch) cloudflare: CloudflareRecovery, + pub(in crate::dispatch) account_id: String, + pub(in crate::dispatch) account_plan_type: Option, + pub(in crate::dispatch) request_id: String, + pub(in crate::dispatch) route: String, + pub(in crate::dispatch) model: String, + pub(in crate::dispatch) display_model: String, + pub(in crate::dispatch) requested_model: String, + pub(in crate::dispatch) client_ip: Option, + pub(in crate::dispatch) request: CodexResponsesRequest, + pub(in crate::dispatch) tuple_schema: Option, + pub(in crate::dispatch) transport: CodexBackendTransport, + pub(in crate::dispatch) rate_limit_headers: Vec<(String, String)>, + pub(in crate::dispatch) rate_limit_header_updates: Option, + pub(in crate::dispatch) turn_state_update: Option, + pub(in crate::dispatch) websocket_pool_decision: Option, + pub(in crate::dispatch) turn_state: Option, + pub(in crate::dispatch) diagnostics: CodexUpstreamDiagnostics, + pub(in crate::dispatch) attempt: ResponseDispatchAttempt, + pub(in crate::dispatch) attempts: Vec, + pub(in crate::dispatch) started_at: Instant, } -pub(super) fn latest_response_id(body: &str) -> Option { +pub(in crate::dispatch) fn latest_response_id(body: &str) -> Option { parse_sse_events(body).ok().and_then(|events| { events.iter().rev().find_map(|event| { serde_json::from_str::(&event.data) @@ -85,7 +85,7 @@ pub(super) fn latest_response_id(body: &str) -> Option { }) } -pub(super) fn premature_close_failed_event( +pub(in crate::dispatch) fn premature_close_failed_event( response_id: Option<&str>, detail: Option<&str>, ) -> String { @@ -101,7 +101,7 @@ pub(super) fn premature_close_failed_event( ) } -pub(super) async fn finalize_live_response_stream( +pub(in crate::dispatch) async fn finalize_live_response_stream( context: LiveResponseStreamContext, body: String, first_token_ms: Option, diff --git a/backend/src/dispatch/responses/live_stream.rs b/backend/src/dispatch/stream/live.rs similarity index 98% rename from backend/src/dispatch/responses/live_stream.rs rename to backend/src/dispatch/stream/live.rs index eb60ee2e0..105dece8c 100644 --- a/backend/src/dispatch/responses/live_stream.rs +++ b/backend/src/dispatch/stream/live.rs @@ -9,10 +9,10 @@ use serde_json::Value; use tokio::sync::{mpsc, oneshot}; use crate::{ - dispatch::responses::{ + dispatch::{ errors::ResponseDispatchStreamError, service::ResponseDispatchStream, - stream_lifecycle::{ + stream::lifecycle::{ finalize_live_response_stream, latest_response_id, premature_close_failed_event, LiveResponseStreamContext, }, @@ -53,7 +53,7 @@ impl Stream for MpscResponseBodyStream { } } -pub(super) fn spawn_live_response_stream( +pub(in crate::dispatch) fn spawn_live_response_stream( context: LiveResponseStreamContext, prefetched: Bytes, mut body: CodexBackendSseStream, diff --git a/backend/src/dispatch/stream/mod.rs b/backend/src/dispatch/stream/mod.rs new file mode 100644 index 000000000..589774e87 --- /dev/null +++ b/backend/src/dispatch/stream/mod.rs @@ -0,0 +1,7 @@ +//! Responses 流式转发与生命周期处理。 + +pub(crate) mod lifecycle; +pub(crate) mod live; +pub(crate) mod prefetch; +pub(crate) mod sse_failure; +pub(crate) mod trace; diff --git a/backend/src/dispatch/responses/prefetch.rs b/backend/src/dispatch/stream/prefetch.rs similarity index 90% rename from backend/src/dispatch/responses/prefetch.rs rename to backend/src/dispatch/stream/prefetch.rs index aa870dfc4..9c7b375ff 100644 --- a/backend/src/dispatch/responses/prefetch.rs +++ b/backend/src/dispatch/stream/prefetch.rs @@ -6,11 +6,11 @@ use crate::upstream::openai::{ transport::CodexBackendSseStream, }; -use super::errors::ResponseDispatchError; +use crate::dispatch::errors::ResponseDispatchError; const MAX_STREAM_PREFETCH_BYTES: usize = 64 * 1024; -pub(super) async fn prefetch_first_sse_chunk( +pub(in crate::dispatch) async fn prefetch_first_sse_chunk( mut body: CodexBackendSseStream, ) -> Result<(Bytes, CodexBackendSseStream), ResponseDispatchError> { let mut prefetched = Vec::new(); diff --git a/backend/src/dispatch/responses/sse_failure.rs b/backend/src/dispatch/stream/sse_failure.rs similarity index 82% rename from backend/src/dispatch/responses/sse_failure.rs rename to backend/src/dispatch/stream/sse_failure.rs index 383aa3626..483265728 100644 --- a/backend/src/dispatch/responses/sse_failure.rs +++ b/backend/src/dispatch/stream/sse_failure.rs @@ -16,11 +16,11 @@ use crate::{ }, }; -pub(super) const STREAM_DISCONNECTED_CODE: &str = "stream_disconnected"; -pub(super) const STREAM_DISCONNECTED_MESSAGE: &str = +pub(in crate::dispatch) const STREAM_DISCONNECTED_CODE: &str = "stream_disconnected"; +pub(in crate::dispatch) const STREAM_DISCONNECTED_MESSAGE: &str = "Upstream stream closed before response.completed"; -pub(super) fn sse_failure_error_body(failure: &ResponsesSseFailure) -> String { +pub(in crate::dispatch) fn sse_failure_error_body(failure: &ResponsesSseFailure) -> String { match failure.upstream_code.as_deref() { Some(code) => serde_json::json!({ "error": { @@ -88,19 +88,21 @@ pub(crate) fn is_auth_sse_failure(failure: &ResponsesSseFailure) -> bool { ) } -pub(super) fn is_model_unsupported_sse_failure(failure: &ResponsesSseFailure) -> bool { +pub(in crate::dispatch) fn is_model_unsupported_sse_failure(failure: &ResponsesSseFailure) -> bool { sse_failure_matches(failure, is_model_unsupported_signal) } -pub(super) fn is_history_recovery_sse_failure(failure: &ResponsesSseFailure) -> bool { +pub(in crate::dispatch) fn is_history_recovery_sse_failure(failure: &ResponsesSseFailure) -> bool { sse_failure_matches(failure, is_history_recovery_signal) } -pub(super) fn sse_failure_invalid_reasoning_replay(failure: &ResponsesSseFailure) -> bool { +pub(in crate::dispatch) fn sse_failure_invalid_reasoning_replay( + failure: &ResponsesSseFailure, +) -> bool { sse_failure_matches(failure, is_invalid_encrypted_content_signal) } -pub(super) fn client_error_invalid_reasoning_replay(error: &CodexClientError) -> bool { +pub(in crate::dispatch) fn client_error_invalid_reasoning_replay(error: &CodexClientError) -> bool { matches!( error, CodexClientError::Upstream { body, .. } if is_invalid_encrypted_content_signal(body) @@ -115,17 +117,7 @@ pub(crate) fn auth_sse_failure_account_status(failure: &ResponsesSseFailure) -> } } -pub(crate) fn sse_failure_account_status(failure: &ResponsesSseFailure) -> Option { - if is_quota_exhausted_sse_failure(failure) { - Some(AccountStatus::QuotaExhausted) - } else if is_auth_sse_failure(failure) { - Some(auth_sse_failure_account_status(failure)) - } else { - None - } -} - -pub(super) fn first_sse_failure( +pub(in crate::dispatch) fn first_sse_failure( prefetched: &[u8], ) -> Result, SseError> { let body = String::from_utf8_lossy(prefetched); @@ -137,7 +129,7 @@ pub(super) fn first_sse_failure( } } -pub(super) fn stream_failure_metadata( +pub(in crate::dispatch) fn stream_failure_metadata( failure: &ResponsesSseFailure, usage: Option, ) -> Value { @@ -169,7 +161,7 @@ fn enrich_stream_failure_source_metadata(metadata: &mut Value, failure: &Respons } } -pub(super) fn stream_failure_source(failure: &ResponsesSseFailure) -> &'static str { +pub(in crate::dispatch) fn stream_failure_source(failure: &ResponsesSseFailure) -> &'static str { if synthetic_stream_disconnected_detail(failure).is_some() { "proxy" } else { @@ -177,7 +169,7 @@ pub(super) fn stream_failure_source(failure: &ResponsesSseFailure) -> &'static s } } -pub(super) fn synthetic_stream_disconnected_detail( +pub(in crate::dispatch) fn synthetic_stream_disconnected_detail( failure: &ResponsesSseFailure, ) -> Option { if failure.upstream_code.as_deref() != Some(STREAM_DISCONNECTED_CODE) { @@ -193,7 +185,7 @@ pub(super) fn synthetic_stream_disconnected_detail( Some(detail) } -pub(super) fn status_code_for_stream_failure(failure: &ResponsesSseFailure) -> i64 { +pub(in crate::dispatch) fn status_code_for_stream_failure(failure: &ResponsesSseFailure) -> i64 { let code = failure .upstream_code .as_deref() @@ -236,6 +228,6 @@ pub(super) fn status_code_for_stream_failure(failure: &ResponsesSseFailure) -> i 502 } -pub(super) fn stream_failure_http_status(failure: &ResponsesSseFailure) -> u16 { +pub(in crate::dispatch) fn stream_failure_http_status(failure: &ResponsesSseFailure) -> u16 { u16::try_from(status_code_for_stream_failure(failure)).unwrap_or(502) } diff --git a/backend/src/dispatch/responses/trace.rs b/backend/src/dispatch/stream/trace.rs similarity index 60% rename from backend/src/dispatch/responses/trace.rs rename to backend/src/dispatch/stream/trace.rs index 02066f3bc..936834b05 100644 --- a/backend/src/dispatch/responses/trace.rs +++ b/backend/src/dispatch/stream/trace.rs @@ -1,19 +1,22 @@ //! Request-level trace state for Responses dispatch. #[derive(Debug, Default)] -pub(super) struct ResponseDispatchTrace { +pub(in crate::dispatch) struct ResponseDispatchTrace { next_attempt_index: i64, attempts: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct ResponseDispatchAttempt { +pub(in crate::dispatch) struct ResponseDispatchAttempt { index: i64, account_id: String, } impl ResponseDispatchTrace { - pub(super) fn start_attempt(&mut self, account_id: &str) -> ResponseDispatchAttempt { + pub(in crate::dispatch) fn start_attempt( + &mut self, + account_id: &str, + ) -> ResponseDispatchAttempt { let attempt = ResponseDispatchAttempt { index: self.next_attempt_index, account_id: account_id.to_string(), @@ -23,17 +26,17 @@ impl ResponseDispatchTrace { attempt } - pub(super) fn attempts(&self) -> &[ResponseDispatchAttempt] { + pub(in crate::dispatch) fn attempts(&self) -> &[ResponseDispatchAttempt] { &self.attempts } } impl ResponseDispatchAttempt { - pub(super) fn index(&self) -> i64 { + pub(in crate::dispatch) fn index(&self) -> i64 { self.index } - pub(super) fn account_id(&self) -> &str { + pub(in crate::dispatch) fn account_id(&self) -> &str { &self.account_id } } diff --git a/backend/src/dispatch/upstream.rs b/backend/src/dispatch/upstream_call.rs similarity index 99% rename from backend/src/dispatch/upstream.rs rename to backend/src/dispatch/upstream_call.rs index f4a2cf91f..e8b0d321c 100644 --- a/backend/src/dispatch/upstream.rs +++ b/backend/src/dispatch/upstream_call.rs @@ -14,8 +14,8 @@ use crate::{ }, }, dispatch::{ - affinity::build_conversation_identity, cloudflare::CloudflareRecovery, - errors::is_retryable_upstream_5xx_error, + affinity::build_conversation_identity, errors::is_retryable_upstream_5xx_error, + recovery::cloudflare::CloudflareRecovery, }, upstream::openai::{ protocol::responses::{CodexCompactRequest, CodexResponsesRequest}, diff --git a/backend/src/main.rs b/backend/src/main.rs index cbcb1c11c..5249dfb69 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -6,7 +6,7 @@ async fn main() -> Result<(), Box> { match arguments.next().as_deref().and_then(|value| value.to_str()) { None | Some("serve") => { reject_extra_arguments(arguments)?; - codex_proxy_rs::bootstrap::application::run().await + codex_proxy_rs::bootstrap::services::run().await } Some("import-sqlite") => { let source = arguments diff --git a/backend/src/models/catalog.rs b/backend/src/models/catalog.rs index aacb97371..43d2dc3a9 100644 --- a/backend/src/models/catalog.rs +++ b/backend/src/models/catalog.rs @@ -2,12 +2,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::upstream::openai::protocol::model_catalog::{BackendModelEntry, BackendReasoningEffort}; - -use super::{ - config::ModelConfig, - info::{CodexModelInfo, ReasoningEffortInfo}, - snapshot::ModelPlanSnapshot, +use super::types::{ + BackendModelEntry, BackendReasoningEffort, CodexModelInfo, ModelConfig, ModelPlanSnapshot, + ReasoningEffortInfo, }; const BUILTIN_CODEX_MODELS: [(&str, &str, bool); 8] = [ @@ -41,6 +38,20 @@ impl ModelPlanSnapshot { models: entries.into_iter().map(normalize_backend_model).collect(), } } + + /// 将上游 JSON 条目解析并标准化为可持久化快照。 + pub fn from_backend_values( + plan_type: impl Into, + entries: Vec, + ) -> Self { + Self::from_backend_entries( + plan_type, + entries + .into_iter() + .filter_map(|entry| serde_json::from_value(entry).ok()) + .collect(), + ) + } } impl ModelCatalog { diff --git a/backend/src/models/config.rs b/backend/src/models/config.rs deleted file mode 100644 index 40e79a39b..000000000 --- a/backend/src/models/config.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! 模型目录配置。 - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -/// 模型目录别名配置。 -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub struct ModelConfig { - /// 模型别名映射。 - pub model_aliases: BTreeMap, -} diff --git a/backend/src/models/info.rs b/backend/src/models/info.rs deleted file mode 100644 index 927a4eedb..000000000 --- a/backend/src/models/info.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! 标准化模型展示信息。 - -use serde::{Deserialize, Serialize}; - -/// 单个 reasoning effort 的标准化展示信息。 -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct ReasoningEffortInfo { - /// 推理强度名。 - pub reasoning_effort: String, - /// 展示描述。 - pub description: String, -} - -/// 对外暴露的单个模型目录条目。 -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct CodexModelInfo { - /// 模型唯一 ID。 - pub id: String, - /// 展示名。 - pub display_name: String, - /// 描述文本。 - pub description: String, - /// 是否默认模型。 - pub is_default: bool, - /// 支持的推理强度列表。 - pub supported_reasoning_efforts: Vec, - /// 默认推理强度。 - pub default_reasoning_effort: String, - /// 输入模态。 - pub input_modalities: Vec, - /// 输出模态。 - pub output_modalities: Vec, - /// 是否支持 personality。 - pub supports_personality: bool, - /// 升级提示。 - pub upgrade: Option, - /// 来源标记。 - pub source: String, - /// 当前上下文窗口。 - #[serde(skip_serializing_if = "Option::is_none")] - pub context_window: Option, - /// 最大上下文窗口。 - #[serde(skip_serializing_if = "Option::is_none")] - pub max_context_window: Option, - /// 最大输出 token 数。 - #[serde(skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - /// 截断策略限制。 - #[serde(skip_serializing_if = "Option::is_none")] - pub truncation_policy_limit: Option, -} diff --git a/backend/src/models/mod.rs b/backend/src/models/mod.rs index 68b555550..0690ad55c 100644 --- a/backend/src/models/mod.rs +++ b/backend/src/models/mod.rs @@ -1,8 +1,6 @@ //! 模型目录领域。 pub mod catalog; -pub mod config; -pub mod info; pub mod service; -pub mod snapshot; pub mod store; +pub mod types; diff --git a/backend/src/models/service.rs b/backend/src/models/service.rs index e3d0c384f..99ad4bb70 100644 --- a/backend/src/models/service.rs +++ b/backend/src/models/service.rs @@ -6,17 +6,17 @@ use std::{ }; use thiserror::Error; +use tokio::sync::watch; use crate::{ - accounts::account::Account, + settings::SettingsSnapshot, upstream::openai::transport::{CodexModelCatalogClient, CodexModelCatalogRequest}, }; use super::{ catalog::ModelCatalog, - config::ModelConfig, - snapshot::ModelPlanSnapshot, store::{ModelSnapshotStore, ModelSnapshotStoreError}, + types::{ModelConfig, ModelPlanSnapshot}, }; /// 模型刷新摘要。 @@ -76,8 +76,10 @@ pub struct ModelPlanRouting { pub struct ModelRefreshPlanAccount { /// 订阅计划类型。 pub plan_type: String, - /// 用于刷新该计划模型列表的账号。 - pub account: Account, + /// 用于访问模型目录的令牌。 + pub access_token: String, + /// 上游账号标识。 + pub account_id: Option, } /// 模型目录服务。 @@ -116,6 +118,19 @@ impl ModelService { self.rebuild_catalog_from_memory(); } + /// 持续接收运行时设置并更新模型别名。 + pub async fn subscribe_settings( + self: Arc, + mut receiver: watch::Receiver, + ) { + while receiver.changed().await.is_ok() { + let settings = receiver.borrow_and_update().clone(); + self.update_config(ModelConfig { + model_aliases: settings.model_aliases, + }); + } + } + /// 从快照存储加载运行时内存模型目录。 pub async fn reload_from_store(&self) -> Result<(), ModelServiceError> { let Some(store) = self.store.as_ref() else { @@ -163,8 +178,8 @@ impl ModelService { for plan_account in plan_accounts { let request = CodexModelCatalogRequest { - access_token: &plan_account.account.access_token, - account_id: plan_account.account.account_id.as_deref(), + access_token: &plan_account.access_token, + account_id: plan_account.account_id.as_deref(), request_id, installation_id, plan_type: &plan_account.plan_type, @@ -184,7 +199,11 @@ impl ModelService { }; let snapshot = - ModelPlanSnapshot::from_backend_entries(plan_account.plan_type.clone(), entries); + ModelPlanSnapshot::from_backend_values(plan_account.plan_type.clone(), entries); + if snapshot.models.is_empty() { + result.failed_plans += 1; + continue; + } result.model_count += snapshot.models.len(); store .replace_plan_snapshot(&snapshot) diff --git a/backend/src/models/snapshot.rs b/backend/src/models/snapshot.rs deleted file mode 100644 index 3f9ef7f6f..000000000 --- a/backend/src/models/snapshot.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! 模型计划快照。 - -use serde::{Deserialize, Serialize}; - -use super::info::CodexModelInfo; - -/// 按计划类型持久化的模型快照。 -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub struct ModelPlanSnapshot { - /// 订阅计划类型。 - pub plan_type: String, - /// 该计划可见的模型列表。 - pub models: Vec, -} diff --git a/backend/src/models/store.rs b/backend/src/models/store.rs index eb6e7b95b..26708a68e 100644 --- a/backend/src/models/store.rs +++ b/backend/src/models/store.rs @@ -8,7 +8,7 @@ use thiserror::Error; use crate::infra::redis::RedisConnection; -use super::{info::CodexModelInfo, snapshot::ModelPlanSnapshot}; +use super::types::{CodexModelInfo, ModelPlanSnapshot}; /// 模型快照存储错误。 #[derive(Debug, Error, PartialEq, Eq)] diff --git a/backend/src/models/types.rs b/backend/src/models/types.rs new file mode 100644 index 000000000..bbfafdf42 --- /dev/null +++ b/backend/src/models/types.rs @@ -0,0 +1,119 @@ +//! 模型目录领域类型。 + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// 模型目录别名配置。 +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub struct ModelConfig { + /// 模型别名映射。 + pub model_aliases: BTreeMap, +} + +/// 单个 reasoning effort 的标准化展示信息。 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ReasoningEffortInfo { + /// 推理强度名。 + pub reasoning_effort: String, + /// 展示描述。 + pub description: String, +} + +/// 对外暴露的单个模型目录条目。 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct CodexModelInfo { + /// 模型唯一 ID。 + pub id: String, + /// 展示名。 + pub display_name: String, + /// 描述文本。 + pub description: String, + /// 是否默认模型。 + pub is_default: bool, + /// 支持的推理强度列表。 + pub supported_reasoning_efforts: Vec, + /// 默认推理强度。 + pub default_reasoning_effort: String, + /// 输入模态。 + pub input_modalities: Vec, + /// 输出模态。 + pub output_modalities: Vec, + /// 是否支持 personality。 + pub supports_personality: bool, + /// 升级提示。 + pub upgrade: Option, + /// 来源标记。 + pub source: String, + /// 当前上下文窗口。 + #[serde(skip_serializing_if = "Option::is_none")] + pub context_window: Option, + /// 最大上下文窗口。 + #[serde(skip_serializing_if = "Option::is_none")] + pub max_context_window: Option, + /// 最大输出 token 数。 + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + /// 截断策略限制。 + #[serde(skip_serializing_if = "Option::is_none")] + pub truncation_policy_limit: Option, +} + +/// 按计划类型持久化的模型快照。 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ModelPlanSnapshot { + /// 订阅计划类型。 + pub plan_type: String, + /// 该计划可见的模型列表。 + pub models: Vec, +} + +/// 上游声明的 reasoning effort 条目。 +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct BackendReasoningEffort { + #[serde(default, rename = "reasoning_effort", alias = "reasoningEffort")] + pub reasoning_effort: Option, + #[serde(default)] + pub effort: Option, + #[serde(default)] + pub description: Option, +} + +/// 上游截断策略定义。 +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct BackendTruncationPolicy { + pub limit: Option, +} + +/// 上游模型目录的原始模型条目。 +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct BackendModelEntry { + pub slug: Option, + pub id: Option, + pub name: Option, + pub display_name: Option, + pub title: Option, + pub description: Option, + pub is_default: Option, + pub default_reasoning_effort: Option, + pub default_reasoning_level: Option, + #[serde(default)] + pub supported_reasoning_efforts: Vec, + #[serde(default)] + pub supported_reasoning_levels: Vec, + pub input_modalities: Option>, + pub output_modalities: Option>, + pub supports_personality: Option, + pub upgrade: Option, + #[serde(alias = "contextWindow")] + pub context_window: Option, + #[serde(alias = "maxContextWindow")] + pub max_context_window: Option, + #[serde(alias = "maxOutputTokens")] + pub max_output_tokens: Option, + #[serde(alias = "truncationPolicy")] + pub truncation_policy: Option, +} diff --git a/backend/src/settings/service.rs b/backend/src/settings/service.rs index ebf1a7d49..ffbd66599 100644 --- a/backend/src/settings/service.rs +++ b/backend/src/settings/service.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, sync::Arc}; use subtle::ConstantTimeEq; -use tokio::sync::RwLock; +use tokio::sync::{watch, Mutex}; use crate::infra::identity::{generate_admin_api_key, hash_credential}; @@ -52,14 +52,17 @@ impl SettingsService { #[derive(Clone)] pub struct RuntimeSettingsService { store: PgSettingsStore, - snapshot: Arc>>, + sender: watch::Sender, + update_lock: Arc>, } impl RuntimeSettingsService { pub fn new(settings: SettingsSnapshot, pool: sqlx::PgPool) -> Self { + let (sender, _) = watch::channel(settings); Self { store: PgSettingsStore::new(pool), - snapshot: Arc::new(RwLock::new(Arc::new(settings))), + sender, + update_lock: Arc::new(Mutex::new(())), } } @@ -72,20 +75,21 @@ impl RuntimeSettingsService { .await } - pub async fn current(&self) -> Arc { - self.snapshot.read().await.clone() + pub fn current(&self) -> SettingsSnapshot { + self.sender.borrow().clone() } - pub async fn update( - &self, - patch: SettingsPatch, - ) -> Result, SettingsError> { - let mut current = self.snapshot.write().await; - let mut next = (**current).clone(); + /// 订阅设置快照变更。 + pub fn subscribe(&self) -> watch::Receiver { + self.sender.subscribe() + } + + pub async fn update(&self, patch: SettingsPatch) -> Result { + let _guard = self.update_lock.lock().await; + let mut next = self.current(); SettingsService::apply_patch(&mut next, patch)?; self.store.save(&next).await?; - let next = Arc::new(next); - *current = next.clone(); + self.sender.send_replace(next.clone()); Ok(next) } @@ -94,7 +98,7 @@ impl RuntimeSettingsService { } pub async fn regenerate_admin_api_key(&self) -> Result { - let current = self.current().await; + let current = self.current(); self.store.ensure(¤t).await?; let key = generate_admin_api_key(); self.store diff --git a/backend/src/telemetry/account_usage/store.rs b/backend/src/telemetry/account_usage/store.rs index d438800da..7f891356c 100644 --- a/backend/src/telemetry/account_usage/store.rs +++ b/backend/src/telemetry/account_usage/store.rs @@ -1,14 +1,114 @@ //! PostgreSQL 账号用量查询存储。 +use std::collections::HashMap; + +use async_trait::async_trait; use chrono::{DateTime, Utc}; use sqlx::{postgres::PgRow, PgPool, Postgres, QueryBuilder, Row}; use thiserror::Error; use crate::infra::{ + format::nonnegative_i64_to_u64, json::{decode_cursor, encode_cursor, Page}, time::parse_rfc3339_utc as parse_rfc3339, }; +const RECORD_USAGE_SQL: &str = r" +insert into account_usage ( + account_id, request_count, empty_response_count, + input_tokens, output_tokens, cached_tokens, reasoning_tokens, total_tokens, + image_input_tokens, image_output_tokens, image_request_count, image_request_failed_count, + window_request_count, window_input_tokens, window_output_tokens, window_cached_tokens, + window_image_input_tokens, window_image_output_tokens, + window_image_request_count, window_image_request_failed_count, + window_started_at, last_used_at +) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) +on conflict(account_id) do update set + request_count = account_usage.request_count + excluded.request_count, + empty_response_count = account_usage.empty_response_count + excluded.empty_response_count, + input_tokens = account_usage.input_tokens + excluded.input_tokens, + output_tokens = account_usage.output_tokens + excluded.output_tokens, + cached_tokens = account_usage.cached_tokens + excluded.cached_tokens, + reasoning_tokens = account_usage.reasoning_tokens + excluded.reasoning_tokens, + total_tokens = account_usage.total_tokens + excluded.total_tokens, + image_input_tokens = account_usage.image_input_tokens + excluded.image_input_tokens, + image_output_tokens = account_usage.image_output_tokens + excluded.image_output_tokens, + image_request_count = account_usage.image_request_count + excluded.image_request_count, + image_request_failed_count = account_usage.image_request_failed_count + excluded.image_request_failed_count, + window_request_count = account_usage.window_request_count + excluded.window_request_count, + window_input_tokens = account_usage.window_input_tokens + excluded.window_input_tokens, + window_output_tokens = account_usage.window_output_tokens + excluded.window_output_tokens, + window_cached_tokens = account_usage.window_cached_tokens + excluded.window_cached_tokens, + window_image_input_tokens = account_usage.window_image_input_tokens + excluded.window_image_input_tokens, + window_image_output_tokens = account_usage.window_image_output_tokens + excluded.window_image_output_tokens, + window_image_request_count = account_usage.window_image_request_count + excluded.window_image_request_count, + window_image_request_failed_count = account_usage.window_image_request_failed_count + excluded.window_image_request_failed_count, + window_started_at = coalesce(account_usage.window_started_at, excluded.window_started_at), + last_used_at = excluded.last_used_at"; + +const RECORD_MODEL_USAGE_SQL: &str = r" +insert into account_model_usage ( + account_id, model, request_count, error_count, + input_tokens, output_tokens, cached_tokens, last_used_at +) values ($1, $2, $3, $4, $5, $6, $7, $8) +on conflict(account_id, model) do update set + request_count = account_model_usage.request_count + excluded.request_count, + error_count = account_model_usage.error_count + excluded.error_count, + input_tokens = account_model_usage.input_tokens + excluded.input_tokens, + output_tokens = account_model_usage.output_tokens + excluded.output_tokens, + cached_tokens = account_model_usage.cached_tokens + excluded.cached_tokens, + last_used_at = excluded.last_used_at"; + +const SYNC_RUNTIME_WINDOW_SQL: &str = r" +insert into account_usage ( + account_id, window_request_count, window_input_tokens, window_output_tokens, + window_cached_tokens, window_image_input_tokens, window_image_output_tokens, + window_image_request_count, window_image_request_failed_count, + window_started_at, window_reset_at, limit_window_seconds +) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +on conflict(account_id) do update set + window_request_count = excluded.window_request_count, + window_input_tokens = excluded.window_input_tokens, + window_output_tokens = excluded.window_output_tokens, + window_cached_tokens = excluded.window_cached_tokens, + window_image_input_tokens = excluded.window_image_input_tokens, + window_image_output_tokens = excluded.window_image_output_tokens, + window_image_request_count = excluded.window_image_request_count, + window_image_request_failed_count = excluded.window_image_request_failed_count, + window_started_at = excluded.window_started_at, + window_reset_at = excluded.window_reset_at, + limit_window_seconds = excluded.limit_window_seconds +where account_usage.window_reset_at is null + or account_usage.window_reset_at <= excluded.window_reset_at + or account_usage.window_reset_at <= $13"; + +const SYNC_RATE_LIMIT_WINDOW_RESET_SQL: &str = r" +insert into account_usage ( + account_id, window_request_count, window_input_tokens, window_output_tokens, + window_cached_tokens, window_image_input_tokens, window_image_output_tokens, + window_image_request_count, window_image_request_failed_count, + window_started_at, window_reset_at, limit_window_seconds +) values ($1, 0, 0, 0, 0, 0, 0, 0, 0, $2, $3, $4) +on conflict(account_id) do update set + window_request_count = 0, + window_input_tokens = 0, + window_output_tokens = 0, + window_cached_tokens = 0, + window_image_input_tokens = 0, + window_image_output_tokens = 0, + window_image_request_count = 0, + window_image_request_failed_count = 0, + window_started_at = excluded.window_started_at, + window_reset_at = excluded.window_reset_at, + limit_window_seconds = coalesce(excluded.limit_window_seconds, account_usage.limit_window_seconds)"; + +const SYNC_RATE_LIMIT_WINDOW_SQL: &str = r" +insert into account_usage (account_id, window_reset_at, limit_window_seconds) +values ($1, $2, $3) +on conflict(account_id) do update set + window_reset_at = excluded.window_reset_at, + limit_window_seconds = coalesce(excluded.limit_window_seconds, account_usage.limit_window_seconds)"; + const LIST_USAGE_SELECT_SQL: &str = r" select au.account_id, @@ -71,6 +171,127 @@ pub enum PgAccountUsageStoreError { /// PostgreSQL 用量存储结果。 pub type PgAccountUsageStoreResult = Result; +/// 账号用量端口错误。 +#[derive(Debug, Error)] +#[error("account usage store operation failed: {message}")] +pub struct AccountUsageStoreError { + message: String, +} + +impl From for AccountUsageStoreError { + fn from(error: PgAccountUsageStoreError) -> Self { + Self { + message: error.to_string(), + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AccountUsageDelta { + pub requests: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, + pub empty_responses: u64, + pub image_input_tokens: u64, + pub image_output_tokens: u64, + pub image_requests: u64, + pub image_request_failures: u64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AccountModelUsageDelta { + pub requests: u64, + pub errors: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct AccountUsageSnapshot { + pub request_count: u64, + pub empty_response_count: u64, + pub image_input_tokens: u64, + pub image_output_tokens: u64, + pub image_request_count: u64, + pub image_request_failed_count: u64, + pub window_request_count: u64, + pub window_input_tokens: u64, + pub window_output_tokens: u64, + pub window_cached_tokens: u64, + pub window_image_input_tokens: u64, + pub window_image_output_tokens: u64, + pub window_image_request_count: u64, + pub window_image_request_failed_count: u64, + pub window_started_at: Option>, + pub window_reset_at: Option>, + pub limit_window_seconds: Option, + pub last_used_at: Option>, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AccountUsageWindow { + pub request_count: u64, + pub input_tokens: u64, + pub output_tokens: u64, + pub cached_tokens: u64, + pub image_input_tokens: u64, + pub image_output_tokens: u64, + pub image_request_count: u64, + pub image_request_failed_count: u64, + pub started_at: Option>, + pub reset_at: Option>, + pub limit_window_seconds: Option, +} + +#[async_trait] +pub trait AccountUsageStore: Send + Sync + 'static { + async fn snapshots( + &self, + account_ids: &[String], + ) -> Result, AccountUsageStoreError>; + + async fn record_usage_delta( + &self, + account_id: &str, + usage: AccountUsageDelta, + ) -> Result<(), AccountUsageStoreError>; + + async fn record_model_usage_delta( + &self, + account_id: &str, + model: &str, + usage: AccountModelUsageDelta, + ) -> Result<(), AccountUsageStoreError>; + + async fn sync_runtime_window( + &self, + account_id: &str, + window: AccountUsageWindow, + ) -> Result<(), AccountUsageStoreError>; + + async fn sync_rate_limit_window( + &self, + account_id: &str, + reset_at: DateTime, + limit_window_seconds: Option, + ) -> Result<(), AccountUsageStoreError>; + + async fn record_request(&self, account_id: &str) -> Result<(), AccountUsageStoreError> { + self.record_usage_delta( + account_id, + AccountUsageDelta { + requests: 1, + ..AccountUsageDelta::default() + }, + ) + .await + } +} + /// 账号用量列表记录。 #[derive(Debug, Clone)] pub struct UsageListRecord { @@ -158,7 +379,6 @@ pub struct PgAccountUsageStore { } impl PgAccountUsageStore { - /// 构造存储。 pub fn new(pool: PgPool) -> Self { Self { pool } } @@ -167,6 +387,172 @@ impl PgAccountUsageStore { &self.pool } + async fn load_snapshots( + &self, + account_ids: &[String], + ) -> PgAccountUsageStoreResult> { + if account_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut builder = QueryBuilder::::new( + r"select + account_id, request_count, empty_response_count, + image_input_tokens, image_output_tokens, + image_request_count, image_request_failed_count, + window_request_count, window_input_tokens, window_output_tokens, window_cached_tokens, + window_image_input_tokens, window_image_output_tokens, + window_image_request_count, window_image_request_failed_count, + window_started_at, window_reset_at, limit_window_seconds, last_used_at +from account_usage +where account_id in (", + ); + let mut separated = builder.separated(", "); + for account_id in account_ids { + separated.push_bind(account_id); + } + separated.push_unseparated(")"); + + let rows = builder.build().fetch_all(&self.pool).await?; + Ok(rows + .iter() + .map(|row| (row.get("account_id"), usage_snapshot_from_row(row))) + .collect()) + } + + async fn write_usage_delta( + &self, + account_id: &str, + delta: AccountUsageDelta, + ) -> PgAccountUsageStoreResult<()> { + let request_count = u64_to_i64_saturating(delta.requests); + let input_tokens = u64_to_i64_saturating(delta.input_tokens); + let output_tokens = u64_to_i64_saturating(delta.output_tokens); + let cached_tokens = u64_to_i64_saturating(delta.cached_tokens); + let image_input_tokens = u64_to_i64_saturating(delta.image_input_tokens); + let image_output_tokens = u64_to_i64_saturating(delta.image_output_tokens); + let image_request_count = u64_to_i64_saturating(delta.image_requests); + let image_request_failed_count = u64_to_i64_saturating(delta.image_request_failures); + let last_used_at = Utc::now(); + sqlx::query(RECORD_USAGE_SQL) + .bind(account_id) + .bind(request_count) + .bind(u64_to_i64_saturating(delta.empty_responses)) + .bind(input_tokens) + .bind(output_tokens) + .bind(cached_tokens) + .bind(u64_to_i64_saturating(delta.reasoning_tokens)) + .bind(u64_to_i64_saturating(delta.total_tokens)) + .bind(image_input_tokens) + .bind(image_output_tokens) + .bind(image_request_count) + .bind(image_request_failed_count) + .bind(request_count) + .bind(input_tokens) + .bind(output_tokens) + .bind(cached_tokens) + .bind(image_input_tokens) + .bind(image_output_tokens) + .bind(image_request_count) + .bind(image_request_failed_count) + .bind(last_used_at) + .bind(last_used_at) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn write_model_usage_delta( + &self, + account_id: &str, + model: &str, + delta: AccountModelUsageDelta, + ) -> PgAccountUsageStoreResult<()> { + let model = model.trim(); + if model.is_empty() { + return Ok(()); + } + sqlx::query(RECORD_MODEL_USAGE_SQL) + .bind(account_id) + .bind(model) + .bind(u64_to_i64_saturating(delta.requests)) + .bind(u64_to_i64_saturating(delta.errors)) + .bind(u64_to_i64_saturating(delta.input_tokens)) + .bind(u64_to_i64_saturating(delta.output_tokens)) + .bind(u64_to_i64_saturating(delta.cached_tokens)) + .bind(Utc::now()) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn replace_runtime_window( + &self, + account_id: &str, + window: AccountUsageWindow, + ) -> PgAccountUsageStoreResult<()> { + sqlx::query(SYNC_RUNTIME_WINDOW_SQL) + .bind(account_id) + .bind(u64_to_i64_saturating(window.request_count)) + .bind(u64_to_i64_saturating(window.input_tokens)) + .bind(u64_to_i64_saturating(window.output_tokens)) + .bind(u64_to_i64_saturating(window.cached_tokens)) + .bind(u64_to_i64_saturating(window.image_input_tokens)) + .bind(u64_to_i64_saturating(window.image_output_tokens)) + .bind(u64_to_i64_saturating(window.image_request_count)) + .bind(u64_to_i64_saturating(window.image_request_failed_count)) + .bind(window.started_at) + .bind(window.reset_at) + .bind(window.limit_window_seconds.map(u64_to_i64_saturating)) + .bind(Utc::now()) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn update_rate_limit_window( + &self, + account_id: &str, + reset_at: DateTime, + limit_window_seconds: Option, + ) -> PgAccountUsageStoreResult<()> { + let existing = sqlx::query( + "select window_reset_at, limit_window_seconds from account_usage where account_id = $1", + ) + .bind(account_id) + .fetch_optional(&self.pool) + .await?; + let should_reset = existing + .as_ref() + .map(|row| { + should_reset_usage_window( + row.get("window_reset_at"), + optional_positive_i64_to_u64(row.get("limit_window_seconds")), + reset_at, + limit_window_seconds, + ) + }) + .unwrap_or_default(); + let limit_window_seconds = limit_window_seconds.map(u64_to_i64_saturating); + if should_reset { + sqlx::query(SYNC_RATE_LIMIT_WINDOW_RESET_SQL) + .bind(account_id) + .bind(Utc::now()) + .bind(reset_at) + .bind(limit_window_seconds) + .execute(&self.pool) + .await?; + } else { + sqlx::query(SYNC_RATE_LIMIT_WINDOW_SQL) + .bind(account_id) + .bind(reset_at) + .bind(limit_window_seconds) + .execute(&self.pool) + .await?; + } + Ok(()) + } + /// 分页列出账号用量。 pub async fn list_usage( &self, @@ -239,6 +625,58 @@ impl PgAccountUsageStore { } } +#[async_trait] +impl AccountUsageStore for PgAccountUsageStore { + async fn snapshots( + &self, + account_ids: &[String], + ) -> Result, AccountUsageStoreError> { + self.load_snapshots(account_ids).await.map_err(Into::into) + } + + async fn record_usage_delta( + &self, + account_id: &str, + usage: AccountUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + self.write_usage_delta(account_id, usage) + .await + .map_err(Into::into) + } + + async fn record_model_usage_delta( + &self, + account_id: &str, + model: &str, + usage: AccountModelUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + self.write_model_usage_delta(account_id, model, usage) + .await + .map_err(Into::into) + } + + async fn sync_runtime_window( + &self, + account_id: &str, + window: AccountUsageWindow, + ) -> Result<(), AccountUsageStoreError> { + self.replace_runtime_window(account_id, window) + .await + .map_err(Into::into) + } + + async fn sync_rate_limit_window( + &self, + account_id: &str, + reset_at: DateTime, + limit_window_seconds: Option, + ) -> Result<(), AccountUsageStoreError> { + self.update_rate_limit_window(account_id, reset_at, limit_window_seconds) + .await + .map_err(Into::into) + } +} + fn usage_list_from_row(row: &PgRow) -> PgAccountUsageStoreResult { Ok(UsageListRecord { account_id: row.get("account_id"), @@ -269,6 +707,69 @@ fn usage_list_from_row(row: &PgRow) -> PgAccountUsageStoreResult AccountUsageSnapshot { + AccountUsageSnapshot { + request_count: nonnegative_i64_to_u64(row.get("request_count")), + empty_response_count: nonnegative_i64_to_u64(row.get("empty_response_count")), + image_input_tokens: nonnegative_i64_to_u64(row.get("image_input_tokens")), + image_output_tokens: nonnegative_i64_to_u64(row.get("image_output_tokens")), + image_request_count: nonnegative_i64_to_u64(row.get("image_request_count")), + image_request_failed_count: nonnegative_i64_to_u64(row.get("image_request_failed_count")), + window_request_count: nonnegative_i64_to_u64(row.get("window_request_count")), + window_input_tokens: nonnegative_i64_to_u64(row.get("window_input_tokens")), + window_output_tokens: nonnegative_i64_to_u64(row.get("window_output_tokens")), + window_cached_tokens: nonnegative_i64_to_u64(row.get("window_cached_tokens")), + window_image_input_tokens: nonnegative_i64_to_u64(row.get("window_image_input_tokens")), + window_image_output_tokens: nonnegative_i64_to_u64(row.get("window_image_output_tokens")), + window_image_request_count: nonnegative_i64_to_u64(row.get("window_image_request_count")), + window_image_request_failed_count: nonnegative_i64_to_u64( + row.get("window_image_request_failed_count"), + ), + window_started_at: row.get("window_started_at"), + window_reset_at: row.get("window_reset_at"), + limit_window_seconds: optional_positive_i64_to_u64(row.get("limit_window_seconds")), + last_used_at: row.get("last_used_at"), + } +} + +pub fn should_reset_usage_window( + existing_reset_at: Option>, + existing_limit_window_seconds: Option, + new_reset_at: DateTime, + new_limit_window_seconds: Option, +) -> bool { + let Some(existing_reset_at) = existing_reset_at else { + return false; + }; + if existing_reset_at == new_reset_at { + return false; + } + + let drift = existing_reset_at + .signed_duration_since(new_reset_at) + .num_seconds() + .unsigned_abs(); + let window_seconds = new_limit_window_seconds + .or(existing_limit_window_seconds) + .unwrap_or(0); + let threshold = if window_seconds > 0 { + window_seconds / 2 + } else { + 3_600 + }; + drift >= threshold +} + +fn u64_to_i64_saturating(value: u64) -> i64 { + value.min(i64::MAX as u64) as i64 +} + +fn optional_positive_i64_to_u64(value: Option) -> Option { + value + .and_then(|value| u64::try_from(value).ok()) + .filter(|value| *value > 0) +} + fn to_page(rows: &[PgRow], limit: u32) -> Page { let has_more = rows.len() > limit as usize; let mut items = Vec::with_capacity(limit as usize); diff --git a/backend/src/telemetry/dashboard.rs b/backend/src/telemetry/dashboard.rs new file mode 100644 index 000000000..7e814b4b4 --- /dev/null +++ b/backend/src/telemetry/dashboard.rs @@ -0,0 +1,539 @@ +//! Dashboard 遥测聚合与展示模型。 + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::infra::{ + format::{ + format_compact_number, format_cost, format_duration_ms, format_percent, format_rate, + format_tokens, nonnegative_i64_to_u64, + }, + time::{china_day_start, china_hour, china_hour_start, china_quarter_hour_start}, +}; + +use super::account_usage::query::{AccountUsageSummary, AccountUsageTimeBucket}; + +const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15; +const HEALTH_TIMELINE_SLOTS: i64 = 24 * 4; +const HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD: u64 = 3; + +#[derive(Debug, Clone, Copy, Default)] +pub struct DashboardAccountCounts { + pub total: u64, + pub enabled: u64, + pub abnormal: u64, +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DashboardTrendKind { + #[default] + Usage, + Latency, + Errors, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardHealthTimelineData { + title: String, + description: String, + reliability_display: String, + points: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardCardsData { + accounts: DashboardAccountsCardData, + traffic: DashboardTrafficCardData, + tokens: DashboardTokenCardData, + cache: DashboardCacheCardData, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardAccountsCardData { + total: String, + total_value: u64, + enabled: String, + enabled_value: u64, + abnormal: String, + abnormal_value: u64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardTrafficCardData { + today_requests: String, + today_requests_value: u64, + yesterday_requests_value: u64, + total_requests: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardTokenCardData { + today_tokens: String, + today_tokens_value: u64, + yesterday_tokens_value: u64, + total_tokens: String, + today_cost_usd: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardCacheCardData { + today_hit_rate: String, + today_hit_rate_value: Option, + yesterday_hit_rate_value: Option, + total_hit_rate: String, + total_cached_tokens: String, + average_first_token_latency_ms: String, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DashboardTrendData { + kind: DashboardTrendKind, + points: Vec, + summary: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardTrendPointData { + time: String, + requests: String, + requests_value: u64, + input_tokens: String, + input_tokens_value: u64, + output_tokens: String, + output_tokens_value: u64, + cached_tokens: String, + cached_tokens_value: u64, + cache_hit_rate_value: f64, + tokens_value: u64, + errors: String, + errors_value: u64, + latency: String, + latency_value: u64, + max_latency: String, + max_latency_value: u64, + min_latency: String, + min_latency_value: u64, + success_rate: String, + success_rate_value: f64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DashboardTrendSummaryData { + label: String, + value: String, + ratio: Option, +} + +#[derive(Debug, Clone, Copy, Default)] +struct UsageWindow { + requests: u64, + input_tokens: u64, + output_tokens: u64, + cached_tokens: u64, + errors: u64, + first_token_latency_sum: u64, + first_token_latency_count: u64, + max_first_token_bucket_latency: u64, + min_first_token_bucket_latency: u64, +} + +impl UsageWindow { + fn tokens(self) -> u64 { + self.input_tokens + self.output_tokens + } + + fn cache_hit_rate(self) -> Option { + (self.input_tokens > 0).then(|| self.cached_tokens as f64 / self.input_tokens as f64) + } + + fn avg_first_token_latency(self) -> Option { + self.first_token_latency_sum + .checked_div(self.first_token_latency_count) + } + + fn max_first_token_bucket_latency(self) -> Option { + (self.max_first_token_bucket_latency > 0).then_some(self.max_first_token_bucket_latency) + } + + fn min_first_token_bucket_latency(self) -> Option { + (self.min_first_token_bucket_latency > 0).then_some(self.min_first_token_bucket_latency) + } +} + +pub fn dashboard_cards( + accounts: DashboardAccountCounts, + buckets: &[AccountUsageTimeBucket], + lifetime_usage: &AccountUsageSummary, +) -> DashboardCardsData { + dashboard_cards_at(accounts, buckets, lifetime_usage, Utc::now()) +} + +pub fn dashboard_cards_at( + accounts: DashboardAccountCounts, + buckets: &[AccountUsageTimeBucket], + lifetime_usage: &AccountUsageSummary, + now: DateTime, +) -> DashboardCardsData { + let today_start = china_day_start(now); + let yesterday_start = today_start - Duration::days(1); + let today = usage_window(buckets, today_start, now); + let yesterday = usage_window(buckets, yesterday_start, today_start); + let today_cost = cost_window(buckets, today_start, now).unwrap_or(0.0); + let total_requests = nonnegative_i64_to_u64(lifetime_usage.request_count); + let total_tokens = nonnegative_i64_to_u64(lifetime_usage.total_tokens); + let total_cached_tokens = nonnegative_i64_to_u64(lifetime_usage.cached_tokens); + + DashboardCardsData { + accounts: DashboardAccountsCardData { + total: format_compact_number(accounts.total), + total_value: accounts.total, + enabled: format_compact_number(accounts.enabled), + enabled_value: accounts.enabled, + abnormal: format_compact_number(accounts.abnormal), + abnormal_value: accounts.abnormal, + }, + traffic: DashboardTrafficCardData { + today_requests: format_compact_number(today.requests), + today_requests_value: today.requests, + yesterday_requests_value: yesterday.requests, + total_requests: format_compact_number(total_requests), + }, + tokens: DashboardTokenCardData { + today_tokens: format_tokens(today.tokens()), + today_tokens_value: today.tokens(), + yesterday_tokens_value: yesterday.tokens(), + total_tokens: format_tokens(total_tokens), + today_cost_usd: format_cost(today_cost), + }, + cache: DashboardCacheCardData { + today_hit_rate: format_rate(today.cache_hit_rate()), + today_hit_rate_value: today.cache_hit_rate(), + yesterday_hit_rate_value: yesterday.cache_hit_rate(), + total_hit_rate: format_rate(summary_cache_hit_rate(lifetime_usage)), + total_cached_tokens: format_tokens(total_cached_tokens), + average_first_token_latency_ms: format_optional_duration_ms( + today.avg_first_token_latency(), + ), + }, + } +} + +pub fn dashboard_trend_data( + buckets: &[AccountUsageTimeBucket], + kind: DashboardTrendKind, +) -> DashboardTrendData { + dashboard_trend_data_at(buckets, kind, Utc::now()) +} + +pub fn dashboard_trend_data_at( + records: &[AccountUsageTimeBucket], + kind: DashboardTrendKind, + now: DateTime, +) -> DashboardTrendData { + let current_hour = china_hour_start(now); + let start = china_day_start(now); + let elapsed_hours = current_hour.signed_duration_since(start).num_hours(); + let mut buckets = (0..=elapsed_hours) + .map(|index| (start + Duration::hours(index), UsageWindow::default())) + .collect::>(); + + for record in records { + if record.bucket_start < start || record.bucket_start > now { + continue; + } + let record_hour = china_hour_start(record.bucket_start); + if let Some((_, bucket)) = buckets + .iter_mut() + .find(|(bucket_start, _)| *bucket_start == record_hour) + { + apply_bucket(bucket, record); + } + } + + let points = buckets + .iter() + .map(|(bucket_start, bucket)| { + let latency = bucket.avg_first_token_latency().unwrap_or(0); + let max_latency = bucket.max_first_token_bucket_latency().unwrap_or(0); + let min_latency = bucket.min_first_token_bucket_latency().unwrap_or(0); + let cache_hit_rate = bucket.cache_hit_rate().unwrap_or(0.0); + let success_rate = if bucket.requests > 0 { + ((bucket.requests - bucket.errors) as f64 / bucket.requests as f64 * 1000.0).round() + / 10.0 + } else { + 0.0 + }; + DashboardTrendPointData { + time: format!("{:02}", china_hour(bucket_start)), + requests: format_compact_number(bucket.requests), + requests_value: bucket.requests, + input_tokens: format_tokens(bucket.input_tokens), + input_tokens_value: bucket.input_tokens, + output_tokens: format_tokens(bucket.output_tokens), + output_tokens_value: bucket.output_tokens, + cached_tokens: format_tokens(bucket.cached_tokens), + cached_tokens_value: bucket.cached_tokens, + cache_hit_rate_value: cache_hit_rate, + tokens_value: bucket.tokens(), + errors: format_compact_number(bucket.errors), + errors_value: bucket.errors, + latency: format_optional_duration_ms(Some(latency)), + latency_value: latency, + max_latency: format_optional_duration_ms(Some(max_latency)), + max_latency_value: max_latency, + min_latency: format_optional_duration_ms(Some(min_latency)), + min_latency_value: min_latency, + success_rate: format_percent(success_rate), + success_rate_value: success_rate, + } + }) + .collect::>(); + + DashboardTrendData { + kind, + summary: trend_summary(kind, &points), + points, + } +} + +pub fn dashboard_health_timeline_data( + buckets: &[AccountUsageTimeBucket], +) -> DashboardHealthTimelineData { + dashboard_health_timeline_data_at(buckets, Utc::now()) +} + +pub fn dashboard_health_timeline_data_at( + records: &[AccountUsageTimeBucket], + now: DateTime, +) -> DashboardHealthTimelineData { + let current_slot = china_quarter_hour_start(now); + let start = china_day_start(now); + let mut buckets = (0..HEALTH_TIMELINE_SLOTS) + .map(|index| { + ( + start + Duration::minutes(HEALTH_TIMELINE_SLOT_MINUTES * index), + UsageWindow::default(), + ) + }) + .collect::>(); + for record in records { + if record.bucket_start < start || record.bucket_start > now { + continue; + } + let record_slot = china_quarter_hour_start(record.bucket_start); + if let Some((_, bucket)) = buckets + .iter_mut() + .find(|(bucket_start, _)| *bucket_start == record_slot) + { + apply_bucket(bucket, record); + } + } + + let (requests, errors) = buckets + .iter() + .filter(|(bucket_start, _)| *bucket_start <= current_slot) + .fold((0, 0), |(requests, errors), (_, bucket)| { + (requests + bucket.requests, errors + bucket.errors) + }); + let reliability = (requests > 0) + .then(|| ((requests - errors) as f64 / requests as f64 * 1000.0).round() / 10.0); + DashboardHealthTimelineData { + title: "请求健康时间线".to_string(), + description: "请求可靠性".to_string(), + reliability_display: reliability + .map(|value| format!("{value:.1}%")) + .unwrap_or_else(|| "-".to_string()), + points: buckets + .into_iter() + .map(|(bucket_start, bucket)| { + let success_rate = if bucket.requests > 0 { + (bucket.requests.saturating_sub(bucket.errors) as f64 / bucket.requests as f64 + * 1000.0) + .round() + / 10.0 + } else { + 0.0 + }; + health_tone(bucket, success_rate, bucket_start > current_slot) + }) + .collect(), + } +} + +fn trend_summary( + kind: DashboardTrendKind, + points: &[DashboardTrendPointData], +) -> Vec { + match kind { + DashboardTrendKind::Usage => vec![ + trend_summary_item( + kind, + "输入", + points.iter().map(|p| p.input_tokens_value).sum(), + None, + ), + trend_summary_item( + kind, + "输出", + points.iter().map(|p| p.output_tokens_value).sum(), + None, + ), + trend_summary_item( + kind, + "缓存", + points.iter().map(|p| p.cached_tokens_value).sum(), + None, + ), + ], + DashboardTrendKind::Latency => { + let samples = points + .iter() + .filter(|point| point.latency_value > 0) + .collect::>(); + let average = if samples.is_empty() { + 0 + } else { + samples.iter().map(|point| point.latency_value).sum::() / samples.len() as u64 + }; + vec![ + trend_summary_item(kind, "平均", average, None), + trend_summary_item( + kind, + "最高", + samples + .iter() + .map(|p| p.max_latency_value) + .max() + .unwrap_or(0), + None, + ), + trend_summary_item( + kind, + "最低", + samples + .iter() + .filter_map(|p| (p.min_latency_value > 0).then_some(p.min_latency_value)) + .min() + .unwrap_or(0), + None, + ), + ] + } + DashboardTrendKind::Errors => { + let errors = points.iter().map(|point| point.errors_value).sum::(); + let requests = points.iter().map(|point| point.requests_value).sum::(); + let success_rate = (requests > 0) + .then(|| ((requests - errors) as f64 / requests as f64 * 1000.0).round() / 10.0); + vec![ + trend_summary_item(kind, "错误数", errors, None), + trend_summary_item(kind, "成功率", 0, success_rate), + trend_summary_item(kind, "总请求", requests, None), + ] + } + } +} + +fn trend_summary_item( + kind: DashboardTrendKind, + label: &str, + value: u64, + ratio: Option, +) -> DashboardTrendSummaryData { + DashboardTrendSummaryData { + label: label.to_string(), + value: match kind { + DashboardTrendKind::Usage => format_tokens(value), + DashboardTrendKind::Latency => format_optional_duration_ms(Some(value)), + DashboardTrendKind::Errors => format_compact_number(value), + }, + ratio: ratio.map(format_percent), + } +} + +fn usage_window( + records: &[AccountUsageTimeBucket], + start: DateTime, + end: DateTime, +) -> UsageWindow { + records + .iter() + .filter(|record| record.bucket_start >= start && record.bucket_start < end) + .fold(UsageWindow::default(), |mut window, record| { + apply_bucket(&mut window, record); + window + }) +} + +fn cost_window( + records: &[AccountUsageTimeBucket], + start: DateTime, + end: DateTime, +) -> Option { + let costs = records + .iter() + .filter(|record| record.bucket_start >= start && record.bucket_start < end) + .filter_map(|record| record.cost_usd) + .collect::>(); + (!costs.is_empty()).then(|| costs.into_iter().sum()) +} + +fn apply_bucket(window: &mut UsageWindow, record: &AccountUsageTimeBucket) { + window.requests += nonnegative_i64_to_u64(record.request_count); + window.input_tokens += nonnegative_i64_to_u64(record.input_tokens); + window.output_tokens += nonnegative_i64_to_u64(record.output_tokens); + window.cached_tokens += nonnegative_i64_to_u64(record.cached_tokens); + window.errors += nonnegative_i64_to_u64(record.error_count); + window.first_token_latency_sum += nonnegative_i64_to_u64(record.first_token_latency_sum); + window.first_token_latency_count += nonnegative_i64_to_u64(record.first_token_latency_count); + if let Some(latency) = first_token_bucket_latency(record) { + window.max_first_token_bucket_latency = window.max_first_token_bucket_latency.max(latency); + window.min_first_token_bucket_latency = if window.min_first_token_bucket_latency == 0 { + latency + } else { + window.min_first_token_bucket_latency.min(latency) + }; + } +} + +fn first_token_bucket_latency(record: &AccountUsageTimeBucket) -> Option { + nonnegative_i64_to_u64(record.first_token_latency_sum) + .checked_div(nonnegative_i64_to_u64(record.first_token_latency_count)) + .filter(|latency| *latency > 0) +} + +fn health_tone(bucket: UsageWindow, success_rate: f64, is_future: bool) -> char { + let successes = bucket.requests.saturating_sub(bucket.errors); + if is_future { + '0' + } else if bucket.requests == 0 { + '1' + } else if successes == 0 { + '2' + } else if success_rate < 90.0 { + '3' + } else if successes < HEALTH_TIMELINE_STABLE_SUCCESS_THRESHOLD { + '4' + } else { + '5' + } +} + +fn summary_cache_hit_rate(summary: &AccountUsageSummary) -> Option { + let input_tokens = nonnegative_i64_to_u64(summary.input_tokens); + (input_tokens > 0) + .then(|| nonnegative_i64_to_u64(summary.cached_tokens) as f64 / input_tokens as f64) +} + +fn format_optional_duration_ms(value: Option) -> String { + format_duration_ms(value.and_then(|value| i64::try_from(value).ok())) +} diff --git a/backend/src/telemetry/diagnostics.rs b/backend/src/telemetry/diagnostics.rs deleted file mode 100644 index f3b6a413f..000000000 --- a/backend/src/telemetry/diagnostics.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! 诊断数据类型与聚合函数。 - -use serde::Serialize; - -use crate::{accounts::pool::AccountCapacitySummary, upstream::openai::fingerprint::Fingerprint}; - -/// 账号池摘要。 -#[derive(Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AccountPoolDiagnostics { - /// 总数。 - pub total: usize, - /// 活跃数。 - pub active: usize, - /// 过期数。 - pub expired: usize, - /// 配额耗尽数。 - pub quota_exhausted: usize, - /// 刷新中数。 - pub refreshing: usize, - /// 禁用数。 - pub disabled: usize, - /// 封禁数。 - pub banned: usize, -} - -/// Account-pool capacity diagnostics. -#[derive(Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AccountCapacityDiagnostics { - /// Maximum concurrent slots per account. - pub max_concurrent_per_account: usize, - /// Total slots available across active accounts. - pub total_slots: usize, - /// Currently occupied slots. - pub used_slots: usize, - /// Currently available slots. - pub available_slots: usize, -} - -/// Runtime fingerprint summary. -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct FingerprintDiagnostics { - /// Fingerprint source label. - pub source: &'static str, - /// Client originator. - pub originator: String, - /// App version. - pub app_version: String, - /// Build number. - pub build_number: String, - /// Platform name. - pub platform: String, - /// Architecture name. - pub arch: String, - /// Chromium major version. - pub chromium_version: String, - /// Expanded user-agent. - pub user_agent: String, - /// 指纹最后更新时间。 - #[serde(skip_serializing_if = "Option::is_none")] - pub updated_at: Option, -} - -/// 构造 fingerprint 诊断数据。 -pub fn fingerprint_diagnostics(fingerprint: &Fingerprint) -> FingerprintDiagnostics { - FingerprintDiagnostics { - source: "runtime", - originator: fingerprint.originator.clone(), - app_version: fingerprint.app_version.clone(), - build_number: fingerprint.build_number.clone(), - platform: fingerprint.platform.clone(), - arch: fingerprint.arch.clone(), - chromium_version: fingerprint.chromium_version.clone(), - user_agent: fingerprint.user_agent(), - updated_at: fingerprint.updated_at.clone(), - } -} - -impl From for AccountCapacityDiagnostics { - fn from(summary: AccountCapacitySummary) -> Self { - Self { - max_concurrent_per_account: summary.max_concurrent_per_account, - total_slots: summary.total_slots, - used_slots: summary.used_slots, - available_slots: summary.available_slots, - } - } -} diff --git a/backend/src/telemetry/mod.rs b/backend/src/telemetry/mod.rs index 2d25be1de..cdda500a0 100644 --- a/backend/src/telemetry/mod.rs +++ b/backend/src/telemetry/mod.rs @@ -3,7 +3,7 @@ pub mod account_usage; pub(crate) mod billing; pub mod buckets; -pub mod diagnostics; +pub mod dashboard; pub mod ops; pub mod rebuild; pub mod recorder; diff --git a/backend/src/upstream/openai/protocol/mod.rs b/backend/src/upstream/openai/protocol/mod.rs index b8dacd0f2..57085bee1 100644 --- a/backend/src/upstream/openai/protocol/mod.rs +++ b/backend/src/upstream/openai/protocol/mod.rs @@ -2,8 +2,6 @@ /// Codex 事件语义与用量提取。 pub mod events; -/// 上游模型目录响应类型。 -pub mod model_catalog; /// Codex Responses / Compact 请求体。 pub mod responses; /// Codex JSON schema 处理。 diff --git a/backend/src/upstream/openai/protocol/model_catalog.rs b/backend/src/upstream/openai/protocol/model_catalog.rs deleted file mode 100644 index 6f6f24536..000000000 --- a/backend/src/upstream/openai/protocol/model_catalog.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! 上游模型目录原始条目。 - -use serde::{Deserialize, Serialize}; - -/// 上游声明的 reasoning effort 条目。 -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct BackendReasoningEffort { - /// reasoning_effort 字段。 - #[serde(default, rename = "reasoning_effort", alias = "reasoningEffort")] - pub reasoning_effort: Option, - /// 上游可能返回的 effort 字段别名。 - #[serde(default)] - pub effort: Option, - /// 展示描述。 - #[serde(default)] - pub description: Option, -} - -/// 上游截断策略定义。 -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct BackendTruncationPolicy { - /// 上下文截断限制。 - pub limit: Option, -} - -/// 上游 `/codex/models` 返回的原始模型条目。 -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct BackendModelEntry { - /// slug。 - pub slug: Option, - /// id。 - pub id: Option, - /// name。 - pub name: Option, - /// display_name。 - pub display_name: Option, - /// title。 - pub title: Option, - /// description。 - pub description: Option, - /// 是否默认模型。 - pub is_default: Option, - /// 默认 reasoning effort。 - pub default_reasoning_effort: Option, - /// 默认 reasoning level。 - pub default_reasoning_level: Option, - /// 支持的 reasoning effort 列表。 - #[serde(default)] - pub supported_reasoning_efforts: Vec, - /// 支持的 reasoning level 列表。 - #[serde(default)] - pub supported_reasoning_levels: Vec, - /// 输入模态。 - pub input_modalities: Option>, - /// 输出模态。 - pub output_modalities: Option>, - /// 是否支持 personality。 - pub supports_personality: Option, - /// 升级提示。 - pub upgrade: Option, - /// contextWindow 别名。 - #[serde(alias = "contextWindow")] - pub context_window: Option, - /// maxContextWindow 别名。 - #[serde(alias = "maxContextWindow")] - pub max_context_window: Option, - /// maxOutputTokens 别名。 - #[serde(alias = "maxOutputTokens")] - pub max_output_tokens: Option, - /// truncationPolicy 别名。 - #[serde(alias = "truncationPolicy")] - pub truncation_policy: Option, -} diff --git a/backend/src/upstream/openai/transport/client.rs b/backend/src/upstream/openai/transport/client.rs index 8011c2c37..cf6ea595e 100644 --- a/backend/src/upstream/openai/transport/client.rs +++ b/backend/src/upstream/openai/transport/client.rs @@ -24,7 +24,6 @@ use uuid::Uuid; use crate::upstream::openai::fingerprint::{Fingerprint, RuntimeFingerprint}; use crate::upstream::openai::protocol::events::{extract_sse_usage, retry_after_seconds_from_body}; -use crate::upstream::openai::protocol::model_catalog::BackendModelEntry; use crate::upstream::openai::protocol::responses::{ http_sse_fallback_allowed, transport_for_request, CodexCompactRequest, CodexResponsesRequest, CodexTransport, @@ -201,7 +200,7 @@ pub trait CodexModelCatalogClient: Send + Sync + 'static { async fn fetch_models( &self, request: &CodexModelCatalogRequest<'_>, - ) -> Result, CodexModelCatalogClientError>; + ) -> Result, CodexModelCatalogClientError>; } // --------------------------------------------------------------------------- @@ -345,7 +344,7 @@ impl CodexModelCatalogClient for CodexBackendClient { async fn fetch_models( &self, request: &CodexModelCatalogRequest<'_>, - ) -> Result, CodexModelCatalogClientError> { + ) -> Result, CodexModelCatalogClientError> { self.fetch_models_with_context(CodexRequestContext { access_token: request.access_token, account_id: request.account_id, @@ -446,7 +445,7 @@ pub(super) async fn read_capped_error_body( // Model entry extraction // --------------------------------------------------------------------------- -fn extract_backend_model_entries(value: &Value) -> Vec { +fn extract_model_entries(value: &Value) -> Vec { let Some(models) = value .pointer("/chat_models/models") .or_else(|| value.get("models")) @@ -460,22 +459,23 @@ fn extract_backend_model_entries(value: &Value) -> Vec { let mut entries = Vec::new(); for model in models { if let Some(nested) = model.get("models").and_then(Value::as_array) { - entries.extend(nested.iter().filter_map(parse_backend_model_entry)); - } else if let Some(entry) = parse_backend_model_entry(model) { - entries.push(entry); + entries.extend(nested.iter().filter(|entry| is_model_entry(entry)).cloned()); + } else if is_model_entry(model) { + entries.push(model.clone()); } } entries } -fn parse_backend_model_entry(value: &Value) -> Option { - let entry = serde_json::from_value::(value.clone()).ok()?; - (entry.slug.is_some() - || entry.id.is_some() - || entry.name.is_some() - || entry.display_name.is_some() - || entry.title.is_some()) - .then_some(entry) +fn is_model_entry(value: &Value) -> bool { + ["slug", "id", "name", "display_name", "title"] + .into_iter() + .any(|key| { + value + .get(key) + .and_then(Value::as_str) + .is_some_and(|value| !value.trim().is_empty()) + }) } // --------------------------------------------------------------------------- diff --git a/backend/src/upstream/openai/transport/client/requests.rs b/backend/src/upstream/openai/transport/client/requests.rs index 2188d815c..12e5a0154 100644 --- a/backend/src/upstream/openai/transport/client/requests.rs +++ b/backend/src/upstream/openai/transport/client/requests.rs @@ -463,7 +463,7 @@ impl CodexBackendClient { pub(super) async fn fetch_models_with_context( &self, context: CodexRequestContext<'_>, - ) -> CodexClientResult> { + ) -> CodexClientResult> { let fingerprint = self.fingerprint.current(); let endpoints = [ format!( @@ -481,7 +481,7 @@ impl CodexBackendClient { continue; } let parsed = response.json::().await?; - let models = extract_backend_model_entries(&parsed); + let models = extract_model_entries(&parsed); if !models.is_empty() { return Ok(models); } diff --git a/backend/tests/accounts/cloudflare.rs b/backend/tests/accounts/cookies/cooldown.rs similarity index 100% rename from backend/tests/accounts/cloudflare.rs rename to backend/tests/accounts/cookies/cooldown.rs diff --git a/backend/tests/accounts/cookies/mod.rs b/backend/tests/accounts/cookies/mod.rs new file mode 100644 index 000000000..65650a1e8 --- /dev/null +++ b/backend/tests/accounts/cookies/mod.rs @@ -0,0 +1,2 @@ +mod cooldown; +mod store; diff --git a/backend/tests/accounts/cookies.rs b/backend/tests/accounts/cookies/store.rs similarity index 100% rename from backend/tests/accounts/cookies.rs rename to backend/tests/accounts/cookies/store.rs diff --git a/backend/tests/accounts/mod.rs b/backend/tests/accounts/mod.rs index 2fd305afb..a4fac2f80 100644 --- a/backend/tests/accounts/mod.rs +++ b/backend/tests/accounts/mod.rs @@ -1,8 +1,6 @@ -pub(crate) mod account_pool; -pub(crate) mod account_store; -pub(crate) mod cloudflare; pub(crate) mod cookies; -pub(crate) mod quota_refresh; -pub(crate) mod refresh_leases; +pub(crate) mod pool; +pub(crate) mod quota; +pub(crate) mod refresh; pub(crate) mod scheduler; -pub(crate) mod token_refresh; +pub(crate) mod store; diff --git a/backend/tests/accounts/account_pool/mod.rs b/backend/tests/accounts/pool/mod.rs similarity index 98% rename from backend/tests/accounts/account_pool/mod.rs rename to backend/tests/accounts/pool/mod.rs index a71e17124..68d6bc546 100644 --- a/backend/tests/accounts/account_pool/mod.rs +++ b/backend/tests/accounts/pool/mod.rs @@ -11,6 +11,7 @@ use codex_proxy_rs::accounts::{ use serde_json::json; mod quota; +mod runtime; mod selection; mod usage_window; diff --git a/backend/tests/accounts/account_pool/quota.rs b/backend/tests/accounts/pool/quota.rs similarity index 100% rename from backend/tests/accounts/account_pool/quota.rs rename to backend/tests/accounts/pool/quota.rs diff --git a/backend/tests/bootstrap/account_pool_restore/mod.rs b/backend/tests/accounts/pool/runtime.rs similarity index 92% rename from backend/tests/bootstrap/account_pool_restore/mod.rs rename to backend/tests/accounts/pool/runtime.rs index 8dd0c8107..e58364a8b 100644 --- a/backend/tests/bootstrap/account_pool_restore/mod.rs +++ b/backend/tests/accounts/pool/runtime.rs @@ -7,6 +7,7 @@ use codex_proxy_rs::accounts::{ }, store::{AccountStore, PgAccountStore}, }; +use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore; use crate::support::storage::init_test_db; @@ -15,9 +16,10 @@ async fn runtime_account_pool_should_restore_capacity_and_clear_runtime_state() let (pool, _dir) = init_test_db("runtime-account-pool").await; insert_account(&pool, "acct_pool").await; - let store = PgAccountStore::new(pool); + let store = PgAccountStore::new(pool.clone()); let service = RuntimeAccountPoolService::new( Arc::new(store) as Arc, + Arc::new(PgAccountUsageStore::new(pool)), AccountPoolOptions { max_concurrent_per_account: 2, ..AccountPoolOptions::default() @@ -58,9 +60,10 @@ async fn runtime_account_pool_should_restore_accounts_by_added_at_and_id() { insert_account(&pool, "acct_c").await; insert_account(&pool, "acct_a").await; - let store = PgAccountStore::new(pool); + let store = PgAccountStore::new(pool.clone()); let service = RuntimeAccountPoolService::new( Arc::new(store) as Arc, + Arc::new(PgAccountUsageStore::new(pool)), AccountPoolOptions { rotation_strategy: RotationStrategy::RoundRobin, ..AccountPoolOptions::default() diff --git a/backend/tests/accounts/account_pool/selection.rs b/backend/tests/accounts/pool/selection.rs similarity index 99% rename from backend/tests/accounts/account_pool/selection.rs rename to backend/tests/accounts/pool/selection.rs index 83200af6b..8a7695e8d 100644 --- a/backend/tests/accounts/account_pool/selection.rs +++ b/backend/tests/accounts/pool/selection.rs @@ -191,6 +191,9 @@ async fn runtime_account_pool_should_persist_expired_status_when_jwt_expiry_is_d .unwrap(); let runtime_pool = codex_proxy_rs::accounts::pool::RuntimeAccountPoolService::new( std::sync::Arc::new(store), + std::sync::Arc::new( + codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore::new(pool.clone()), + ), AccountPoolOptions::default(), 0, ); @@ -271,6 +274,9 @@ async fn runtime_account_pool_should_persist_quota_window_reset_when_cooldown_ex .unwrap(); let runtime_pool = codex_proxy_rs::accounts::pool::RuntimeAccountPoolService::new( std::sync::Arc::new(store), + std::sync::Arc::new( + codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore::new(pool.clone()), + ), AccountPoolOptions::default(), 0, ); diff --git a/backend/tests/accounts/account_pool/usage_window.rs b/backend/tests/accounts/pool/usage_window.rs similarity index 100% rename from backend/tests/accounts/account_pool/usage_window.rs rename to backend/tests/accounts/pool/usage_window.rs diff --git a/backend/tests/accounts/quota_refresh.rs b/backend/tests/accounts/quota.rs similarity index 94% rename from backend/tests/accounts/quota_refresh.rs rename to backend/tests/accounts/quota.rs index eb23f1fbf..3cd422fb7 100644 --- a/backend/tests/accounts/quota_refresh.rs +++ b/backend/tests/accounts/quota.rs @@ -8,6 +8,7 @@ use codex_proxy_rs::accounts::quota::{ quota_snapshot_reset_at, RuntimeQuotaRefreshService, }; use codex_proxy_rs::accounts::store::{NewAccount, PgAccountStore}; +use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore; use codex_proxy_rs::upstream::openai::transport::CodexBackendClient; use secrecy::SecretString; use serde_json::{json, Value}; @@ -311,8 +312,12 @@ async fn quota_refresh_service_should_send_usage_cookie_when_cookie_store_is_con server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = - RuntimeQuotaRefreshService::new(store, Arc::new(codex)).with_cookie_store(cookies); + let service = RuntimeQuotaRefreshService::new( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + ) + .with_cookie_store(cookies); let mut last_refreshed = HashMap::new(); let summary = service @@ -388,7 +393,11 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_ server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = RuntimeQuotaRefreshService::new(store.clone(), Arc::new(codex)); + let service = RuntimeQuotaRefreshService::new( + store.clone(), + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + ); let mut last_refreshed = HashMap::new(); let summary = service @@ -459,7 +468,11 @@ async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() { server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = RuntimeQuotaRefreshService::new(store, Arc::new(codex)); + let service = RuntimeQuotaRefreshService::new( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + ); let mut last_refreshed = HashMap::new(); let summary = service @@ -490,8 +503,12 @@ async fn quota_refresh_service_should_skip_recent_locked_account_before_cooldown server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = - RuntimeQuotaRefreshService::with_min_refresh_interval_secs(store, Arc::new(codex), 1_800); + let service = RuntimeQuotaRefreshService::with_min_refresh_interval_secs( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + 1_800, + ); let mut last_refreshed = HashMap::new(); last_refreshed.insert("acct-quota".to_string(), tokio::time::Instant::now()); @@ -525,8 +542,12 @@ async fn quota_refresh_service_should_skip_recent_locked_account_inside_cooldown server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = - RuntimeQuotaRefreshService::with_min_refresh_interval_secs(store, Arc::new(codex), 1_800); + let service = RuntimeQuotaRefreshService::with_min_refresh_interval_secs( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + 1_800, + ); let mut last_refreshed = HashMap::new(); last_refreshed.insert("acct-quota".to_string(), tokio::time::Instant::now()); @@ -577,8 +598,12 @@ async fn quota_refresh_service_should_bypass_recent_skip_after_cooldown_grace() server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = - RuntimeQuotaRefreshService::with_min_refresh_interval_secs(store, Arc::new(codex), 1_800); + let service = RuntimeQuotaRefreshService::with_min_refresh_interval_secs( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + 1_800, + ); let mut last_refreshed = HashMap::new(); last_refreshed.insert("acct-quota".to_string(), tokio::time::Instant::now()); @@ -627,8 +652,12 @@ async fn quota_refresh_service_should_stagger_multiple_locked_account_requests() server.uri(), crate::support::fingerprint::runtime_test_fingerprint(), ); - let service = RuntimeQuotaRefreshService::new(store, Arc::new(codex)) - .with_request_spacing(StdDuration::from_millis(200)); + let service = RuntimeQuotaRefreshService::new( + store, + PgAccountUsageStore::new(pool.clone()), + Arc::new(codex), + ) + .with_request_spacing(StdDuration::from_millis(200)); let refresh = tokio::spawn(async move { let mut last_refreshed = HashMap::new(); diff --git a/backend/tests/accounts/token_refresh/failures.rs b/backend/tests/accounts/refresh/failures.rs similarity index 100% rename from backend/tests/accounts/token_refresh/failures.rs rename to backend/tests/accounts/refresh/failures.rs diff --git a/backend/tests/accounts/refresh_leases.rs b/backend/tests/accounts/refresh/lease.rs similarity index 100% rename from backend/tests/accounts/refresh_leases.rs rename to backend/tests/accounts/refresh/lease.rs diff --git a/backend/tests/accounts/token_refresh/mod.rs b/backend/tests/accounts/refresh/mod.rs similarity index 99% rename from backend/tests/accounts/token_refresh/mod.rs rename to backend/tests/accounts/refresh/mod.rs index 54adc7652..2084b9887 100644 --- a/backend/tests/accounts/token_refresh/mod.rs +++ b/backend/tests/accounts/refresh/mod.rs @@ -24,6 +24,7 @@ use tokio::{ use crate::support::storage::{create_test_redis, init_test_db}; mod failures; +mod lease; mod scheduling; mod success; diff --git a/backend/tests/accounts/token_refresh/scheduling.rs b/backend/tests/accounts/refresh/scheduling.rs similarity index 100% rename from backend/tests/accounts/token_refresh/scheduling.rs rename to backend/tests/accounts/refresh/scheduling.rs diff --git a/backend/tests/accounts/token_refresh/success.rs b/backend/tests/accounts/refresh/success.rs similarity index 100% rename from backend/tests/accounts/token_refresh/success.rs rename to backend/tests/accounts/refresh/success.rs diff --git a/backend/tests/accounts/account_store.rs b/backend/tests/accounts/store.rs similarity index 94% rename from backend/tests/accounts/account_store.rs rename to backend/tests/accounts/store.rs index c8e81588f..d46792a2d 100644 --- a/backend/tests/accounts/account_store.rs +++ b/backend/tests/accounts/store.rs @@ -1,9 +1,12 @@ use chrono::{DateTime, Duration, Utc}; use codex_proxy_rs::accounts::{ - account::{AccountStatus, AccountUsageDelta}, + account::AccountStatus, quota::{quota_snapshot_limit_reached, quota_snapshot_reset_at}, store::{AccountClaimsUpdate, AccountStore, NewAccount, PgAccountStore}, }; +use codex_proxy_rs::telemetry::account_usage::store::{ + AccountUsageDelta, AccountUsageStore, AccountUsageWindow, PgAccountUsageStore, +}; use secrecy::{ExposeSecret, SecretString}; use serde_json::json; @@ -324,10 +327,11 @@ async fn account_store_claims_update_should_preserve_optional_claims_when_refres async fn account_store_should_accumulate_usage_counters() { let (pool, _dir) = account_store_parts("accounts", 7).await; let repo = PgAccountStore::new(pool.clone()); + let usage_store = PgAccountUsageStore::new(pool.clone()); seed_new_account(&repo, "acct_a").await; - AccountStore::record_usage_delta( - &repo, + AccountUsageStore::record_usage_delta( + &usage_store, "acct_a", AccountUsageDelta { input_tokens: 10, @@ -342,8 +346,8 @@ async fn account_store_should_accumulate_usage_counters() { ) .await .unwrap(); - AccountStore::record_usage_delta( - &repo, + AccountUsageStore::record_usage_delta( + &usage_store, "acct_a", AccountUsageDelta { input_tokens: 8, @@ -397,7 +401,7 @@ async fn account_store_should_accumulate_usage_counters() { #[tokio::test] async fn account_store_should_restore_window_usage_into_runtime_pool_accounts() { let (pool, _dir) = account_store_parts("accounts", 33).await; - let repo = PgAccountStore::new(pool.clone()); + let usage_store = PgAccountUsageStore::new(pool.clone()); let window_started_at = pg_timestamp(Utc::now() - Duration::minutes(2)); let window_reset_at = pg_timestamp(Utc::now() + Duration::minutes(3)); @@ -412,10 +416,11 @@ async fn account_store_should_restore_window_usage_into_runtime_pool_accounts() .await .unwrap(); - let account = AccountStore::list_pool_accounts(&repo) + let account = AccountUsageStore::snapshots(&usage_store, &["acct_a".to_string()]) .await .unwrap() - .remove(0); + .remove("acct_a") + .unwrap(); assert_eq!(account.window_request_count, 7); assert_eq!(account.image_input_tokens, 31); @@ -681,7 +686,7 @@ async fn account_store_runtime_state_sync_should_not_clear_newer_future_quota_co stale_account.quota_verify_required = true; stale_account.quota_cooldown_until = None; - repo.sync_runtime_account_state(&stale_account, false) + repo.sync_runtime_account_state(&stale_account) .await .unwrap(); let stored: (String, bool, bool, Option>) = sqlx::query_as( @@ -734,9 +739,25 @@ async fn account_store_runtime_state_sync_should_not_regress_newer_usage_window( stale_account.window_started_at = Some(now); stale_account.limit_window_seconds = Some(60); - repo.sync_runtime_account_state(&stale_account, true) - .await - .unwrap(); + AccountUsageStore::sync_runtime_window( + &PgAccountUsageStore::new(pool.clone()), + "acct_a", + AccountUsageWindow { + request_count: stale_account.window_request_count, + input_tokens: stale_account.window_input_tokens, + output_tokens: stale_account.window_output_tokens, + cached_tokens: stale_account.window_cached_tokens, + image_input_tokens: stale_account.window_image_input_tokens, + image_output_tokens: stale_account.window_image_output_tokens, + image_request_count: stale_account.window_image_request_count, + image_request_failed_count: stale_account.window_image_request_failed_count, + started_at: stale_account.window_started_at, + reset_at: stale_account.window_reset_at, + limit_window_seconds: stale_account.limit_window_seconds, + }, + ) + .await + .unwrap(); let stored: (i64, i64, Option>, Option) = sqlx::query_as( "select window_request_count, window_input_tokens, window_reset_at, limit_window_seconds from account_usage where account_id = $1", ) diff --git a/backend/tests/api/admin/accounts/exporting.rs b/backend/tests/api/admin/accounts_routes/export.rs similarity index 100% rename from backend/tests/api/admin/accounts/exporting.rs rename to backend/tests/api/admin/accounts_routes/export.rs diff --git a/backend/tests/api/admin/accounts/importing.rs b/backend/tests/api/admin/accounts_routes/import.rs similarity index 99% rename from backend/tests/api/admin/accounts/importing.rs rename to backend/tests/api/admin/accounts_routes/import.rs index 829f2fa77..9916192d7 100644 --- a/backend/tests/api/admin/accounts/importing.rs +++ b/backend/tests/api/admin/accounts_routes/import.rs @@ -24,9 +24,7 @@ async fn admin_accounts_import_should_store_cpr_account_tokens() { stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app @@ -626,9 +624,7 @@ async fn admin_accounts_import_should_fetch_usage_to_complete_missing_plan_and_q stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let access_token = test_jwt_with_exp( None, @@ -840,9 +836,7 @@ async fn admin_accounts_import_should_require_admin_session_cookie() { stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app diff --git a/backend/tests/api/admin/accounts/lifecycle.rs b/backend/tests/api/admin/accounts_routes/lifecycle.rs similarity index 98% rename from backend/tests/api/admin/accounts/lifecycle.rs rename to backend/tests/api/admin/accounts_routes/lifecycle.rs index fb1fe4489..68b4259d3 100644 --- a/backend/tests/api/admin/accounts/lifecycle.rs +++ b/backend/tests/api/admin/accounts_routes/lifecycle.rs @@ -10,20 +10,17 @@ async fn admin_accounts_lifecycle_should_update_and_delete_accounts() { .bind(crate::support::storage::timestamp("2026-06-18T00:00:00Z")).bind(crate::support::storage::timestamp("2026-06-18T00:00:00Z")).execute(&pool).await.unwrap(); seed_account_related_rows(&pool, "acct_lifecycle").await; let config = test_config(test_database_url()); - let stores = background_task_stores(pool.clone(), redis); + let stores = background_task_stores(pool.clone(), redis.clone()); let fingerprint = crate::support::fingerprint::test_fingerprint(); let services = Arc::new(Services::new( &config, stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; - state - .services - .background_tasks - .session_affinity + let state = AppState::from(services.as_ref()); + let session_affinity = + codex_proxy_rs::dispatch::affinity::RedisSessionAffinityStore::new(redis); + session_affinity .upsert( "resp_lifecycle", &codex_proxy_rs::dispatch::affinity::SessionAffinityEntry { @@ -115,10 +112,7 @@ async fn admin_accounts_lifecycle_should_update_and_delete_accounts() { assert_eq!(deleted.status(), StatusCode::OK); assert_eq!(response_json(deleted).await["data"]["deleted"], 1); assert_account_related_rows_deleted(&pool, "acct_lifecycle").await; - assert!(state - .services - .background_tasks - .session_affinity + assert!(session_affinity .get( "resp_lifecycle", chrono::Utc::now(), @@ -897,9 +891,7 @@ async fn admin_account_create_should_derive_claims_and_store_plain_tokens() { stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state.clone()); let response = app.oneshot( @@ -933,9 +925,7 @@ async fn admin_account_manual_create_should_reject_missing_invalid_expired_or_un stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let cases = [ diff --git a/backend/tests/api/admin/accounts/list.rs b/backend/tests/api/admin/accounts_routes/list.rs similarity index 98% rename from backend/tests/api/admin/accounts/list.rs rename to backend/tests/api/admin/accounts_routes/list.rs index 56cfa499b..836e066d7 100644 --- a/backend/tests/api/admin/accounts/list.rs +++ b/backend/tests/api/admin/accounts_routes/list.rs @@ -21,9 +21,7 @@ async fn admin_accounts_list_should_not_expose_account_tokens() { stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app @@ -78,10 +76,9 @@ async fn admin_accounts_list_should_derive_refreshing_display_status_from_active .await .unwrap(); let now = chrono::Utc::now(); - assert!(state - .services - .background_tasks - .refresh_leases + let refresh_leases = + codex_proxy_rs::accounts::refresh::RedisRefreshLeaseStore::new(state.redis.clone()); + assert!(refresh_leases .try_acquire( "acct_refreshing", "test-owner", @@ -128,10 +125,9 @@ async fn admin_accounts_list_should_ignore_refresh_lease_for_expired_account() { .await .unwrap(); let now = chrono::Utc::now(); - assert!(state - .services - .background_tasks - .refresh_leases + let refresh_leases = + codex_proxy_rs::accounts::refresh::RedisRefreshLeaseStore::new(state.redis.clone()); + assert!(refresh_leases .try_acquire( "acct_expired", "stale-owner", diff --git a/backend/tests/api/admin/accounts/mod.rs b/backend/tests/api/admin/accounts_routes/mod.rs similarity index 84% rename from backend/tests/api/admin/accounts/mod.rs rename to backend/tests/api/admin/accounts_routes/mod.rs index 9d5ce1956..eab86bf48 100644 --- a/backend/tests/api/admin/accounts/mod.rs +++ b/backend/tests/api/admin/accounts_routes/mod.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{ops::Deref, sync::Arc}; use axum::{ body::Body, @@ -10,8 +10,9 @@ use codex_proxy_rs::{ cookies::PgCookieStore, store::{NewAccount, PgAccountStore}, }, + api::AppState, bootstrap::services::Services, - bootstrap::state::AppState, + infra::redis::RedisConnection, }; use secrecy::{ExposeSecret, SecretString}; use serde_json::{json, Value}; @@ -27,13 +28,26 @@ use crate::support::{ storage::{background_task_stores, create_test_redis, init_test_db, test_database_url}, }; -mod exporting; -mod importing; +mod export; +mod import; mod lifecycle; mod list; mod oauth; +mod probe; mod quota; -mod testing; + +struct AdminAccountsTestState { + app_state: AppState, + redis: RedisConnection, +} + +impl Deref for AdminAccountsTestState { + type Target = AppState; + + fn deref(&self) -> &Self::Target { + &self.app_state + } +} struct UsageAccountSeed<'a> { id: &'a str, @@ -96,7 +110,12 @@ async fn post_admin_account(app: &axum::Router, payload: Value) -> axum::respons async fn admin_accounts_test_app( db_name: &str, key_byte: u8, -) -> (axum::Router, AppState, PgPool, tempfile::TempDir) { +) -> ( + axum::Router, + AdminAccountsTestState, + PgPool, + tempfile::TempDir, +) { admin_accounts_test_app_with_api_base_url( db_name, key_byte, @@ -109,7 +128,12 @@ async fn admin_accounts_test_app_with_api_base_url( db_name: &str, key_byte: u8, api_base_url: String, -) -> (axum::Router, AppState, PgPool, tempfile::TempDir) { +) -> ( + axum::Router, + AdminAccountsTestState, + PgPool, + tempfile::TempDir, +) { admin_accounts_test_app_with_overrides(db_name, key_byte, api_base_url, None).await } @@ -117,7 +141,12 @@ async fn admin_accounts_test_app_with_oauth_token_endpoint( db_name: &str, key_byte: u8, oauth_token_endpoint: String, -) -> (axum::Router, AppState, PgPool, tempfile::TempDir) { +) -> ( + axum::Router, + AdminAccountsTestState, + PgPool, + tempfile::TempDir, +) { admin_accounts_test_app_with_overrides( db_name, key_byte, @@ -132,7 +161,12 @@ async fn admin_accounts_test_app_with_api_base_url_and_oauth_token_endpoint( key_byte: u8, api_base_url: String, oauth_token_endpoint: String, -) -> (axum::Router, AppState, PgPool, tempfile::TempDir) { +) -> ( + axum::Router, + AdminAccountsTestState, + PgPool, + tempfile::TempDir, +) { admin_accounts_test_app_with_overrides( db_name, key_byte, @@ -147,7 +181,12 @@ async fn admin_accounts_test_app_with_overrides( _key_byte: u8, api_base_url: String, oauth_token_endpoint: Option, -) -> (axum::Router, AppState, PgPool, tempfile::TempDir) { +) -> ( + axum::Router, + AdminAccountsTestState, + PgPool, + tempfile::TempDir, +) { let (pool, dir) = init_test_db(db_name).await; let redis = create_test_redis(db_name).await; seed_admin_session(&pool, &redis, "session_1").await; @@ -156,18 +195,24 @@ async fn admin_accounts_test_app_with_overrides( if let Some(oauth_token_endpoint) = oauth_token_endpoint { config.auth.oauth_token_endpoint = oauth_token_endpoint; } - let stores = background_task_stores(pool.clone(), redis); + let stores = background_task_stores(pool.clone(), redis.clone()); let fingerprint = crate::support::fingerprint::test_fingerprint(); let services = Arc::new(Services::new( &config, stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state.clone()); - (app, state, pool, dir) + ( + app, + AdminAccountsTestState { + app_state: state, + redis, + }, + pool, + dir, + ) } async fn seed_account(pool: &PgPool, account: NewAccount) { diff --git a/backend/tests/api/admin/accounts/oauth.rs b/backend/tests/api/admin/accounts_routes/oauth.rs similarity index 100% rename from backend/tests/api/admin/accounts/oauth.rs rename to backend/tests/api/admin/accounts_routes/oauth.rs diff --git a/backend/tests/api/admin/accounts/testing.rs b/backend/tests/api/admin/accounts_routes/probe.rs similarity index 96% rename from backend/tests/api/admin/accounts/testing.rs rename to backend/tests/api/admin/accounts_routes/probe.rs index 6962e25b6..6ab7fcbcc 100644 --- a/backend/tests/api/admin/accounts/testing.rs +++ b/backend/tests/api/admin/accounts_routes/probe.rs @@ -1,11 +1,11 @@ use super::*; use axum::body::to_bytes; use codex_proxy_rs::{ - models::{snapshot::ModelPlanSnapshot, store::ModelSnapshotStore}, - upstream::openai::protocol::{ - model_catalog::BackendModelEntry, - sse::{encode_sse_event, parse_sse_events}, + models::{ + store::ModelSnapshotStore, + types::{BackendModelEntry, ModelPlanSnapshot}, }, + upstream::openai::protocol::sse::{encode_sse_event, parse_sse_events}, }; use wiremock::{ matchers::{header, method, path}, @@ -158,13 +158,9 @@ async fn account_models_should_fetch_missing_plan_snapshot_with_account_token() let body = response_json(response).await; assert_eq!(body["data"]["models"][0]["id"], "gpt-plan-b-live"); assert_eq!(body["data"]["models"][0]["label"], "GPT Plan B Live"); - let stored = state - .services - .background_tasks - .model_snapshots - .list_plan_snapshots() - .await - .unwrap(); + let model_snapshots = + codex_proxy_rs::models::store::RedisModelSnapshotStore::new(state.redis.clone()); + let stored = model_snapshots.list_plan_snapshots().await.unwrap(); assert!(stored.iter().any(|snapshot| snapshot.plan_type == "plan-b")); } @@ -228,7 +224,7 @@ async fn account_models_should_return_models_for_account_plan_snapshot() { assert_eq!(body["data"]["models"][0]["label"], "GPT Plan B Only"); } -async fn seed_model_snapshot(state: &AppState, plan_type: &str) { +async fn seed_model_snapshot(state: &AdminAccountsTestState, plan_type: &str) { let snapshot = ModelPlanSnapshot::from_backend_entries( plan_type, vec![ @@ -244,17 +240,14 @@ async fn seed_model_snapshot(state: &AppState, plan_type: &str) { }, ], ); - state - .services - .background_tasks - .model_snapshots + codex_proxy_rs::models::store::RedisModelSnapshotStore::new(state.redis.clone()) .replace_plan_snapshot(&snapshot) .await .unwrap(); } async fn seed_single_model_snapshot( - state: &AppState, + state: &AdminAccountsTestState, plan_type: &str, model_id: &str, display_name: &str, @@ -267,10 +260,7 @@ async fn seed_single_model_snapshot( ..BackendModelEntry::default() }], ); - state - .services - .background_tasks - .model_snapshots + codex_proxy_rs::models::store::RedisModelSnapshotStore::new(state.redis.clone()) .replace_plan_snapshot(&snapshot) .await .unwrap(); diff --git a/backend/tests/api/admin/accounts/quota.rs b/backend/tests/api/admin/accounts_routes/quota.rs similarity index 100% rename from backend/tests/api/admin/accounts/quota.rs rename to backend/tests/api/admin/accounts_routes/quota.rs diff --git a/backend/tests/api/admin/auth/mod.rs b/backend/tests/api/admin/auth_routes/mod.rs similarity index 62% rename from backend/tests/api/admin/auth/mod.rs rename to backend/tests/api/admin/auth_routes/mod.rs index 096bc7a27..5e2f5fc60 100644 --- a/backend/tests/api/admin/auth/mod.rs +++ b/backend/tests/api/admin/auth_routes/mod.rs @@ -1,3 +1,2 @@ pub(crate) mod password; pub(crate) mod session; -pub(crate) mod session_store; diff --git a/backend/tests/api/admin/auth/password.rs b/backend/tests/api/admin/auth_routes/password.rs similarity index 100% rename from backend/tests/api/admin/auth/password.rs rename to backend/tests/api/admin/auth_routes/password.rs diff --git a/backend/tests/api/admin/auth/session.rs b/backend/tests/api/admin/auth_routes/session.rs similarity index 98% rename from backend/tests/api/admin/auth/session.rs rename to backend/tests/api/admin/auth_routes/session.rs index 99a97e18d..bf7f1bed3 100644 --- a/backend/tests/api/admin/auth/session.rs +++ b/backend/tests/api/admin/auth_routes/session.rs @@ -185,9 +185,7 @@ async fn admin_login_test_app( stores, runtime_fingerprint(fingerprint), )); - let state = codex_proxy_rs::bootstrap::state::AppState { - services: (*services).clone(), - }; + let state = codex_proxy_rs::api::AppState::from(services.as_ref()); ( codex_proxy_rs::api::router::router().with_state(state), dir, diff --git a/backend/tests/api/admin/monitoring/dashboard.rs b/backend/tests/api/admin/dashboard_routes.rs similarity index 99% rename from backend/tests/api/admin/monitoring/dashboard.rs rename to backend/tests/api/admin/dashboard_routes.rs index 5535fce1b..728657efe 100644 --- a/backend/tests/api/admin/monitoring/dashboard.rs +++ b/backend/tests/api/admin/dashboard_routes.rs @@ -4,7 +4,8 @@ use axum::{ }; use chrono::{Duration, Utc}; use codex_proxy_rs::{ - bootstrap::{services::Services, state::AppState}, + api::AppState, + bootstrap::services::Services, infra::time::{china_datetime, china_day_start, china_hour}, telemetry::{usage::store::PgUsageRecordStore, usage::types::UsageRecord}, upstream::openai::fingerprint::Fingerprint, @@ -643,9 +644,7 @@ async fn dashboard_test_app( stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); ( codex_proxy_rs::api::router::router().with_state(state), PgUsageRecordStore::new(pool.clone()), diff --git a/backend/tests/api/admin/keys/authorization.rs b/backend/tests/api/admin/keys_routes/authorization.rs similarity index 100% rename from backend/tests/api/admin/keys/authorization.rs rename to backend/tests/api/admin/keys_routes/authorization.rs diff --git a/backend/tests/api/admin/keys/lifecycle.rs b/backend/tests/api/admin/keys_routes/lifecycle.rs similarity index 100% rename from backend/tests/api/admin/keys/lifecycle.rs rename to backend/tests/api/admin/keys_routes/lifecycle.rs diff --git a/backend/tests/api/admin/keys/mod.rs b/backend/tests/api/admin/keys_routes/mod.rs similarity index 91% rename from backend/tests/api/admin/keys/mod.rs rename to backend/tests/api/admin/keys_routes/mod.rs index 1acf331e2..6f351e3f1 100644 --- a/backend/tests/api/admin/keys/mod.rs +++ b/backend/tests/api/admin/keys_routes/mod.rs @@ -2,7 +2,7 @@ use axum::{ body::Body, http::{Request, StatusCode}, }; -use codex_proxy_rs::{bootstrap::services::Services, bootstrap::state::AppState}; +use codex_proxy_rs::{api::AppState, bootstrap::services::Services}; use serde_json::json; use tower::util::ServiceExt; @@ -16,7 +16,6 @@ use crate::support::{ mod authorization; mod lifecycle; -mod store; async fn admin_client_key_test_app(db_name: &str) -> (axum::Router, tempfile::TempDir) { let (pool, dir) = init_test_db(db_name).await; @@ -30,9 +29,7 @@ async fn admin_client_key_test_app(db_name: &str) -> (axum::Router, tempfile::Te stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); (codex_proxy_rs::api::router::router().with_state(state), dir) } diff --git a/backend/tests/api/admin/mod.rs b/backend/tests/api/admin/mod.rs index 44419e4d3..3c42f663b 100644 --- a/backend/tests/api/admin/mod.rs +++ b/backend/tests/api/admin/mod.rs @@ -1,7 +1,8 @@ -pub(crate) mod accounts; -pub(crate) mod auth; -pub(crate) mod keys; -pub(crate) mod monitoring; +pub(crate) mod accounts_routes; +pub(crate) mod auth_routes; +pub(crate) mod dashboard_routes; +pub(crate) mod keys_routes; pub(crate) mod response; -pub(crate) mod settings; -pub(crate) mod system; +pub(crate) mod settings_routes; +pub(crate) mod system_routes; +pub(crate) mod usage_routes; diff --git a/backend/tests/api/admin/monitoring/mod.rs b/backend/tests/api/admin/monitoring/mod.rs deleted file mode 100644 index 22de2ac66..000000000 --- a/backend/tests/api/admin/monitoring/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod dashboard; -pub(crate) mod ops_error_store; -pub(crate) mod usage_record_routes; -pub(crate) mod usage_record_store; diff --git a/backend/tests/api/admin/response/mod.rs b/backend/tests/api/admin/response.rs similarity index 100% rename from backend/tests/api/admin/response/mod.rs rename to backend/tests/api/admin/response.rs diff --git a/backend/tests/api/admin/settings/mod.rs b/backend/tests/api/admin/settings_routes/mod.rs similarity index 100% rename from backend/tests/api/admin/settings/mod.rs rename to backend/tests/api/admin/settings_routes/mod.rs diff --git a/backend/tests/api/admin/settings/routes.rs b/backend/tests/api/admin/settings_routes/routes.rs similarity index 93% rename from backend/tests/api/admin/settings/routes.rs rename to backend/tests/api/admin/settings_routes/routes.rs index b10c06d9b..6bed0e4e2 100644 --- a/backend/tests/api/admin/settings/routes.rs +++ b/backend/tests/api/admin/settings_routes/routes.rs @@ -10,10 +10,11 @@ use axum::{ use chrono::Utc; use codex_proxy_rs::{ accounts::pool::AccountAcquireRequest, + api::AppState, bootstrap::{ config::AppConfig, services::{apply_settings_to_config, settings_snapshot_from_config, Services}, - state::AppState, + tasks::coordinator::TaskCoordinator, }, settings::service::RuntimeSettingsService, }; @@ -263,9 +264,7 @@ async fn admin_settings_update_should_persist_runtime_settings_to_database() { let fingerprint = crate::support::fingerprint::test_fingerprint(); let services = Services::new(&config, stores, runtime_fingerprint(fingerprint)); let services = std::sync::Arc::new(services); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app @@ -363,10 +362,9 @@ async fn admin_settings_update_should_apply_runtime_services() { stores, runtime_fingerprint(fingerprint), )); + let settings_tasks = TaskCoordinator::start_settings_subscriptions(&services); services.account_pool.restore_from_store().await.unwrap(); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app @@ -396,6 +394,29 @@ async fn admin_settings_update_should_apply_runtime_services() { .unwrap(); assert_eq!(response.status(), StatusCode::OK); + tokio::time::timeout(StdDuration::from_secs(1), async { + loop { + let resolved = services + .models + .catalog() + .await + .resolve_model_id("runtime-alias"); + let refresh_policy = services.refresh_policy.snapshot(); + let capacity = services.account_pool.capacity_summary(Utc::now()).await; + if resolved == "gpt-5.5" + && refresh_policy.refresh_margin_seconds == 120 + && refresh_policy.refresh_concurrency == 3 + && capacity.total_slots == 4 + && capacity.max_concurrent_per_account == 2 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("runtime settings subscribers should observe the persisted update"); + let resolved = services .models .catalog() @@ -435,6 +456,7 @@ async fn admin_settings_update_should_apply_runtime_services() { .wait_for_request_interval(&repeated) .await; assert!(started_at.elapsed() >= StdDuration::from_millis(20)); + settings_tasks.shutdown().await; } #[tokio::test] @@ -450,9 +472,7 @@ async fn admin_settings_update_should_reject_unsupported_or_invalid_fields() { stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); let response = app @@ -485,9 +505,7 @@ async fn admin_settings_test_app(db_name: &str) -> (axum::Router, tempfile::Temp stores, runtime_fingerprint(fingerprint), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); (codex_proxy_rs::api::router::router().with_state(state), dir) } diff --git a/backend/tests/api/admin/system/mod.rs b/backend/tests/api/admin/system_routes/mod.rs similarity index 99% rename from backend/tests/api/admin/system/mod.rs rename to backend/tests/api/admin/system_routes/mod.rs index 711ed01bf..9d2e0b7cb 100644 --- a/backend/tests/api/admin/system/mod.rs +++ b/backend/tests/api/admin/system_routes/mod.rs @@ -9,7 +9,7 @@ use axum::{ body::{to_bytes, Body}, http::{header::CONTENT_TYPE, Request, StatusCode}, }; -use codex_proxy_rs::{bootstrap::services::Services, bootstrap::state::AppState}; +use codex_proxy_rs::{api::AppState, bootstrap::services::Services}; use flate2::{write::GzEncoder, Compression}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; @@ -1274,9 +1274,7 @@ async fn admin_system_test_app(db_name: &str) -> (axum::Router, tempfile::TempDi stores, runtime_fingerprint(test_fingerprint()), )); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); (app, dir) } diff --git a/backend/tests/api/admin/monitoring/usage_record_routes.rs b/backend/tests/api/admin/usage_routes.rs similarity index 98% rename from backend/tests/api/admin/monitoring/usage_record_routes.rs rename to backend/tests/api/admin/usage_routes.rs index 856c0ed4c..cb28102c7 100644 --- a/backend/tests/api/admin/monitoring/usage_record_routes.rs +++ b/backend/tests/api/admin/usage_routes.rs @@ -3,7 +3,8 @@ use axum::{ http::{Request, StatusCode}, }; use codex_proxy_rs::{ - bootstrap::{services::Services, state::AppState}, + api::AppState, + bootstrap::services::Services, telemetry::{ ops::store::PgOpsErrorLogStore, ops::types::OpsErrorLog, usage::store::PgUsageRecordStore, usage::types::UsageRecord, @@ -135,7 +136,7 @@ async fn monitoring_test_app( stores, runtime_fingerprint(crate::support::fingerprint::test_fingerprint()), ); - let app = codex_proxy_rs::api::router::router().with_state(AppState { services }); + let app = codex_proxy_rs::api::router::router().with_state(AppState::from(&services)); ( app, PgUsageRecordStore::new(pool.clone()), diff --git a/backend/tests/api/http/web_assets/mod.rs b/backend/tests/api/assets.rs similarity index 98% rename from backend/tests/api/http/web_assets/mod.rs rename to backend/tests/api/assets.rs index 0b5072810..4a0cb50b2 100644 --- a/backend/tests/api/http/web_assets/mod.rs +++ b/backend/tests/api/assets.rs @@ -181,9 +181,7 @@ async fn router_with_dist_and_database(dist: &Path, label: &str) -> (Router, PgP stores, runtime_fingerprint(fingerprint), )); - let state = codex_proxy_rs::bootstrap::state::AppState { - services: (*services).clone(), - }; + let state = codex_proxy_rs::api::AppState::from(services.as_ref()); ( codex_proxy_rs::api::router::router_with_assets(dist).with_state(state), pool, diff --git a/backend/tests/api/client/mod.rs b/backend/tests/api/client/mod.rs index c8a1717a8..9c6839d14 100644 --- a/backend/tests/api/client/mod.rs +++ b/backend/tests/api/client/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod auth; pub(crate) mod key_auth; -pub(crate) mod openai; +pub(crate) mod models; +pub(crate) mod responses; diff --git a/backend/tests/api/client/openai/models_routes.rs b/backend/tests/api/client/models.rs similarity index 97% rename from backend/tests/api/client/openai/models_routes.rs rename to backend/tests/api/client/models.rs index 246643af4..9352b85eb 100644 --- a/backend/tests/api/client/openai/models_routes.rs +++ b/backend/tests/api/client/models.rs @@ -5,7 +5,7 @@ use axum::{ http::{Request, StatusCode}, Router, }; -use codex_proxy_rs::{bootstrap::services::Services, bootstrap::state::AppState}; +use codex_proxy_rs::{api::AppState, bootstrap::services::Services}; use tower::util::ServiceExt; use crate::support::{ @@ -154,9 +154,7 @@ async fn build_test_app( .initialize_hot_path_state() .await .expect("hot path state should initialize"); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); let app = codex_proxy_rs::api::router::router().with_state(state); (app, plaintext, dir) diff --git a/backend/tests/api/client/openai/mod.rs b/backend/tests/api/client/openai/mod.rs deleted file mode 100644 index 562894937..000000000 --- a/backend/tests/api/client/openai/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod models_routes; -pub(crate) mod responses_routes; diff --git a/backend/tests/api/client/openai/responses_routes.rs b/backend/tests/api/client/responses.rs similarity index 97% rename from backend/tests/api/client/openai/responses_routes.rs rename to backend/tests/api/client/responses.rs index e017040a3..f627763c7 100644 --- a/backend/tests/api/client/openai/responses_routes.rs +++ b/backend/tests/api/client/responses.rs @@ -2,7 +2,7 @@ use axum::{ body::{to_bytes, Body}, http::{Request, StatusCode}, }; -use codex_proxy_rs::{bootstrap::services::Services, bootstrap::state::AppState}; +use codex_proxy_rs::{api::AppState, bootstrap::services::Services}; use tower::util::ServiceExt; use crate::support::{ @@ -144,9 +144,7 @@ async fn test_app_with_client_api_key() -> (axum::Router, String, tempfile::Temp .initialize_hot_path_state() .await .expect("hot path state should initialize"); - let state = AppState { - services: (*services).clone(), - }; + let state = AppState::from(services.as_ref()); ( codex_proxy_rs::api::router::router().with_state(state), plaintext, diff --git a/backend/tests/api/http/mod.rs b/backend/tests/api/http/mod.rs deleted file mode 100644 index a92cb06bc..000000000 --- a/backend/tests/api/http/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) mod trace_middleware; -pub(crate) mod web_assets; diff --git a/backend/tests/api/middleware/mod.rs b/backend/tests/api/middleware/mod.rs new file mode 100644 index 000000000..fc1f25c36 --- /dev/null +++ b/backend/tests/api/middleware/mod.rs @@ -0,0 +1 @@ +mod trace; diff --git a/backend/tests/api/http/trace_middleware/mod.rs b/backend/tests/api/middleware/trace.rs similarity index 100% rename from backend/tests/api/http/trace_middleware/mod.rs rename to backend/tests/api/middleware/trace.rs diff --git a/backend/tests/api/mod.rs b/backend/tests/api/mod.rs index e9410c3dd..8cf086ab2 100644 --- a/backend/tests/api/mod.rs +++ b/backend/tests/api/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod admin; +pub(crate) mod assets; pub(crate) mod client; -pub(crate) mod http; +pub(crate) mod middleware; diff --git a/backend/tests/auth/mod.rs b/backend/tests/auth/mod.rs new file mode 100644 index 000000000..3d6074864 --- /dev/null +++ b/backend/tests/auth/mod.rs @@ -0,0 +1 @@ +mod store; diff --git a/backend/tests/api/admin/auth/session_store.rs b/backend/tests/auth/store.rs similarity index 100% rename from backend/tests/api/admin/auth/session_store.rs rename to backend/tests/auth/store.rs diff --git a/backend/tests/bootstrap/mod.rs b/backend/tests/bootstrap/mod.rs index 0ce8f5300..a9721b8b4 100644 --- a/backend/tests/bootstrap/mod.rs +++ b/backend/tests/bootstrap/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod account_pool_restore; pub(crate) mod config; pub(crate) mod import_sqlite; pub(crate) mod services; diff --git a/backend/tests/bootstrap/tasks/fingerprint.rs b/backend/tests/bootstrap/tasks/fingerprint_update.rs similarity index 100% rename from backend/tests/bootstrap/tasks/fingerprint.rs rename to backend/tests/bootstrap/tasks/fingerprint_update.rs diff --git a/backend/tests/bootstrap/tasks/mod.rs b/backend/tests/bootstrap/tasks/mod.rs index b1e1ca30f..a1c8b9c9c 100644 --- a/backend/tests/bootstrap/tasks/mod.rs +++ b/backend/tests/bootstrap/tasks/mod.rs @@ -1,13 +1,16 @@ -use std::{sync::Arc, time::Duration as StdDuration}; +use std::{collections::HashMap, sync::Arc, time::Duration as StdDuration}; use async_trait::async_trait; use chrono::{Duration, Utc}; use codex_proxy_rs::accounts::account::{Account, AccountStatus}; -use codex_proxy_rs::accounts::account::{AccountModelUsageDelta, AccountUsageDelta}; use codex_proxy_rs::accounts::cookies::PgCookieStore; use codex_proxy_rs::accounts::store::{AccountStore, AccountStoreResult}; -use codex_proxy_rs::models::config::ModelConfig; use codex_proxy_rs::models::service::ModelService; +use codex_proxy_rs::models::types::ModelConfig; +use codex_proxy_rs::telemetry::account_usage::store::{ + AccountModelUsageDelta, AccountUsageDelta, AccountUsageSnapshot, AccountUsageStore, + AccountUsageStoreError, AccountUsageWindow, +}; use codex_proxy_rs::upstream::openai::fingerprint::{PgFingerprintStore, RuntimeFingerprint}; use wiremock::{ matchers::{method, path}, @@ -18,7 +21,7 @@ use crate::support::storage::init_test_db; mod cleanup; mod coordinator; -mod fingerprint; +mod fingerprint_update; mod model_refresh; struct FakeAccountStore; @@ -56,23 +59,6 @@ impl AccountStore for FakeAccountStore { Ok(true) } - async fn record_usage_delta( - &self, - _account_id: &str, - _usage: AccountUsageDelta, - ) -> AccountStoreResult<()> { - Ok(()) - } - - async fn record_model_usage_delta( - &self, - _account_id: &str, - _model: &str, - _usage: AccountModelUsageDelta, - ) -> AccountStoreResult<()> { - Ok(()) - } - async fn get_quota_json(&self, _account_id: &str) -> AccountStoreResult> { Ok(None) } @@ -87,20 +73,53 @@ impl AccountStore for FakeAccountStore { Ok(false) } - async fn sync_runtime_account_state( - &self, - _account: &Account, - _sync_usage_window: bool, - ) -> AccountStoreResult { + async fn sync_runtime_account_state(&self, _account: &Account) -> AccountStoreResult { Ok(false) } +} + +struct FakeAccountUsageStore; + +#[async_trait] +impl AccountUsageStore for FakeAccountUsageStore { + async fn snapshots( + &self, + _account_ids: &[String], + ) -> Result, AccountUsageStoreError> { + Ok(HashMap::new()) + } + + async fn record_usage_delta( + &self, + _account_id: &str, + _usage: AccountUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + Ok(()) + } + + async fn record_model_usage_delta( + &self, + _account_id: &str, + _model: &str, + _usage: AccountModelUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + Ok(()) + } + + async fn sync_runtime_window( + &self, + _account_id: &str, + _window: AccountUsageWindow, + ) -> Result<(), AccountUsageStoreError> { + Ok(()) + } async fn sync_rate_limit_window( &self, _account_id: &str, _reset_at: chrono::DateTime, _limit_window_seconds: Option, - ) -> AccountStoreResult<()> { + ) -> Result<(), AccountUsageStoreError> { Ok(()) } } diff --git a/backend/tests/bootstrap/tasks/model_refresh.rs b/backend/tests/bootstrap/tasks/model_refresh.rs index e2c587e24..278e4c10a 100644 --- a/backend/tests/bootstrap/tasks/model_refresh.rs +++ b/backend/tests/bootstrap/tasks/model_refresh.rs @@ -8,14 +8,11 @@ use std::{ use codex_proxy_rs::{ accounts::pool::{AccountAcquireRequest, AccountPoolOptions, RuntimeAccountPoolService}, models::{ - snapshot::ModelPlanSnapshot, store::{ModelSnapshotStore, ModelSnapshotStoreResult}, + types::{BackendModelEntry, ModelPlanSnapshot}, }, - upstream::openai::{ - protocol::model_catalog::BackendModelEntry, - transport::{ - CodexModelCatalogClient, CodexModelCatalogClientError, CodexModelCatalogRequest, - }, + upstream::openai::transport::{ + CodexModelCatalogClient, CodexModelCatalogClientError, CodexModelCatalogRequest, }, }; @@ -25,6 +22,7 @@ async fn model_refresh_task_should_start_and_shutdown() { let account_store = Arc::new(FakeAccountStore); let account_pool = Arc::new(RuntimeAccountPoolService::new( account_store.clone(), + Arc::new(FakeAccountUsageStore), AccountPoolOptions::default(), 0, )); @@ -52,16 +50,23 @@ async fn model_refresh_task_should_sync_model_plan_allowlist_to_account_pool() { Some(store), Some(upstream), )); + let request_usage_records = Arc::new(AtomicUsize::new(0)); + let model_request_usage_records = Arc::new(AtomicUsize::new(0)); let account_store = Arc::new(PlanAccountStore { accounts: vec![ plan_account("acct-plus", "plus"), plan_account("acct-team", "team"), ], - request_usage_records: Arc::new(AtomicUsize::new(0)), - model_request_usage_records: Arc::new(AtomicUsize::new(0)), + request_usage_records: request_usage_records.clone(), + model_request_usage_records: model_request_usage_records.clone(), + }); + let usage_store = Arc::new(PlanAccountUsageStore { + request_usage_records, + model_request_usage_records, }); let account_pool = Arc::new(RuntimeAccountPoolService::new( account_store.clone(), + usage_store, AccountPoolOptions::default(), 0, )); @@ -147,12 +152,15 @@ impl CodexModelCatalogClient for FakeModelCatalogClient { async fn fetch_models( &self, request: &CodexModelCatalogRequest<'_>, - ) -> Result, CodexModelCatalogClientError> { + ) -> Result, CodexModelCatalogClientError> { Ok(self .models_by_plan .get(request.plan_type) .cloned() - .unwrap_or_default()) + .unwrap_or_default() + .into_iter() + .map(|model| serde_json::to_value(model).unwrap()) + .collect()) } } @@ -192,30 +200,6 @@ impl AccountStore for PlanAccountStore { Ok(true) } - async fn record_usage_delta( - &self, - _account_id: &str, - usage: AccountUsageDelta, - ) -> AccountStoreResult<()> { - if usage.requests > 0 { - self.request_usage_records.fetch_add(1, Ordering::SeqCst); - } - Ok(()) - } - - async fn record_model_usage_delta( - &self, - _account_id: &str, - _model: &str, - usage: AccountModelUsageDelta, - ) -> AccountStoreResult<()> { - if usage.requests > 0 { - self.model_request_usage_records - .fetch_add(1, Ordering::SeqCst); - } - Ok(()) - } - async fn get_quota_json(&self, _account_id: &str) -> AccountStoreResult> { Ok(None) } @@ -230,20 +214,63 @@ impl AccountStore for PlanAccountStore { Ok(false) } - async fn sync_runtime_account_state( - &self, - _account: &Account, - _sync_usage_window: bool, - ) -> AccountStoreResult { + async fn sync_runtime_account_state(&self, _account: &Account) -> AccountStoreResult { Ok(false) } +} + +struct PlanAccountUsageStore { + request_usage_records: Arc, + model_request_usage_records: Arc, +} + +#[async_trait] +impl AccountUsageStore for PlanAccountUsageStore { + async fn snapshots( + &self, + _account_ids: &[String], + ) -> Result, AccountUsageStoreError> { + Ok(HashMap::new()) + } + + async fn record_usage_delta( + &self, + _account_id: &str, + usage: AccountUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + if usage.requests > 0 { + self.request_usage_records.fetch_add(1, Ordering::SeqCst); + } + Ok(()) + } + + async fn record_model_usage_delta( + &self, + _account_id: &str, + _model: &str, + usage: AccountModelUsageDelta, + ) -> Result<(), AccountUsageStoreError> { + if usage.requests > 0 { + self.model_request_usage_records + .fetch_add(1, Ordering::SeqCst); + } + Ok(()) + } + + async fn sync_runtime_window( + &self, + _account_id: &str, + _window: AccountUsageWindow, + ) -> Result<(), AccountUsageStoreError> { + Ok(()) + } async fn sync_rate_limit_window( &self, _account_id: &str, _reset_at: chrono::DateTime, _limit_window_seconds: Option, - ) -> AccountStoreResult<()> { + ) -> Result<(), AccountUsageStoreError> { Ok(()) } } diff --git a/backend/tests/dispatch/affinity.rs b/backend/tests/dispatch/affinity/mod.rs similarity index 100% rename from backend/tests/dispatch/affinity.rs rename to backend/tests/dispatch/affinity/mod.rs diff --git a/backend/tests/dispatch/mod.rs b/backend/tests/dispatch/mod.rs index 74cc09489..7b4c0d241 100644 --- a/backend/tests/dispatch/mod.rs +++ b/backend/tests/dispatch/mod.rs @@ -1,2 +1,2 @@ pub(crate) mod affinity; -pub(crate) mod chat_upstream; +pub(crate) mod service; diff --git a/backend/tests/dispatch/chat_upstream/compact_routes.rs b/backend/tests/dispatch/service/compact_routes.rs similarity index 100% rename from backend/tests/dispatch/chat_upstream/compact_routes.rs rename to backend/tests/dispatch/service/compact_routes.rs diff --git a/backend/tests/dispatch/chat_upstream/mod.rs b/backend/tests/dispatch/service/mod.rs similarity index 98% rename from backend/tests/dispatch/chat_upstream/mod.rs rename to backend/tests/dispatch/service/mod.rs index bba69b68e..46f63f0e7 100644 --- a/backend/tests/dispatch/chat_upstream/mod.rs +++ b/backend/tests/dispatch/service/mod.rs @@ -14,9 +14,9 @@ use chrono::{Duration, Utc}; use codex_proxy_rs::{ accounts::{account::AccountStatus, cookies::PgCookieStore, pool::AccountAcquireRequest}, api::router, + api::AppState, bootstrap::config::AppConfig, bootstrap::services::{Services, UsageRecordOptions}, - bootstrap::state::AppState, dispatch::affinity::{ build_conversation_identity, compute_variant_hash, prepare_variant_identity, SessionAffinityEntry, @@ -276,7 +276,7 @@ async fn test_app_state_with_storage( .initialize_hot_path_state() .await .expect("hot path state should initialize"); - AppState { services } + AppState::from(&services) } async fn test_app_with_account(base_url: String) -> (axum::Router, String, tempfile::TempDir) { @@ -420,12 +420,7 @@ async fn test_app_with_account_pool_config_and_affinity( ) .await; if let Some(response_id) = affinity_response_id { - insert_session_affinity( - &state.services.background_tasks.session_affinity, - response_id, - "acct_chat", - ) - .await; + insert_session_affinity(&state.services.session_affinity, response_id, "acct_chat").await; } state .services @@ -452,11 +447,10 @@ async fn test_app_with_two_accounts_and_affinity( .await; state .services - .background_tasks .session_affinity - .upsert( - "resp_previous", - &SessionAffinityEntry { + .record( + "resp_previous".to_string(), + SessionAffinityEntry { account_id: "acct_z".to_string(), conversation_id: "conv_affinity".to_string(), turn_state: Some("turn_affinity".to_string()), @@ -466,7 +460,6 @@ async fn test_app_with_two_accounts_and_affinity( variant_hash: None, created_at: now, }, - Duration::hours(4), ) .await .unwrap(); @@ -511,11 +504,10 @@ async fn test_app_with_two_accounts_and_affinity_status( .await; state .services - .background_tasks .session_affinity - .upsert( - "resp_affinity_risk", - &SessionAffinityEntry { + .record( + "resp_affinity_risk".to_string(), + SessionAffinityEntry { account_id: "acct_affinity".to_string(), conversation_id: "conv_affinity_risk".to_string(), turn_state: Some("turn_affinity_risk".to_string()), @@ -525,7 +517,6 @@ async fn test_app_with_two_accounts_and_affinity_status( variant_hash: None, created_at: now, }, - Duration::hours(4), ) .await .unwrap(); @@ -692,14 +683,14 @@ async fn insert_model_snapshot(redis: &codex_proxy_rs::infra::redis::RedisConnec } async fn insert_session_affinity( - store: &codex_proxy_rs::dispatch::affinity::RedisSessionAffinityStore, + service: &codex_proxy_rs::dispatch::affinity::RuntimeSessionAffinityService, response_id: &str, account_id: &str, ) { - store - .upsert( - response_id, - &SessionAffinityEntry { + service + .record( + response_id.to_string(), + SessionAffinityEntry { account_id: account_id.to_string(), conversation_id: format!("conv_{response_id}"), turn_state: Some("turn-stale".to_string()), @@ -709,7 +700,6 @@ async fn insert_session_affinity( variant_hash: None, created_at: Utc::now(), }, - Duration::hours(4), ) .await .unwrap(); diff --git a/backend/tests/dispatch/chat_upstream/responses_http.rs b/backend/tests/dispatch/service/responses_http.rs similarity index 99% rename from backend/tests/dispatch/chat_upstream/responses_http.rs rename to backend/tests/dispatch/service/responses_http.rs index 042d5b742..2cec338bf 100644 --- a/backend/tests/dispatch/chat_upstream/responses_http.rs +++ b/backend/tests/dispatch/service/responses_http.rs @@ -1229,11 +1229,9 @@ async fn responses_should_preserve_explicit_history_when_banned_affinity_switche let captured = upstream.await.unwrap(); let affinity = state .services - .background_tasks .session_affinity - .get("resp_affinity_risk", Utc::now(), chrono::Duration::hours(4)) - .await - .unwrap(); + .lookup("resp_affinity_risk", Utc::now()) + .await; assert_eq!(response.status(), StatusCode::OK); assert_eq!( @@ -1284,11 +1282,9 @@ async fn responses_should_preserve_explicit_history_when_quota_affinity_switches let captured = upstream.await.unwrap(); let affinity = state .services - .background_tasks .session_affinity - .get("resp_affinity_risk", Utc::now(), chrono::Duration::hours(4)) - .await - .unwrap(); + .lookup("resp_affinity_risk", Utc::now()) + .await; assert_eq!(response.status(), StatusCode::OK); assert_eq!( diff --git a/backend/tests/dispatch/chat_upstream/responses_recovery.rs b/backend/tests/dispatch/service/responses_recovery.rs similarity index 100% rename from backend/tests/dispatch/chat_upstream/responses_recovery.rs rename to backend/tests/dispatch/service/responses_recovery.rs diff --git a/backend/tests/dispatch/chat_upstream/responses_websocket.rs b/backend/tests/dispatch/service/responses_websocket.rs similarity index 99% rename from backend/tests/dispatch/chat_upstream/responses_websocket.rs rename to backend/tests/dispatch/service/responses_websocket.rs index 0413b9283..e544cd923 100644 --- a/backend/tests/dispatch/chat_upstream/responses_websocket.rs +++ b/backend/tests/dispatch/service/responses_websocket.rs @@ -880,11 +880,9 @@ async fn responses_websocket_should_strip_implicit_affinity_when_switching_accou let (first_payload, second_payload, second_headers) = upstream.await.unwrap(); let affinity = state .services - .background_tasks .session_affinity - .get(&first_response_id, Utc::now(), chrono::Duration::hours(4)) - .await - .unwrap(); + .lookup(&first_response_id, Utc::now()) + .await; assert!(first_payload["prompt_cache_key"].as_str().is_some()); assert_eq!( @@ -1525,15 +1523,9 @@ async fn responses_websocket_should_route_previous_response_id_to_recorded_accou .contains("\"id\":\"resp_affinity_first\"")); let stored_affinity = state .services - .background_tasks .session_affinity - .get( - "resp_affinity_first", - Utc::now(), - chrono::Duration::hours(4), - ) + .lookup("resp_affinity_first", Utc::now()) .await - .unwrap() .unwrap(); assert_eq!(stored_affinity.account_id, "acct_primary"); assert!(!stored_affinity.conversation_id.is_empty()); diff --git a/backend/tests/dispatch/chat_upstream/usage_logging.rs b/backend/tests/dispatch/service/usage_logging.rs similarity index 100% rename from backend/tests/dispatch/chat_upstream/usage_logging.rs rename to backend/tests/dispatch/service/usage_logging.rs diff --git a/backend/tests/infra/storage_schema/mod.rs b/backend/tests/infra/database.rs similarity index 100% rename from backend/tests/infra/storage_schema/mod.rs rename to backend/tests/infra/database.rs diff --git a/backend/tests/infra/log_rotation/mod.rs b/backend/tests/infra/logging.rs similarity index 100% rename from backend/tests/infra/log_rotation/mod.rs rename to backend/tests/infra/logging.rs diff --git a/backend/tests/infra/mod.rs b/backend/tests/infra/mod.rs index 0d3408b5d..a7bf05c0e 100644 --- a/backend/tests/infra/mod.rs +++ b/backend/tests/infra/mod.rs @@ -1,5 +1,5 @@ +pub(crate) mod database; pub(crate) mod format; -pub(crate) mod log_rotation; +pub(crate) mod logging; pub(crate) mod paths; -pub(crate) mod storage_schema; pub(crate) mod time; diff --git a/backend/tests/keys/mod.rs b/backend/tests/keys/mod.rs new file mode 100644 index 000000000..3d6074864 --- /dev/null +++ b/backend/tests/keys/mod.rs @@ -0,0 +1 @@ +mod store; diff --git a/backend/tests/api/admin/keys/store.rs b/backend/tests/keys/store.rs similarity index 100% rename from backend/tests/api/admin/keys/store.rs rename to backend/tests/keys/store.rs diff --git a/backend/tests/main.rs b/backend/tests/main.rs index 67a5380f8..445a63de3 100644 --- a/backend/tests/main.rs +++ b/backend/tests/main.rs @@ -1,8 +1,10 @@ mod accounts; mod api; +mod auth; mod bootstrap; mod dispatch; mod infra; +mod keys; mod models; mod support; mod telemetry; diff --git a/backend/tests/models/catalog.rs b/backend/tests/models/catalog.rs index 1c668fdb8..d6a345651 100644 --- a/backend/tests/models/catalog.rs +++ b/backend/tests/models/catalog.rs @@ -10,14 +10,13 @@ use async_trait::async_trait; use codex_proxy_rs::accounts::account::{Account, AccountStatus}; use codex_proxy_rs::models::catalog::ModelCatalog; -use codex_proxy_rs::models::config::ModelConfig; use codex_proxy_rs::models::service::{ ModelRefreshPlanAccount, ModelRefreshResult, ModelService, ModelServiceError, }; -use codex_proxy_rs::models::snapshot::ModelPlanSnapshot; use codex_proxy_rs::models::store::{ModelSnapshotStore, ModelSnapshotStoreResult}; -use codex_proxy_rs::upstream::openai::protocol::model_catalog::{ - BackendModelEntry, BackendReasoningEffort, BackendTruncationPolicy, +use codex_proxy_rs::models::types::{ + BackendModelEntry, BackendReasoningEffort, BackendTruncationPolicy, ModelConfig, + ModelPlanSnapshot, }; use codex_proxy_rs::upstream::openai::transport::{ CodexModelCatalogClient, CodexModelCatalogClientError, CodexModelCatalogRequest, @@ -236,7 +235,8 @@ fn active_account(id: &str, plan_type: &str) -> Account { fn refresh_plan_account(plan_type: &str, account: Account) -> ModelRefreshPlanAccount { ModelRefreshPlanAccount { plan_type: plan_type.to_string(), - account, + access_token: account.access_token, + account_id: account.account_id, } } @@ -289,7 +289,7 @@ impl CodexModelCatalogClient for FakeModelCatalogClient { async fn fetch_models( &self, request: &CodexModelCatalogRequest<'_>, - ) -> Result, CodexModelCatalogClientError> { + ) -> Result, CodexModelCatalogClientError> { self.requested_plan_types .lock() .await @@ -298,6 +298,9 @@ impl CodexModelCatalogClient for FakeModelCatalogClient { .models_by_plan .get(request.plan_type) .cloned() - .unwrap_or_default()) + .unwrap_or_default() + .into_iter() + .map(|model| serde_json::to_value(model).unwrap()) + .collect()) } } diff --git a/backend/tests/models/store.rs b/backend/tests/models/store.rs index d3f411752..ca5df5b89 100644 --- a/backend/tests/models/store.rs +++ b/backend/tests/models/store.rs @@ -1,9 +1,6 @@ -use codex_proxy_rs::{ - models::{ - snapshot::ModelPlanSnapshot, - store::{ModelSnapshotStore, RedisModelSnapshotStore}, - }, - upstream::openai::protocol::model_catalog::BackendModelEntry, +use codex_proxy_rs::models::{ + store::{ModelSnapshotStore, RedisModelSnapshotStore}, + types::{BackendModelEntry, ModelPlanSnapshot}, }; use crate::support::storage::create_test_redis; diff --git a/backend/tests/telemetry/mod.rs b/backend/tests/telemetry/mod.rs index c24290dac..5acb4c7af 100644 --- a/backend/tests/telemetry/mod.rs +++ b/backend/tests/telemetry/mod.rs @@ -1 +1,3 @@ +pub(crate) mod ops; pub(crate) mod rebuild; +pub(crate) mod usage; diff --git a/backend/tests/telemetry/ops/mod.rs b/backend/tests/telemetry/ops/mod.rs new file mode 100644 index 000000000..3d6074864 --- /dev/null +++ b/backend/tests/telemetry/ops/mod.rs @@ -0,0 +1 @@ +mod store; diff --git a/backend/tests/api/admin/monitoring/ops_error_store.rs b/backend/tests/telemetry/ops/store.rs similarity index 100% rename from backend/tests/api/admin/monitoring/ops_error_store.rs rename to backend/tests/telemetry/ops/store.rs diff --git a/backend/tests/telemetry/usage/mod.rs b/backend/tests/telemetry/usage/mod.rs new file mode 100644 index 000000000..3d6074864 --- /dev/null +++ b/backend/tests/telemetry/usage/mod.rs @@ -0,0 +1 @@ +mod store; diff --git a/backend/tests/api/admin/monitoring/usage_record_store.rs b/backend/tests/telemetry/usage/store.rs similarity index 100% rename from backend/tests/api/admin/monitoring/usage_record_store.rs rename to backend/tests/telemetry/usage/store.rs diff --git a/backend/tests/upstream/openai/protocol/usage_rate_limits.rs b/backend/tests/upstream/openai/protocol/events.rs similarity index 100% rename from backend/tests/upstream/openai/protocol/usage_rate_limits.rs rename to backend/tests/upstream/openai/protocol/events.rs diff --git a/backend/tests/upstream/openai/protocol/mod.rs b/backend/tests/upstream/openai/protocol/mod.rs index 03c3a4f11..ab39d9a23 100644 --- a/backend/tests/upstream/openai/protocol/mod.rs +++ b/backend/tests/upstream/openai/protocol/mod.rs @@ -45,7 +45,7 @@ use serde_json::json; use crate::support::assertions::assert_substrings_appear_in_order; -mod codex_websocket; -mod openai_responses; -mod response_first_event; -mod usage_rate_limits; +mod events; +mod responses; +mod sse; +mod websocket; diff --git a/backend/tests/upstream/openai/protocol/openai_responses.rs b/backend/tests/upstream/openai/protocol/responses.rs similarity index 100% rename from backend/tests/upstream/openai/protocol/openai_responses.rs rename to backend/tests/upstream/openai/protocol/responses.rs diff --git a/backend/tests/upstream/openai/protocol/response_first_event.rs b/backend/tests/upstream/openai/protocol/sse.rs similarity index 100% rename from backend/tests/upstream/openai/protocol/response_first_event.rs rename to backend/tests/upstream/openai/protocol/sse.rs diff --git a/backend/tests/upstream/openai/protocol/codex_websocket.rs b/backend/tests/upstream/openai/protocol/websocket.rs similarity index 100% rename from backend/tests/upstream/openai/protocol/codex_websocket.rs rename to backend/tests/upstream/openai/protocol/websocket.rs diff --git a/backend/tests/upstream/openai/transport/headers.rs b/backend/tests/upstream/openai/transport/headers.rs index 406989b14..c0427135d 100644 --- a/backend/tests/upstream/openai/transport/headers.rs +++ b/backend/tests/upstream/openai/transport/headers.rs @@ -675,7 +675,7 @@ async fn codex_backend_client_models_should_use_original_auxiliary_headers() { .await .unwrap(); - assert_eq!(models[0].slug.as_deref(), Some("gpt-5.5")); + assert_eq!(models[0]["slug"], "gpt-5.5"); let requests = server.received_requests().await.unwrap(); let models_request = requests .iter() diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 72b332687..d11701d19 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -22,7 +22,7 @@ ENV CPR_GIT_SHA=${CPR_GIT_SHA} ENV CPR_BUILD_TIME=${CPR_BUILD_TIME} ENV CPR_BUILD_TYPE=${CPR_BUILD_TYPE} COPY backend/Cargo.toml backend/Cargo.lock ./ -COPY backend/build ./build +COPY backend/build.rs ./build.rs COPY backend/src ./src COPY release/version.yaml /app/release/version.yaml RUN --mount=type=cache,target=/usr/local/cargo/registry \ diff --git a/docs/architecture.md b/docs/architecture.md index 02f7b9f8b..4650ab537 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,16 +39,16 @@ | --- | --- | --- | --- | | L0 | `infra` | PostgreSQL 连接/迁移、Redis 连接/键前缀、时间、JSON、日志、身份生成、路径、格式化 | 出现任何业务概念 | | L1 | `upstream` | 纯上游客户端：协议编解码、HTTP/SSE/WebSocket 传输、指纹、OAuth token client。**每家 provider 一个子目录**（`openai/`，规划中 `cloudflare/`） | 依赖任何领域模块；出现账号池/调度/遥测概念 | -| L1 | `telemetry` | 遥测域：usage_records / ops_error_logs / request_time_buckets / account_usage 的**写入与查询**、billing、rebuild | 被冠以 admin 语义；HTTP handler | -| L1 | `keys` | 客户端 API key 域：store、鉴权校验、管理操作 | — | +| L1 | `telemetry` | 遥测域：usage_records / ops_error_logs / request_time_buckets / account_usage 的**写入与查询**、billing、dashboard 聚合、rebuild | 被冠以 admin 语义；HTTP handler | +| L1 | `keys` | 客户端 API key 域：store、内存缓存、鉴权校验、管理操作 | — | | L1 | `auth` | 管理端认证域：admin 用户（PG）、会话（Redis，token_hash 键，database.md §4B.1） | 客户端 key 概念（那是 `keys`） | -| L1 | `settings` | 运行时设置域：校验、持久化、并发串行化的当前快照、admin API key 校验 | 持有账号池、模型或刷新服务引用（§2.3） | +| L1 | `settings` | 运行时设置域：持久化、快照、变更广播（watch）、admin API key 校验 | 直接调用上层服务推送变更（§2.3） | | L1 | `update` | 自更新域：release 查询、下载、校验、解压、替换、状态 | — | | L2 | `accounts` | 账号域：实体、池、调度器、store、quota、token 刷新、Cookie、导入导出、管理操作 | — | | L2 | `models` | 模型目录域：catalog、别名、plan 快照 | — | | L3 | `dispatch` | 请求编排域：账号获取、affinity、上游调用、流转发、失败恢复、重试、遥测记录 | HTTP 类型（axum）出现在签名 | | L4 | `api` | 全部入站 HTTP：`api/client`（/v1）、`api/admin`（/api/admin）、路由组合、中间件、SPA 资源 | 写 SQL；承载领域逻辑 | -| 装配层 | `bootstrap` | 进程装配：启动配置、服务构造、AppState、后台任务、关闭；`api` 只读取其 AppState 门面 | 被 L0–L3 领域层依赖 | +| L5 | `bootstrap` | 进程装配：启动配置、服务构造、AppState、后台任务、关闭 | 被任何层依赖 | ### 2.2 依赖方向 @@ -57,8 +57,10 @@ | 允许的同层/特殊依赖 | 理由 | | --- | --- | | `accounts` → `models` | 候选过滤需要模型允许列表；单向 | +| `accounts` → `settings` | 池参数/刷新策略订阅设置快照（watch 类型）；单向 | +| `auth` → `settings` | admin API key 哈希存放在 runtime_settings 表，经 settings 的 store 方法读取；单向 | -**全库硬禁**（CI 脚本断言，§9）： +**全库硬禁**： - L1–L3 任何模块 import `crate::api` / `crate::bootstrap`。 - `dispatch`、`accounts`、`telemetry` 等领域模块 import `axum`。 @@ -68,7 +70,7 @@ ### 2.3 控制反转两处（消除向上依赖） -1. **设置传播**：`RuntimeSettingsService` 只负责校验、持久化和当前快照，不持有任何上层消费者。`bootstrap::Services::update_settings` 在持久化成功后同步把同一快照应用到账号池、刷新策略与模型配置；更新串行执行，不保留无人消费的 watch 通道，也不存在低层向上回调。 +1. **设置传播**：现状 `RuntimeSettingsService` 持有账号池与刷新策略的引用、主动推送更新（低层持有高层引用）。终态：`settings` 只暴露 `tokio::sync::watch::Receiver`；`accounts`（池、刷新策略）、`models`（别名）各自订阅。装配在 `bootstrap` 完成。 2. **遥测写入**：现状 proxy 热路径 import `admin::monitoring` 的 Admin* 服务。终态：`dispatch` 依赖 `telemetry::Recorder`（L3 → L1，方向正确），admin 侧只是同一域的查询消费者。 --- @@ -77,58 +79,117 @@ ```text backend/src/ -├── main.rs lib.rs -├── infra/ # database.rs、redis.rs、migrations/ 与通用基础能力 -├── upstream/openai/ # 纯 OpenAI/Codex client -│ ├── token_client.rs -│ ├── protocol/ # schema / responses / events / SSE / WebSocket -│ ├── transport/ # HTTP、TLS、WebSocket 与连接池；大文件可按职责建子文件 -│ └── fingerprint/ # types / store / runtime / updater -├── telemetry/ # 四类账本与查询 -│ ├── recorder.rs # 成功/失败事实的唯一领域写入口 -│ ├── billing.rs diagnostics.rs rebuild.rs -│ ├── usage/ # types.rs、query.rs、store.rs、store/query.rs -│ ├── ops/ # types.rs、query.rs、store.rs -│ ├── buckets/ # query.rs、store.rs -│ └── account_usage/ # query.rs、store.rs -├── keys/ # types / PG store / verifier / manage -├── auth/ # types / PG+Redis store / session service -├── settings/ # types / PG store / serialized current-snapshot service -├── update/ # types / state / release / download / archive / service -├── accounts/ -│ ├── account.rs service.rs import.rs window.rs -│ ├── pool.rs pool/{runtime,types}.rs -│ ├── scheduler.rs scheduler/{candidates,feedback,strategy/...}.rs -│ ├── store.rs store/{queries,rows,write}.rs -│ ├── quota/{service,runtime}.rs -│ ├── refresh/{lease,service,runtime}.rs refresh/runtime/types.rs -│ ├── cookies/store.rs -│ └── manage/{service,types,lifecycle,import,export,oauth,probe,quota,quota_view}.rs -├── models/ # config / info / snapshot / catalog / service / Redis store -├── dispatch/ -│ ├── errors.rs upstream.rs cloudflare.rs exhaustion.rs -│ ├── implicit_resume.rs reasoning_replay.rs -│ ├── affinity/{types,identity,store,service,resolve}.rs -│ └── responses/ # Responses 编排是一个完整子能力，不做无收益扁平化 -│ ├── service.rs service/{complete,stream,compact}.rs -│ ├── errors.rs event_recording.rs live_stream.rs -│ └── prefetch.rs sse_failure.rs stream_lifecycle.rs trace.rs -├── api/ -│ ├── router.rs assets.rs middleware/ -│ ├── client/ # /v1 路由、鉴权与协议响应 -│ └── admin/ # /api/admin handlers 与 DTO -│ ├── accounts_routes.rs accounts_routes/{actions,stats}.rs -│ ├── dashboard_routes.rs dashboard_routes/data.rs -│ └── keys/ops/settings/system/usage 等 routes -└── bootstrap/ - ├── application.rs config.rs services.rs state.rs shutdown.rs - ├── import_sqlite.rs import_sqlite/{core,telemetry}.rs - └── tasks/ # periodic、retention、token/quota/model refresh 等装配任务 +├── main.rs # 子命令入口：serve（默认）/ import-sqlite / rebuild-buckets +├── lib.rs +├── infra/ # L0 +│ ├── mod.rs +│ ├── database.rs # PgPool 连接/池参数、迁移框架（PG 谱系 0001 终态基线）、ping +│ ├── redis.rs # Redis ConnectionManager、键前缀、ping（database.md §1） +│ ├── migrations/ # *.sql（PG） +│ ├── identity.rs time.rs json.rs format.rs logging.rs paths.rs +├── upstream/ # L1 纯上游客户端（每家 provider 一个子目录） +│ ├── mod.rs # 第二家 provider 出现时，UpstreamClient trait 落此 +│ └── openai/ +│ ├── mod.rs +│ ├── token_client.rs +│ ├── protocol/ +│ │ ├── mod.rs schema.rs responses.rs events.rs sse.rs websocket.rs +│ ├── transport/ +│ │ ├── mod.rs client.rs endpoints.rs headers.rs tls.rs +│ │ ├── response_meta.rs usage.rs diagnostics.rs +│ │ ├── websocket.rs websocket_pool.rs websocket_pump.rs +│ └── fingerprint/ +│ ├── mod.rs types.rs store.rs runtime.rs updater.rs +├── telemetry/ # L1 遥测域（原 admin/monitoring + proxy/dispatch/usage_events） +│ ├── mod.rs +│ ├── recorder.rs # 唯一写入口：成功/失败两条 PG 事务（database.md §5.2） +│ ├── billing.rs +│ ├── rebuild.rs # rebuild-buckets（database.md §7） +│ ├── dashboard.rs # Dashboard 聚合查询（无 HTTP；口径 database.md §5.3） +│ ├── usage/ # usage_records +│ │ ├── mod.rs types.rs store.rs query.rs +│ ├── ops/ # ops_error_logs（含 /api/admin/ops/errors 的查询面） +│ │ ├── mod.rs types.rs store.rs query.rs +│ ├── buckets/ # request_time_buckets +│ │ ├── mod.rs store.rs query.rs +│ └── account_usage/ # account_usage / account_model_usage +│ ├── mod.rs store.rs query.rs +├── keys/ # L1 客户端 key 域（原 admin/keys/service.rs 拆分） +│ ├── mod.rs types.rs store.rs service.rs manage.rs +│ │ # 鉴权 = 完整 key → PG unique 点查（database.md §4.3），无进程内鉴权缓存 +├── auth/ # L1 管理端认证域（原 admin/auth/service.rs） +│ ├── mod.rs types.rs store.rs service.rs # store = PG admin_users + Redis 会话键（§4B.1） +├── settings/ # L1 运行时设置域（原 config/settings.rs） +│ ├── mod.rs types.rs store.rs service.rs # service 持 watch 广播 +├── update/ # L1 自更新域（原 admin/system/*） +│ ├── mod.rs types.rs state.rs release.rs download.rs archive.rs service.rs +├── accounts/ # L2 账号域（原 upstream/accounts + upstream/scheduler + admin/accounts/service） +│ ├── mod.rs +│ ├── account.rs # 实体（原 model.rs） +│ ├── window.rs +│ ├── pool.rs # 池：存储态/槽位/过滤（§5.4 行数预算内需拆出 pool/ 子文件） +│ ├── scheduler.rs # 调度门面（原 scheduler/mod.rs 的类型） +│ ├── scheduler/ +│ │ ├── candidates.rs feedback.rs +│ │ └── strategy/ +│ │ ├── mod.rs smart.rs quota_reset.rs round_robin.rs sticky.rs +│ ├── store.rs # AccountStore trait + PG 门面 +│ ├── store/ +│ │ ├── queries.rs rows.rs +│ ├── quota/ +│ │ ├── mod.rs service.rs runtime.rs +│ ├── refresh/ # 原 token_refresh +│ │ ├── mod.rs policy.rs lease.rs service.rs # lease = Redis SET NX PX（§4B.2） +│ ├── cookies/ +│ │ ├── mod.rs store.rs +│ ├── import.rs # 导入 payload 解析（原 upstream/accounts/importing.rs） +│ └── manage/ # 管理操作（原 admin/accounts/service/*） +│ ├── mod.rs types.rs service.rs lifecycle.rs +│ ├── import.rs export.rs oauth.rs probe.rs quota.rs quota_view.rs +├── models/ # L2 模型目录域 +│ ├── mod.rs types.rs catalog.rs service.rs store.rs # store = Redis 快照 HASH（§4B.4） +├── dispatch/ # L3 请求编排域（原 proxy/dispatch） +│ ├── mod.rs +│ ├── service.rs # 编排入口（原 responses/service.rs，按 §5.4 拆分） +│ ├── upstream_call.rs # 原 dispatch/upstream.rs +│ ├── errors.rs # 原 dispatch/errors.rs + responses/errors.rs 合并 +│ ├── recording.rs # 原 responses/event_recording.rs，调 telemetry::recorder +│ ├── affinity/ # 原 session_affinity.rs（683 行三合一）+ responses/affinity.rs +│ │ ├── mod.rs types.rs identity.rs store.rs service.rs resolve.rs +│ │ │ # store = Redis 键 + 二级索引（§4B.3），无内存映射副本 +│ ├── recovery/ +│ │ ├── mod.rs auth.rs cloudflare.rs exhaustion.rs +│ │ ├── implicit_resume.rs reasoning_replay.rs +│ └── stream/ +│ ├── mod.rs live.rs lifecycle.rs prefetch.rs sse_failure.rs trace.rs +├── api/ # L4 全部入站 HTTP（原 http + web + proxy 入口 + admin routes） +│ ├── mod.rs +│ ├── router.rs # 总路由 + healthz（infra 的 PG ping + Redis ping，任一失败 503） +│ ├── assets.rs # SPA 静态资源（原 web/assets.rs） +│ ├── middleware/ +│ │ ├── mod.rs request_id.rs trace.rs +│ ├── client/ # /v1（原 proxy/auth + proxy/openai） +│ │ ├── mod.rs router.rs auth.rs +│ │ ├── responses.rs models.rs errors.rs sse.rs +│ └── admin/ # /api/admin（原 admin 的全部 routes 与 DTO） +│ ├── mod.rs router.rs response.rs session.rs # session = 鉴权提取器/中间件 +│ ├── auth_routes.rs settings_routes.rs system_routes.rs +│ ├── accounts_routes.rs keys_routes.rs +│ ├── usage_routes.rs ops_routes.rs dashboard_routes.rs # ops_routes = /api/admin/ops/errors +└── bootstrap/ # L5 进程装配（原 runtime + config/schema,loader） + ├── mod.rs + ├── config.rs # 启动配置 schema + loader（原 config/schema.rs + loader.rs；含 redis.url） + ├── services.rs state.rs shutdown.rs + ├── import_sqlite.rs # 一次性 SQLite v3 → PG 导入命令（database.md §6；§5.5 的唯一跨域写豁免） + └── tasks/ + ├── mod.rs coordinator.rs periodic.rs cleanup.rs + ├── cookie_cleanup.rs retention_trim.rs # retention_trim 新增（database.md §7） + ├── model_refresh.rs token_refresh.rs quota_refresh.rs + ├── fingerprint_update.rs + # session_cleanup / session_affinity_cleanup 不复存在：Redis TTL 接管（database.md §4B） ``` -这是已落地结构，不是待办蓝图。为控制单文件行数而增加的同名子目录属于同一能力，不构成抽象层。Dashboard 的跨域只读组合保留在 `api/admin/dashboard_routes/data.rs`：它只组装 presentation read model，不写 SQL、不改变领域状态；把它下沉到 `telemetry` 反而会制造 telemetry → accounts 的向上依赖。 - -`backend/tests/` 顶层按 `src/` 领域镜像；子目录可按公开路由或行为场景组织，因此 `monitoring`、`token_refresh` 等历史行为名称可保留在测试中。`fixtures/`、`support/` 是测试基础设施，不要求与生产文件逐一同名。 +`backend/tests/` 目录结构**镜像 `src/`**：模块搬家时测试目录同步搬家，`fixtures/`、`support/` 不动。 --- @@ -187,7 +248,7 @@ backend/src/ ### 5.2 存储边界（SQL 与 Redis 同规） -业务 SQL 与 Redis 命令只允许出现在 `store.rs` / `store/` / `query.rs`。明确的基础设施/维护例外只有：`infra/database.rs`（迁移与 ping）、`infra/redis.rs`（连接与 ping）、`bootstrap/import_sqlite*`（一次性跨库导入）、`telemetry/rebuild.rs`（账本重建命令）、`accounts/refresh/lease.rs`（Redis 租约适配器）。`api` 层与领域 `service.rs` 出现存储命令即为违规；healthz 只调用 infra 的 ping 门面。 +SQL 文本与 Redis 命令只允许出现在 `store.rs` / `store/` / `query.rs`。`api` 层与 `service.rs` 出现 `sqlx::query` 或 `redis::` 调用即为违规（现状 `http/router.rs` 的 healthz 直接拿某个 store 的连接池执行裸 SQL——终态 `infra::database` 提供 `ping(pool)`、`infra::redis` 提供 `ping(conn)`，router 只调它们）。 ### 5.3 HTTP 边界 @@ -195,21 +256,21 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 ### 5.4 文件行数预算 -- 软上限 **400 行**（超过即在 review 中说明理由）；硬上限 **800 行**（超过必须拆，CI 直接阻断，§9）。 -- 迁移前超硬线清单（11 个）及最终落点： +- 软上限 **400 行**（超过即在 review 中说明理由）；硬上限 **800 行**（超过必须拆）。 +- 现状超硬线清单（11 个，全部有既定拆法）： | 文件（现路径） | 行数 | 拆法 | | --- | --- | --- | -| `proxy/dispatch/responses/service.rs` | 1913 | → `dispatch/responses/service.rs` + `service/{complete,stream,compact}.rs` | -| `upstream/transport/websocket.rs` | 1470 | → `upstream/openai/transport/websocket.rs` + `websocket/{stream,types}.rs` + `websocket_pump.rs` | -| `admin/accounts/routes.rs` | 1455 | → `api/admin/accounts_routes.rs` + `accounts_routes/{actions,stats}.rs`；领域操作进入 `accounts/manage` | -| `upstream/accounts/pool.rs` | 1420 | → `accounts/pool.rs` + `pool/{runtime,types}.rs` | -| `upstream/transport/client.rs` | 1251 | → `upstream/openai/transport/client.rs` + `client/requests.rs` | -| `upstream/accounts/store/mod.rs` | 1126 | → `accounts/store.rs` + `store/{queries,rows,write}.rs` | -| `admin/monitoring/usage_record_store.rs` | 1114 | → `telemetry/usage/store.rs` + `store/query.rs`；查询门面独立为 `usage/query.rs` | -| `admin/monitoring/dashboard.rs` | 1080 | → 账本查询进入 `telemetry`，跨域 presentation 组合进入 `api/admin/dashboard_routes/{data}.rs` | -| `upstream/protocol/websocket.rs` | 1033 | → `upstream/openai/protocol/websocket.rs` + `websocket/validation.rs` | -| `upstream/accounts/token_refresh/runtime.rs` | 953 | → `accounts/refresh/{service,lease,runtime}.rs` + `runtime/types.rs` | +| `proxy/dispatch/responses/service.rs` | 1913 | → `dispatch/service.rs`（编排骨架）+ `upstream_call.rs` + `stream/*` + `recovery/*` 已分走的逻辑归位 | +| `upstream/transport/websocket.rs` | 1470 | → `websocket.rs`（连接/握手）+ `websocket_frames.rs`（帧编解码） | +| `admin/accounts/routes.rs` | 1455 | → `api/admin/accounts_routes.rs` 按子资源拆（accounts / oauth / quota / import-export） | +| `upstream/accounts/pool.rs` | 1420 | → `accounts/pool.rs`（门面）+ `pool/state.rs`（槽位/在途）+ `pool/filters.rs` | +| `upstream/transport/client.rs` | 1251 | → `client.rs` + `client_sse.rs`（SSE 请求路径） | +| `upstream/accounts/store/mod.rs` | 1126 | → `store.rs` 门面 + `store/queries.rs` 按读写再分 | +| `admin/monitoring/usage_record_store.rs` | 1114 | PR0 原地重写（拆表后 metadata 聚合全数删除，行数大减），PR1 搬家时拆 `telemetry/usage/{store,query}.rs` | +| `admin/monitoring/dashboard.rs` | 1080 | → `telemetry/dashboard.rs`（聚合）+ `api/admin/dashboard_routes.rs`（HTTP） | +| `upstream/protocol/websocket.rs` | 1033 | → `websocket.rs`（事件转换）+ `websocket_errors.rs`（错误帧解析） | +| `upstream/accounts/token_refresh/runtime.rs` | 953 | → `accounts/refresh/{service,lease,policy}.rs` | | `admin/system/updater/mod.rs` | 811 | → `update/{service,state}.rs` | ### 5.5 一域一账本 @@ -233,7 +294,7 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 | `cpr:affinity:*` | `dispatch`（affinity/store.rs） | | `cpr:models:plan_snapshots` | `models` | -跨域读通过属主域的 query/service 方法，不直接跨域写。**跨域写唯一豁免**：`bootstrap/import_sqlite*`（一次性导入命令，database.md §6）按依赖序直写全部 PG 表；`telemetry/rebuild.rs` 仍只写 telemetry 自有账本，不是跨域豁免。v3 源库退役后导入模块可删。 +跨域读通过属主域的 query/service 方法，不直接跨域写。**唯一豁免**：`bootstrap/import_sqlite.rs`（一次性导入命令，database.md §6）按依赖序直写全部 PG 表；v3 源库退役后该文件可删。 --- @@ -243,7 +304,7 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 1. **热路径反向依赖管理端**：`proxy/dispatch/usage_events.rs`、`event_recording.rs` import `admin::monitoring` 的 `Admin*Service` 写账本；`runtime/services.rs` 用 Admin 命名的服务装配核心链路。遥测是领域层，管理端只是它的一个读者。→ 建 `telemetry` 域（PR1）。 2. **核心编排寄居 proxy**：`proxy/dispatch/`（约 5600 行，含 1913 行 service）是系统心脏，却挂在客户端 HTTP 入口下；`SqliteSessionAffinityStore`（DB store）住在 proxy 里。→ 建 `dispatch` 域（PR4）。 -3. **config 模块越权**：`config/settings.rs`（601 行）= 持久化 + 热更新推送 + admin API key 校验，还持有账号池引用向上推送。config 应只剩启动 schema + loader。→ 建 `settings` 域，由 bootstrap 在持久化成功后统一应用快照（PR2）。 +3. **config 模块越权**：`config/settings.rs`（601 行）= 持久化 + 热更新推送 + admin API key 校验，还持有账号池引用向上推送。config 应只剩启动 schema + loader。→ 建 `settings` 域 + watch 反转（PR2）。 4. **admin/keys 四合一**：`admin/keys/service.rs`（698 行）同时装着 proxy 鉴权用的 `ClientKeyService`、运行时缓存、SQLite store、管理端 service——客户端鉴权路径穿过 admin 模块。→ 建 `keys` 域（PR2）。 5. **web/http 假分家**：`web/` 只有 69 行 assets，是路由的 fallback 组成部分。→ 并入 `api`（PR5）。 6. healthz 穿透三层拿 store 连接池执行裸 SQL（§5.2）。 @@ -291,17 +352,17 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 | `proxy/dispatch/usage_events.rs` | `telemetry/recorder.rs` †（Admin* 类型消失） | | `proxy/dispatch/session_affinity.rs` | `dispatch/affinity/{types,store,service}.rs` † | | `proxy/dispatch/responses/affinity.rs` | `dispatch/affinity/resolve.rs` | -| `proxy/dispatch/responses/service.rs` | `dispatch/responses/service.rs` + `service/{complete,stream,compact}.rs` † | -| `proxy/dispatch/responses/{live_stream,stream_lifecycle,prefetch,sse_failure,trace}.rs` | `dispatch/responses/` 下同名文件 | -| `proxy/dispatch/responses/event_recording.rs` | `dispatch/responses/event_recording.rs`（只依赖 `telemetry::Recorder`） | -| `proxy/dispatch/responses/errors.rs` + `proxy/dispatch/errors.rs` | `dispatch/responses/errors.rs` + `dispatch/errors.rs`（协议编排错误与通用调度错误分开） | -| `proxy/dispatch/{auth_recovery,cloudflare,exhaustion,implicit_resume,reasoning_replay}.rs` | 通用逻辑保留在 `dispatch/` 根；Responses 专属恢复留在 `dispatch/responses/` | -| `proxy/dispatch/upstream.rs` | `dispatch/upstream.rs` | +| `proxy/dispatch/responses/service.rs` | `dispatch/service.rs` † | +| `proxy/dispatch/responses/{live_stream,stream_lifecycle,prefetch,sse_failure,trace}.rs` | `dispatch/stream/{live,lifecycle,prefetch,sse_failure,trace}.rs` | +| `proxy/dispatch/responses/event_recording.rs` | `dispatch/recording.rs` | +| `proxy/dispatch/responses/errors.rs` + `proxy/dispatch/errors.rs` | `dispatch/errors.rs` † | +| `proxy/dispatch/{auth_recovery,cloudflare,exhaustion,implicit_resume,reasoning_replay}.rs` | `dispatch/recovery/{auth,cloudflare,exhaustion,implicit_resume,reasoning_replay}.rs` | +| `proxy/dispatch/upstream.rs` | `dispatch/upstream_call.rs` | | `admin/monitoring/usage_record_{model,store,service}.rs` | `telemetry/usage/{types,store,query}.rs` †（重写已在 PR0 完成，此处纯搬家） | | `admin/monitoring/ops_error_{model,store,service}.rs` | `telemetry/ops/{types,store,query}.rs` † | | `admin/monitoring/account_usage_{store,service}.rs` | `telemetry/account_usage/{store,query}.rs` † | | `admin/monitoring/billing.rs` | `telemetry/billing.rs` | -| `admin/monitoring/dashboard.rs` | 各账本查询进入 `telemetry/*/query.rs`；跨域只读组合进入 `api/admin/dashboard_routes/{data}.rs` † | +| `admin/monitoring/dashboard.rs` | `telemetry/dashboard.rs` + `api/admin/dashboard_routes.rs` † | | `admin/monitoring/diagnostics.rs` | `telemetry/diagnostics.rs` | | `admin/monitoring/usage_record_routes.rs` | `api/admin/usage_routes.rs` | | `admin/keys/service.rs` | `keys/{types,store,service,manage}.rs` †（进程内鉴权缓存消亡，database.md §4.3） | @@ -311,7 +372,7 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 | `admin/accounts/routes.rs` | `api/admin/accounts_routes.rs` †（按子资源拆） | | `admin/accounts/quota_view.rs` | `accounts/manage/quota_view.rs` | | `admin/accounts/service/*` | `accounts/manage/*`（`importing`→`import`、`exporting`→`export`、`testing`→`probe`、`contracts`→`types`） | -| `admin/{response,update_payload}.rs` | `api/admin/{response,update_payload}.rs` | +| `admin/{response,update_payload}.rs` | `api/admin/response.rs`、`update/types.rs` | | `admin/router.rs` | `api/admin/router.rs` | | `admin/settings/routes.rs` | `api/admin/settings_routes.rs` | | `admin/system/{routes,state}.rs`、`admin/system/updater/*` | `api/admin/system_routes.rs`、`update/{state,service,release,download,archive}.rs` † | @@ -319,12 +380,12 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 | `upstream/accounts/{pool,window}.rs` | `accounts/{pool,window}.rs`（pool 拆 †） | | `upstream/accounts/store/*` | `accounts/store.rs` + `accounts/store/{queries,rows}.rs` † | | `upstream/accounts/quota/*` | `accounts/quota/{service,runtime}.rs` † | -| `upstream/accounts/token_refresh/*` | `accounts/refresh/{lease,service,runtime}.rs` + `runtime/types.rs` † | +| `upstream/accounts/token_refresh/*` | `accounts/refresh/{policy,lease,service}.rs` † | | `upstream/accounts/cookies.rs` | `accounts/cookies/store.rs` | | `upstream/accounts/importing.rs` | `accounts/import.rs` | -| `upstream/accounts/service.rs` | `accounts/service.rs` | +| `upstream/accounts/service.rs` | 并入 `accounts/mod.rs` 门面导出（32 行） | | `upstream/scheduler/*` | `accounts/scheduler.rs` + `accounts/scheduler/*`（mod.rs 门面类型移出） | -| `upstream/models/*` | `models/{config,info,snapshot,catalog,service,store}.rs`（各文件保留真实职责） | +| `upstream/models/*` | `models/{types,catalog,service,store}.rs`（`backend_entry`+`info`+`snapshot`+`config` 并入 `types.rs`/`service.rs`） | | `upstream/fingerprint.rs` | `upstream/openai/fingerprint/{types,store,runtime,updater}.rs` †（Repository→Store） | | `upstream/{protocol,transport}/*`、`upstream/token_client.rs` | `upstream/openai/` 下同名路径（websocket/client 按 §5.4 拆） | | `infra/*` | 原地保留 | @@ -334,41 +395,28 @@ axum/tower 类型只允许出现在 `api` 与 `bootstrap`。领域函数签名 ## 8. 执行记录（PR0 + 六个顺序搬家阶段） -以下阶段已于 **2026-07-10** 全部完成。固定验收：`cargo fmt --check`、`cargo clippy -- -D warnings`、`cargo test`、`backend/scripts/check_architecture.sh`、`backend/tests` 镜像目录同步搬移。**PR0 是唯一的迁移行为变更阶段**（存储引擎与 schema 语义一次到位）；PR1 起为搬家、更名与拆分。 +以下阶段已于 **2026-07-10** 全部完成。固定验收：`cargo fmt --check`、`cargo clippy -- -D warnings`、`cargo test`、`backend/tests` 镜像目录同步搬移。**PR0 是唯一的迁移行为变更阶段**（存储引擎与 schema 语义一次到位）；PR1 起为搬家、更名与拆分。 | PR | 内容 | 关键动作 | | --- | --- | --- | | **PR0** | **存储终态切换**（database.md §7 全部 12 条，在现有目录结构内完成） | Cargo：sqlx 加 postgres、新增 redis；`infra/database.rs` 改 PG + 0001 终态基线、新增 `infra/redis.rs`；全部 store 换 PG 方言 / 运行态四键空间改 Redis；成功/失败事实拆表 + provider 维度 + 两条写入事务；client key 可取回、session/admin-key 哈希化 + `client_api_key_id` 归因贯通；Dashboard 口径换源；trim 周期任务化；`import-sqlite` / `rebuild-buckets` 子命令；tests support 换 PG+Redis;deploy compose 加双服务。验收对齐 database.md §6.4 | | **PR1** | `telemetry` 域成立 | monitoring 全部迁出 admin；`usage_events.rs` → `recorder.rs`；Admin* 遥测类型按 §4.3 更名；dispatch/api 改依赖 telemetry；启用"禁止 `crate::admin::monitoring`"检查 | -| **PR2** | `keys` / `auth` / `settings` / `update` 域成立 | 四个域从 admin/config 拆出；settings 只保留持久化与串行快照，消费者传播由 bootstrap 统一执行（§2.3）；config 只剩启动 schema+loader | +| **PR2** | `keys` / `auth` / `settings` / `update` 域成立 | 四个域从 admin/config 拆出；settings 改 watch 广播（§2.3）；config 只剩 schema+loader | | **PR3** | `accounts` / `models` 域成立 | `upstream/accounts`、`upstream/scheduler`、`admin/accounts/service` 三处合并为 `accounts`；`upstream` 缩为纯客户端并落成 `openai/` 子目录；动名词文件更名；Repository→Store | -| **PR4** | `dispatch` 域成立 | proxy/dispatch 整体迁出；保留内聚的 `responses` 子能力并按行数拆 service 子文件；affinity store 独立成文件；`proxy` 只剩 api/client 的料 | +| **PR4** | `dispatch` 域成立 | proxy/dispatch 整体迁出并按目标树重组；affinity store 独立成文件；`proxy` 只剩 api/client 的料 | | **PR5** | `api` / `bootstrap` 壳成立 | http + web + proxy 入口 + admin routes 合并为 `api`；runtime + config 装配部分改名 `bootstrap`；删除 `crate::{admin,proxy,runtime,web,http,config}` 六个旧根模块；启用全部依赖检查 | -| **PR6** | 行数预算清账 | §5.4 清单内 11 个文件拆到硬线以下；启用行数预算 CI 阻断 | +| **PR6** | 行数预算清账 | §5.4 清单内 11 个文件拆到硬线以下 | 顺序依据：存储切换触及所有 store、无法按域切片，必须整体先行（PR0）——先切引擎再搬家，每个 store 文件只被"重写"一次、"移动"一次，不叠加；PR1–4 自底向上消除旧模块的存在必要；PR5 收壳；PR6 纯内部拆分无跨域影响。原"PR1 与数据库改造互锁"的约定由 PR0 替代，PR2 中的哈希化条目亦已并入 PR0。 --- -## 9. CI 强制 +## 9. 仓库级约定 -`backend/scripts/check_architecture.sh` 是可执行规则的唯一来源，并由 `.github/workflows/ci.yml` 在 clippy 后调用；本文不复制脚本，避免规则双写漂移。当前强制项包括： - -- 领域层不得依赖 `api` / `bootstrap`，且同时识别 `crate::api::...` 与 `use crate::{ api::... }` 两种写法；`upstream` 不得依赖任何领域模块。 -- axum 只进入 `api` / `bootstrap`；API 不得执行 SQL 或 Redis 命令。 -- 存储命令只能位于 §5.2 列出的 adapter/query/维护入口；文件名、类型前缀、旧模块路径、Repository 命名与 Arc 别名均有阻断。 -- `mod.rs` 不超过 50 行，其他 Rust 文件不超过 800 行。 - -全部规则已启用；后续新增代码一旦恢复旧路径、越层依赖、违规命名或突破硬行数上限，CI 直接失败。 - ---- - -## 10. 仓库级约定 - -- `backend/tests/` 顶层镜像 `src/` 领域，不在 `src/` 写测试模块。测试依赖本机（或 CI 服务容器）的 PostgreSQL 与 Redis：`CPR_TEST_DATABASE_URL` 下每测试建独立数据库、`CPR_TEST_REDIS_URL` 下每测试随机键前缀（database.md §7）。 -- `architecture.md`（本文）与 `database.md` 是架构/存储权威；`docs/` 可保留故障复盘或修复记录，但不得另行定义架构、schema 或运行契约。行为实现细节以模块文档注释为准。 +- `backend/tests/` 镜像 `src/`，不在 `src/` 写测试模块。测试依赖本机（或 CI 服务容器）的 PostgreSQL 与 Redis：`CPR_TEST_DATABASE_URL` 下每测试建独立数据库、`CPR_TEST_REDIS_URL` 下每测试随机键前缀（database.md §7）。 +- `docs/` 只保留权威文档：`architecture.md`（本文）、`database.md`。行为细节（调度策略、WS 池、更新流程等）以模块文档注释为准（`scheduler/mod.rs` 是范本），不再在 docs 里维护第二份会腐化的副本。 - `deploy/` 随 PR0 扩容（compose 增 postgres/redis 服务与健康检查依赖）；`release/`、`skills/` 现状合格，不动。 -- 前端联动（全部随 PR0）：keys 列表继续返回完整 key，复制与 CCSwitch 导入长期保持可用且交互/文案不变；admin API key 状态不再有脱敏值（哈希不可逆，只余"已启用"）；成功记录响应不再有 `level` 字段；错误明细/筛选改走 `/api/admin/ops/errors`；Dashboard 只修正取数来源，既有卡片文案不变。 +- 前端联动（全部随 PR0）：keys 列表继续返回完整 key，并长期提供复制与 CCSwitch 导入；admin API key 状态不再有脱敏值（哈希不可逆，只余"已启用"）；成功记录响应不再有 `level` 字段；错误明细/筛选改走 `/api/admin/ops/errors`；Dashboard "总请求/总 token" 卡片改为 account_usage 来源并标注"成功累计"。 --- diff --git a/docs/database.md b/docs/database.md index 39f8c635a..0b949ff7e 100644 --- a/docs/database.md +++ b/docs/database.md @@ -772,7 +772,7 @@ codex-proxy-rs import-sqlite <旧库路径.sqlite> 1. **连接层**(`infra/database.rs` + `infra/redis.rs`):PgPool(参数见 §1)+ 迁移框架(PG 谱系 0001 起);Redis ConnectionManager + 键前缀 + `PING` 健康检查。`Cargo.toml`:sqlx 加 `postgres`(保留 `sqlite` 仅供导入命令读源库),新增 `redis` crate。 2. **配置**(`bootstrap/config.rs`、`deploy/config.example.yaml`):`database.url` 改 `postgres://…`,新增 `redis.url`。 3. **鉴权归因**(`keys/{store,service,manage}.rs`、`api/client/auth.rs`):key 创建落唯一 `key` 列；鉴权按完整 key 做 PG unique 点查(删除进程内鉴权缓存);鉴权返回 key `id`,调用链把 `client_api_key_id` 装进请求上下文直至事件写入。管理端列表持续返回完整 key，前端长期保留复制与 CCSwitch 导入入口。 -4. **事件结构**(`telemetry/{usage,ops}`、`telemetry/recorder.rs`、`dispatch/responses/event_recording.rs`):成功/失败事件携带 `client_api_key_id` 与 `provider`;token、requested/upstream_model、service_tier、first_token_ms 从 metadata 移到一等字段;metadata 不再写已提升字段;UsageRecord 删除 `level`(成功事实无等级)。 +4. **事件结构**(`telemetry/{usage,ops}`、`telemetry/recorder.rs`、`dispatch/recording.rs`):成功/失败事件携带 `client_api_key_id` 与 `provider`;token、requested/upstream_model、service_tier、first_token_ms 从 metadata 移到一等字段;metadata 不再写已提升字段;UsageRecord 删除 `level`(成功事实无等级)。 5. **写入路径**(`telemetry/usage/store.rs`、`telemetry/ops/store.rs`、`telemetry/buckets/store.rs`):§5.2 的两条 PG 事务;错误路径对桶只 `error_count + 1`;`__unknown__` sentinel;min/max 延迟使用 nullable-safe 的 `least()`/`greatest()`;内联 trim 移除。 6. **查询路径**(`telemetry/{usage,ops,account_usage}`、`api/admin/{usage,ops,dashboard,accounts}_routes*`):summary/分布/趋势全部走列,删除所有 `->>` 聚合与 metadata LIKE;新增 `/api/admin/ops/errors` 错误明细查询面;Dashboard 按 §5.3 只改取数来源，不改既有文案。 7. **运行态改 Redis**(`auth/store.rs` 会话、`accounts/refresh/lease.rs`、`dispatch/affinity/store.rs`、`models/store.rs`):按 §4B 契约实现;删除 admin 会话/亲和清理任务与亲和重启恢复;账号删除路径挂 Redis 亲和级联(§4B.3)。