From bccbe2a8c5efca0d42abee87cdbb2396ddc7ebf0 Mon Sep 17 00:00:00 2001 From: zyycn Date: Mon, 29 Jun 2026 22:19:43 +0800 Subject: [PATCH] feat: align account scheduling and quota updates Co-authored-by: Codex --- docs/ws-ts-alignment.md | 101 ++++++- src/proxy/dispatch/chat.rs | 8 +- src/proxy/dispatch/responses.rs | 259 +++++++++++++++++- src/runtime/tasks/coordinator.rs | 1 + src/runtime/tasks/quota_refresh.rs | 7 + src/upstream/accounts/pool.rs | 131 +++++---- src/upstream/accounts/quota/runtime.rs | 29 +- src/upstream/accounts/store.rs | 9 +- src/upstream/protocol/responses.rs | 52 +++- src/upstream/transport/client.rs | 20 +- .../dispatch/chat_upstream/chat_routes.rs | 60 ++++ .../chat_upstream/responses_websocket.rs | 17 ++ .../dispatch/chat_upstream/usage_logging.rs | 114 +++++++- tests/runtime/account_pool_restore/mod.rs | 32 ++- tests/upstream/accounts/account_pool/quota.rs | 29 +- .../accounts/account_pool/selection.rs | 229 ++++++++++++++++ tests/upstream/accounts/account_repository.rs | 78 ++++++ tests/upstream/accounts/quota_refresh.rs | 62 +++++ .../accounts/components/AccountQuotaPanel.vue | 42 ++- .../components/AccountOverviewCard.vue | 4 +- web/src/views/settings/index.vue | 10 +- 21 files changed, 1175 insertions(+), 119 deletions(-) diff --git a/docs/ws-ts-alignment.md b/docs/ws-ts-alignment.md index b88b7c858..9185dde95 100644 --- a/docs/ws-ts-alignment.md +++ b/docs/ws-ts-alignment.md @@ -110,23 +110,34 @@ base_url + account_id + prompt_cache_key/client_conversation_id/previous_respons ## 请求头差异 -TS 版本在 WebSocket 和 HTTP SSE 链路中使用: +本轮按 `/home/zyy/桌面/Codes/codex-proxy` 当前源码和 `docs/flows.all.txt` 抓包重新核对,结论需要区分 TS 参考实现和官方 Codex Desktop 真实链路。 + +TS 版本当前在 WebSocket 和 HTTP SSE 链路中使用: ```text -session_id: x-client-request-id: +session_id: +``` + +官方 Codex Desktop 抓包中的 WebSocket 请求使用: + +```text +x-client-request-id: +session-id: +thread-id: ``` Rust 版本当前上游请求头使用: ```text +x-client-request-id: session-id: thread-id: ``` 同时 Rust 在 `client_metadata` 内写入 `session_id` / `thread_id`。 -这里不能只看本地测试是否通过,需要做一次真实 header audit。若没有明确兼容原因,应优先贴近 TS 版和官方客户端链路,避免因为 header 命名差异影响上游会话亲和、缓存或连接复用。 +因此 header 不能简单按 TS 版 `session_id` 回退。TS 参考实现和官方客户端抓包在会话 header 命名上不一致;Rust 当前更贴近官方抓包,并且现有 header 测试已断言不发送 `session_id` 请求头。后续若要调整,应先用真实链路 A/B 验证,而不是仅以 TS 源码为准。 ## 首 Token 统计 @@ -161,15 +172,15 @@ thread-id: ## 修复优先级 1. 已完成:核对活跃 WebSocket 流的 `WEBSOCKET_RECEIVE_IDLE_TIMEOUT=20s`,对齐 TS 版本 active stream 不做每帧 20s 超时的行为;首帧前仍保留 stale reuse 保护。 -2. 把代理合成的 `stream_disconnected` 明确标记为 synthetic/proxy-side failure,并在 usage/log 中保留原始错误 detail。 -3. 给 `live upstream stream ended with response.failed` 增加 `request_id`、`response_id`、transport、pool decision、first token、latency 和 failure detail。 +2. 已完成:把代理合成的 `stream_disconnected` 明确标记为 synthetic/proxy-side failure,并在 usage/log 中保留原始错误 detail。 +3. 已完成:给 `live upstream stream ended with response.failed` 增加 `request_id`、`response_id`、transport、pool decision、first token、latency 和 failure detail。 4. 已完成:修 `response.failed` 分类,去掉“未分类 response.failed 默认 503”的行为,并收窄到 TS allowlist。 5. 已完成:调整 WS terminal frame 回收策略;普通 terminal 释放当前 stream 并回池,分类上游错误才 discard。 -6. 部分完成:已补 Responses WebSocket terminal 语义测试;`/v1/chat/completions` 仍需补同一套路由级回归。 -7. 补 pool key/session affinity 单测,尤其是只有 `previous_response_id` 的连续请求。 -8. 做真实 header audit,再决定是否把 `session-id` / `thread-id` 改为或补齐 `session_id`。 -9. 补首 token 统计测试,分别覆盖 HTTP SSE、WebSocket、metadata-only 前缀事件、首帧前失败和已有输出后断流。 -10. 增加 stream close / pool decision 的结构化日志字段,并补一条真正代表流结束状态的日志;不要只依赖 HTTP trace 的 `completed HTTP request status=200`。 +6. 已完成:已补 Responses WebSocket terminal 语义测试,并补 `/v1/chat/completions` 路由级回归,确认未分类 `response.failed` 不触发账号轮换或 WS discard 放大。 +7. 已验证:Rust 在只有 `previous_response_id` 时会通过 session affinity 继承原 conversation identity,并已有 `responses_websocket_should_route_previous_response_id_to_recorded_account` 回归覆盖。 +8. 已完成:基于 `docs/flows.all.txt` 做真实 header audit;官方客户端使用 `session-id` / `thread-id`,Rust 当前保持该形态,不补齐 TS 版 `session_id` 请求头。 +9. 已完成:补首 token 统计判定和测试,覆盖 HTTP SSE、WebSocket、metadata-only 前缀事件、首帧前失败和已有输出后断流;`response.created`、`response.in_progress`、`response.metadata`、`codex.rate_limits` 和上游失败 terminal 不会误写 `firstTokenMs`。 +10. 已完成:补 `live response stream finalized` 结构化日志,作为真正代表流结束状态的日志;字段包含 request/response、transport、pool decision、first token、latency、完成/失败状态和失败细节,不再只依赖 HTTP trace 的 `completed HTTP request status=200`。 ## 真实链路回归 @@ -182,6 +193,76 @@ thread-id: - 人为触发或观察上游 `response.failed`,确认客户端收到的是正确 SSE terminal 语义,usage 不误写为代理 502/503。 - 对比管理端请求明细、后端日志和客户端侧耗时,确认 request id 能串起来。 +### 2026-06-29 额度、调度、缓存真实链路 + +运行环境:本地 `target/debug/codex-proxy-rs`,`config.yaml`,SQLite `.runtime/data/codex-proxy-rs.sqlite`,日志 `.runtime/logs/codex-proxy-rs.2026-06-29.log`。本轮使用管理端 API Key 创建临时客户端 key,真实请求不记录明文 key。 + +本地回归: + +- `cargo test --test main quota -- --nocapture`:39 passed。 +- `cargo test --test main usage_logging -- --nocapture`:15 passed。 +- `cargo test --test main responses_websocket -- --nocapture`:30 passed。 +- `cargo test --test main chat_completions_should_mark_quota_exhausted_after_402_and_fallback -- --nocapture`:1 passed。 +- `cargo build`:通过。 + +真实链路结果: + +- 启动恢复:数据库中 `acct_a73f281b0a2c4fdebae8b17ab88b2270` 原为额度耗尽,启动后恢复为 `status=quota_exhausted`、`quota_limit_reached=1`、`remaining=0`、`cooldown_until=2026-07-29T08:45:10+00:00`;本轮真实请求没有再调度到该账号。 +- HTTP SSE:`real_quota_sse_20260629_2057` 成功完成,transport=`http_sse`,`firstTokenMs=2623`,usage=`20/57/77`,账号 `acct_fa9f0172d2084eaf86ddbd0e5d0c1c16` 写入 `x-codex-primary-used-percent=5`,`quota_fetched_at/updated_at` 更新到本轮请求结束时间。 +- WebSocket fallback 日志:`real_quota_ws_20260629_2055` 上游 WS 握手返回 `tls handshake eof`,触发 fallback warn;日志字段已包含 `request_id`、`account_id`、`transport=websocket`、`fallback_transport=http_sse`、`fallback_reason=websocket_error` 和 error。 +- WebSocket 成功:`real_chain_ws_20260629_2109` 成功完成,transport=`websocket`,pool=`new`,`firstTokenMs=7052`,usage=`85/3526/3611`;账号 `acct_fa9...` 额度从 5% 被动更新到 14%。 +- 缓存命中:相同长上下文和同一 `prompt_cache_key` 连续请求中,`real_cache_repeat_20260629_1_ws` 为 transport=`websocket`、pool=`new`、`cachedTokens=0`;`real_cache_repeat_20260629_3_ws` 回到同账号并复用连接,transport=`websocket`、pool=`reuse`、`cachedTokens=15104`、`firstTokenMs=2738`,账号 `acct_81aaba2a4e084162924b17d6f55e8a10` 额度从 6% 更新到 9%。 +- `/v1/chat/completions`:`real_chat_stream_20260629_2114` 成功完成,kind=`v1.chat`,transport=`http_sse`,`firstTokenMs=8824`,usage=`84/752/836`,账号 `acct_fa9...` 额度继续更新到 20%。 + +真实链路观察: + +- WS 和 HTTP SSE 成功路径都会写 usage,并会把上游 rate limit headers 被动同步回账号 quota。 +- 额度耗尽账号会恢复为 `quota_exhausted` 并跳过调度。 +- 同账号、同 `prompt_cache_key`、WS pool reuse 时可以观察到 prompt cache 命中;跨账号或 WS 失败 fallback 的样本不能用来判断缓存未命中。 +- 强制 HTTP SSE 并携带 `previous_response_id` 会被上游拒绝:`Unsupported parameter: previous_response_id`。真实连续上下文应走默认/WS 路径。 + +### 2026-06-29 TS 调度策略对齐 + +本轮重新对齐 TS 版本 `/home/zyy/桌面/Codes/codex-proxy` 的三个调度模式: + +- `least_used` / 智能分配:按 TS 比较器选择候选账号,顺序为 quota exhausted 靠后、两个账号都有 `window_reset_at` 时更早 reset 优先、更低 `request_count` 优先、最后按 LRU 选择更久未使用账号;只在完整比较结果相同的首组选项内轮转。 +- `round_robin` / 轮询:候选账号顺序使用账号恢复/插入顺序,不再被账号 ID 排序覆盖;cursor 语义对齐 TS:选择前对当前候选数量取模,选择后只自增。 +- `sticky` / 粘性:优先最近使用账号;当 `last_used` 完全相同时保留候选顺序,冷启动时选择候选首项。 +- `least_used` 和 `round_robin` 共用同一个 rotation cursor;切换调度策略时重置 cursor。 +- `prompt_cache_key` 不新增独立账号硬绑定。TS 版本也是通过 `previous_response_id` / implicit resume 命中 session affinity 后把账号作为 preferred hint;Rust 保持同一语义。缓存命中仍要求后续请求落在同账号,并且 WS pool key 的账号 + conversation identity 一致。 + +本地回归: + +- `cargo test --test main account_pool -- --nocapture`:38 passed。 +- `cargo test --test main responses_websocket -- --nocapture`:30 passed。 +- `cargo test --test main responses_http -- --nocapture`:23 passed。 +- `cargo build`:通过。 +- `cargo clippy --all-targets --all-features --locked -- -D warnings`:通过。 +- `pnpm -C web build`:通过。 + +真实链路回归: + +- 运行环境:本地 `target/debug/codex-proxy-rs`,审计目录 `.runtime/real-chain-scheduler-20260629_214555/ws-audit`,日志 `.runtime/logs/codex-proxy-rs.2026-06-29.log`。本轮使用管理端 API Key 创建临时客户端 key,结束后已删除临时 key,并将 `rotationStrategy` 恢复为 `least_used`。 +- `least_used`:`real_sched_least_20260629_214555` 走 `/v1/responses` stream,status=200,选中 `acct_81aaba2a4e084162924b17d6f55e8a10`,transport=`websocket`,pool=`new`,`firstTokenMs=8317`,usage=`70/503/0`。该账号是本轮候选中更符合 TS 智能分配比较器的低使用/LRU 账号。 +- `round_robin`:依次执行 `real_sched_rr1_20260629_214555`、`real_sched_rr2_20260629_214555`、`real_sched_rr3_20260629_214555`,均 status=200,选中账号顺序为 `acct_7951dd589a5c4046afdd7080d58f8501` -> `acct_81aaba2a4e084162924b17d6f55e8a10` -> `acct_fa9f0172d2084eaf86ddbd0e5d0c1c16`。其中第一轮后 `acct_795...` 根据上游 quota header 被更新为 `status=quota_exhausted`、`quota_limit_reached=1`、`quota_cooldown_until=2026-07-29T03:09:21+00:00`,后续轮询跳过该账号。 +- `sticky` + 缓存:`real_sched_sticky_cache1_20260629_214555` 和 `real_sched_sticky_cache2_20260629_214555` 均选中 `acct_fa9f0172d2084eaf86ddbd0e5d0c1c16`,验证 sticky 不切换账号。第一次 pool=`new`、`firstTokenMs=7532`、usage=`6581/2031/0`;第二次 pool=`reuse`、`firstTokenMs=1575`、usage=`6581/1737/6400`,同账号、同 `prompt_cache_key`、WS 复用时缓存命中。 + +真实链路观察: + +- 三个调度策略均能完成真实 `/v1/responses` stream,usage 记录中的 `account_id`、`transport`、`websocketPool.kind` 和日志 `live response stream finalized` 一致。 +- 额度耗尽会实时写回账号状态,并从后续调度候选中排除。 +- 切换账号会丢缓存的根因不是 `prompt_cache_key` 丢失,而是缓存依赖同账号和同 conversation/WS pool identity;sticky 场景已验证同账号复用下缓存命中。 + +#### 智能分配二次审计 + +2026-06-29 重新按 TS `src/auth/rotation-strategy.ts` / `src/auth/account-lifecycle.ts` 审计 Rust `AccountPool::select_least_used`: + +- 比较器顺序一致:quota limited 靠后 -> 两边都有 `window_reset_at` 时更早 reset 优先 -> 更低 `request_count` 优先 -> 更早 `last_used` 优先。 +- 缺失 `window_reset_at` 的账号不会被惩罚;只要另一侧没有同时具备可比较窗口,就回落到 `request_count`。 +- 首组选项完全相等时使用共享 rotation cursor 轮转,候选数组/账号池插入顺序不被排序副作用改变。 +- acquire 前候选过滤和 TS 对齐到同一层级:active、并发槽位、exclude、Cloudflare 冷却、quota limited 跳过、模型计划限制、tier priority,再进入 `least_used`。 +- 已补 Rust 回归覆盖 TS 边界用例:缺失窗口、双边无窗口、quota limited 内部按 reset 排序、least_used 不改变候选顺序;`cargo test --test main account_pool -- --nocapture`:42 passed。 + ## 源码锚点 Rust 版: diff --git a/src/proxy/dispatch/chat.rs b/src/proxy/dispatch/chat.rs index 573ff3f73..63e81e2ad 100644 --- a/src/proxy/dispatch/chat.rs +++ b/src/proxy/dispatch/chat.rs @@ -147,7 +147,7 @@ impl ChatDispatchService { }}; } - let (account_id, response) = loop { + let (account, response) = loop { let acquire_request = AccountAcquireRequest::new(&request.model, Utc::now()) .with_exclude_account_ids(excluded_account_ids.iter().cloned()); let acquired = match self.account_pool.acquire_with(&acquire_request).await { @@ -246,7 +246,7 @@ impl ChatDispatchService { self.account_pool.release(&release_account_id).await; match response_result { - Ok(response) => break (release_account_id, response), + Ok(response) => break (account, response), Err(error) if is_rate_limit_upstream_error(&error) => { rate_limited_count += 1; last_rate_limit_error = Some(upstream_error_body(&error)); @@ -351,6 +351,7 @@ impl ChatDispatchService { } } }; + let account_id = account.id.clone(); let body = match crate::proxy::openai::chat::chat_completion_from_codex_sse( &response.body, &display_model, @@ -375,6 +376,9 @@ impl ChatDispatchService { }; let response_id = body.get("id").and_then(Value::as_str); self.cloudflare.reset_account_recovery(&account_id).await; + self.account_pool + .sync_passive_rate_limit_headers(&account, &response.rate_limit_headers) + .await; if let Some(ref usage) = response.usage { self.account_pool .record_token_usage(&account_id, &request.model, usage) diff --git a/src/proxy/dispatch/responses.rs b/src/proxy/dispatch/responses.rs index 7c28340e1..faad19d7d 100644 --- a/src/proxy/dispatch/responses.rs +++ b/src/proxy/dispatch/responses.rs @@ -592,6 +592,9 @@ impl ResponseDispatchService { &response.set_cookie_headers, ) .await; + self.account_pool + .sync_passive_rate_limit_headers(&account, &response.rate_limit_headers) + .await; let collected_response = match response_from_codex_sse(&response.body, tuple_schema.as_ref()) { Ok(collected_response) => collected_response, @@ -855,9 +858,6 @@ impl ResponseDispatchService { CollectedResponse::Completed(body) => { let response_id = body.get("id").and_then(Value::as_str); self.cloudflare.reset_account_recovery(&account.id).await; - self.account_pool - .sync_passive_rate_limit_headers(&account, &response.rate_limit_headers) - .await; if let Some(usage) = response.usage { self.account_pool .record_response_usage( @@ -1208,6 +1208,9 @@ impl ResponseDispatchService { self.cloudflare .capture_set_cookie_headers(&release_account_id, &set_cookie_headers) .await; + self.account_pool + .sync_passive_rate_limit_headers(&account, &rate_limit_headers) + .await; let (prefetched, body) = match prefetch_first_sse_chunk(response.body).await { Ok(prefetched) => prefetched, Err(ResponseDispatchError::Upstream(error)) @@ -1366,6 +1369,7 @@ impl ResponseDispatchService { usage_records: Arc::clone(&self.usage_records), cloudflare: self.cloudflare.clone(), account_id: account.id, + account_plan_type: account.plan_type, request_id: request_id.to_string(), route: route.to_string(), model: request.model.clone(), @@ -1760,6 +1764,9 @@ impl ResponseDispatchService { ) .await; self.cloudflare.reset_account_recovery(&account.id).await; + self.account_pool + .sync_passive_rate_limit_headers(&account, &response.rate_limit_headers) + .await; let usage = extract_usage(&response.body); if let Some(usage) = usage { self.account_pool @@ -2058,6 +2065,7 @@ struct LiveResponseStreamContext { usage_records: Arc, cloudflare: CloudflareRecovery, account_id: String, + account_plan_type: Option, request_id: String, route: String, model: String, @@ -2380,12 +2388,20 @@ fn latest_response_id(body: &str) -> Option { }) } +const STREAM_DISCONNECTED_CODE: &str = "stream_disconnected"; +const STREAM_DISCONNECTED_MESSAGE: &str = "Upstream stream closed before response.completed"; + fn premature_close_failed_event(response_id: Option<&str>, detail: Option<&str>) -> String { let message = match detail.filter(|value| !value.trim().is_empty()) { - Some(detail) => format!("Upstream stream closed before response.completed: {detail}"), - None => "Upstream stream closed before response.completed".to_string(), + Some(detail) => format!("{STREAM_DISCONNECTED_MESSAGE}: {detail}"), + None => STREAM_DISCONNECTED_MESSAGE.to_string(), }; - response_failed_sse_event_with_id(response_id, "server_error", "stream_disconnected", &message) + response_failed_sse_event_with_id( + response_id, + "server_error", + STREAM_DISCONNECTED_CODE, + &message, + ) } async fn finalize_live_response_stream( @@ -2394,6 +2410,14 @@ async fn finalize_live_response_stream( first_token_ms: Option, ) { let rate_limit_headers = live_response_rate_limit_headers(&context).await; + context + .account_pool + .sync_passive_rate_limit_headers_for_account( + &context.account_id, + context.account_plan_type.as_deref(), + &rate_limit_headers, + ) + .await; let turn_state = live_response_turn_state(&context).await; let usage = match extract_sse_usage(&body) { Ok(Some(usage)) => { @@ -2453,10 +2477,29 @@ async fn finalize_live_response_stream( ) .await; } + let response_id = latest_response_id(&body); + let latency_ms = elapsed_millis_i64(context.started_at); + let failure_source = stream_failure_source(&failure); + let failure_detail = synthetic_stream_disconnected_detail(&failure); + let websocket_pool_kind = context + .websocket_pool_decision + .map(|decision| decision.kind()); + let websocket_pool_reason = context + .websocket_pool_decision + .and_then(|decision| decision.reason()); tracing::warn!( account_id = %context.account_id, + request_id = %context.request_id, + response_id = response_id.as_deref().unwrap_or(""), + transport = %backend_transport_name(context.transport), + websocket_pool_kind = ?websocket_pool_kind, + websocket_pool_reason = ?websocket_pool_reason, + first_token_ms = ?first_token_ms, + latency_ms, event = %failure.event, code = ?failure.upstream_code.as_deref(), + failure_source = %failure_source, + failure_detail = ?failure_detail.as_deref(), "live upstream stream ended with response.failed" ); let mut metadata = stream_failure_metadata(&failure, usage); @@ -2859,6 +2902,7 @@ async fn record_live_response_stream_event( ensure_stream_metadata_flag(&mut metadata); enrich_event_route_metadata(&mut metadata, &context.route); enrich_live_response_stream_metadata(context, rate_limit_headers, &mut metadata, body); + log_live_response_stream_finalized(context, status_code, level, message, &metadata, body); let mut event = UsageRecord::new(response_event_kind(&context.route), level, message); event.request_id = Some(context.request_id.clone()); event.account_id = Some(context.account_id.clone()); @@ -2880,6 +2924,79 @@ async fn record_live_response_stream_event( } } +fn log_live_response_stream_finalized( + context: &LiveResponseStreamContext, + status_code: i64, + level: UsageRecordLevel, + message: &str, + metadata: &Value, + body: &str, +) { + let response_id = metadata_string_field(metadata, "responseId") + .map(ToString::to_string) + .or_else(|| latest_response_id(body)); + let first_token_ms = metadata.get("firstTokenMs").and_then(Value::as_i64); + let websocket_pool_kind = context + .websocket_pool_decision + .map(WebSocketPoolDecision::kind); + let websocket_pool_reason = context + .websocket_pool_decision + .and_then(WebSocketPoolDecision::reason); + let completed = metadata.get("completed").and_then(Value::as_bool); + let failed = metadata.get("failed").and_then(Value::as_bool); + let upstream_code = metadata_string_field(metadata, "upstreamCode"); + let failure_class = metadata_string_field(metadata, "failureClass"); + let failure_source = metadata_string_field(metadata, "failureSource"); + let failure_detail = metadata_string_field(metadata, "failureDetail"); + + macro_rules! emit_stream_finalized_log { + ($level:expr) => { + tracing::event!( + $level, + account_id = %context.account_id, + request_id = %context.request_id, + route = %context.route, + model = %context.display_model, + status_code, + usage_level = ?level, + event_message = %message, + transport = %backend_transport_name(context.transport), + websocket_pool_kind = ?websocket_pool_kind, + websocket_pool_reason = ?websocket_pool_reason, + response_id = response_id.as_deref().unwrap_or(""), + first_token_ms = ?first_token_ms, + latency_ms = elapsed_millis_i64(context.started_at), + completed = ?completed, + failed = ?failed, + upstream_code = ?upstream_code, + failure_class = ?failure_class, + failure_source = ?failure_source, + failure_detail = ?failure_detail, + "live response stream finalized" + ); + }; + } + + match level { + UsageRecordLevel::Debug => { + emit_stream_finalized_log!(tracing::Level::DEBUG); + } + UsageRecordLevel::Info => { + emit_stream_finalized_log!(tracing::Level::INFO); + } + UsageRecordLevel::Warn => { + emit_stream_finalized_log!(tracing::Level::WARN); + } + UsageRecordLevel::Error => { + emit_stream_finalized_log!(tracing::Level::ERROR); + } + } +} + +fn metadata_string_field<'a>(metadata: &'a Value, field: &str) -> Option<&'a str> { + metadata.get(field).and_then(Value::as_str) +} + async fn live_response_rate_limit_headers( context: &LiveResponseStreamContext, ) -> Vec<(String, String)> { @@ -2898,14 +3015,54 @@ async fn live_response_turn_state(context: &LiveResponseStreamContext) -> Option } fn stream_failure_metadata(failure: &ResponsesSseFailure, usage: Option) -> Value { - serde_json::json!({ + let mut metadata = serde_json::json!({ "stream": true, "failed": true, "failureEvent": failure.event, "failureMessage": failure.message, "upstreamCode": failure.upstream_code, "usage": usage, - }) + }); + enrich_stream_failure_source_metadata(&mut metadata, failure); + metadata +} + +fn enrich_stream_failure_source_metadata(metadata: &mut Value, failure: &ResponsesSseFailure) { + let Some(object) = metadata.as_object_mut() else { + return; + }; + object.insert( + "failureSource".to_string(), + Value::String(stream_failure_source(failure).to_string()), + ); + if let Some(detail) = synthetic_stream_disconnected_detail(failure) { + object.insert("synthetic".to_string(), Value::Bool(true)); + if !detail.is_empty() { + object.insert("failureDetail".to_string(), Value::String(detail)); + } + } +} + +fn stream_failure_source(failure: &ResponsesSseFailure) -> &'static str { + if synthetic_stream_disconnected_detail(failure).is_some() { + "proxy" + } else { + "upstream" + } +} + +fn synthetic_stream_disconnected_detail(failure: &ResponsesSseFailure) -> Option { + if failure.upstream_code.as_deref() != Some(STREAM_DISCONNECTED_CODE) { + return None; + } + let detail = failure + .message + .strip_prefix(STREAM_DISCONNECTED_MESSAGE)? + .strip_prefix(": ") + .unwrap_or_default() + .trim() + .to_string(); + Some(detail) } fn insert_first_token_ms(metadata: &mut Value, first_token_ms: Option) { @@ -3292,7 +3449,10 @@ fn elapsed_millis_i64(started_at: Instant) -> i64 { #[cfg(test)] mod tests { - use crate::upstream::protocol::responses::response_body_has_first_event; + use super::{stream_failure_metadata, STREAM_DISCONNECTED_CODE, STREAM_DISCONNECTED_MESSAGE}; + use crate::upstream::protocol::responses::{ + response_body_has_first_event, ResponsesSseFailure, + }; #[test] fn first_event_detection_should_match_text_delta() { @@ -3305,13 +3465,68 @@ data: {"delta":"hello"} } #[test] - fn first_event_detection_should_match_response_created() { + fn first_event_detection_should_match_reasoning_delta() { + let body = br#"event: response.reasoning_summary_text.delta +data: {"delta":"thinking"} + +"#; + + assert!(response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_match_function_call_arguments_delta() { + let body = br#"event: response.function_call_arguments.delta +data: {"delta":"{\"path\""} + +"#; + + assert!(response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_match_type_when_event_name_is_missing() { + let body = br#"data: {"type":"response.output_text.delta","delta":"hello"} + +"#; + + assert!(response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_ignore_response_created() { let body = br#"event: response.created data: {"type":"response.created","response":{"id":"resp_1","status":"in_progress"}} "#; - assert!(response_body_has_first_event(body)); + assert!(!response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_ignore_metadata_and_rate_limit_events() { + let body = br#"event: codex.rate_limits +data: {"type":"codex.rate_limits","rate_limits":{"primary":{"used_percent":42}}} + +event: response.metadata +data: {"type":"response.metadata","x-codex-turn-state":"state"} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_1","status":"in_progress"}} + +"#; + + assert!(!response_body_has_first_event(body)); + } + + #[test] + fn first_event_detection_should_ignore_empty_delta() { + let body = br#"event: response.output_text.delta +data: {"delta":""} + +"#; + + assert!(!response_body_has_first_event(body)); } #[test] @@ -3333,8 +3548,28 @@ data: {"delta":"hello"}"#; #[test] fn first_event_detection_should_match_crlf_frames() { - let body = b"event: response.created\r\ndata: {\"type\":\"response.created\"}\r\n\r\n"; + let body = b"event: response.output_text.delta\r\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\r\n\r\n"; assert!(response_body_has_first_event(body)); } + + #[test] + fn stream_failure_metadata_should_mark_synthetic_proxy_disconnect() { + let failure = ResponsesSseFailure { + event: "response.failed".to_string(), + message: format!( + "{STREAM_DISCONNECTED_MESSAGE}: websocket receive idle timeout after 20s" + ), + upstream_code: Some(STREAM_DISCONNECTED_CODE.to_string()), + }; + + let metadata = stream_failure_metadata(&failure, None); + + assert_eq!(metadata["failureSource"], "proxy"); + assert_eq!(metadata["synthetic"], true); + assert_eq!( + metadata["failureDetail"], + "websocket receive idle timeout after 20s" + ); + } } diff --git a/src/runtime/tasks/coordinator.rs b/src/runtime/tasks/coordinator.rs index 46f21aad4..351faa4db 100644 --- a/src/runtime/tasks/coordinator.rs +++ b/src/runtime/tasks/coordinator.rs @@ -95,6 +95,7 @@ impl TaskCoordinator { ) .with_installation_id(services.installation_id.clone()) .with_cookie_store(stores.cookies.clone()) + .with_account_pool(services.account_pool.clone()) .start(), ); coordinator.push( diff --git a/src/runtime/tasks/quota_refresh.rs b/src/runtime/tasks/quota_refresh.rs index 39ed8dda5..61ea76dd3 100644 --- a/src/runtime/tasks/quota_refresh.rs +++ b/src/runtime/tasks/quota_refresh.rs @@ -8,6 +8,7 @@ use tracing::{debug, info, warn}; use crate::{ upstream::accounts::{ cookies::SqliteCookieStore, + pool::RuntimeAccountPoolService, quota::{QuotaRefreshSummary, RuntimeQuotaRefreshService}, store::SqliteAccountStore, }, @@ -52,6 +53,12 @@ impl QuotaRefreshTask { self } + /// 设置运行时账号池,用于刷新后同步内存调度状态。 + pub fn with_account_pool(mut self, account_pool: Arc) -> Self { + self.service = self.service.with_account_pool(account_pool); + self + } + /// 启动后台刷新任务。 pub fn start(self) -> SchedulerHandle { let (shutdown_tx, mut shutdown_rx) = tokio::sync::mpsc::channel(1); diff --git a/src/upstream/accounts/pool.rs b/src/upstream/accounts/pool.rs index 1572d2c9b..607edbd9e 100644 --- a/src/upstream/accounts/pool.rs +++ b/src/upstream/accounts/pool.rs @@ -8,6 +8,7 @@ use std::{ }; use chrono::{DateTime, Duration, Utc}; +use indexmap::IndexMap; use thiserror::Error; use crate::upstream::accounts::quota::{quota_snapshot_limit_reached, quota_snapshot_reset_at}; @@ -167,11 +168,10 @@ pub struct AccountWindowUsageDelta { /// 纯内存账号池,负责账号调度和运行时状态维护。 #[derive(Debug)] pub struct AccountPool { - accounts: BTreeMap, + accounts: IndexMap, slots: BTreeMap>>, options: AccountPoolOptions, - round_robin_cursor: usize, - least_used_cursor: usize, + rotation_cursor: usize, } impl Default for AccountPool { @@ -184,11 +184,10 @@ impl AccountPool { /// 使用指定配置创建账号池。 pub fn with_options(options: AccountPoolOptions) -> Self { Self { - accounts: BTreeMap::new(), + accounts: IndexMap::new(), slots: BTreeMap::new(), options, - round_robin_cursor: 0, - least_used_cursor: 0, + rotation_cursor: 0, } } @@ -204,7 +203,7 @@ impl AccountPool { /// 移除账号及其在途槽位。 pub fn remove(&mut self, account_id: &str) -> bool { - let removed = self.accounts.remove(account_id).is_some(); + let removed = self.accounts.shift_remove(account_id).is_some(); self.slots.remove(account_id); removed } @@ -213,12 +212,14 @@ impl AccountPool { pub fn clear(&mut self) { self.accounts.clear(); self.slots.clear(); - self.round_robin_cursor = 0; - self.least_used_cursor = 0; + self.rotation_cursor = 0; } /// 替换账号池运行参数。 pub fn set_options(&mut self, options: AccountPoolOptions) { + if self.options.rotation_strategy != options.rotation_strategy { + self.rotation_cursor = 0; + } self.options = options; } @@ -276,6 +277,11 @@ impl AccountPool { account.quota_verify_required = false; account.quota_limit_reached = limit_reached; account.quota_cooldown_until = limit_reached.then_some(cooldown_until).flatten(); + match (account.status, limit_reached) { + (AccountStatus::Active, true) => account.status = AccountStatus::QuotaExhausted, + (AccountStatus::QuotaExhausted, false) => account.status = AccountStatus::Active, + _ => {} + } if let Some(cooldown_until) = account.quota_cooldown_until { account.window_reset_at = account .window_reset_at @@ -484,33 +490,39 @@ impl AccountPool { } fn select_least_used(&mut self, candidates: &[Account]) -> Option { - let best_key = candidates + let mut sorted = candidates.iter().collect::>(); + sorted.sort_by(|a, b| compare_least_used(a, b)); + let best = *sorted.first()?; + let tied_count = sorted .iter() - .map(LeastUsedGroupKey::from_account) - .min_by(compare_least_used_group)?; - let mut tied = candidates - .iter() - .filter(|account| LeastUsedGroupKey::from_account(account) == best_key) - .collect::>(); - tied.sort_by(compare_lru_then_id); - let index = self.least_used_cursor % tied.len(); - self.least_used_cursor = self.least_used_cursor.wrapping_add(1); - Some((*tied[index]).clone()) + .take_while(|account| compare_least_used(best, account) == Ordering::Equal) + .count(); + let index = self.rotation_cursor % tied_count; + self.rotation_cursor = self.rotation_cursor.wrapping_add(1); + Some((*sorted[index]).clone()) } fn select_sticky(candidates: &[Account]) -> Option { - candidates - .iter() - .max_by_key(|account| account.last_used_at.clone()) - .cloned() + let mut selected = candidates.first()?; + for candidate in &candidates[1..] { + if compare_last_used( + candidate.last_used_at.as_deref(), + selected.last_used_at.as_deref(), + ) == Ordering::Greater + { + selected = candidate; + } + } + Some((*selected).clone()) } fn select_round_robin(&mut self, candidates: &[Account]) -> Option { if candidates.is_empty() { return None; } - let index = self.round_robin_cursor % candidates.len(); - self.round_robin_cursor = (index + 1) % candidates.len(); + self.rotation_cursor %= candidates.len(); + let index = self.rotation_cursor; + self.rotation_cursor = self.rotation_cursor.wrapping_add(1); Some(candidates[index].clone()) } @@ -962,28 +974,39 @@ impl RuntimeAccountPoolService { &self, account: &Account, headers: &[(String, String)], + ) { + self.sync_passive_rate_limit_headers_for_account( + &account.id, + account.plan_type.as_deref(), + headers, + ) + .await; + } + + /// 根据上游返回的 rate-limit header 被动同步指定账号 quota 状态。 + pub async fn sync_passive_rate_limit_headers_for_account( + &self, + account_id: &str, + plan_type: Option<&str>, + headers: &[(String, String)], ) { let Some(rate_limits) = parse_rate_limit_headers(headers) else { return; }; - let existing_quota = match self.store.get_quota_json(&account.id).await { + let existing_quota = match self.store.get_quota_json(account_id).await { Ok(Some(quota_json)) => serde_json::from_str::("a_json).ok(), Ok(None) => None, Err(error) => { tracing::warn!( - account_id = %account.id, + account_id = %account_id, error = %error, "failed to read existing quota json before passive rate-limit sync" ); None } }; - let quota = rate_limit_quota( - &rate_limits, - account.plan_type.as_deref(), - existing_quota.as_ref(), - ); - self.apply_quota_snapshot(&account.id, "a).await; + let quota = rate_limit_quota(&rate_limits, plan_type, existing_quota.as_ref()); + self.apply_quota_snapshot(account_id, "a).await; } /// 读取运行时账号快照。 @@ -1122,6 +1145,9 @@ fn refresh_quota_window(account: &mut Account, now: DateTime) { account.quota_verify_required = true; account.quota_limit_reached = false; account.quota_cooldown_until = None; + if account.status == AccountStatus::QuotaExhausted { + account.status = AccountStatus::Active; + } } } @@ -1194,28 +1220,12 @@ fn refresh_cloudflare_cooldown(account: &mut Account, now: DateTime) { } } -#[derive(Debug, Clone, PartialEq, Eq)] -struct LeastUsedGroupKey { - quota_limited: bool, - window_reset_at: Option>, - request_count: u64, -} - -impl LeastUsedGroupKey { - fn from_account(account: &Account) -> Self { - Self { - quota_limited: account.quota_limit_reached, - window_reset_at: account.window_reset_at, - request_count: account.request_count, - } - } -} - -fn compare_least_used_group(a: &LeastUsedGroupKey, b: &LeastUsedGroupKey) -> Ordering { - a.quota_limited - .cmp(&b.quota_limited) +fn compare_least_used(a: &Account, b: &Account) -> Ordering { + a.quota_limit_reached + .cmp(&b.quota_limit_reached) .then_with(|| compare_window_reset(a.window_reset_at, b.window_reset_at)) .then_with(|| a.request_count.cmp(&b.request_count)) + .then_with(|| compare_last_used(a.last_used_at.as_deref(), b.last_used_at.as_deref())) } fn compare_window_reset(a: Option>, b: Option>) -> Ordering { @@ -1225,8 +1235,13 @@ fn compare_window_reset(a: Option>, b: Option>) -> O } } -fn compare_lru_then_id(a: &&Account, b: &&Account) -> Ordering { - a.last_used_at - .cmp(&b.last_used_at) - .then_with(|| a.id.cmp(&b.id)) +fn compare_last_used(a: Option<&str>, b: Option<&str>) -> Ordering { + last_used_millis(a).cmp(&last_used_millis(b)) +} + +fn last_used_millis(value: Option<&str>) -> i64 { + value + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + .map(|datetime| datetime.timestamp_millis()) + .unwrap_or(0) } diff --git a/src/upstream/accounts/quota/runtime.rs b/src/upstream/accounts/quota/runtime.rs index 9231476be..0fc3cce60 100644 --- a/src/upstream/accounts/quota/runtime.rs +++ b/src/upstream/accounts/quota/runtime.rs @@ -8,6 +8,7 @@ use tracing::warn; use crate::upstream::accounts::{ model::AccountStatus, + pool::RuntimeAccountPoolService, quota::{ quota_from_usage, quota_snapshot_limit_reached, quota_snapshot_limit_window_seconds, quota_snapshot_reset_at, @@ -30,6 +31,7 @@ pub struct RuntimeQuotaRefreshService { request_spacing: Duration, installation_id: Option, cookie_store: Option, + account_pool: Option>, } impl RuntimeQuotaRefreshService { @@ -42,6 +44,7 @@ impl RuntimeQuotaRefreshService { request_spacing: Self::default_request_spacing(), installation_id: None, cookie_store: None, + account_pool: None, } } @@ -62,6 +65,7 @@ impl RuntimeQuotaRefreshService { request_spacing: Self::default_request_spacing(), installation_id: None, cookie_store: None, + account_pool: None, } } @@ -83,6 +87,12 @@ impl RuntimeQuotaRefreshService { self } + /// 设置运行时账号池,用于刷新后同步内存调度状态。 + pub fn with_account_pool(mut self, account_pool: Arc) -> Self { + self.account_pool = Some(account_pool); + self + } + /// 执行一次配额锁定或待验证账号刷新,复用调用方传入的最近刷新记录。 pub async fn refresh_locked_accounts( &self, @@ -103,8 +113,9 @@ impl RuntimeQuotaRefreshService { let mut candidates = accounts .into_iter() .filter(|account| { - account.status == AccountStatus::Active - && (account.quota_limit_reached || account.quota_verify_required) + account.status == AccountStatus::QuotaExhausted + || (account.status == AccountStatus::Active + && (account.quota_limit_reached || account.quota_verify_required)) }) .peekable(); @@ -167,6 +178,7 @@ impl RuntimeQuotaRefreshService { ) .await?; } + self.sync_runtime_pool_account(&account.id).await; } Err(error) => { warn!( @@ -194,6 +206,19 @@ impl RuntimeQuotaRefreshService { .ok() .flatten() } + + async fn sync_runtime_pool_account(&self, account_id: &str) { + let Some(account_pool) = &self.account_pool else { + return; + }; + if let Err(error) = account_pool.sync_account_from_repository(account_id).await { + warn!( + account_id = %account_id, + error = %error, + "failed to sync runtime account pool after quota refresh" + ); + } + } } /// 单次配额刷新摘要。 diff --git a/src/upstream/accounts/store.rs b/src/upstream/accounts/store.rs index 6c205a407..a261af469 100644 --- a/src/upstream/accounts/store.rs +++ b/src/upstream/accounts/store.rs @@ -54,7 +54,7 @@ select au.last_used_at as usage_last_used_at from accounts a left join account_usage au on au.account_id = a.id -order by a.added_at desc, a.id desc"; +order by a.rowid asc"; const GET_POOL_ACCOUNT_SQL: &str = r" select @@ -376,6 +376,12 @@ set quota_json = ?, quota_fetched_at = ?, plan_type = coalesce(?, plan_type), + status = case + when status in ('disabled', 'banned', 'expired', 'refreshing') then status + when ? = 1 then 'quota_exhausted' + when status = 'quota_exhausted' then 'active' + else status + end, quota_limit_reached = ?, quota_verify_required = 0, quota_cooldown_until = ?, @@ -1372,6 +1378,7 @@ impl SqliteAccountStore { .bind(&now) .bind(&plan_type) .bind(if limit_reached { 1 } else { 0 }) + .bind(if limit_reached { 1 } else { 0 }) .bind(&cooldown) .bind(&now) .bind(account_id) diff --git a/src/upstream/protocol/responses.rs b/src/upstream/protocol/responses.rs index d11c8ba96..9c58d9cf0 100644 --- a/src/upstream/protocol/responses.rs +++ b/src/upstream/protocol/responses.rs @@ -7,7 +7,7 @@ use crate::upstream::{ models::ParsedModelName, protocol::{ schema::reconvert_tuple_values, - sse::{parse_sse_events, SseError}, + sse::{parse_sse_events, SseError, SseEvent}, }, }; @@ -146,12 +146,50 @@ pub fn response_body_has_first_event(body_bytes: &[u8]) -> bool { let Some(complete_body) = complete_sse_body_prefix(&body) else { return false; }; - parse_sse_events(complete_body).is_ok_and(|events| { - events.iter().any(|event| { - let data = event.data.trim(); - !data.is_empty() && data != "[DONE]" - }) - }) + parse_sse_events(complete_body) + .is_ok_and(|events| events.iter().any(response_sse_event_has_first_output)) +} + +fn response_sse_event_has_first_output(event: &SseEvent) -> bool { + let data = event.data.trim(); + if data.is_empty() || data == "[DONE]" { + return false; + } + + let value = serde_json::from_str::(data).ok(); + let event_type = event.event.as_deref().or_else(|| { + value + .as_ref() + .and_then(|value| value.get("type")) + .and_then(Value::as_str) + }); + + match event_type { + Some("response.output_text.delta") + | Some("response.reasoning_summary_text.delta") + | Some("response.reasoning_text.delta") + | Some("response.function_call_arguments.delta") + | Some("response.custom_tool_call_input.delta") => value + .as_ref() + .and_then(|value| value.get("delta")) + .and_then(Value::as_str) + .is_some_and(|delta| !delta.is_empty()), + Some("response.output_text.done") => value + .as_ref() + .and_then(|value| value.get("text")) + .and_then(Value::as_str) + .is_some_and(|text| !text.is_empty()), + Some("response.function_call_arguments.done") => value + .as_ref() + .and_then(|value| value.get("arguments")) + .and_then(Value::as_str) + .is_some_and(|arguments| !arguments.is_empty()), + Some("response.output_item.added" | "response.output_item.done") => value + .as_ref() + .and_then(|value| value.get("item")) + .is_some_and(Value::is_object), + _ => false, + } } fn complete_sse_body_prefix(body: &str) -> Option<&str> { diff --git a/src/upstream/transport/client.rs b/src/upstream/transport/client.rs index b5a3114a0..62a4b08f3 100644 --- a/src/upstream/transport/client.rs +++ b/src/upstream/transport/client.rs @@ -488,7 +488,15 @@ impl CodexBackendClient { if http_sse_fallback_allowed(&upstream_request) && websocket_error_allows_http_fallback(&error) => { - tracing::warn!(error = %error, "websocket response failed; falling back to HTTP SSE"); + tracing::warn!( + request_id = %context.request_id, + account_id = pool_account_id.or(context.account_id).unwrap_or_default(), + transport = "websocket", + fallback_transport = "http_sse", + fallback_reason = "websocket_error", + error = %error, + "websocket response failed; falling back to HTTP SSE" + ); self.create_response_http_sse(&upstream_request, context, started_at) .await } @@ -530,7 +538,15 @@ impl CodexBackendClient { if http_sse_fallback_allowed(&upstream_request) && websocket_error_allows_http_fallback(&error) => { - tracing::warn!(error = %error, "websocket response stream failed; falling back to HTTP SSE"); + tracing::warn!( + request_id = %context.request_id, + account_id = pool_account_id.or(context.account_id).unwrap_or_default(), + transport = "websocket", + fallback_transport = "http_sse", + fallback_reason = "websocket_error", + error = %error, + "websocket response stream failed; falling back to HTTP SSE" + ); self.create_response_stream_http_sse(&upstream_request, context) .await } diff --git a/tests/proxy/dispatch/chat_upstream/chat_routes.rs b/tests/proxy/dispatch/chat_upstream/chat_routes.rs index 4401a6012..2c8e3f1c0 100644 --- a/tests/proxy/dispatch/chat_upstream/chat_routes.rs +++ b/tests/proxy/dispatch/chat_upstream/chat_routes.rs @@ -211,6 +211,66 @@ async fn chat_completions_with_user_should_use_and_reuse_websocket() { ); } +#[tokio::test] +async fn chat_completions_websocket_should_report_unmapped_response_failed_without_rotation() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut websocket = accept_async(stream).await.unwrap(); + let message = websocket.next().await.unwrap().unwrap(); + let payload = serde_json::from_str::(&message.into_text().unwrap()) + .expect("websocket payload should be json"); + websocket + .send(Message::Text( + response_failed_websocket_message( + "resp_chat_failed_terminal", + "policy_violation", + "Terminal policy failure", + ) + .into(), + )) + .await + .unwrap(); + payload + }); + let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(base_url).await; + + let response = app + .oneshot(chat_json_request( + &api_key, + "req_chat_ws_unmapped_failed", + "chat-ws-failed-user", + "Trigger a terminal response.failed", + )) + .await + .unwrap(); + let status = response.status(); + let body = response_json(response).await; + let payload = upstream.await.unwrap(); + let event = latest_usage_record(&pool, "v1.chat").await; + let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); + + assert_eq!(status, StatusCode::BAD_GATEWAY); + assert_eq!(body["error"]["code"], "invalid_upstream_response"); + assert_eq!(payload["type"], "response.create"); + assert_eq!( + event.request_id.as_deref(), + Some("req_chat_ws_unmapped_failed") + ); + assert_eq!(event.account_id.as_deref(), Some("acct_chat")); + assert_eq!(event.route.as_deref(), Some("/v1/chat/completions")); + assert_eq!(event.status_code, Some(502)); + assert_eq!(metadata["route"], "/v1/chat/completions"); + assert_eq!(metadata["apiKind"], "chat"); + assert_eq!(metadata["stream"], false); + assert_eq!(metadata["transport"], "websocket"); + assert_eq!(metadata["failureClass"], "invalid_sse"); + assert!(metadata["error"] + .as_str() + .is_some_and(|error| error.contains("Terminal policy failure"))); +} + #[tokio::test] async fn chat_completions_stream_should_translate_codex_sse_to_openai_chunks() { let server = MockServer::start().await; diff --git a/tests/proxy/dispatch/chat_upstream/responses_websocket.rs b/tests/proxy/dispatch/chat_upstream/responses_websocket.rs index fbc2b000c..9b5dd76e6 100644 --- a/tests/proxy/dispatch/chat_upstream/responses_websocket.rs +++ b/tests/proxy/dispatch/chat_upstream/responses_websocket.rs @@ -2129,6 +2129,23 @@ async fn responses_stream_with_previous_response_id_should_record_websocket_audi assert_eq!(metadata["transport"], "websocket"); assert_eq!(metadata["usage"]["inputTokens"], 3); assert_eq!(metadata["usage"]["outputTokens"], 1); + assert!( + metadata["firstTokenMs"] + .as_i64() + .is_some_and(|value| value > 0), + "websocket stream usage metadata should include first token latency: {metadata:?}", + ); assert_rate_limit_header(&metadata, "x-codex-primary-used-percent", "44"); assert_rate_limit_header(&metadata, "x-codex-primary-window-minutes", "5"); + let stored_quota: (Option,) = + sqlx::query_as("select quota_json from accounts where id = ?") + .bind("acct_chat") + .fetch_one(&pool) + .await + .unwrap(); + let quota: Value = serde_json::from_str(stored_quota.0.as_deref().unwrap()).unwrap(); + assert_eq!( + quota["snapshots"][0]["primary"]["used_percent"].as_f64(), + Some(44.0) + ); } diff --git a/tests/proxy/dispatch/chat_upstream/usage_logging.rs b/tests/proxy/dispatch/chat_upstream/usage_logging.rs index 2ec71a655..3f2b1966d 100644 --- a/tests/proxy/dispatch/chat_upstream/usage_logging.rs +++ b/tests/proxy/dispatch/chat_upstream/usage_logging.rs @@ -22,6 +22,7 @@ async fn responses_should_use_imported_account_record_usage_cookie_and_usage_rec let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(server.uri()).await; let response = app + .clone() .oneshot( Request::builder() .method("POST") @@ -101,6 +102,7 @@ async fn chat_completions_non_stream_should_record_first_token_latency() { let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(server.uri()).await; let response = app + .clone() .oneshot( Request::builder() .method("POST") @@ -158,6 +160,7 @@ async fn responses_usage_record_should_store_resolved_upstream_model_after_alias }) .await; let response = app + .clone() .oneshot( Request::builder() .method("POST") @@ -359,6 +362,7 @@ async fn responses_should_passively_cache_rate_limit_headers() { .unwrap(); let response = app + .clone() .oneshot( Request::builder() .method("POST") @@ -380,8 +384,8 @@ async fn responses_should_passively_cache_rate_limit_headers() { .unwrap(); assert_eq!(response.status(), StatusCode::OK); - let stored: (String, i64, Option, i64) = sqlx::query_as( - "select quota_json, quota_limit_reached, quota_cooldown_until, quota_verify_required from accounts where id = ?", + let stored: (String, String, i64, Option, i64) = sqlx::query_as( + "select quota_json, status, quota_limit_reached, quota_cooldown_until, quota_verify_required from accounts where id = ?", ) .bind("acct_chat") .fetch_one(&pool) @@ -392,9 +396,10 @@ async fn responses_should_passively_cache_rate_limit_headers() { assert_eq!(quota["snapshots"][0]["primary"]["limit_reached"], true); assert_eq!(quota["snapshots"][0]["primary"]["reset_at"], reset_at); assert_eq!(quota["credits"]["balance"], 12); - assert_eq!(stored.1, 1); - assert!(stored.2.is_some()); - assert_eq!(stored.3, 0); + assert_eq!(stored.1, "quota_exhausted"); + assert_eq!(stored.2, 1); + assert!(stored.3.is_some()); + assert_eq!(stored.4, 0); let window: (i64, i64, i64, i64, Option, Option, Option) = sqlx::query_as( "select window_request_count, window_input_tokens, window_output_tokens, window_cached_tokens, window_started_at, window_reset_at, limit_window_seconds from account_usage where account_id = ?", @@ -415,6 +420,28 @@ async fn responses_should_passively_cache_rate_limit_headers() { reset_at ); assert_eq!(window.6, Some(300)); + + let second_response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "input": [], + "stream": false, + "use_websocket": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_ne!(second_response.status(), StatusCode::OK); } #[tokio::test] @@ -721,8 +748,85 @@ data: {"delta":"partial before logged disconnect"} assert_eq!(metadata["stream"], true); assert_eq!(metadata["failed"], true); assert_eq!(metadata["upstreamCode"], "stream_disconnected"); + assert_eq!(metadata["failureSource"], "proxy"); + assert_eq!(metadata["synthetic"], true); assert_rate_limit_header(&metadata, "retry-after", "11"); assert_rate_limit_header(&metadata, "x-codex-primary-used-percent", "88"); + let stored_quota: (Option,) = + sqlx::query_as("select quota_json from accounts where id = ?") + .bind("acct_chat") + .fetch_one(&pool) + .await + .unwrap(); + let quota: Value = serde_json::from_str(stored_quota.0.as_deref().unwrap()).unwrap(); + assert_eq!( + quota["snapshots"][0]["primary"]["used_percent"].as_f64(), + Some(88.0) + ); +} + +#[tokio::test] +async fn responses_stream_should_not_record_first_token_when_metadata_only_prefix_fails() { + let first_frame = r#"event: response.created +data: {"type":"response.created","response":{"id":"resp_metadata_only","status":"in_progress"}} + +event: codex.rate_limits +data: {"type":"codex.rate_limits","rate_limits":{"primary":{"used_percent":42.0}}} + +event: response.metadata +data: {"type":"response.metadata","x-codex-turn-state":"state"} + +"#; + let (base_url, first_chunk_sent, close_upstream) = + spawn_chunked_sse_upstream_then_clean_close(first_frame).await; + + let (app, api_key, pool, _dir) = test_app_with_account_pool_and_logging(base_url).await; + let response_task = tokio::spawn(async move { + app.oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header("authorization", format!("Bearer {api_key}")) + .header("content-type", "application/json") + .header("x-request-id", "req_stream_metadata_only_disconnect") + .body(Body::from( + json!({ + "model": "gpt-5.5", + "input": [{"role": "user", "content": "metadata then close"}], + "stream": true, + "use_websocket": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap() + }); + first_chunk_sent.await.unwrap(); + let response = response_task.await.unwrap(); + close_upstream.send(()).unwrap(); + let body = response_text(response).await; + let event = latest_response_usage_record(&pool).await; + let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap(); + let first_token_bucket: (i64, i64) = sqlx::query_as( + "select coalesce(sum(first_token_latency_sum), 0), coalesce(sum(first_token_latency_count), 0) from usage_time_buckets", + ) + .fetch_one(&pool) + .await + .unwrap(); + + assert!(body.contains("stream_disconnected")); + assert_eq!(event.level, "error"); + assert_eq!( + event.request_id.as_deref(), + Some("req_stream_metadata_only_disconnect") + ); + assert_eq!(event.status_code, Some(502)); + assert_eq!(metadata["stream"], true); + assert_eq!(metadata["failed"], true); + assert!(metadata.get("firstTokenMs").is_none()); + assert_eq!(first_token_bucket, (0, 0)); } #[tokio::test] diff --git a/tests/runtime/account_pool_restore/mod.rs b/tests/runtime/account_pool_restore/mod.rs index 9f382b2f2..8fac6ae60 100644 --- a/tests/runtime/account_pool_restore/mod.rs +++ b/tests/runtime/account_pool_restore/mod.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use chrono::Utc; use codex_proxy_rs::upstream::accounts::{ - pool::{AccountAcquireRequest, AccountPoolOptions, RuntimeAccountPoolService}, + pool::{ + AccountAcquireRequest, AccountPoolOptions, RotationStrategy, RuntimeAccountPoolService, + }, store::{AccountStore, SqliteAccountStore}, }; @@ -50,6 +52,34 @@ async fn runtime_account_pool_should_restore_capacity_and_clear_runtime_state() assert_eq!(restored_capacity.available_slots, 2); } +#[tokio::test] +async fn runtime_account_pool_should_restore_accounts_in_insert_order() { + let (pool, _dir) = init_test_db("runtime-account-pool-order.sqlite").await; + insert_account(&pool, "acct_c").await; + insert_account(&pool, "acct_a").await; + + let store = SqliteAccountStore::new(pool); + let service = RuntimeAccountPoolService::new( + Arc::new(store) as Arc, + AccountPoolOptions { + rotation_strategy: RotationStrategy::RoundRobin, + ..AccountPoolOptions::default() + }, + 0, + ); + + service + .restore_from_repository() + .await + .expect("account pool should restore"); + let acquired = service + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", Utc::now())) + .await + .expect("active account should be acquired"); + + assert_eq!(acquired.account.id, "acct_c"); +} + async fn insert_account(pool: &sqlx::SqlitePool, id: &str) { sqlx::query( "insert into accounts (id, email, chatgpt_account_id, chatgpt_user_id, access_token, access_token_expires_at, status, added_at, updated_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", diff --git a/tests/upstream/accounts/account_pool/quota.rs b/tests/upstream/accounts/account_pool/quota.rs index c858186be..bbcdb3f96 100644 --- a/tests/upstream/accounts/account_pool/quota.rs +++ b/tests/upstream/accounts/account_pool/quota.rs @@ -14,6 +14,31 @@ fn account_pool_should_skip_accounts_with_cached_quota_limit() { assert_eq!(acquire_account(&mut pool, "gpt-5.5").unwrap().id, "usable"); } +#[test] +fn account_pool_should_mark_quota_state_as_quota_exhausted_status() { + let now = fixed_time(); + let mut pool = AccountPool::default(); + pool.insert(crate::support::accounts::test_account( + "limited", + AccountStatus::Active, + )); + + pool.apply_quota_state("limited", true, Some(now + Duration::seconds(60))); + + let limited = pool.get("limited").unwrap(); + assert_eq!(limited.status, AccountStatus::QuotaExhausted); + assert!(limited.quota_limit_reached); + assert!(pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .is_none()); + + pool.apply_quota_state("limited", false, None); + + let restored = pool.get("limited").unwrap(); + assert_eq!(restored.status, AccountStatus::Active); + assert!(!restored.quota_limit_reached); +} + #[test] fn account_pool_should_reuse_quota_limited_accounts_after_cooldown() { let now = fixed_time(); @@ -81,7 +106,8 @@ fn account_pool_should_not_replace_known_window_length_with_cooldown_seconds() { #[test] fn acquire_should_refresh_expired_cooldowns_before_selecting_account() { let now = fixed_time(); - let mut account = crate::support::accounts::test_account("acct_a", AccountStatus::Active); + let mut account = + crate::support::accounts::test_account("acct_a", AccountStatus::QuotaExhausted); account.quota_limit_reached = true; account.quota_cooldown_until = Some(now - Duration::seconds(1)); account.cloudflare_cooldown_until = Some(now - Duration::seconds(1)); @@ -97,6 +123,7 @@ fn acquire_should_refresh_expired_cooldowns_before_selecting_account() { assert!(!acquired.account.quota_limit_reached); assert!(acquired.account.quota_verify_required); + assert_eq!(acquired.account.status, AccountStatus::Active); assert!(acquired.account.quota_cooldown_until.is_none()); assert!(acquired.account.cloudflare_cooldown_until.is_none()); assert_eq!(acquired.account.window_request_count, 1); diff --git a/tests/upstream/accounts/account_pool/selection.rs b/tests/upstream/accounts/account_pool/selection.rs index 20d459c04..c5055c648 100644 --- a/tests/upstream/accounts/account_pool/selection.rs +++ b/tests/upstream/accounts/account_pool/selection.rs @@ -52,6 +52,96 @@ fn account_pool_should_rotate_round_robin_across_candidates() { assert!(acquire_account(&mut pool, "gpt-5.5").is_none()); } +#[test] +fn round_robin_should_use_account_insert_order() { + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::RoundRobin, + ..AccountPoolOptions::default() + }); + pool.insert(crate::support::accounts::test_account( + "acct_c", + AccountStatus::Active, + )); + pool.insert(crate::support::accounts::test_account( + "acct_a", + AccountStatus::Active, + )); + + let acquired = acquire_account(&mut pool, "gpt-5.5").unwrap(); + + assert_eq!(acquired.id, "acct_c"); +} + +#[test] +fn round_robin_should_keep_ts_cursor_when_candidates_recover() { + let mut pool = AccountPool::with_options(AccountPoolOptions { + max_concurrent_per_account: 1, + rotation_strategy: RotationStrategy::RoundRobin, + ..AccountPoolOptions::default() + }); + pool.insert(crate::support::accounts::test_account( + "acct_a", + AccountStatus::Active, + )); + pool.insert(crate::support::accounts::test_account( + "acct_b", + AccountStatus::Active, + )); + pool.insert(crate::support::accounts::test_account( + "acct_c", + AccountStatus::Active, + )); + + let first = acquire_account(&mut pool, "gpt-5.5").unwrap(); + let second = acquire_account(&mut pool, "gpt-5.5").unwrap(); + pool.release(&first.id); + pool.release(&second.id); + let third = acquire_account(&mut pool, "gpt-5.5").unwrap(); + + assert_eq!(third.id, "acct_c"); +} + +#[test] +fn sticky_should_use_account_insert_order_when_last_used_ties() { + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::Sticky, + ..AccountPoolOptions::default() + }); + pool.insert(crate::support::accounts::test_account( + "acct_a", + AccountStatus::Active, + )); + pool.insert(crate::support::accounts::test_account( + "acct_b", + AccountStatus::Active, + )); + + let acquired = acquire_account(&mut pool, "gpt-5.5").unwrap(); + + assert_eq!(acquired.id, "acct_a"); +} + +#[test] +fn sticky_should_prefer_most_recently_used_account() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::Sticky, + ..AccountPoolOptions::default() + }); + let mut older = crate::support::accounts::test_account("older", AccountStatus::Active); + older.last_used_at = Some((now - Duration::minutes(10)).to_rfc3339()); + let mut newer = crate::support::accounts::test_account("newer", AccountStatus::Active); + newer.last_used_at = Some((now - Duration::minutes(1)).to_rfc3339()); + pool.insert(older); + pool.insert(newer); + + let acquired = pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap(); + + assert_eq!(acquired.account.id, "newer"); +} + #[test] fn acquire_should_skip_accounts_with_expired_token_metadata() { let now = fixed_time(); @@ -269,6 +359,42 @@ fn account_pool_should_rotate_tied_least_used_accounts() { assert_ne!(first.id, second.id); } +#[test] +fn least_used_should_prefer_lru_before_tie_rotation() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + max_concurrent_per_account: 1, + rotation_strategy: RotationStrategy::LeastUsed, + ..AccountPoolOptions::default() + }); + pool.insert(crate::support::accounts::test_account( + "seed", + AccountStatus::Active, + )); + let mut older = crate::support::accounts::test_account("older", AccountStatus::Active); + older.last_used_at = Some((now - Duration::minutes(10)).to_rfc3339()); + let mut newer = crate::support::accounts::test_account("newer", AccountStatus::Active); + newer.last_used_at = Some((now - Duration::minutes(1)).to_rfc3339()); + pool.insert(older); + pool.insert(newer); + + assert_eq!( + pool.acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap() + .account + .id, + "seed" + ); + let acquired = pool + .acquire_with(&AccountAcquireRequest::new( + "gpt-5.5", + now + Duration::seconds(1), + )) + .unwrap(); + + assert_eq!(acquired.account.id, "older"); +} + #[test] fn least_used_should_prefer_lower_runtime_request_count() { let now = fixed_time(); @@ -289,3 +415,106 @@ fn least_used_should_prefer_lower_runtime_request_count() { assert_eq!(acquired.account.id, "quiet"); } + +#[test] +fn least_used_should_not_penalize_missing_window_reset() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::LeastUsed, + ..AccountPoolOptions::default() + }); + let mut fresh = crate::support::accounts::test_account("fresh", AccountStatus::Active); + fresh.request_count = 1; + let mut known_window = + crate::support::accounts::test_account("known_window", AccountStatus::Active); + known_window.request_count = 5; + known_window.window_reset_at = Some(now + Duration::days(1)); + pool.insert(known_window); + pool.insert(fresh); + + let acquired = pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap(); + + assert_eq!(acquired.account.id, "fresh"); +} + +#[test] +fn least_used_should_fall_through_to_request_count_without_window_resets() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::LeastUsed, + ..AccountPoolOptions::default() + }); + let mut busier = crate::support::accounts::test_account("busier", AccountStatus::Active); + busier.request_count = 3; + let mut quieter = crate::support::accounts::test_account("quieter", AccountStatus::Active); + quieter.request_count = 1; + pool.insert(busier); + pool.insert(quieter); + + let acquired = pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap(); + + assert_eq!(acquired.account.id, "quieter"); +} + +#[test] +fn least_used_should_sort_quota_limited_accounts_by_reset_when_not_skipping() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + skip_quota_limited: false, + rotation_strategy: RotationStrategy::LeastUsed, + ..AccountPoolOptions::default() + }); + let mut later = crate::support::accounts::test_account("later", AccountStatus::Active); + later.quota_limit_reached = true; + later.window_reset_at = Some(now + Duration::days(3)); + let mut sooner = crate::support::accounts::test_account("sooner", AccountStatus::Active); + sooner.quota_limit_reached = true; + sooner.window_reset_at = Some(now + Duration::days(1)); + pool.insert(later); + pool.insert(sooner); + + let acquired = pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap(); + + assert_eq!(acquired.account.id, "sooner"); +} + +#[test] +fn least_used_should_not_mutate_candidate_order() { + let now = fixed_time(); + let mut pool = AccountPool::with_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::LeastUsed, + ..AccountPoolOptions::default() + }); + let mut first = crate::support::accounts::test_account("first", AccountStatus::Active); + first.request_count = 5; + let mut second = crate::support::accounts::test_account("second", AccountStatus::Active); + second.request_count = 3; + let mut third = crate::support::accounts::test_account("third", AccountStatus::Active); + third.request_count = 1; + pool.insert(first); + pool.insert(second); + pool.insert(third); + + let selected = pool + .acquire_with(&AccountAcquireRequest::new("gpt-5.5", now)) + .unwrap(); + pool.release(&selected.account.id); + pool.set_options(AccountPoolOptions { + rotation_strategy: RotationStrategy::RoundRobin, + ..AccountPoolOptions::default() + }); + let acquired = pool + .acquire_with(&AccountAcquireRequest::new( + "gpt-5.5", + now + Duration::seconds(1), + )) + .unwrap(); + + assert_eq!(acquired.account.id, "first"); +} diff --git a/tests/upstream/accounts/account_repository.rs b/tests/upstream/accounts/account_repository.rs index f3ba7c508..13c63251b 100644 --- a/tests/upstream/accounts/account_repository.rs +++ b/tests/upstream/accounts/account_repository.rs @@ -312,6 +312,84 @@ async fn account_repository_should_update_quota_json_and_fetched_at() { ); } +#[tokio::test] +async fn account_repository_should_map_quota_snapshot_to_account_status() { + let (pool, _dir) = sqlite_account_store_parts("accounts.sqlite", 37).await; + let repo = SqliteAccountStore::new(pool.clone()); + seed_repo_account(&pool, "acct_a", "2026-06-11T00:00:00Z").await; + let reset_at = Utc::now() + Duration::minutes(5); + + let limited = repo + .apply_quota_snapshot( + "acct_a", + r#"{"plan_type":"free","monthly_limit":{"limit_reached":true}}"#, + true, + Some(reset_at), + ) + .await + .unwrap(); + let limited_status: (String, i64, Option) = sqlx::query_as( + "select status, quota_limit_reached, quota_cooldown_until from accounts where id = ?", + ) + .bind("acct_a") + .fetch_one(&pool) + .await + .unwrap(); + + assert!(limited); + assert_eq!(limited_status.0, "quota_exhausted"); + assert_eq!(limited_status.1, 1); + assert!(limited_status.2.is_some()); + + let restored = repo + .apply_quota_snapshot( + "acct_a", + r#"{"plan_type":"free","monthly_limit":{"limit_reached":false}}"#, + false, + None, + ) + .await + .unwrap(); + let restored_status: (String, i64, Option) = sqlx::query_as( + "select status, quota_limit_reached, quota_cooldown_until from accounts where id = ?", + ) + .bind("acct_a") + .fetch_one(&pool) + .await + .unwrap(); + + assert!(restored); + assert_eq!(restored_status.0, "active"); + assert_eq!(restored_status.1, 0); + assert!(restored_status.2.is_none()); +} + +#[tokio::test] +async fn account_repository_should_not_override_disabled_status_from_quota_snapshot() { + let (pool, _dir) = sqlite_account_store_parts("accounts.sqlite", 38).await; + let repo = SqliteAccountStore::new(pool.clone()); + seed_repo_account(&pool, "acct_a", "2026-06-11T00:00:00Z").await; + repo.set_status("acct_a", AccountStatus::Disabled) + .await + .unwrap(); + + repo.apply_quota_snapshot( + "acct_a", + r#"{"plan_type":"free","monthly_limit":{"limit_reached":true}}"#, + true, + Some(Utc::now() + Duration::minutes(5)), + ) + .await + .unwrap(); + let status: (String,) = sqlx::query_as("select status from accounts where id = ?") + .bind("acct_a") + .fetch_one(&pool) + .await + .unwrap(); + + assert_eq!(status.0, "disabled"); +} + async fn sqlite_account_store_parts( db_name: &str, _key_byte: u8, diff --git a/tests/upstream/accounts/quota_refresh.rs b/tests/upstream/accounts/quota_refresh.rs index 97262ca57..0e59a5fcc 100644 --- a/tests/upstream/accounts/quota_refresh.rs +++ b/tests/upstream/accounts/quota_refresh.rs @@ -322,6 +322,68 @@ async fn quota_refresh_service_should_fetch_usage_for_quota_locked_accounts_and_ ); } +#[tokio::test] +async fn quota_refresh_service_should_refresh_quota_exhausted_accounts() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "plan_type": "plus", + "rate_limit": { + "allowed": true, + "limit_reached": false, + "primary_window": { + "used_percent": 12, + "reset_at": 1_800_000_000, + "limit_window_seconds": 18_000 + } + } + }))) + .expect(1) + .mount(&server) + .await; + + let dir = tempfile::tempdir().expect("temp dir"); + let db = dir.path().join("quota-refresh-exhausted.sqlite"); + let pool = connect_sqlite(&format!("sqlite://{}", db.display())) + .await + .expect("sqlite pool"); + let store = SqliteAccountStore::new(pool.clone()); + insert_quota_locked_account(&store, &pool, "acct-quota", "access-token").await; + store + .set_status("acct-quota", AccountStatus::QuotaExhausted) + .await + .expect("status should update"); + sqlx::query( + "update accounts set quota_limit_reached = 0, quota_verify_required = 0 where id = ?", + ) + .bind("acct-quota") + .execute(&pool) + .await + .expect("quota flags should reset"); + let codex = CodexBackendClient::new( + reqwest::Client::new(), + server.uri(), + crate::support::fingerprint::test_fingerprint(), + ); + let service = RuntimeQuotaRefreshService::new(store, Arc::new(codex)); + + let mut last_refreshed = HashMap::new(); + let summary = service + .refresh_locked_accounts(&mut last_refreshed) + .await + .expect("quota refresh should succeed"); + let stored: (String, i64) = + sqlx::query_as("select status, quota_limit_reached from accounts where id = ?") + .bind("acct-quota") + .fetch_one(&pool) + .await + .unwrap(); + + assert_eq!(summary.refreshed, 1); + assert_eq!(stored, ("active".to_string(), 0)); +} + #[tokio::test] async fn quota_refresh_service_should_stagger_multiple_locked_account_requests() { let server = MockServer::start().await; diff --git a/web/src/views/accounts/components/AccountQuotaPanel.vue b/web/src/views/accounts/components/AccountQuotaPanel.vue index 38596e20c..5f0ce2874 100644 --- a/web/src/views/accounts/components/AccountQuotaPanel.vue +++ b/web/src/views/accounts/components/AccountQuotaPanel.vue @@ -21,8 +21,25 @@ function quotaWindowPercent(window?: any) { return clamp(window?.usedPercent ?? 0, 0, 100) } -function quotaWindowBarWidth(window?: any) { - return `${quotaWindowPercent(window)}%` +function quotaWindowBarStyle(window?: any) { + const percent = quotaWindowPercent(window) + return { + width: `${percent}%`, + minWidth: percent > 0 ? '8px' : '0', + } +} + +function quotaWindowBarClass(window?: any) { + if (window?.usedPercent === null || window?.usedPercent === undefined) { + return 'bg-(--cp-default-border-hover)' + } + if (window.usedPercent >= 95) { + return 'bg-(--cp-danger)' + } + if (window.usedPercent >= 80) { + return 'bg-(--cp-warning)' + } + return 'bg-(--cp-success)' } @@ -58,10 +75,11 @@ function quotaWindowBarWidth(window?: any) { {{ window.labelDisplay }} {{ window.usedPercentDisplay }} -
+
{{ window.labelDisplay }} {{ window.usedPercentDisplay }}
-
+
@@ -104,10 +123,11 @@ function quotaWindowBarWidth(window?: any) { {{ window.labelDisplay }} {{ window.usedPercentDisplay }}
-
+
{ const s = props.rotationStrategy if (!s) return '—' const map: Record = { - least_used: '最少使用优先', + least_used: '智能分配(推荐)', round_robin: '轮询', - sticky: '会话粘性', + sticky: '粘滞', } return map[s] || s }) diff --git a/web/src/views/settings/index.vue b/web/src/views/settings/index.vue index 210f39753..e9409f75a 100644 --- a/web/src/views/settings/index.vue +++ b/web/src/views/settings/index.vue @@ -66,19 +66,19 @@ const rotationOptions: Array<{ description: string }> = [ { - label: '智能分配', + label: '智能分配(推荐)', value: 'least_used', - description: '优先使用压力更低的账号,适合长期稳定运行。', + description: '优先使用即将刷新额度的账号,最大化总使用量。', }, { label: '轮询', value: 'round_robin', - description: '按顺序分配请求,让账号池保持均匀节奏。', + description: '按顺序轮流使用各账号。', }, { - label: '粘性', + label: '粘滞', value: 'sticky', - description: '持续使用同一账号,直到遇到限速或不可用。', + description: '持续使用同一账号,直到限速或额度耗尽。', }, ]