From ed9b98c319e281f7ef468c7bbbb4f90c795d3647 Mon Sep 17 00:00:00 2001 From: Zyy Date: Fri, 12 Jun 2026 12:19:32 +0800 Subject: [PATCH] feat: add upstream account fallback handling Co-authored-by: Codex --- docs/implementation-status.md | 6 +- .../2026-06-12-openai-gpt-codex-parity.md | 15 +- ...26-06-12-openai-gpt-codex-parity-design.md | 4 +- src/accounts/model.rs | 2 + src/accounts/pool.rs | 52 ++- src/accounts/repository.rs | 1 + src/codex/client.rs | 52 ++- src/http/admin.rs | 1 + src/http/v1.rs | 268 +++++++++++-- tests/account_pool_scheduling_test.rs | 22 ++ tests/v1_upstream_route_test.rs | 369 +++++++++++++++++- 11 files changed, 733 insertions(+), 59 deletions(-) diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 63c89a1bc..73252ed50 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -13,14 +13,14 @@ Update this file before each feature commit. | Logging and pagination | Completed | `eb0341e` + `fc16c48` + `b0c10f2` + `ee0ed90` + `fdb9374` + `5bf012c` | `cargo test event_logs_are_cursor_paginated`; `cargo test admin_logs_are_cursor_paginated_and_include_request_id`; `cargo test rotating_writer_splits_files_when_size_limit_is_reached`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, admin `/admin/logs` query API, admin `/admin/usage-stats` and `/admin/usage-stats/summary`, event log repository, runtime tracing initialization, HTTP trace layer, date/size-based file rotation, v1 response lifecycle events with request/account/route/model/status/latency/non-secret metadata, and non-secret `account.refresh` events for upstream-401 refresh failures. | | TLS headers and fingerprint | Completed | `1bdd08c` | `cargo test codex_headers_include_desktop_identity_and_turn_state`; `cargo tree \| rg 'reqwest\|rustls'`; full `fmt/test/clippy` before commit | Adds Codex Desktop fingerprint model, exact identity/request headers, and pinned reqwest/rustls client builder. | | Cookie persistence | Completed | `4710926` + `5d3cc6e` | `cargo test cookie_jar_captures_and_replays_account_scoped_cookies`; `cargo test cookie_repository_encrypts_and_replays_account_scoped_cookies`; full `fmt/test/clippy` before commit | Adds account-scoped in-memory Cookie jar and encrypted SQLite Cookie repository; Cookies are replayed only for the same account/domain. | -| Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; full `fmt/test/clippy` before commit | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, 401-triggered refresh and single retry for `/v1/responses`, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process import-to-pool sync, and startup pool snapshots from SQLite with usage-based last-used metadata. Still missing durable release-time slot state, broader upstream retry/fallback wiring, and admin account CRUD. | +| Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` + `pending` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; Task 5 targeted `cargo test --test v1_upstream_route_test` and `cargo test --test account_pool_scheduling_test`; full `fmt/test/clippy` before commit | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, 401-triggered refresh and single retry for `/v1/responses`, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process import-to-pool sync, startup pool snapshots from SQLite with usage-based last-used metadata, in-memory 429 quota cooldown marking, Cloudflare cooldown mutation, and pool status mutation for upstream fallback. Still missing durable release-time slot state, streaming/Chat upstream fallback wiring, durable quota cooldown persistence, and admin account CRUD. | | Account repository | Completed | `14fe706` | `cargo test --test account_repository_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds encrypted SQLite account token storage, decrypting reads, cursor pagination, status/label updates, release-time usage counter persistence, and refresh-token-preserving token updates. | | Translation | Completed | `eff182e` + `b9c30ba` | `cargo test --test routes_chat_test`; `cargo test --test chat_completions_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` | Adds OpenAI Chat request types, expanded Codex Responses request type, OpenAI Chat to Codex translation for system/developer instructions, multimodal content, tool/function calls, response formats, reasoning effort, service tier, and parallel tool calls. Codex SSE now converts back to OpenAI chat completions and chat chunks with text deltas, reasoning content, tool calls, usage details, and OpenAI-compatible error bodies. | -| HTTP routes | Partial | `ab0ad8f` + `3830b21` + `6179d44` + `cfec8ba` + `6341336` + `228febd` + `c2cf7a6` + `e9fdea0` + `ee0ed90` + `fdb9374` + `9ca5c7d` + `b9c30ba` + `350ffbe` | `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test models_route_returns_openai_compatible_codex_model_list`; `cargo test --test admin_settings_route_test`; `cargo test --test model_catalog_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test chat_completions_route_test`; `cargo test --test codex_websocket_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | Adds Axum router, health route, `/v1/models`, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/debug/models`, `/admin/accounts`, `/admin/accounts/import`, `/admin/api-keys`, `/admin/usage-stats`, `/admin/usage-stats/summary`, `/v1/responses` HTTP SSE collection with output item/delta reconstruction plus `output_text` sync for non-streaming clients, default streaming Responses passthrough when `stream` is omitted, WebSocket-backed Responses streaming for `previous_response_id`, Responses field parity for service tier, tool choice, parallel tool calls, text format, prompt cache key, include, client metadata, reasoning include default, and Codex context headers, plus a dedicated `/v1/chat/completions` route backed by imported Codex accounts. `/v1` unauthorized and model errors remain OpenAI-compatible, global request-id response headers are preserved, `gpt-5.5` remains the default/test model baseline, and `/admin/settings` is read-only with admin-session auth and lower camelCase envelope. `/admin/accounts` reads metadata only and does not decrypt tokens for list views. Still missing OAuth/device/CLI account login flows, admin account mutation/health/quota/Cookie routes, full admin API-key mutation/export/import routes, settings mutation/persistence, diagnostics, backend model refresh, model cache persistence, and live Chat streaming transform. | +| HTTP routes | Partial | `ab0ad8f` + `3830b21` + `6179d44` + `cfec8ba` + `6341336` + `228febd` + `c2cf7a6` + `e9fdea0` + `ee0ed90` + `fdb9374` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `pending` | `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test models_route_returns_openai_compatible_codex_model_list`; `cargo test --test admin_settings_route_test`; `cargo test --test model_catalog_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test chat_completions_route_test`; `cargo test --test codex_websocket_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe`; Task 5 targeted verification before `pending` | Adds Axum router, health route, `/v1/models`, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/debug/models`, `/admin/accounts`, `/admin/accounts/import`, `/admin/api-keys`, `/admin/usage-stats`, `/admin/usage-stats/summary`, `/v1/responses` HTTP SSE collection with output item/delta reconstruction plus `output_text` sync for non-streaming clients, default streaming Responses passthrough when `stream` is omitted, WebSocket-backed Responses streaming for `previous_response_id`, non-streaming `/v1/responses` fallback to another imported account after 429/402/403 account-state failures, Responses field parity for service tier, tool choice, parallel tool calls, text format, prompt cache key, include, client metadata, reasoning include default, and Codex context headers, plus a dedicated `/v1/chat/completions` route backed by imported Codex accounts. `/v1` unauthorized and model errors remain OpenAI-compatible, global request-id response headers are preserved, `gpt-5.5` remains the default/test model baseline, and `/admin/settings` is read-only with admin-session auth and lower camelCase envelope. `/admin/accounts` reads metadata only and does not decrypt tokens for list views. Still missing OAuth/device/CLI account login flows, admin account mutation/health/quota/Cookie routes, full admin API-key mutation/export/import routes, settings mutation/persistence, diagnostics, backend model refresh, model cache persistence, live Chat streaming transform, and streaming upstream fallback. | | Admin/frontend response contract | Completed | `1c34ea7` + `b0c10f2` | `cargo test admin_envelope_serializes_lower_camel_case_request_id`; `cargo test admin_logs_reject_missing_admin_session_cookie`; `cargo test v1_response_has_request_id_header_without_admin_body_field`; full `fmt/test/clippy` before commit | Adds reusable admin envelopes with body codes, lower camelCase JSON, paginated list shape, requestId bodies, admin session cookie enforcement, and keeps `/v1/*` OpenAI-compatible. | | HTTP auth boundary | Completed | `cec4607` + `e9fdea0` | `cargo test --test http_auth_test`; `cargo test --test routes_responses_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_logs_route_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds a real `src/http/auth.rs` boundary for client API key and admin session cookie parsing. `/v1/*` accepts only enabled local `Bearer cpr_...` client API keys verified from SQLite, admin session parsing ignores client API keys, and route handlers no longer own ad hoc credential parsing. | | Model catalog | Partial | `6341336` | `cargo test --test model_catalog_test`; `cargo test model_catalog_route_returns_codex_metadata_without_alias_entries`; full `fmt/test/clippy` before commit | Adds a native static Codex model catalog with `gpt-5.5` default, configured aliases, reasoning/service-tier suffix parsing, OpenAI-compatible `/v1/models`, extended catalog/detail/info routes, and debug summary. Still missing backend Codex model fetching per plan, cache persistence, `/admin/refresh-models`, plan allowlist integration from fetched models, and full model metadata parity. | -| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` + `5bf012c` + `9ca5c7d` + `b9c30ba` + `350ffbe` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test codex_websocket_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, sends WebSocket `response.create` requests when `previous_response_id` requires upstream history, replays/captures encrypted Cookies, forwards Codex request context headers, normalizes `service_tier: fast` to upstream `priority`, collects `response.completed`, reconstructs missing assistant output from `response.output_item.done` or text deltas, synchronizes `output_text` for non-streaming clients, passes through Responses streaming SSE by default, adapts Chat Completions responses/chunks, refreshes and retries once after upstream 401 when a refresh token is stored, records token usage, writes v1 lifecycle event logs, records refresh failure classification for upstream 401, and restores the account pool from SQLite on startup. Still missing SSE error/failed mapping, rate-limit header capture, retry/fallback across accounts beyond refresh retry, 429 quota cache, 402/403 classification, previous-response account affinity, and live Chat streaming transform. | +| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` + `5bf012c` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `pending` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test codex_websocket_test`; Task 5 targeted `cargo test --test v1_upstream_route_test` and `cargo test --test account_pool_scheduling_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, sends WebSocket `response.create` requests when `previous_response_id` requires upstream history, replays/captures encrypted Cookies, forwards Codex request context headers, normalizes `service_tier: fast` to upstream `priority`, collects `response.completed`, reconstructs missing assistant output from `response.output_item.done` or text deltas, synchronizes `output_text` for non-streaming clients, passes through Responses streaming SSE by default, adapts Chat Completions responses/chunks, refreshes and retries once after upstream 401 when a refresh token is stored, parses Retry-After plus Codex reset metadata from upstream errors, records 429 request attempts, classifies non-streaming upstream 429/402/403 into account cooldown/quota/banned/Cloudflare state changes, retries non-streaming `/v1/responses` with another imported account when available, records token usage, writes v1 lifecycle event logs, records refresh failure classification for upstream 401, and restores the account pool from SQLite on startup. Still missing SSE error/failed mapping, successful rate-limit header capture, streaming/Chat retry/fallback beyond refresh retry, durable 429 quota cooldown persistence, previous-response account affinity, and live Chat streaming transform. | | Codex WebSocket boundary | Completed | `cc0516d` + `b9c30ba` + `350ffbe` | `cargo test --test codex_websocket_test`; `cargo test --test v1_upstream_route_test v1_responses_should_use_websocket_for_previous_response_id_streaming`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | Adds `src/codex/websocket.rs` with tested transport classification for HTTP SSE versus WebSocket-only requests. `previous_response_id` and explicit WebSocket requests use typed transport selection, local-only `use_websocket` is not serialized into upstream JSON, WebSocket `response.create` carries Codex headers and parity fields, incoming WebSocket messages are emitted as SSE-compatible events, and `/v1/responses` returns WebSocket-backed `text/event-stream` output for `previous_response_id` streaming requests. | | Fingerprint updates | Partial | `dd26d31` + `e32220a` | `cargo test update_manifest_updates_app_version_and_build_number`; `cargo test fingerprint_updater_fetches_manifest_and_persists_history`; full `fmt/test/clippy` before commit | Parses a JSON update manifest and persists app version/build number history to SQLite. Still missing original appcast XML polling, update-state persistence, runtime header fingerprint selection from the latest stored fingerprint, and optional local Codex.app Chromium extraction. | | Runtime docs and packaging | Completed | `33b63f3` | `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; `cargo run`; `curl -s -i http://127.0.0.1:8080/health` | README scope, `.env.example`, Dockerfile, compose file, and health smoke test are complete. | diff --git a/docs/superpowers/plans/2026-06-12-openai-gpt-codex-parity.md b/docs/superpowers/plans/2026-06-12-openai-gpt-codex-parity.md index 08b5b61d4..870f92ed3 100644 --- a/docs/superpowers/plans/2026-06-12-openai-gpt-codex-parity.md +++ b/docs/superpowers/plans/2026-06-12-openai-gpt-codex-parity.md @@ -8,7 +8,7 @@ **Tech Stack:** Rust, Axum, Reqwest/rustls, serde, sqlx/SQLite, wiremock, tokio, futures, optional WebSocket transport dependency. -**Checkpoint 2026-06-12:** Commit `b9c30ba` completed Tasks 1-3 for the in-scope OpenAI GPT/Codex path: Chat Completions route/translation/output, Responses request field parity, default Responses streaming, Codex context headers, and local-only WebSocket flag serialization cleanup. Commit `350ffbe` completed Task 4: `previous_response_id` requests use WebSocket `response.create` and return SSE-compatible output. Remaining major work starts at Task 5: account fallback/rate limits, model refresh/cache, and scoped admin operations. +**Checkpoint 2026-06-12:** Commit `b9c30ba` completed Tasks 1-3 for the in-scope OpenAI GPT/Codex path: Chat Completions route/translation/output, Responses request field parity, default Responses streaming, Codex context headers, and local-only WebSocket flag serialization cleanup. Commit `350ffbe` completed Task 4: `previous_response_id` requests use WebSocket `response.create` and return SSE-compatible output. The current pending stage partially completes Task 5 for non-streaming `/v1/responses`: 429/402/403 account-state classification and fallback retry across imported accounts. Remaining major work continues with streaming/Chat fallback, model refresh/cache, and scoped admin operations. --- @@ -72,13 +72,16 @@ - Modify: `src/http/v1.rs` - Modify: `src/accounts/pool.rs` - Modify: `src/accounts/repository.rs` -- Test: `tests/upstream_fallback_test.rs` +- Test: `tests/v1_upstream_route_test.rs` - Test: `tests/account_pool_scheduling_test.rs` -- [ ] Write failing tests for 429 quota marking, retry-after handling, fallback account retry, refresh retry preservation, and exhausted no-account responses. -- [ ] Run: `cargo test --test upstream_fallback_test --test account_pool_scheduling_test` -- [ ] Add error classification and fallback acquire/release paths without adding proxy-pool support. -- [ ] Run: `cargo test --test upstream_fallback_test --test account_pool_scheduling_test` +- [x] Write failing tests for non-streaming Responses 429 retry-after handling, fallback account retry, 402 quota exhaustion, 403 banned classification, and Cloudflare 403 cooldown. +- [x] Run targeted tests: `cargo test --test v1_upstream_route_test v1_responses_should_retry_next_account_after_429_retry_after`; `cargo test --test v1_upstream_route_test v1_responses_should_mark_quota_exhausted_after_402_and_retry_next_account`; `cargo test --test v1_upstream_route_test v1_responses_should_mark_banned_after_403_and_retry_next_account`; `cargo test --test v1_upstream_route_test v1_responses_should_cool_down_cloudflare_403_and_retry_next_account` +- [x] Add non-streaming Responses error classification and fallback acquire/release paths without adding proxy-pool support. +- [x] Run: `cargo test --test v1_upstream_route_test` +- [x] Run: `cargo test --test account_pool_scheduling_test` +- [ ] Extend fallback classification and account retry to streaming HTTP SSE, WebSocket-backed Responses streaming, and Chat Completions. +- [ ] Add tests for exhausted no-account responses, refresh retry preservation under fallback, successful rate-limit header capture, and durable quota cooldown persistence if needed. ### Task 6: Model Catalog Refresh diff --git a/docs/superpowers/specs/2026-06-12-openai-gpt-codex-parity-design.md b/docs/superpowers/specs/2026-06-12-openai-gpt-codex-parity-design.md index 60dbe71ef..3547efddd 100644 --- a/docs/superpowers/specs/2026-06-12-openai-gpt-codex-parity-design.md +++ b/docs/superpowers/specs/2026-06-12-openai-gpt-codex-parity-design.md @@ -18,6 +18,8 @@ OpenAI Chat compatibility is implemented as a real route, not an alias to Respon Responses compatibility keeps native Responses passthrough semantics but adds the TS implementation's in-scope fields: reasoning/service tier, tool choice, parallel tool calls, text formats, prompt cache key, include, client metadata, and Codex context headers. `previous_response_id` uses WebSocket transport rather than being rejected. +Current staged progress: Tasks 1-4 are implemented for the OpenAI GPT/Codex scope. Task 5 is partially staged for non-streaming `/v1/responses`: upstream 429/402/403 responses classify account state and retry another imported account when available. Streaming Responses, WebSocket streaming fallback, and Chat Completions fallback remain Task 5 follow-ups. + ## Components - `src/translation/openai_to_codex.rs`: OpenAI Chat request schema and Chat-to-Codex translation, including messages, tools/functions, multimodal input, response formats, reasoning effort, and service tier. @@ -38,7 +40,7 @@ For Responses, the route validates the local client API key, parses native Respo ## Error Handling -Client authentication errors stay OpenAI-compatible. Model misses return `model_not_found`. Upstream 401 triggers refresh and one retry for the same account. 429 and quota exhaustion update account state and retry another account when available. Retryable transport/empty-response failures can retry another account. Terminal upstream errors are formatted as OpenAI-compatible errors for Chat and Responses-compatible failed events for streaming Responses. +Client authentication errors stay OpenAI-compatible. Model misses return `model_not_found`. Upstream 401 triggers refresh and one retry for the same account. For non-streaming Responses, 429 applies a quota cooldown and records the failed attempt, 402 marks quota exhausted, Cloudflare 403 applies a short cooldown, non-Cloudflare 403 marks banned, and each retryable account-state error tries another imported account when available. Retryable transport/empty-response failures can retry another account in a later Task 5 pass. Terminal upstream errors are formatted as OpenAI-compatible errors for Chat and Responses-compatible failed events for streaming Responses. ## Testing diff --git a/src/accounts/model.rs b/src/accounts/model.rs index 956678f0c..813650168 100644 --- a/src/accounts/model.rs +++ b/src/accounts/model.rs @@ -24,6 +24,7 @@ pub struct Account { pub access_token_expires_at: Option>, pub status: AccountStatus, pub quota_limit_reached: bool, + pub quota_cooldown_until: Option>, pub cloudflare_cooldown_until: Option>, pub added_at: String, pub last_used_at: Option, @@ -43,6 +44,7 @@ impl Account { access_token_expires_at: Some(Utc::now() + Duration::hours(1)), status, quota_limit_reached: false, + quota_cooldown_until: None, cloudflare_cooldown_until: None, added_at: Utc::now().to_rfc3339(), last_used_at: None, diff --git a/src/accounts/pool.rs b/src/accounts/pool.rs index f01391324..d4b8b7fb6 100644 --- a/src/accounts/pool.rs +++ b/src/accounts/pool.rs @@ -108,6 +108,44 @@ impl AccountPool { self.accounts.insert(account.id.clone(), account); } + pub fn set_status(&mut self, account_id: &str, status: AccountStatus) -> bool { + let Some(account) = self.accounts.get_mut(account_id) else { + return false; + }; + account.status = status; + if status != AccountStatus::Active { + self.slots.remove(account_id); + } + true + } + + pub fn mark_quota_limited_until( + &mut self, + account_id: &str, + cooldown_until: DateTime, + ) -> bool { + let Some(account) = self.accounts.get_mut(account_id) else { + return false; + }; + account.quota_limit_reached = true; + account.quota_cooldown_until = Some(cooldown_until); + self.slots.remove(account_id); + true + } + + pub fn set_cloudflare_cooldown_until( + &mut self, + account_id: &str, + cooldown_until: DateTime, + ) -> bool { + let Some(account) = self.accounts.get_mut(account_id) else { + return false; + }; + account.cloudflare_cooldown_until = Some(cooldown_until); + self.slots.remove(account_id); + true + } + pub fn acquire(&mut self, model: &str) -> Option { self.acquire_with(AccountAcquireRequest::new(model, Utc::now())) .map(|acquired| acquired.account) @@ -153,8 +191,7 @@ impl AccountPool { .accounts .values() .filter(|account| { - account.status == AccountStatus::Active - && (!self.options.skip_quota_limited || !account.quota_limit_reached) + account.status == AccountStatus::Active && self.is_quota_available(account, now) }) .count(); let total_slots = active_accounts * self.options.max_concurrent_per_account; @@ -223,7 +260,7 @@ impl AccountPool { fn is_base_available(&self, account: &Account, request: &AccountAcquireRequest) -> bool { account.status == AccountStatus::Active && self.slot_count(&account.id) < self.options.max_concurrent_per_account - && (!self.options.skip_quota_limited || !account.quota_limit_reached) + && self.is_quota_available(account, request.now) && self.is_model_allowed(account, &request.model) && !request .exclude_account_ids @@ -234,6 +271,15 @@ impl AccountPool { .is_none_or(|cooldown_until| request.now >= cooldown_until) } + fn is_quota_available(&self, account: &Account, now: DateTime) -> bool { + if !self.options.skip_quota_limited || !account.quota_limit_reached { + return true; + } + account + .quota_cooldown_until + .is_some_and(|cooldown_until| now >= cooldown_until) + } + fn slot_count(&self, account_id: &str) -> usize { self.slots.get(account_id).map_or(0, VecDeque::len) } diff --git a/src/accounts/repository.rs b/src/accounts/repository.rs index 83405117f..ccaf7a8ee 100644 --- a/src/accounts/repository.rs +++ b/src/accounts/repository.rs @@ -284,6 +284,7 @@ impl AccountRepository { access_token_expires_at: stored.access_token_expires_at, status: stored.status, quota_limit_reached: false, + quota_cooldown_until: None, cloudflare_cooldown_until: None, added_at: stored.added_at.to_rfc3339(), last_used_at: row.get("usage_last_used_at"), diff --git a/src/codex/client.rs b/src/codex/client.rs index d1622b092..f97154262 100644 --- a/src/codex/client.rs +++ b/src/codex/client.rs @@ -1,7 +1,8 @@ use reqwest::{ - header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE, COOKIE, SET_COOKIE}, + header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE, COOKIE, RETRY_AFTER, SET_COOKIE}, Client, Response as ReqwestResponse, StatusCode, }; +use serde_json::Value; use thiserror::Error; use crate::{ @@ -33,7 +34,11 @@ pub enum CodexClientError { #[error("invalid upstream SSE response: {0}")] InvalidSse(#[from] SseError), #[error("upstream returned status {status}: {body}")] - Upstream { status: StatusCode, body: String }, + Upstream { + status: StatusCode, + body: String, + retry_after_seconds: Option, + }, } pub type CodexClientResult = Result; @@ -108,9 +113,15 @@ impl CodexBackendClient { let status = response.status(); let turn_state = turn_state(&response); let set_cookie_headers = set_cookie_headers(&response); + let retry_after_seconds = retry_after_seconds(response.headers(), None); let body = response.text().await?; if !status.is_success() { - return Err(CodexClientError::Upstream { status, body }); + return Err(CodexClientError::Upstream { + status, + retry_after_seconds: retry_after_seconds + .or_else(|| retry_after_seconds_from_body(&body)), + body, + }); } let usage = extract_sse_usage(&body)?; @@ -132,8 +143,14 @@ impl CodexBackendClient { let turn_state = turn_state(&response); let set_cookie_headers = set_cookie_headers(&response); if !status.is_success() { + let retry_after_seconds = retry_after_seconds(response.headers(), None); let body = response.text().await?; - return Err(CodexClientError::Upstream { status, body }); + return Err(CodexClientError::Upstream { + status, + retry_after_seconds: retry_after_seconds + .or_else(|| retry_after_seconds_from_body(&body)), + body, + }); } Ok(CodexBackendStream { @@ -248,3 +265,30 @@ fn set_cookie_headers(response: &ReqwestResponse) -> Vec { .filter_map(|value| value.to_str().ok().map(ToString::to_string)) .collect() } + +fn retry_after_seconds(headers: &HeaderMap, body: Option<&str>) -> Option { + headers + .get(RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse::().ok()) + .filter(|seconds| *seconds > 0) + .or_else(|| body.and_then(retry_after_seconds_from_body)) +} + +fn retry_after_seconds_from_body(body: &str) -> Option { + let value = serde_json::from_str::(body).ok()?; + let error = value.get("error").unwrap_or(&value); + if let Some(seconds) = error + .get("resets_in_seconds") + .and_then(Value::as_u64) + .filter(|seconds| *seconds > 0) + { + return Some(seconds); + } + let resets_at = error.get("resets_at").and_then(Value::as_u64)?; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok()? + .as_secs(); + (resets_at > now).then_some(resets_at - now) +} diff --git a/src/http/admin.rs b/src/http/admin.rs index d9ae55de8..6811de680 100644 --- a/src/http/admin.rs +++ b/src/http/admin.rs @@ -786,6 +786,7 @@ pub async fn import_accounts( access_token_expires_at: None, status, quota_limit_reached: false, + quota_cooldown_until: None, cloudflare_cooldown_until: None, added_at: chrono::Utc::now().to_rfc3339(), last_used_at: None, diff --git a/src/http/v1.rs b/src/http/v1.rs index 523ad2e38..d656e66bc 100644 --- a/src/http/v1.rs +++ b/src/http/v1.rs @@ -10,7 +10,7 @@ use axum::{ response::{IntoResponse, Response}, Extension, Json, }; -use chrono::Utc; +use chrono::{Duration, Utc}; use futures::{stream, StreamExt}; use reqwest::Url; use secrecy::SecretString; @@ -49,6 +49,9 @@ use crate::{ }; const MODEL_CREATED_TIMESTAMP: i64 = 1_700_000_000; +const DEFAULT_RATE_LIMIT_BACKOFF_SECONDS: u64 = 60; +const MAX_RATE_LIMIT_BACKOFF_SECONDS: u64 = 86_400 * 7; +const CLOUDFLARE_CHALLENGE_COOLDOWN_SECONDS: u64 = 10; #[derive(Deserialize)] struct ResponsesBody { @@ -159,8 +162,8 @@ pub async fn responses( let Some(acquired) = acquired else { return no_available_accounts_response().into_response(); }; - let account = acquired.account; - let log_context = V1LogContext::new( + let mut account = acquired.account; + let mut log_context = V1LogContext::new( request_id.as_str(), &account.id, &codex_request.model, @@ -172,29 +175,62 @@ pub async fn responses( return responses_stream(state, codex_request, account, log_context).await; } - let response = send_codex_request_with_refresh_retry( - &state, - &codex_request, - &account, - request_id.as_str(), - ) - .await; - state.account_pool().lock().await.release(&account.id); + let mut excluded_account_ids = Vec::new(); + let response = loop { + let response = send_codex_request_with_refresh_retry( + &state, + &codex_request, + &account, + request_id.as_str(), + ) + .await; + state.account_pool().lock().await.release(&account.id); - let response = match response { - Ok(response) => response, - Err(error) => { - let error_response = codex_client_error_response(error); - log_v1_response( - &state, - &log_context, - error_response.0, - EventLevel::Error, - "v1 responses upstream request failed", - json!({"stream": false}), - ) - .await; - return error_response.into_response(); + match response { + Ok(response) => break response, + Err(error) => { + if let Some(retry) = classify_upstream_account_retry(&error) { + apply_upstream_account_retry(&state, &account, retry).await; + excluded_account_ids.push(account.id.clone()); + log_v1_response( + &state, + &log_context, + retry.status(), + EventLevel::Warn, + "v1 responses upstream retrying with fallback account", + retry.metadata(false), + ) + .await; + let fallback = { + state.account_pool().lock().await.acquire_with( + AccountAcquireRequest::new(&codex_request.model, Utc::now()) + .with_exclude_account_ids(excluded_account_ids.iter().cloned()), + ) + }; + if let Some(fallback) = fallback { + account = fallback.account; + log_context = V1LogContext::new( + request_id.as_str(), + &account.id, + &codex_request.model, + client_stream, + started_at, + ); + continue; + } + } + let error_response = codex_client_error_response(error); + log_v1_response( + &state, + &log_context, + error_response.0, + EventLevel::Error, + "v1 responses upstream request failed", + json!({"stream": false}), + ) + .await; + return error_response.into_response(); + } } }; if persist_upstream_cookies(&state, &account.id, &response.set_cookie_headers) @@ -587,11 +623,19 @@ async fn send_codex_request_with_refresh_retry( request_id: &str, ) -> Result { match send_codex_request(state, request, account, request_id).await { - Err(CodexClientError::Upstream { status, body }) if status == StatusCode::UNAUTHORIZED => { + Err(CodexClientError::Upstream { + status, + body, + retry_after_seconds, + }) if status == StatusCode::UNAUTHORIZED => { let Some(refreshed) = refresh_account_after_unauthorized(state, request, account, request_id).await else { - return Err(CodexClientError::Upstream { status, body }); + return Err(CodexClientError::Upstream { + status, + body, + retry_after_seconds, + }); }; send_codex_request(state, request, &refreshed, request_id).await } @@ -642,11 +686,19 @@ async fn send_codex_stream_request_with_refresh_retry( request_id: &str, ) -> Result { match send_codex_stream_request(state, request, account, request_id).await { - Err(CodexClientError::Upstream { status, body }) if status == StatusCode::UNAUTHORIZED => { + Err(CodexClientError::Upstream { + status, + body, + retry_after_seconds, + }) if status == StatusCode::UNAUTHORIZED => { let Some(refreshed) = refresh_account_after_unauthorized(state, request, account, request_id).await else { - return Err(CodexClientError::Upstream { status, body }); + return Err(CodexClientError::Upstream { + status, + body, + retry_after_seconds, + }); }; send_codex_stream_request(state, request, &refreshed, request_id).await } @@ -690,6 +742,146 @@ async fn send_codex_stream_request( .await } +#[derive(Debug, Clone, Copy)] +enum UpstreamAccountRetry { + RateLimited { retry_after_seconds: u64 }, + QuotaExhausted, + CloudflareChallenge { cooldown_seconds: u64 }, + Banned, +} + +impl UpstreamAccountRetry { + fn status(self) -> StatusCode { + match self { + Self::RateLimited { .. } => StatusCode::TOO_MANY_REQUESTS, + Self::QuotaExhausted => StatusCode::PAYMENT_REQUIRED, + Self::CloudflareChallenge { .. } => StatusCode::FORBIDDEN, + Self::Banned => StatusCode::FORBIDDEN, + } + } + + fn metadata(self, stream: bool) -> Value { + match self { + Self::RateLimited { + retry_after_seconds, + } => json!({ + "stream": stream, + "retry": true, + "reason": "rateLimited", + "retryAfterSeconds": retry_after_seconds, + }), + Self::QuotaExhausted => json!({ + "stream": stream, + "retry": true, + "reason": "quotaExhausted", + }), + Self::CloudflareChallenge { cooldown_seconds } => json!({ + "stream": stream, + "retry": true, + "reason": "cloudflareChallenge", + "cooldownSeconds": cooldown_seconds, + }), + Self::Banned => json!({ + "stream": stream, + "retry": true, + "reason": "banned", + }), + } + } +} + +fn classify_upstream_account_retry(error: &CodexClientError) -> Option { + match error { + CodexClientError::Upstream { + status, + retry_after_seconds, + .. + } if *status == StatusCode::TOO_MANY_REQUESTS => Some(UpstreamAccountRetry::RateLimited { + retry_after_seconds: retry_after_seconds + .unwrap_or(DEFAULT_RATE_LIMIT_BACKOFF_SECONDS) + .min(MAX_RATE_LIMIT_BACKOFF_SECONDS), + }), + CodexClientError::Upstream { status, .. } if *status == StatusCode::PAYMENT_REQUIRED => { + Some(UpstreamAccountRetry::QuotaExhausted) + } + CodexClientError::Upstream { status, body, .. } if *status == StatusCode::FORBIDDEN => { + if is_cloudflare_challenge(body) { + Some(UpstreamAccountRetry::CloudflareChallenge { + cooldown_seconds: CLOUDFLARE_CHALLENGE_COOLDOWN_SECONDS, + }) + } else { + Some(UpstreamAccountRetry::Banned) + } + } + _ => None, + } +} + +fn is_cloudflare_challenge(body: &str) -> bool { + let lower = body.to_ascii_lowercase(); + lower.contains("cf-mitigated") + || lower.contains("cf-chl-bypass") + || lower.contains("_cf_chl") + || lower.contains("cf_chl") + || lower.contains("attention required") + || lower.contains("just a moment") +} + +async fn apply_upstream_account_retry( + state: &AppState, + account: &Account, + retry: UpstreamAccountRetry, +) { + match retry { + UpstreamAccountRetry::RateLimited { + retry_after_seconds, + } => { + let cooldown_until = Utc::now() + Duration::seconds(retry_after_seconds as i64); + state + .account_pool() + .lock() + .await + .mark_quota_limited_until(&account.id, cooldown_until); + if record_request_attempt(state, &account.id).await.is_err() { + tracing::warn!( + account_id = %account.id, + "failed to record rate-limited account attempt" + ); + } + } + UpstreamAccountRetry::QuotaExhausted => { + set_account_status(state, account, AccountStatus::QuotaExhausted).await; + } + UpstreamAccountRetry::CloudflareChallenge { cooldown_seconds } => { + let cooldown_until = Utc::now() + Duration::seconds(cooldown_seconds as i64); + state + .account_pool() + .lock() + .await + .set_cloudflare_cooldown_until(&account.id, cooldown_until); + } + UpstreamAccountRetry::Banned => { + set_account_status(state, account, AccountStatus::Banned).await; + } + } +} + +async fn set_account_status(state: &AppState, account: &Account, status: AccountStatus) { + if let Some(repo) = state.account_repository() { + if repo.set_status(&account.id, status).await.is_err() { + tracing::warn!( + account_id = %account.id, + "failed to persist upstream account status" + ); + } + } + state + .account_pool() + .lock() + .await + .set_status(&account.id, status); +} + async fn responses_stream( state: AppState, request: CodexResponsesRequest, @@ -1052,6 +1244,22 @@ async fn record_usage(state: &AppState, account_id: &str, usage: TokenUsage) -> .map_err(|_| ()) } +async fn record_request_attempt(state: &AppState, account_id: &str) -> Result<(), ()> { + let Some(repo) = state.account_repository() else { + return Ok(()); + }; + repo.record_usage( + account_id, + UsageDelta { + input_tokens: 0, + output_tokens: 0, + cached_tokens: 0, + }, + ) + .await + .map_err(|_| ()) +} + #[derive(Clone)] struct V1LogContext { request_id: String, @@ -1343,7 +1551,7 @@ fn codex_client_error_response(error: CodexClientError) -> (StatusCode, Json ( + CodexClientError::Upstream { status, body, .. } => ( status, Json(openai_error( &format!( diff --git a/tests/account_pool_scheduling_test.rs b/tests/account_pool_scheduling_test.rs index bc1e2815d..876d05283 100644 --- a/tests/account_pool_scheduling_test.rs +++ b/tests/account_pool_scheduling_test.rs @@ -54,6 +54,28 @@ fn account_pool_should_skip_accounts_with_cached_quota_limit() { assert_eq!(pool.acquire("gpt-5.5").unwrap().id, "usable"); } +#[test] +fn account_pool_should_reuse_quota_limited_accounts_after_cooldown() { + let now = fixed_time(); + let mut pool = AccountPool::default(); + pool.insert(Account::test("limited", AccountStatus::Active)); + pool.mark_quota_limited_until("limited", now + Duration::seconds(30)); + + assert!(pool + .acquire_with(AccountAcquireRequest::new("gpt-5.5", now)) + .is_none()); + assert_eq!( + pool.acquire_with(AccountAcquireRequest::new( + "gpt-5.5", + now + Duration::seconds(31) + )) + .unwrap() + .account + .id, + "limited" + ); +} + #[test] fn account_pool_should_prefer_configured_tier_priority() { let mut pool = AccountPool::with_options(AccountPoolOptions { diff --git a/tests/v1_upstream_route_test.rs b/tests/v1_upstream_route_test.rs index 7822ebf1e..064f46e70 100644 --- a/tests/v1_upstream_route_test.rs +++ b/tests/v1_upstream_route_test.rs @@ -56,6 +56,13 @@ struct FailingTokenRefresher { failure: RefreshFailure, } +struct ImportAccount { + id: &'static str, + account_id: &'static str, + token: &'static str, + refresh_token: &'static str, +} + #[async_trait] impl TokenRefresher for StaticTokenRefresher { async fn refresh(&self, _refresh_token: &str) -> Result { @@ -129,6 +136,22 @@ fn test_config(database_url: String, base_url: String) -> AppConfig { } async fn build_imported_app(base_url: String) -> ImportedApp { + build_imported_app_with_accounts( + base_url, + &[ImportAccount { + id: "acct_imported", + account_id: "chatgpt-account", + token: "access-secret", + refresh_token: "refresh-secret", + }], + ) + .await +} + +async fn build_imported_app_with_accounts( + base_url: String, + accounts: &[ImportAccount], +) -> ImportedApp { let tempdir = tempfile::tempdir().unwrap(); let db = tempdir.path().join("v1-upstream.sqlite"); let url = format!("sqlite://{}", db.display()); @@ -147,6 +170,18 @@ async fn build_imported_app(base_url: String) -> ImportedApp { secret_box.clone(), hasher, )); + let accounts = accounts + .iter() + .map(|account| { + json!({ + "id": account.id, + "accountId": account.account_id, + "token": account.token, + "refreshToken": account.refresh_token, + "status": "active" + }) + }) + .collect::>(); let import_response = app .clone() @@ -156,18 +191,7 @@ async fn build_imported_app(base_url: String) -> ImportedApp { .uri("/admin/accounts/import") .header("content-type", "application/json") .header("cookie", "cpr_admin_session=session_1") - .body(Body::from( - json!({ - "accounts": [{ - "id": "acct_imported", - "accountId": "chatgpt-account", - "token": "access-secret", - "refreshToken": "refresh-secret", - "status": "active" - }] - }) - .to_string(), - )) + .body(Body::from(json!({ "accounts": accounts }).to_string())) .unwrap(), ) .await @@ -337,6 +361,327 @@ async fn v1_responses_should_use_imported_account_and_record_usage() { assert_eq!(event.4["usage"]["inputTokens"], 7); } +#[tokio::test] +async fn v1_responses_should_retry_next_account_after_429_retry_after() { + let server = MockServer::start().await; + let success_sse = concat!( + "event: response.completed\n", + "data: {\"response\":{\"id\":\"resp_after_429\",\"object\":\"response\",\"usage\":{\"input_tokens\":5,\"output_tokens\":2}}}\n", + "\n", + ); + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-a")) + .respond_with( + ResponseTemplate::new(429) + .insert_header("retry-after", "120") + .set_body_json(json!({ + "error": { + "message": "rate limited", + "resets_in_seconds": 120 + } + })), + ) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-b")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(success_sse), + ) + .mount(&server) + .await; + let imported = build_imported_app_with_accounts( + server.uri(), + &[ + ImportAccount { + id: "acct_a", + account_id: "chatgpt-a", + token: "access-a", + refresh_token: "refresh-a", + }, + ImportAccount { + id: "acct_b", + account_id: "chatgpt-b", + token: "access-b", + refresh_token: "refresh-b", + }, + ], + ) + .await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header( + "authorization", + format!("Bearer {}", imported.client_api_key), + ) + .header("content-type", "application/json") + .body(Body::from( + r#"{"model":"gpt-5.5","input":[],"stream":false}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["id"], "resp_after_429"); + let usage_a: (i64, i64, i64) = sqlx::query_as( + "select request_count, input_tokens, output_tokens from account_usage where account_id = ?", + ) + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(usage_a, (1, 0, 0)); + let usage_b: (i64, i64, i64) = sqlx::query_as( + "select request_count, input_tokens, output_tokens from account_usage where account_id = ?", + ) + .bind("acct_b") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(usage_b, (1, 5, 2)); +} + +#[tokio::test] +async fn v1_responses_should_mark_quota_exhausted_after_402_and_retry_next_account() { + let server = MockServer::start().await; + let success_sse = concat!( + "event: response.completed\n", + "data: {\"response\":{\"id\":\"resp_after_402\",\"object\":\"response\",\"usage\":{\"input_tokens\":6,\"output_tokens\":3}}}\n", + "\n", + ); + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-a")) + .respond_with(ResponseTemplate::new(402).set_body_json(json!({ + "error": {"message": "quota exhausted"} + }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-b")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(success_sse), + ) + .mount(&server) + .await; + let imported = build_imported_app_with_accounts( + server.uri(), + &[ + ImportAccount { + id: "acct_a", + account_id: "chatgpt-a", + token: "access-a", + refresh_token: "refresh-a", + }, + ImportAccount { + id: "acct_b", + account_id: "chatgpt-b", + token: "access-b", + refresh_token: "refresh-b", + }, + ], + ) + .await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header( + "authorization", + format!("Bearer {}", imported.client_api_key), + ) + .header("content-type", "application/json") + .body(Body::from( + r#"{"model":"gpt-5.5","input":[],"stream":false}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["id"], "resp_after_402"); + let account_status: (String,) = sqlx::query_as("select status from accounts where id = ?") + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(account_status.0, "quota_exhausted"); +} + +#[tokio::test] +async fn v1_responses_should_mark_banned_after_403_and_retry_next_account() { + let server = MockServer::start().await; + let success_sse = concat!( + "event: response.completed\n", + "data: {\"response\":{\"id\":\"resp_after_403\",\"object\":\"response\",\"usage\":{\"input_tokens\":6,\"output_tokens\":3}}}\n", + "\n", + ); + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-a")) + .respond_with(ResponseTemplate::new(403).set_body_json(json!({ + "error": {"message": "account banned"} + }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-b")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(success_sse), + ) + .mount(&server) + .await; + let imported = build_imported_app_with_accounts( + server.uri(), + &[ + ImportAccount { + id: "acct_a", + account_id: "chatgpt-a", + token: "access-a", + refresh_token: "refresh-a", + }, + ImportAccount { + id: "acct_b", + account_id: "chatgpt-b", + token: "access-b", + refresh_token: "refresh-b", + }, + ], + ) + .await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header( + "authorization", + format!("Bearer {}", imported.client_api_key), + ) + .header("content-type", "application/json") + .body(Body::from( + r#"{"model":"gpt-5.5","input":[],"stream":false}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["id"], "resp_after_403"); + let account_status: (String,) = sqlx::query_as("select status from accounts where id = ?") + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(account_status.0, "banned"); +} + +#[tokio::test] +async fn v1_responses_should_cool_down_cloudflare_403_and_retry_next_account() { + let server = MockServer::start().await; + let success_sse = concat!( + "event: response.completed\n", + "data: {\"response\":{\"id\":\"resp_after_cf\",\"object\":\"response\",\"usage\":{\"input_tokens\":6,\"output_tokens\":3}}}\n", + "\n", + ); + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-a")) + .respond_with(ResponseTemplate::new(403).set_body_string( + "Just a moment...cf_chl challenge", + )) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-b")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_string(success_sse), + ) + .mount(&server) + .await; + let imported = build_imported_app_with_accounts( + server.uri(), + &[ + ImportAccount { + id: "acct_a", + account_id: "chatgpt-a", + token: "access-a", + refresh_token: "refresh-a", + }, + ImportAccount { + id: "acct_b", + account_id: "chatgpt-b", + token: "access-b", + refresh_token: "refresh-b", + }, + ], + ) + .await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header( + "authorization", + format!("Bearer {}", imported.client_api_key), + ) + .header("content-type", "application/json") + .body(Body::from( + r#"{"model":"gpt-5.5","input":[],"stream":false}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["id"], "resp_after_cf"); + let account_status: (String,) = sqlx::query_as("select status from accounts where id = ?") + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(account_status.0, "active"); +} + #[tokio::test] async fn v1_responses_should_forward_parity_fields_and_context_headers() { let server = MockServer::start().await;