diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 532922fe3..5330af875 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -15,15 +15,15 @@ Update this file before each feature commit. | Diagnostics | Completed | `pending` | `cargo test --test diagnostics_route_test admin_diagnostics`; `cargo test --test diagnostics_route_test debug_upstream`; `cargo test --test diagnostics_route_test` | Adds admin-session-gated `GET /admin/diagnostics` plus local-only `GET /debug/diagnostics`, `GET /debug/fingerprint`, and `GET /debug/upstream` with runtime package metadata, config/database/log/security paths, Codex backend transport settings, static Desktop fingerprint summary and standalone fingerprint debug output, account repository/pool state, runtime capacity summary, retained settings summary, and a Codex models endpoint connectivity probe without tokens or upstream bodies. Remote forwarded debug requests are rejected with `403`. | | TLS headers and fingerprint | Completed | `1bdd08c` | `cargo test codex_headers_include_desktop_identity_and_turn_state`; `cargo tree \| rg 'reqwest\|rustls'`; full `fmt/test/clippy` before commit | Adds Codex Desktop fingerprint model, exact identity/request headers, and pinned reqwest/rustls client builder. | | Cookie persistence | Completed | `4710926` + `5d3cc6e` + `203270f` | `cargo test cookie_jar_captures_and_replays_account_scoped_cookies`; `cargo test cookie_repository_encrypts_and_replays_account_scoped_cookies`; full `fmt/test/clippy` before `203270f` | Adds account-scoped in-memory Cookie jar and encrypted SQLite Cookie repository; Cookies are replayed only for the same account/domain. Admin-pasted browser Cookie headers are stored encrypted and can be cleared per account. | -| Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` + `3ecd5b6` + `3960a35` + `db6cb98` + `93cfee3` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test codex_websocket_test`; Task 5 targeted `cargo test --test v1_upstream_route_test`, `cargo test --test chat_completions_route_test`, `cargo test --test account_pool_scheduling_test`, and WebSocket fallback route tests; Task 7 targeted `cargo test --test admin_accounts_route_test`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `fmt/test/clippy` before `db6cb98` | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, OpenAI OAuth device-code polling into the shared validated account import path, OpenAI OAuth PKCE authorization-code exchange into the same path, 401-triggered refresh and single retry, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process manual/import/CLI/device/PKCE import-to-pool sync, startup pool snapshots from SQLite with usage-based last-used metadata, in-memory 429 quota cooldown marking, Cloudflare cooldown mutation, pool status mutation for upstream fallback, admin label/status/delete/runtime refresh synchronization, admin reset-usage last-used clearing, runtime pool clearing on logout, shared fallback acquisition for non-streaming Responses, HTTP SSE Responses setup, Chat Completions, and non-history WebSocket handshakes before request body send. `previous_response_id` WebSocket requests remain account-affine on classified failures. Still missing durable release-time slot state and durable quota cooldown persistence because the current account schema has no dedicated cooldown column. | -| Account repository | Completed | `14fe706` + `db6cb98` + `72438c6` + `869b6f8` + `93cfee3` + `a32a53c` + `b879dd6` | `cargo test --test account_repository_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `72438c6`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; quota warnings targeted and full gate before `b879dd6` | Adds encrypted SQLite account token storage, decrypting reads, metadata-only reads for auth status, cursor pagination, full export reads, status/label updates, account deletion, account-wide delete for logout, release-time usage counter persistence, refresh-token-preserving token updates, ChatGPT account/user identity lookup, JWT-claim metadata/token updates that preserve existing labels, quota JSON caching, quota warning snapshot reads, and per-account local usage counter reset. | +| Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` + `3ecd5b6` + `3960a35` + `db6cb98` + `93cfee3` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` + `pending` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test account_pool_startup_test app_state_should_restore_persisted_cooldowns_into_runtime_pool`; `cargo test --test codex_websocket_test`; Task 5 targeted `cargo test --test v1_upstream_route_test`, `cargo test --test chat_completions_route_test`, `cargo test --test account_pool_scheduling_test`, and WebSocket fallback route tests; Task 7 targeted `cargo test --test admin_accounts_route_test`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `fmt/test/clippy` before `db6cb98` | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, OpenAI OAuth device-code polling into the shared validated account import path, OpenAI OAuth PKCE authorization-code exchange into the same path, 401-triggered refresh and single retry, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process manual/import/CLI/device/PKCE import-to-pool sync, startup pool snapshots from SQLite with usage-based last-used metadata and persisted quota/Cloudflare cooldown state, SQLite-backed 429 quota cooldown marking, Cloudflare cooldown mutation, pool status mutation for upstream fallback, admin label/status/delete/runtime refresh synchronization, admin reset-usage last-used clearing, runtime pool clearing on logout, shared fallback acquisition for non-streaming Responses, HTTP SSE Responses setup, Chat Completions, and non-history WebSocket handshakes before request body send. `previous_response_id` WebSocket requests remain account-affine on classified failures. Still missing durable release-time slot state. | +| Account repository | Completed | `14fe706` + `db6cb98` + `72438c6` + `869b6f8` + `93cfee3` + `a32a53c` + `b879dd6` + `pending` | `cargo test --test account_repository_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo test --test account_pool_startup_test app_state_should_restore_persisted_cooldowns_into_runtime_pool`; `cargo test --test v1_upstream_fallback_test v1_responses_should_`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `72438c6`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; quota warnings targeted and full gate before `b879dd6` | Adds encrypted SQLite account token storage, decrypting reads, metadata-only reads for auth status, cursor pagination, full export reads, status/label updates, account deletion, account-wide delete for logout, release-time usage counter persistence, refresh-token-preserving token updates, ChatGPT account/user identity lookup, JWT-claim metadata/token updates that preserve existing labels, quota JSON caching, quota warning snapshot reads, persisted quota/Cloudflare cooldown fields for runtime pool restore, and per-account local usage counter reset. | | Translation | Completed | `eff182e` + `b9c30ba` | `cargo test --test routes_chat_test`; `cargo test --test chat_completions_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` | Adds OpenAI Chat request types, expanded Codex Responses request type, OpenAI Chat to Codex translation for system/developer instructions, multimodal content, tool/function calls, response formats, reasoning effort, service tier, and parallel tool calls. Codex SSE now converts back to OpenAI chat completions and chat chunks with text deltas, reasoning content, tool calls, usage details, and OpenAI-compatible error bodies. | | HTTP routes | Partial | `ab0ad8f` + `3830b21` + `6179d44` + `cfec8ba` + `6341336` + `228febd` + `c2cf7a6` + `e9fdea0` + `ee0ed90` + `fdb9374` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `3ecd5b6` + `3960a35` + `1738f24` + `0b6cc7e` + `f75db60` + `4b19846` + `db6cb98` + `16eb4ed` + `8af350d` + `203270f` + `960bbb2` + `72438c6` + `16938e6` + `869b6f8` + `e1f35e0` + `93cfee3` + `bfe561d` + `cf5ff0b` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` | `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test models_route_returns_openai_compatible_codex_model_list`; `cargo test --test admin_settings_route_test`; `cargo test --test model_catalog_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test cli_auth_import_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test chat_completions_route_test`; `cargo test --test codex_websocket_test`; targeted HTTP SSE error tests before `bfe561d`; targeted local API key import/export tests before `cf5ff0b`; WebSocket fallback targeted route tests; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe`; Task 5 targeted verification before `3ecd5b6` and `3960a35`; full `fmt/test/clippy` before `1738f24`; full `fmt/test/clippy` plus real-file smoke before `0b6cc7e`; full `fmt/test/clippy` before `f75db60`; full `fmt/test/clippy` before `4b19846`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `16eb4ed`; full `fmt/test/clippy` before `8af350d`; full `fmt/test/clippy` before `203270f`; full `fmt/test/clippy` before `960bbb2`; full `fmt/test/clippy` before `72438c6`; full `fmt/test/clippy` before `16938e6` | Adds Axum router, health route, `/v1/models`, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/debug/models`, `/debug/diagnostics`, `/debug/fingerprint`, `/debug/upstream`, `/admin/diagnostics`, `/admin/auth/status`, `/admin/auth/logout`, `/admin/auth/login-start`, `/admin/auth/code-relay`, `/admin/auth/callback`, `/auth/callback`, `/admin/auth/device-login`, `/admin/auth/device-poll/{device_code}`, `/admin/accounts`, `/admin/accounts/quota-warnings`, `/admin/accounts/export`, `/admin/accounts/batch-delete`, `/admin/accounts/batch-status`, `/admin/accounts/health-check`, `/admin/accounts/{id}`, `/admin/accounts/{id}/label`, `/admin/accounts/{id}/status`, `/admin/accounts/{id}/refresh`, `/admin/accounts/{id}/reset-usage`, `/admin/accounts/{id}/quota`, `/admin/accounts/{id}/cookies`, `/admin/accounts/import`, `/admin/accounts/import-cli`, `/admin/api-keys`, `/admin/api-keys/export`, `/admin/api-keys/import`, `/admin/api-keys/batch-delete`, `/admin/api-keys/{id}`, `/admin/api-keys/{id}/label`, `/admin/api-keys/{id}/status`, `/admin/usage-stats`, `/admin/usage-stats/summary`, `/admin/refresh-models`, `/v1/responses` HTTP SSE collection with output item/delta reconstruction plus `output_text` sync for non-streaming clients, default streaming Responses passthrough when `stream` is omitted, WebSocket-backed Responses for `previous_response_id`, explicit non-history WebSocket Responses via `use_websocket`/`useWebSocket`, non-streaming and HTTP SSE setup fallback to another imported account after 429/402/403 account-state failures, WebSocket setup fallback after classified handshake failures when no `previous_response_id` is present, Responses field parity for service tier, tool choice, parallel tool calls, text format, prompt cache key, include, client metadata, reasoning include default, and Codex context headers, plus a dedicated `/v1/chat/completions` route backed by imported Codex accounts with fallback for buffered non-streaming and client-streaming output conversion. `/v1` unauthorized and model errors remain OpenAI-compatible, cached backend model snapshots are used by `/v1/models*` when present, global request-id response headers are preserved, `gpt-5.5` remains the default/test model baseline, and `/admin/settings` supports admin-session-gated read/write access with lower camelCase envelopes. `/admin/auth/status` returns sanitized authenticated state, first account metadata, and pool summary without decrypting or returning tokens. `/admin/auth/logout` clears all account rows and the runtime pool while relying on SQLite foreign-key cascades for usage/cookies. `/admin/auth/login-start` creates a short-lived in-memory PKCE session and returns the OpenAI authorization URL; `/admin/auth/code-relay`, `/admin/auth/callback`, and the registered `/auth/callback` redirect path exchange returned authorization codes and import tokens through the same JWT-claim validated path. `/admin/auth/device-login` starts the OpenAI OAuth device-code flow, and `/admin/auth/device-poll/{device_code}` handles pending/slow_down states or imports returned tokens without returning secrets. `/admin/accounts` reads metadata only and does not decrypt tokens for list views. Manual `POST /admin/accounts` accepts only `token` and/or `refreshToken`, rejects invalid/expired JWTs and missing ChatGPT account claims, derives metadata from JWT claims, ignores caller metadata/status/label, exchanges refresh-token-only inputs through `AppState`'s OpenAI token refresher, preserves or rotates refresh tokens, stores secrets encrypted, synchronizes the runtime account pool, updates existing accounts by `chatgpt_account_id` + `chatgpt_user_id`, and never echoes tokens. `/admin/accounts/quota-warnings` returns active warning/critical quota windows computed from cached SQLite quota JSON without contacting upstream. `/admin/accounts/import` accepts native and Sub2API OpenAI OAuth account exports, returns `sourceFormat`, and ignores proxy-only metadata. `/admin/accounts/import-cli` reads Codex CLI `auth.json` from `$CODEX_HOME` or an explicit admin fixture path, uses the same validated JWT-claim account import path as manual add, ignores caller metadata, and returns only sanitized import counts. `/admin/accounts/export` returns native or Sub2API OpenAI OAuth exports without proxy fields. `/admin/accounts/{id}/label`, `/status`, `/refresh`, `/reset-usage`, `/quota`, delete, batch-delete, batch-status, and `/admin/accounts/health-check` mutate or inspect SQLite plus runtime account pool without returning secrets; health/quota call Codex backend usage endpoints through imported account tokens and classify invalid/quota/banned/Cloudflare responses into account state or cooldown. `/admin/accounts/{id}/cookies` stores/reads/deletes encrypted per-account browser cookies. `/admin/api-keys/{id}/label`, `/status`, delete, and batch-delete mutate local client key rows used by `/v1` auth; `/admin/api-keys/export` returns metadata only without plaintext/keyHash/pepper, and `/admin/api-keys/import` rotates metadata into new local keys with plaintext returned once. These are local auth utilities, not TS provider key pool parity. `/admin/refresh-models` fetches backend model lists through imported accounts only, stores plan snapshots, and syncs runtime model-plan allowlists. Still missing live Chat streaming transform. | | Admin quota warnings | Completed | `b879dd6` | `cargo test --test admin_accounts_cookies_quota_test quota_warnings`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo fmt --check`; `cargo test`; `cargo clippy --all-targets --all-features --locked -- -D warnings` | Adds `/admin/accounts/quota-warnings`, backed by cached SQLite quota snapshots and configured warning thresholds. The route is admin-session gated, returns lower camelCase admin envelopes, and does not call upstream Codex while listing active warnings. | | Admin/frontend response contract | Completed | `1c34ea7` + `b0c10f2` | `cargo test admin_envelope_serializes_lower_camel_case_request_id`; `cargo test admin_logs_reject_missing_admin_session_cookie`; `cargo test v1_response_has_request_id_header_without_admin_body_field`; full `fmt/test/clippy` before commit | Adds reusable admin envelopes with body codes, lower camelCase JSON, paginated list shape, requestId bodies, admin session cookie enforcement, and keeps `/v1/*` OpenAI-compatible. | | HTTP auth boundary | Completed | `cec4607` + `e9fdea0` | `cargo test --test http_auth_test`; `cargo test --test routes_responses_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_logs_route_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds a real `src/http/auth.rs` boundary for client API key and admin session cookie parsing. `/v1/*` accepts only enabled local `Bearer cpr_...` client API keys verified from SQLite, admin session parsing ignores client API keys, and route handlers no longer own ad hoc credential parsing. | | Model catalog | Partial | `6341336` + `1738f24` + `f75db60` + `4b19846` | `cargo test --test model_catalog_test`; `cargo test model_catalog_route_returns_codex_metadata_without_alias_entries`; `cargo test --test storage_schema_test`; `cargo test --test routes_responses_test model_catalog_route_returns_cached_backend_models`; `cargo test --test admin_models_route_test`; full `fmt/test/clippy` before `1738f24`; full `fmt/test/clippy` before `f75db60`; full `fmt/test/clippy` before `4b19846` | Adds a native static Codex model catalog with `gpt-5.5` default, configured aliases, reasoning/service-tier suffix parsing including `none` and `minimal`, OpenAI-compatible `/v1/models`, extended catalog/detail/info routes, debug summary, SQLite-backed backend model snapshots by plan, cached backend catalog merge, model-to-plan allowlist derivation, and an admin refresh route that probes Codex model endpoints with one imported account per plan and syncs the runtime account-pool allowlist. Still missing full model metadata parity. | -| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` + `5bf012c` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `3ecd5b6` + `3960a35` + `bfe561d` + `6d18d2c` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test codex_websocket_test`; `cargo test --test admin_accounts_route_test`; Task 5 targeted `cargo test --test v1_upstream_route_test`, `cargo test --test chat_completions_route_test`, `cargo test --test account_pool_scheduling_test`, and WebSocket route tests; HTTP SSE error mapping targeted `cargo test --test v1_upstream_route_test v1_responses_non_stream_should_return_upstream_error` and `cargo test --test v1_upstream_route_test v1_responses_stream_should_passthrough`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, sends WebSocket `response.create` requests when `previous_response_id` requires upstream history, keeps `previous_response_id` requests account-affine on classified WebSocket handshake failures for streaming and non-streaming clients, replays/captures encrypted Cookies, captures successful WebSocket rate-limit headers, forwards Codex request context headers, fetches Codex usage/quota through imported account tokens for admin health/quota operations, normalizes `service_tier: fast` to upstream `priority`, collects `response.completed`, reconstructs missing assistant output from `response.output_item.done` or text deltas, synchronizes `output_text` for non-streaming clients, maps non-streaming HTTP SSE `error` and `response.failed` terminal frames to OpenAI-compatible `upstream_error` bodies, passes through Responses streaming SSE by default while marking `error` and `response.failed` terminal frames as failed lifecycle logs, adapts Chat Completions responses/chunks, refreshes and retries once after upstream 401 when a refresh token is stored, parses Retry-After plus Codex reset metadata from upstream errors, records 429 request attempts, classifies upstream 429/402/403 into account cooldown/quota/banned/Cloudflare state changes, retries non-streaming `/v1/responses`, HTTP SSE `/v1/responses` setup, non-history WebSocket setup, and `/v1/chat/completions` with another imported account when available, records token usage, writes v1 lifecycle event logs, records refresh failure classification for upstream 401, and restores the account pool from SQLite on startup. Still missing durable 429 quota cooldown persistence because the current account schema has no dedicated cooldown column, and live Chat streaming transform. | +| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` + `5bf012c` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `3ecd5b6` + `3960a35` + `bfe561d` + `6d18d2c` + `pending` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test v1_upstream_fallback_test v1_responses_should_`; `cargo test --test account_pool_startup_test`; `cargo test --test account_pool_startup_test app_state_should_restore_persisted_cooldowns_into_runtime_pool`; `cargo test --test codex_websocket_test`; `cargo test --test admin_accounts_route_test`; Task 5 targeted `cargo test --test v1_upstream_route_test`, `cargo test --test chat_completions_route_test`, `cargo test --test account_pool_scheduling_test`, and WebSocket route tests; HTTP SSE error mapping targeted `cargo test --test v1_upstream_route_test v1_responses_non_stream_should_return_upstream_error` and `cargo test --test v1_upstream_route_test v1_responses_stream_should_passthrough`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, sends WebSocket `response.create` requests when `previous_response_id` requires upstream history, keeps `previous_response_id` requests account-affine on classified WebSocket handshake failures for streaming and non-streaming clients, replays/captures encrypted Cookies, captures successful WebSocket rate-limit headers, forwards Codex request context headers, fetches Codex usage/quota through imported account tokens for admin health/quota operations, normalizes `service_tier: fast` to upstream `priority`, collects `response.completed`, reconstructs missing assistant output from `response.output_item.done` or text deltas, synchronizes `output_text` for non-streaming clients, maps non-streaming HTTP SSE `error` and `response.failed` terminal frames to OpenAI-compatible `upstream_error` bodies, passes through Responses streaming SSE by default while marking `error` and `response.failed` terminal frames as failed lifecycle logs, adapts Chat Completions responses/chunks, refreshes and retries once after upstream 401 when a refresh token is stored, parses Retry-After plus Codex reset metadata from upstream errors, records 429 request attempts, persists 429 quota cooldowns and Cloudflare cooldowns to SQLite, classifies upstream 429/402/403 into account cooldown/quota/banned/Cloudflare state changes, retries non-streaming `/v1/responses`, HTTP SSE `/v1/responses` setup, non-history WebSocket setup, and `/v1/chat/completions` with another imported account when available, records token usage, writes v1 lifecycle event logs, records refresh failure classification for upstream 401, and restores the account pool from SQLite on startup. Still missing path-block Cookie clearing, richer OpenAI-compatible error mapping, and live Chat streaming transform. | | Codex WebSocket boundary | Completed | `cc0516d` + `b9c30ba` + `350ffbe` + `6d18d2c` | `cargo test --test codex_websocket_test`; `cargo test --test v1_upstream_route_test v1_responses_should_use_websocket_for_previous_response_id_streaming`; WebSocket route fallback tests; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe` | Adds `src/codex/websocket.rs` with tested transport classification for HTTP SSE versus WebSocket-only requests. `previous_response_id` and explicit WebSocket requests use typed transport selection, local-only `use_websocket` is not serialized into upstream JSON, WebSocket `response.create` carries Codex headers and parity fields, incoming WebSocket messages are emitted as SSE-compatible events, handshake HTTP failures surface as upstream errors before request body send, successful handshake headers capture turn state, cookies, and rate-limit headers, non-history WebSocket setup can fallback to another imported account, and `previous_response_id` WebSocket failures remain on the original account. | | Fingerprint updates | Partial | `dd26d31` + `e32220a` | `cargo test update_manifest_updates_app_version_and_build_number`; `cargo test fingerprint_updater_fetches_manifest_and_persists_history`; full `fmt/test/clippy` before commit | Parses a JSON update manifest and persists app version/build number history to SQLite. Still missing original appcast XML polling, update-state persistence, runtime header fingerprint selection from the latest stored fingerprint, and optional local Codex.app Chromium extraction. | | Runtime docs and packaging | Completed | `33b63f3` | `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; `cargo run`; `curl -s -i http://127.0.0.1:8080/health` | README scope, root runtime config, Dockerfile, compose file, and health smoke test are complete. | diff --git a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md index ec67700a8..b77244a63 100644 --- a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md +++ b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md @@ -2979,8 +2979,8 @@ The Rust project has no legacy compatibility requirement, but the in-scope behav - [x] **Diagnostics:** expand health/admin diagnostics to include authenticated state, pool summary, capacity summary, transport/fingerprint status, paths, runtime metadata, and test-connection checks. `/admin/diagnostics` returns the non-secret runtime summary behind admin-session auth. Local-only `/debug/diagnostics` returns runtime metadata, paths, account repository/pool state, capacity summary, transport config, and static fingerprint status without secrets. Local-only `/debug/fingerprint` returns the runtime Desktop fingerprint summary. Local-only `/debug/upstream` probes the Codex models endpoint with runtime TLS/fingerprint headers and returns only status metadata. Keep production-local gating for sensitive debug endpoints. - [ ] **Logs and usage stats:** add `/admin/logs/state`, clear/detail, error-log equivalents if retained, and richer bounded history queries if needed. `GET/PATCH /admin/logs/state`, `/admin/logs/{id}`, `DELETE /admin/logs`, `/admin/logs` kind/level/request/account/route/model/status/search filters, log file rotation, SQLite event pagination, `/admin/usage-stats` pagination, and `/admin/usage-stats/summary` are complete. Remaining: decide whether to retain old error-log grouping/raw/count/seen/report APIs and richer bounded body-history queries. - [x] **Model catalog:** replace the hard-coded `/v1/models` list with a model store that supports static defaults, configured aliases, `-low/-medium/-high/-xhigh/-fast/-flex` suffix parsing, backend model fetching per plan, cache persistence, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/admin/refresh-models`, and `/debug/models`. Static defaults, aliases, suffix parsing, cached backend snapshots, plan allowlists, admin refresh, read-only model routes, and debug summary are complete. -- [ ] **Account scheduling parity:** extend account acquisition beyond `active + least last_used`. Completed in the in-memory pool: max concurrent slots per account, stale slot cleanup, least-used/round-robin/sticky strategies, quota-cache skip, tier priority, model-plan filtering, exclude IDs, preferred account from session affinity, Cloudflare cooldown skip, request-staggering metadata, capacity summary, settings-backed construction, import-to-pool sync, startup pool snapshots from SQLite, HTTP fallback acquisition across alternate accounts, WebSocket fallback for non-history requests, previous-response account affinity, and runtime pool clearing on logout. Remaining: durable release-time slot state and durable quota-cache/cooldown restore. -- [ ] **Upstream Codex lifecycle:** implement account acquire -> Codex request -> SSE/collect-or-stream -> rate-limit capture -> usage release -> retry/fallback. HTTP SSE collection, streaming passthrough, 401-triggered refresh retry, Codex request context fields, SSE `error`/`response.failed` mapping, 429/402/403 account-state classification, successful WebSocket rate-limit capture, and fallback across alternate accounts for non-streaming Responses, HTTP SSE setup, non-history WebSocket setup, and Chat Completions are complete. Remaining: durable 429 quota cooldown persistence, path-block Cookie clearing, and richer OpenAI-compatible error mapping. +- [ ] **Account scheduling parity:** extend account acquisition beyond `active + least last_used`. Completed in the in-memory pool: max concurrent slots per account, stale slot cleanup, least-used/round-robin/sticky strategies, quota-cache skip, tier priority, model-plan filtering, exclude IDs, preferred account from session affinity, Cloudflare cooldown skip, request-staggering metadata, capacity summary, settings-backed construction, import-to-pool sync, startup pool snapshots from SQLite, durable quota/Cloudflare cooldown restore, HTTP fallback acquisition across alternate accounts, WebSocket fallback for non-history requests, previous-response account affinity, and runtime pool clearing on logout. Remaining: durable release-time slot state. +- [ ] **Upstream Codex lifecycle:** implement account acquire -> Codex request -> SSE/collect-or-stream -> rate-limit capture -> usage release -> retry/fallback. HTTP SSE collection, streaming passthrough, 401-triggered refresh retry, Codex request context fields, SSE `error`/`response.failed` mapping, 429/402/403 account-state classification, durable 429 quota cooldown persistence, successful WebSocket rate-limit capture, and fallback across alternate accounts for non-streaming Responses, HTTP SSE setup, non-history WebSocket setup, and Chat Completions are complete. Remaining: path-block Cookie clearing and richer OpenAI-compatible error mapping. - [ ] **Session affinity and previous response:** persist an in-memory response-id to account mapping with TTL, prompt-cache/conversation identity, turn-state replay, and preferred account selection. `src/codex/websocket.rs` implements the WebSocket `response.create` path for `previous_response_id`, and classified WebSocket failures for `previous_response_id` stay account-affine for streaming and non-streaming clients. Response-id TTL cleanup and broader affinity persistence still need to be finished. - [ ] **Fingerprint auto-update parity:** poll the real Codex Desktop appcast/update source, parse app version/build number, persist update state, select the latest stored fingerprint for runtime headers, and optionally extract Chromium version from a local Codex.app path when configured. @@ -2998,7 +2998,7 @@ This pass re-read the TypeScript routes and core modules under `/home/zyy/Codes/ - [ ] **Logs and error diagnostics:** Rust has cursor-paginated SQLite event logs, rotating files, `GET/PATCH /admin/logs/state`, detail-by-id, clear, and admin log filters for kind, level, request ID, account ID, route, model, status code, and message/metadata search. Missing direction/body-history filters, bounded in-memory request/response body capture, and the original local error-log grouping/raw/count/seen/report/clear APIs if retained for troubleshooting. - [ ] **Usage and quota background jobs:** Rust records cumulative account usage from `/v1/*` and computes admin quota warnings from cached SQLite quota snapshots. Missing usage history snapshots (`raw`, `five_min`, `hourly`, `daily`), retention cleanup, passive quota-window recovery, active quota refresher for dirty/limit-reached accounts, and startup background scheduling. - [ ] **Model catalog:** Rust has a static default catalog, aliases, suffix parsing, read routes, debug summary, backend Codex model fetch/probe, per-plan snapshots, model-plan index feeding account scheduling, `/admin/refresh-models`, and runtime cache persistence. Missing richer metadata fields and optional external model-cache loading if retained. -- [ ] **Account scheduling and retry:** Rust has in-memory slots, stale cleanup, strategies, tier priority, preferred/exclude IDs, SQLite startup restore, request-staggering metadata, 429/402/403 fallback across alternate accounts, Cloudflare cooldown skip, successful WebSocket rate-limit header application, non-history WebSocket fallback, previous-response history affinity, and status/token/logout mutation sync for the runtime pool. Missing strict request-interval sleep enforcement, durable quota/cache restore, Cloudflare path-block tracker, cooldown clearing, and richer capacity/health endpoints. +- [ ] **Account scheduling and retry:** Rust has in-memory slots, stale cleanup, strategies, tier priority, preferred/exclude IDs, SQLite startup restore, durable quota/Cloudflare cooldown restore, request-staggering metadata, 429/402/403 fallback across alternate accounts, Cloudflare cooldown skip, successful WebSocket rate-limit header application, non-history WebSocket fallback, previous-response history affinity, and status/token/logout mutation sync for the runtime pool. Missing strict request-interval sleep enforcement, Cloudflare path-block tracker, cooldown clearing, and richer capacity/health endpoints. - [ ] **Refresh scheduler parity:** Rust has typed refresh policy and 401-triggered refresh retry. Missing long-running refresh timers at `exp - margin`, crash recovery for `refreshing`/`expired` accounts, exponential backoff, recovery scheduling, permanent-failure thresholding, per-account in-flight suppression, cross-process refresh locks, and disk refresh-token sync before consuming one-time RTs. - [ ] **Codex Desktop fingerprint and transport parity:** Rust uses pinned `reqwest`/`rustls` and core headers, but exact parity is not proven. Missing header-order/default-header parity (`sec-ch-ua`, accept language/encoding/fetch headers), `x-codex-installation-id`, runtime selection of the newest stored fingerprint, Sparkle appcast XML polling, update-state/version-state persistence, optional local Codex.app Chromium extraction, and real Desktop TLS/header capture comparison. `x-openai-internal-codex-residency` currently needs real-client verification because the builder inserts `global` and the reqwest client overwrites it with `us`. - [x] **Health and diagnostics:** Rust `/health` returns `status: ok`, `/admin/auth/status` returns admin-session-gated authenticated state plus account pool summary, `/admin/diagnostics` returns the non-secret runtime diagnostics summary, local-only `/debug/diagnostics` returns capacity summary, transport/fingerprint status, paths, runtime metadata, and retained settings summary without secrets, and local-only `/debug/upstream` probes the Codex models endpoint without returning tokens or upstream bodies. diff --git a/migrations/0005_account_cooldowns.sql b/migrations/0005_account_cooldowns.sql new file mode 100644 index 000000000..b3f77fde2 --- /dev/null +++ b/migrations/0005_account_cooldowns.sql @@ -0,0 +1,3 @@ +alter table accounts add column quota_limit_reached integer not null default 0; +alter table accounts add column quota_cooldown_until text; +alter table accounts add column cloudflare_cooldown_until text; diff --git a/src/codex/accounts/repository.rs b/src/codex/accounts/repository.rs index 3027bc04a..311a26708 100644 --- a/src/codex/accounts/repository.rs +++ b/src/codex/accounts/repository.rs @@ -322,7 +322,7 @@ impl AccountRepository { pub async fn list_pool_accounts(&self) -> AccountRepositoryResult> { let rows = sqlx::query( - "select accounts.id, email, accounts.account_id, user_id, label, plan_type, access_token_cipher, refresh_token_cipher, access_token_expires_at, status, added_at, updated_at, account_usage.last_used_at as usage_last_used_at from accounts left join account_usage on account_usage.account_id = accounts.id order by added_at desc, accounts.id desc", + "select accounts.id, email, accounts.account_id, user_id, label, plan_type, access_token_cipher, refresh_token_cipher, access_token_expires_at, status, added_at, updated_at, quota_limit_reached, quota_cooldown_until, cloudflare_cooldown_until, account_usage.last_used_at as usage_last_used_at from accounts left join account_usage on account_usage.account_id = accounts.id order by added_at desc, accounts.id desc", ) .fetch_all(&self.pool) .await?; @@ -345,9 +345,13 @@ impl AccountRepository { refresh_token, access_token_expires_at: stored.access_token_expires_at, status: stored.status, - quota_limit_reached: false, - quota_cooldown_until: None, - cloudflare_cooldown_until: None, + quota_limit_reached: row.get::("quota_limit_reached") != 0, + quota_cooldown_until: parse_optional_rfc3339( + row.get::, _>("quota_cooldown_until"), + )?, + cloudflare_cooldown_until: parse_optional_rfc3339( + row.get::, _>("cloudflare_cooldown_until"), + )?, added_at: stored.added_at.to_rfc3339(), last_used_at: row.get("usage_last_used_at"), }); @@ -545,6 +549,38 @@ impl AccountRepository { Ok(result.rows_affected() > 0) } + pub async fn set_quota_cooldown_until( + &self, + id: &str, + cooldown_until: DateTime, + ) -> AccountRepositoryResult { + let result = sqlx::query( + "update accounts set quota_limit_reached = 1, quota_cooldown_until = ?, updated_at = ? where id = ?", + ) + .bind(cooldown_until.to_rfc3339()) + .bind(Utc::now().to_rfc3339()) + .bind(id) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn set_cloudflare_cooldown_until( + &self, + id: &str, + cooldown_until: DateTime, + ) -> AccountRepositoryResult { + let result = sqlx::query( + "update accounts set cloudflare_cooldown_until = ?, updated_at = ? where id = ?", + ) + .bind(cooldown_until.to_rfc3339()) + .bind(Utc::now().to_rfc3339()) + .bind(id) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() > 0) + } + fn account_from_row( &self, row: &sqlx::sqlite::SqliteRow, diff --git a/src/codex/accounts/service/health.rs b/src/codex/accounts/service/health.rs index 05080a3a8..7c873656c 100644 --- a/src/codex/accounts/service/health.rs +++ b/src/codex/accounts/service/health.rs @@ -174,6 +174,9 @@ pub(super) async fn apply_codex_account_error( retry_after_seconds, }) => { let cooldown_until = Utc::now() + chrono::Duration::seconds(retry_after_seconds as i64); + let _ = repo + .set_quota_cooldown_until(&account.id, cooldown_until) + .await; service .account_pool .lock() @@ -183,6 +186,9 @@ pub(super) async fn apply_codex_account_error( } Some(CodexAccountErrorAction::CloudflareChallenge { cooldown_seconds }) => { let cooldown_until = Utc::now() + chrono::Duration::seconds(cooldown_seconds as i64); + let _ = repo + .set_cloudflare_cooldown_until(&account.id, cooldown_until) + .await; service .account_pool .lock() diff --git a/src/codex/upstream/mod.rs b/src/codex/upstream/mod.rs index 934f67a66..03b8bd0d0 100644 --- a/src/codex/upstream/mod.rs +++ b/src/codex/upstream/mod.rs @@ -451,6 +451,18 @@ async fn apply_upstream_account_retry_with_deps( retry_after_seconds, } => { let cooldown_until = Utc::now() + Duration::seconds(retry_after_seconds as i64); + if let Some(repo) = deps.account_repository.as_ref() { + if repo + .set_quota_cooldown_until(&account.id, cooldown_until) + .await + .is_err() + { + tracing::warn!( + account_id = %account.id, + "failed to persist quota cooldown" + ); + } + } deps.account_pool .lock() .await @@ -467,6 +479,18 @@ async fn apply_upstream_account_retry_with_deps( } UpstreamAccountRetry::CloudflareChallenge { cooldown_seconds } => { let cooldown_until = Utc::now() + Duration::seconds(cooldown_seconds as i64); + if let Some(repo) = deps.account_repository.as_ref() { + if repo + .set_cloudflare_cooldown_until(&account.id, cooldown_until) + .await + .is_err() + { + tracing::warn!( + account_id = %account.id, + "failed to persist Cloudflare cooldown" + ); + } + } deps.account_pool .lock() .await diff --git a/tests/account_pool_startup_test.rs b/tests/account_pool_startup_test.rs index d410c0f4c..c0cdbb581 100644 --- a/tests/account_pool_startup_test.rs +++ b/tests/account_pool_startup_test.rs @@ -1,4 +1,4 @@ -use chrono::Utc; +use chrono::{Duration, Utc}; use secrecy::SecretString; use codex_proxy_rs::{ @@ -123,3 +123,51 @@ async fn app_state_should_restore_account_pool_from_sqlite_accounts() { assert_eq!(acquired.access_token, "access-secret"); assert!(acquired.last_used_at.is_some()); } + +#[tokio::test] +async fn app_state_should_restore_persisted_cooldowns_into_runtime_pool() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("startup-pool-cooldown.sqlite"); + let url = format!("sqlite://{}", db.display()); + let pool = connect_sqlite(&url).await.unwrap(); + let secret_box = SecretBox::new([32u8; 32]); + let repo = codex_proxy_rs::codex::accounts::repository::AccountRepository::new( + pool.clone(), + secret_box.clone(), + ); + repo.insert(NewAccount { + id: "acct_cooling".to_string(), + email: Some("cooling@example.com".to_string()), + account_id: Some("chatgpt-cooling".to_string()), + user_id: None, + label: None, + plan_type: Some("plus".to_string()), + access_token: SecretString::new("access-cooling".to_string().into()), + refresh_token: Some(SecretString::new("refresh-cooling".to_string().into())), + access_token_expires_at: Some(Utc::now() + Duration::hours(1)), + status: AccountStatus::Active, + }) + .await + .unwrap(); + let cooldown_until = (Utc::now() + Duration::minutes(10)).to_rfc3339(); + sqlx::query( + "update accounts set quota_limit_reached = 1, quota_cooldown_until = ?, cloudflare_cooldown_until = ? where id = ?", + ) + .bind(&cooldown_until) + .bind(&cooldown_until) + .bind("acct_cooling") + .execute(&pool) + .await + .unwrap(); + let state = AppState::with_pool_and_secret_box(test_config(url), pool, secret_box); + + let restored = state.reload_account_pool_from_repository().await.unwrap(); + + assert_eq!(restored, 1); + assert!(state + .services + .accounts + .acquire_runtime_account("gpt-5.5") + .await + .is_none()); +} diff --git a/tests/v1_upstream_fallback_test.rs b/tests/v1_upstream_fallback_test.rs index c21d9ad02..75e778d49 100644 --- a/tests/v1_upstream_fallback_test.rs +++ b/tests/v1_upstream_fallback_test.rs @@ -2,6 +2,7 @@ use axum::{ body::Body, http::{Request, StatusCode}, }; +use chrono::{DateTime, Utc}; use secrecy::ExposeSecret; use serde_json::{json, Value}; use tower::ServiceExt; @@ -36,6 +37,7 @@ const STREAM_AFTER_REFRESH_SSE: &str = #[tokio::test] async fn v1_responses_should_retry_next_account_after_429_retry_after() { + let started_at = Utc::now(); let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -120,6 +122,18 @@ async fn v1_responses_should_retry_next_account_after_429_retry_after() { .await .unwrap(); assert_eq!(usage_b, (1, 5, 2)); + let cooldown: (i64, Option) = sqlx::query_as( + "select quota_limit_reached, quota_cooldown_until from accounts where id = ?", + ) + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(cooldown.0, 1); + let cooldown_until = DateTime::parse_from_rfc3339(cooldown.1.as_deref().unwrap()) + .unwrap() + .with_timezone(&Utc); + assert!(cooldown_until > started_at); } #[tokio::test] @@ -342,6 +356,7 @@ async fn v1_responses_should_mark_banned_after_403_and_retry_next_account() { #[tokio::test] async fn v1_responses_should_cool_down_cloudflare_403_and_retry_next_account() { + let started_at = Utc::now(); let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/codex/responses")) @@ -409,6 +424,16 @@ async fn v1_responses_should_cool_down_cloudflare_403_and_retry_next_account() { .await .unwrap(); assert_eq!(account_status.0, "active"); + let (cooldown_until,): (Option,) = + sqlx::query_as("select cloudflare_cooldown_until from accounts where id = ?") + .bind("acct_a") + .fetch_one(&imported.pool) + .await + .unwrap(); + let cooldown_until = DateTime::parse_from_rfc3339(cooldown_until.as_deref().unwrap()) + .unwrap() + .with_timezone(&Utc); + assert!(cooldown_until > started_at); } #[tokio::test]