From f66c69c7fa4b970c916417171deccd1cfd3f1409 Mon Sep 17 00:00:00 2001 From: zyycn Date: Sat, 13 Jun 2026 11:16:34 +0800 Subject: [PATCH] feat: add upstream diagnostics probe Co-authored-by: Codex --- docs/api.md | 20 +++ docs/implementation-status.md | 4 +- .../plans/2026-06-11-codex-proxy-rs.md | 4 +- src/app/router.rs | 3 +- src/app/state.rs | 8 +- src/codex/transport/client.rs | 24 ++++ src/http/diagnostics.rs | 29 +++- src/service/diagnostics.rs | 132 ++++++++++++++++++ src/service/mod.rs | 1 + tests/diagnostics_route_test.rs | 76 ++++++++++ 10 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 src/service/diagnostics.rs diff --git a/docs/api.md b/docs/api.md index 9490bd5ab..f8735836d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -58,6 +58,26 @@ Success response: } ``` +### `GET /debug/upstream` + +Runs a local-only Codex backend connectivity probe against the Codex models endpoint using the runtime TLS and Desktop fingerprint headers. The route rejects forwarded remote requests using `x-forwarded-for` or `x-real-ip`, returns only status metadata, and does not expose account tokens or upstream response bodies. + +Success response for a reachable backend that rejects the synthetic diagnostic token: + +```json +{ + "target": "codexModels", + "status": "reachable", + "backendBaseUrl": "https://chatgpt.com/backend-api", + "endpoint": "https://chatgpt.com/backend-api/codex/models?client_version=26.519.81530", + "reachable": true, + "statusCode": 401, + "authorization": "rejected", + "durationMs": 23, + "error": null +} +``` + ### `GET /debug/diagnostics` Returns a local-only runtime diagnostics summary. The route is intended for production-local troubleshooting and rejects forwarded remote requests using `x-forwarded-for` or `x-real-ip`. diff --git a/docs/implementation-status.md b/docs/implementation-status.md index edd9c78bd..038f3048d 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -12,13 +12,13 @@ Update this file before each feature commit. | Admin auth and client API keys | Partial | `24d5799` + `4e0d1e5` + `e9fdea0` + `8af350d` + `960bbb2` | `cargo test client_api_key_has_proxy_prefix_and_verifies_against_hash`; `cargo test admin_password_hash_is_not_a_client_api_key`; `cargo test --test admin_login_route_test`; `cargo test --test admin_api_keys_route_test`; `cargo fmt --check`; `cargo test`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; full `fmt/test/clippy` before `8af350d`; full `fmt/test/clippy` before `960bbb2`; current local key import/export targeted `cargo test --test admin_api_keys_route_test` | Admin passwords use Argon2id, client API keys use `cpr_` prefix plus HMAC-SHA256 with server-side pepper; `/admin/login` issues HttpOnly sessions and rejects client API keys as admin credentials. `/admin/api-keys` can create/list local client keys, status-enable/status-disable local client keys, label local client keys, delete local client keys, batch-delete local client keys, export local client key metadata, import metadata by rotating to new local `cpr_` keys, and `/v1/*` verifies only enabled keys against SQLite instead of accepting any `cpr_` string. These routes are Rust-local client auth utilities, not TypeScript third-party provider API-key parity. Provider API key pools and provider key import/export remain out of scope; local `cpr_` key plaintext and key hashes are not exportable after HMAC storage, and import returns newly generated plaintext values only once. | | Secret encryption | Completed | `105aa58` + `228febd` | `cargo test secret_box_encrypts_and_decrypts_without_storing_plaintext`; `cargo test secret_box_load_or_create_should_reuse_existing_master_key_file`; full `fmt/test/clippy` before commit | AES-256-GCM `SecretBox` stores upstream secrets as `v1::` without plaintext and can load/create the runtime master key file used by repositories. | | Logging and pagination | Completed | `eb0341e` + `fc16c48` + `b0c10f2` + `ee0ed90` + `fdb9374` + `5bf012c` + `pending` | `cargo test event_logs_are_cursor_paginated`; `cargo test admin_logs_are_cursor_paginated_and_include_request_id`; `cargo test --test admin_logs_route_test admin_logs_`; `cargo test rotating_writer_splits_files_when_size_limit_is_reached`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, admin `/admin/logs` query/detail/state/clear APIs, admin `/admin/usage-stats` and `/admin/usage-stats/summary`, event log repository, runtime tracing initialization, HTTP trace layer, date/size-based file rotation, v1 response lifecycle events with request/account/route/model/status/latency/non-secret metadata, and non-secret `account.refresh` events for upstream-401 refresh failures. | -| Diagnostics | Partial | `pending` | `cargo test --test diagnostics_route_test` | Adds local-only `GET /debug/diagnostics` and `GET /debug/fingerprint` with runtime package metadata, config/database/log/security paths, Codex backend transport settings, static Desktop fingerprint summary and standalone fingerprint debug output, account repository/pool state, runtime capacity summary, and retained settings summary without tokens or client API keys. Remote forwarded requests are rejected with `403`. Still missing active upstream test-connection checks and richer admin diagnostics surfaces. | +| Diagnostics | Partial | `pending` | `cargo test --test diagnostics_route_test debug_upstream`; `cargo test --test diagnostics_route_test` | Adds local-only `GET /debug/diagnostics`, `GET /debug/fingerprint`, and `GET /debug/upstream` with runtime package metadata, config/database/log/security paths, Codex backend transport settings, static Desktop fingerprint summary and standalone fingerprint debug output, account repository/pool state, runtime capacity summary, retained settings summary, and a Codex models endpoint connectivity probe without tokens or upstream bodies. Remote forwarded requests are rejected with `403`. Still missing richer admin diagnostics surfaces. | | TLS headers and fingerprint | Completed | `1bdd08c` | `cargo test codex_headers_include_desktop_identity_and_turn_state`; `cargo tree \| rg 'reqwest\|rustls'`; full `fmt/test/clippy` before commit | Adds Codex Desktop fingerprint model, exact identity/request headers, and pinned reqwest/rustls client builder. | | Cookie persistence | Completed | `4710926` + `5d3cc6e` + `203270f` | `cargo test cookie_jar_captures_and_replays_account_scoped_cookies`; `cargo test cookie_repository_encrypts_and_replays_account_scoped_cookies`; full `fmt/test/clippy` before `203270f` | Adds account-scoped in-memory Cookie jar and encrypted SQLite Cookie repository; Cookies are replayed only for the same account/domain. Admin-pasted browser Cookie headers are stored encrypted and can be cleared per account. | | Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` + `3ecd5b6` + `3960a35` + `db6cb98` + `93cfee3` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; `cargo test --test codex_websocket_test`; Task 5 targeted `cargo test --test v1_upstream_route_test`, `cargo test --test chat_completions_route_test`, `cargo test --test account_pool_scheduling_test`, and WebSocket fallback route tests; Task 7 targeted `cargo test --test admin_accounts_route_test`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `fmt/test/clippy` before `db6cb98` | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, OpenAI OAuth device-code polling into the shared validated account import path, OpenAI OAuth PKCE authorization-code exchange into the same path, 401-triggered refresh and single retry, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process manual/import/CLI/device/PKCE import-to-pool sync, startup pool snapshots from SQLite with usage-based last-used metadata, in-memory 429 quota cooldown marking, Cloudflare cooldown mutation, pool status mutation for upstream fallback, admin label/status/delete/runtime refresh synchronization, admin reset-usage last-used clearing, runtime pool clearing on logout, shared fallback acquisition for non-streaming Responses, HTTP SSE Responses setup, Chat Completions, and non-history WebSocket handshakes before request body send. `previous_response_id` WebSocket requests remain account-affine on classified failures. Still missing durable release-time slot state and durable quota cooldown persistence because the current account schema has no dedicated cooldown column. | | Account repository | Completed | `14fe706` + `db6cb98` + `72438c6` + `869b6f8` + `93cfee3` + `a32a53c` + `b879dd6` | `cargo test --test account_repository_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `72438c6`; auth status/logout targeted `cargo test --test admin_accounts_route_test`; quota warnings targeted and full gate before `b879dd6` | Adds encrypted SQLite account token storage, decrypting reads, metadata-only reads for auth status, cursor pagination, full export reads, status/label updates, account deletion, account-wide delete for logout, release-time usage counter persistence, refresh-token-preserving token updates, ChatGPT account/user identity lookup, JWT-claim metadata/token updates that preserve existing labels, quota JSON caching, quota warning snapshot reads, and per-account local usage counter reset. | | Translation | Completed | `eff182e` + `b9c30ba` | `cargo test --test routes_chat_test`; `cargo test --test chat_completions_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` | Adds OpenAI Chat request types, expanded Codex Responses request type, OpenAI Chat to Codex translation for system/developer instructions, multimodal content, tool/function calls, response formats, reasoning effort, service tier, and parallel tool calls. Codex SSE now converts back to OpenAI chat completions and chat chunks with text deltas, reasoning content, tool calls, usage details, and OpenAI-compatible error bodies. | -| HTTP routes | Partial | `ab0ad8f` + `3830b21` + `6179d44` + `cfec8ba` + `6341336` + `228febd` + `c2cf7a6` + `e9fdea0` + `ee0ed90` + `fdb9374` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `3ecd5b6` + `3960a35` + `1738f24` + `0b6cc7e` + `f75db60` + `4b19846` + `db6cb98` + `16eb4ed` + `8af350d` + `203270f` + `960bbb2` + `72438c6` + `16938e6` + `869b6f8` + `e1f35e0` + `93cfee3` + `bfe561d` + `cf5ff0b` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` | `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test models_route_returns_openai_compatible_codex_model_list`; `cargo test --test admin_settings_route_test`; `cargo test --test model_catalog_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test cli_auth_import_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test chat_completions_route_test`; `cargo test --test codex_websocket_test`; targeted HTTP SSE error tests before `bfe561d`; targeted local API key import/export tests before `cf5ff0b`; WebSocket fallback targeted route tests; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe`; Task 5 targeted verification before `3ecd5b6` and `3960a35`; full `fmt/test/clippy` before `1738f24`; full `fmt/test/clippy` plus real-file smoke before `0b6cc7e`; full `fmt/test/clippy` before `f75db60`; full `fmt/test/clippy` before `4b19846`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `16eb4ed`; full `fmt/test/clippy` before `8af350d`; full `fmt/test/clippy` before `203270f`; full `fmt/test/clippy` before `960bbb2`; full `fmt/test/clippy` before `72438c6`; full `fmt/test/clippy` before `16938e6` | Adds Axum router, health route, `/v1/models`, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/debug/models`, `/debug/diagnostics`, `/debug/fingerprint`, `/admin/auth/status`, `/admin/auth/logout`, `/admin/auth/login-start`, `/admin/auth/code-relay`, `/admin/auth/callback`, `/auth/callback`, `/admin/auth/device-login`, `/admin/auth/device-poll/{device_code}`, `/admin/accounts`, `/admin/accounts/quota-warnings`, `/admin/accounts/export`, `/admin/accounts/batch-delete`, `/admin/accounts/batch-status`, `/admin/accounts/health-check`, `/admin/accounts/{id}`, `/admin/accounts/{id}/label`, `/admin/accounts/{id}/status`, `/admin/accounts/{id}/refresh`, `/admin/accounts/{id}/reset-usage`, `/admin/accounts/{id}/quota`, `/admin/accounts/{id}/cookies`, `/admin/accounts/import`, `/admin/accounts/import-cli`, `/admin/api-keys`, `/admin/api-keys/export`, `/admin/api-keys/import`, `/admin/api-keys/batch-delete`, `/admin/api-keys/{id}`, `/admin/api-keys/{id}/label`, `/admin/api-keys/{id}/status`, `/admin/usage-stats`, `/admin/usage-stats/summary`, `/admin/refresh-models`, `/v1/responses` HTTP SSE collection with output item/delta reconstruction plus `output_text` sync for non-streaming clients, default streaming Responses passthrough when `stream` is omitted, WebSocket-backed Responses for `previous_response_id`, explicit non-history WebSocket Responses via `use_websocket`/`useWebSocket`, non-streaming and HTTP SSE setup fallback to another imported account after 429/402/403 account-state failures, WebSocket setup fallback after classified handshake failures when no `previous_response_id` is present, Responses field parity for service tier, tool choice, parallel tool calls, text format, prompt cache key, include, client metadata, reasoning include default, and Codex context headers, plus a dedicated `/v1/chat/completions` route backed by imported Codex accounts with fallback for buffered non-streaming and client-streaming output conversion. `/v1` unauthorized and model errors remain OpenAI-compatible, cached backend model snapshots are used by `/v1/models*` when present, global request-id response headers are preserved, `gpt-5.5` remains the default/test model baseline, and `/admin/settings` supports admin-session-gated read/write access with lower camelCase envelopes. `/admin/auth/status` returns sanitized authenticated state, first account metadata, and pool summary without decrypting or returning tokens. `/admin/auth/logout` clears all account rows and the runtime pool while relying on SQLite foreign-key cascades for usage/cookies. `/admin/auth/login-start` creates a short-lived in-memory PKCE session and returns the OpenAI authorization URL; `/admin/auth/code-relay`, `/admin/auth/callback`, and the registered `/auth/callback` redirect path exchange returned authorization codes and import tokens through the same JWT-claim validated path. `/admin/auth/device-login` starts the OpenAI OAuth device-code flow, and `/admin/auth/device-poll/{device_code}` handles pending/slow_down states or imports returned tokens without returning secrets. `/admin/accounts` reads metadata only and does not decrypt tokens for list views. Manual `POST /admin/accounts` accepts only `token` and/or `refreshToken`, rejects invalid/expired JWTs and missing ChatGPT account claims, derives metadata from JWT claims, ignores caller metadata/status/label, exchanges refresh-token-only inputs through `AppState`'s OpenAI token refresher, preserves or rotates refresh tokens, stores secrets encrypted, synchronizes the runtime account pool, updates existing accounts by `chatgpt_account_id` + `chatgpt_user_id`, and never echoes tokens. `/admin/accounts/quota-warnings` returns active warning/critical quota windows computed from cached SQLite quota JSON without contacting upstream. `/admin/accounts/import` accepts native and Sub2API OpenAI OAuth account exports, returns `sourceFormat`, and ignores proxy-only metadata. `/admin/accounts/import-cli` reads Codex CLI `auth.json` from `$CODEX_HOME` or an explicit admin fixture path, uses the same validated JWT-claim account import path as manual add, ignores caller metadata, and returns only sanitized import counts. `/admin/accounts/export` returns native or Sub2API OpenAI OAuth exports without proxy fields. `/admin/accounts/{id}/label`, `/status`, `/refresh`, `/reset-usage`, `/quota`, delete, batch-delete, batch-status, and `/admin/accounts/health-check` mutate or inspect SQLite plus runtime account pool without returning secrets; health/quota call Codex backend usage endpoints through imported account tokens and classify invalid/quota/banned/Cloudflare responses into account state or cooldown. `/admin/accounts/{id}/cookies` stores/reads/deletes encrypted per-account browser cookies. `/admin/api-keys/{id}/label`, `/status`, delete, and batch-delete mutate local client key rows used by `/v1` auth; `/admin/api-keys/export` returns metadata only without plaintext/keyHash/pepper, and `/admin/api-keys/import` rotates metadata into new local keys with plaintext returned once. These are local auth utilities, not TS provider key pool parity. `/admin/refresh-models` fetches backend model lists through imported accounts only, stores plan snapshots, and syncs runtime model-plan allowlists. Still missing upstream test-connection diagnostics and live Chat streaming transform. | +| HTTP routes | Partial | `ab0ad8f` + `3830b21` + `6179d44` + `cfec8ba` + `6341336` + `228febd` + `c2cf7a6` + `e9fdea0` + `ee0ed90` + `fdb9374` + `9ca5c7d` + `b9c30ba` + `350ffbe` + `3ecd5b6` + `3960a35` + `1738f24` + `0b6cc7e` + `f75db60` + `4b19846` + `db6cb98` + `16eb4ed` + `8af350d` + `203270f` + `960bbb2` + `72438c6` + `16938e6` + `869b6f8` + `e1f35e0` + `93cfee3` + `bfe561d` + `cf5ff0b` + `6d18d2c` + `a32a53c` + `1d6c301` + `a3b4d2b` | `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test models_route_returns_openai_compatible_codex_model_list`; `cargo test --test admin_settings_route_test`; `cargo test --test model_catalog_test`; `cargo test --test admin_accounts_route_test`; `cargo test --test cli_auth_import_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test chat_completions_route_test`; `cargo test --test codex_websocket_test`; targeted HTTP SSE error tests before `bfe561d`; targeted local API key import/export tests before `cf5ff0b`; WebSocket fallback targeted route tests; auth status/logout targeted `cargo test --test admin_accounts_route_test`; device login targeted `cargo test --test admin_accounts_route_test admin_auth_device`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; PKCE auth-flow targeted `cargo test --test admin_accounts_route_test admin_auth_`, `cargo test --test oauth_refresh_test`, and `cargo test --test admin_accounts_route_test`; full `cargo fmt --check && cargo test && cargo clippy --all-targets --all-features --locked -- -D warnings` before `b9c30ba` and `350ffbe`; Task 5 targeted verification before `3ecd5b6` and `3960a35`; full `fmt/test/clippy` before `1738f24`; full `fmt/test/clippy` plus real-file smoke before `0b6cc7e`; full `fmt/test/clippy` before `f75db60`; full `fmt/test/clippy` before `4b19846`; full `fmt/test/clippy` before `db6cb98`; full `fmt/test/clippy` before `16eb4ed`; full `fmt/test/clippy` before `8af350d`; full `fmt/test/clippy` before `203270f`; full `fmt/test/clippy` before `960bbb2`; full `fmt/test/clippy` before `72438c6`; full `fmt/test/clippy` before `16938e6` | Adds Axum router, health route, `/v1/models`, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/debug/models`, `/debug/diagnostics`, `/debug/fingerprint`, `/debug/upstream`, `/admin/auth/status`, `/admin/auth/logout`, `/admin/auth/login-start`, `/admin/auth/code-relay`, `/admin/auth/callback`, `/auth/callback`, `/admin/auth/device-login`, `/admin/auth/device-poll/{device_code}`, `/admin/accounts`, `/admin/accounts/quota-warnings`, `/admin/accounts/export`, `/admin/accounts/batch-delete`, `/admin/accounts/batch-status`, `/admin/accounts/health-check`, `/admin/accounts/{id}`, `/admin/accounts/{id}/label`, `/admin/accounts/{id}/status`, `/admin/accounts/{id}/refresh`, `/admin/accounts/{id}/reset-usage`, `/admin/accounts/{id}/quota`, `/admin/accounts/{id}/cookies`, `/admin/accounts/import`, `/admin/accounts/import-cli`, `/admin/api-keys`, `/admin/api-keys/export`, `/admin/api-keys/import`, `/admin/api-keys/batch-delete`, `/admin/api-keys/{id}`, `/admin/api-keys/{id}/label`, `/admin/api-keys/{id}/status`, `/admin/usage-stats`, `/admin/usage-stats/summary`, `/admin/refresh-models`, `/v1/responses` HTTP SSE collection with output item/delta reconstruction plus `output_text` sync for non-streaming clients, default streaming Responses passthrough when `stream` is omitted, WebSocket-backed Responses for `previous_response_id`, explicit non-history WebSocket Responses via `use_websocket`/`useWebSocket`, non-streaming and HTTP SSE setup fallback to another imported account after 429/402/403 account-state failures, WebSocket setup fallback after classified handshake failures when no `previous_response_id` is present, Responses field parity for service tier, tool choice, parallel tool calls, text format, prompt cache key, include, client metadata, reasoning include default, and Codex context headers, plus a dedicated `/v1/chat/completions` route backed by imported Codex accounts with fallback for buffered non-streaming and client-streaming output conversion. `/v1` unauthorized and model errors remain OpenAI-compatible, cached backend model snapshots are used by `/v1/models*` when present, global request-id response headers are preserved, `gpt-5.5` remains the default/test model baseline, and `/admin/settings` supports admin-session-gated read/write access with lower camelCase envelopes. `/admin/auth/status` returns sanitized authenticated state, first account metadata, and pool summary without decrypting or returning tokens. `/admin/auth/logout` clears all account rows and the runtime pool while relying on SQLite foreign-key cascades for usage/cookies. `/admin/auth/login-start` creates a short-lived in-memory PKCE session and returns the OpenAI authorization URL; `/admin/auth/code-relay`, `/admin/auth/callback`, and the registered `/auth/callback` redirect path exchange returned authorization codes and import tokens through the same JWT-claim validated path. `/admin/auth/device-login` starts the OpenAI OAuth device-code flow, and `/admin/auth/device-poll/{device_code}` handles pending/slow_down states or imports returned tokens without returning secrets. `/admin/accounts` reads metadata only and does not decrypt tokens for list views. Manual `POST /admin/accounts` accepts only `token` and/or `refreshToken`, rejects invalid/expired JWTs and missing ChatGPT account claims, derives metadata from JWT claims, ignores caller metadata/status/label, exchanges refresh-token-only inputs through `AppState`'s OpenAI token refresher, preserves or rotates refresh tokens, stores secrets encrypted, synchronizes the runtime account pool, updates existing accounts by `chatgpt_account_id` + `chatgpt_user_id`, and never echoes tokens. `/admin/accounts/quota-warnings` returns active warning/critical quota windows computed from cached SQLite quota JSON without contacting upstream. `/admin/accounts/import` accepts native and Sub2API OpenAI OAuth account exports, returns `sourceFormat`, and ignores proxy-only metadata. `/admin/accounts/import-cli` reads Codex CLI `auth.json` from `$CODEX_HOME` or an explicit admin fixture path, uses the same validated JWT-claim account import path as manual add, ignores caller metadata, and returns only sanitized import counts. `/admin/accounts/export` returns native or Sub2API OpenAI OAuth exports without proxy fields. `/admin/accounts/{id}/label`, `/status`, `/refresh`, `/reset-usage`, `/quota`, delete, batch-delete, batch-status, and `/admin/accounts/health-check` mutate or inspect SQLite plus runtime account pool without returning secrets; health/quota call Codex backend usage endpoints through imported account tokens and classify invalid/quota/banned/Cloudflare responses into account state or cooldown. `/admin/accounts/{id}/cookies` stores/reads/deletes encrypted per-account browser cookies. `/admin/api-keys/{id}/label`, `/status`, delete, and batch-delete mutate local client key rows used by `/v1` auth; `/admin/api-keys/export` returns metadata only without plaintext/keyHash/pepper, and `/admin/api-keys/import` rotates metadata into new local keys with plaintext returned once. These are local auth utilities, not TS provider key pool parity. `/admin/refresh-models` fetches backend model lists through imported accounts only, stores plan snapshots, and syncs runtime model-plan allowlists. Still missing live Chat streaming transform. | | Admin quota warnings | Completed | `b879dd6` | `cargo test --test admin_accounts_cookies_quota_test quota_warnings`; `cargo test --test admin_accounts_cookies_quota_test`; `cargo fmt --check`; `cargo test`; `cargo clippy --all-targets --all-features --locked -- -D warnings` | Adds `/admin/accounts/quota-warnings`, backed by cached SQLite quota snapshots and configured warning thresholds. The route is admin-session gated, returns lower camelCase admin envelopes, and does not call upstream Codex while listing active warnings. | | Admin/frontend response contract | Completed | `1c34ea7` + `b0c10f2` | `cargo test admin_envelope_serializes_lower_camel_case_request_id`; `cargo test admin_logs_reject_missing_admin_session_cookie`; `cargo test v1_response_has_request_id_header_without_admin_body_field`; full `fmt/test/clippy` before commit | Adds reusable admin envelopes with body codes, lower camelCase JSON, paginated list shape, requestId bodies, admin session cookie enforcement, and keeps `/v1/*` OpenAI-compatible. | | HTTP auth boundary | Completed | `cec4607` + `e9fdea0` | `cargo test --test http_auth_test`; `cargo test --test routes_responses_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_logs_route_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds a real `src/http/auth.rs` boundary for client API key and admin session cookie parsing. `/v1/*` accepts only enabled local `Bearer cpr_...` client API keys verified from SQLite, admin session parsing ignores client API keys, and route handlers no longer own ad hoc credential parsing. | diff --git a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md index 84c4ddbeb..856410bfa 100644 --- a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md +++ b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md @@ -2976,7 +2976,7 @@ The Rust project has no legacy compatibility requirement, but the in-scope behav - [x] **Login routes:** implement OAuth PKCE login-start/code-relay/callback, device-login/device-poll, CLI auth import, manual access-token import, and logout as account-login flows. CLI auth import, validated manual token import, device login/poll, OAuth PKCE login-start/code-relay/callback, admin-session-gated auth status, and logout are complete. Keep this separate from admin password login and client API key creation. - [x] **Client API key management:** implement admin CRUD for local client API keys: list/create/delete/batch-delete/label/status/export/import. Create/list/status/delete/batch-delete/label/export/import and `/v1/*` SQLite verification are complete. Remove provider/model binding semantics from the TypeScript version; Rust client API keys authorize only local `/v1/*` access. - [x] **Settings:** implement admin settings for server/runtime fields that remain in scope: default model, reasoning effort, service tier, model aliases, refresh enabled/margin/concurrency, max concurrent per account, request interval, rotation strategy, tier priority, quota refresh/thresholds/skip exhausted, logs state/capacity/body capture, and usage history retention. GET and PATCH `/admin/settings` are admin-session gated, PATCH validates retained fields, writes a root `local.yaml` overlay, and rejects proxy URL, Ollama, third-party provider, OpenAI official key, proxy API key, and Electron/self-update settings. Existing constructed runtime services still use construction-time config until restart or a later runtime reconfiguration task. -- [ ] **Diagnostics:** expand health/admin diagnostics to include authenticated state, pool summary, capacity summary, transport/fingerprint status, paths, runtime metadata, and test-connection checks. Local-only `/debug/diagnostics` now returns runtime metadata, paths, account repository/pool state, capacity summary, transport config, and static fingerprint status without secrets. Remaining: active upstream test-connection checks and richer admin diagnostics surfaces. Keep production-local gating for sensitive debug endpoints. +- [ ] **Diagnostics:** expand health/admin diagnostics to include authenticated state, pool summary, capacity summary, transport/fingerprint status, paths, runtime metadata, and test-connection checks. Local-only `/debug/diagnostics` now returns runtime metadata, paths, account repository/pool state, capacity summary, transport config, and static fingerprint status without secrets. Local-only `/debug/fingerprint` returns the runtime Desktop fingerprint summary. Local-only `/debug/upstream` probes the Codex models endpoint with runtime TLS/fingerprint headers and returns only status metadata. Remaining: richer admin diagnostics surfaces. Keep production-local gating for sensitive debug endpoints. - [ ] **Logs and usage stats:** add `/admin/logs/state`, clear/detail, error-log equivalents if retained, and richer bounded history queries if needed. `/admin/logs/state`, `/admin/logs/{id}`, `DELETE /admin/logs`, log file rotation, SQLite event pagination, `/admin/usage-stats` pagination, and `/admin/usage-stats/summary` are complete. Remaining: decide whether to retain old error-log grouping/raw/count/seen/report APIs and richer bounded body-history queries. - [x] **Model catalog:** replace the hard-coded `/v1/models` list with a model store that supports static defaults, configured aliases, `-low/-medium/-high/-xhigh/-fast/-flex` suffix parsing, backend model fetching per plan, cache persistence, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/admin/refresh-models`, and `/debug/models`. Static defaults, aliases, suffix parsing, cached backend snapshots, plan allowlists, admin refresh, read-only model routes, and debug summary are complete. - [ ] **Account scheduling parity:** extend account acquisition beyond `active + least last_used`. Completed in the in-memory pool: max concurrent slots per account, stale slot cleanup, least-used/round-robin/sticky strategies, quota-cache skip, tier priority, model-plan filtering, exclude IDs, preferred account from session affinity, Cloudflare cooldown skip, request-staggering metadata, capacity summary, settings-backed construction, import-to-pool sync, startup pool snapshots from SQLite, HTTP fallback acquisition across alternate accounts, WebSocket fallback for non-history requests, previous-response account affinity, and runtime pool clearing on logout. Remaining: durable release-time slot state and durable quota-cache/cooldown restore. @@ -3001,7 +3001,7 @@ This pass re-read the TypeScript routes and core modules under `/home/zyy/Codes/ - [ ] **Account scheduling and retry:** Rust has in-memory slots, stale cleanup, strategies, tier priority, preferred/exclude IDs, SQLite startup restore, request-staggering metadata, 429/402/403 fallback across alternate accounts, Cloudflare cooldown skip, successful WebSocket rate-limit header application, non-history WebSocket fallback, previous-response history affinity, and status/token/logout mutation sync for the runtime pool. Missing strict request-interval sleep enforcement, durable quota/cache restore, Cloudflare path-block tracker, cooldown clearing, and richer capacity/health endpoints. - [ ] **Refresh scheduler parity:** Rust has typed refresh policy and 401-triggered refresh retry. Missing long-running refresh timers at `exp - margin`, crash recovery for `refreshing`/`expired` accounts, exponential backoff, recovery scheduling, permanent-failure thresholding, per-account in-flight suppression, cross-process refresh locks, and disk refresh-token sync before consuming one-time RTs. - [ ] **Codex Desktop fingerprint and transport parity:** Rust uses pinned `reqwest`/`rustls` and core headers, but exact parity is not proven. Missing header-order/default-header parity (`sec-ch-ua`, accept language/encoding/fetch headers), `x-codex-installation-id`, runtime selection of the newest stored fingerprint, Sparkle appcast XML polling, update-state/version-state persistence, optional local Codex.app Chromium extraction, and real Desktop TLS/header capture comparison. `x-openai-internal-codex-residency` currently needs real-client verification because the builder inserts `global` and the reqwest client overwrites it with `us`. -- [ ] **Health and diagnostics:** Rust `/health` returns `status: ok`, `/admin/auth/status` returns admin-session-gated authenticated state plus account pool summary, and local-only `/debug/diagnostics` returns capacity summary, transport/fingerprint status, paths, runtime metadata, and retained settings summary without secrets. Missing richer upstream test-connection/probe routes. +- [ ] **Health and diagnostics:** Rust `/health` returns `status: ok`, `/admin/auth/status` returns admin-session-gated authenticated state plus account pool summary, local-only `/debug/diagnostics` returns capacity summary, transport/fingerprint status, paths, runtime metadata, and retained settings summary without secrets, and local-only `/debug/upstream` probes the Codex models endpoint without returning tokens or upstream bodies. Missing richer admin diagnostics surfaces. - [ ] **Background service lifecycle:** Rust startup restores accounts and initializes tracing, but does not start/stop scheduler tasks for account refresh, model refresh, quota refresh, active quota refresh, session cleanup, fingerprint polling, or graceful shutdown drains. Ollama, proxy health timers, self-update, Electron, and provider router startup remain intentionally deleted. ### Keep Removed diff --git a/src/app/router.rs b/src/app/router.rs index 5f92b4921..f86a111ad 100644 --- a/src/app/router.rs +++ b/src/app/router.rs @@ -3,7 +3,7 @@ use tower_http::trace::TraceLayer; use crate::http::{ admin, - diagnostics::{debug_fingerprint, diagnostics}, + diagnostics::{debug_fingerprint, debug_upstream, diagnostics}, health::health, middleware::attach_request_id, v1, @@ -16,6 +16,7 @@ pub fn build_router(state: AppState) -> Router { .route("/health", get(health)) .route("/debug/diagnostics", get(diagnostics)) .route("/debug/fingerprint", get(debug_fingerprint)) + .route("/debug/upstream", get(debug_upstream)) .merge(v1::router()) .merge(admin::router()) .with_state(state) diff --git a/src/app/state.rs b/src/app/state.rs index 5d0c24961..7578b1e33 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -17,8 +17,9 @@ use crate::codex::upstream::CodexUpstreamService; use crate::config::AppConfig; use crate::logs::repository::EventLogRepository; use crate::service::{ - admin_auth::AdminAuthService, api_key::ApiKeyService, chat::ChatService, log::LogService, - responses::ResponsesService, settings::SettingsService, usage::UsageService, + admin_auth::AdminAuthService, api_key::ApiKeyService, chat::ChatService, + diagnostics::DiagnosticsService, log::LogService, responses::ResponsesService, + settings::SettingsService, usage::UsageService, }; use crate::utils::crypto::SecretBox; @@ -35,6 +36,7 @@ pub struct AppServices { pub logs: LogService, pub usage: UsageService, pub settings: SettingsService, + pub diagnostics: DiagnosticsService, pub chat: ChatService, pub responses: ResponsesService, pub models: ModelService, @@ -192,6 +194,7 @@ impl AppState { config.clone(), local_config_path.unwrap_or_else(|| PathBuf::from("local.yaml")), ); + let diagnostics = DiagnosticsService::new(config.clone()); let accounts = account_service( &config, account_repository(pool_ref, secret_box_ref), @@ -229,6 +232,7 @@ impl AppState { logs, usage, settings, + diagnostics, chat, responses, models, diff --git a/src/codex/transport/client.rs b/src/codex/transport/client.rs index 806e66568..5657060b9 100644 --- a/src/codex/transport/client.rs +++ b/src/codex/transport/client.rs @@ -72,6 +72,12 @@ pub struct CodexBackendResponse { pub rate_limit_headers: Vec<(String, String)>, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CodexConnectivityProbe { + pub endpoint: String, + pub status: StatusCode, +} + pub struct CodexBackendStream { pub response: ReqwestResponse, pub turn_state: Option, @@ -203,6 +209,24 @@ impl CodexBackendClient { Err(CodexClientError::ModelsUnavailable) } + pub async fn probe_models_endpoint( + &self, + context: CodexRequestContext<'_>, + ) -> CodexClientResult { + let endpoint = format!( + "{}/codex/models?client_version={}", + self.base_url, self.fingerprint.app_version + ); + let mut headers = self.request_headers(context)?; + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + let response = self.client.get(&endpoint).headers(headers).send().await?; + + Ok(CodexConnectivityProbe { + endpoint, + status: response.status(), + }) + } + pub async fn fetch_usage(&self, context: CodexRequestContext<'_>) -> CodexClientResult { let mut last_invalid_body = None; for endpoint in self.usage_endpoints() { diff --git a/src/http/diagnostics.rs b/src/http/diagnostics.rs index 76ff9a4f8..59b80a4d3 100644 --- a/src/http/diagnostics.rs +++ b/src/http/diagnostics.rs @@ -1,5 +1,5 @@ use axum::{ - extract::State, + extract::{Extension, State}, http::{HeaderMap, StatusCode}, response::IntoResponse, Json, @@ -11,6 +11,7 @@ use crate::{ app::state::AppState, codex::{accounts::pool::AccountCapacitySummary, fingerprint::model::Fingerprint}, config::AppConfig, + http::middleware::RequestId, service::admin_auth::AdminAuthPoolSummary, }; @@ -166,6 +167,32 @@ pub async fn debug_fingerprint(headers: HeaderMap) -> impl IntoResponse { .into_response() } +pub async fn debug_upstream( + State(state): State, + Extension(request_id): Extension, + headers: HeaderMap, +) -> impl IntoResponse { + if !is_local_debug_request(&headers) { + return ( + StatusCode::FORBIDDEN, + Json(json!({ "error": "debug endpoint is local-only" })), + ) + .into_response(); + } + + ( + StatusCode::OK, + Json( + state + .services + .diagnostics + .probe_upstream(request_id.as_str()) + .await, + ), + ) + .into_response() +} + fn paths_diagnostics(state: &AppState, config: &AppConfig) -> PathDiagnostics { PathDiagnostics { config: "config.yaml", diff --git a/src/service/diagnostics.rs b/src/service/diagnostics.rs new file mode 100644 index 000000000..80fe96cf0 --- /dev/null +++ b/src/service/diagnostics.rs @@ -0,0 +1,132 @@ +use std::time::Instant; + +use reqwest::StatusCode; +use serde::Serialize; + +use crate::{ + codex::{ + fingerprint::model::Fingerprint, + transport::client::{ + build_reqwest_client, CodexBackendClient, CodexClientError, CodexRequestContext, + }, + }, + config::AppConfig, +}; + +#[derive(Clone)] +pub struct DiagnosticsService { + config: AppConfig, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpstreamProbeResult { + pub target: &'static str, + pub status: &'static str, + pub backend_base_url: String, + pub endpoint: Option, + pub reachable: bool, + pub status_code: Option, + pub authorization: &'static str, + pub duration_ms: u64, + pub error: Option, +} + +impl DiagnosticsService { + pub fn new(config: AppConfig) -> Self { + Self { config } + } + + pub async fn probe_upstream(&self, request_id: &str) -> UpstreamProbeResult { + let started_at = Instant::now(); + let fingerprint = Fingerprint::default_codex_desktop(); + let client = match build_reqwest_client(self.config.tls.force_http11) { + Ok(client) => { + CodexBackendClient::new(client, self.config.api.base_url.clone(), fingerprint) + } + Err(error) => { + return self.unreachable_probe( + None, + started_at, + format!("failed to build HTTP client: {error}"), + ); + } + }; + + match client + .probe_models_endpoint(CodexRequestContext { + access_token: "diagnostic-probe", + account_id: None, + request_id, + turn_state: None, + turn_metadata: None, + beta_features: None, + include_timing_metrics: None, + version: None, + codex_window_id: None, + parent_thread_id: None, + cookie_header: None, + }) + .await + { + Ok(probe) => self.reachable_probe(probe.endpoint, probe.status, started_at), + Err(error) => self.unreachable_probe(None, started_at, public_probe_error(&error)), + } + } + + fn reachable_probe( + &self, + endpoint: String, + status: StatusCode, + started_at: Instant, + ) -> UpstreamProbeResult { + UpstreamProbeResult { + target: "codexModels", + status: "reachable", + backend_base_url: self.config.api.base_url.clone(), + endpoint: Some(endpoint), + reachable: true, + status_code: Some(status.as_u16()), + authorization: authorization_status(status), + duration_ms: elapsed_millis(started_at), + error: None, + } + } + + fn unreachable_probe( + &self, + endpoint: Option, + started_at: Instant, + error: String, + ) -> UpstreamProbeResult { + UpstreamProbeResult { + target: "codexModels", + status: "unreachable", + backend_base_url: self.config.api.base_url.clone(), + endpoint, + reachable: false, + status_code: None, + authorization: "unknown", + duration_ms: elapsed_millis(started_at), + error: Some(error), + } + } +} + +fn authorization_status(status: StatusCode) -> &'static str { + if status.is_success() { + "accepted" + } else if status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN { + "rejected" + } else { + "unknown" + } +} + +fn elapsed_millis(started_at: Instant) -> u64 { + started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64 +} + +fn public_probe_error(error: &CodexClientError) -> String { + error.to_string().chars().take(200).collect() +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 3f44abf34..db8ff6d0b 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,6 +1,7 @@ pub mod admin_auth; pub mod api_key; pub mod chat; +pub mod diagnostics; pub mod log; pub mod responses; pub mod settings; diff --git a/tests/diagnostics_route_test.rs b/tests/diagnostics_route_test.rs index 8ea9690fb..48ad0510e 100644 --- a/tests/diagnostics_route_test.rs +++ b/tests/diagnostics_route_test.rs @@ -3,6 +3,10 @@ use axum::{ http::{Request, StatusCode}, }; use tower::ServiceExt; +use wiremock::{ + matchers::{header, method, path}, + Mock, MockServer, ResponseTemplate, +}; mod common; @@ -125,3 +129,75 @@ async fn debug_fingerprint_should_reject_forwarded_remote_requests() { assert_eq!(response.status(), StatusCode::FORBIDDEN); } + +#[tokio::test] +async fn debug_upstream_should_probe_codex_models_endpoint_without_returning_secrets() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/codex/models")) + .and(header("originator", "Codex Desktop")) + .and(header("x-client-request-id", "req_debug_probe")) + .respond_with(ResponseTemplate::new(401).set_body_json(serde_json::json!({ + "error": { + "message": "missing or invalid token" + } + }))) + .expect(1) + .mount(&server) + .await; + let imported = build_imported_app(server.uri()).await; + + let response = imported + .app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri("/debug/upstream") + .header("x-request-id", "req_debug_probe") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["target"], "codexModels"); + assert_eq!(body["backendBaseUrl"], server.uri()); + assert_eq!(body["reachable"], true); + assert_eq!(body["statusCode"], 401); + assert_eq!(body["authorization"], "rejected"); + assert!(body["endpoint"].as_str().unwrap().contains("/codex/models")); + + let serialized = serde_json::to_string(&body).unwrap(); + assert!(!serialized.contains("access-secret")); + assert!(!serialized.contains("refresh-secret")); +} + +#[tokio::test] +async fn debug_upstream_should_reject_forwarded_remote_requests_without_probe() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/codex/models")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + let imported = build_imported_app(server.uri()).await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("GET") + .uri("/debug/upstream") + .header("x-forwarded-for", "203.0.113.50") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); +}