From 5bf012c64ef8188b8076b0af41179b44b0f50b84 Mon Sep 17 00:00:00 2001 From: zyycn Date: Fri, 12 Jun 2026 06:58:28 +0800 Subject: [PATCH] feat: log token refresh failures Co-authored-by: Codex --- docs/api.md | 1 + docs/implementation-status.md | 4 +- .../plans/2026-06-11-codex-proxy-rs.md | 3 +- src/http/v1.rs | 89 +++++++++++++++--- tests/v1_upstream_route_test.rs | 93 ++++++++++++++++++- 5 files changed, 171 insertions(+), 19 deletions(-) diff --git a/docs/api.md b/docs/api.md index ef0dc624f..70a910355 100644 --- a/docs/api.md +++ b/docs/api.md @@ -48,6 +48,7 @@ When the client omits `stream` or sets `"stream": false`, the Rust route collect When the client sets `"stream": true`, the Rust route returns `text/event-stream` and passes through upstream SSE frames while collecting usage for the account after the stream finishes. Both modes capture upstream `Set-Cookie`, replay encrypted account-scoped Cookies, record account usage when usage appears in SSE, and write a `v1.response` event log with `requestId`, `accountId`, `route`, `model`, `statusCode`, `latencyMs`, and non-secret metadata. If Codex returns `401` and the account has a stored refresh token, the route refreshes the account through the OpenAI OAuth token endpoint once, persists the rotated access token, preserves the old refresh token when the server omits a new one, updates the in-process account pool, and retries the same request once. +Refresh failures are recorded as non-secret `account.refresh` event logs with the request id, account id, model, `trigger`, `failure`, and resulting `accountStatus`. `previous_response_id` and explicit WebSocket-only requests are rejected until the WebSocket transport is implemented and verified. diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 5608d59b6..f9bfcf4e1 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -10,7 +10,7 @@ Update this file before each feature commit. | SQLite storage | Completed | `ffec8b5` + `b0c10f2` | `cargo test sqlite_schema_creates_accounts_and_event_tables`; `cargo test connect_sqlite_creates_parent_directory_for_file_database`; full `fmt/test/clippy` before commit | Creates SQLite WAL connector, parent directory bootstrap, sqlx-managed schema for accounts/API keys/sessions/cookies/fingerprints/events, and event/account indexes. | | Admin auth and client API keys | Partial | `24d5799` + `4e0d1e5` + `e9fdea0` | `cargo test client_api_key_has_proxy_prefix_and_verifies_against_hash`; `cargo test admin_password_hash_is_not_a_client_api_key`; `cargo test --test admin_login_route_test`; `cargo test --test admin_api_keys_route_test`; full `fmt/test/clippy` before commit | Admin passwords use Argon2id, client API keys use `cpr_` prefix plus HMAC-SHA256 with server-side pepper; `/admin/login` issues HttpOnly sessions and rejects client API keys as admin credentials. `/admin/api-keys` can create/list local client keys, and `/v1/*` now verifies enabled keys against SQLite instead of accepting any `cpr_` string. Still missing delete/batch-delete/label/status/export/import key management. | | Secret encryption | Completed | `105aa58` + `228febd` | `cargo test secret_box_encrypts_and_decrypts_without_storing_plaintext`; `cargo test secret_box_load_or_create_should_reuse_existing_master_key_file`; full `fmt/test/clippy` before commit | AES-256-GCM `SecretBox` stores upstream secrets as `v1::` without plaintext and can load/create the runtime master key file used by repositories. | -| Logging and pagination | Completed | `eb0341e` + `fc16c48` + `b0c10f2` + `ee0ed90` + `fdb9374` | `cargo test event_logs_are_cursor_paginated`; `cargo test admin_logs_are_cursor_paginated_and_include_request_id`; `cargo test rotating_writer_splits_files_when_size_limit_is_reached`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, admin `/admin/logs` query API, admin `/admin/usage-stats` and `/admin/usage-stats/summary`, event log repository, runtime tracing initialization, HTTP trace layer, date/size-based file rotation, and v1 response lifecycle events with request/account/route/model/status/latency/non-secret metadata. | +| Logging and pagination | Completed | `eb0341e` + `fc16c48` + `b0c10f2` + `ee0ed90` + `fdb9374` + `pending` | `cargo test event_logs_are_cursor_paginated`; `cargo test admin_logs_are_cursor_paginated_and_include_request_id`; `cargo test rotating_writer_splits_files_when_size_limit_is_reached`; `cargo test --test admin_usage_stats_route_test`; `cargo test --test v1_upstream_route_test`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, admin `/admin/logs` query API, admin `/admin/usage-stats` and `/admin/usage-stats/summary`, event log repository, runtime tracing initialization, HTTP trace layer, date/size-based file rotation, v1 response lifecycle events with request/account/route/model/status/latency/non-secret metadata, and non-secret `account.refresh` events for upstream-401 refresh failures. | | TLS headers and fingerprint | Completed | `1bdd08c` | `cargo test codex_headers_include_desktop_identity_and_turn_state`; `cargo tree \| rg 'reqwest\|rustls'`; full `fmt/test/clippy` before commit | Adds Codex Desktop fingerprint model, exact identity/request headers, and pinned reqwest/rustls client builder. | | Cookie persistence | Completed | `4710926` + `5d3cc6e` | `cargo test cookie_jar_captures_and_replays_account_scoped_cookies`; `cargo test cookie_repository_encrypts_and_replays_account_scoped_cookies`; full `fmt/test/clippy` before commit | Adds account-scoped in-memory Cookie jar and encrypted SQLite Cookie repository; Cookies are replayed only for the same account/domain. | | Account pool and refresh | Partial | `0f97c5c` + `2f25db8` + `4dc0c9a` + `c2cf7a6` + `636fa79` + `fdb9374` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; `cargo test --test account_pool_scheduling_test`; `cargo test --test account_refresh_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; full `fmt/test/clippy` before commit | Implements base account status filtering, refresh scheduler semantics, real OpenAI OAuth refresh-token exchange, 401-triggered refresh and single retry for `/v1/responses`, max concurrent slots, stale slot cleanup, least-used/round-robin/sticky strategies, tier priority, quota-cache filtering, Cloudflare cooldown skip, request staggering metadata, preferred account selection, exclude IDs, capacity summary, settings-backed pool construction, same-process import-to-pool sync, and startup pool snapshots from SQLite with usage-based last-used metadata. Still missing durable release-time slot state, broader upstream retry/fallback wiring, and admin account CRUD. | @@ -20,7 +20,7 @@ Update this file before each feature commit. | Admin/frontend response contract | Completed | `1c34ea7` + `b0c10f2` | `cargo test admin_envelope_serializes_lower_camel_case_request_id`; `cargo test admin_logs_reject_missing_admin_session_cookie`; `cargo test v1_response_has_request_id_header_without_admin_body_field`; full `fmt/test/clippy` before commit | Adds reusable admin envelopes with body codes, lower camelCase JSON, paginated list shape, requestId bodies, admin session cookie enforcement, and keeps `/v1/*` OpenAI-compatible. | | HTTP auth boundary | Completed | `cec4607` + `e9fdea0` | `cargo test --test http_auth_test`; `cargo test --test routes_responses_test`; `cargo test --test admin_api_keys_route_test`; `cargo test --test admin_logs_route_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds a real `src/http/auth.rs` boundary for client API key and admin session cookie parsing. `/v1/*` accepts only enabled local `Bearer cpr_...` client API keys verified from SQLite, admin session parsing ignores client API keys, and route handlers no longer own ad hoc credential parsing. | | Model catalog | Partial | `6341336` | `cargo test --test model_catalog_test`; `cargo test model_catalog_route_returns_codex_metadata_without_alias_entries`; full `fmt/test/clippy` before commit | Adds a native static Codex model catalog with `gpt-5.5` default, configured aliases, reasoning/service-tier suffix parsing, OpenAI-compatible `/v1/models`, extended catalog/detail/info routes, and debug summary. Still missing backend Codex model fetching per plan, cache persistence, `/admin/refresh-models`, plan allowlist integration from fetched models, and full model metadata parity. | -| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; full `fmt/test/clippy` before commit | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, replays/captures encrypted Cookies, collects `response.completed` for non-streaming clients, passes through client `stream:true` SSE, refreshes and retries once after upstream 401 when a refresh token is stored, records token usage, writes v1 lifecycle event logs, and restores the account pool from SQLite on startup. Still missing rate-limit header capture, retry/fallback across accounts beyond refresh retry, 429 quota cache, 402/403 classification, and previous-response/session affinity. | +| Upstream lifecycle | Partial | `88cad87` + `225fbff` + `c2cf7a6` + `636fa79` + `ee0ed90` + `fdb9374` + `pending` | `cargo test responses_route_rejects_non_codex_provider_models`; `cargo test v1_requires_client_api_key_not_admin_cookie`; `cargo test --test codex_sse_usage_test`; `cargo test --test codex_client_test`; `cargo test --test oauth_refresh_test`; `cargo test --test v1_upstream_route_test`; `cargo test --test account_pool_startup_test`; full `fmt/test/clippy` before commit | `/v1` enforces client API key shape, rejects non-Codex provider models through the model catalog, returns OpenAI-compatible errors, includes reusable SSE/usage helpers, sends Codex Desktop HTTP SSE requests through imported accounts, replays/captures encrypted Cookies, collects `response.completed` for non-streaming clients, passes through client `stream:true` SSE, refreshes and retries once after upstream 401 when a refresh token is stored, records token usage, writes v1 lifecycle event logs, records refresh failure classification for upstream 401, and restores the account pool from SQLite on startup. Still missing rate-limit header capture, retry/fallback across accounts beyond refresh retry, 429 quota cache, 402/403 classification, and previous-response/session affinity. | | Codex WebSocket boundary | Completed | `cc0516d` | `cargo test --test codex_websocket_test`; `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test` | Adds `src/codex/websocket.rs` with tested transport classification for HTTP SSE versus WebSocket-only requests. `previous_response_id` and explicit WebSocket requests now fail through a typed boundary instead of being silently downgraded; real WebSocket transport remains an upstream lifecycle follow-up. | | Fingerprint updates | Partial | `dd26d31` + `e32220a` | `cargo test update_manifest_updates_app_version_and_build_number`; `cargo test fingerprint_updater_fetches_manifest_and_persists_history`; full `fmt/test/clippy` before commit | Parses a JSON update manifest and persists app version/build number history to SQLite. Still missing original appcast XML polling, update-state persistence, runtime header fingerprint selection from the latest stored fingerprint, and optional local Codex.app Chromium extraction. | | Runtime docs and packaging | Completed | `33b63f3` | `cargo fmt --check`; `cargo clippy --all-targets --all-features --locked -- -D warnings`; `cargo test`; `cargo run`; `curl -s -i http://127.0.0.1:8080/health` | README scope, `.env.example`, Dockerfile, compose file, and health smoke test are complete. | diff --git a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md index 313ce0b53..a481a7bc5 100644 --- a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md +++ b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md @@ -2968,6 +2968,7 @@ git commit -m "docs: document codex-proxy-rs scope" - [x] Split local client API key auth from admin login. `/admin/api-keys` can create/list local `cpr_` keys, plaintext is returned only once, list responses never expose plaintext/hash, and `/v1/*` verifies enabled keys from SQLite instead of accepting any `cpr_` string. - [x] Add tested `/v1/responses` streaming passthrough and lifecycle event logs. Client `stream:true` returns `text/event-stream`, keeps the account slot until stream completion or disconnect, parses usage after SSE completion, writes account usage, and records non-secret `v1.response` logs with request/account/route/model/status/latency metadata. Non-streaming calls now write the same lifecycle log after `response.completed`. - [x] Add tested admin usage-stat read APIs. `/admin/usage-stats` returns cursor-paginated account usage counters, `/admin/usage-stats/summary` returns global totals, and both require the admin session cookie while exposing only non-secret account metadata. +- [x] Add tested upstream-401 refresh failure event logs. When refresh fails, the service records a non-secret `account.refresh` event with request/account/model context, failure classification, and resulting account status. - [x] Add real OpenAI OAuth refresh-token exchange and `/v1/responses` 401 retry wiring. The Rust service refreshes access tokens through the OAuth token endpoint, preserves the existing refresh token when the server omits a replacement, persists the rotated access token, updates the in-process account pool, and retries the failed non-streaming or streaming request once. ## 2026-06-11 Original Project Gap Audit @@ -2986,7 +2987,7 @@ The Rust project has no legacy compatibility requirement, but the in-scope behav - [ ] **Logs and usage stats:** add `/admin/logs/state`, clear/detail, error-log equivalents if retained, and richer bounded history queries if needed. Keep log file rotation and SQLite event pagination. Initial `/admin/usage-stats` pagination and `/admin/usage-stats/summary` are complete. - [ ] **Model catalog:** replace the hard-coded `/v1/models` list with a model store that supports static defaults, configured aliases, `-low/-medium/-high/-xhigh/-fast/-flex` suffix parsing, backend model fetching per plan, cache persistence, `/v1/models/catalog`, `/v1/models/{id}`, `/v1/models/{id}/info`, `/admin/refresh-models`, and `/debug/models`. Static defaults, aliases, suffix parsing, read-only model routes, and debug summary are complete. Backend fetch/cache, plan snapshots, and admin refresh remain. - [ ] **Account scheduling parity:** extend account acquisition beyond `active + least last_used`. Completed in the in-memory pool: max concurrent slots per account, stale slot cleanup, least-used/round-robin/sticky strategies, quota-cache skip, tier priority, model-plan filtering, exclude IDs, preferred account from session affinity, Cloudflare cooldown skip, request-staggering metadata, capacity summary, settings-backed construction, import-to-pool sync, and startup pool snapshots from SQLite. Remaining: durable release-time slot state and upstream retry/fallback wiring. -- [ ] **Upstream Codex lifecycle:** implement account acquire -> Codex request -> SSE/collect-or-stream -> rate-limit capture -> usage release -> retry/fallback. Initial HTTP SSE collection, streaming passthrough, and 401-triggered refresh retry are complete: `/v1/responses` acquires imported/restored accounts, sends Codex Desktop headers, replays/captures Cookies, collects `response.completed` for non-streaming clients, passes through `stream:true` SSE, refreshes and retries once after upstream 401, records usage, writes lifecycle logs, and returns the correct OpenAI-compatible JSON or SSE shape. Remaining: rate-limit capture, retry/fallback across alternate accounts, 429 quota cache, 402 quota exhausted, 403 ban/Cloudflare distinction, path-block Cookie clearing, and richer OpenAI-compatible error mapping. +- [ ] **Upstream Codex lifecycle:** implement account acquire -> Codex request -> SSE/collect-or-stream -> rate-limit capture -> usage release -> retry/fallback. Initial HTTP SSE collection, streaming passthrough, and 401-triggered refresh retry are complete: `/v1/responses` acquires imported/restored accounts, sends Codex Desktop headers, replays/captures Cookies, collects `response.completed` for non-streaming clients, passes through `stream:true` SSE, refreshes and retries once after upstream 401, records refresh failure classification when refresh cannot recover the account, records usage, writes lifecycle logs, and returns the correct OpenAI-compatible JSON or SSE shape. Remaining: rate-limit capture, retry/fallback across alternate accounts, 429 quota cache, 402 quota exhausted, 403 ban/Cloudflare distinction, path-block Cookie clearing, and richer OpenAI-compatible error mapping. - [ ] **Session affinity and previous response:** persist an in-memory response-id to account mapping with TTL, prompt-cache/conversation identity, turn-state replay, and preferred account selection. `src/codex/websocket.rs` must either implement the WebSocket path with verified TLS/header behavior or the API must reject WebSocket-only cases explicitly with documented limitations until fingerprint parity is proven. - [ ] **Fingerprint auto-update parity:** poll the real Codex Desktop appcast/update source, parse app version/build number, persist update state, select the latest stored fingerprint for runtime headers, and optionally extract Chromium version from a local Codex.app path when configured. diff --git a/src/http/v1.rs b/src/http/v1.rs index fe56a1afa..2e1d727ab 100644 --- a/src/http/v1.rs +++ b/src/http/v1.rs @@ -267,7 +267,9 @@ async fn send_codex_request_with_refresh_retry( ) -> Result { match send_codex_request(state, request, account, request_id).await { Err(CodexClientError::Upstream { status, body }) if status == StatusCode::UNAUTHORIZED => { - let Some(refreshed) = refresh_account_after_unauthorized(state, account).await else { + let Some(refreshed) = + refresh_account_after_unauthorized(state, request, account, request_id).await + else { return Err(CodexClientError::Upstream { status, body }); }; send_codex_request(state, request, &refreshed, request_id).await @@ -314,7 +316,9 @@ async fn send_codex_stream_request_with_refresh_retry( ) -> Result { match send_codex_stream_request(state, request, account, request_id).await { Err(CodexClientError::Upstream { status, body }) if status == StatusCode::UNAUTHORIZED => { - let Some(refreshed) = refresh_account_after_unauthorized(state, account).await else { + let Some(refreshed) = + refresh_account_after_unauthorized(state, request, account, request_id).await + else { return Err(CodexClientError::Upstream { status, body }); }; send_codex_stream_request(state, request, &refreshed, request_id).await @@ -447,7 +451,9 @@ async fn responses_stream( async fn refresh_account_after_unauthorized( state: &AppState, + request: &CodexResponsesRequest, account: &Account, + request_id: &str, ) -> Option { if !state.config().auth.refresh_enabled { return None; @@ -457,7 +463,7 @@ async fn refresh_account_after_unauthorized( match refresher.refresh(refresh_token).await { Ok(tokens) => persist_refreshed_account(state, account, tokens).await, Err(failure) => { - mark_refresh_failure(state, account, failure).await; + mark_refresh_failure(state, account, failure, request_id, &request.model).await; None } } @@ -494,16 +500,23 @@ async fn persist_refreshed_account( Some(refreshed) } -async fn mark_refresh_failure(state: &AppState, account: &Account, failure: RefreshFailure) { - let Some(status) = status_for_refresh_failure(failure) else { - return; - }; - if let Some(repo) = state.account_repository() { - let _ = repo.set_status(&account.id, status).await; +async fn mark_refresh_failure( + state: &AppState, + account: &Account, + failure: RefreshFailure, + request_id: &str, + model: &str, +) { + let status = status_for_refresh_failure(failure); + if let Some(status) = status { + if let Some(repo) = state.account_repository() { + let _ = repo.set_status(&account.id, status).await; + } + let mut updated = account.clone(); + updated.status = status; + state.account_pool().lock().await.insert(updated); } - let mut updated = account.clone(); - updated.status = status; - state.account_pool().lock().await.insert(updated); + log_account_refresh_failure(state, account, failure, status, request_id, model).await; } fn status_for_refresh_failure(failure: RefreshFailure) -> Option { @@ -516,6 +529,58 @@ fn status_for_refresh_failure(failure: RefreshFailure) -> Option } } +async fn log_account_refresh_failure( + state: &AppState, + account: &Account, + failure: RefreshFailure, + status: Option, + request_id: &str, + model: &str, +) { + let Some(repo) = state.event_logs() else { + return; + }; + let mut event = EventLog::new( + "account.refresh", + EventLevel::Warn, + "account refresh failed after upstream 401", + ); + event.request_id = Some(request_id.to_string()); + event.account_id = Some(account.id.clone()); + event.route = Some("/v1/responses".to_string()); + event.model = Some(model.to_string()); + event.status_code = Some(i64::from(StatusCode::UNAUTHORIZED.as_u16())); + event.metadata = json!({ + "trigger": "upstream_401", + "failure": refresh_failure_value(failure), + "accountStatus": status.map(account_status_value), + }); + if let Err(error) = repo.insert(event).await { + tracing::warn!(?error, "failed to insert account refresh event log"); + } +} + +fn refresh_failure_value(failure: RefreshFailure) -> &'static str { + match failure { + RefreshFailure::InvalidGrant => "invalidGrant", + RefreshFailure::QuotaExhausted => "quotaExhausted", + RefreshFailure::Banned => "banned", + RefreshFailure::Disabled => "disabled", + RefreshFailure::Transport => "transport", + } +} + +fn account_status_value(status: AccountStatus) -> &'static str { + match status { + AccountStatus::Active => "active", + AccountStatus::Expired => "expired", + AccountStatus::QuotaExhausted => "quotaExhausted", + AccountStatus::Refreshing => "refreshing", + AccountStatus::Disabled => "disabled", + AccountStatus::Banned => "banned", + } +} + async fn persist_upstream_cookies( state: &AppState, account_id: &str, diff --git a/tests/v1_upstream_route_test.rs b/tests/v1_upstream_route_test.rs index 8553b9568..c8f435b60 100644 --- a/tests/v1_upstream_route_test.rs +++ b/tests/v1_upstream_route_test.rs @@ -48,6 +48,11 @@ struct StaticTokenRefresher { refresh_token: Option, } +#[derive(Clone)] +struct FailingTokenRefresher { + failure: RefreshFailure, +} + #[async_trait] impl TokenRefresher for StaticTokenRefresher { async fn refresh(&self, _refresh_token: &str) -> Result { @@ -58,6 +63,13 @@ impl TokenRefresher for StaticTokenRefresher { } } +#[async_trait] +impl TokenRefresher for FailingTokenRefresher { + async fn refresh(&self, _refresh_token: &str) -> Result { + Err(self.failure) + } +} + fn test_config(database_url: String, base_url: String) -> AppConfig { AppConfig { server: ServerConfig { @@ -169,6 +181,23 @@ async fn build_imported_app(base_url: String) -> ImportedApp { } async fn build_imported_app_with_refresher(base_url: String, access_token: &str) -> ImportedApp { + build_imported_app_with_token_refresher( + base_url, + StaticTokenRefresher { + access_token: access_token.to_string(), + refresh_token: None, + }, + ) + .await +} + +async fn build_imported_app_with_token_refresher( + base_url: String, + token_refresher: C, +) -> ImportedApp +where + C: TokenRefresher, +{ let tempdir = tempfile::tempdir().unwrap(); let db = tempdir.path().join("v1-upstream-refresh.sqlite"); let url = format!("sqlite://{}", db.display()); @@ -187,10 +216,7 @@ async fn build_imported_app_with_refresher(base_url: String, access_token: &str) pool.clone(), secret_box.clone(), hasher, - StaticTokenRefresher { - access_token: access_token.to_string(), - refresh_token: None, - }, + token_refresher, ), ); @@ -514,6 +540,65 @@ async fn v1_responses_should_refresh_after_401_and_retry_stream() { assert_eq!(usage, (1, 4, 2, 0)); } +#[tokio::test] +async fn v1_responses_should_log_refresh_failure_after_401() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/codex/responses")) + .and(header("authorization", "Bearer access-secret")) + .respond_with(ResponseTemplate::new(401).set_body_json(json!({ + "error": {"code": "token_revoked"} + }))) + .mount(&server) + .await; + let imported = build_imported_app_with_token_refresher( + server.uri(), + FailingTokenRefresher { + failure: RefreshFailure::InvalidGrant, + }, + ) + .await; + + let response = imported + .app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/responses") + .header( + "authorization", + format!("Bearer {}", imported.client_api_key), + ) + .header("content-type", "application/json") + .header("x-request-id", "req_refresh_failed") + .body(Body::from(r#"{"model":"gpt-5.5","input":[]}"#)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let account_status: (String,) = sqlx::query_as("select status from accounts where id = ?") + .bind("acct_imported") + .fetch_one(&imported.pool) + .await + .unwrap(); + assert_eq!(account_status.0, "expired"); + let refresh_event: (String, String, String) = sqlx::query_as( + "select level, message, metadata_json from event_logs where request_id = ? and kind = 'account.refresh'", + ) + .bind("req_refresh_failed") + .fetch_one(&imported.pool) + .await + .unwrap(); + let metadata: Value = serde_json::from_str(&refresh_event.2).unwrap(); + assert_eq!(refresh_event.0, "warn"); + assert_eq!(refresh_event.1, "account refresh failed after upstream 401"); + assert_eq!(metadata["failure"], "invalidGrant"); + assert_eq!(metadata["accountStatus"], "expired"); + assert_eq!(metadata["trigger"], "upstream_401"); +} + async fn seed_admin_session(pool: &sqlx::SqlitePool, session_id: &str) { let now = Utc::now().to_rfc3339(); sqlx::query(