mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-17 16:06:57 +00:00
8747 lines
311 KiB
Rust
8747 lines
311 KiB
Rust
//! 服务组装。
|
|
|
|
use std::{
|
|
collections::BTreeMap,
|
|
path::PathBuf,
|
|
pin::Pin,
|
|
sync::{Arc, RwLock as StdRwLock},
|
|
task::{Context, Poll},
|
|
time::{Duration as StdDuration, Instant},
|
|
};
|
|
|
|
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
|
|
use bytes::Bytes;
|
|
use chrono::{DateTime, Duration, Utc};
|
|
use codex_proxy_adapters::{
|
|
codex::client::{
|
|
CodexBackendClient, CodexBackendResponse, CodexBackendSseStream,
|
|
CodexBackendStreamingResponse, CodexBackendTransport, CodexClientError,
|
|
CodexCompactResponse, CodexRateLimitHeaderUpdates, CodexRequestContext,
|
|
CodexTurnStateUpdate,
|
|
},
|
|
codex::fingerprint::FingerprintRepository,
|
|
oauth::openai::default_openai_oauth_client,
|
|
sqlite::{
|
|
accounts::{
|
|
AccountClaimsUpdate, AccountQuotaSnapshot, AccountUsageListRecord, AccountUsageSummary,
|
|
ImportedAccountUpdate, NewAccount, SqliteAccountStore, StoredAccount,
|
|
StoredAccountMetadata,
|
|
},
|
|
admin_sessions::SqliteAdminSessionStore,
|
|
client_keys::{CreatedClientApiKey, SqliteClientKeyStore, StoredClientApiKey},
|
|
cookies::SqliteCookieStore,
|
|
events::{EventLogFilter, SqliteEventLogStore},
|
|
refresh_leases::SqliteRefreshLeaseStore,
|
|
session_affinity::{SqliteSessionAffinityStore, SqliteSessionAffinityStoreError},
|
|
},
|
|
};
|
|
use codex_proxy_core::{
|
|
accounts::{
|
|
cloudflare::{CloudflareChallengeCooldownTracker, CloudflarePathBlockTracker},
|
|
jwt::{jwt_expiry, JwtExpiry},
|
|
model::{Account, AccountStatus},
|
|
pool::{
|
|
AccountAcquireRequest, AccountCapacitySummary, AccountPool, AccountPoolOptions,
|
|
AccountWindowUsageDelta, AcquiredAccount, RotationStrategy,
|
|
},
|
|
ports::{AccountStore, AccountStoreError},
|
|
},
|
|
admin::{
|
|
auth::AdminAuthService,
|
|
client_keys::ClientKeyService,
|
|
settings::{
|
|
AdminQuotaWarningThresholds, AdminSettings, AdminSettingsPatch, SettingsService,
|
|
SettingsServiceError,
|
|
},
|
|
},
|
|
auth::{
|
|
oauth::{
|
|
DeviceCode, OAuthConfig, OAuthError, PkceLogin, PkceSessionStore, RefreshFailure,
|
|
TokenPair,
|
|
},
|
|
ports::{OAuthClient, TokenRefresher},
|
|
},
|
|
events::{
|
|
model::{EventLevel, EventLog},
|
|
service::EventLogService,
|
|
},
|
|
gateway::{
|
|
conversation::{build_conversation_identity, ensure_prompt_cache_key},
|
|
fingerprint::Fingerprint,
|
|
ports::CodexModelCatalogClient,
|
|
},
|
|
models::{
|
|
model::ModelConfig,
|
|
ports::ModelSnapshotStore,
|
|
service::{ModelRefreshResult, ModelService, ModelServiceError},
|
|
},
|
|
protocol::{
|
|
codex::{
|
|
events::{
|
|
extract_sse_usage, extract_usage, parse_rate_limit_headers, rate_limit_quota,
|
|
TokenUsage,
|
|
},
|
|
responses::{CodexCompactRequest, CodexResponsesRequest},
|
|
sse::{encode_sse_event, parse_sse_events, SseError},
|
|
},
|
|
openai::chat::chat_completion_from_codex_sse,
|
|
openai::responses::{
|
|
completed_response_metadata, reconvert_responses_sse_event_tuple_values,
|
|
response_from_codex_sse, CollectedResponse, ResponsesSseFailure,
|
|
},
|
|
},
|
|
serving::{
|
|
affinity::{
|
|
compute_variant_hash, hash_instructions, prepare_variant_identity,
|
|
SessionAffinityEntry, SessionAffinityMap,
|
|
},
|
|
fallback::{
|
|
status_code_is_quota_exhausted, status_code_is_rate_limited,
|
|
status_code_is_transient_upstream,
|
|
},
|
|
implicit_resume::{
|
|
continuation_input_start, implicit_resume_allowed, ImplicitResumeSnapshot,
|
|
},
|
|
quota::{
|
|
quota_from_usage, quota_reached, quota_snapshot_limit_reached,
|
|
quota_snapshot_limit_window_seconds, quota_snapshot_reset_at,
|
|
},
|
|
reasoning_replay::ReasoningReplayCache,
|
|
recovery::status_code_allows_same_account_retry,
|
|
responses::apply_response_model_options,
|
|
},
|
|
usage::service::UsageService,
|
|
};
|
|
use codex_proxy_platform::{
|
|
config::{AppConfig, ConfigWriteError, QuotaWarningThresholds},
|
|
identity::{hash_admin_password, verify_admin_password},
|
|
json::Page,
|
|
};
|
|
use futures::{stream, Stream, StreamExt};
|
|
use secrecy::{ExposeSecret, SecretString};
|
|
use serde_json::{Map, Value};
|
|
use thiserror::Error;
|
|
use tokio::{
|
|
sync::{mpsc, oneshot, RwLock},
|
|
time::sleep,
|
|
};
|
|
|
|
use crate::{repositories::Repositories, upstream};
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum ExhaustedAccountClass {
|
|
QuotaExhausted,
|
|
RateLimited,
|
|
Expired,
|
|
Disabled,
|
|
Banned,
|
|
CloudflareChallenge,
|
|
CloudflarePathBlocked,
|
|
ModelUnsupported,
|
|
}
|
|
|
|
/// 运行时服务集合。
|
|
#[derive(Clone)]
|
|
pub struct Services {
|
|
/// 模型目录服务。
|
|
pub models: Arc<ModelService>,
|
|
/// 管理端模型服务。
|
|
pub admin_models: Arc<AdminModelService>,
|
|
/// 账号存储。
|
|
pub accounts: Arc<dyn AccountStore>,
|
|
/// 模型快照存储。
|
|
pub model_snapshots: Arc<dyn ModelSnapshotStore>,
|
|
/// 客户端 API Key 服务。
|
|
pub client_keys: Arc<ClientKeyService>,
|
|
/// 管理端客户端 API Key 服务。
|
|
pub admin_client_keys: Arc<AdminClientKeyService>,
|
|
/// 管理员会话服务。
|
|
pub admin_sessions: Arc<AdminSessionService>,
|
|
/// 管理端设置服务。
|
|
pub settings: Arc<RuntimeSettingsService>,
|
|
/// 管理端账号服务。
|
|
pub admin_accounts: Arc<AdminAccountService>,
|
|
/// 管理端 OAuth 服务。
|
|
pub admin_oauth: Arc<AdminOAuthService>,
|
|
/// 管理端日志服务。
|
|
pub logs: Arc<AdminLogService>,
|
|
/// 管理端用量服务。
|
|
pub usage: Arc<AdminUsageService>,
|
|
/// 运行时账号池服务。
|
|
pub account_pool: Arc<RuntimeAccountPoolService>,
|
|
/// OpenAI Chat Completions 调度服务。
|
|
pub chat: Arc<ChatDispatchService>,
|
|
/// OpenAI Responses 调度服务。
|
|
pub responses: Arc<ResponseDispatchService>,
|
|
/// 运行时会话亲和性服务。
|
|
pub session_affinity: Arc<RuntimeSessionAffinityService>,
|
|
/// Codex 上游客户端。
|
|
pub codex: Arc<CodexBackendClient>,
|
|
/// 当前运行时指纹。
|
|
pub fingerprint: Fingerprint,
|
|
/// 当前运行时 Codex installation id。
|
|
pub installation_id: Option<String>,
|
|
/// 后台任务需要的具体存储适配器。
|
|
pub background_tasks: BackgroundTaskStores,
|
|
}
|
|
|
|
/// Codex upstream connectivity probe result.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct RuntimeUpstreamProbe {
|
|
/// Probed target name.
|
|
pub target: &'static str,
|
|
/// Configured backend base URL.
|
|
pub backend_base_url: String,
|
|
/// Full endpoint URL.
|
|
pub endpoint: String,
|
|
/// Whether the endpoint responded at transport level.
|
|
pub reachable: bool,
|
|
/// HTTP status code, when available.
|
|
pub status_code: Option<u16>,
|
|
/// Authentication outcome inferred from status.
|
|
pub authorization: &'static str,
|
|
}
|
|
|
|
/// 后台任务需要的具体存储适配器集合。
|
|
#[derive(Clone)]
|
|
pub struct BackgroundTaskStores {
|
|
/// 账号存储。
|
|
pub accounts: SqliteAccountStore,
|
|
/// 管理员会话存储。
|
|
pub admin_sessions: SqliteAdminSessionStore,
|
|
/// Cookie 存储。
|
|
pub cookies: SqliteCookieStore,
|
|
/// 指纹存储。
|
|
pub fingerprints: FingerprintRepository,
|
|
/// 会话亲和性存储。
|
|
pub session_affinity: SqliteSessionAffinityStore,
|
|
/// 账号刷新租约存储。
|
|
pub refresh_leases: SqliteRefreshLeaseStore,
|
|
}
|
|
|
|
impl Services {
|
|
/// 从仓储和上游适配器构造运行时服务集合。
|
|
pub fn new(config: &AppConfig, repositories: Repositories, fingerprint: Fingerprint) -> Self {
|
|
Self::with_installation_id(config, repositories, fingerprint, None)
|
|
}
|
|
|
|
/// 从仓储、上游适配器和 installation id 构造运行时服务集合。
|
|
pub fn with_installation_id(
|
|
config: &AppConfig,
|
|
repositories: Repositories,
|
|
fingerprint: Fingerprint,
|
|
installation_id: Option<String>,
|
|
) -> Self {
|
|
Self::with_installation_id_and_local_config_path(
|
|
config,
|
|
repositories,
|
|
fingerprint,
|
|
installation_id,
|
|
"local.yaml",
|
|
)
|
|
}
|
|
|
|
/// 从仓储、上游适配器、installation id 和本地配置路径构造运行时服务集合。
|
|
pub fn with_installation_id_and_local_config_path(
|
|
config: &AppConfig,
|
|
repositories: Repositories,
|
|
fingerprint: Fingerprint,
|
|
installation_id: Option<String>,
|
|
local_config_path: impl Into<PathBuf>,
|
|
) -> Self {
|
|
let token_refresher: Arc<dyn TokenRefresher> =
|
|
Arc::new(default_openai_oauth_client(oauth_config(config)));
|
|
let oauth_client: Arc<dyn OAuthClient> =
|
|
Arc::new(default_openai_oauth_client(oauth_config(config)));
|
|
Self::with_installation_id_local_config_path_and_oauth_clients(
|
|
config,
|
|
repositories,
|
|
fingerprint,
|
|
installation_id,
|
|
local_config_path,
|
|
token_refresher,
|
|
oauth_client,
|
|
)
|
|
}
|
|
|
|
/// 从仓储、上游适配器、installation id、本地配置路径和 token refresher 构造服务集合。
|
|
pub fn with_installation_id_local_config_path_and_token_refresher(
|
|
config: &AppConfig,
|
|
repositories: Repositories,
|
|
fingerprint: Fingerprint,
|
|
installation_id: Option<String>,
|
|
local_config_path: impl Into<PathBuf>,
|
|
token_refresher: Arc<dyn TokenRefresher>,
|
|
) -> Self {
|
|
let oauth_client: Arc<dyn OAuthClient> =
|
|
Arc::new(default_openai_oauth_client(oauth_config(config)));
|
|
Self::with_installation_id_local_config_path_and_oauth_clients(
|
|
config,
|
|
repositories,
|
|
fingerprint,
|
|
installation_id,
|
|
local_config_path,
|
|
token_refresher,
|
|
oauth_client,
|
|
)
|
|
}
|
|
|
|
/// 从仓储、上游适配器、installation id、本地配置路径和 OAuth 端口构造服务集合。
|
|
pub fn with_installation_id_local_config_path_and_oauth_clients(
|
|
config: &AppConfig,
|
|
repositories: Repositories,
|
|
fingerprint: Fingerprint,
|
|
installation_id: Option<String>,
|
|
local_config_path: impl Into<PathBuf>,
|
|
token_refresher: Arc<dyn TokenRefresher>,
|
|
oauth_client: Arc<dyn OAuthClient>,
|
|
) -> Self {
|
|
let installation_id = installation_id.filter(|id| !id.trim().is_empty());
|
|
let runtime_fingerprint = fingerprint.clone();
|
|
let model_config = ModelConfig {
|
|
default_model: config.model.default_model.clone(),
|
|
default_reasoning_effort: config.model.default_reasoning_effort.clone(),
|
|
service_tier: config.model.service_tier.clone(),
|
|
aliases: config.model.aliases.clone(),
|
|
};
|
|
let account_store = repositories.accounts;
|
|
let cookies = repositories.cookies;
|
|
let background_tasks = BackgroundTaskStores {
|
|
accounts: account_store.clone(),
|
|
admin_sessions: repositories.admin_sessions.clone(),
|
|
cookies: cookies.clone(),
|
|
fingerprints: repositories.fingerprints.clone(),
|
|
session_affinity: repositories.session_affinity.clone(),
|
|
refresh_leases: repositories.refresh_leases.clone(),
|
|
};
|
|
let accounts = Arc::new(account_store.clone()) as Arc<dyn AccountStore>;
|
|
let account_pool = Arc::new(RuntimeAccountPoolService::new(
|
|
accounts.clone(),
|
|
account_pool_options(config),
|
|
config.auth.request_interval_ms,
|
|
));
|
|
let model_snapshots = Arc::new(repositories.model_snapshots) as Arc<dyn ModelSnapshotStore>;
|
|
let model_catalog_client =
|
|
upstream::model_catalog_client(config.api.base_url.clone(), fingerprint.clone())
|
|
as Arc<dyn CodexModelCatalogClient>;
|
|
let models = Arc::new(ModelService::new(
|
|
model_config,
|
|
Some(model_snapshots.clone()),
|
|
Some(model_catalog_client),
|
|
Some(Arc::new(tokio::sync::Mutex::new(BTreeMap::new()))),
|
|
));
|
|
let client_key_store = repositories.client_keys;
|
|
let client_keys = Arc::new(ClientKeyService::new(Arc::new(client_key_store.clone())));
|
|
let admin_client_keys = Arc::new(AdminClientKeyService::new(client_key_store));
|
|
let admin_sessions = Arc::new(AdminSessionService::new(
|
|
repositories.admin_sessions,
|
|
config.admin.default_username.clone(),
|
|
config.admin.session_ttl_minutes,
|
|
));
|
|
let settings = Arc::new(RuntimeSettingsService::with_local_config_path(
|
|
config.clone(),
|
|
local_config_path,
|
|
));
|
|
let admin_models = Arc::new(AdminModelService::new(
|
|
models.clone(),
|
|
accounts.clone(),
|
|
installation_id.clone(),
|
|
));
|
|
let codex = upstream::codex_backend_client(
|
|
config.api.base_url.clone(),
|
|
fingerprint,
|
|
&config.ws_pool,
|
|
);
|
|
let admin_accounts = Arc::new(AdminAccountService::new(
|
|
account_store.clone(),
|
|
cookies.clone(),
|
|
config.quota.warning_thresholds.clone(),
|
|
codex.clone(),
|
|
account_pool.clone(),
|
|
token_refresher,
|
|
config.auth.refresh_margin_seconds,
|
|
installation_id.clone(),
|
|
));
|
|
let admin_oauth = Arc::new(AdminOAuthService::new(oauth_config(config), oauth_client));
|
|
let logs = Arc::new(AdminLogService::new(
|
|
repositories.event_logs,
|
|
config.logging.enabled,
|
|
config.logging.capacity,
|
|
config.logging.capture_body,
|
|
));
|
|
let session_affinity = Arc::new(RuntimeSessionAffinityService::new(
|
|
repositories.session_affinity,
|
|
));
|
|
let usage = Arc::new(AdminUsageService::new(account_store));
|
|
let cloudflare_recovery = CloudflareRecovery::new(
|
|
cookies,
|
|
CloudflarePathBlockTracker::new(),
|
|
CloudflareChallengeCooldownTracker::new(),
|
|
);
|
|
let chat = Arc::new(ChatDispatchService::new(
|
|
account_pool.clone(),
|
|
models.clone(),
|
|
codex.clone(),
|
|
logs.clone(),
|
|
installation_id.clone(),
|
|
cloudflare_recovery.clone(),
|
|
));
|
|
let responses = Arc::new(ResponseDispatchService::new(
|
|
account_pool.clone(),
|
|
models.clone(),
|
|
codex.clone(),
|
|
session_affinity.clone(),
|
|
logs.clone(),
|
|
installation_id.clone(),
|
|
cloudflare_recovery,
|
|
));
|
|
|
|
Self {
|
|
models,
|
|
admin_models,
|
|
accounts,
|
|
model_snapshots,
|
|
client_keys,
|
|
admin_client_keys,
|
|
admin_sessions,
|
|
settings,
|
|
admin_accounts,
|
|
admin_oauth,
|
|
logs,
|
|
usage,
|
|
account_pool,
|
|
chat,
|
|
responses,
|
|
session_affinity,
|
|
codex,
|
|
fingerprint: runtime_fingerprint,
|
|
installation_id,
|
|
background_tasks,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Services {
|
|
/// Probe the Codex models endpoint without exposing account secrets.
|
|
pub async fn probe_codex_models_endpoint(&self, request_id: &str) -> RuntimeUpstreamProbe {
|
|
let config = self.settings.current();
|
|
let probe = self
|
|
.codex
|
|
.probe_models_endpoint(CodexRequestContext {
|
|
access_token: "",
|
|
account_id: None,
|
|
request_id,
|
|
turn_state: None,
|
|
turn_metadata: None,
|
|
beta_features: None,
|
|
include_timing_metrics: None,
|
|
version: None,
|
|
codex_window_id: None,
|
|
parent_thread_id: None,
|
|
cookie_header: None,
|
|
installation_id: self.installation_id.as_deref(),
|
|
session_id: None,
|
|
})
|
|
.await;
|
|
|
|
match probe {
|
|
Ok(probe) => RuntimeUpstreamProbe {
|
|
target: "codexModels",
|
|
backend_base_url: config.api.base_url.clone(),
|
|
endpoint: probe.endpoint,
|
|
reachable: true,
|
|
status_code: Some(probe.status.as_u16()),
|
|
authorization: if probe.status.as_u16() == 401 {
|
|
"rejected"
|
|
} else {
|
|
"unknown"
|
|
},
|
|
},
|
|
Err(_) => RuntimeUpstreamProbe {
|
|
target: "codexModels",
|
|
backend_base_url: config.api.base_url.clone(),
|
|
endpoint: format!(
|
|
"{}/codex/models?client_version={}",
|
|
config.api.base_url.trim_end_matches('/'),
|
|
self.fingerprint.app_version
|
|
),
|
|
reachable: false,
|
|
status_code: None,
|
|
authorization: "unknown",
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
fn account_pool_options(config: &AppConfig) -> AccountPoolOptions {
|
|
AccountPoolOptions {
|
|
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
|
stale_slot_ttl: Duration::minutes(5),
|
|
rotation_strategy: match config.auth.rotation_strategy.as_str() {
|
|
"round_robin" => RotationStrategy::RoundRobin,
|
|
"sticky" => RotationStrategy::Sticky,
|
|
_ => RotationStrategy::LeastUsed,
|
|
},
|
|
skip_quota_limited: config.quota.skip_exhausted,
|
|
tier_priority: config.auth.tier_priority.clone(),
|
|
model_plan_allowlist: BTreeMap::new(),
|
|
}
|
|
}
|
|
|
|
/// 运行时账号池服务。
|
|
#[derive(Clone)]
|
|
pub struct RuntimeAccountPoolService {
|
|
accounts: Arc<dyn AccountStore>,
|
|
pool: Arc<tokio::sync::Mutex<AccountPool>>,
|
|
request_interval: StdDuration,
|
|
}
|
|
|
|
impl RuntimeAccountPoolService {
|
|
/// 构造运行时账号池服务。
|
|
pub fn new(
|
|
accounts: Arc<dyn AccountStore>,
|
|
options: AccountPoolOptions,
|
|
request_interval_ms: u64,
|
|
) -> Self {
|
|
Self {
|
|
accounts,
|
|
pool: Arc::new(tokio::sync::Mutex::new(AccountPool::with_options(options))),
|
|
request_interval: StdDuration::from_millis(request_interval_ms),
|
|
}
|
|
}
|
|
|
|
/// 从账号存储恢复账号池内容。
|
|
pub async fn restore_from_repository(&self) -> Result<usize, RuntimeAccountPoolError> {
|
|
let accounts = self.accounts.list_pool_accounts().await?;
|
|
let restored = accounts.len();
|
|
let mut pool = self.pool.lock().await;
|
|
pool.clear();
|
|
for account in accounts {
|
|
pool.insert(account);
|
|
}
|
|
Ok(restored)
|
|
}
|
|
|
|
/// 从账号存储同步单个账号到运行时账号池;账号已不存在时从池中移除。
|
|
pub async fn sync_account_from_repository(
|
|
&self,
|
|
account_id: &str,
|
|
) -> Result<bool, RuntimeAccountPoolError> {
|
|
let account = self.accounts.get_pool_account(account_id).await?;
|
|
let mut pool = self.pool.lock().await;
|
|
if let Some(account) = account {
|
|
pool.insert(account);
|
|
return Ok(true);
|
|
}
|
|
Ok(pool.remove(account_id))
|
|
}
|
|
|
|
/// 获取运行时账号池中的账号快照。
|
|
pub async fn account_snapshot(&self, account_id: &str) -> Option<Account> {
|
|
self.pool.lock().await.get(account_id)
|
|
}
|
|
|
|
/// 从运行时账号池移除账号。
|
|
pub async fn remove_account(&self, account_id: &str) -> bool {
|
|
self.pool.lock().await.remove(account_id)
|
|
}
|
|
|
|
/// 清空运行时账号池。
|
|
pub async fn clear(&self) {
|
|
self.pool.lock().await.clear();
|
|
}
|
|
|
|
/// 从账号池获取指定模型可用账号。
|
|
pub async fn acquire(&self, model: &str, now: DateTime<Utc>) -> Option<AcquiredAccount> {
|
|
self.acquire_with(AccountAcquireRequest::new(model, now))
|
|
.await
|
|
}
|
|
|
|
/// 使用完整获取请求从账号池获取账号。
|
|
pub async fn acquire_with(&self, request: AccountAcquireRequest) -> Option<AcquiredAccount> {
|
|
let acquired = self.pool.lock().await.acquire_with(request)?;
|
|
if let Err(error) = self.accounts.record_request(&acquired.account.id).await {
|
|
tracing::warn!(
|
|
account_id = acquired.account.id,
|
|
error = %error,
|
|
"failed to persist account request usage"
|
|
);
|
|
}
|
|
Some(acquired)
|
|
}
|
|
|
|
/// 等待同一账号前一个在途请求满足配置的发送间隔。
|
|
pub async fn wait_for_request_interval(&self, acquired: &AcquiredAccount) {
|
|
if self.request_interval.is_zero() {
|
|
return;
|
|
}
|
|
let Some(previous_slot_at) = acquired.previous_slot_at else {
|
|
return;
|
|
};
|
|
let elapsed = Utc::now()
|
|
.signed_duration_since(previous_slot_at)
|
|
.to_std()
|
|
.unwrap_or_default();
|
|
if elapsed < self.request_interval {
|
|
sleep(self.request_interval - elapsed).await;
|
|
}
|
|
}
|
|
|
|
/// 释放账号的一个在途槽位。
|
|
pub async fn release(&self, account_id: &str) {
|
|
self.pool.lock().await.release(account_id);
|
|
}
|
|
|
|
/// Return a snapshot of runtime account-pool capacity.
|
|
pub async fn capacity_summary(&self, now: DateTime<Utc>) -> AccountCapacitySummary {
|
|
self.pool.lock().await.capacity_summary(now)
|
|
}
|
|
|
|
/// Return a snapshot of runtime account-pool capacity using the current time.
|
|
pub async fn capacity_summary_now(&self) -> AccountCapacitySummary {
|
|
self.capacity_summary(Utc::now()).await
|
|
}
|
|
|
|
/// 标记账号因配额限流进入冷却。
|
|
pub async fn mark_quota_limited_until(
|
|
&self,
|
|
account_id: &str,
|
|
cooldown_until: DateTime<Utc>,
|
|
) -> bool {
|
|
let persisted = match self
|
|
.accounts
|
|
.mark_quota_limited_until(account_id, cooldown_until)
|
|
.await
|
|
{
|
|
Ok(persisted) => persisted,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist quota cooldown state"
|
|
);
|
|
false
|
|
}
|
|
};
|
|
let in_memory = self
|
|
.pool
|
|
.lock()
|
|
.await
|
|
.mark_quota_limited_until(account_id, cooldown_until);
|
|
persisted || in_memory
|
|
}
|
|
|
|
/// 应用已经验证过的账号配额快照。
|
|
pub async fn apply_quota_snapshot(&self, account_id: &str, quota: &Value) -> bool {
|
|
let limit_reached = quota_snapshot_limit_reached(quota);
|
|
let reset_at = quota_snapshot_reset_at(quota);
|
|
let cooldown_until = limit_reached.then_some(reset_at).flatten();
|
|
let quota_json = quota.to_string();
|
|
let persisted = match self
|
|
.accounts
|
|
.apply_quota_snapshot(account_id, "a_json, limit_reached, cooldown_until)
|
|
.await
|
|
{
|
|
Ok(persisted) => persisted,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist verified quota snapshot"
|
|
);
|
|
false
|
|
}
|
|
};
|
|
let in_memory =
|
|
self.pool
|
|
.lock()
|
|
.await
|
|
.apply_quota_state(account_id, limit_reached, cooldown_until);
|
|
|
|
if let Some(reset_at) = reset_at {
|
|
let limit_window_seconds = quota_snapshot_limit_window_seconds(quota);
|
|
if let Err(error) = self
|
|
.accounts
|
|
.sync_rate_limit_window(account_id, reset_at, limit_window_seconds)
|
|
.await
|
|
{
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist verified quota window"
|
|
);
|
|
}
|
|
self.pool.lock().await.sync_rate_limit_window(
|
|
account_id,
|
|
reset_at,
|
|
limit_window_seconds,
|
|
);
|
|
}
|
|
|
|
persisted || in_memory
|
|
}
|
|
|
|
/// 标记账号处于 Cloudflare 冷却期。
|
|
pub async fn set_cloudflare_cooldown_until(
|
|
&self,
|
|
account_id: &str,
|
|
cooldown_until: DateTime<Utc>,
|
|
) -> bool {
|
|
let persisted = match self
|
|
.accounts
|
|
.set_cloudflare_cooldown_until(account_id, cooldown_until)
|
|
.await
|
|
{
|
|
Ok(persisted) => persisted,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist Cloudflare cooldown state"
|
|
);
|
|
false
|
|
}
|
|
};
|
|
let in_memory = self
|
|
.pool
|
|
.lock()
|
|
.await
|
|
.set_cloudflare_cooldown_until(account_id, cooldown_until);
|
|
persisted || in_memory
|
|
}
|
|
|
|
/// 更新账号状态。
|
|
pub async fn set_status(&self, account_id: &str, status: AccountStatus) -> bool {
|
|
let persisted = match self.accounts.set_status(account_id, status).await {
|
|
Ok(persisted) => persisted,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist account status"
|
|
);
|
|
false
|
|
}
|
|
};
|
|
let in_memory = self.pool.lock().await.set_status(account_id, status);
|
|
persisted || in_memory
|
|
}
|
|
|
|
/// 清零运行时账号池中的累计和窗口用量。
|
|
pub async fn reset_usage(&self, account_id: &str) -> bool {
|
|
self.pool.lock().await.reset_usage(account_id)
|
|
}
|
|
|
|
/// 记录账号成功响应的 token 用量。
|
|
pub async fn record_token_usage(&self, account_id: &str, usage: TokenUsage) {
|
|
self.record_response_usage(account_id, usage, false).await;
|
|
}
|
|
|
|
/// 记录 Responses 成功响应的 token 与工具用量。
|
|
pub async fn record_response_usage(
|
|
&self,
|
|
account_id: &str,
|
|
usage: TokenUsage,
|
|
image_generation_requested: bool,
|
|
) {
|
|
let image_request_succeeded = image_generation_requested && usage.image_output_tokens > 0;
|
|
let image_request_failed = image_generation_requested && !image_request_succeeded;
|
|
let mut persisted_usage = UsageService::account_delta_from_token_usage(usage);
|
|
persisted_usage.image_requests = bool_to_u64(image_request_succeeded);
|
|
persisted_usage.image_request_failures = bool_to_u64(image_request_failed);
|
|
if let Err(error) = self
|
|
.accounts
|
|
.record_usage_delta(account_id, persisted_usage)
|
|
.await
|
|
{
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist account token usage"
|
|
);
|
|
}
|
|
self.pool.lock().await.record_window_token_usage(
|
|
account_id,
|
|
AccountWindowUsageDelta {
|
|
input_tokens: usage.input_tokens,
|
|
output_tokens: usage.output_tokens,
|
|
cached_tokens: usage.cached_tokens,
|
|
image_input_tokens: usage.image_input_tokens,
|
|
image_output_tokens: usage.image_output_tokens,
|
|
image_request_succeeded,
|
|
image_request_failed,
|
|
},
|
|
);
|
|
}
|
|
|
|
/// 记录 Responses 空响应尝试。
|
|
pub async fn record_empty_response_attempt(
|
|
&self,
|
|
account_id: &str,
|
|
image_generation_requested: bool,
|
|
) {
|
|
let usage = codex_proxy_core::accounts::usage::AccountUsageDelta {
|
|
empty_responses: 1,
|
|
image_request_failures: bool_to_u64(image_generation_requested),
|
|
..codex_proxy_core::accounts::usage::AccountUsageDelta::default()
|
|
};
|
|
if let Err(error) = self.accounts.record_usage_delta(account_id, usage).await {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist empty response usage"
|
|
);
|
|
}
|
|
if image_generation_requested {
|
|
self.pool.lock().await.record_window_token_usage(
|
|
account_id,
|
|
AccountWindowUsageDelta {
|
|
image_request_failed: true,
|
|
..AccountWindowUsageDelta::default()
|
|
},
|
|
);
|
|
}
|
|
}
|
|
|
|
/// 将上游成功响应头里的 rate-limit 状态被动写回配额和窗口缓存。
|
|
pub async fn sync_passive_rate_limit_headers(
|
|
&self,
|
|
account: &Account,
|
|
headers: &[(String, String)],
|
|
) {
|
|
let Some(rate_limits) = parse_rate_limit_headers(headers) else {
|
|
return;
|
|
};
|
|
let existing_quota = match self.accounts.get_quota_json(&account.id).await {
|
|
Ok(Some(quota_json)) => serde_json::from_str::<Value>("a_json).ok(),
|
|
Ok(None) => None,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id = %account.id,
|
|
error = %error,
|
|
"failed to read existing quota json before passive rate-limit sync"
|
|
);
|
|
None
|
|
}
|
|
};
|
|
let quota = rate_limit_quota(
|
|
&rate_limits,
|
|
account.plan_type.as_deref(),
|
|
existing_quota.as_ref(),
|
|
);
|
|
self.apply_quota_snapshot(&account.id, "a).await;
|
|
}
|
|
}
|
|
|
|
/// 运行时账号池错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum RuntimeAccountPoolError {
|
|
/// 账号存储访问失败。
|
|
#[error("account store error: {0}")]
|
|
Store(#[from] AccountStoreError),
|
|
}
|
|
|
|
const MAX_QUOTA_VERIFY_ATTEMPTS: usize = 5;
|
|
const QUOTA_VERIFY_LIMIT_REACHED_MESSAGE: &str = "Upstream usage quota still reports limit_reached";
|
|
|
|
#[derive(Clone)]
|
|
pub(crate) struct CloudflareRecovery {
|
|
cookies: SqliteCookieStore,
|
|
path_block_tracker: CloudflarePathBlockTracker,
|
|
challenge_cooldowns: CloudflareChallengeCooldownTracker,
|
|
}
|
|
|
|
impl CloudflareRecovery {
|
|
fn new(
|
|
cookies: SqliteCookieStore,
|
|
path_block_tracker: CloudflarePathBlockTracker,
|
|
challenge_cooldowns: CloudflareChallengeCooldownTracker,
|
|
) -> Self {
|
|
Self {
|
|
cookies,
|
|
path_block_tracker,
|
|
challenge_cooldowns,
|
|
}
|
|
}
|
|
|
|
async fn cookie_header_for_request(
|
|
&self,
|
|
account_id: &str,
|
|
request_path: &str,
|
|
) -> Option<String> {
|
|
match self
|
|
.cookies
|
|
.cookie_header_for_request(account_id, "chatgpt.com", request_path)
|
|
.await
|
|
{
|
|
Ok(cookie_header) => cookie_header,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to read account cookies for upstream request"
|
|
);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn capture_set_cookie_headers(&self, account_id: &str, headers: &[String]) {
|
|
for header in headers {
|
|
if let Err(error) = self.cookies.capture_set_cookie(account_id, header).await {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to persist upstream set-cookie header"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn apply_challenge(&self, account_pool: &RuntimeAccountPoolService, account_id: &str) {
|
|
self.delete_account_cookies(account_id, "Cloudflare challenge")
|
|
.await;
|
|
let cooldown = self
|
|
.challenge_cooldowns
|
|
.record_challenge(account_id, Utc::now())
|
|
.await;
|
|
account_pool
|
|
.set_cloudflare_cooldown_until(account_id, cooldown.cooldown_until)
|
|
.await;
|
|
tracing::warn!(
|
|
account_id,
|
|
challenge_count = cooldown.challenge_count,
|
|
delay_seconds = cooldown.delay_seconds,
|
|
"upstream returned Cloudflare challenge"
|
|
);
|
|
}
|
|
|
|
async fn apply_path_block(&self, account_pool: &RuntimeAccountPoolService, account_id: &str) {
|
|
self.delete_account_cookies(account_id, "Cloudflare path-block")
|
|
.await;
|
|
let now = Utc::now();
|
|
let count = self
|
|
.path_block_tracker
|
|
.record_path_block(account_id, now)
|
|
.await;
|
|
if self
|
|
.path_block_tracker
|
|
.should_disable(account_id, now)
|
|
.await
|
|
{
|
|
account_pool
|
|
.set_status(account_id, AccountStatus::Disabled)
|
|
.await;
|
|
}
|
|
tracing::warn!(
|
|
account_id,
|
|
path_block_count = count,
|
|
"upstream returned Cloudflare path-block"
|
|
);
|
|
}
|
|
|
|
async fn reset_account_recovery(&self, account_id: &str) {
|
|
self.path_block_tracker.reset(account_id).await;
|
|
self.challenge_cooldowns.reset(account_id).await;
|
|
}
|
|
|
|
async fn delete_account_cookies(&self, account_id: &str, reason: &str) {
|
|
if let Err(error) = self.cookies.delete_account_cookies(account_id).await {
|
|
tracing::warn!(
|
|
account_id,
|
|
reason,
|
|
error = %error,
|
|
"failed to delete account cookies after Cloudflare recovery signal"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 默认会话亲和性 TTL 秒数。
|
|
const DEFAULT_SESSION_AFFINITY_TTL_SECS: i64 = 4 * 60 * 60;
|
|
|
|
/// 运行时会话亲和性服务。
|
|
#[derive(Clone)]
|
|
pub struct RuntimeSessionAffinityService {
|
|
store: SqliteSessionAffinityStore,
|
|
map: Arc<tokio::sync::RwLock<SessionAffinityMap>>,
|
|
ttl: Duration,
|
|
}
|
|
|
|
/// 默认 reasoning replay TTL 秒数。
|
|
const DEFAULT_REASONING_REPLAY_TTL_SECS: i64 = 55 * 60;
|
|
const IMPLICIT_RESUME_MAX_AGE_SECS: i64 = DEFAULT_REASONING_REPLAY_TTL_SECS;
|
|
|
|
impl RuntimeSessionAffinityService {
|
|
/// 构造运行时会话亲和性服务。
|
|
pub fn new(store: SqliteSessionAffinityStore) -> Self {
|
|
let ttl = Duration::seconds(DEFAULT_SESSION_AFFINITY_TTL_SECS);
|
|
Self {
|
|
store,
|
|
map: Arc::new(tokio::sync::RwLock::new(SessionAffinityMap::new(ttl))),
|
|
ttl,
|
|
}
|
|
}
|
|
|
|
/// 从 SQLite 恢复未过期的会话亲和性记录。
|
|
pub async fn restore_from_repository(
|
|
&self,
|
|
now: DateTime<Utc>,
|
|
) -> Result<usize, RuntimeSessionAffinityError> {
|
|
let records = self.store.list_active(now).await?;
|
|
Ok(self.map.write().await.restore(records, now))
|
|
}
|
|
|
|
/// 记录并持久化响应 ID 的亲和性条目。
|
|
pub async fn record(
|
|
&self,
|
|
response_id: String,
|
|
entry: SessionAffinityEntry,
|
|
) -> Result<(), RuntimeSessionAffinityError> {
|
|
self.store.upsert(&response_id, &entry, self.ttl).await?;
|
|
self.map.write().await.record(response_id, entry);
|
|
Ok(())
|
|
}
|
|
|
|
/// 根据响应 ID 查找账号 ID。
|
|
pub async fn lookup_account(&self, response_id: &str, now: DateTime<Utc>) -> Option<String> {
|
|
self.map.read().await.lookup_account(response_id, now)
|
|
}
|
|
|
|
/// 根据响应 ID 查找对话 ID。
|
|
pub async fn lookup_conversation_id(
|
|
&self,
|
|
response_id: &str,
|
|
now: DateTime<Utc>,
|
|
) -> Option<String> {
|
|
self.map
|
|
.read()
|
|
.await
|
|
.lookup_conversation_id(response_id, now)
|
|
}
|
|
|
|
/// 根据响应 ID 查找 turn state。
|
|
pub async fn lookup_turn_state(&self, response_id: &str, now: DateTime<Utc>) -> Option<String> {
|
|
self.map.read().await.lookup_turn_state(response_id, now)
|
|
}
|
|
|
|
/// 根据响应 ID 查找指令哈希。
|
|
pub async fn lookup_instructions_hash(
|
|
&self,
|
|
response_id: &str,
|
|
now: DateTime<Utc>,
|
|
) -> Option<String> {
|
|
self.map
|
|
.read()
|
|
.await
|
|
.lookup_instructions_hash(response_id, now)
|
|
}
|
|
|
|
/// 根据响应 ID 查找函数调用 ID 列表。
|
|
pub async fn lookup_function_call_ids(
|
|
&self,
|
|
response_id: &str,
|
|
now: DateTime<Utc>,
|
|
) -> Vec<String> {
|
|
self.map
|
|
.read()
|
|
.await
|
|
.lookup_function_call_ids(response_id, now)
|
|
}
|
|
|
|
/// 查找指定对话和变体下最新的响应 ID。
|
|
pub async fn lookup_latest_response_by_conversation(
|
|
&self,
|
|
conversation_id: &str,
|
|
max_age: Option<Duration>,
|
|
variant_hash: Option<&str>,
|
|
now: DateTime<Utc>,
|
|
) -> Option<String> {
|
|
self.map
|
|
.read()
|
|
.await
|
|
.lookup_latest_response_by_conversation(conversation_id, max_age, variant_hash, now)
|
|
}
|
|
|
|
/// 删除响应 ID 的内存亲和性映射。
|
|
pub async fn forget(&self, response_id: &str) -> bool {
|
|
self.map.write().await.forget(response_id)
|
|
}
|
|
}
|
|
|
|
/// 运行时会话亲和性错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum RuntimeSessionAffinityError {
|
|
/// 存储访问失败。
|
|
#[error("session affinity store error: {0}")]
|
|
Store(#[from] SqliteSessionAffinityStoreError),
|
|
}
|
|
|
|
pub(crate) fn oauth_config(config: &AppConfig) -> OAuthConfig {
|
|
OAuthConfig {
|
|
client_id: config.auth.oauth_client_id.clone(),
|
|
auth_endpoint: config.auth.oauth_auth_endpoint.clone(),
|
|
device_code_endpoint: oauth_device_code_endpoint(&config.auth.oauth_token_endpoint),
|
|
token_endpoint: config.auth.oauth_token_endpoint.clone(),
|
|
}
|
|
}
|
|
|
|
fn oauth_device_code_endpoint(token_endpoint: &str) -> String {
|
|
token_endpoint
|
|
.strip_suffix("/token")
|
|
.map(|prefix| format!("{prefix}/device/code"))
|
|
.unwrap_or_else(|| "https://auth.openai.com/oauth/device/code".to_string())
|
|
}
|
|
|
|
/// 管理端模型服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminModelService {
|
|
models: Arc<ModelService>,
|
|
accounts: Arc<dyn AccountStore>,
|
|
installation_id: Option<String>,
|
|
}
|
|
|
|
impl AdminModelService {
|
|
/// 构造管理端模型服务。
|
|
pub fn new(
|
|
models: Arc<ModelService>,
|
|
accounts: Arc<dyn AccountStore>,
|
|
installation_id: Option<String>,
|
|
) -> Self {
|
|
Self {
|
|
models,
|
|
accounts,
|
|
installation_id,
|
|
}
|
|
}
|
|
|
|
/// 用当前活跃账号刷新上游模型快照。
|
|
pub async fn refresh_backend_models(
|
|
&self,
|
|
request_id: &str,
|
|
) -> Result<ModelRefreshResult, AdminModelError> {
|
|
let accounts = self
|
|
.accounts
|
|
.list_pool_accounts()
|
|
.await
|
|
.map_err(|_| AdminModelError::ListAccounts)?;
|
|
self.models
|
|
.refresh_backend_models_with_installation_id(
|
|
&accounts,
|
|
request_id,
|
|
self.installation_id.as_deref(),
|
|
)
|
|
.await
|
|
.map_err(AdminModelError::from)
|
|
}
|
|
}
|
|
|
|
/// 管理端模型错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminModelError {
|
|
/// 列出账号失败。
|
|
#[error("failed to list accounts")]
|
|
ListAccounts,
|
|
/// 没有可用账号。
|
|
#[error("no active accounts available for model refresh")]
|
|
NoAccounts,
|
|
/// 模型快照存储不可用。
|
|
#[error("model snapshot store is unavailable")]
|
|
SnapshotStoreUnavailable,
|
|
/// 上游客户端不可用。
|
|
#[error("model upstream client is unavailable")]
|
|
UpstreamClientUnavailable,
|
|
/// 写入快照失败。
|
|
#[error("failed to store model snapshot")]
|
|
StoreSnapshot,
|
|
/// 读取快照失败。
|
|
#[error("failed to load model snapshots")]
|
|
LoadSnapshots,
|
|
/// 所有计划刷新失败。
|
|
#[error("all model refresh plans failed")]
|
|
AllPlansFailed(ModelRefreshResult),
|
|
}
|
|
|
|
impl From<ModelServiceError> for AdminModelError {
|
|
fn from(error: ModelServiceError) -> Self {
|
|
match error {
|
|
ModelServiceError::SnapshotStoreUnavailable => Self::SnapshotStoreUnavailable,
|
|
ModelServiceError::UpstreamClientUnavailable => Self::UpstreamClientUnavailable,
|
|
ModelServiceError::NoAccounts => Self::NoAccounts,
|
|
ModelServiceError::StoreSnapshot => Self::StoreSnapshot,
|
|
ModelServiceError::LoadSnapshots => Self::LoadSnapshots,
|
|
ModelServiceError::AllPlansFailed(result) => Self::AllPlansFailed(result),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 管理端 OAuth 服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminOAuthService {
|
|
config: OAuthConfig,
|
|
client: Arc<dyn OAuthClient>,
|
|
sessions: Arc<tokio::sync::Mutex<PkceSessionStore>>,
|
|
}
|
|
|
|
impl AdminOAuthService {
|
|
/// 构造管理端 OAuth 服务。
|
|
pub fn new(config: OAuthConfig, client: Arc<dyn OAuthClient>) -> Self {
|
|
Self {
|
|
config,
|
|
client,
|
|
sessions: Arc::new(tokio::sync::Mutex::new(PkceSessionStore::default())),
|
|
}
|
|
}
|
|
|
|
/// 开始 PKCE 登录。
|
|
pub async fn start_pkce_login(&self, return_host: &str) -> PkceLogin {
|
|
self.sessions
|
|
.lock()
|
|
.await
|
|
.start_login(return_host, &self.config)
|
|
}
|
|
|
|
/// 请求设备码登录信息。
|
|
pub async fn request_device_code(&self) -> Result<DeviceCode, AdminOAuthError> {
|
|
self.client
|
|
.request_device_code()
|
|
.await
|
|
.map_err(AdminOAuthError::OAuth)
|
|
}
|
|
|
|
/// 轮询设备码 token。
|
|
pub async fn poll_device_token(
|
|
&self,
|
|
device_code: &str,
|
|
) -> Result<AdminDevicePoll, AdminOAuthError> {
|
|
match self.client.poll_device_token(device_code).await {
|
|
Ok(tokens) => Ok(AdminDevicePoll::Authorized(tokens)),
|
|
Err(error) => {
|
|
if let Some(code) = error.pending_code() {
|
|
Ok(AdminDevicePoll::Pending { code })
|
|
} else {
|
|
Err(AdminOAuthError::OAuth(error))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 使用回调 code/state 完成 PKCE token 交换。
|
|
pub async fn exchange_callback(
|
|
&self,
|
|
code: &str,
|
|
state: &str,
|
|
) -> Result<AdminOAuthCallback, AdminOAuthError> {
|
|
let session = self
|
|
.sessions
|
|
.lock()
|
|
.await
|
|
.try_acquire(state)
|
|
.ok_or(AdminOAuthError::InvalidState)?;
|
|
|
|
match self
|
|
.client
|
|
.exchange_code(code, &session.code_verifier, &session.redirect_uri)
|
|
.await
|
|
{
|
|
Ok(tokens) => {
|
|
self.sessions.lock().await.complete(state);
|
|
Ok(AdminOAuthCallback {
|
|
tokens,
|
|
return_host: session.return_host,
|
|
})
|
|
}
|
|
Err(error) => {
|
|
self.sessions.lock().await.release(state);
|
|
Err(AdminOAuthError::OAuth(error))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 设备码轮询结果。
|
|
#[derive(Debug, Clone)]
|
|
pub enum AdminDevicePoll {
|
|
/// 授权还未完成。
|
|
Pending {
|
|
/// OAuth 标准 pending 错误码。
|
|
code: &'static str,
|
|
},
|
|
/// 已换取 token。
|
|
Authorized(TokenPair),
|
|
}
|
|
|
|
/// PKCE 回调换取的 token 和返回 host。
|
|
#[derive(Debug, Clone)]
|
|
pub struct AdminOAuthCallback {
|
|
/// OAuth token 对。
|
|
pub tokens: TokenPair,
|
|
/// 登录前的管理端 host。
|
|
pub return_host: String,
|
|
}
|
|
|
|
/// 管理端 OAuth 错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminOAuthError {
|
|
/// callback URL 或 query 缺少必需字段。
|
|
#[error("invalid OAuth callback")]
|
|
InvalidCallback,
|
|
/// OAuth state 不存在、过期或正在处理。
|
|
#[error("invalid OAuth state")]
|
|
InvalidState,
|
|
/// OAuth 上游错误。
|
|
#[error("{0}")]
|
|
OAuth(OAuthError),
|
|
}
|
|
|
|
/// 管理端用量服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminUsageService {
|
|
store: SqliteAccountStore,
|
|
}
|
|
|
|
impl AdminUsageService {
|
|
/// 构造管理端用量服务。
|
|
pub fn new(store: SqliteAccountStore) -> Self {
|
|
Self { store }
|
|
}
|
|
|
|
/// 分页列出账号用量统计。
|
|
pub async fn list(
|
|
&self,
|
|
cursor: Option<String>,
|
|
limit: u32,
|
|
) -> Result<Page<AdminUsageRecord>, AdminUsageError> {
|
|
let page = self
|
|
.store
|
|
.list_usage(cursor, limit)
|
|
.await
|
|
.map_err(|_| AdminUsageError::List)?;
|
|
Ok(Page {
|
|
items: page.items.into_iter().map(AdminUsageRecord::from).collect(),
|
|
next_cursor: page.next_cursor,
|
|
})
|
|
}
|
|
|
|
/// 汇总账号用量统计。
|
|
pub async fn summary(&self) -> Result<AdminUsageSummary, AdminUsageError> {
|
|
self.store
|
|
.usage_summary()
|
|
.await
|
|
.map(AdminUsageSummary::from)
|
|
.map_err(|_| AdminUsageError::Summary)
|
|
}
|
|
}
|
|
|
|
/// 管理端用量错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminUsageError {
|
|
/// 列表失败。
|
|
#[error("failed to list account usage")]
|
|
List,
|
|
/// 汇总失败。
|
|
#[error("failed to summarize account usage")]
|
|
Summary,
|
|
}
|
|
|
|
/// 管理端用量记录。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminUsageRecord {
|
|
/// 账号 ID。
|
|
pub account_id: String,
|
|
/// 邮箱。
|
|
pub email: Option<String>,
|
|
/// 管理标签。
|
|
pub label: Option<String>,
|
|
/// 计划类型。
|
|
pub plan_type: Option<String>,
|
|
/// 请求数。
|
|
pub request_count: i64,
|
|
/// 空响应数。
|
|
pub empty_response_count: i64,
|
|
/// 输入 token 数。
|
|
pub input_tokens: i64,
|
|
/// 输出 token 数。
|
|
pub output_tokens: i64,
|
|
/// 缓存 token 数。
|
|
pub cached_tokens: i64,
|
|
/// reasoning token 数。
|
|
pub reasoning_tokens: i64,
|
|
/// 上游返回的总 token 数。
|
|
pub total_tokens: i64,
|
|
/// 图片输入 token 数。
|
|
pub image_input_tokens: i64,
|
|
/// 图片输出 token 数。
|
|
pub image_output_tokens: i64,
|
|
/// 图片请求数。
|
|
pub image_request_count: i64,
|
|
/// 图片请求失败数。
|
|
pub image_request_failed_count: i64,
|
|
/// 最近使用时间。
|
|
pub last_used_at: Option<chrono::DateTime<chrono::Utc>>,
|
|
}
|
|
|
|
/// 管理端用量汇总。
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct AdminUsageSummary {
|
|
/// 有用量记录的账号数。
|
|
pub account_count: i64,
|
|
/// 请求总数。
|
|
pub request_count: i64,
|
|
/// 空响应总数。
|
|
pub empty_response_count: i64,
|
|
/// 输入 token 总数。
|
|
pub input_tokens: i64,
|
|
/// 输出 token 总数。
|
|
pub output_tokens: i64,
|
|
/// 缓存 token 总数。
|
|
pub cached_tokens: i64,
|
|
/// reasoning token 总数。
|
|
pub reasoning_tokens: i64,
|
|
/// 上游返回 token 总数。
|
|
pub total_tokens: i64,
|
|
/// 图片输入 token 总数。
|
|
pub image_input_tokens: i64,
|
|
/// 图片输出 token 总数。
|
|
pub image_output_tokens: i64,
|
|
/// 图片请求总数。
|
|
pub image_request_count: i64,
|
|
/// 图片请求失败总数。
|
|
pub image_request_failed_count: i64,
|
|
}
|
|
|
|
/// OpenAI Chat Completions 调度服务。
|
|
#[derive(Clone)]
|
|
pub struct ChatDispatchService {
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
models: Arc<ModelService>,
|
|
codex: Arc<CodexBackendClient>,
|
|
logs: Arc<AdminLogService>,
|
|
installation_id: Option<String>,
|
|
cloudflare: CloudflareRecovery,
|
|
}
|
|
|
|
impl ChatDispatchService {
|
|
/// 构造 Chat Completions 调度服务。
|
|
pub(crate) fn new(
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
models: Arc<ModelService>,
|
|
codex: Arc<CodexBackendClient>,
|
|
logs: Arc<AdminLogService>,
|
|
installation_id: Option<String>,
|
|
cloudflare: CloudflareRecovery,
|
|
) -> Self {
|
|
Self {
|
|
account_pool,
|
|
models,
|
|
codex,
|
|
logs,
|
|
installation_id,
|
|
cloudflare,
|
|
}
|
|
}
|
|
|
|
/// 调度非流式 Chat Completions 请求到 Codex Responses 上游。
|
|
pub async fn complete(
|
|
&self,
|
|
request_id: &str,
|
|
mut request: CodexResponsesRequest,
|
|
requested_model: &str,
|
|
) -> Result<Value, ChatDispatchError> {
|
|
let started_at = Instant::now();
|
|
let catalog = self.models.catalog().await;
|
|
let parsed_model = catalog.parse_model_name(requested_model);
|
|
let display_model =
|
|
codex_proxy_core::models::catalog::ModelCatalog::build_display_model_name(
|
|
&parsed_model,
|
|
);
|
|
apply_response_model_options(&mut request, &parsed_model, self.models.config());
|
|
let include_reasoning = request
|
|
.reasoning
|
|
.as_ref()
|
|
.and_then(|reasoning| reasoning.get("effort"))
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|effort| !effort.trim().is_empty());
|
|
let tuple_schema = request.tuple_schema.clone();
|
|
let mut excluded_account_ids = Vec::new();
|
|
let mut rate_limited_count = 0usize;
|
|
let mut last_rate_limit_error = None;
|
|
let mut quota_exhausted_count = 0usize;
|
|
let mut last_quota_error = None;
|
|
let mut expired_count = 0usize;
|
|
let mut last_auth_error = None;
|
|
let mut disabled_count = 0usize;
|
|
let mut last_disabled_auth_error = None;
|
|
let mut banned_count = 0usize;
|
|
let mut last_banned_auth_error = None;
|
|
let mut last_banned_status_code: Option<u16> = None;
|
|
let mut cloudflare_challenge_count = 0usize;
|
|
let mut last_cloudflare_challenge_error = None;
|
|
let mut cloudflare_path_block_count = 0usize;
|
|
let mut last_cloudflare_path_block_error = None;
|
|
let mut model_unsupported_count = 0usize;
|
|
let mut last_model_unsupported_error = None;
|
|
let mut model_unsupported_retry_used = false;
|
|
let mut quota_verify_attempts = 0usize;
|
|
let mut last_failed_account_id = None;
|
|
macro_rules! return_dispatch_error {
|
|
($error:expr) => {{
|
|
let error = $error;
|
|
self.record_chat_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_failed_account_id.as_deref(),
|
|
Some(backend_transport_name(requested_response_transport(
|
|
&request,
|
|
))),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}};
|
|
($error:expr, account_id: $account_id:expr, transport: $transport:expr) => {{
|
|
let error = $error;
|
|
self.record_chat_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
$account_id,
|
|
$transport,
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}};
|
|
}
|
|
let (account_id, response) = loop {
|
|
let acquire_request = AccountAcquireRequest::new(&request.model, Utc::now())
|
|
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
|
let acquired = match self.account_pool.acquire_with(acquire_request).await {
|
|
Some(acquired) => acquired,
|
|
None if quota_exhausted_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::QuotaExhausted {
|
|
count: quota_exhausted_count,
|
|
upstream_error: last_quota_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if rate_limited_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::RateLimited {
|
|
count: rate_limited_count,
|
|
upstream_error: last_rate_limit_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if expired_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::Expired {
|
|
count: expired_count,
|
|
upstream_error: last_auth_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if disabled_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::Disabled {
|
|
count: disabled_count,
|
|
upstream_error: last_disabled_auth_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if banned_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::Banned {
|
|
count: banned_count,
|
|
upstream_error: last_banned_auth_error.unwrap_or_default(),
|
|
status_code: last_banned_status_code.unwrap_or(403),
|
|
});
|
|
}
|
|
None if cloudflare_challenge_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::CloudflareChallenge {
|
|
count: cloudflare_challenge_count,
|
|
upstream_error: last_cloudflare_challenge_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if cloudflare_path_block_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::CloudflarePathBlocked {
|
|
count: cloudflare_path_block_count,
|
|
upstream_error: last_cloudflare_path_block_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None if model_unsupported_count > 0 => {
|
|
return_dispatch_error!(ChatDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count,
|
|
upstream_error: last_model_unsupported_error.unwrap_or_default(),
|
|
});
|
|
}
|
|
None => return_dispatch_error!(ChatDispatchError::NoActiveAccount),
|
|
};
|
|
let acquired = match verify_acquired_quota_if_required(
|
|
self.account_pool.as_ref(),
|
|
self.codex.as_ref(),
|
|
&self.cloudflare,
|
|
self.installation_id.as_deref(),
|
|
request_id,
|
|
acquired,
|
|
&mut excluded_account_ids,
|
|
&mut quota_verify_attempts,
|
|
)
|
|
.await
|
|
{
|
|
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
|
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string());
|
|
continue;
|
|
}
|
|
QuotaVerificationDecision::MaxAttemptsReached => {
|
|
return_dispatch_error!(ChatDispatchError::RateLimited {
|
|
count: rate_limited_count + 1,
|
|
upstream_error: QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string(),
|
|
});
|
|
}
|
|
};
|
|
self.account_pool.wait_for_request_interval(&acquired).await;
|
|
let account = acquired.account;
|
|
let release_account_id = account.id.clone();
|
|
let response_result = create_response_with_account(
|
|
&self.codex,
|
|
self.installation_id.as_deref(),
|
|
&self.cloudflare,
|
|
&request,
|
|
request_id,
|
|
&account,
|
|
)
|
|
.await;
|
|
self.account_pool.release(&release_account_id).await;
|
|
|
|
match response_result {
|
|
Ok(response) => break (release_account_id, response),
|
|
Err(error) if is_rate_limit_upstream_error(&error) => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(upstream_error_body(&error));
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
let cooldown_until = rate_limit_cooldown_until(&error, Utc::now());
|
|
self.account_pool
|
|
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(upstream_error_body(&error));
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_auth_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
let account_status = auth_failure_account_status(&error);
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
|
cloudflare_challenge_count += 1;
|
|
last_cloudflare_challenge_error =
|
|
Some(cloudflare_challenge_error_message().to_string());
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
self.cloudflare
|
|
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
|
cloudflare_path_block_count += 1;
|
|
last_cloudflare_path_block_error =
|
|
Some(cloudflare_path_block_error_message().to_string());
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
self.cloudflare
|
|
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
if model_unsupported_retry_used {
|
|
return_dispatch_error!(
|
|
ChatDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
},
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(requested_response_transport(
|
|
&request
|
|
)))
|
|
);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_banned_upstream_error(&error) => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error_body(&error));
|
|
last_failed_account_id = Some(release_account_id.clone());
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::Banned)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) => {
|
|
return_dispatch_error!(
|
|
ChatDispatchError::Upstream(error),
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(requested_response_transport(
|
|
&request
|
|
)))
|
|
);
|
|
}
|
|
}
|
|
};
|
|
let body = match chat_completion_from_codex_sse(
|
|
&response.body,
|
|
&display_model,
|
|
include_reasoning,
|
|
tuple_schema.as_ref(),
|
|
) {
|
|
Ok(Some(body)) => body,
|
|
Ok(None) => {
|
|
return_dispatch_error!(
|
|
ChatDispatchError::EmptyUpstreamResponse,
|
|
account_id: Some(&account_id),
|
|
transport: Some(backend_transport_name(response.transport))
|
|
);
|
|
}
|
|
Err(error) => {
|
|
return_dispatch_error!(
|
|
ChatDispatchError::InvalidSse(error),
|
|
account_id: Some(&account_id),
|
|
transport: Some(backend_transport_name(response.transport))
|
|
);
|
|
}
|
|
};
|
|
self.cloudflare.reset_account_recovery(&account_id).await;
|
|
if let Some(usage) = response.usage {
|
|
self.account_pool
|
|
.record_token_usage(&account_id, usage)
|
|
.await;
|
|
}
|
|
record_response_event(ResponseEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &account_id,
|
|
route: "/v1/chat/completions",
|
|
model: requested_model,
|
|
started_at,
|
|
status_code: 200,
|
|
level: EventLevel::Info,
|
|
message: "v1 chat completions completed",
|
|
metadata: serde_json::json!({
|
|
"stream": false,
|
|
"transport": backend_transport_name(response.transport),
|
|
"usage": response.usage,
|
|
}),
|
|
rate_limit_headers: &response.rate_limit_headers,
|
|
})
|
|
.await;
|
|
Ok(body)
|
|
}
|
|
|
|
async fn record_chat_dispatch_error(
|
|
&self,
|
|
request_id: &str,
|
|
requested_model: &str,
|
|
started_at: Instant,
|
|
account_id: Option<&str>,
|
|
transport: Option<&str>,
|
|
error: &ChatDispatchError,
|
|
) {
|
|
record_chat_dispatch_error_event(ChatDispatchErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id,
|
|
route: "/v1/chat/completions",
|
|
model: requested_model,
|
|
started_at,
|
|
transport,
|
|
error,
|
|
})
|
|
.await;
|
|
}
|
|
}
|
|
|
|
/// Chat Completions 调度错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum ChatDispatchError {
|
|
/// 账号存储失败。
|
|
#[error("failed to list runtime accounts")]
|
|
AccountStore,
|
|
/// 没有活跃账号。
|
|
#[error("no active account is available")]
|
|
NoActiveAccount,
|
|
/// 所有账号都因配额耗尽不可用。
|
|
#[error("all accounts exhausted by quota")]
|
|
QuotaExhausted {
|
|
/// 配额耗尽账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因限流不可用。
|
|
#[error("all accounts exhausted by rate limit")]
|
|
RateLimited {
|
|
/// 限流账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因认证失效不可用。
|
|
#[error("all accounts exhausted by expired auth")]
|
|
Expired {
|
|
/// 认证失效账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因 token 已确认不可用而被禁用。
|
|
#[error("all accounts exhausted by disabled auth")]
|
|
Disabled {
|
|
/// 禁用账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都被上游封禁。
|
|
#[error("all accounts exhausted by banned auth")]
|
|
Banned {
|
|
/// 封禁账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
/// 应返回给客户端的上游 HTTP 状态。
|
|
status_code: u16,
|
|
},
|
|
/// 所有账号都因 Cloudflare challenge 暂不可用。
|
|
#[error("all accounts exhausted by Cloudflare challenge")]
|
|
CloudflareChallenge {
|
|
/// Cloudflare challenge 账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误说明。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因 Cloudflare path-block 暂不可用。
|
|
#[error("all accounts exhausted by Cloudflare path-block")]
|
|
CloudflarePathBlocked {
|
|
/// Cloudflare path-block 账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误说明。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都不支持请求模型。
|
|
#[error("all accounts do not support the requested model")]
|
|
ModelUnsupported {
|
|
/// 不支持模型的账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 上游请求失败。
|
|
#[error("upstream request failed: {0}")]
|
|
Upstream(#[from] CodexClientError),
|
|
/// 上游 SSE 无法解析。
|
|
#[error("invalid upstream SSE response: {0}")]
|
|
InvalidSse(#[from] SseError),
|
|
/// 上游响应没有完成事件。
|
|
#[error("upstream response did not include response.completed")]
|
|
EmptyUpstreamResponse,
|
|
}
|
|
|
|
/// OpenAI Responses 调度服务。
|
|
#[derive(Clone)]
|
|
pub struct ResponseDispatchService {
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
models: Arc<ModelService>,
|
|
codex: Arc<CodexBackendClient>,
|
|
session_affinity: Arc<RuntimeSessionAffinityService>,
|
|
reasoning_replay: Arc<tokio::sync::Mutex<ReasoningReplayCache>>,
|
|
logs: Arc<AdminLogService>,
|
|
installation_id: Option<String>,
|
|
cloudflare: CloudflareRecovery,
|
|
}
|
|
|
|
/// Responses live SSE 响应体流。
|
|
pub type ResponseBodyStream =
|
|
Pin<Box<dyn Stream<Item = Result<Bytes, ResponseDispatchStreamError>> + Send + 'static>>;
|
|
|
|
/// Responses live SSE 调度结果。
|
|
pub struct ResponseDispatchStream {
|
|
/// 可直接转为 HTTP body 的 SSE 字节流。
|
|
pub body: ResponseBodyStream,
|
|
}
|
|
|
|
struct MpscResponseBodyStream {
|
|
receiver: mpsc::Receiver<Result<Bytes, ResponseDispatchStreamError>>,
|
|
cancel: Option<oneshot::Sender<()>>,
|
|
}
|
|
|
|
impl Drop for MpscResponseBodyStream {
|
|
fn drop(&mut self) {
|
|
if let Some(cancel) = self.cancel.take() {
|
|
let _ = cancel.send(());
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Stream for MpscResponseBodyStream {
|
|
type Item = Result<Bytes, ResponseDispatchStreamError>;
|
|
|
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
self.receiver.poll_recv(cx)
|
|
}
|
|
}
|
|
|
|
impl ResponseDispatchService {
|
|
/// 构造 Responses 调度服务。
|
|
pub(crate) fn new(
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
models: Arc<ModelService>,
|
|
codex: Arc<CodexBackendClient>,
|
|
session_affinity: Arc<RuntimeSessionAffinityService>,
|
|
logs: Arc<AdminLogService>,
|
|
installation_id: Option<String>,
|
|
cloudflare: CloudflareRecovery,
|
|
) -> Self {
|
|
Self {
|
|
account_pool,
|
|
models,
|
|
codex,
|
|
session_affinity,
|
|
reasoning_replay: Arc::new(tokio::sync::Mutex::new(ReasoningReplayCache::new(
|
|
Duration::seconds(DEFAULT_REASONING_REPLAY_TTL_SECS),
|
|
))),
|
|
logs,
|
|
installation_id,
|
|
cloudflare,
|
|
}
|
|
}
|
|
|
|
async fn prepare_response_session(
|
|
&self,
|
|
request: &mut CodexResponsesRequest,
|
|
) -> Option<ImplicitResumeSnapshot> {
|
|
prepare_variant_identity(request);
|
|
if let Some(previous_response_id) = request.previous_response_id.clone() {
|
|
if request.prompt_cache_key.is_none() {
|
|
request.prompt_cache_key = self
|
|
.session_affinity
|
|
.lookup_conversation_id(&previous_response_id, Utc::now())
|
|
.await;
|
|
}
|
|
if request.turn_state.as_deref().is_none_or(str::is_empty) {
|
|
request.turn_state = self
|
|
.session_affinity
|
|
.lookup_turn_state(&previous_response_id, Utc::now())
|
|
.await;
|
|
}
|
|
ensure_prompt_cache_key(request);
|
|
return None;
|
|
}
|
|
|
|
ensure_prompt_cache_key(request);
|
|
self.apply_implicit_resume(request).await
|
|
}
|
|
|
|
async fn apply_implicit_resume(
|
|
&self,
|
|
request: &mut CodexResponsesRequest,
|
|
) -> Option<ImplicitResumeSnapshot> {
|
|
let continuation_start = continuation_input_start(&request.input);
|
|
if continuation_start == 0 || continuation_start >= request.input.len() {
|
|
return None;
|
|
}
|
|
let conversation_id = request
|
|
.prompt_cache_key
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())?
|
|
.to_string();
|
|
let snapshot = ImplicitResumeSnapshot::capture(request);
|
|
let variant_hash = compute_variant_hash(request);
|
|
let now = Utc::now();
|
|
let previous_response_id = self
|
|
.session_affinity
|
|
.lookup_latest_response_by_conversation(
|
|
&conversation_id,
|
|
Some(Duration::seconds(IMPLICIT_RESUME_MAX_AGE_SECS)),
|
|
Some(&variant_hash),
|
|
now,
|
|
)
|
|
.await?;
|
|
let current_instructions_hash = hash_instructions(Some(&request.instructions));
|
|
if self
|
|
.session_affinity
|
|
.lookup_instructions_hash(&previous_response_id, now)
|
|
.await
|
|
.as_deref()
|
|
!= Some(current_instructions_hash.as_str())
|
|
{
|
|
return None;
|
|
}
|
|
let stored_function_call_ids = self
|
|
.session_affinity
|
|
.lookup_function_call_ids(&previous_response_id, now)
|
|
.await;
|
|
if !implicit_resume_allowed(
|
|
&request.input[continuation_start..],
|
|
&request.input,
|
|
&stored_function_call_ids,
|
|
) {
|
|
return None;
|
|
}
|
|
let account_id = self
|
|
.session_affinity
|
|
.lookup_account(&previous_response_id, now)
|
|
.await?;
|
|
let replay_items = self.reasoning_replay.lock().await.lookup(
|
|
&previous_response_id,
|
|
&account_id,
|
|
&conversation_id,
|
|
&variant_hash,
|
|
now,
|
|
);
|
|
let continuation = request.input[continuation_start..].to_vec();
|
|
let mut input = replay_items;
|
|
input.extend(continuation);
|
|
|
|
request.previous_response_id = Some(previous_response_id.clone());
|
|
request.use_websocket = true;
|
|
request.force_http_sse = false;
|
|
request.input = input;
|
|
if let Some(turn_state) = self
|
|
.session_affinity
|
|
.lookup_turn_state(&previous_response_id, now)
|
|
.await
|
|
{
|
|
request.turn_state = Some(turn_state);
|
|
}
|
|
|
|
Some(snapshot)
|
|
}
|
|
|
|
async fn preferred_account_id_for_request(
|
|
&self,
|
|
request: &CodexResponsesRequest,
|
|
now: DateTime<Utc>,
|
|
) -> Option<String> {
|
|
let previous_response_id = request.previous_response_id.as_deref()?;
|
|
self.session_affinity
|
|
.lookup_account(previous_response_id, now)
|
|
.await
|
|
}
|
|
|
|
async fn recover_request_history(
|
|
&self,
|
|
request: &mut CodexResponsesRequest,
|
|
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
|
) {
|
|
if let Some(previous_response_id) = request.previous_response_id.as_deref() {
|
|
self.session_affinity.forget(previous_response_id).await;
|
|
}
|
|
if let Some(snapshot) = implicit_resume.take() {
|
|
snapshot.restore(request);
|
|
request.previous_response_id = None;
|
|
request.turn_state = None;
|
|
} else {
|
|
strip_request_history(request);
|
|
}
|
|
}
|
|
|
|
async fn apply_cascading_ban_defense(
|
|
&self,
|
|
request: &mut CodexResponsesRequest,
|
|
implicit_resume: &mut Option<ImplicitResumeSnapshot>,
|
|
preferred_account_id: Option<&str>,
|
|
acquired_account_id: &str,
|
|
explicit_previous_response_id: Option<&str>,
|
|
) -> bool {
|
|
let Some(preferred_account_id) =
|
|
preferred_account_id.filter(|account_id| *account_id != acquired_account_id)
|
|
else {
|
|
return false;
|
|
};
|
|
let has_history = request.previous_response_id.is_some()
|
|
|| request
|
|
.turn_state
|
|
.as_deref()
|
|
.is_some_and(|value| !value.is_empty());
|
|
if !has_history {
|
|
return false;
|
|
}
|
|
let Some(preferred_account) = self
|
|
.account_pool
|
|
.account_snapshot(preferred_account_id)
|
|
.await
|
|
else {
|
|
return false;
|
|
};
|
|
if !matches!(
|
|
preferred_account.status,
|
|
AccountStatus::Banned | AccountStatus::Disabled
|
|
) {
|
|
return false;
|
|
}
|
|
|
|
let response_id_to_forget = explicit_previous_response_id
|
|
.or(request.previous_response_id.as_deref())
|
|
.map(str::to_string);
|
|
if let Some(snapshot) = implicit_resume.take() {
|
|
snapshot.restore(request);
|
|
request.previous_response_id = None;
|
|
request.turn_state = None;
|
|
} else {
|
|
strip_request_history(request);
|
|
}
|
|
if let Some(response_id) = response_id_to_forget {
|
|
self.session_affinity.forget(&response_id).await;
|
|
}
|
|
true
|
|
}
|
|
|
|
async fn evict_reasoning_replay(&self, request: &CodexResponsesRequest, account_id: &str) {
|
|
evict_reasoning_replay(&self.reasoning_replay, request, account_id).await;
|
|
}
|
|
|
|
async fn record_response_affinity(
|
|
&self,
|
|
request: &CodexResponsesRequest,
|
|
account_id: &str,
|
|
body: &str,
|
|
turn_state: Option<String>,
|
|
usage: Option<TokenUsage>,
|
|
) {
|
|
record_response_affinity(
|
|
&self.session_affinity,
|
|
&self.reasoning_replay,
|
|
request,
|
|
account_id,
|
|
body,
|
|
turn_state,
|
|
usage,
|
|
)
|
|
.await;
|
|
}
|
|
|
|
/// 调度非流式 Responses 请求到 Codex Responses 上游。
|
|
pub async fn complete(
|
|
&self,
|
|
request_id: &str,
|
|
route: &str,
|
|
mut request: CodexResponsesRequest,
|
|
requested_model: &str,
|
|
) -> Result<Value, ResponseDispatchError> {
|
|
let started_at = Instant::now();
|
|
let catalog = self.models.catalog().await;
|
|
let parsed_model = catalog.parse_model_name(requested_model);
|
|
apply_response_model_options(&mut request, &parsed_model, self.models.config());
|
|
let tuple_schema = request.tuple_schema.clone();
|
|
let image_generation_requested = request.expects_image_generation();
|
|
let now = Utc::now();
|
|
let explicit_previous_response_id = request.previous_response_id.clone();
|
|
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
|
let preferred_account_id = self.preferred_account_id_for_request(&request, now).await;
|
|
let mut acquire_request = AccountAcquireRequest::new(&request.model, now);
|
|
if let Some(preferred_account_id) = preferred_account_id.as_deref() {
|
|
acquire_request = acquire_request.with_preferred_account_id(preferred_account_id);
|
|
}
|
|
let mut excluded_account_ids = Vec::new();
|
|
let mut rate_limited_count = 0usize;
|
|
let mut last_rate_limit_error = None;
|
|
let mut quota_exhausted_count = 0usize;
|
|
let mut last_quota_error = None;
|
|
let mut expired_count = 0usize;
|
|
let mut last_auth_error = None;
|
|
let mut disabled_count = 0usize;
|
|
let mut last_disabled_auth_error = None;
|
|
let mut banned_count = 0usize;
|
|
let mut last_banned_auth_error = None;
|
|
let mut last_banned_status_code: Option<u16> = None;
|
|
let mut cloudflare_challenge_count = 0usize;
|
|
let mut last_cloudflare_challenge_error = None;
|
|
let mut cloudflare_path_block_count = 0usize;
|
|
let mut last_cloudflare_path_block_error = None;
|
|
let mut model_unsupported_count = 0usize;
|
|
let mut last_model_unsupported_error = None;
|
|
let mut model_unsupported_retry_used = false;
|
|
let mut history_recovery_used = false;
|
|
let mut last_exhausted_account_class = None;
|
|
let mut empty_response_retries = 0u8;
|
|
let mut quota_verify_attempts = 0usize;
|
|
let mut last_attempted_account_id = None;
|
|
const MAX_EMPTY_RESPONSE_RETRIES: u8 = 2;
|
|
let (account, response, collected_response) = loop {
|
|
let mut attempt_acquire_request = acquire_request
|
|
.clone()
|
|
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
|
attempt_acquire_request.now = Utc::now();
|
|
let acquired = match self
|
|
.account_pool
|
|
.acquire_with(attempt_acquire_request)
|
|
.await
|
|
{
|
|
Some(acquired) => acquired,
|
|
None => {
|
|
let error = match last_exhausted_account_class {
|
|
Some(ExhaustedAccountClass::QuotaExhausted) => {
|
|
ResponseDispatchError::QuotaExhausted {
|
|
count: quota_exhausted_count,
|
|
upstream_error: last_quota_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::RateLimited) => {
|
|
ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count,
|
|
upstream_error: last_rate_limit_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::Expired) => ResponseDispatchError::Expired {
|
|
count: expired_count,
|
|
upstream_error: last_auth_error.unwrap_or_default(),
|
|
},
|
|
Some(ExhaustedAccountClass::Disabled) => ResponseDispatchError::Disabled {
|
|
count: disabled_count,
|
|
upstream_error: last_disabled_auth_error.unwrap_or_default(),
|
|
},
|
|
Some(ExhaustedAccountClass::Banned) => ResponseDispatchError::Banned {
|
|
count: banned_count,
|
|
upstream_error: last_banned_auth_error.unwrap_or_default(),
|
|
status_code: last_banned_status_code.unwrap_or(403),
|
|
},
|
|
Some(ExhaustedAccountClass::CloudflareChallenge) => {
|
|
ResponseDispatchError::CloudflareChallenge {
|
|
count: cloudflare_challenge_count,
|
|
upstream_error: last_cloudflare_challenge_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::CloudflarePathBlocked) => {
|
|
ResponseDispatchError::CloudflarePathBlocked {
|
|
count: cloudflare_path_block_count,
|
|
upstream_error: last_cloudflare_path_block_error
|
|
.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::ModelUnsupported) => {
|
|
ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count,
|
|
upstream_error: last_model_unsupported_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
None => ResponseDispatchError::NoActiveAccount,
|
|
};
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(requested_response_transport(
|
|
&request,
|
|
))),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
};
|
|
let acquired_account_id = acquired.account.id.clone();
|
|
let acquired = match verify_acquired_quota_if_required(
|
|
self.account_pool.as_ref(),
|
|
self.codex.as_ref(),
|
|
&self.cloudflare,
|
|
self.installation_id.as_deref(),
|
|
request_id,
|
|
acquired,
|
|
&mut excluded_account_ids,
|
|
&mut quota_verify_attempts,
|
|
)
|
|
.await
|
|
{
|
|
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
|
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string());
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::RateLimited);
|
|
continue;
|
|
}
|
|
QuotaVerificationDecision::MaxAttemptsReached => {
|
|
let error = ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count + 1,
|
|
upstream_error: QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string(),
|
|
};
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&acquired_account_id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(requested_response_transport(
|
|
&request,
|
|
))),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
};
|
|
self.apply_cascading_ban_defense(
|
|
&mut request,
|
|
&mut implicit_resume,
|
|
preferred_account_id.as_deref(),
|
|
&acquired.account.id,
|
|
explicit_previous_response_id.as_deref(),
|
|
)
|
|
.await;
|
|
self.account_pool.wait_for_request_interval(&acquired).await;
|
|
let account = acquired.account;
|
|
let release_account_id = account.id.clone();
|
|
last_attempted_account_id = Some(release_account_id.clone());
|
|
let response_result = create_response_with_account_retrying_5xx(
|
|
&self.codex,
|
|
self.installation_id.as_deref(),
|
|
&self.cloudflare,
|
|
&request,
|
|
request_id,
|
|
&account,
|
|
)
|
|
.await;
|
|
self.account_pool.release(&release_account_id).await;
|
|
if let Err(error) = &response_result {
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(
|
|
&release_account_id,
|
|
upstream_error_set_cookie_headers(error),
|
|
)
|
|
.await;
|
|
}
|
|
|
|
match response_result {
|
|
Ok(response) => {
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(
|
|
&release_account_id,
|
|
&response.set_cookie_headers,
|
|
)
|
|
.await;
|
|
let collected_response =
|
|
match response_from_codex_sse(&response.body, tuple_schema.as_ref()) {
|
|
Ok(collected_response) => collected_response,
|
|
Err(error) => {
|
|
let error = ResponseDispatchError::InvalidSse(error);
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&release_account_id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(response.transport)),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
};
|
|
if matches!(collected_response, CollectedResponse::Empty) {
|
|
self.account_pool
|
|
.record_empty_response_attempt(
|
|
&release_account_id,
|
|
image_generation_requested,
|
|
)
|
|
.await;
|
|
empty_response_retries += 1;
|
|
if empty_response_retries <= MAX_EMPTY_RESPONSE_RETRIES {
|
|
continue;
|
|
}
|
|
}
|
|
if let CollectedResponse::Failed(failure) = &collected_response {
|
|
if is_history_recovery_sse_failure(failure) && !history_recovery_used {
|
|
if sse_failure_invalid_reasoning_replay(failure) {
|
|
self.evict_reasoning_replay(&request, &release_account_id)
|
|
.await;
|
|
}
|
|
self.recover_request_history(&mut request, &mut implicit_resume)
|
|
.await;
|
|
history_recovery_used = true;
|
|
continue;
|
|
}
|
|
if is_model_unsupported_sse_failure(failure) {
|
|
let upstream_error = sse_failure_error_body(failure);
|
|
if model_unsupported_retry_used {
|
|
let error = ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
};
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&release_account_id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(response.transport)),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::ModelUnsupported);
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
continue;
|
|
}
|
|
if is_quota_exhausted_sse_failure(failure) {
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(failure.message.clone());
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::QuotaExhausted);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
continue;
|
|
}
|
|
if is_auth_sse_failure(failure) {
|
|
let upstream_error = sse_failure_error_body(failure);
|
|
let account_status = auth_sse_failure_account_status(failure);
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Disabled);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code =
|
|
Some(stream_failure_http_status(failure));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Banned);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Expired);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
continue;
|
|
}
|
|
}
|
|
break (account, response, collected_response);
|
|
}
|
|
Err(error) if is_rate_limit_upstream_error(&error) => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::RateLimited);
|
|
let cooldown_until = rate_limit_cooldown_until(&error, Utc::now());
|
|
self.account_pool
|
|
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error)
|
|
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
|
|
{
|
|
if client_error_invalid_reasoning_replay(&error) {
|
|
self.evict_reasoning_replay(&request, &release_account_id)
|
|
.await;
|
|
}
|
|
self.recover_request_history(&mut request, &mut implicit_resume)
|
|
.await;
|
|
history_recovery_used = true;
|
|
}
|
|
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::QuotaExhausted);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_auth_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
let account_status = auth_failure_account_status(&error);
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Disabled);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Banned);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Expired);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
|
cloudflare_challenge_count += 1;
|
|
last_cloudflare_challenge_error =
|
|
Some(cloudflare_challenge_error_message().to_string());
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::CloudflareChallenge);
|
|
self.cloudflare
|
|
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
|
cloudflare_path_block_count += 1;
|
|
last_cloudflare_path_block_error =
|
|
Some(cloudflare_path_block_error_message().to_string());
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::CloudflarePathBlocked);
|
|
self.cloudflare
|
|
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
if model_unsupported_retry_used {
|
|
let error = ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
};
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&release_account_id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(requested_response_transport(
|
|
&request,
|
|
))),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::ModelUnsupported);
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_banned_upstream_error(&error) => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Banned);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::Banned)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) => {
|
|
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &release_account_id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
stream: false,
|
|
transport: requested_response_transport(&request),
|
|
error: &error,
|
|
})
|
|
.await;
|
|
return Err(ResponseDispatchError::Upstream(error));
|
|
}
|
|
}
|
|
};
|
|
|
|
match collected_response {
|
|
CollectedResponse::Completed(body) => {
|
|
self.cloudflare.reset_account_recovery(&account.id).await;
|
|
self.account_pool
|
|
.sync_passive_rate_limit_headers(&account, &response.rate_limit_headers)
|
|
.await;
|
|
if let Some(usage) = response.usage {
|
|
self.account_pool
|
|
.record_response_usage(&account.id, usage, image_generation_requested)
|
|
.await;
|
|
}
|
|
self.record_response_affinity(
|
|
&request,
|
|
&account.id,
|
|
&response.body,
|
|
response.turn_state.clone(),
|
|
response.usage,
|
|
)
|
|
.await;
|
|
record_response_event(ResponseEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &account.id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
status_code: 200,
|
|
level: EventLevel::Info,
|
|
message: "v1 responses completed",
|
|
metadata: serde_json::json!({
|
|
"stream": false,
|
|
"transport": backend_transport_name(response.transport),
|
|
"usage": response.usage,
|
|
}),
|
|
rate_limit_headers: &response.rate_limit_headers,
|
|
})
|
|
.await;
|
|
Ok(body)
|
|
}
|
|
CollectedResponse::Failed(failure) => {
|
|
let error = ResponseDispatchError::Failed(failure);
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&account.id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(response.transport)),
|
|
&error,
|
|
)
|
|
.await;
|
|
Err(error)
|
|
}
|
|
CollectedResponse::MissingCompleted => {
|
|
let error = ResponseDispatchError::MissingCompleted;
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&account.id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(response.transport)),
|
|
&error,
|
|
)
|
|
.await;
|
|
Err(error)
|
|
}
|
|
CollectedResponse::Empty => {
|
|
let error = ResponseDispatchError::EmptyUpstreamResponse;
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
Some(&account.id),
|
|
false,
|
|
false,
|
|
Some(backend_transport_name(response.transport)),
|
|
&error,
|
|
)
|
|
.await;
|
|
Err(error)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[expect(
|
|
clippy::too_many_arguments,
|
|
reason = "keeps response dispatch logging call sites explicit at branch exits"
|
|
)]
|
|
async fn record_response_dispatch_error(
|
|
&self,
|
|
request_id: &str,
|
|
route: &str,
|
|
requested_model: &str,
|
|
started_at: Instant,
|
|
account_id: Option<&str>,
|
|
stream: bool,
|
|
compact: bool,
|
|
transport: Option<&str>,
|
|
error: &ResponseDispatchError,
|
|
) {
|
|
record_response_dispatch_error_event(ResponseDispatchErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
stream,
|
|
compact,
|
|
transport,
|
|
error,
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// 调度 Responses compact 请求到 Codex compact 上游。
|
|
pub async fn compact(
|
|
&self,
|
|
request_id: &str,
|
|
mut request: CodexCompactRequest,
|
|
requested_model: &str,
|
|
) -> Result<Value, ResponseDispatchError> {
|
|
let started_at = Instant::now();
|
|
let catalog = self.models.catalog().await;
|
|
let parsed_model = catalog.parse_model_name(requested_model);
|
|
request.model = parsed_model.model_id;
|
|
let mut excluded_account_ids = Vec::new();
|
|
let mut rate_limited_count = 0usize;
|
|
let mut last_rate_limit_error = None;
|
|
let mut quota_exhausted_count = 0usize;
|
|
let mut last_quota_error = None;
|
|
let mut expired_count = 0usize;
|
|
let mut last_auth_error = None;
|
|
let mut disabled_count = 0usize;
|
|
let mut last_disabled_auth_error = None;
|
|
let mut banned_count = 0usize;
|
|
let mut last_banned_auth_error = None;
|
|
let mut last_banned_status_code: Option<u16> = None;
|
|
let mut cloudflare_challenge_count = 0usize;
|
|
let mut last_cloudflare_challenge_error = None;
|
|
let mut cloudflare_path_block_count = 0usize;
|
|
let mut last_cloudflare_path_block_error = None;
|
|
let mut model_unsupported_count = 0usize;
|
|
let mut last_model_unsupported_error = None;
|
|
let mut model_unsupported_retry_used = false;
|
|
let mut quota_verify_attempts = 0usize;
|
|
let mut last_attempted_account_id = None::<String>;
|
|
|
|
loop {
|
|
let acquire_request = AccountAcquireRequest::new(&request.model, Utc::now())
|
|
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
|
let acquired = match self.account_pool.acquire_with(acquire_request).await {
|
|
Some(acquired) => acquired,
|
|
None if quota_exhausted_count > 0 => {
|
|
let error = ResponseDispatchError::QuotaExhausted {
|
|
count: quota_exhausted_count,
|
|
upstream_error: last_quota_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if rate_limited_count > 0 => {
|
|
let error = ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count,
|
|
upstream_error: last_rate_limit_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if expired_count > 0 => {
|
|
let error = ResponseDispatchError::Expired {
|
|
count: expired_count,
|
|
upstream_error: last_auth_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if disabled_count > 0 => {
|
|
let error = ResponseDispatchError::Disabled {
|
|
count: disabled_count,
|
|
upstream_error: last_disabled_auth_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if banned_count > 0 => {
|
|
let error = ResponseDispatchError::Banned {
|
|
count: banned_count,
|
|
upstream_error: last_banned_auth_error.unwrap_or_default(),
|
|
status_code: last_banned_status_code.unwrap_or(403),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if cloudflare_challenge_count > 0 => {
|
|
let error = ResponseDispatchError::CloudflareChallenge {
|
|
count: cloudflare_challenge_count,
|
|
upstream_error: last_cloudflare_challenge_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if cloudflare_path_block_count > 0 => {
|
|
let error = ResponseDispatchError::CloudflarePathBlocked {
|
|
count: cloudflare_path_block_count,
|
|
upstream_error: last_cloudflare_path_block_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None if model_unsupported_count > 0 => {
|
|
let error = ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count,
|
|
upstream_error: last_model_unsupported_error.unwrap_or_default(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
None => {
|
|
let error = ResponseDispatchError::NoActiveAccount;
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
};
|
|
last_attempted_account_id = Some(acquired.account.id.clone());
|
|
let acquired = match verify_acquired_quota_if_required(
|
|
self.account_pool.as_ref(),
|
|
self.codex.as_ref(),
|
|
&self.cloudflare,
|
|
self.installation_id.as_deref(),
|
|
request_id,
|
|
acquired,
|
|
&mut excluded_account_ids,
|
|
&mut quota_verify_attempts,
|
|
)
|
|
.await
|
|
{
|
|
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
|
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string());
|
|
continue;
|
|
}
|
|
QuotaVerificationDecision::MaxAttemptsReached => {
|
|
let error = ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count + 1,
|
|
upstream_error: QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string(),
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
};
|
|
let account = acquired.account;
|
|
let release_account_id = account.id.clone();
|
|
let response_result = create_compact_response_with_account_retrying_5xx(
|
|
&self.codex,
|
|
self.installation_id.as_deref(),
|
|
&self.cloudflare,
|
|
&request,
|
|
request_id,
|
|
&account,
|
|
)
|
|
.await;
|
|
self.account_pool.release(&release_account_id).await;
|
|
if let Err(error) = &response_result {
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(
|
|
&release_account_id,
|
|
upstream_error_set_cookie_headers(error),
|
|
)
|
|
.await;
|
|
}
|
|
|
|
match response_result {
|
|
Ok(response) => {
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(
|
|
&release_account_id,
|
|
&response.set_cookie_headers,
|
|
)
|
|
.await;
|
|
self.cloudflare.reset_account_recovery(&account.id).await;
|
|
let usage = extract_usage(&response.body);
|
|
if let Some(usage) = usage {
|
|
self.account_pool
|
|
.record_token_usage(&account.id, usage)
|
|
.await;
|
|
}
|
|
record_response_event(ResponseEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &account.id,
|
|
route: "/v1/responses/compact",
|
|
model: requested_model,
|
|
started_at,
|
|
status_code: 200,
|
|
level: EventLevel::Info,
|
|
message: "v1 responses compact completed",
|
|
metadata: serde_json::json!({
|
|
"stream": false,
|
|
"compact": true,
|
|
"usage": usage,
|
|
}),
|
|
rate_limit_headers: &response.rate_limit_headers,
|
|
})
|
|
.await;
|
|
return Ok(response.body);
|
|
}
|
|
Err(error) if is_rate_limit_upstream_error(&error) => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(upstream_error_body(&error));
|
|
let cooldown_until = rate_limit_cooldown_until(&error, Utc::now());
|
|
self.account_pool
|
|
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(upstream_error_body(&error));
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_auth_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
let account_status = auth_failure_account_status(&error);
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
|
cloudflare_challenge_count += 1;
|
|
last_cloudflare_challenge_error =
|
|
Some(cloudflare_challenge_error_message().to_string());
|
|
self.cloudflare
|
|
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
|
cloudflare_path_block_count += 1;
|
|
last_cloudflare_path_block_error =
|
|
Some(cloudflare_path_block_error_message().to_string());
|
|
self.cloudflare
|
|
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
|
let upstream_error = upstream_error_body(&error);
|
|
if model_unsupported_retry_used {
|
|
let error = ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
};
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
Some(&release_account_id),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_banned_upstream_error(&error) => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error_body(&error));
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::Banned)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) => {
|
|
let error = ResponseDispatchError::Upstream(error);
|
|
self.record_compact_dispatch_error(
|
|
request_id,
|
|
requested_model,
|
|
started_at,
|
|
Some(&release_account_id),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn record_compact_dispatch_error(
|
|
&self,
|
|
request_id: &str,
|
|
requested_model: &str,
|
|
started_at: Instant,
|
|
account_id: Option<&str>,
|
|
error: &ResponseDispatchError,
|
|
) {
|
|
record_response_dispatch_error_event(ResponseDispatchErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id,
|
|
route: "/v1/responses/compact",
|
|
model: requested_model,
|
|
started_at,
|
|
stream: false,
|
|
compact: true,
|
|
transport: Some("http"),
|
|
error,
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// 调度流式 Responses 请求到 Codex Responses 上游。
|
|
pub async fn stream(
|
|
&self,
|
|
request_id: &str,
|
|
route: &str,
|
|
mut request: CodexResponsesRequest,
|
|
requested_model: &str,
|
|
) -> Result<ResponseDispatchStream, ResponseDispatchError> {
|
|
let catalog = self.models.catalog().await;
|
|
let parsed_model = catalog.parse_model_name(requested_model);
|
|
apply_response_model_options(&mut request, &parsed_model, self.models.config());
|
|
request.stream = true;
|
|
let started_at = Instant::now();
|
|
let tuple_schema = request.tuple_schema.clone();
|
|
let now = Utc::now();
|
|
let explicit_previous_response_id = request.previous_response_id.clone();
|
|
let mut implicit_resume = self.prepare_response_session(&mut request).await;
|
|
let preferred_account_id = self.preferred_account_id_for_request(&request, now).await;
|
|
let mut acquire_request = AccountAcquireRequest::new(&request.model, now);
|
|
if let Some(preferred_account_id) = preferred_account_id.as_deref() {
|
|
acquire_request = acquire_request.with_preferred_account_id(preferred_account_id);
|
|
}
|
|
let mut excluded_account_ids = Vec::new();
|
|
let mut rate_limited_count = 0usize;
|
|
let mut last_rate_limit_error = None;
|
|
let mut quota_exhausted_count = 0usize;
|
|
let mut last_quota_error = None;
|
|
let mut expired_count = 0usize;
|
|
let mut last_auth_error = None;
|
|
let mut disabled_count = 0usize;
|
|
let mut last_disabled_auth_error = None;
|
|
let mut banned_count = 0usize;
|
|
let mut last_banned_auth_error = None;
|
|
let mut last_banned_status_code: Option<u16> = None;
|
|
let mut cloudflare_challenge_count = 0usize;
|
|
let mut last_cloudflare_challenge_error = None;
|
|
let mut cloudflare_path_block_count = 0usize;
|
|
let mut last_cloudflare_path_block_error = None;
|
|
let mut model_unsupported_count = 0usize;
|
|
let mut last_model_unsupported_error = None;
|
|
let mut model_unsupported_retry_used = false;
|
|
let mut history_recovery_used = false;
|
|
let mut last_exhausted_account_class = None;
|
|
let mut quota_verify_attempts = 0usize;
|
|
let mut last_attempted_account_id = None::<String>;
|
|
macro_rules! return_stream_dispatch_error {
|
|
($error:expr) => {{
|
|
let error = $error;
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
last_attempted_account_id.as_deref(),
|
|
true,
|
|
false,
|
|
Some(backend_transport_name(requested_response_transport(
|
|
&request,
|
|
))),
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}};
|
|
($error:expr, account_id: $account_id:expr, transport: $transport:expr) => {{
|
|
let error = $error;
|
|
self.record_response_dispatch_error(
|
|
request_id,
|
|
route,
|
|
requested_model,
|
|
started_at,
|
|
$account_id,
|
|
true,
|
|
false,
|
|
$transport,
|
|
&error,
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}};
|
|
}
|
|
loop {
|
|
let mut attempt_acquire_request = acquire_request
|
|
.clone()
|
|
.with_exclude_account_ids(excluded_account_ids.iter().cloned());
|
|
attempt_acquire_request.now = Utc::now();
|
|
let acquired = match self
|
|
.account_pool
|
|
.acquire_with(attempt_acquire_request)
|
|
.await
|
|
{
|
|
Some(acquired) => acquired,
|
|
None => {
|
|
let error = match last_exhausted_account_class {
|
|
Some(ExhaustedAccountClass::QuotaExhausted) => {
|
|
ResponseDispatchError::QuotaExhausted {
|
|
count: quota_exhausted_count,
|
|
upstream_error: last_quota_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::RateLimited) => {
|
|
ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count,
|
|
upstream_error: last_rate_limit_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::Expired) => ResponseDispatchError::Expired {
|
|
count: expired_count,
|
|
upstream_error: last_auth_error.unwrap_or_default(),
|
|
},
|
|
Some(ExhaustedAccountClass::Disabled) => ResponseDispatchError::Disabled {
|
|
count: disabled_count,
|
|
upstream_error: last_disabled_auth_error.unwrap_or_default(),
|
|
},
|
|
Some(ExhaustedAccountClass::Banned) => ResponseDispatchError::Banned {
|
|
count: banned_count,
|
|
upstream_error: last_banned_auth_error.unwrap_or_default(),
|
|
status_code: last_banned_status_code.unwrap_or(403),
|
|
},
|
|
Some(ExhaustedAccountClass::CloudflareChallenge) => {
|
|
ResponseDispatchError::CloudflareChallenge {
|
|
count: cloudflare_challenge_count,
|
|
upstream_error: last_cloudflare_challenge_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::CloudflarePathBlocked) => {
|
|
ResponseDispatchError::CloudflarePathBlocked {
|
|
count: cloudflare_path_block_count,
|
|
upstream_error: last_cloudflare_path_block_error
|
|
.unwrap_or_default(),
|
|
}
|
|
}
|
|
Some(ExhaustedAccountClass::ModelUnsupported) => {
|
|
ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count,
|
|
upstream_error: last_model_unsupported_error.unwrap_or_default(),
|
|
}
|
|
}
|
|
None => ResponseDispatchError::NoActiveAccount,
|
|
};
|
|
return_stream_dispatch_error!(error);
|
|
}
|
|
};
|
|
let acquired_account_id = acquired.account.id.clone();
|
|
let acquired = match verify_acquired_quota_if_required(
|
|
self.account_pool.as_ref(),
|
|
self.codex.as_ref(),
|
|
&self.cloudflare,
|
|
self.installation_id.as_deref(),
|
|
request_id,
|
|
acquired,
|
|
&mut excluded_account_ids,
|
|
&mut quota_verify_attempts,
|
|
)
|
|
.await
|
|
{
|
|
QuotaVerificationDecision::Ready(acquired) => *acquired,
|
|
QuotaVerificationDecision::RetryWithAnotherAccount => {
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string());
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::RateLimited);
|
|
continue;
|
|
}
|
|
QuotaVerificationDecision::MaxAttemptsReached => {
|
|
return_stream_dispatch_error!(
|
|
ResponseDispatchError::RateLimited {
|
|
count: rate_limited_count + 1,
|
|
upstream_error: QUOTA_VERIFY_LIMIT_REACHED_MESSAGE.to_string(),
|
|
},
|
|
account_id: Some(&acquired_account_id),
|
|
transport: Some(backend_transport_name(requested_response_transport(
|
|
&request
|
|
)))
|
|
);
|
|
}
|
|
};
|
|
self.apply_cascading_ban_defense(
|
|
&mut request,
|
|
&mut implicit_resume,
|
|
preferred_account_id.as_deref(),
|
|
&acquired.account.id,
|
|
explicit_previous_response_id.as_deref(),
|
|
)
|
|
.await;
|
|
self.account_pool.wait_for_request_interval(&acquired).await;
|
|
let account = acquired.account;
|
|
let release_account_id = account.id.clone();
|
|
last_attempted_account_id = Some(release_account_id.clone());
|
|
let response_result = create_response_stream_with_account_retrying_5xx(
|
|
&self.codex,
|
|
self.installation_id.as_deref(),
|
|
&self.cloudflare,
|
|
&request,
|
|
request_id,
|
|
&account,
|
|
)
|
|
.await;
|
|
if let Err(error) = &response_result {
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(
|
|
&release_account_id,
|
|
upstream_error_set_cookie_headers(error),
|
|
)
|
|
.await;
|
|
}
|
|
|
|
match response_result {
|
|
Ok(response) => {
|
|
let transport = response.transport;
|
|
let set_cookie_headers = response.set_cookie_headers;
|
|
let rate_limit_headers = response.rate_limit_headers;
|
|
let rate_limit_header_updates = response.rate_limit_header_updates;
|
|
let turn_state_update = response.turn_state_update;
|
|
let turn_state = response.turn_state;
|
|
self.cloudflare
|
|
.capture_set_cookie_headers(&release_account_id, &set_cookie_headers)
|
|
.await;
|
|
let (prefetched, body) = match prefetch_first_sse_chunk(response.body).await {
|
|
Ok(prefetched) => prefetched,
|
|
Err(ResponseDispatchError::Upstream(error))
|
|
if is_history_recovery_upstream_error(&error)
|
|
&& !history_recovery_used =>
|
|
{
|
|
self.account_pool.release(&release_account_id).await;
|
|
if client_error_invalid_reasoning_replay(&error) {
|
|
self.evict_reasoning_replay(&request, &release_account_id)
|
|
.await;
|
|
}
|
|
self.recover_request_history(&mut request, &mut implicit_resume)
|
|
.await;
|
|
history_recovery_used = true;
|
|
continue;
|
|
}
|
|
Err(error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
if let ResponseDispatchError::Upstream(upstream_error) = &error {
|
|
record_response_upstream_error_event(
|
|
ResponseUpstreamErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &release_account_id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
stream: true,
|
|
transport,
|
|
error: upstream_error,
|
|
},
|
|
)
|
|
.await;
|
|
return Err(error);
|
|
}
|
|
return_stream_dispatch_error!(
|
|
error,
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(transport))
|
|
);
|
|
}
|
|
};
|
|
let first_failure = match first_sse_failure(&prefetched) {
|
|
Ok(failure) => failure,
|
|
Err(error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
return_stream_dispatch_error!(
|
|
ResponseDispatchError::InvalidSse(error),
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(transport))
|
|
);
|
|
}
|
|
};
|
|
if let Some(failure) = first_failure {
|
|
if is_history_recovery_sse_failure(&failure) && !history_recovery_used {
|
|
self.account_pool.release(&release_account_id).await;
|
|
if sse_failure_invalid_reasoning_replay(&failure) {
|
|
self.evict_reasoning_replay(&request, &release_account_id)
|
|
.await;
|
|
}
|
|
self.recover_request_history(&mut request, &mut implicit_resume)
|
|
.await;
|
|
history_recovery_used = true;
|
|
continue;
|
|
}
|
|
if is_model_unsupported_sse_failure(&failure) {
|
|
let upstream_error = sse_failure_error_body(&failure);
|
|
if model_unsupported_retry_used {
|
|
self.account_pool.release(&release_account_id).await;
|
|
return_stream_dispatch_error!(
|
|
ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
},
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(transport))
|
|
);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
self.account_pool.release(&account.id).await;
|
|
continue;
|
|
}
|
|
if is_quota_exhausted_sse_failure(&failure) {
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(failure.message.clone());
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::QuotaExhausted);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
self.account_pool.release(&account.id).await;
|
|
continue;
|
|
}
|
|
if is_auth_sse_failure(&failure) {
|
|
let upstream_error = sse_failure_error_body(&failure);
|
|
let account_status = auth_sse_failure_account_status(&failure);
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Disabled);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code =
|
|
Some(stream_failure_http_status(&failure));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Banned);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::Expired);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
self.account_pool.release(&account.id).await;
|
|
continue;
|
|
}
|
|
self.account_pool.release(&release_account_id).await;
|
|
record_prefetched_response_stream_failure_event(
|
|
ResponseStreamFailureEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &release_account_id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
transport,
|
|
request: &request,
|
|
failure: &failure,
|
|
rate_limit_headers: &rate_limit_headers,
|
|
prefetched: &prefetched,
|
|
},
|
|
)
|
|
.await;
|
|
return Err(ResponseDispatchError::Failed(failure.clone()));
|
|
}
|
|
return Ok(spawn_live_response_stream(
|
|
LiveResponseStreamContext {
|
|
account_pool: Arc::clone(&self.account_pool),
|
|
session_affinity: Arc::clone(&self.session_affinity),
|
|
reasoning_replay: Arc::clone(&self.reasoning_replay),
|
|
logs: Arc::clone(&self.logs),
|
|
cloudflare: self.cloudflare.clone(),
|
|
account_id: account.id,
|
|
request_id: request_id.to_string(),
|
|
route: route.to_string(),
|
|
model: requested_model.to_string(),
|
|
request,
|
|
tuple_schema,
|
|
transport,
|
|
rate_limit_headers,
|
|
rate_limit_header_updates,
|
|
turn_state_update,
|
|
turn_state,
|
|
started_at,
|
|
},
|
|
prefetched,
|
|
body,
|
|
));
|
|
}
|
|
Err(error) if is_rate_limit_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
rate_limited_count += 1;
|
|
last_rate_limit_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::RateLimited);
|
|
let cooldown_until = rate_limit_cooldown_until(&error, Utc::now());
|
|
self.account_pool
|
|
.mark_quota_limited_until(&release_account_id, cooldown_until)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_quota_exhausted_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
quota_exhausted_count += 1;
|
|
last_quota_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::QuotaExhausted);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::QuotaExhausted)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error)
|
|
if is_history_recovery_upstream_error(&error) && !history_recovery_used =>
|
|
{
|
|
self.account_pool.release(&release_account_id).await;
|
|
if client_error_invalid_reasoning_replay(&error) {
|
|
self.evict_reasoning_replay(&request, &release_account_id)
|
|
.await;
|
|
}
|
|
self.recover_request_history(&mut request, &mut implicit_resume)
|
|
.await;
|
|
history_recovery_used = true;
|
|
}
|
|
Err(error) if is_auth_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
let upstream_error = upstream_error_body(&error);
|
|
let account_status = auth_failure_account_status(&error);
|
|
match account_status {
|
|
AccountStatus::Disabled => {
|
|
disabled_count += 1;
|
|
last_disabled_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Disabled);
|
|
}
|
|
AccountStatus::Banned => {
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Banned);
|
|
}
|
|
_ => {
|
|
expired_count += 1;
|
|
last_auth_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Expired);
|
|
}
|
|
}
|
|
self.account_pool
|
|
.set_status(&release_account_id, account_status)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_challenge_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
cloudflare_challenge_count += 1;
|
|
last_cloudflare_challenge_error =
|
|
Some(cloudflare_challenge_error_message().to_string());
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::CloudflareChallenge);
|
|
self.cloudflare
|
|
.apply_challenge(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_cloudflare_path_block_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
cloudflare_path_block_count += 1;
|
|
last_cloudflare_path_block_error =
|
|
Some(cloudflare_path_block_error_message().to_string());
|
|
last_exhausted_account_class =
|
|
Some(ExhaustedAccountClass::CloudflarePathBlocked);
|
|
self.cloudflare
|
|
.apply_path_block(self.account_pool.as_ref(), &release_account_id)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_model_unsupported_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
let upstream_error = upstream_error_body(&error);
|
|
if model_unsupported_retry_used {
|
|
return_stream_dispatch_error!(
|
|
ResponseDispatchError::ModelUnsupported {
|
|
count: model_unsupported_count + 1,
|
|
upstream_error,
|
|
},
|
|
account_id: Some(&release_account_id),
|
|
transport: Some(backend_transport_name(requested_response_transport(
|
|
&request
|
|
)))
|
|
);
|
|
}
|
|
model_unsupported_count += 1;
|
|
last_model_unsupported_error = Some(upstream_error);
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::ModelUnsupported);
|
|
model_unsupported_retry_used = true;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) if is_banned_upstream_error(&error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
banned_count += 1;
|
|
last_banned_status_code = Some(upstream_error_http_status(&error));
|
|
last_banned_auth_error = Some(upstream_error_body(&error));
|
|
last_exhausted_account_class = Some(ExhaustedAccountClass::Banned);
|
|
self.account_pool
|
|
.set_status(&release_account_id, AccountStatus::Banned)
|
|
.await;
|
|
excluded_account_ids.push(release_account_id);
|
|
}
|
|
Err(error) => {
|
|
self.account_pool.release(&release_account_id).await;
|
|
record_response_upstream_error_event(ResponseUpstreamErrorEventRecord {
|
|
logs: &self.logs,
|
|
request_id,
|
|
account_id: &release_account_id,
|
|
route,
|
|
model: requested_model,
|
|
started_at,
|
|
stream: true,
|
|
transport: requested_response_transport(&request),
|
|
error: &error,
|
|
})
|
|
.await;
|
|
return Err(ResponseDispatchError::Upstream(error));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
enum QuotaVerificationDecision {
|
|
Ready(Box<AcquiredAccount>),
|
|
RetryWithAnotherAccount,
|
|
MaxAttemptsReached,
|
|
}
|
|
|
|
#[expect(
|
|
clippy::too_many_arguments,
|
|
reason = "quota verification needs shared services plus mutable retry state at dispatch loop boundaries"
|
|
)]
|
|
async fn verify_acquired_quota_if_required(
|
|
account_pool: &RuntimeAccountPoolService,
|
|
codex: &CodexBackendClient,
|
|
cloudflare: &CloudflareRecovery,
|
|
installation_id: Option<&str>,
|
|
request_id: &str,
|
|
acquired: AcquiredAccount,
|
|
excluded_account_ids: &mut Vec<String>,
|
|
verify_attempts: &mut usize,
|
|
) -> QuotaVerificationDecision {
|
|
if !acquired.account.quota_verify_required {
|
|
return QuotaVerificationDecision::Ready(Box::new(acquired));
|
|
}
|
|
|
|
let account_id = acquired.account.id.clone();
|
|
let cookie_header = cloudflare
|
|
.cookie_header_for_request(&account_id, "/codex/usage")
|
|
.await;
|
|
let usage = codex
|
|
.fetch_usage(CodexRequestContext {
|
|
access_token: &acquired.account.access_token,
|
|
account_id: acquired.account.account_id.as_deref(),
|
|
request_id,
|
|
turn_state: None,
|
|
turn_metadata: None,
|
|
beta_features: None,
|
|
include_timing_metrics: None,
|
|
version: None,
|
|
codex_window_id: None,
|
|
parent_thread_id: None,
|
|
cookie_header: cookie_header.as_deref(),
|
|
installation_id,
|
|
session_id: None,
|
|
})
|
|
.await;
|
|
|
|
let raw = match usage {
|
|
Ok(raw) => raw,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id = %account_id,
|
|
error = %error,
|
|
"failed to verify dirty quota before upstream request"
|
|
);
|
|
return QuotaVerificationDecision::Ready(Box::new(acquired));
|
|
}
|
|
};
|
|
|
|
let quota = quota_from_usage(&raw);
|
|
account_pool.apply_quota_snapshot(&account_id, "a).await;
|
|
if quota_snapshot_limit_reached("a) {
|
|
account_pool.release(&account_id).await;
|
|
excluded_account_ids.push(account_id);
|
|
*verify_attempts += 1;
|
|
if *verify_attempts >= MAX_QUOTA_VERIFY_ATTEMPTS {
|
|
return QuotaVerificationDecision::MaxAttemptsReached;
|
|
}
|
|
return QuotaVerificationDecision::RetryWithAnotherAccount;
|
|
}
|
|
|
|
QuotaVerificationDecision::Ready(Box::new(acquired_with_verified_quota(acquired, "a)))
|
|
}
|
|
|
|
fn acquired_with_verified_quota(mut acquired: AcquiredAccount, quota: &Value) -> AcquiredAccount {
|
|
let limit_reached = quota_snapshot_limit_reached(quota);
|
|
acquired.account.quota_verify_required = false;
|
|
acquired.account.quota_limit_reached = limit_reached;
|
|
acquired.account.quota_cooldown_until = limit_reached
|
|
.then_some(quota_snapshot_reset_at(quota))
|
|
.flatten();
|
|
if let Some(reset_at) = quota_snapshot_reset_at(quota) {
|
|
acquired.account.window_reset_at = Some(reset_at);
|
|
if let Some(limit_window_seconds) = quota_snapshot_limit_window_seconds(quota) {
|
|
acquired.account.limit_window_seconds = Some(limit_window_seconds);
|
|
}
|
|
}
|
|
acquired
|
|
}
|
|
|
|
async fn create_response_with_account(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexResponsesRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexBackendResponse, CodexClientError> {
|
|
let cookie_header = cloudflare
|
|
.cookie_header_for_request(&account.id, "/codex/responses")
|
|
.await;
|
|
let identity = build_conversation_identity(
|
|
request.prompt_cache_key.as_deref(),
|
|
request.codex_window_id.as_deref(),
|
|
&account.id,
|
|
);
|
|
codex
|
|
.create_response(
|
|
request,
|
|
CodexRequestContext {
|
|
access_token: &account.access_token,
|
|
account_id: account.account_id.as_deref(),
|
|
request_id,
|
|
turn_state: request.turn_state.as_deref(),
|
|
turn_metadata: request.turn_metadata.as_deref(),
|
|
beta_features: request.beta_features.as_deref(),
|
|
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
|
version: request.version.as_deref(),
|
|
codex_window_id: identity.window_id.as_deref(),
|
|
parent_thread_id: request.parent_thread_id.as_deref(),
|
|
cookie_header: cookie_header.as_deref(),
|
|
installation_id,
|
|
session_id: identity.conversation_id.as_deref(),
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn create_response_stream_with_account(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexResponsesRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexBackendStreamingResponse, CodexClientError> {
|
|
let cookie_header = cloudflare
|
|
.cookie_header_for_request(&account.id, "/codex/responses")
|
|
.await;
|
|
let identity = build_conversation_identity(
|
|
request.prompt_cache_key.as_deref(),
|
|
request.codex_window_id.as_deref(),
|
|
&account.id,
|
|
);
|
|
codex
|
|
.create_response_stream(
|
|
request,
|
|
CodexRequestContext {
|
|
access_token: &account.access_token,
|
|
account_id: account.account_id.as_deref(),
|
|
request_id,
|
|
turn_state: request.turn_state.as_deref(),
|
|
turn_metadata: request.turn_metadata.as_deref(),
|
|
beta_features: request.beta_features.as_deref(),
|
|
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
|
version: request.version.as_deref(),
|
|
codex_window_id: identity.window_id.as_deref(),
|
|
parent_thread_id: request.parent_thread_id.as_deref(),
|
|
cookie_header: cookie_header.as_deref(),
|
|
installation_id,
|
|
session_id: identity.conversation_id.as_deref(),
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn create_compact_response_with_account(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexCompactRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexCompactResponse, CodexClientError> {
|
|
let cookie_header = cloudflare
|
|
.cookie_header_for_request(&account.id, "/codex/responses/compact")
|
|
.await;
|
|
codex
|
|
.create_compact_response(
|
|
request,
|
|
CodexRequestContext {
|
|
access_token: &account.access_token,
|
|
account_id: account.account_id.as_deref(),
|
|
request_id,
|
|
turn_state: None,
|
|
turn_metadata: None,
|
|
beta_features: None,
|
|
include_timing_metrics: None,
|
|
version: None,
|
|
codex_window_id: None,
|
|
parent_thread_id: None,
|
|
cookie_header: cookie_header.as_deref(),
|
|
installation_id,
|
|
session_id: None,
|
|
},
|
|
)
|
|
.await
|
|
}
|
|
|
|
const MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT: usize = 2;
|
|
|
|
async fn create_response_with_account_retrying_5xx(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexResponsesRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexBackendResponse, CodexClientError> {
|
|
let mut retries = 0;
|
|
loop {
|
|
let result = create_response_with_account(
|
|
codex,
|
|
installation_id,
|
|
cloudflare,
|
|
request,
|
|
request_id,
|
|
account,
|
|
)
|
|
.await;
|
|
match result {
|
|
Err(error)
|
|
if is_retryable_upstream_5xx_error(&error)
|
|
&& retries < MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT =>
|
|
{
|
|
retries += 1;
|
|
}
|
|
result => return result,
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn create_response_stream_with_account_retrying_5xx(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexResponsesRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexBackendStreamingResponse, CodexClientError> {
|
|
let mut retries = 0;
|
|
loop {
|
|
let result = create_response_stream_with_account(
|
|
codex,
|
|
installation_id,
|
|
cloudflare,
|
|
request,
|
|
request_id,
|
|
account,
|
|
)
|
|
.await;
|
|
match result {
|
|
Err(error)
|
|
if is_retryable_upstream_5xx_error(&error)
|
|
&& retries < MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT =>
|
|
{
|
|
retries += 1;
|
|
}
|
|
result => return result,
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn create_compact_response_with_account_retrying_5xx(
|
|
codex: &CodexBackendClient,
|
|
installation_id: Option<&str>,
|
|
cloudflare: &CloudflareRecovery,
|
|
request: &CodexCompactRequest,
|
|
request_id: &str,
|
|
account: &Account,
|
|
) -> Result<CodexCompactResponse, CodexClientError> {
|
|
let mut retries = 0;
|
|
loop {
|
|
let result = create_compact_response_with_account(
|
|
codex,
|
|
installation_id,
|
|
cloudflare,
|
|
request,
|
|
request_id,
|
|
account,
|
|
)
|
|
.await;
|
|
match result {
|
|
Err(error)
|
|
if is_retryable_upstream_5xx_error(&error)
|
|
&& retries < MAX_UPSTREAM_5XX_RETRIES_PER_ACCOUNT =>
|
|
{
|
|
retries += 1;
|
|
}
|
|
result => return result,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, .. } if status_code_is_rate_limited(status.as_u16())
|
|
)
|
|
}
|
|
|
|
fn is_retryable_upstream_5xx_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, body, .. }
|
|
if status_code_is_transient_upstream(status.as_u16())
|
|
&& status_code_allows_same_account_retry(status.as_u16())
|
|
&& !is_history_recovery_signal(body)
|
|
)
|
|
}
|
|
|
|
fn is_quota_exhausted_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, .. } if status_code_is_quota_exhausted(status.as_u16())
|
|
)
|
|
}
|
|
|
|
fn is_auth_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, .. } if status.as_u16() == 401
|
|
)
|
|
}
|
|
|
|
fn is_cloudflare_challenge_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, body, .. }
|
|
if status.as_u16() == 403 && is_cloudflare_challenge_signal(body)
|
|
)
|
|
}
|
|
|
|
fn is_cloudflare_path_block_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, body, .. }
|
|
if status.as_u16() == 404 && body.trim().is_empty()
|
|
)
|
|
}
|
|
|
|
fn is_model_unsupported_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, body, .. }
|
|
if status.is_client_error()
|
|
&& !matches!(status.as_u16(), 401 | 402 | 403 | 404 | 429)
|
|
&& is_model_unsupported_signal(body)
|
|
)
|
|
}
|
|
|
|
fn is_history_recovery_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { body, .. } if is_history_recovery_signal(body)
|
|
)
|
|
}
|
|
|
|
fn is_banned_upstream_error(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { status, body, .. }
|
|
if status.as_u16() == 403 && is_banned_auth_signal(body)
|
|
)
|
|
}
|
|
|
|
fn auth_failure_account_status(error: &CodexClientError) -> AccountStatus {
|
|
match error {
|
|
CodexClientError::Upstream { body, .. } if is_banned_auth_signal(body) => {
|
|
AccountStatus::Banned
|
|
}
|
|
_ => AccountStatus::Expired,
|
|
}
|
|
}
|
|
|
|
fn upstream_error_body(error: &CodexClientError) -> String {
|
|
match error {
|
|
CodexClientError::Upstream { body, .. } => body.clone(),
|
|
error => error.to_string(),
|
|
}
|
|
}
|
|
|
|
fn upstream_error_set_cookie_headers(error: &CodexClientError) -> &[String] {
|
|
match error {
|
|
CodexClientError::Upstream {
|
|
set_cookie_headers, ..
|
|
} => set_cookie_headers,
|
|
_ => &[],
|
|
}
|
|
}
|
|
|
|
fn sse_failure_error_body(failure: &ResponsesSseFailure) -> String {
|
|
match failure.upstream_code.as_deref() {
|
|
Some(code) => serde_json::json!({
|
|
"error": {
|
|
"code": code,
|
|
"message": failure.message.as_str(),
|
|
}
|
|
})
|
|
.to_string(),
|
|
None => failure.message.clone(),
|
|
}
|
|
}
|
|
|
|
fn is_quota_exhausted_sse_failure(failure: &ResponsesSseFailure) -> bool {
|
|
failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.is_some_and(|code| matches!(code, "quota_exceeded" | "insufficient_quota"))
|
|
|| failure.message.to_ascii_lowercase().contains("quota")
|
|
}
|
|
|
|
fn is_auth_sse_failure(failure: &ResponsesSseFailure) -> bool {
|
|
failure.upstream_code.as_deref().is_some_and(|code| {
|
|
let code = code.to_ascii_lowercase();
|
|
matches!(
|
|
code.as_str(),
|
|
"token_invalid"
|
|
| "token_expired"
|
|
| "token_revoked"
|
|
| "account_deactivated"
|
|
| "unauthorized"
|
|
| "invalid_api_key"
|
|
)
|
|
}) || {
|
|
let message = failure.message.to_ascii_lowercase();
|
|
message.contains("token revoked")
|
|
|| message.contains("token invalid")
|
|
|| message.contains("token expired")
|
|
}
|
|
}
|
|
|
|
fn is_model_unsupported_sse_failure(failure: &ResponsesSseFailure) -> bool {
|
|
failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.is_some_and(is_model_unsupported_signal)
|
|
|| is_model_unsupported_signal(&failure.message)
|
|
}
|
|
|
|
fn is_history_recovery_sse_failure(failure: &ResponsesSseFailure) -> bool {
|
|
failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.is_some_and(is_history_recovery_signal)
|
|
|| is_history_recovery_signal(&failure.message)
|
|
}
|
|
|
|
fn sse_failure_invalid_reasoning_replay(failure: &ResponsesSseFailure) -> bool {
|
|
failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.is_some_and(is_invalid_encrypted_content_signal)
|
|
|| is_invalid_encrypted_content_signal(&failure.message)
|
|
}
|
|
|
|
fn client_error_invalid_reasoning_replay(error: &CodexClientError) -> bool {
|
|
matches!(
|
|
error,
|
|
CodexClientError::Upstream { body, .. } if is_invalid_encrypted_content_signal(body)
|
|
)
|
|
}
|
|
|
|
fn auth_sse_failure_account_status(failure: &ResponsesSseFailure) -> AccountStatus {
|
|
if failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.is_some_and(is_banned_auth_signal)
|
|
|| is_banned_auth_signal(&failure.message)
|
|
{
|
|
AccountStatus::Banned
|
|
} else {
|
|
AccountStatus::Expired
|
|
}
|
|
}
|
|
|
|
fn is_banned_auth_signal(value: &str) -> bool {
|
|
let value = value.to_ascii_lowercase();
|
|
value.contains("account_deactivated")
|
|
|| value.contains("account deactivated")
|
|
|| value.contains("account has been deactivated")
|
|
|| value.contains("deactivated")
|
|
|| value.contains("banned")
|
|
}
|
|
|
|
fn is_cloudflare_challenge_signal(value: &str) -> bool {
|
|
let value = value.to_ascii_lowercase();
|
|
value.contains("cf-mitigated")
|
|
|| value.contains("cf-chl-bypass")
|
|
|| value.contains("_cf_chl")
|
|
|| value.contains("cf_chl")
|
|
|| value.contains("attention required")
|
|
|| value.contains("just a moment")
|
|
}
|
|
|
|
fn is_model_unsupported_signal(value: &str) -> bool {
|
|
let value = value.to_ascii_lowercase();
|
|
value.contains("model_not_supported")
|
|
|| value.contains("model_not_available")
|
|
|| (value.contains("model")
|
|
&& (value.contains("not supported")
|
|
|| value.contains("not available")
|
|
|| value.contains("not_supported")
|
|
|| value.contains("not_available")))
|
|
}
|
|
|
|
fn is_history_recovery_signal(value: &str) -> bool {
|
|
let value = value.to_ascii_lowercase();
|
|
value.contains("previous_response_not_found")
|
|
|| (value.contains("previous response") && value.contains("not found"))
|
|
|| value.contains("no tool output found for function call")
|
|
|| is_invalid_encrypted_content_signal(&value)
|
|
}
|
|
|
|
fn is_invalid_encrypted_content_signal(value: &str) -> bool {
|
|
let value = value.to_ascii_lowercase();
|
|
value.contains("invalid_encrypted_content")
|
|
|| (value.contains("invalid") && value.contains("encrypted") && value.contains("content"))
|
|
}
|
|
|
|
fn strip_request_history(request: &mut CodexResponsesRequest) {
|
|
request.previous_response_id = None;
|
|
request.turn_state = None;
|
|
}
|
|
|
|
fn cloudflare_challenge_error_message() -> &'static str {
|
|
"Upstream blocked the request (Cloudflare challenge)"
|
|
}
|
|
|
|
fn cloudflare_path_block_error_message() -> &'static str {
|
|
"Upstream blocked the request (Cloudflare path-block)"
|
|
}
|
|
|
|
fn rate_limit_cooldown_until(error: &CodexClientError, now: DateTime<Utc>) -> DateTime<Utc> {
|
|
let retry_after_seconds = match error {
|
|
CodexClientError::Upstream {
|
|
retry_after_seconds,
|
|
..
|
|
} => retry_after_seconds.unwrap_or(60),
|
|
_ => 60,
|
|
};
|
|
now + Duration::seconds(retry_after_seconds.min(i64::MAX as u64) as i64)
|
|
}
|
|
|
|
const MAX_STREAM_PREFETCH_BYTES: usize = 64 * 1024;
|
|
const DONE_SSE_FRAME_TEXT: &str = "data: [DONE]\n\n";
|
|
const DONE_SSE_FRAME: &[u8] = b"data: [DONE]\n\n";
|
|
|
|
async fn prefetch_first_sse_chunk(
|
|
mut body: CodexBackendSseStream,
|
|
) -> Result<(Bytes, CodexBackendSseStream), ResponseDispatchError> {
|
|
let mut prefetched = Vec::new();
|
|
while !contains_sse_event_separator(&prefetched) {
|
|
let Some(next) = body.next().await else {
|
|
if prefetched.is_empty() {
|
|
return Err(ResponseDispatchError::EmptyUpstreamResponse);
|
|
}
|
|
return Err(ResponseDispatchError::MissingCompleted);
|
|
};
|
|
let chunk = next.map_err(ResponseDispatchError::Upstream)?;
|
|
prefetched.extend_from_slice(&chunk);
|
|
if prefetched.len() > MAX_STREAM_PREFETCH_BYTES {
|
|
return Err(ResponseDispatchError::InvalidSse(
|
|
SseError::BufferExceeded {
|
|
max_bytes: MAX_STREAM_PREFETCH_BYTES,
|
|
},
|
|
));
|
|
}
|
|
}
|
|
|
|
Ok((Bytes::from(prefetched), body))
|
|
}
|
|
|
|
fn contains_sse_event_separator(bytes: &[u8]) -> bool {
|
|
bytes.windows(2).any(|window| window == b"\n\n")
|
|
|| bytes.windows(4).any(|window| window == b"\r\n\r\n")
|
|
}
|
|
|
|
fn first_sse_failure(prefetched: &[u8]) -> Result<Option<ResponsesSseFailure>, SseError> {
|
|
let body = String::from_utf8_lossy(prefetched);
|
|
match response_from_codex_sse(&body, None)? {
|
|
CollectedResponse::Failed(failure) => Ok(Some(failure)),
|
|
CollectedResponse::Completed(_)
|
|
| CollectedResponse::MissingCompleted
|
|
| CollectedResponse::Empty => Ok(None),
|
|
}
|
|
}
|
|
|
|
struct LiveResponseStreamContext {
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
session_affinity: Arc<RuntimeSessionAffinityService>,
|
|
reasoning_replay: Arc<tokio::sync::Mutex<ReasoningReplayCache>>,
|
|
logs: Arc<AdminLogService>,
|
|
cloudflare: CloudflareRecovery,
|
|
account_id: String,
|
|
request_id: String,
|
|
route: String,
|
|
model: String,
|
|
request: CodexResponsesRequest,
|
|
tuple_schema: Option<Value>,
|
|
transport: CodexBackendTransport,
|
|
rate_limit_headers: Vec<(String, String)>,
|
|
rate_limit_header_updates: Option<CodexRateLimitHeaderUpdates>,
|
|
turn_state_update: Option<CodexTurnStateUpdate>,
|
|
turn_state: Option<String>,
|
|
started_at: Instant,
|
|
}
|
|
|
|
fn spawn_live_response_stream(
|
|
context: LiveResponseStreamContext,
|
|
prefetched: Bytes,
|
|
mut body: CodexBackendSseStream,
|
|
) -> ResponseDispatchStream {
|
|
let (sender, receiver) = mpsc::channel(8);
|
|
let (cancel_sender, mut cancel_receiver) = oneshot::channel();
|
|
tokio::spawn(async move {
|
|
let mut tuple_transformer = context
|
|
.tuple_schema
|
|
.clone()
|
|
.map(TupleSseEventTransformer::new);
|
|
let mut body_bytes = Vec::new();
|
|
if !send_live_response_stream_chunk(
|
|
&sender,
|
|
&mut body_bytes,
|
|
tuple_transformer.as_mut(),
|
|
prefetched,
|
|
)
|
|
.await
|
|
{
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
}
|
|
|
|
loop {
|
|
let next = tokio::select! {
|
|
_ = &mut cancel_receiver => {
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
}
|
|
next = body.next() => next,
|
|
};
|
|
let Some(next) = next else {
|
|
break;
|
|
};
|
|
match next {
|
|
Ok(chunk) => {
|
|
if !send_live_response_stream_chunk(
|
|
&sender,
|
|
&mut body_bytes,
|
|
tuple_transformer.as_mut(),
|
|
chunk,
|
|
)
|
|
.await
|
|
{
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
}
|
|
}
|
|
Err(error) => {
|
|
if !flush_live_response_stream_transformer(
|
|
&sender,
|
|
&mut body_bytes,
|
|
tuple_transformer.as_mut(),
|
|
)
|
|
.await
|
|
{
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
}
|
|
let detail = error.to_string();
|
|
let Some(body_text) =
|
|
send_live_response_stream_tail(&sender, &mut body_bytes, Some(&detail))
|
|
.await
|
|
else {
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
};
|
|
finalize_live_response_stream(context, body_text).await;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
if !flush_live_response_stream_transformer(
|
|
&sender,
|
|
&mut body_bytes,
|
|
tuple_transformer.as_mut(),
|
|
)
|
|
.await
|
|
{
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
}
|
|
let Some(body_text) = send_live_response_stream_tail(&sender, &mut body_bytes, None).await
|
|
else {
|
|
context.account_pool.release(&context.account_id).await;
|
|
return;
|
|
};
|
|
|
|
finalize_live_response_stream(context, body_text).await;
|
|
});
|
|
|
|
ResponseDispatchStream {
|
|
body: Box::pin(MpscResponseBodyStream {
|
|
receiver,
|
|
cancel: Some(cancel_sender),
|
|
}),
|
|
}
|
|
}
|
|
|
|
async fn send_live_response_stream_chunk(
|
|
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
|
|
body_bytes: &mut Vec<u8>,
|
|
transformer: Option<&mut TupleSseEventTransformer>,
|
|
chunk: Bytes,
|
|
) -> bool {
|
|
let chunks = match transformer {
|
|
Some(transformer) => transformer.push(&chunk),
|
|
None => vec![chunk],
|
|
};
|
|
send_live_response_stream_chunks(sender, body_bytes, chunks).await
|
|
}
|
|
|
|
async fn flush_live_response_stream_transformer(
|
|
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
|
|
body_bytes: &mut Vec<u8>,
|
|
transformer: Option<&mut TupleSseEventTransformer>,
|
|
) -> bool {
|
|
let Some(transformer) = transformer else {
|
|
return true;
|
|
};
|
|
send_live_response_stream_chunks(sender, body_bytes, transformer.finish()).await
|
|
}
|
|
|
|
async fn send_live_response_stream_chunks(
|
|
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
|
|
body_bytes: &mut Vec<u8>,
|
|
chunks: Vec<Bytes>,
|
|
) -> bool {
|
|
for chunk in chunks {
|
|
body_bytes.extend_from_slice(&chunk);
|
|
if sender.send(Ok(chunk)).await.is_err() {
|
|
return false;
|
|
}
|
|
}
|
|
true
|
|
}
|
|
|
|
struct TupleSseEventTransformer {
|
|
tuple_schema: Value,
|
|
pending: Vec<u8>,
|
|
}
|
|
|
|
impl TupleSseEventTransformer {
|
|
fn new(tuple_schema: Value) -> Self {
|
|
Self {
|
|
tuple_schema,
|
|
pending: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn push(&mut self, chunk: &[u8]) -> Vec<Bytes> {
|
|
self.pending.extend_from_slice(chunk);
|
|
let mut chunks = Vec::new();
|
|
while let Some(frame_end) = next_sse_frame_end(&self.pending) {
|
|
let frame = self.pending.drain(..frame_end).collect::<Vec<_>>();
|
|
chunks.push(self.transform_frame(&frame));
|
|
}
|
|
chunks
|
|
}
|
|
|
|
fn finish(&mut self) -> Vec<Bytes> {
|
|
if self.pending.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
let frame = std::mem::take(&mut self.pending);
|
|
vec![self.transform_frame(&frame)]
|
|
}
|
|
|
|
fn transform_frame(&self, frame: &[u8]) -> Bytes {
|
|
let frame_text = String::from_utf8_lossy(frame);
|
|
let Ok(events) = parse_sse_events(&frame_text) else {
|
|
return Bytes::copy_from_slice(frame);
|
|
};
|
|
let [event] = events.as_slice() else {
|
|
return Bytes::copy_from_slice(frame);
|
|
};
|
|
let Ok(data) = serde_json::from_str::<Value>(&event.data) else {
|
|
return Bytes::copy_from_slice(frame);
|
|
};
|
|
let transformed = reconvert_responses_sse_event_tuple_values(
|
|
event.event.as_deref(),
|
|
data,
|
|
&self.tuple_schema,
|
|
);
|
|
Bytes::from(encode_sse_event(
|
|
event.event.as_deref().unwrap_or_default(),
|
|
&transformed.to_string(),
|
|
))
|
|
}
|
|
}
|
|
|
|
fn next_sse_frame_end(bytes: &[u8]) -> Option<usize> {
|
|
let lf_lf = bytes
|
|
.windows(2)
|
|
.position(|window| window == b"\n\n")
|
|
.map(|position| position + 2);
|
|
let crlf_crlf = bytes
|
|
.windows(4)
|
|
.position(|window| window == b"\r\n\r\n")
|
|
.map(|position| position + 4);
|
|
match (lf_lf, crlf_crlf) {
|
|
(Some(left), Some(right)) => Some(left.min(right)),
|
|
(Some(end), None) | (None, Some(end)) => Some(end),
|
|
(None, None) => None,
|
|
}
|
|
}
|
|
|
|
async fn send_live_response_stream_tail(
|
|
sender: &mpsc::Sender<Result<Bytes, ResponseDispatchStreamError>>,
|
|
body_bytes: &mut Vec<u8>,
|
|
failure_detail: Option<&str>,
|
|
) -> Option<String> {
|
|
let mut body_text = String::from_utf8_lossy(body_bytes).to_string();
|
|
if !sse_body_has_terminal_event(&body_text) {
|
|
if let Some(separator) = missing_sse_event_separator(&body_text) {
|
|
body_text.push_str(separator);
|
|
body_bytes.extend_from_slice(separator.as_bytes());
|
|
if sender
|
|
.send(Ok(Bytes::copy_from_slice(separator.as_bytes())))
|
|
.await
|
|
.is_err()
|
|
{
|
|
return None;
|
|
}
|
|
}
|
|
let failure = premature_close_failed_event(failure_detail);
|
|
body_text.push_str(&failure);
|
|
body_bytes.extend_from_slice(failure.as_bytes());
|
|
if sender.send(Ok(Bytes::from(failure))).await.is_err() {
|
|
return None;
|
|
}
|
|
}
|
|
|
|
if !sse_body_has_done(&body_text) {
|
|
body_text.push_str(DONE_SSE_FRAME_TEXT);
|
|
body_bytes.extend_from_slice(DONE_SSE_FRAME);
|
|
if sender
|
|
.send(Ok(Bytes::from_static(DONE_SSE_FRAME)))
|
|
.await
|
|
.is_err()
|
|
{
|
|
return None;
|
|
}
|
|
}
|
|
|
|
Some(body_text)
|
|
}
|
|
|
|
fn sse_body_has_terminal_event(body: &str) -> bool {
|
|
parse_sse_events(body).is_ok_and(|events| {
|
|
events.iter().any(|event| {
|
|
matches!(
|
|
event.event.as_deref(),
|
|
Some("response.completed" | "response.failed" | "error")
|
|
)
|
|
})
|
|
})
|
|
}
|
|
|
|
fn missing_sse_event_separator(body: &str) -> Option<&'static str> {
|
|
if body.is_empty()
|
|
|| body.ends_with("\n\n")
|
|
|| body.ends_with("\r\n\r\n")
|
|
|| body.ends_with("\r\r")
|
|
{
|
|
None
|
|
} else if body.ends_with('\n') || body.ends_with('\r') {
|
|
Some("\n")
|
|
} else {
|
|
Some("\n\n")
|
|
}
|
|
}
|
|
|
|
fn premature_close_failed_event(detail: Option<&str>) -> String {
|
|
let message = match detail.filter(|value| !value.trim().is_empty()) {
|
|
Some(detail) => format!("Upstream stream closed before response.completed: {detail}"),
|
|
None => "Upstream stream closed before response.completed".to_string(),
|
|
};
|
|
response_stream_failed_sse_event("server_error", "stream_disconnected", &message)
|
|
}
|
|
|
|
fn response_stream_failed_sse_event(error_type: &str, code: &str, message: &str) -> String {
|
|
let error = serde_json::json!({
|
|
"type": error_type,
|
|
"code": code,
|
|
"message": message,
|
|
});
|
|
let data = serde_json::json!({
|
|
"type": "response.failed",
|
|
"response": {
|
|
"id": format!("resp_proxy_{}", uuid::Uuid::new_v4().simple()),
|
|
"status": "failed",
|
|
"error": error,
|
|
},
|
|
"error": error,
|
|
});
|
|
encode_sse_event("response.failed", &data.to_string())
|
|
}
|
|
|
|
fn sse_body_has_done(body: &str) -> bool {
|
|
body.trim_end_matches(['\r', '\n'])
|
|
.ends_with("data: [DONE]")
|
|
}
|
|
|
|
async fn finalize_live_response_stream(context: LiveResponseStreamContext, body: String) {
|
|
let rate_limit_headers = live_response_rate_limit_headers(&context).await;
|
|
let turn_state = live_response_turn_state(&context).await;
|
|
let usage = match extract_sse_usage(&body) {
|
|
Ok(Some(usage)) => {
|
|
context
|
|
.account_pool
|
|
.record_token_usage(&context.account_id, usage)
|
|
.await;
|
|
Some(usage)
|
|
}
|
|
Ok(None) => None,
|
|
Err(error) => {
|
|
tracing::warn!(account_id = %context.account_id, error = %error, "failed to extract streaming token usage");
|
|
None
|
|
}
|
|
};
|
|
|
|
match response_from_codex_sse(&body, context.tuple_schema.as_ref()) {
|
|
Ok(CollectedResponse::Completed(completed)) => {
|
|
context
|
|
.cloudflare
|
|
.reset_account_recovery(&context.account_id)
|
|
.await;
|
|
let response_id = completed.get("id").and_then(Value::as_str);
|
|
record_response_affinity(
|
|
&context.session_affinity,
|
|
&context.reasoning_replay,
|
|
&context.request,
|
|
&context.account_id,
|
|
&body,
|
|
turn_state,
|
|
usage,
|
|
)
|
|
.await;
|
|
record_live_response_stream_event(
|
|
&context,
|
|
200,
|
|
EventLevel::Info,
|
|
"v1 responses stream completed",
|
|
serde_json::json!({
|
|
"stream": true,
|
|
"completed": true,
|
|
"responseId": response_id,
|
|
"usage": usage,
|
|
}),
|
|
&rate_limit_headers,
|
|
&body,
|
|
)
|
|
.await;
|
|
}
|
|
Ok(CollectedResponse::Failed(failure)) => {
|
|
if sse_failure_invalid_reasoning_replay(&failure) {
|
|
evict_reasoning_replay(
|
|
&context.reasoning_replay,
|
|
&context.request,
|
|
&context.account_id,
|
|
)
|
|
.await;
|
|
}
|
|
tracing::warn!(
|
|
account_id = %context.account_id,
|
|
event = %failure.event,
|
|
code = ?failure.upstream_code.as_deref(),
|
|
"live upstream stream ended with response.failed"
|
|
);
|
|
record_live_response_stream_event(
|
|
&context,
|
|
status_code_for_stream_failure(&failure),
|
|
EventLevel::Error,
|
|
"v1 responses stream failed",
|
|
stream_failure_metadata(&failure, usage),
|
|
&rate_limit_headers,
|
|
&body,
|
|
)
|
|
.await;
|
|
}
|
|
Ok(CollectedResponse::MissingCompleted | CollectedResponse::Empty) => {
|
|
tracing::warn!(
|
|
account_id = %context.account_id,
|
|
"live upstream stream ended without response.completed"
|
|
);
|
|
record_live_response_stream_event(
|
|
&context,
|
|
502,
|
|
EventLevel::Error,
|
|
"v1 responses stream ended without response.completed",
|
|
serde_json::json!({
|
|
"stream": true,
|
|
"failed": true,
|
|
"upstreamCode": "missing_completed",
|
|
"usage": usage,
|
|
}),
|
|
&rate_limit_headers,
|
|
&body,
|
|
)
|
|
.await;
|
|
}
|
|
Err(error) => {
|
|
tracing::warn!(account_id = %context.account_id, error = %error, "failed to parse completed live stream");
|
|
record_live_response_stream_event(
|
|
&context,
|
|
502,
|
|
EventLevel::Warn,
|
|
"v1 responses stream SSE response invalid",
|
|
serde_json::json!({
|
|
"stream": true,
|
|
"sseParseError": error.to_string(),
|
|
"usage": usage,
|
|
}),
|
|
&rate_limit_headers,
|
|
&body,
|
|
)
|
|
.await;
|
|
}
|
|
}
|
|
|
|
context.account_pool.release(&context.account_id).await;
|
|
}
|
|
|
|
async fn record_response_affinity(
|
|
session_affinity: &Arc<RuntimeSessionAffinityService>,
|
|
reasoning_replay: &Arc<tokio::sync::Mutex<ReasoningReplayCache>>,
|
|
request: &CodexResponsesRequest,
|
|
account_id: &str,
|
|
body: &str,
|
|
turn_state: Option<String>,
|
|
usage: Option<TokenUsage>,
|
|
) {
|
|
let Some(conversation_id) = request
|
|
.prompt_cache_key
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
else {
|
|
return;
|
|
};
|
|
let metadata = match completed_response_metadata(body) {
|
|
Ok(Some(metadata)) => metadata,
|
|
Ok(None) => return,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
error = %error,
|
|
"failed to parse completed response metadata for session affinity"
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
|
|
let variant_hash = compute_variant_hash(request);
|
|
let entry = SessionAffinityEntry {
|
|
account_id: account_id.to_string(),
|
|
conversation_id: conversation_id.to_string(),
|
|
turn_state: turn_state
|
|
.filter(|value| !value.trim().is_empty())
|
|
.or_else(|| request.turn_state.clone()),
|
|
instructions_hash: Some(hash_instructions(Some(&request.instructions))),
|
|
input_tokens: usage.map(|usage| usage.input_tokens),
|
|
function_call_ids: metadata.function_call_ids,
|
|
variant_hash: Some(variant_hash.clone()),
|
|
created_at: Utc::now(),
|
|
};
|
|
if let Err(error) = session_affinity
|
|
.record(metadata.response_id.clone(), entry)
|
|
.await
|
|
{
|
|
tracing::warn!(
|
|
error = %error,
|
|
response_id = %metadata.response_id,
|
|
account_id = %account_id,
|
|
"failed to record session affinity"
|
|
);
|
|
}
|
|
|
|
reasoning_replay.lock().await.record(
|
|
metadata.response_id,
|
|
account_id,
|
|
conversation_id,
|
|
&variant_hash,
|
|
&metadata.replay_items,
|
|
Utc::now(),
|
|
);
|
|
}
|
|
|
|
async fn evict_reasoning_replay(
|
|
reasoning_replay: &Arc<tokio::sync::Mutex<ReasoningReplayCache>>,
|
|
request: &CodexResponsesRequest,
|
|
account_id: &str,
|
|
) {
|
|
let Some(conversation_id) = request
|
|
.prompt_cache_key
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
else {
|
|
return;
|
|
};
|
|
let variant_hash = compute_variant_hash(request);
|
|
let evicted = reasoning_replay.lock().await.evict_by_identity(
|
|
account_id,
|
|
conversation_id,
|
|
&variant_hash,
|
|
Utc::now(),
|
|
);
|
|
if evicted > 0 {
|
|
tracing::info!(
|
|
account_id = %account_id,
|
|
conversation_id = %conversation_id,
|
|
variant_hash = %variant_hash,
|
|
evicted,
|
|
"evicted reasoning replay after invalid encrypted content"
|
|
);
|
|
}
|
|
}
|
|
|
|
struct ResponseEventRecord<'a> {
|
|
logs: &'a AdminLogService,
|
|
request_id: &'a str,
|
|
account_id: &'a str,
|
|
route: &'a str,
|
|
model: &'a str,
|
|
started_at: Instant,
|
|
status_code: i64,
|
|
level: EventLevel,
|
|
message: &'a str,
|
|
metadata: Value,
|
|
rate_limit_headers: &'a [(String, String)],
|
|
}
|
|
|
|
struct ResponseStreamFailureEventRecord<'a> {
|
|
logs: &'a AdminLogService,
|
|
request_id: &'a str,
|
|
account_id: &'a str,
|
|
route: &'a str,
|
|
model: &'a str,
|
|
started_at: Instant,
|
|
transport: CodexBackendTransport,
|
|
request: &'a CodexResponsesRequest,
|
|
failure: &'a ResponsesSseFailure,
|
|
rate_limit_headers: &'a [(String, String)],
|
|
prefetched: &'a [u8],
|
|
}
|
|
|
|
struct ResponseUpstreamErrorEventRecord<'a> {
|
|
logs: &'a AdminLogService,
|
|
request_id: &'a str,
|
|
account_id: &'a str,
|
|
route: &'a str,
|
|
model: &'a str,
|
|
started_at: Instant,
|
|
stream: bool,
|
|
transport: CodexBackendTransport,
|
|
error: &'a CodexClientError,
|
|
}
|
|
|
|
struct ResponseDispatchErrorEventRecord<'a> {
|
|
logs: &'a AdminLogService,
|
|
request_id: &'a str,
|
|
account_id: Option<&'a str>,
|
|
route: &'a str,
|
|
model: &'a str,
|
|
started_at: Instant,
|
|
stream: bool,
|
|
compact: bool,
|
|
transport: Option<&'a str>,
|
|
error: &'a ResponseDispatchError,
|
|
}
|
|
|
|
struct ChatDispatchErrorEventRecord<'a> {
|
|
logs: &'a AdminLogService,
|
|
request_id: &'a str,
|
|
account_id: Option<&'a str>,
|
|
route: &'a str,
|
|
model: &'a str,
|
|
started_at: Instant,
|
|
transport: Option<&'a str>,
|
|
error: &'a ChatDispatchError,
|
|
}
|
|
|
|
async fn record_response_event(mut record: ResponseEventRecord<'_>) {
|
|
enrich_event_route_metadata(&mut record.metadata, record.route);
|
|
if let Some(object) = record.metadata.as_object_mut() {
|
|
if !record.rate_limit_headers.is_empty() {
|
|
object
|
|
.entry("rateLimitHeaders".to_string())
|
|
.or_insert_with(|| serde_json::json!(record.rate_limit_headers));
|
|
}
|
|
}
|
|
let mut event = EventLog::new(
|
|
response_event_kind(record.route),
|
|
record.level,
|
|
record.message,
|
|
);
|
|
event.request_id = Some(record.request_id.to_string());
|
|
event.account_id = Some(record.account_id.to_string());
|
|
event.route = Some(record.route.to_string());
|
|
event.model = Some(record.model.to_string());
|
|
event.status_code = Some(record.status_code);
|
|
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
|
event.metadata = record.metadata;
|
|
if let Err(error) = record.logs.record(event).await {
|
|
tracing::warn!(account_id = record.account_id, error = %error, "failed to record response event");
|
|
}
|
|
}
|
|
|
|
async fn record_response_dispatch_error_event(record: ResponseDispatchErrorEventRecord<'_>) {
|
|
let mut metadata = response_dispatch_error_metadata(
|
|
record.error,
|
|
record.stream,
|
|
record.compact,
|
|
record.transport,
|
|
);
|
|
enrich_event_route_metadata(&mut metadata, record.route);
|
|
|
|
let mut event = EventLog::new(
|
|
response_event_kind(record.route),
|
|
EventLevel::Error,
|
|
"v1 responses dispatch failed",
|
|
);
|
|
event.request_id = Some(record.request_id.to_string());
|
|
event.account_id = record.account_id.map(ToString::to_string);
|
|
event.route = Some(record.route.to_string());
|
|
event.model = Some(record.model.to_string());
|
|
event.status_code = Some(status_code_for_response_dispatch_error(record.error));
|
|
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
|
event.metadata = metadata;
|
|
if let Err(error) = record.logs.record(event).await {
|
|
tracing::warn!(account_id = record.account_id.unwrap_or(""), error = %error, "failed to record response dispatch error event");
|
|
}
|
|
}
|
|
|
|
async fn record_chat_dispatch_error_event(record: ChatDispatchErrorEventRecord<'_>) {
|
|
let mut metadata = chat_dispatch_error_metadata(record.error, record.transport);
|
|
enrich_event_route_metadata(&mut metadata, record.route);
|
|
|
|
let mut event = EventLog::new(
|
|
response_event_kind(record.route),
|
|
EventLevel::Error,
|
|
"v1 chat dispatch failed",
|
|
);
|
|
event.request_id = Some(record.request_id.to_string());
|
|
event.account_id = record.account_id.map(ToString::to_string);
|
|
event.route = Some(record.route.to_string());
|
|
event.model = Some(record.model.to_string());
|
|
event.status_code = Some(status_code_for_chat_dispatch_error(record.error));
|
|
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
|
event.metadata = metadata;
|
|
if let Err(error) = record.logs.record(event).await {
|
|
tracing::warn!(account_id = record.account_id.unwrap_or(""), error = %error, "failed to record chat dispatch error event");
|
|
}
|
|
}
|
|
|
|
async fn record_response_upstream_error_event(record: ResponseUpstreamErrorEventRecord<'_>) {
|
|
record_response_event(ResponseEventRecord {
|
|
logs: record.logs,
|
|
request_id: record.request_id,
|
|
account_id: record.account_id,
|
|
route: record.route,
|
|
model: record.model,
|
|
started_at: record.started_at,
|
|
status_code: status_code_for_upstream_error(record.error),
|
|
level: EventLevel::Error,
|
|
message: "v1 responses upstream request failed",
|
|
metadata: upstream_error_metadata(record.error, record.stream, record.transport),
|
|
rate_limit_headers: &[],
|
|
})
|
|
.await;
|
|
}
|
|
|
|
async fn record_prefetched_response_stream_failure_event(
|
|
record: ResponseStreamFailureEventRecord<'_>,
|
|
) {
|
|
let mut metadata = stream_failure_metadata(record.failure, None);
|
|
if let Some(object) = metadata.as_object_mut() {
|
|
if record.transport == CodexBackendTransport::WebSocket {
|
|
object.insert(
|
|
"transport".to_string(),
|
|
Value::String("websocket".to_string()),
|
|
);
|
|
}
|
|
object.insert("requestBody".to_string(), serde_json::json!(record.request));
|
|
object.insert(
|
|
"responseBody".to_string(),
|
|
Value::String(String::from_utf8_lossy(record.prefetched).to_string()),
|
|
);
|
|
}
|
|
record_response_event(ResponseEventRecord {
|
|
logs: record.logs,
|
|
request_id: record.request_id,
|
|
account_id: record.account_id,
|
|
route: record.route,
|
|
model: record.model,
|
|
started_at: record.started_at,
|
|
status_code: status_code_for_stream_failure(record.failure),
|
|
level: EventLevel::Error,
|
|
message: "v1 responses stream failed",
|
|
metadata,
|
|
rate_limit_headers: record.rate_limit_headers,
|
|
})
|
|
.await;
|
|
}
|
|
|
|
fn requested_response_transport(request: &CodexResponsesRequest) -> CodexBackendTransport {
|
|
if request.use_websocket && !request.force_http_sse {
|
|
CodexBackendTransport::WebSocket
|
|
} else {
|
|
CodexBackendTransport::HttpSse
|
|
}
|
|
}
|
|
|
|
fn upstream_error_metadata(
|
|
error: &CodexClientError,
|
|
stream: bool,
|
|
transport: CodexBackendTransport,
|
|
) -> Value {
|
|
let mut metadata = serde_json::json!({
|
|
"stream": stream,
|
|
"failed": true,
|
|
"transport": backend_transport_name(transport),
|
|
"errorKind": "upstream",
|
|
"error": error.to_string(),
|
|
});
|
|
if let Some(object) = metadata.as_object_mut() {
|
|
if let CodexClientError::Upstream {
|
|
status,
|
|
body,
|
|
retry_after_seconds,
|
|
..
|
|
} = error
|
|
{
|
|
object.insert(
|
|
"upstreamStatus".to_string(),
|
|
Value::Number(serde_json::Number::from(status.as_u16())),
|
|
);
|
|
object.insert("upstreamBody".to_string(), Value::String(body.clone()));
|
|
if let Some(retry_after_seconds) = retry_after_seconds {
|
|
object.insert(
|
|
"retryAfterSeconds".to_string(),
|
|
Value::Number(serde_json::Number::from(*retry_after_seconds)),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
metadata
|
|
}
|
|
|
|
fn chat_dispatch_error_metadata(error: &ChatDispatchError, transport: Option<&str>) -> Value {
|
|
let mut metadata = serde_json::json!({
|
|
"stream": false,
|
|
"failed": true,
|
|
"errorKind": "dispatch",
|
|
"error": error.to_string(),
|
|
});
|
|
let Some(object) = metadata.as_object_mut() else {
|
|
return metadata;
|
|
};
|
|
if let Some(transport) = transport {
|
|
object.insert(
|
|
"transport".to_string(),
|
|
Value::String(transport.to_string()),
|
|
);
|
|
}
|
|
|
|
match error {
|
|
ChatDispatchError::QuotaExhausted {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "quota_exhausted", *count, upstream_error),
|
|
ChatDispatchError::RateLimited {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "rate_limited", *count, upstream_error),
|
|
ChatDispatchError::Expired {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "expired", *count, upstream_error),
|
|
ChatDispatchError::Disabled {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "disabled", *count, upstream_error),
|
|
ChatDispatchError::Banned {
|
|
count,
|
|
upstream_error,
|
|
status_code,
|
|
} => {
|
|
insert_exhausted_dispatch_metadata(object, "banned", *count, upstream_error);
|
|
object.insert(
|
|
"upstreamStatus".to_string(),
|
|
Value::Number(serde_json::Number::from(*status_code)),
|
|
);
|
|
}
|
|
ChatDispatchError::CloudflareChallenge {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(
|
|
object,
|
|
"cloudflare_challenge",
|
|
*count,
|
|
upstream_error,
|
|
),
|
|
ChatDispatchError::CloudflarePathBlocked {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(
|
|
object,
|
|
"cloudflare_path_block",
|
|
*count,
|
|
upstream_error,
|
|
),
|
|
ChatDispatchError::ModelUnsupported {
|
|
count,
|
|
upstream_error,
|
|
} => {
|
|
insert_exhausted_dispatch_metadata(object, "model_unsupported", *count, upstream_error)
|
|
}
|
|
ChatDispatchError::Upstream(error) => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("upstream".to_string()),
|
|
);
|
|
if let CodexClientError::Upstream {
|
|
status,
|
|
body,
|
|
retry_after_seconds,
|
|
..
|
|
} = error
|
|
{
|
|
object.insert(
|
|
"upstreamStatus".to_string(),
|
|
Value::Number(serde_json::Number::from(status.as_u16())),
|
|
);
|
|
object.insert("upstreamBody".to_string(), Value::String(body.clone()));
|
|
if let Some(retry_after_seconds) = retry_after_seconds {
|
|
object.insert(
|
|
"retryAfterSeconds".to_string(),
|
|
Value::Number(serde_json::Number::from(*retry_after_seconds)),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
ChatDispatchError::NoActiveAccount => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("no_active_account".to_string()),
|
|
);
|
|
}
|
|
ChatDispatchError::AccountStore => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("account_store".to_string()),
|
|
);
|
|
}
|
|
ChatDispatchError::InvalidSse(_) => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("invalid_sse".to_string()),
|
|
);
|
|
}
|
|
ChatDispatchError::EmptyUpstreamResponse => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("empty_upstream_response".to_string()),
|
|
);
|
|
}
|
|
}
|
|
metadata
|
|
}
|
|
|
|
fn response_dispatch_error_metadata(
|
|
error: &ResponseDispatchError,
|
|
stream: bool,
|
|
compact: bool,
|
|
transport: Option<&str>,
|
|
) -> Value {
|
|
let mut metadata = serde_json::json!({
|
|
"stream": stream,
|
|
"failed": true,
|
|
"errorKind": "dispatch",
|
|
"error": error.to_string(),
|
|
});
|
|
let Some(object) = metadata.as_object_mut() else {
|
|
return metadata;
|
|
};
|
|
if compact {
|
|
object.insert("compact".to_string(), Value::Bool(true));
|
|
}
|
|
if let Some(transport) = transport {
|
|
object.insert(
|
|
"transport".to_string(),
|
|
Value::String(transport.to_string()),
|
|
);
|
|
}
|
|
|
|
match error {
|
|
ResponseDispatchError::QuotaExhausted {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "quota_exhausted", *count, upstream_error),
|
|
ResponseDispatchError::RateLimited {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "rate_limited", *count, upstream_error),
|
|
ResponseDispatchError::Expired {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "expired", *count, upstream_error),
|
|
ResponseDispatchError::Disabled {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(object, "disabled", *count, upstream_error),
|
|
ResponseDispatchError::Banned {
|
|
count,
|
|
upstream_error,
|
|
status_code,
|
|
} => {
|
|
insert_exhausted_dispatch_metadata(object, "banned", *count, upstream_error);
|
|
object.insert(
|
|
"upstreamStatus".to_string(),
|
|
Value::Number(serde_json::Number::from(*status_code)),
|
|
);
|
|
}
|
|
ResponseDispatchError::CloudflareChallenge {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(
|
|
object,
|
|
"cloudflare_challenge",
|
|
*count,
|
|
upstream_error,
|
|
),
|
|
ResponseDispatchError::CloudflarePathBlocked {
|
|
count,
|
|
upstream_error,
|
|
} => insert_exhausted_dispatch_metadata(
|
|
object,
|
|
"cloudflare_path_block",
|
|
*count,
|
|
upstream_error,
|
|
),
|
|
ResponseDispatchError::ModelUnsupported {
|
|
count,
|
|
upstream_error,
|
|
} => {
|
|
insert_exhausted_dispatch_metadata(object, "model_unsupported", *count, upstream_error)
|
|
}
|
|
ResponseDispatchError::Upstream(error) => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("upstream".to_string()),
|
|
);
|
|
if let CodexClientError::Upstream {
|
|
status,
|
|
body,
|
|
retry_after_seconds,
|
|
..
|
|
} = error
|
|
{
|
|
object.insert(
|
|
"upstreamStatus".to_string(),
|
|
Value::Number(serde_json::Number::from(status.as_u16())),
|
|
);
|
|
object.insert("upstreamBody".to_string(), Value::String(body.clone()));
|
|
if let Some(retry_after_seconds) = retry_after_seconds {
|
|
object.insert(
|
|
"retryAfterSeconds".to_string(),
|
|
Value::Number(serde_json::Number::from(*retry_after_seconds)),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
ResponseDispatchError::NoActiveAccount => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("no_active_account".to_string()),
|
|
);
|
|
}
|
|
ResponseDispatchError::AccountStore => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("account_store".to_string()),
|
|
);
|
|
}
|
|
ResponseDispatchError::InvalidSse(_) => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("invalid_sse".to_string()),
|
|
);
|
|
}
|
|
ResponseDispatchError::MissingCompleted => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("missing_completed".to_string()),
|
|
);
|
|
}
|
|
ResponseDispatchError::EmptyUpstreamResponse => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("empty_upstream_response".to_string()),
|
|
);
|
|
}
|
|
ResponseDispatchError::Failed(failure) => {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String("response_failed".to_string()),
|
|
);
|
|
object.insert(
|
|
"failureEvent".to_string(),
|
|
Value::String(failure.event.clone()),
|
|
);
|
|
object.insert(
|
|
"failureMessage".to_string(),
|
|
Value::String(failure.message.clone()),
|
|
);
|
|
object.insert(
|
|
"upstreamCode".to_string(),
|
|
failure
|
|
.upstream_code
|
|
.as_ref()
|
|
.map(|code| Value::String(code.clone()))
|
|
.unwrap_or(Value::Null),
|
|
);
|
|
}
|
|
}
|
|
metadata
|
|
}
|
|
|
|
fn insert_exhausted_dispatch_metadata(
|
|
object: &mut Map<String, Value>,
|
|
failure_class: &str,
|
|
count: usize,
|
|
upstream_error: &str,
|
|
) {
|
|
object.insert(
|
|
"failureClass".to_string(),
|
|
Value::String(failure_class.to_string()),
|
|
);
|
|
object.insert(
|
|
"exhaustedCount".to_string(),
|
|
Value::Number(serde_json::Number::from(count as u64)),
|
|
);
|
|
object.insert(
|
|
"upstreamError".to_string(),
|
|
Value::String(upstream_error.to_string()),
|
|
);
|
|
}
|
|
|
|
fn status_code_for_response_dispatch_error(error: &ResponseDispatchError) -> i64 {
|
|
match error {
|
|
ResponseDispatchError::NoActiveAccount | ResponseDispatchError::AccountStore => 503,
|
|
ResponseDispatchError::QuotaExhausted { .. } => 402,
|
|
ResponseDispatchError::RateLimited { .. } => 429,
|
|
ResponseDispatchError::Expired { .. } | ResponseDispatchError::Disabled { .. } => 401,
|
|
ResponseDispatchError::Banned { status_code, .. } => i64::from(*status_code),
|
|
ResponseDispatchError::CloudflareChallenge { .. }
|
|
| ResponseDispatchError::CloudflarePathBlocked { .. }
|
|
| ResponseDispatchError::InvalidSse(_)
|
|
| ResponseDispatchError::MissingCompleted
|
|
| ResponseDispatchError::EmptyUpstreamResponse
|
|
| ResponseDispatchError::Failed(_) => 502,
|
|
ResponseDispatchError::ModelUnsupported { .. } => 400,
|
|
ResponseDispatchError::Upstream(error) => status_code_for_upstream_error(error),
|
|
}
|
|
}
|
|
|
|
fn status_code_for_chat_dispatch_error(error: &ChatDispatchError) -> i64 {
|
|
match error {
|
|
ChatDispatchError::NoActiveAccount | ChatDispatchError::AccountStore => 503,
|
|
ChatDispatchError::QuotaExhausted { .. } => 402,
|
|
ChatDispatchError::RateLimited { .. } => 429,
|
|
ChatDispatchError::Expired { .. } | ChatDispatchError::Disabled { .. } => 401,
|
|
ChatDispatchError::Banned { status_code, .. } => i64::from(*status_code),
|
|
ChatDispatchError::CloudflareChallenge { .. }
|
|
| ChatDispatchError::CloudflarePathBlocked { .. }
|
|
| ChatDispatchError::InvalidSse(_)
|
|
| ChatDispatchError::EmptyUpstreamResponse => 502,
|
|
ChatDispatchError::ModelUnsupported { .. } => 400,
|
|
ChatDispatchError::Upstream(error) => status_code_for_upstream_error(error),
|
|
}
|
|
}
|
|
|
|
fn status_code_for_upstream_error(error: &CodexClientError) -> i64 {
|
|
match error {
|
|
CodexClientError::Upstream { status, .. } => i64::from(status.as_u16()),
|
|
_ => 502,
|
|
}
|
|
}
|
|
|
|
fn upstream_error_http_status(error: &CodexClientError) -> u16 {
|
|
match error {
|
|
CodexClientError::Upstream { status, .. } => status.as_u16(),
|
|
_ => 502,
|
|
}
|
|
}
|
|
|
|
fn backend_transport_name(transport: CodexBackendTransport) -> &'static str {
|
|
match transport {
|
|
CodexBackendTransport::HttpSse => "http_sse",
|
|
CodexBackendTransport::WebSocket => "websocket",
|
|
}
|
|
}
|
|
|
|
async fn record_live_response_stream_event(
|
|
context: &LiveResponseStreamContext,
|
|
status_code: i64,
|
|
level: EventLevel,
|
|
message: &str,
|
|
mut metadata: Value,
|
|
rate_limit_headers: &[(String, String)],
|
|
body: &str,
|
|
) {
|
|
ensure_stream_metadata_flag(&mut metadata);
|
|
enrich_event_route_metadata(&mut metadata, &context.route);
|
|
enrich_live_response_stream_metadata(context, rate_limit_headers, &mut metadata, body);
|
|
let mut event = EventLog::new(response_event_kind(&context.route), level, message);
|
|
event.request_id = Some(context.request_id.clone());
|
|
event.account_id = Some(context.account_id.clone());
|
|
event.route = Some(context.route.clone());
|
|
event.model = Some(context.model.clone());
|
|
event.status_code = Some(status_code);
|
|
event.latency_ms = Some(elapsed_millis_i64(context.started_at));
|
|
event.metadata = metadata;
|
|
if let Err(error) = context.logs.record(event).await {
|
|
tracing::warn!(account_id = %context.account_id, error = %error, "failed to record live response stream event");
|
|
}
|
|
}
|
|
|
|
fn response_event_kind(route: &str) -> &'static str {
|
|
if route == "/v1/chat/completions" {
|
|
"v1.chat"
|
|
} else {
|
|
"v1.response"
|
|
}
|
|
}
|
|
|
|
fn response_api_kind(route: &str) -> &'static str {
|
|
if route == "/v1/chat/completions" {
|
|
"chat"
|
|
} else {
|
|
"responses"
|
|
}
|
|
}
|
|
|
|
fn enrich_event_route_metadata(metadata: &mut Value, route: &str) {
|
|
let Some(object) = metadata.as_object_mut() else {
|
|
return;
|
|
};
|
|
object
|
|
.entry("route".to_string())
|
|
.or_insert_with(|| Value::String(route.to_string()));
|
|
object
|
|
.entry("apiKind".to_string())
|
|
.or_insert_with(|| Value::String(response_api_kind(route).to_string()));
|
|
}
|
|
|
|
fn ensure_stream_metadata_flag(metadata: &mut Value) {
|
|
let Some(object) = metadata.as_object_mut() else {
|
|
*metadata = serde_json::json!({ "stream": true });
|
|
return;
|
|
};
|
|
object
|
|
.entry("stream".to_string())
|
|
.or_insert(Value::Bool(true));
|
|
}
|
|
|
|
fn enrich_live_response_stream_metadata(
|
|
context: &LiveResponseStreamContext,
|
|
rate_limit_headers: &[(String, String)],
|
|
metadata: &mut Value,
|
|
body: &str,
|
|
) {
|
|
let Some(object) = metadata.as_object_mut() else {
|
|
return;
|
|
};
|
|
object
|
|
.entry("transport".to_string())
|
|
.or_insert_with(|| Value::String(backend_transport_name(context.transport).to_string()));
|
|
if !rate_limit_headers.is_empty() {
|
|
object
|
|
.entry("rateLimitHeaders".to_string())
|
|
.or_insert_with(|| serde_json::json!(rate_limit_headers));
|
|
}
|
|
object
|
|
.entry("requestBody".to_string())
|
|
.or_insert_with(|| serde_json::json!(context.request));
|
|
object
|
|
.entry("responseBody".to_string())
|
|
.or_insert_with(|| Value::String(body.to_string()));
|
|
}
|
|
|
|
async fn live_response_rate_limit_headers(
|
|
context: &LiveResponseStreamContext,
|
|
) -> Vec<(String, String)> {
|
|
let mut headers = context.rate_limit_headers.clone();
|
|
if let Some(updates) = &context.rate_limit_header_updates {
|
|
headers.extend(updates.lock().await.iter().cloned());
|
|
}
|
|
headers
|
|
}
|
|
|
|
async fn live_response_turn_state(context: &LiveResponseStreamContext) -> Option<String> {
|
|
if let Some(update) = &context.turn_state_update {
|
|
return update.lock().await.clone();
|
|
}
|
|
context.turn_state.clone()
|
|
}
|
|
|
|
fn elapsed_millis_i64(started_at: Instant) -> i64 {
|
|
started_at.elapsed().as_millis().min(i64::MAX as u128) as i64
|
|
}
|
|
|
|
fn bool_to_u64(value: bool) -> u64 {
|
|
if value {
|
|
1
|
|
} else {
|
|
0
|
|
}
|
|
}
|
|
|
|
fn stream_failure_metadata(failure: &ResponsesSseFailure, usage: Option<TokenUsage>) -> Value {
|
|
serde_json::json!({
|
|
"stream": true,
|
|
"failed": true,
|
|
"failureEvent": failure.event,
|
|
"failureMessage": failure.message,
|
|
"upstreamCode": failure.upstream_code,
|
|
"usage": usage,
|
|
})
|
|
}
|
|
|
|
fn status_code_for_stream_failure(failure: &ResponsesSseFailure) -> i64 {
|
|
let code = failure
|
|
.upstream_code
|
|
.as_deref()
|
|
.unwrap_or("error")
|
|
.to_ascii_lowercase();
|
|
if code.contains("model") && (code.contains("not_supported") || code.contains("not_available"))
|
|
{
|
|
return 400;
|
|
}
|
|
if code.contains("invalid_request") || code.contains("not_found") {
|
|
return 400;
|
|
}
|
|
if code.contains("rate_limit") || code.contains("usage_limit") {
|
|
return 429;
|
|
}
|
|
if code.contains("unauthorized")
|
|
|| code.contains("invalid_api_key")
|
|
|| code == "token_invalid"
|
|
|| code == "token_expired"
|
|
|| code == "account_deactivated"
|
|
{
|
|
return 401;
|
|
}
|
|
if code.contains("forbidden") || code.contains("banned") {
|
|
return 403;
|
|
}
|
|
if code.contains("payment") || code.contains("quota") {
|
|
return 402;
|
|
}
|
|
502
|
|
}
|
|
|
|
fn stream_failure_http_status(failure: &ResponsesSseFailure) -> u16 {
|
|
u16::try_from(status_code_for_stream_failure(failure)).unwrap_or(502)
|
|
}
|
|
|
|
/// Responses 调度错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum ResponseDispatchError {
|
|
/// 账号存储失败。
|
|
#[error("failed to list runtime accounts")]
|
|
AccountStore,
|
|
/// 没有活跃账号。
|
|
#[error("no active account is available")]
|
|
NoActiveAccount,
|
|
/// 所有账号都因配额耗尽不可用。
|
|
#[error("all accounts exhausted by quota")]
|
|
QuotaExhausted {
|
|
/// 配额耗尽账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因限流不可用。
|
|
#[error("all accounts exhausted by rate limit")]
|
|
RateLimited {
|
|
/// 限流账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因认证失效不可用。
|
|
#[error("all accounts exhausted by expired auth")]
|
|
Expired {
|
|
/// 认证失效账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因 token 已确认不可用而被禁用。
|
|
#[error("all accounts exhausted by disabled auth")]
|
|
Disabled {
|
|
/// 禁用账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都被上游封禁。
|
|
#[error("all accounts exhausted by banned auth")]
|
|
Banned {
|
|
/// 封禁账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
/// 应返回给客户端的上游 HTTP 状态。
|
|
status_code: u16,
|
|
},
|
|
/// 所有账号都因 Cloudflare challenge 暂不可用。
|
|
#[error("all accounts exhausted by Cloudflare challenge")]
|
|
CloudflareChallenge {
|
|
/// Cloudflare challenge 账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误说明。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有账号都因 Cloudflare path-block 暂不可用。
|
|
#[error("all accounts exhausted by Cloudflare path-block")]
|
|
CloudflarePathBlocked {
|
|
/// Cloudflare path-block 账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误说明。
|
|
upstream_error: String,
|
|
},
|
|
/// 所有候选账号都不支持当前模型。
|
|
#[error("all accounts exhausted by unsupported model")]
|
|
ModelUnsupported {
|
|
/// 模型不支持账号数量。
|
|
count: usize,
|
|
/// 最后一个上游错误体。
|
|
upstream_error: String,
|
|
},
|
|
/// 上游请求失败。
|
|
#[error("upstream request failed: {0}")]
|
|
Upstream(#[from] CodexClientError),
|
|
/// 上游 SSE 无法解析。
|
|
#[error("invalid upstream SSE response: {0}")]
|
|
InvalidSse(#[from] SseError),
|
|
/// 上游响应没有完成事件。
|
|
#[error("upstream response did not include response.completed")]
|
|
MissingCompleted,
|
|
/// 上游响应完成但没有可见输出。
|
|
#[error("upstream response did not include visible output")]
|
|
EmptyUpstreamResponse,
|
|
/// 上游返回失败事件。
|
|
#[error("upstream response failed: {0:?}")]
|
|
Failed(ResponsesSseFailure),
|
|
}
|
|
|
|
/// Responses live SSE body stream error.
|
|
#[derive(Debug, Error)]
|
|
pub enum ResponseDispatchStreamError {
|
|
/// 上游字节流读取失败。
|
|
#[error("upstream stream failed: {0}")]
|
|
Upstream(#[from] CodexClientError),
|
|
}
|
|
|
|
/// 管理端日志服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminLogService {
|
|
store: SqliteEventLogStore,
|
|
settings: Arc<RwLock<AdminLogSettings>>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy)]
|
|
struct AdminLogSettings {
|
|
enabled: bool,
|
|
capacity: u32,
|
|
capture_body: bool,
|
|
}
|
|
|
|
impl AdminLogService {
|
|
/// 构造管理端日志服务。
|
|
pub fn new(
|
|
store: SqliteEventLogStore,
|
|
enabled: bool,
|
|
capacity: u32,
|
|
capture_body: bool,
|
|
) -> Self {
|
|
Self {
|
|
store,
|
|
settings: Arc::new(RwLock::new(AdminLogSettings {
|
|
enabled,
|
|
capacity,
|
|
capture_body,
|
|
})),
|
|
}
|
|
}
|
|
|
|
/// 分页查询日志。
|
|
pub async fn list(
|
|
&self,
|
|
cursor: Option<String>,
|
|
limit: u32,
|
|
filter: AdminLogFilter,
|
|
) -> Result<Page<EventLog>, AdminLogError> {
|
|
self.store
|
|
.list(filter.into(), cursor, limit)
|
|
.await
|
|
.map_err(|_| AdminLogError::List)
|
|
}
|
|
|
|
/// 按 ID 读取日志。
|
|
pub async fn get(&self, id: &str) -> Result<Option<EventLog>, AdminLogError> {
|
|
self.store.get(id).await.map_err(|_| AdminLogError::Get)
|
|
}
|
|
|
|
/// 读取日志状态。
|
|
pub async fn state(&self) -> Result<AdminLogState, AdminLogError> {
|
|
let settings = *self.settings.read().await;
|
|
Ok(AdminLogState {
|
|
enabled: settings.enabled,
|
|
capacity: settings.capacity,
|
|
capture_body: settings.capture_body,
|
|
stored_count: self.store.count().await.map_err(|_| AdminLogError::Count)?,
|
|
})
|
|
}
|
|
|
|
/// 更新日志状态。
|
|
pub async fn update_state(
|
|
&self,
|
|
update: AdminLogStateUpdate,
|
|
) -> Result<AdminLogState, AdminLogError> {
|
|
if matches!(update.capacity, Some(0)) {
|
|
return Err(AdminLogError::InvalidCapacity);
|
|
}
|
|
|
|
let trim_capacity = {
|
|
let mut settings = self.settings.write().await;
|
|
if let Some(enabled) = update.enabled {
|
|
settings.enabled = enabled;
|
|
}
|
|
if let Some(capacity) = update.capacity {
|
|
settings.capacity = capacity;
|
|
}
|
|
if let Some(capture_body) = update.capture_body {
|
|
settings.capture_body = capture_body;
|
|
}
|
|
update.capacity
|
|
};
|
|
|
|
if let Some(capacity) = trim_capacity {
|
|
self.store
|
|
.trim_to_capacity(capacity)
|
|
.await
|
|
.map_err(|_| AdminLogError::Trim)?;
|
|
}
|
|
|
|
self.state().await
|
|
}
|
|
|
|
/// 清空日志。
|
|
pub async fn clear(&self) -> Result<AdminClearLogs, AdminLogError> {
|
|
self.store
|
|
.clear()
|
|
.await
|
|
.map(|cleared| AdminClearLogs { cleared })
|
|
.map_err(|_| AdminLogError::Clear)
|
|
}
|
|
|
|
async fn record(&self, mut event: EventLog) -> Result<(), AdminLogError> {
|
|
let settings = *self.settings.read().await;
|
|
let policy = EventLogService::new(settings.enabled);
|
|
if !policy.should_record(&event) {
|
|
return Ok(());
|
|
}
|
|
apply_capture_body_policy(&mut event, settings.capture_body);
|
|
self.store
|
|
.append(&event)
|
|
.await
|
|
.map_err(|_| AdminLogError::Append)?;
|
|
self.store
|
|
.trim_to_capacity(settings.capacity)
|
|
.await
|
|
.map_err(|_| AdminLogError::Trim)?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn apply_capture_body_policy(event: &mut EventLog, capture_body: bool) {
|
|
if capture_body {
|
|
return;
|
|
}
|
|
let Some(metadata) = event.metadata.as_object_mut() else {
|
|
return;
|
|
};
|
|
for key in [
|
|
"body",
|
|
"rawBody",
|
|
"requestBody",
|
|
"responseBody",
|
|
"upstreamBody",
|
|
] {
|
|
metadata.remove(key);
|
|
}
|
|
}
|
|
|
|
/// 日志查询过滤器。
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct AdminLogFilter {
|
|
/// 事件类别。
|
|
pub kind: Option<String>,
|
|
/// 事件等级。
|
|
pub level: Option<EventLevel>,
|
|
/// 请求 ID。
|
|
pub request_id: Option<String>,
|
|
/// 账号 ID。
|
|
pub account_id: Option<String>,
|
|
/// 路由。
|
|
pub route: Option<String>,
|
|
/// 模型。
|
|
pub model: Option<String>,
|
|
/// HTTP 状态码。
|
|
pub status_code: Option<i64>,
|
|
/// 上游传输方式。
|
|
pub transport: Option<String>,
|
|
/// 同一请求内的上游尝试序号。
|
|
pub attempt_index: Option<i64>,
|
|
/// 上游 HTTP 状态码。
|
|
pub upstream_status_code: Option<i64>,
|
|
/// 失败分类。
|
|
pub failure_class: Option<String>,
|
|
/// 上游响应 ID。
|
|
pub response_id: Option<String>,
|
|
/// 上游请求 ID。
|
|
pub upstream_request_id: Option<String>,
|
|
/// 搜索关键词。
|
|
pub search: Option<String>,
|
|
}
|
|
|
|
/// 日志状态。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminLogState {
|
|
/// 是否启用。
|
|
pub enabled: bool,
|
|
/// 内存容量。
|
|
pub capacity: u32,
|
|
/// 是否捕获请求体。
|
|
pub capture_body: bool,
|
|
/// 已存储数量。
|
|
pub stored_count: u64,
|
|
}
|
|
|
|
/// 日志状态更新。
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct AdminLogStateUpdate {
|
|
/// 是否启用。
|
|
pub enabled: Option<bool>,
|
|
/// 日志容量。
|
|
pub capacity: Option<u32>,
|
|
/// 是否捕获请求体。
|
|
pub capture_body: Option<bool>,
|
|
}
|
|
|
|
/// 清空日志结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminClearLogs {
|
|
/// 清理数量。
|
|
pub cleared: u64,
|
|
}
|
|
|
|
/// 管理端日志错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminLogError {
|
|
/// 列表失败。
|
|
#[error("failed to list event logs")]
|
|
List,
|
|
/// 读取失败。
|
|
#[error("failed to get event log")]
|
|
Get,
|
|
/// 计数失败。
|
|
#[error("failed to count event logs")]
|
|
Count,
|
|
/// 清空失败。
|
|
#[error("failed to clear event logs")]
|
|
Clear,
|
|
/// 写入失败。
|
|
#[error("failed to append event log")]
|
|
Append,
|
|
/// 裁剪失败。
|
|
#[error("failed to trim event logs")]
|
|
Trim,
|
|
/// 日志容量非法。
|
|
#[error("log capacity must be greater than zero")]
|
|
InvalidCapacity,
|
|
}
|
|
|
|
/// 管理端账号服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminAccountService {
|
|
store: SqliteAccountStore,
|
|
cookies: SqliteCookieStore,
|
|
quota_thresholds: QuotaWarningThresholds,
|
|
codex: Arc<CodexBackendClient>,
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
token_refresher: Arc<dyn TokenRefresher>,
|
|
refresh_margin_seconds: u64,
|
|
installation_id: Option<String>,
|
|
}
|
|
|
|
impl AdminAccountService {
|
|
/// 构造管理端账号服务。
|
|
#[expect(
|
|
clippy::too_many_arguments,
|
|
reason = "constructor wires service dependencies from runtime bootstrap"
|
|
)]
|
|
pub fn new(
|
|
store: SqliteAccountStore,
|
|
cookies: SqliteCookieStore,
|
|
quota_thresholds: QuotaWarningThresholds,
|
|
codex: Arc<CodexBackendClient>,
|
|
account_pool: Arc<RuntimeAccountPoolService>,
|
|
token_refresher: Arc<dyn TokenRefresher>,
|
|
refresh_margin_seconds: u64,
|
|
installation_id: Option<String>,
|
|
) -> Self {
|
|
Self {
|
|
store,
|
|
cookies,
|
|
quota_thresholds,
|
|
codex,
|
|
account_pool,
|
|
token_refresher,
|
|
refresh_margin_seconds,
|
|
installation_id,
|
|
}
|
|
}
|
|
|
|
fn next_refresh_at_for_expires_at(&self, expires_at: DateTime<Utc>) -> DateTime<Utc> {
|
|
let margin_seconds = self.refresh_margin_seconds.min(i64::MAX as u64) as i64;
|
|
expires_at - Duration::seconds(margin_seconds)
|
|
}
|
|
|
|
/// 分页列出账号元数据。
|
|
pub async fn list(
|
|
&self,
|
|
cursor: Option<String>,
|
|
limit: u32,
|
|
) -> Result<Page<AdminAccountMetadata>, AdminAccountError> {
|
|
let page = self
|
|
.store
|
|
.list_metadata(cursor, limit)
|
|
.await
|
|
.map_err(|_| AdminAccountError::List)?;
|
|
Ok(Page {
|
|
items: page
|
|
.items
|
|
.into_iter()
|
|
.map(AdminAccountMetadata::from)
|
|
.collect(),
|
|
next_cursor: page.next_cursor,
|
|
})
|
|
}
|
|
|
|
/// 返回管理端认证状态摘要。
|
|
pub async fn auth_status(&self) -> Result<AdminAuthStatus, AdminAccountError> {
|
|
let mut cursor = None;
|
|
let mut pool = AdminAuthPoolStatus::default();
|
|
let mut user = None;
|
|
loop {
|
|
let page = self
|
|
.store
|
|
.list_metadata(cursor, 200)
|
|
.await
|
|
.map_err(|_| AdminAccountError::List)?;
|
|
for account in page.items {
|
|
pool.record(account.status);
|
|
if user.is_none() && account.status == AccountStatus::Active {
|
|
user = Some(AdminAccountMetadata::from(account));
|
|
}
|
|
}
|
|
if page.next_cursor.is_none() {
|
|
break;
|
|
}
|
|
cursor = page.next_cursor;
|
|
}
|
|
|
|
Ok(AdminAuthStatus {
|
|
authenticated: pool.total > 0,
|
|
user,
|
|
pool,
|
|
})
|
|
}
|
|
|
|
/// 清空管理端 OAuth 登录账号。
|
|
pub async fn logout(&self) -> Result<AdminAuthLogout, AdminAccountError> {
|
|
let deleted = self
|
|
.store
|
|
.delete_all()
|
|
.await
|
|
.map_err(|_| AdminAccountError::Delete)?;
|
|
self.account_pool.clear().await;
|
|
Ok(AdminAuthLogout {
|
|
success: true,
|
|
deleted,
|
|
})
|
|
}
|
|
|
|
/// 导出账号;包含可重新导入的 token,只应暴露给管理端会话。
|
|
pub async fn export(
|
|
&self,
|
|
ids: Vec<String>,
|
|
) -> Result<Vec<AdminStoredAccount>, AdminAccountError> {
|
|
if ids.is_empty() {
|
|
let mut accounts = Vec::new();
|
|
let mut cursor = None;
|
|
loop {
|
|
let page = self
|
|
.store
|
|
.list(cursor, 200)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Export)?;
|
|
accounts.extend(page.items.into_iter().map(AdminStoredAccount::from));
|
|
if page.next_cursor.is_none() {
|
|
return Ok(accounts);
|
|
}
|
|
cursor = page.next_cursor;
|
|
}
|
|
}
|
|
|
|
let mut accounts = Vec::with_capacity(ids.len());
|
|
for id in ids {
|
|
match self.store.get(&id).await {
|
|
Ok(Some(account)) => accounts.push(AdminStoredAccount::from(account)),
|
|
Ok(None) => {}
|
|
Err(_) => return Err(AdminAccountError::Export),
|
|
}
|
|
}
|
|
Ok(accounts)
|
|
}
|
|
|
|
/// 导入 native 账号导出数据。
|
|
pub async fn import(&self, payload: &Value) -> Result<ImportedAccounts, AdminAccountError> {
|
|
let parsed = parse_account_import_payload(payload)?;
|
|
let source_format = parsed.source.as_str();
|
|
let entries = parsed.entries;
|
|
if entries.is_empty() {
|
|
return Err(AdminAccountError::NoImportableAccounts);
|
|
}
|
|
|
|
let mut imported = 0u32;
|
|
let mut skipped = 0u32;
|
|
for entry in entries {
|
|
match self.import_entry(entry, parsed.source).await? {
|
|
ImportedAccountState::Imported(account_id) => {
|
|
imported += 1;
|
|
self.sync_account_pool(&account_id).await?;
|
|
}
|
|
ImportedAccountState::Skipped => skipped += 1,
|
|
}
|
|
}
|
|
|
|
Ok(ImportedAccounts {
|
|
imported,
|
|
skipped,
|
|
source_format,
|
|
})
|
|
}
|
|
|
|
/// 手动创建或更新一个经 JWT claims 校验的账号。
|
|
pub async fn create(
|
|
&self,
|
|
token: Option<String>,
|
|
refresh_token: Option<String>,
|
|
) -> Result<AdminAccountMetadata, AdminAccountError> {
|
|
let provided_refresh_token = normalize_nonempty(refresh_token);
|
|
let tokens =
|
|
if let Some(access_token) = normalize_nonempty(token.map(normalize_bearer_token)) {
|
|
ManualCreateTokens {
|
|
access_token,
|
|
refresh_token_for_new: provided_refresh_token.clone(),
|
|
refresh_token_for_existing: provided_refresh_token,
|
|
}
|
|
} else if let Some(refresh_token) = provided_refresh_token {
|
|
let token_pair = self
|
|
.token_refresher
|
|
.refresh(&refresh_token)
|
|
.await
|
|
.map_err(AdminAccountError::RefreshTokenExchange)?;
|
|
let access_token =
|
|
normalize_nonempty(Some(normalize_bearer_token(token_pair.access_token)))
|
|
.ok_or(AdminAccountError::TokenRequired)?;
|
|
ManualCreateTokens {
|
|
access_token,
|
|
refresh_token_for_new: token_pair
|
|
.refresh_token
|
|
.clone()
|
|
.or_else(|| Some(refresh_token.clone())),
|
|
refresh_token_for_existing: token_pair.refresh_token,
|
|
}
|
|
} else {
|
|
return Err(AdminAccountError::TokenRequired);
|
|
};
|
|
|
|
let claims = manual_account_claims(&tokens.access_token, chrono::Utc::now())
|
|
.map_err(AdminAccountError::InvalidToken)?;
|
|
let existing = self
|
|
.store
|
|
.find_by_chatgpt_identity(&claims.account_id, claims.user_id.as_deref())
|
|
.await
|
|
.map_err(|_| AdminAccountError::Inspect)?;
|
|
|
|
let account_id = if let Some(existing) = existing {
|
|
let refresh_token = tokens.refresh_token_for_existing;
|
|
let updated = self
|
|
.store
|
|
.update_from_claims(
|
|
&existing.id,
|
|
AccountClaimsUpdate {
|
|
email: claims.email.clone(),
|
|
account_id: Some(claims.account_id.clone()),
|
|
user_id: claims.user_id.clone(),
|
|
plan_type: claims.plan_type.clone(),
|
|
access_token: SecretString::new(tokens.access_token.into()),
|
|
refresh_token: refresh_token.map(|token| SecretString::new(token.into())),
|
|
access_token_expires_at: Some(claims.expires_at),
|
|
next_refresh_at: Some(
|
|
self.next_refresh_at_for_expires_at(claims.expires_at),
|
|
),
|
|
status: AccountStatus::Active,
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|_| AdminAccountError::UpdateClaims)?;
|
|
if !updated {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
existing.id
|
|
} else {
|
|
let id = normalized_account_id(None);
|
|
let refresh_token = tokens.refresh_token_for_new;
|
|
self.store
|
|
.insert(NewAccount {
|
|
id: id.clone(),
|
|
email: claims.email.clone(),
|
|
account_id: Some(claims.account_id.clone()),
|
|
user_id: claims.user_id.clone(),
|
|
label: None,
|
|
plan_type: claims.plan_type.clone(),
|
|
access_token: SecretString::new(tokens.access_token.into()),
|
|
refresh_token: refresh_token.map(|token| SecretString::new(token.into())),
|
|
access_token_expires_at: Some(claims.expires_at),
|
|
status: AccountStatus::Active,
|
|
})
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
id
|
|
};
|
|
|
|
self.sync_account_pool(&account_id).await?;
|
|
|
|
self.store
|
|
.get(&account_id)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Inspect)?
|
|
.map(AdminAccountMetadata::from)
|
|
.ok_or(AdminAccountError::NotFound)
|
|
}
|
|
|
|
/// 导入 Codex CLI 的 auth.json 内容。
|
|
pub async fn import_codex_cli_auth(
|
|
&self,
|
|
payload: &Value,
|
|
) -> Result<AdminAccountMetadata, AdminAccountError> {
|
|
let token = first_string(payload, &["access_token", "accessToken", "token"]);
|
|
let refresh_token = first_string(payload, &["refresh_token", "refreshToken"]);
|
|
if token.is_none() && refresh_token.is_none() {
|
|
return Err(AdminAccountError::NoImportableAccounts);
|
|
}
|
|
self.create(token, refresh_token).await
|
|
}
|
|
|
|
async fn import_entry(
|
|
&self,
|
|
entry: AccountImportEntry,
|
|
source: AccountImportSource,
|
|
) -> Result<ImportedAccountState, AdminAccountError> {
|
|
let Some(resolved_tokens) = self
|
|
.resolve_import_tokens(entry.token, entry.refresh_token)
|
|
.await?
|
|
else {
|
|
return Ok(ImportedAccountState::Skipped);
|
|
};
|
|
let label = normalize_label(entry.label);
|
|
if label
|
|
.as_ref()
|
|
.is_some_and(|label| label.chars().count() > 64)
|
|
{
|
|
return Err(AdminAccountError::LabelTooLong);
|
|
}
|
|
|
|
let access_token_expires_at = entry
|
|
.access_token_expires_at
|
|
.as_deref()
|
|
.map(parse_account_import_datetime)
|
|
.transpose()?;
|
|
let quota_fetched_at = entry
|
|
.quota_fetched_at
|
|
.as_deref()
|
|
.map(parse_account_import_datetime)
|
|
.transpose()?;
|
|
let quota_json = entry.cached_quota.as_ref().map(Value::to_string);
|
|
let quota_verify_required = entry.quota_verify_required.unwrap_or(false);
|
|
let status = normalized_imported_account_status(
|
|
parse_account_import_status(entry.status.as_deref())?,
|
|
source,
|
|
&resolved_tokens.access_token,
|
|
);
|
|
let account_id = self
|
|
.import_target_account_id(
|
|
entry.id.as_deref(),
|
|
resolved_tokens.claims.as_ref(),
|
|
entry.account_id.as_deref(),
|
|
entry.user_id.as_deref(),
|
|
)
|
|
.await?;
|
|
let access_token_expires_at = resolved_tokens
|
|
.claims
|
|
.as_ref()
|
|
.map(|claims| claims.expires_at)
|
|
.or(access_token_expires_at);
|
|
let next_refresh_at = access_token_expires_at
|
|
.map(|expires_at| self.next_refresh_at_for_expires_at(expires_at));
|
|
let claims = resolved_tokens.claims.as_ref();
|
|
let account = NewAccount {
|
|
id: account_id.clone(),
|
|
email: claims
|
|
.and_then(|claims| claims.email.clone())
|
|
.or_else(|| normalize_nonempty(entry.email)),
|
|
account_id: claims
|
|
.map(|claims| claims.account_id.clone())
|
|
.or_else(|| normalize_nonempty(entry.account_id)),
|
|
user_id: claims
|
|
.and_then(|claims| claims.user_id.clone())
|
|
.or_else(|| normalize_nonempty(entry.user_id)),
|
|
label,
|
|
plan_type: claims
|
|
.and_then(|claims| claims.plan_type.clone())
|
|
.or_else(|| normalize_nonempty(entry.plan_type)),
|
|
access_token: SecretString::new(resolved_tokens.access_token.into()),
|
|
refresh_token: resolved_tokens
|
|
.refresh_token
|
|
.map(|token| SecretString::new(token.into())),
|
|
access_token_expires_at,
|
|
status,
|
|
};
|
|
|
|
match self.store.get(&account_id).await {
|
|
Ok(Some(_)) => {
|
|
let updated = self
|
|
.store
|
|
.update_from_import(ImportedAccountUpdate {
|
|
account,
|
|
quota_json,
|
|
quota_fetched_at,
|
|
quota_verify_required,
|
|
})
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
if !updated {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
self.store
|
|
.set_next_refresh_at(&account_id, next_refresh_at)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
return Ok(ImportedAccountState::Imported(account_id));
|
|
}
|
|
Ok(None) => {}
|
|
Err(_) => return Err(AdminAccountError::Inspect),
|
|
}
|
|
|
|
self.store
|
|
.insert(account)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
|
|
self.store
|
|
.set_next_refresh_at(&account_id, next_refresh_at)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
|
|
if quota_json.is_some()
|
|
|| quota_fetched_at.is_some()
|
|
|| entry.quota_verify_required.is_some()
|
|
{
|
|
self.store
|
|
.apply_import_quota_state(
|
|
&account_id,
|
|
quota_json.as_deref(),
|
|
quota_fetched_at,
|
|
quota_verify_required,
|
|
)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Import)?;
|
|
}
|
|
|
|
Ok(ImportedAccountState::Imported(account_id))
|
|
}
|
|
|
|
async fn resolve_import_tokens(
|
|
&self,
|
|
token: Option<String>,
|
|
refresh_token: Option<String>,
|
|
) -> Result<Option<ResolvedImportTokens>, AdminAccountError> {
|
|
let mut refresh_token = normalize_nonempty(refresh_token);
|
|
let Some(access_token) = normalize_nonempty(token.map(normalize_bearer_token)) else {
|
|
let Some(existing_refresh_token) = refresh_token else {
|
|
return Ok(None);
|
|
};
|
|
let refreshed = self
|
|
.token_refresher
|
|
.refresh(&existing_refresh_token)
|
|
.await
|
|
.map_err(AdminAccountError::RefreshTokenExchange)?;
|
|
let access_token =
|
|
normalize_nonempty(Some(normalize_bearer_token(refreshed.access_token)))
|
|
.ok_or(AdminAccountError::TokenRequired)?;
|
|
refresh_token = refreshed.refresh_token.or(Some(existing_refresh_token));
|
|
let claims = manual_account_claims(&access_token, chrono::Utc::now())
|
|
.map_err(AdminAccountError::InvalidToken)?;
|
|
return Ok(Some(ResolvedImportTokens {
|
|
access_token,
|
|
refresh_token,
|
|
claims: Some(claims),
|
|
}));
|
|
};
|
|
|
|
if let Ok(claims) = manual_account_claims(&access_token, chrono::Utc::now()) {
|
|
return Ok(Some(ResolvedImportTokens {
|
|
access_token,
|
|
refresh_token,
|
|
claims: Some(claims),
|
|
}));
|
|
}
|
|
|
|
let Some(existing_refresh_token) = refresh_token else {
|
|
return Ok(Some(ResolvedImportTokens {
|
|
access_token,
|
|
refresh_token: None,
|
|
claims: None,
|
|
}));
|
|
};
|
|
let refreshed = self
|
|
.token_refresher
|
|
.refresh(&existing_refresh_token)
|
|
.await
|
|
.map_err(AdminAccountError::RefreshTokenExchange)?;
|
|
let access_token = normalize_nonempty(Some(normalize_bearer_token(refreshed.access_token)))
|
|
.ok_or(AdminAccountError::TokenRequired)?;
|
|
refresh_token = refreshed.refresh_token.or(Some(existing_refresh_token));
|
|
let claims = manual_account_claims(&access_token, chrono::Utc::now())
|
|
.map_err(AdminAccountError::InvalidToken)?;
|
|
Ok(Some(ResolvedImportTokens {
|
|
access_token,
|
|
refresh_token,
|
|
claims: Some(claims),
|
|
}))
|
|
}
|
|
|
|
async fn import_target_account_id(
|
|
&self,
|
|
id: Option<&str>,
|
|
claims: Option<&ManualAccountClaims>,
|
|
account_id: Option<&str>,
|
|
user_id: Option<&str>,
|
|
) -> Result<String, AdminAccountError> {
|
|
let provided_id = normalize_nonempty_str(id).map(ToString::to_string);
|
|
if let Some(id) = provided_id.as_deref() {
|
|
match self.store.get(id).await {
|
|
Ok(Some(_)) => return Ok(id.to_string()),
|
|
Ok(None) => {}
|
|
Err(_) => return Err(AdminAccountError::Inspect),
|
|
}
|
|
}
|
|
|
|
let chatgpt_account_id = claims
|
|
.map(|claims| claims.account_id.as_str())
|
|
.or_else(|| normalize_nonempty_str(account_id));
|
|
let chatgpt_user_id = claims
|
|
.and_then(|claims| claims.user_id.as_deref())
|
|
.or_else(|| normalize_nonempty_str(user_id));
|
|
if let Some(chatgpt_account_id) = chatgpt_account_id {
|
|
if let Some(existing) = self
|
|
.store
|
|
.find_by_chatgpt_identity(chatgpt_account_id, chatgpt_user_id)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Inspect)?
|
|
{
|
|
return Ok(existing.id);
|
|
}
|
|
}
|
|
|
|
Ok(provided_id.unwrap_or_else(|| normalized_account_id(None)))
|
|
}
|
|
|
|
/// 更新账号标签。
|
|
pub async fn update_label(
|
|
&self,
|
|
account_id: &str,
|
|
label: Option<String>,
|
|
) -> Result<bool, AdminAccountError> {
|
|
if label
|
|
.as_ref()
|
|
.is_some_and(|label| label.chars().count() > 64)
|
|
{
|
|
return Err(AdminAccountError::LabelTooLong);
|
|
}
|
|
let updated = self
|
|
.store
|
|
.set_label(account_id, label)
|
|
.await
|
|
.map_err(|_| AdminAccountError::UpdateLabel)?;
|
|
if updated {
|
|
self.sync_account_pool_best_effort(account_id, "account label update")
|
|
.await;
|
|
}
|
|
Ok(updated)
|
|
}
|
|
|
|
/// 更新账号状态。
|
|
pub async fn update_status(
|
|
&self,
|
|
account_id: &str,
|
|
status: &str,
|
|
) -> Result<Option<UpdatedAccountStatus>, AdminAccountError> {
|
|
let status = parse_account_status(status)?;
|
|
match self.store.set_status(account_id, status).await {
|
|
Ok(true) => {
|
|
self.sync_account_pool_best_effort(account_id, "account status update")
|
|
.await;
|
|
self.evict_account_websocket_pool(account_id).await;
|
|
Ok(Some(UpdatedAccountStatus {
|
|
id: account_id.to_string(),
|
|
status,
|
|
}))
|
|
}
|
|
Ok(false) => Ok(None),
|
|
Err(_) => Err(AdminAccountError::UpdateStatus),
|
|
}
|
|
}
|
|
|
|
/// 删除账号。
|
|
pub async fn delete(&self, account_id: &str) -> Result<bool, AdminAccountError> {
|
|
let deleted = self
|
|
.store
|
|
.delete(account_id)
|
|
.await
|
|
.map_err(|_| AdminAccountError::Delete)?;
|
|
if deleted {
|
|
self.account_pool.remove_account(account_id).await;
|
|
}
|
|
Ok(deleted)
|
|
}
|
|
|
|
/// 批量删除账号。
|
|
pub async fn batch_delete(
|
|
&self,
|
|
ids: Vec<String>,
|
|
) -> Result<BatchDeleteAccounts, AdminAccountError> {
|
|
if ids.is_empty() {
|
|
return Err(AdminAccountError::EmptyIds);
|
|
}
|
|
|
|
let mut deleted = 0u32;
|
|
let mut not_found = Vec::new();
|
|
for id in ids {
|
|
match self.store.delete(&id).await {
|
|
Ok(true) => {
|
|
deleted += 1;
|
|
self.account_pool.remove_account(&id).await;
|
|
}
|
|
Ok(false) => not_found.push(id),
|
|
Err(_) => return Err(AdminAccountError::Delete),
|
|
}
|
|
}
|
|
|
|
Ok(BatchDeleteAccounts { deleted, not_found })
|
|
}
|
|
|
|
/// 批量更新账号状态。
|
|
pub async fn batch_update_status(
|
|
&self,
|
|
ids: Vec<String>,
|
|
status: &str,
|
|
) -> Result<BatchUpdateAccountStatus, AdminAccountError> {
|
|
if ids.is_empty() {
|
|
return Err(AdminAccountError::EmptyIds);
|
|
}
|
|
let status = parse_batch_account_status(status)?;
|
|
|
|
let mut updated = 0u32;
|
|
let mut not_found = Vec::new();
|
|
for id in ids {
|
|
match self.store.set_status(&id, status).await {
|
|
Ok(true) => {
|
|
updated += 1;
|
|
self.sync_account_pool_best_effort(&id, "account batch status update")
|
|
.await;
|
|
self.evict_account_websocket_pool(&id).await;
|
|
}
|
|
Ok(false) => not_found.push(id),
|
|
Err(_) => return Err(AdminAccountError::UpdateStatus),
|
|
}
|
|
}
|
|
|
|
Ok(BatchUpdateAccountStatus { updated, not_found })
|
|
}
|
|
|
|
/// 使用账号 refresh token 刷新 access token。
|
|
pub async fn refresh_account(
|
|
&self,
|
|
account_id: &str,
|
|
) -> Result<AdminAccountRefresh, AdminAccountError> {
|
|
let account = match self.store.get(account_id).await {
|
|
Ok(Some(account)) => account,
|
|
Ok(None) => return Err(AdminAccountError::NotFound),
|
|
Err(_) => return Err(AdminAccountError::Inspect),
|
|
};
|
|
let previous_status = account.status;
|
|
let Some(refresh_token) = account.refresh_token.as_ref() else {
|
|
return Err(AdminAccountError::TokenRequired);
|
|
};
|
|
|
|
match self
|
|
.token_refresher
|
|
.refresh(refresh_token.expose_secret())
|
|
.await
|
|
{
|
|
Ok(tokens) => {
|
|
let access_token =
|
|
normalize_nonempty(Some(normalize_bearer_token(tokens.access_token)))
|
|
.ok_or(AdminAccountError::TokenRequired)?;
|
|
let claims = manual_account_claims(&access_token, chrono::Utc::now())
|
|
.map_err(AdminAccountError::InvalidToken)?;
|
|
let updated = self
|
|
.store
|
|
.update_from_claims(
|
|
account_id,
|
|
AccountClaimsUpdate {
|
|
email: claims.email,
|
|
account_id: Some(claims.account_id),
|
|
user_id: claims.user_id,
|
|
plan_type: claims.plan_type,
|
|
access_token: SecretString::new(access_token.into()),
|
|
refresh_token: tokens
|
|
.refresh_token
|
|
.map(|token| SecretString::new(token.into())),
|
|
access_token_expires_at: Some(claims.expires_at),
|
|
next_refresh_at: Some(
|
|
self.next_refresh_at_for_expires_at(claims.expires_at),
|
|
),
|
|
status: AccountStatus::Active,
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|_| AdminAccountError::UpdateClaims)?;
|
|
if !updated {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
self.sync_account_pool(account_id).await?;
|
|
Ok(AdminAccountRefresh {
|
|
id: account_id.to_string(),
|
|
previous_status,
|
|
outcome: AdminAccountProbeOutcome::Alive,
|
|
status: Some(AccountStatus::Active),
|
|
error: None,
|
|
})
|
|
}
|
|
Err(failure) => {
|
|
let status = refresh_failure_status(failure);
|
|
let updated = self
|
|
.store
|
|
.set_status(account_id, status)
|
|
.await
|
|
.map_err(|_| AdminAccountError::UpdateStatus)?;
|
|
if !updated {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
if refresh_failure_status_clears_next_refresh_at(status) {
|
|
let cleared = self
|
|
.store
|
|
.set_next_refresh_at(account_id, None)
|
|
.await
|
|
.map_err(|_| AdminAccountError::UpdateStatus)?;
|
|
if !cleared {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
}
|
|
self.sync_account_pool_best_effort(account_id, "account refresh failure")
|
|
.await;
|
|
Ok(AdminAccountRefresh {
|
|
id: account_id.to_string(),
|
|
previous_status,
|
|
outcome: AdminAccountProbeOutcome::Dead,
|
|
status: Some(status),
|
|
error: Some(failure.to_string()),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 重置账号本地用量计数。
|
|
pub async fn reset_usage(
|
|
&self,
|
|
account_id: &str,
|
|
) -> Result<AdminAccountResetUsage, AdminAccountError> {
|
|
match self.store.get(account_id).await {
|
|
Ok(Some(_)) => {}
|
|
Ok(None) => return Err(AdminAccountError::NotFound),
|
|
Err(_) => return Err(AdminAccountError::Inspect),
|
|
}
|
|
|
|
self.store
|
|
.reset_usage(account_id)
|
|
.await
|
|
.map_err(|_| AdminAccountError::ResetUsage)?;
|
|
self.sync_account_pool(account_id).await?;
|
|
|
|
Ok(AdminAccountResetUsage {
|
|
id: account_id.to_string(),
|
|
reset: true,
|
|
})
|
|
}
|
|
|
|
/// 读取账号 Cookie 请求头。
|
|
pub async fn cookies(&self, account_id: &str) -> Result<Option<String>, AdminAccountError> {
|
|
self.ensure_cookie_account_exists(account_id).await?;
|
|
self.cookies
|
|
.cookie_header(account_id, "chatgpt.com")
|
|
.await
|
|
.map_err(|_| AdminAccountError::LoadCookies)
|
|
}
|
|
|
|
/// 设置账号 Cookie 请求头。
|
|
pub async fn set_cookies(
|
|
&self,
|
|
account_id: &str,
|
|
cookie_header: &str,
|
|
) -> Result<Option<String>, AdminAccountError> {
|
|
self.ensure_cookie_account_exists(account_id).await?;
|
|
match self
|
|
.cookies
|
|
.set_cookie_header(account_id, cookie_header)
|
|
.await
|
|
{
|
|
Ok(0) => Err(AdminAccountError::NoValidCookies),
|
|
Ok(_) => self
|
|
.cookies
|
|
.cookie_header(account_id, "chatgpt.com")
|
|
.await
|
|
.map_err(|_| AdminAccountError::LoadCookies),
|
|
Err(_) => Err(AdminAccountError::StoreCookies),
|
|
}
|
|
}
|
|
|
|
/// 删除账号 Cookie。
|
|
pub async fn delete_cookies(&self, account_id: &str) -> Result<(), AdminAccountError> {
|
|
self.ensure_cookie_account_exists(account_id).await?;
|
|
self.cookies
|
|
.delete_account_cookies(account_id)
|
|
.await
|
|
.map(|_| ())
|
|
.map_err(|_| AdminAccountError::DeleteCookies)
|
|
}
|
|
|
|
/// 返回基于缓存配额快照的账号预警。
|
|
pub async fn quota_warnings(&self) -> Result<AdminAccountQuotaWarnings, AdminAccountError> {
|
|
let snapshots = self
|
|
.store
|
|
.list_quota_snapshots()
|
|
.await
|
|
.map_err(|_| AdminAccountError::QuotaWarnings)?;
|
|
Ok(quota_warnings_from_snapshots(
|
|
snapshots,
|
|
&self.quota_thresholds,
|
|
))
|
|
}
|
|
|
|
/// 拉取并持久化单个账号的 Codex usage 配额快照。
|
|
pub async fn account_quota(
|
|
&self,
|
|
account_id: &str,
|
|
request_id: &str,
|
|
) -> Result<AdminAccountQuota, AdminAccountError> {
|
|
let account = match self.store.get(account_id).await {
|
|
Ok(Some(account)) => account,
|
|
Ok(None) => return Err(AdminAccountError::NotFound),
|
|
Err(_) => return Err(AdminAccountError::Inspect),
|
|
};
|
|
if account.status != AccountStatus::Active {
|
|
return Err(AdminAccountError::Inactive(account.status));
|
|
}
|
|
|
|
let raw = self
|
|
.codex
|
|
.fetch_usage(CodexRequestContext {
|
|
access_token: account.access_token.expose_secret(),
|
|
account_id: account.account_id.as_deref(),
|
|
request_id,
|
|
turn_state: None,
|
|
turn_metadata: None,
|
|
beta_features: None,
|
|
include_timing_metrics: None,
|
|
version: None,
|
|
codex_window_id: None,
|
|
parent_thread_id: None,
|
|
cookie_header: None,
|
|
installation_id: self.installation_id.as_deref(),
|
|
session_id: None,
|
|
})
|
|
.await
|
|
.map_err(|error| AdminAccountError::FetchQuota(error.to_string()))?;
|
|
let quota = quota_from_usage(&raw);
|
|
let reset_at = quota_snapshot_reset_at("a);
|
|
let updated = self
|
|
.store
|
|
.apply_quota_snapshot(
|
|
&account.id,
|
|
"a.to_string(),
|
|
quota_snapshot_limit_reached("a),
|
|
quota_snapshot_limit_reached("a)
|
|
.then_some(reset_at)
|
|
.flatten(),
|
|
)
|
|
.await
|
|
.map_err(|_| AdminAccountError::StoreQuota)?;
|
|
if !updated {
|
|
return Err(AdminAccountError::NotFound);
|
|
}
|
|
if let Some(reset_at) = reset_at {
|
|
self.store
|
|
.sync_rate_limit_window(
|
|
&account.id,
|
|
reset_at,
|
|
quota_snapshot_limit_window_seconds("a),
|
|
)
|
|
.await
|
|
.map_err(|_| AdminAccountError::StoreQuota)?;
|
|
}
|
|
|
|
Ok(AdminAccountQuota { quota, raw })
|
|
}
|
|
|
|
/// 对账号执行 refresh-token 健康探测。
|
|
pub async fn health_check_accounts(
|
|
&self,
|
|
ids: Option<Vec<String>>,
|
|
concurrency: usize,
|
|
stagger_ms: u64,
|
|
_request_id: &str,
|
|
) -> Result<Vec<AdminAccountProbeResult>, AdminAccountError> {
|
|
let accounts = self.health_check_candidates(ids).await?;
|
|
let concurrency = concurrency.max(1);
|
|
let results = stream::iter(accounts.into_iter().enumerate())
|
|
.map(|(index, account)| {
|
|
let service = self.clone();
|
|
async move {
|
|
if stagger_ms > 0 && index > 0 {
|
|
let multiplier = index.min(concurrency);
|
|
tokio::time::sleep(std::time::Duration::from_millis(
|
|
stagger_ms.saturating_mul(multiplier as u64),
|
|
))
|
|
.await;
|
|
}
|
|
service.probe_account_refresh(account).await
|
|
}
|
|
})
|
|
.buffer_unordered(concurrency)
|
|
.collect::<Vec<_>>()
|
|
.await;
|
|
Ok(results)
|
|
}
|
|
|
|
async fn health_check_candidates(
|
|
&self,
|
|
ids: Option<Vec<String>>,
|
|
) -> Result<Vec<StoredAccount>, AdminAccountError> {
|
|
if let Some(ids) = ids {
|
|
let mut accounts = Vec::with_capacity(ids.len());
|
|
for id in ids {
|
|
match self.store.get(&id).await {
|
|
Ok(Some(account)) => accounts.push(account),
|
|
Ok(None) => {}
|
|
Err(_) => return Err(AdminAccountError::HealthCheck),
|
|
}
|
|
}
|
|
return Ok(accounts);
|
|
}
|
|
|
|
let mut accounts = Vec::new();
|
|
let mut cursor = None;
|
|
loop {
|
|
let page = self
|
|
.store
|
|
.list(cursor, 200)
|
|
.await
|
|
.map_err(|_| AdminAccountError::HealthCheck)?;
|
|
accounts.extend(page.items);
|
|
if page.next_cursor.is_none() {
|
|
return Ok(accounts);
|
|
}
|
|
cursor = page.next_cursor;
|
|
}
|
|
}
|
|
|
|
async fn probe_account_refresh(&self, account: StoredAccount) -> AdminAccountProbeResult {
|
|
let started_at = Instant::now();
|
|
let previous_status = account.status;
|
|
if account.status == AccountStatus::Disabled {
|
|
return skipped_admin_account_probe_result(account, "manually disabled");
|
|
}
|
|
let Some(refresh_token) = account.refresh_token.as_ref() else {
|
|
return skipped_admin_account_probe_result(account, "no refresh token");
|
|
};
|
|
|
|
match self
|
|
.token_refresher
|
|
.refresh(refresh_token.expose_secret())
|
|
.await
|
|
{
|
|
Ok(tokens) => {
|
|
let Some(access_token) =
|
|
normalize_nonempty(Some(normalize_bearer_token(tokens.access_token)))
|
|
else {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
"token or refreshToken is required".to_string(),
|
|
started_at,
|
|
);
|
|
};
|
|
let claims = match manual_account_claims(&access_token, Utc::now()) {
|
|
Ok(claims) => claims,
|
|
Err(error) => {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
error.to_string(),
|
|
started_at,
|
|
);
|
|
}
|
|
};
|
|
|
|
match self
|
|
.store
|
|
.update_from_claims(
|
|
&account.id,
|
|
AccountClaimsUpdate {
|
|
email: claims.email,
|
|
account_id: Some(claims.account_id),
|
|
user_id: claims.user_id,
|
|
plan_type: claims.plan_type,
|
|
access_token: SecretString::new(access_token.into()),
|
|
refresh_token: tokens
|
|
.refresh_token
|
|
.map(|token| SecretString::new(token.into())),
|
|
access_token_expires_at: Some(claims.expires_at),
|
|
next_refresh_at: Some(
|
|
self.next_refresh_at_for_expires_at(claims.expires_at),
|
|
),
|
|
status: AccountStatus::Active,
|
|
},
|
|
)
|
|
.await
|
|
{
|
|
Ok(true) => {
|
|
self.sync_account_pool_best_effort(&account.id, "account health refresh")
|
|
.await;
|
|
AdminAccountProbeResult {
|
|
id: account.id,
|
|
email: account.email,
|
|
previous_status,
|
|
outcome: AdminAccountProbeOutcome::Alive,
|
|
status: Some(AccountStatus::Active),
|
|
error: None,
|
|
duration_ms: Some(started_at.elapsed().as_millis()),
|
|
}
|
|
}
|
|
Ok(false) => dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::NotFound.to_string(),
|
|
started_at,
|
|
),
|
|
Err(_) => dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::UpdateClaims.to_string(),
|
|
started_at,
|
|
),
|
|
}
|
|
}
|
|
Err(failure) => {
|
|
let status = health_check_failure_status(failure);
|
|
if let Some(status) = status {
|
|
match self.store.set_status(&account.id, status).await {
|
|
Ok(true) => {
|
|
if refresh_failure_status_clears_next_refresh_at(status) {
|
|
match self.store.set_next_refresh_at(&account.id, None).await {
|
|
Ok(true) => {}
|
|
Ok(false) => {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::NotFound.to_string(),
|
|
started_at,
|
|
);
|
|
}
|
|
Err(_) => {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::UpdateStatus.to_string(),
|
|
started_at,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
self.sync_account_pool_best_effort(
|
|
&account.id,
|
|
"account health refresh failure",
|
|
)
|
|
.await;
|
|
}
|
|
Ok(false) => {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::NotFound.to_string(),
|
|
started_at,
|
|
);
|
|
}
|
|
Err(_) => {
|
|
return dead_admin_account_probe_result(
|
|
account,
|
|
previous_status,
|
|
AdminAccountError::UpdateStatus.to_string(),
|
|
started_at,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
AdminAccountProbeResult {
|
|
id: account.id,
|
|
email: account.email,
|
|
previous_status,
|
|
outcome: AdminAccountProbeOutcome::Dead,
|
|
status,
|
|
error: Some(failure.to_string()),
|
|
duration_ms: Some(started_at.elapsed().as_millis()),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn ensure_cookie_account_exists(
|
|
&self,
|
|
account_id: &str,
|
|
) -> Result<(), AdminAccountError> {
|
|
match self.cookies.account_exists(account_id).await {
|
|
Ok(true) => Ok(()),
|
|
Ok(false) => Err(AdminAccountError::NotFound),
|
|
Err(_) => Err(AdminAccountError::Inspect),
|
|
}
|
|
}
|
|
|
|
async fn sync_account_pool(&self, account_id: &str) -> Result<(), AdminAccountError> {
|
|
self.account_pool
|
|
.sync_account_from_repository(account_id)
|
|
.await
|
|
.map(|_| ())
|
|
.map_err(|_| AdminAccountError::SyncAccountPool)
|
|
}
|
|
|
|
async fn sync_account_pool_best_effort(&self, account_id: &str, operation: &str) {
|
|
if let Err(error) = self
|
|
.account_pool
|
|
.sync_account_from_repository(account_id)
|
|
.await
|
|
{
|
|
tracing::warn!(
|
|
account_id,
|
|
operation,
|
|
error = %error,
|
|
"failed to sync runtime account pool after admin account update"
|
|
);
|
|
}
|
|
}
|
|
|
|
async fn evict_account_websocket_pool(&self, account_id: &str) {
|
|
self.codex.evict_websocket_account(account_id).await;
|
|
match self.store.get(account_id).await {
|
|
Ok(Some(account)) => {
|
|
if let Some(upstream_account_id) = account
|
|
.account_id
|
|
.as_deref()
|
|
.filter(|value| *value != account_id)
|
|
{
|
|
self.codex
|
|
.evict_websocket_account(upstream_account_id)
|
|
.await;
|
|
}
|
|
}
|
|
Ok(None) => {}
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
account_id,
|
|
error = %error,
|
|
"failed to inspect account while evicting websocket pool"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn skipped_admin_account_probe_result(
|
|
account: StoredAccount,
|
|
error: &str,
|
|
) -> AdminAccountProbeResult {
|
|
AdminAccountProbeResult {
|
|
id: account.id,
|
|
email: account.email,
|
|
previous_status: account.status,
|
|
outcome: AdminAccountProbeOutcome::Skipped,
|
|
status: Some(account.status),
|
|
error: Some(error.to_string()),
|
|
duration_ms: None,
|
|
}
|
|
}
|
|
|
|
fn dead_admin_account_probe_result(
|
|
account: StoredAccount,
|
|
previous_status: AccountStatus,
|
|
error: String,
|
|
started_at: Instant,
|
|
) -> AdminAccountProbeResult {
|
|
AdminAccountProbeResult {
|
|
id: account.id,
|
|
email: account.email,
|
|
previous_status,
|
|
outcome: AdminAccountProbeOutcome::Dead,
|
|
status: None,
|
|
error: Some(error),
|
|
duration_ms: Some(started_at.elapsed().as_millis()),
|
|
}
|
|
}
|
|
|
|
fn health_check_failure_status(failure: RefreshFailure) -> Option<AccountStatus> {
|
|
match failure {
|
|
RefreshFailure::InvalidGrant => Some(AccountStatus::Disabled),
|
|
RefreshFailure::QuotaExhausted => Some(AccountStatus::QuotaExhausted),
|
|
RefreshFailure::Banned => Some(AccountStatus::Banned),
|
|
RefreshFailure::Disabled => Some(AccountStatus::Disabled),
|
|
RefreshFailure::RetryableTransport => None,
|
|
RefreshFailure::Transport => None,
|
|
}
|
|
}
|
|
|
|
/// 管理端账号错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminAccountError {
|
|
/// 列表失败。
|
|
#[error("failed to list accounts")]
|
|
List,
|
|
/// 导出失败。
|
|
#[error("failed to export accounts")]
|
|
Export,
|
|
/// 导入失败。
|
|
#[error("failed to import accounts")]
|
|
Import,
|
|
/// 检查账号失败。
|
|
#[error("failed to inspect account")]
|
|
Inspect,
|
|
/// 更新标签失败。
|
|
#[error("failed to update account label")]
|
|
UpdateLabel,
|
|
/// 更新状态失败。
|
|
#[error("failed to update account status")]
|
|
UpdateStatus,
|
|
/// 删除失败。
|
|
#[error("failed to delete account")]
|
|
Delete,
|
|
/// 重置用量失败。
|
|
#[error("failed to reset account usage")]
|
|
ResetUsage,
|
|
/// 账号不存在。
|
|
#[error("account not found")]
|
|
NotFound,
|
|
/// 读取 Cookie 失败。
|
|
#[error("failed to load account cookies")]
|
|
LoadCookies,
|
|
/// 写入 Cookie 失败。
|
|
#[error("failed to store account cookies")]
|
|
StoreCookies,
|
|
/// 删除 Cookie 失败。
|
|
#[error("failed to delete account cookies")]
|
|
DeleteCookies,
|
|
/// 根据 JWT claims 更新账号失败。
|
|
#[error("failed to update account claims")]
|
|
UpdateClaims,
|
|
/// 读取配额预警失败。
|
|
#[error("failed to load account quota warnings")]
|
|
QuotaWarnings,
|
|
/// 写入配额快照失败。
|
|
#[error("failed to store account quota")]
|
|
StoreQuota,
|
|
/// 拉取配额失败。
|
|
#[error("failed to fetch account quota: {0}")]
|
|
FetchQuota(String),
|
|
/// 健康检查失败。
|
|
#[error("failed to health-check accounts")]
|
|
HealthCheck,
|
|
/// 账号非 active,不能执行需要上游访问的操作。
|
|
#[error("account is {0:?}, cannot query quota")]
|
|
Inactive(AccountStatus),
|
|
/// token 为空。
|
|
#[error("token or refreshToken is required")]
|
|
TokenRequired,
|
|
/// token 非法。
|
|
#[error("{0}")]
|
|
InvalidToken(&'static str),
|
|
/// refresh token 换取 access token 失败。
|
|
#[error("failed to exchange refreshToken: {0}")]
|
|
RefreshTokenExchange(RefreshFailure),
|
|
/// 同步运行时账号池失败。
|
|
#[error("failed to sync runtime account pool")]
|
|
SyncAccountPool,
|
|
/// 没有有效 Cookie。
|
|
#[error("No valid cookies found")]
|
|
NoValidCookies,
|
|
/// 标签过长。
|
|
#[error("account label must be 64 characters or fewer")]
|
|
LabelTooLong,
|
|
/// 状态值无效。
|
|
#[error("unsupported account status: {0}")]
|
|
InvalidStatus(String),
|
|
/// ID 列表为空。
|
|
#[error("account ids are required")]
|
|
EmptyIds,
|
|
/// 没有可导入账号。
|
|
#[error("No importable accounts found")]
|
|
NoImportableAccounts,
|
|
/// access token 过期时间非法。
|
|
#[error("invalid accessTokenExpiresAt")]
|
|
InvalidAccessTokenExpiresAt,
|
|
}
|
|
|
|
/// 管理端账号元数据。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminAccountMetadata {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 邮箱。
|
|
pub email: Option<String>,
|
|
/// ChatGPT 账号 ID。
|
|
pub account_id: Option<String>,
|
|
/// ChatGPT 用户 ID。
|
|
pub user_id: Option<String>,
|
|
/// 管理标签。
|
|
pub label: Option<String>,
|
|
/// 计划类型。
|
|
pub plan_type: Option<String>,
|
|
/// access token 过期时间。
|
|
pub access_token_expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
|
/// 账号状态。
|
|
pub status: codex_proxy_core::accounts::model::AccountStatus,
|
|
/// 创建时间。
|
|
pub added_at: chrono::DateTime<chrono::Utc>,
|
|
/// 更新时间。
|
|
pub updated_at: chrono::DateTime<chrono::Utc>,
|
|
}
|
|
|
|
/// 管理端账号配额拉取结果。
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct AdminAccountQuota {
|
|
/// 归一化后的配额快照。
|
|
pub quota: Value,
|
|
/// Codex usage 原始响应。
|
|
pub raw: Value,
|
|
}
|
|
|
|
/// 管理端可导出的完整账号数据。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminStoredAccount {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 邮箱。
|
|
pub email: Option<String>,
|
|
/// ChatGPT 账号 ID。
|
|
pub account_id: Option<String>,
|
|
/// ChatGPT 用户 ID。
|
|
pub user_id: Option<String>,
|
|
/// 管理标签。
|
|
pub label: Option<String>,
|
|
/// 计划类型。
|
|
pub plan_type: Option<String>,
|
|
/// access token 明文。
|
|
pub access_token: String,
|
|
/// refresh token 明文。
|
|
pub refresh_token: Option<String>,
|
|
/// access token 过期时间。
|
|
pub access_token_expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
|
/// 账号状态。
|
|
pub status: codex_proxy_core::accounts::model::AccountStatus,
|
|
/// 创建时间。
|
|
pub added_at: chrono::DateTime<chrono::Utc>,
|
|
/// 更新时间。
|
|
pub updated_at: chrono::DateTime<chrono::Utc>,
|
|
}
|
|
|
|
/// 管理端账号配额预警集合。
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct AdminAccountQuotaWarnings {
|
|
/// 预警列表。
|
|
pub warnings: Vec<AdminAccountQuotaWarning>,
|
|
/// 产生预警的快照中最新的拉取时间。
|
|
pub updated_at: Option<chrono::DateTime<chrono::Utc>>,
|
|
}
|
|
|
|
/// 管理端账号配额预警。
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct AdminAccountQuotaWarning {
|
|
/// 账号 ID。
|
|
pub account_id: String,
|
|
/// 邮箱。
|
|
pub email: Option<String>,
|
|
/// 配额窗口。
|
|
pub window: AdminQuotaWarningWindow,
|
|
/// 预警级别。
|
|
pub level: AdminQuotaWarningLevel,
|
|
/// 已使用百分比。
|
|
pub used_percent: f64,
|
|
/// 重置时间戳。
|
|
pub reset_at: Option<i64>,
|
|
}
|
|
|
|
/// 配额预警窗口。
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum AdminQuotaWarningWindow {
|
|
/// 主窗口。
|
|
Primary,
|
|
/// 次窗口。
|
|
Secondary,
|
|
}
|
|
|
|
impl AdminQuotaWarningWindow {
|
|
/// 返回 API 字符串值。
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Primary => "primary",
|
|
Self::Secondary => "secondary",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 配额预警级别。
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum AdminQuotaWarningLevel {
|
|
/// 普通预警。
|
|
Warning,
|
|
/// 临界预警。
|
|
Critical,
|
|
}
|
|
|
|
impl AdminQuotaWarningLevel {
|
|
/// 返回 API 字符串值。
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Warning => "warning",
|
|
Self::Critical => "critical",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 账号健康探测结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminAccountProbeResult {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 邮箱。
|
|
pub email: Option<String>,
|
|
/// 探测前状态。
|
|
pub previous_status: AccountStatus,
|
|
/// 探测结果。
|
|
pub outcome: AdminAccountProbeOutcome,
|
|
/// 探测后的状态。
|
|
pub status: Option<AccountStatus>,
|
|
/// 错误信息。
|
|
pub error: Option<String>,
|
|
/// 耗时毫秒。
|
|
pub duration_ms: Option<u128>,
|
|
}
|
|
|
|
/// 账号健康探测结果类型。
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum AdminAccountProbeOutcome {
|
|
/// 上游 usage 请求成功。
|
|
Alive,
|
|
/// 上游 usage 请求失败。
|
|
Dead,
|
|
/// 未执行上游探测。
|
|
Skipped,
|
|
}
|
|
|
|
impl AdminAccountProbeOutcome {
|
|
/// 返回 API 字符串值。
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Alive => "alive",
|
|
Self::Dead => "dead",
|
|
Self::Skipped => "skipped",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 账号导入结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct ImportedAccounts {
|
|
/// 成功导入数量。
|
|
pub imported: u32,
|
|
/// 跳过数量。
|
|
pub skipped: u32,
|
|
/// 导入适配器格式。
|
|
pub source_format: &'static str,
|
|
}
|
|
|
|
/// 管理端认证状态摘要。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminAuthStatus {
|
|
/// 是否存在已导入账号。
|
|
pub authenticated: bool,
|
|
/// 当前可展示的 active 账号元数据。
|
|
pub user: Option<AdminAccountMetadata>,
|
|
/// 账号池状态计数。
|
|
pub pool: AdminAuthPoolStatus,
|
|
}
|
|
|
|
/// 管理端认证状态中的账号池计数。
|
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
|
pub struct AdminAuthPoolStatus {
|
|
/// 账号总数。
|
|
pub total: u32,
|
|
/// active 账号数。
|
|
pub active: u32,
|
|
/// expired 账号数。
|
|
pub expired: u32,
|
|
/// quota_exhausted 账号数。
|
|
pub quota_exhausted: u32,
|
|
/// refreshing 账号数。
|
|
pub refreshing: u32,
|
|
/// disabled 账号数。
|
|
pub disabled: u32,
|
|
/// banned 账号数。
|
|
pub banned: u32,
|
|
}
|
|
|
|
impl AdminAuthPoolStatus {
|
|
fn record(&mut self, status: AccountStatus) {
|
|
self.total += 1;
|
|
match status {
|
|
AccountStatus::Active => self.active += 1,
|
|
AccountStatus::Expired => self.expired += 1,
|
|
AccountStatus::QuotaExhausted => self.quota_exhausted += 1,
|
|
AccountStatus::Refreshing => self.refreshing += 1,
|
|
AccountStatus::Disabled => self.disabled += 1,
|
|
AccountStatus::Banned => self.banned += 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// 管理端登出结果。
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub struct AdminAuthLogout {
|
|
/// 是否成功。
|
|
pub success: bool,
|
|
/// 删除账号数。
|
|
pub deleted: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ManualCreateTokens {
|
|
access_token: String,
|
|
refresh_token_for_new: Option<String>,
|
|
refresh_token_for_existing: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct AccountImportEntry {
|
|
id: Option<String>,
|
|
email: Option<String>,
|
|
account_id: Option<String>,
|
|
user_id: Option<String>,
|
|
label: Option<String>,
|
|
plan_type: Option<String>,
|
|
token: Option<String>,
|
|
refresh_token: Option<String>,
|
|
access_token_expires_at: Option<String>,
|
|
status: Option<String>,
|
|
cached_quota: Option<Value>,
|
|
quota_fetched_at: Option<String>,
|
|
quota_verify_required: Option<bool>,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ResolvedImportTokens {
|
|
access_token: String,
|
|
refresh_token: Option<String>,
|
|
claims: Option<ManualAccountClaims>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
enum ImportedAccountState {
|
|
Imported(String),
|
|
Skipped,
|
|
}
|
|
|
|
/// 账号状态更新结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct UpdatedAccountStatus {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 新状态。
|
|
pub status: codex_proxy_core::accounts::model::AccountStatus,
|
|
}
|
|
|
|
/// 批量删除账号结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct BatchDeleteAccounts {
|
|
/// 成功删除数量。
|
|
pub deleted: u32,
|
|
/// 未找到的账号 ID。
|
|
pub not_found: Vec<String>,
|
|
}
|
|
|
|
/// 批量状态更新结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct BatchUpdateAccountStatus {
|
|
/// 成功更新数量。
|
|
pub updated: u32,
|
|
/// 未找到的账号 ID。
|
|
pub not_found: Vec<String>,
|
|
}
|
|
|
|
/// 管理端手动刷新账号结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminAccountRefresh {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 刷新前状态。
|
|
pub previous_status: codex_proxy_core::accounts::model::AccountStatus,
|
|
/// 刷新结果。
|
|
pub outcome: AdminAccountProbeOutcome,
|
|
/// 刷新后状态。
|
|
pub status: Option<codex_proxy_core::accounts::model::AccountStatus>,
|
|
/// 错误信息。
|
|
pub error: Option<String>,
|
|
}
|
|
|
|
/// 管理端重置用量结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminAccountResetUsage {
|
|
/// 账号 ID。
|
|
pub id: String,
|
|
/// 是否已处理。
|
|
pub reset: bool,
|
|
}
|
|
|
|
/// 运行时设置服务。
|
|
#[derive(Clone)]
|
|
pub struct RuntimeSettingsService {
|
|
current: Arc<StdRwLock<Arc<AppConfig>>>,
|
|
local_config_path: Arc<PathBuf>,
|
|
}
|
|
|
|
impl RuntimeSettingsService {
|
|
/// 构造运行时设置服务。
|
|
pub fn new(config: AppConfig) -> Self {
|
|
Self::with_local_config_path(config, "local.yaml")
|
|
}
|
|
|
|
/// 构造带本地配置覆盖路径的运行时设置服务。
|
|
pub fn with_local_config_path(
|
|
config: AppConfig,
|
|
local_config_path: impl Into<PathBuf>,
|
|
) -> Self {
|
|
Self {
|
|
current: Arc::new(StdRwLock::new(Arc::new(config))),
|
|
local_config_path: Arc::new(local_config_path.into()),
|
|
}
|
|
}
|
|
|
|
/// 返回当前配置快照。
|
|
pub fn current(&self) -> Arc<AppConfig> {
|
|
self.current
|
|
.read()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
|
.clone()
|
|
}
|
|
|
|
/// Return the local settings overlay path configured for this runtime.
|
|
pub fn local_config_path(&self) -> Arc<PathBuf> {
|
|
self.local_config_path.clone()
|
|
}
|
|
|
|
/// 更新当前设置并写入本地配置覆盖文件。
|
|
pub async fn update(
|
|
&self,
|
|
patch: AdminSettingsPatch,
|
|
) -> Result<Arc<AppConfig>, RuntimeSettingsError> {
|
|
let mut next = (*self.current()).clone();
|
|
let mut settings = admin_settings_from_config(&next);
|
|
SettingsService::apply_patch(&mut settings, patch)?;
|
|
apply_admin_settings_to_config(&mut next, settings);
|
|
next.write_settings_overlay(self.local_config_path.as_ref())
|
|
.await?;
|
|
let next = Arc::new(next);
|
|
*self
|
|
.current
|
|
.write()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner) = next.clone();
|
|
Ok(next)
|
|
}
|
|
}
|
|
|
|
/// 运行时设置错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum RuntimeSettingsError {
|
|
/// 设置补丁验证失败。
|
|
#[error(transparent)]
|
|
InvalidField(#[from] SettingsServiceError),
|
|
/// 本地配置覆盖写入失败。
|
|
#[error(transparent)]
|
|
Persist(#[from] ConfigWriteError),
|
|
}
|
|
|
|
fn admin_settings_from_config(config: &AppConfig) -> AdminSettings {
|
|
AdminSettings {
|
|
default_model: config.model.default_model.clone(),
|
|
default_reasoning_effort: config.model.default_reasoning_effort.clone(),
|
|
service_tier: config.model.service_tier.clone(),
|
|
model_aliases: config.model.aliases.clone(),
|
|
refresh_enabled: config.auth.refresh_enabled,
|
|
refresh_margin_seconds: config.auth.refresh_margin_seconds,
|
|
refresh_concurrency: config.auth.refresh_concurrency,
|
|
max_concurrent_per_account: config.auth.max_concurrent_per_account,
|
|
request_interval_ms: config.auth.request_interval_ms,
|
|
rotation_strategy: config.auth.rotation_strategy.clone(),
|
|
tier_priority: config.auth.tier_priority.clone(),
|
|
quota_refresh_interval_minutes: config.quota.refresh_interval_minutes,
|
|
quota_warning_thresholds: AdminQuotaWarningThresholds {
|
|
primary: config.quota.warning_thresholds.primary.clone(),
|
|
secondary: config.quota.warning_thresholds.secondary.clone(),
|
|
},
|
|
quota_skip_exhausted: config.quota.skip_exhausted,
|
|
logs_enabled: config.logging.enabled,
|
|
logs_capacity: config.logging.capacity,
|
|
logs_capture_body: config.logging.capture_body,
|
|
usage_history_retention_days: config.usage_stats.history_retention_days,
|
|
}
|
|
}
|
|
|
|
fn apply_admin_settings_to_config(config: &mut AppConfig, settings: AdminSettings) {
|
|
config.model.default_model = settings.default_model;
|
|
config.model.default_reasoning_effort = settings.default_reasoning_effort;
|
|
config.model.service_tier = settings.service_tier;
|
|
config.model.aliases = settings.model_aliases;
|
|
config.auth.refresh_enabled = settings.refresh_enabled;
|
|
config.auth.refresh_margin_seconds = settings.refresh_margin_seconds;
|
|
config.auth.refresh_concurrency = settings.refresh_concurrency;
|
|
config.auth.max_concurrent_per_account = settings.max_concurrent_per_account;
|
|
config.auth.request_interval_ms = settings.request_interval_ms;
|
|
config.auth.rotation_strategy = settings.rotation_strategy;
|
|
config.auth.tier_priority = settings.tier_priority;
|
|
config.quota.refresh_interval_minutes = settings.quota_refresh_interval_minutes;
|
|
config.quota.warning_thresholds = QuotaWarningThresholds {
|
|
primary: settings.quota_warning_thresholds.primary,
|
|
secondary: settings.quota_warning_thresholds.secondary,
|
|
};
|
|
config.quota.skip_exhausted = settings.quota_skip_exhausted;
|
|
config.logging.enabled = settings.logs_enabled;
|
|
config.logging.capacity = settings.logs_capacity;
|
|
config.logging.capture_body = settings.logs_capture_body;
|
|
config.usage_stats.history_retention_days = settings.usage_history_retention_days;
|
|
}
|
|
|
|
/// 管理员会话服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminSessionService {
|
|
store: SqliteAdminSessionStore,
|
|
auth: AdminAuthService,
|
|
default_username: String,
|
|
session_ttl_minutes: u64,
|
|
}
|
|
|
|
impl AdminSessionService {
|
|
/// 构造管理员会话服务。
|
|
pub fn new(
|
|
store: SqliteAdminSessionStore,
|
|
default_username: String,
|
|
session_ttl_minutes: u64,
|
|
) -> Self {
|
|
Self {
|
|
store,
|
|
auth: AdminAuthService::new(default_username.clone()),
|
|
default_username,
|
|
session_ttl_minutes,
|
|
}
|
|
}
|
|
|
|
/// 校验管理员会话是否存在且未过期。
|
|
pub async fn validate(&self, session_id: Option<&str>) -> Result<bool, AdminSessionError> {
|
|
let Some(session_id) = session_id else {
|
|
return Ok(false);
|
|
};
|
|
self.store
|
|
.validate_session(session_id)
|
|
.await
|
|
.map_err(|_| AdminSessionError::Validate)
|
|
}
|
|
|
|
/// 如果还没有管理员用户,则根据配置密码创建默认管理员。
|
|
pub async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminSessionError> {
|
|
let password_hash =
|
|
hash_admin_password(password).map_err(|_| AdminSessionError::HashPassword)?;
|
|
self.store
|
|
.ensure_default_admin(&password_hash)
|
|
.await
|
|
.map_err(|_| AdminSessionError::CreateAdmin)
|
|
}
|
|
|
|
/// 使用管理员用户名和密码创建会话。
|
|
pub async fn login(
|
|
&self,
|
|
username: Option<&str>,
|
|
password: &str,
|
|
) -> Result<Option<AdminLoginSession>, AdminSessionError> {
|
|
let username = username.unwrap_or(&self.default_username);
|
|
if !self.auth.username_matches(username) {
|
|
return Ok(None);
|
|
}
|
|
|
|
let Some(admin) = self
|
|
.store
|
|
.load_first_admin()
|
|
.await
|
|
.map_err(|_| AdminSessionError::LoadAdmin)?
|
|
else {
|
|
return Ok(None);
|
|
};
|
|
let password_matches = verify_admin_password(password, &admin.password_hash)
|
|
.map_err(|_| AdminSessionError::VerifyPassword)?;
|
|
if !password_matches {
|
|
return Ok(None);
|
|
}
|
|
|
|
let session_id = format!("sess_{}", uuid::Uuid::new_v4().simple());
|
|
let ttl_minutes = self.session_ttl_minutes.min(i64::MAX as u64) as i64;
|
|
let expires_at = Utc::now() + Duration::minutes(ttl_minutes);
|
|
self.store
|
|
.create_session(&session_id, &admin.id, expires_at)
|
|
.await
|
|
.map_err(|_| AdminSessionError::CreateSession)?;
|
|
|
|
Ok(Some(AdminLoginSession {
|
|
session_id,
|
|
expires_at,
|
|
}))
|
|
}
|
|
}
|
|
|
|
/// 管理员登录成功后的会话。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminLoginSession {
|
|
/// 会话 ID。
|
|
pub session_id: String,
|
|
/// 过期时间。
|
|
pub expires_at: DateTime<Utc>,
|
|
}
|
|
|
|
/// 管理员会话错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminSessionError {
|
|
/// 会话校验失败。
|
|
#[error("failed to validate admin session")]
|
|
Validate,
|
|
/// 密码哈希失败。
|
|
#[error("failed to hash admin password")]
|
|
HashPassword,
|
|
/// 创建管理员失败。
|
|
#[error("failed to create default admin user")]
|
|
CreateAdmin,
|
|
/// 读取管理员失败。
|
|
#[error("failed to load admin user")]
|
|
LoadAdmin,
|
|
/// 密码校验失败。
|
|
#[error("failed to verify admin password")]
|
|
VerifyPassword,
|
|
/// 创建会话失败。
|
|
#[error("failed to create admin session")]
|
|
CreateSession,
|
|
}
|
|
|
|
/// 管理端客户端 API Key 服务。
|
|
#[derive(Clone)]
|
|
pub struct AdminClientKeyService {
|
|
store: SqliteClientKeyStore,
|
|
}
|
|
|
|
impl AdminClientKeyService {
|
|
/// 构造管理端客户端 API Key 服务。
|
|
pub fn new(store: SqliteClientKeyStore) -> Self {
|
|
Self { store }
|
|
}
|
|
|
|
/// 创建新的客户端 API Key。
|
|
pub async fn create(
|
|
&self,
|
|
name: &str,
|
|
) -> Result<AdminCreatedClientApiKey, AdminClientKeyError> {
|
|
let name = name.trim();
|
|
if name.is_empty() {
|
|
return Err(AdminClientKeyError::EmptyName);
|
|
}
|
|
self.store
|
|
.create(name)
|
|
.await
|
|
.map(AdminCreatedClientApiKey::from)
|
|
.map_err(|_| AdminClientKeyError::Create)
|
|
}
|
|
|
|
/// 分页列出客户端 API Key。
|
|
pub async fn list(
|
|
&self,
|
|
cursor: Option<String>,
|
|
limit: u32,
|
|
) -> Result<Page<AdminStoredClientApiKey>, AdminClientKeyError> {
|
|
let page = self
|
|
.store
|
|
.list(cursor, limit)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::List)?;
|
|
Ok(Page {
|
|
items: page
|
|
.items
|
|
.into_iter()
|
|
.map(AdminStoredClientApiKey::from)
|
|
.collect(),
|
|
next_cursor: page.next_cursor,
|
|
})
|
|
}
|
|
|
|
/// 更新客户端 API Key 标签。
|
|
pub async fn update_label(
|
|
&self,
|
|
key_id: &str,
|
|
label: Option<String>,
|
|
) -> Result<Option<AdminStoredClientApiKey>, AdminClientKeyError> {
|
|
if label
|
|
.as_ref()
|
|
.is_some_and(|label| label.chars().count() > 64)
|
|
{
|
|
return Err(AdminClientKeyError::LabelTooLong);
|
|
}
|
|
self.store
|
|
.set_label(key_id, label)
|
|
.await
|
|
.map(|key| key.map(AdminStoredClientApiKey::from))
|
|
.map_err(|_| AdminClientKeyError::UpdateLabel)
|
|
}
|
|
|
|
/// 更新客户端 API Key 启用状态。
|
|
pub async fn update_status(
|
|
&self,
|
|
key_id: &str,
|
|
status: &str,
|
|
) -> Result<Option<UpdatedClientApiKeyStatus>, AdminClientKeyError> {
|
|
let enabled = parse_client_key_status(status)?;
|
|
match self.store.set_enabled(key_id, enabled).await {
|
|
Ok(true) => Ok(Some(UpdatedClientApiKeyStatus {
|
|
id: key_id.to_string(),
|
|
enabled,
|
|
})),
|
|
Ok(false) => Ok(None),
|
|
Err(_) => Err(AdminClientKeyError::UpdateStatus),
|
|
}
|
|
}
|
|
|
|
/// 删除客户端 API Key。
|
|
pub async fn delete(&self, key_id: &str) -> Result<bool, AdminClientKeyError> {
|
|
self.store
|
|
.delete(key_id)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::Delete)
|
|
}
|
|
|
|
/// 批量删除客户端 API Key。
|
|
pub async fn batch_delete(
|
|
&self,
|
|
ids: Vec<String>,
|
|
) -> Result<BatchDeleteClientApiKeys, AdminClientKeyError> {
|
|
if ids.is_empty() {
|
|
return Err(AdminClientKeyError::EmptyIds);
|
|
}
|
|
|
|
let mut deleted = 0u32;
|
|
let mut not_found = Vec::new();
|
|
for id in ids {
|
|
match self.store.delete(&id).await {
|
|
Ok(true) => deleted += 1,
|
|
Ok(false) => not_found.push(id),
|
|
Err(_) => return Err(AdminClientKeyError::Delete),
|
|
}
|
|
}
|
|
|
|
Ok(BatchDeleteClientApiKeys { deleted, not_found })
|
|
}
|
|
|
|
/// 导出客户端 API Key 元数据,不包含明文和哈希材料。
|
|
pub async fn export(
|
|
&self,
|
|
ids: Vec<String>,
|
|
) -> Result<Vec<AdminStoredClientApiKey>, AdminClientKeyError> {
|
|
if ids.is_empty() {
|
|
let mut all_keys = Vec::new();
|
|
let mut cursor = None;
|
|
loop {
|
|
let page = self
|
|
.store
|
|
.list(cursor, 200)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::Export)?;
|
|
all_keys.extend(page.items.into_iter().map(AdminStoredClientApiKey::from));
|
|
if page.next_cursor.is_none() {
|
|
return Ok(all_keys);
|
|
}
|
|
cursor = page.next_cursor;
|
|
}
|
|
}
|
|
|
|
let mut keys = Vec::with_capacity(ids.len());
|
|
for id in ids {
|
|
match self.store.get(&id).await {
|
|
Ok(Some(key)) => keys.push(AdminStoredClientApiKey::from(key)),
|
|
Ok(None) => {}
|
|
Err(_) => return Err(AdminClientKeyError::Export),
|
|
}
|
|
}
|
|
Ok(keys)
|
|
}
|
|
|
|
/// 导入导出的客户端 API Key 元数据,并轮换为新的本地明文。
|
|
pub async fn import(
|
|
&self,
|
|
payload: &Value,
|
|
) -> Result<ImportedClientApiKeys, AdminClientKeyError> {
|
|
let entries = parse_client_key_import_payload(payload);
|
|
if entries.is_empty() {
|
|
return Err(AdminClientKeyError::NoImportableKeys);
|
|
}
|
|
|
|
let mut imported = 0u32;
|
|
let mut skipped = 0u32;
|
|
let mut keys = Vec::with_capacity(entries.len());
|
|
for entry in entries {
|
|
let name = entry.name.trim();
|
|
if name.is_empty() {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
if entry
|
|
.label
|
|
.as_ref()
|
|
.is_some_and(|label| label.chars().count() > 64)
|
|
{
|
|
return Err(AdminClientKeyError::LabelTooLong);
|
|
}
|
|
|
|
let mut created = self.create(name).await?;
|
|
if entry.label.is_some() || !entry.enabled {
|
|
self.store
|
|
.set_label(&created.id, entry.label)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::Import)?;
|
|
if !entry.enabled {
|
|
self.store
|
|
.set_enabled(&created.id, false)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::Import)?;
|
|
}
|
|
let Some(stored) = self
|
|
.store
|
|
.get(&created.id)
|
|
.await
|
|
.map_err(|_| AdminClientKeyError::Import)?
|
|
else {
|
|
return Err(AdminClientKeyError::Import);
|
|
};
|
|
created.label = stored.label;
|
|
created.enabled = stored.enabled;
|
|
}
|
|
imported += 1;
|
|
keys.push(ImportedClientApiKey {
|
|
source_id: entry.source_id,
|
|
source_prefix: entry.source_prefix,
|
|
key: created,
|
|
});
|
|
}
|
|
|
|
Ok(ImportedClientApiKeys {
|
|
imported,
|
|
skipped,
|
|
keys,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// 管理端可见的客户端 API Key 元数据。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminStoredClientApiKey {
|
|
/// API Key 记录 ID。
|
|
pub id: String,
|
|
/// API Key 名称。
|
|
pub name: String,
|
|
/// 管理员可见标签。
|
|
pub label: Option<String>,
|
|
/// 明文 API Key 的短前缀。
|
|
pub prefix: String,
|
|
/// 是否允许用于 `/v1` 认证。
|
|
pub enabled: bool,
|
|
/// 创建时间。
|
|
pub created_at: String,
|
|
/// 最近一次成功使用时间。
|
|
pub last_used_at: Option<String>,
|
|
}
|
|
|
|
/// 新建客户端 API Key 后的一次性结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct AdminCreatedClientApiKey {
|
|
/// API Key 记录 ID。
|
|
pub id: String,
|
|
/// API Key 名称。
|
|
pub name: String,
|
|
/// 管理员可见标签。
|
|
pub label: Option<String>,
|
|
/// 明文 API Key 的短前缀。
|
|
pub prefix: String,
|
|
/// 是否允许用于 `/v1` 认证。
|
|
pub enabled: bool,
|
|
/// 创建时间。
|
|
pub created_at: String,
|
|
/// 最近一次成功使用时间。
|
|
pub last_used_at: Option<String>,
|
|
/// 仅返回一次的明文 API Key。
|
|
pub plaintext: String,
|
|
}
|
|
|
|
/// 客户端 API Key 状态更新结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct UpdatedClientApiKeyStatus {
|
|
/// API Key 记录 ID。
|
|
pub id: String,
|
|
/// 是否启用。
|
|
pub enabled: bool,
|
|
}
|
|
|
|
/// 批量删除客户端 API Key 的结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct BatchDeleteClientApiKeys {
|
|
/// 成功删除数量。
|
|
pub deleted: u32,
|
|
/// 未找到的 ID。
|
|
pub not_found: Vec<String>,
|
|
}
|
|
|
|
/// 导入后的客户端 API Key。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct ImportedClientApiKey {
|
|
/// 来源 ID。
|
|
pub source_id: Option<String>,
|
|
/// 来源短前缀。
|
|
pub source_prefix: Option<String>,
|
|
/// 新建的本地 API Key。
|
|
pub key: AdminCreatedClientApiKey,
|
|
}
|
|
|
|
/// 客户端 API Key 导入结果。
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct ImportedClientApiKeys {
|
|
/// 成功导入数量。
|
|
pub imported: u32,
|
|
/// 跳过数量。
|
|
pub skipped: u32,
|
|
/// 新建 API Key 列表。
|
|
pub keys: Vec<ImportedClientApiKey>,
|
|
}
|
|
|
|
/// 管理端客户端 API Key 错误。
|
|
#[derive(Debug, Error)]
|
|
pub enum AdminClientKeyError {
|
|
/// 列表失败。
|
|
#[error("failed to list client API keys")]
|
|
List,
|
|
/// 导出失败。
|
|
#[error("failed to export client API keys")]
|
|
Export,
|
|
/// 导入失败。
|
|
#[error("failed to import client API keys")]
|
|
Import,
|
|
/// 创建失败。
|
|
#[error("failed to create client API key")]
|
|
Create,
|
|
/// 删除失败。
|
|
#[error("failed to delete client API key")]
|
|
Delete,
|
|
/// 更新标签失败。
|
|
#[error("failed to update client API key label")]
|
|
UpdateLabel,
|
|
/// 更新状态失败。
|
|
#[error("failed to update client API key status")]
|
|
UpdateStatus,
|
|
/// 状态值无效。
|
|
#[error("unsupported client API key status: {0}")]
|
|
InvalidStatus(String),
|
|
/// 名称为空。
|
|
#[error("client API key name is required")]
|
|
EmptyName,
|
|
/// ID 列表为空。
|
|
#[error("client API key ids are required")]
|
|
EmptyIds,
|
|
/// 标签过长。
|
|
#[error("client API key label must be 64 characters or fewer")]
|
|
LabelTooLong,
|
|
/// 没有可导入的 API Key。
|
|
#[error("no importable client API keys found")]
|
|
NoImportableKeys,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ClientApiKeyImportEntry {
|
|
source_id: Option<String>,
|
|
source_prefix: Option<String>,
|
|
name: String,
|
|
label: Option<String>,
|
|
enabled: bool,
|
|
}
|
|
|
|
fn parse_client_key_status(status: &str) -> Result<bool, AdminClientKeyError> {
|
|
match status.trim().to_ascii_lowercase().as_str() {
|
|
"active" => Ok(true),
|
|
"disabled" => Ok(false),
|
|
other => Err(AdminClientKeyError::InvalidStatus(other.to_string())),
|
|
}
|
|
}
|
|
|
|
fn parse_account_status(
|
|
status: &str,
|
|
) -> Result<codex_proxy_core::accounts::model::AccountStatus, AdminAccountError> {
|
|
match status.trim().to_ascii_lowercase().as_str() {
|
|
"active" => Ok(codex_proxy_core::accounts::model::AccountStatus::Active),
|
|
"disabled" => Ok(codex_proxy_core::accounts::model::AccountStatus::Disabled),
|
|
"expired" => Ok(codex_proxy_core::accounts::model::AccountStatus::Expired),
|
|
"quota_exhausted" => Ok(codex_proxy_core::accounts::model::AccountStatus::QuotaExhausted),
|
|
"refreshing" => Ok(codex_proxy_core::accounts::model::AccountStatus::Refreshing),
|
|
"banned" => Ok(codex_proxy_core::accounts::model::AccountStatus::Banned),
|
|
other => Err(AdminAccountError::InvalidStatus(other.to_string())),
|
|
}
|
|
}
|
|
|
|
fn parse_batch_account_status(
|
|
status: &str,
|
|
) -> Result<codex_proxy_core::accounts::model::AccountStatus, AdminAccountError> {
|
|
match status.trim().to_ascii_lowercase().as_str() {
|
|
"active" => Ok(codex_proxy_core::accounts::model::AccountStatus::Active),
|
|
"disabled" => Ok(codex_proxy_core::accounts::model::AccountStatus::Disabled),
|
|
other => Err(AdminAccountError::InvalidStatus(other.to_string())),
|
|
}
|
|
}
|
|
|
|
fn refresh_failure_status(
|
|
failure: RefreshFailure,
|
|
) -> codex_proxy_core::accounts::model::AccountStatus {
|
|
match failure {
|
|
RefreshFailure::InvalidGrant => codex_proxy_core::accounts::model::AccountStatus::Disabled,
|
|
RefreshFailure::QuotaExhausted => {
|
|
codex_proxy_core::accounts::model::AccountStatus::QuotaExhausted
|
|
}
|
|
RefreshFailure::Banned => codex_proxy_core::accounts::model::AccountStatus::Banned,
|
|
RefreshFailure::Disabled => codex_proxy_core::accounts::model::AccountStatus::Disabled,
|
|
RefreshFailure::RetryableTransport => {
|
|
codex_proxy_core::accounts::model::AccountStatus::Active
|
|
}
|
|
RefreshFailure::Transport => codex_proxy_core::accounts::model::AccountStatus::Active,
|
|
}
|
|
}
|
|
|
|
fn refresh_failure_status_clears_next_refresh_at(
|
|
status: codex_proxy_core::accounts::model::AccountStatus,
|
|
) -> bool {
|
|
!matches!(
|
|
status,
|
|
codex_proxy_core::accounts::model::AccountStatus::Active
|
|
)
|
|
}
|
|
|
|
const ACCOUNT_IMPORT_ENVELOPE_KEYS: &[&str] =
|
|
&["code", "message", "data", "requestId", "request_id"];
|
|
const ACCOUNT_IMPORT_CONTAINER_KEYS: &[&str] = &["sourceFormat", "source_format", "accounts"];
|
|
const ACCOUNT_IMPORT_ACCOUNT_KEYS: &[&str] = &[
|
|
"id",
|
|
"email",
|
|
"accountId",
|
|
"account_id",
|
|
"userId",
|
|
"user_id",
|
|
"label",
|
|
"planType",
|
|
"plan_type",
|
|
"token",
|
|
"accessToken",
|
|
"access_token",
|
|
"refreshToken",
|
|
"refresh_token",
|
|
"accessTokenExpiresAt",
|
|
"access_token_expires_at",
|
|
"status",
|
|
"addedAt",
|
|
"added_at",
|
|
"updatedAt",
|
|
"updated_at",
|
|
];
|
|
const SUB2API_ACCOUNT_IMPORT_KEYS: &[&str] = &[
|
|
"id",
|
|
"email",
|
|
"accountId",
|
|
"account_id",
|
|
"userId",
|
|
"user_id",
|
|
"label",
|
|
"planType",
|
|
"plan_type",
|
|
"token",
|
|
"accessToken",
|
|
"access_token",
|
|
"refreshToken",
|
|
"refresh_token",
|
|
"status",
|
|
"addedAt",
|
|
"added_at",
|
|
"cachedQuota",
|
|
"cached_quota",
|
|
"quotaFetchedAt",
|
|
"quota_fetched_at",
|
|
"quotaVerifyRequired",
|
|
"quota_verify_required",
|
|
"proxyApiKey",
|
|
"proxy_api_key",
|
|
"usage",
|
|
];
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum AccountImportSource {
|
|
Native,
|
|
Sub2api,
|
|
}
|
|
|
|
impl AccountImportSource {
|
|
fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Native => "native",
|
|
Self::Sub2api => "sub2api",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct ParsedAccountImport {
|
|
source: AccountImportSource,
|
|
entries: Vec<AccountImportEntry>,
|
|
}
|
|
|
|
fn parse_account_import_payload(payload: &Value) -> Result<ParsedAccountImport, AdminAccountError> {
|
|
let payload = payload
|
|
.get("data")
|
|
.filter(|data| data.get("accounts").is_some())
|
|
.map(|data| {
|
|
ensure_account_import_keys(payload, ACCOUNT_IMPORT_ENVELOPE_KEYS)?;
|
|
Ok(data)
|
|
})
|
|
.transpose()?
|
|
.unwrap_or(payload);
|
|
|
|
if let Some(accounts) = payload.get("accounts") {
|
|
ensure_account_import_keys(payload, ACCOUNT_IMPORT_CONTAINER_KEYS)?;
|
|
let accounts = accounts
|
|
.as_array()
|
|
.ok_or(AdminAccountError::NoImportableAccounts)?;
|
|
let source = account_import_source(payload, accounts)?;
|
|
return Ok(ParsedAccountImport {
|
|
source,
|
|
entries: parse_account_import_entries(accounts, source)?,
|
|
});
|
|
}
|
|
if let Some(accounts) = payload.as_array() {
|
|
let source = account_import_source(payload, accounts)?;
|
|
return Ok(ParsedAccountImport {
|
|
source,
|
|
entries: parse_account_import_entries(accounts, source)?,
|
|
});
|
|
}
|
|
|
|
let source = account_import_source(payload, std::slice::from_ref(payload))?;
|
|
Ok(ParsedAccountImport {
|
|
source,
|
|
entries: account_import_entry_from_value(payload, source)?
|
|
.into_iter()
|
|
.collect(),
|
|
})
|
|
}
|
|
|
|
fn parse_account_import_entries(
|
|
accounts: &[Value],
|
|
source: AccountImportSource,
|
|
) -> Result<Vec<AccountImportEntry>, AdminAccountError> {
|
|
let mut entries = Vec::new();
|
|
for account in accounts {
|
|
if let Some(entry) = account_import_entry_from_value(account, source)? {
|
|
entries.push(entry);
|
|
}
|
|
}
|
|
Ok(entries)
|
|
}
|
|
|
|
fn account_import_entry_from_value(
|
|
value: &Value,
|
|
source: AccountImportSource,
|
|
) -> Result<Option<AccountImportEntry>, AdminAccountError> {
|
|
let Some(account) = value.as_object() else {
|
|
return Ok(None);
|
|
};
|
|
let allowed_keys = match source {
|
|
AccountImportSource::Native => ACCOUNT_IMPORT_ACCOUNT_KEYS,
|
|
AccountImportSource::Sub2api => SUB2API_ACCOUNT_IMPORT_KEYS,
|
|
};
|
|
if account
|
|
.keys()
|
|
.any(|key| !allowed_keys.contains(&key.as_str()))
|
|
{
|
|
return Err(AdminAccountError::NoImportableAccounts);
|
|
}
|
|
|
|
let token = first_string(value, &["token", "accessToken", "access_token"]);
|
|
let refresh_token = first_string(value, &["refreshToken", "refresh_token"]);
|
|
if token.is_none() && refresh_token.is_none() {
|
|
return Ok(None);
|
|
}
|
|
|
|
Ok(Some(AccountImportEntry {
|
|
id: first_string(value, &["id"]),
|
|
email: first_string(value, &["email"]),
|
|
account_id: first_string(value, &["accountId", "account_id"]),
|
|
user_id: first_string(value, &["userId", "user_id"]),
|
|
label: first_string(value, &["label"]),
|
|
plan_type: first_string(value, &["planType", "plan_type"]),
|
|
token,
|
|
refresh_token,
|
|
access_token_expires_at: first_string(
|
|
value,
|
|
&["accessTokenExpiresAt", "access_token_expires_at"],
|
|
),
|
|
status: first_string(value, &["status"]),
|
|
cached_quota: (source == AccountImportSource::Sub2api)
|
|
.then(|| first_value(value, &["cachedQuota", "cached_quota"]))
|
|
.flatten(),
|
|
quota_fetched_at: (source == AccountImportSource::Sub2api)
|
|
.then(|| first_string(value, &["quotaFetchedAt", "quota_fetched_at"]))
|
|
.flatten(),
|
|
quota_verify_required: (source == AccountImportSource::Sub2api)
|
|
.then(|| first_bool(value, &["quotaVerifyRequired", "quota_verify_required"]))
|
|
.flatten(),
|
|
}))
|
|
}
|
|
|
|
fn ensure_account_import_keys(
|
|
value: &Value,
|
|
allowed_keys: &[&str],
|
|
) -> Result<(), AdminAccountError> {
|
|
let Some(object) = value.as_object() else {
|
|
return Err(AdminAccountError::NoImportableAccounts);
|
|
};
|
|
if object
|
|
.keys()
|
|
.all(|key| allowed_keys.contains(&key.as_str()))
|
|
{
|
|
Ok(())
|
|
} else {
|
|
Err(AdminAccountError::NoImportableAccounts)
|
|
}
|
|
}
|
|
|
|
fn account_import_source(
|
|
value: &Value,
|
|
accounts: &[Value],
|
|
) -> Result<AccountImportSource, AdminAccountError> {
|
|
if let Some(source_format) = first_string(value, &["sourceFormat", "source_format"]) {
|
|
return match source_format.trim().to_ascii_lowercase().as_str() {
|
|
"native" => Ok(AccountImportSource::Native),
|
|
"sub2api" => Ok(AccountImportSource::Sub2api),
|
|
_ => Err(AdminAccountError::NoImportableAccounts),
|
|
};
|
|
}
|
|
|
|
if accounts.iter().any(account_import_entry_looks_sub2api) {
|
|
Ok(AccountImportSource::Sub2api)
|
|
} else {
|
|
Ok(AccountImportSource::Native)
|
|
}
|
|
}
|
|
|
|
fn account_import_entry_looks_sub2api(value: &Value) -> bool {
|
|
let Some(account) = value.as_object() else {
|
|
return false;
|
|
};
|
|
[
|
|
"proxyApiKey",
|
|
"proxy_api_key",
|
|
"usage",
|
|
"cachedQuota",
|
|
"cached_quota",
|
|
"quotaFetchedAt",
|
|
"quota_fetched_at",
|
|
"quotaVerifyRequired",
|
|
"quota_verify_required",
|
|
]
|
|
.iter()
|
|
.any(|key| account.contains_key(*key))
|
|
}
|
|
|
|
fn parse_account_import_status(
|
|
status: Option<&str>,
|
|
) -> Result<codex_proxy_core::accounts::model::AccountStatus, AdminAccountError> {
|
|
parse_account_status(status.unwrap_or("active"))
|
|
}
|
|
|
|
fn normalized_imported_account_status(
|
|
status: AccountStatus,
|
|
source: AccountImportSource,
|
|
access_token: &str,
|
|
) -> AccountStatus {
|
|
if source == AccountImportSource::Sub2api
|
|
&& status == AccountStatus::Active
|
|
&& jwt_expiry(access_token, Utc::now()) != JwtExpiry::Valid
|
|
{
|
|
AccountStatus::Expired
|
|
} else {
|
|
status
|
|
}
|
|
}
|
|
|
|
fn parse_account_import_datetime(
|
|
value: &str,
|
|
) -> Result<chrono::DateTime<chrono::Utc>, AdminAccountError> {
|
|
chrono::DateTime::parse_from_rfc3339(value)
|
|
.map(|value| value.with_timezone(&chrono::Utc))
|
|
.map_err(|_| AdminAccountError::InvalidAccessTokenExpiresAt)
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
struct ManualAccountClaims {
|
|
account_id: String,
|
|
user_id: Option<String>,
|
|
email: Option<String>,
|
|
plan_type: Option<String>,
|
|
expires_at: chrono::DateTime<chrono::Utc>,
|
|
}
|
|
|
|
fn manual_account_claims(
|
|
token: &str,
|
|
now: chrono::DateTime<chrono::Utc>,
|
|
) -> Result<ManualAccountClaims, &'static str> {
|
|
let payload = decode_jwt_payload(token).ok_or("Invalid JWT format")?;
|
|
let exp = payload
|
|
.get("exp")
|
|
.and_then(Value::as_i64)
|
|
.ok_or("Token is expired")?;
|
|
if now.timestamp() >= exp {
|
|
return Err("Token is expired");
|
|
}
|
|
let expires_at =
|
|
chrono::DateTime::<chrono::Utc>::from_timestamp(exp, 0).ok_or("Invalid JWT exp claim")?;
|
|
let auth = payload
|
|
.get("https://api.openai.com/auth")
|
|
.and_then(Value::as_object)
|
|
.ok_or("Token missing chatgpt_account_id claim")?;
|
|
let account_id =
|
|
string_claim(auth, "chatgpt_account_id").ok_or("Token missing chatgpt_account_id claim")?;
|
|
let profile = payload
|
|
.get("https://api.openai.com/profile")
|
|
.and_then(Value::as_object);
|
|
let user_id = string_claim(auth, "chatgpt_user_id")
|
|
.or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_user_id")));
|
|
let plan_type = string_claim(auth, "chatgpt_plan_type")
|
|
.or_else(|| profile.and_then(|profile| string_claim(profile, "chatgpt_plan_type")));
|
|
let email = profile.and_then(|profile| string_claim(profile, "email"));
|
|
|
|
Ok(ManualAccountClaims {
|
|
account_id,
|
|
user_id,
|
|
email,
|
|
plan_type,
|
|
expires_at,
|
|
})
|
|
}
|
|
|
|
fn decode_jwt_payload(token: &str) -> Option<Map<String, Value>> {
|
|
let mut parts = token.split('.');
|
|
let _header = parts.next()?;
|
|
let payload = parts.next()?;
|
|
if payload.is_empty() {
|
|
return None;
|
|
}
|
|
let bytes = URL_SAFE_NO_PAD.decode(payload).ok()?;
|
|
serde_json::from_slice::<Value>(&bytes)
|
|
.ok()?
|
|
.as_object()
|
|
.cloned()
|
|
}
|
|
|
|
fn string_claim(map: &Map<String, Value>, key: &str) -> Option<String> {
|
|
map.get(key)
|
|
.and_then(Value::as_str)
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(ToString::to_string)
|
|
}
|
|
|
|
fn normalize_bearer_token(value: String) -> String {
|
|
value
|
|
.trim()
|
|
.strip_prefix("Bearer ")
|
|
.or_else(|| value.trim().strip_prefix("bearer "))
|
|
.unwrap_or(value.trim())
|
|
.trim()
|
|
.to_string()
|
|
}
|
|
|
|
fn normalized_account_id(id: Option<String>) -> String {
|
|
normalize_nonempty(id).unwrap_or_else(|| format!("acct_{}", uuid::Uuid::new_v4().simple()))
|
|
}
|
|
|
|
fn normalize_label(value: Option<String>) -> Option<String> {
|
|
normalize_nonempty(value)
|
|
}
|
|
|
|
fn normalize_nonempty(value: Option<String>) -> Option<String> {
|
|
value
|
|
.map(|value| value.trim().to_string())
|
|
.filter(|value| !value.is_empty())
|
|
}
|
|
|
|
fn normalize_nonempty_str(value: Option<&str>) -> Option<&str> {
|
|
value.map(str::trim).filter(|value| !value.is_empty())
|
|
}
|
|
|
|
fn parse_client_key_import_payload(payload: &Value) -> Vec<ClientApiKeyImportEntry> {
|
|
let payload = payload
|
|
.get("data")
|
|
.filter(|data| data.get("apiKeys").is_some() || data.get("keys").is_some())
|
|
.unwrap_or(payload);
|
|
|
|
if let Some(keys) = payload.get("apiKeys").and_then(Value::as_array) {
|
|
return keys
|
|
.iter()
|
|
.filter_map(client_key_import_entry_from_value)
|
|
.collect();
|
|
}
|
|
if let Some(keys) = payload.get("keys").and_then(Value::as_array) {
|
|
return keys
|
|
.iter()
|
|
.filter_map(client_key_import_entry_from_value)
|
|
.collect();
|
|
}
|
|
if let Some(keys) = payload.as_array() {
|
|
return keys
|
|
.iter()
|
|
.filter_map(client_key_import_entry_from_value)
|
|
.collect();
|
|
}
|
|
|
|
client_key_import_entry_from_value(payload)
|
|
.into_iter()
|
|
.collect()
|
|
}
|
|
|
|
fn client_key_import_entry_from_value(value: &Value) -> Option<ClientApiKeyImportEntry> {
|
|
value.as_object()?;
|
|
let name = first_string(value, &["name"])?;
|
|
Some(ClientApiKeyImportEntry {
|
|
source_id: first_string(value, &["id", "sourceId"]),
|
|
source_prefix: first_string(value, &["prefix", "sourcePrefix"]),
|
|
name,
|
|
label: first_string(value, &["label"]),
|
|
enabled: client_key_import_enabled(value),
|
|
})
|
|
}
|
|
|
|
fn client_key_import_enabled(value: &Value) -> bool {
|
|
if let Some(enabled) = value.get("enabled").and_then(Value::as_bool) {
|
|
return enabled;
|
|
}
|
|
!first_string(value, &["status"])
|
|
.unwrap_or_else(|| "active".to_string())
|
|
.trim()
|
|
.eq_ignore_ascii_case("disabled")
|
|
}
|
|
|
|
fn first_string(value: &Value, keys: &[&str]) -> Option<String> {
|
|
keys.iter()
|
|
.find_map(|key| value.get(key).and_then(Value::as_str))
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(ToString::to_string)
|
|
}
|
|
|
|
fn first_bool(value: &Value, keys: &[&str]) -> Option<bool> {
|
|
keys.iter()
|
|
.find_map(|key| value.get(key).and_then(Value::as_bool))
|
|
}
|
|
|
|
fn first_value(value: &Value, keys: &[&str]) -> Option<Value> {
|
|
keys.iter()
|
|
.find_map(|key| value.get(key).filter(|value| !value.is_null()))
|
|
.cloned()
|
|
}
|
|
|
|
fn quota_warnings_from_snapshots(
|
|
snapshots: Vec<AccountQuotaSnapshot>,
|
|
thresholds: &QuotaWarningThresholds,
|
|
) -> AdminAccountQuotaWarnings {
|
|
let primary_thresholds = sorted_thresholds(&thresholds.primary);
|
|
let secondary_thresholds = sorted_thresholds(&thresholds.secondary);
|
|
let mut warnings = Vec::new();
|
|
let mut updated_at = None;
|
|
|
|
for snapshot in snapshots {
|
|
let Ok(quota) = serde_json::from_str::<Value>(&snapshot.quota_json) else {
|
|
continue;
|
|
};
|
|
let before_len = warnings.len();
|
|
if let Some(warning) = warning_from_quota_window(
|
|
&snapshot.account_id,
|
|
snapshot.email.as_deref(),
|
|
"a,
|
|
"rate_limit",
|
|
AdminQuotaWarningWindow::Primary,
|
|
&primary_thresholds,
|
|
) {
|
|
warnings.push(warning);
|
|
}
|
|
if let Some(warning) = warning_from_quota_window(
|
|
&snapshot.account_id,
|
|
snapshot.email.as_deref(),
|
|
"a,
|
|
"secondary_rate_limit",
|
|
AdminQuotaWarningWindow::Secondary,
|
|
&secondary_thresholds,
|
|
) {
|
|
warnings.push(warning);
|
|
}
|
|
if warnings.len() > before_len {
|
|
updated_at = max_optional_datetime(updated_at, snapshot.quota_fetched_at);
|
|
}
|
|
}
|
|
|
|
AdminAccountQuotaWarnings {
|
|
warnings,
|
|
updated_at,
|
|
}
|
|
}
|
|
|
|
fn warning_from_quota_window(
|
|
account_id: &str,
|
|
email: Option<&str>,
|
|
quota: &Value,
|
|
field: &str,
|
|
window: AdminQuotaWarningWindow,
|
|
thresholds: &[u8],
|
|
) -> Option<AdminAccountQuotaWarning> {
|
|
let quota_window = quota.get(field).filter(|value| !value.is_null())?;
|
|
let used_percent = quota_window
|
|
.get("used_percent")
|
|
.and_then(Value::as_f64)
|
|
.filter(|value| value.is_finite())?;
|
|
let level = warning_level(used_percent, thresholds)?;
|
|
|
|
Some(AdminAccountQuotaWarning {
|
|
account_id: account_id.to_string(),
|
|
email: email.map(ToString::to_string),
|
|
window,
|
|
level,
|
|
used_percent,
|
|
reset_at: quota_window.get("reset_at").and_then(Value::as_i64),
|
|
})
|
|
}
|
|
|
|
fn warning_level(used_percent: f64, thresholds: &[u8]) -> Option<AdminQuotaWarningLevel> {
|
|
let matched_index = thresholds
|
|
.iter()
|
|
.rposition(|threshold| quota_reached(used_percent, f64::from(*threshold)))?;
|
|
if matched_index + 1 == thresholds.len() {
|
|
Some(AdminQuotaWarningLevel::Critical)
|
|
} else {
|
|
Some(AdminQuotaWarningLevel::Warning)
|
|
}
|
|
}
|
|
|
|
fn sorted_thresholds(thresholds: &[u8]) -> Vec<u8> {
|
|
let mut thresholds = thresholds.to_vec();
|
|
thresholds.sort_unstable();
|
|
thresholds.dedup();
|
|
thresholds
|
|
}
|
|
|
|
fn max_optional_datetime(
|
|
current: Option<chrono::DateTime<chrono::Utc>>,
|
|
candidate: Option<chrono::DateTime<chrono::Utc>>,
|
|
) -> Option<chrono::DateTime<chrono::Utc>> {
|
|
match (current, candidate) {
|
|
(Some(current), Some(candidate)) => Some(current.max(candidate)),
|
|
(Some(current), None) => Some(current),
|
|
(None, Some(candidate)) => Some(candidate),
|
|
(None, None) => None,
|
|
}
|
|
}
|
|
|
|
impl From<StoredAccount> for AdminStoredAccount {
|
|
fn from(account: StoredAccount) -> Self {
|
|
Self {
|
|
id: account.id,
|
|
email: account.email,
|
|
account_id: account.account_id,
|
|
user_id: account.user_id,
|
|
label: account.label,
|
|
plan_type: account.plan_type,
|
|
access_token: account.access_token.expose_secret().to_string(),
|
|
refresh_token: account
|
|
.refresh_token
|
|
.map(|token| token.expose_secret().to_string()),
|
|
access_token_expires_at: account.access_token_expires_at,
|
|
status: account.status,
|
|
added_at: account.added_at,
|
|
updated_at: account.updated_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<StoredAccount> for AdminAccountMetadata {
|
|
fn from(account: StoredAccount) -> Self {
|
|
Self {
|
|
id: account.id,
|
|
email: account.email,
|
|
account_id: account.account_id,
|
|
user_id: account.user_id,
|
|
label: account.label,
|
|
plan_type: account.plan_type,
|
|
access_token_expires_at: account.access_token_expires_at,
|
|
status: account.status,
|
|
added_at: account.added_at,
|
|
updated_at: account.updated_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<StoredClientApiKey> for AdminStoredClientApiKey {
|
|
fn from(key: StoredClientApiKey) -> Self {
|
|
Self {
|
|
id: key.id,
|
|
name: key.name,
|
|
label: key.label,
|
|
prefix: key.prefix,
|
|
enabled: key.enabled,
|
|
created_at: key.created_at,
|
|
last_used_at: key.last_used_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<CreatedClientApiKey> for AdminCreatedClientApiKey {
|
|
fn from(key: CreatedClientApiKey) -> Self {
|
|
Self {
|
|
id: key.id,
|
|
name: key.name,
|
|
label: key.label,
|
|
prefix: key.prefix,
|
|
enabled: key.enabled,
|
|
created_at: key.created_at,
|
|
last_used_at: key.last_used_at,
|
|
plaintext: key.plaintext,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<StoredAccountMetadata> for AdminAccountMetadata {
|
|
fn from(account: StoredAccountMetadata) -> Self {
|
|
Self {
|
|
id: account.id,
|
|
email: account.email,
|
|
account_id: account.account_id,
|
|
user_id: account.user_id,
|
|
label: account.label,
|
|
plan_type: account.plan_type,
|
|
access_token_expires_at: account.access_token_expires_at,
|
|
status: account.status,
|
|
added_at: account.added_at,
|
|
updated_at: account.updated_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<AccountUsageListRecord> for AdminUsageRecord {
|
|
fn from(usage: AccountUsageListRecord) -> Self {
|
|
Self {
|
|
account_id: usage.account_id,
|
|
email: usage.email,
|
|
label: usage.label,
|
|
plan_type: usage.plan_type,
|
|
request_count: usage.request_count,
|
|
empty_response_count: usage.empty_response_count,
|
|
input_tokens: usage.input_tokens,
|
|
output_tokens: usage.output_tokens,
|
|
cached_tokens: usage.cached_tokens,
|
|
reasoning_tokens: usage.reasoning_tokens,
|
|
total_tokens: usage.total_tokens,
|
|
image_input_tokens: usage.image_input_tokens,
|
|
image_output_tokens: usage.image_output_tokens,
|
|
image_request_count: usage.image_request_count,
|
|
image_request_failed_count: usage.image_request_failed_count,
|
|
last_used_at: usage.last_used_at,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<AccountUsageSummary> for AdminUsageSummary {
|
|
fn from(summary: AccountUsageSummary) -> Self {
|
|
Self {
|
|
account_count: summary.account_count,
|
|
request_count: summary.request_count,
|
|
empty_response_count: summary.empty_response_count,
|
|
input_tokens: summary.input_tokens,
|
|
output_tokens: summary.output_tokens,
|
|
cached_tokens: summary.cached_tokens,
|
|
reasoning_tokens: summary.reasoning_tokens,
|
|
total_tokens: summary.total_tokens,
|
|
image_input_tokens: summary.image_input_tokens,
|
|
image_output_tokens: summary.image_output_tokens,
|
|
image_request_count: summary.image_request_count,
|
|
image_request_failed_count: summary.image_request_failed_count,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<AdminLogFilter> for EventLogFilter {
|
|
fn from(filter: AdminLogFilter) -> Self {
|
|
Self {
|
|
kind: filter.kind,
|
|
level: filter.level,
|
|
request_id: filter.request_id,
|
|
account_id: filter.account_id,
|
|
route: filter.route,
|
|
model: filter.model,
|
|
status_code: filter.status_code,
|
|
transport: filter.transport,
|
|
attempt_index: filter.attempt_index,
|
|
upstream_status_code: filter.upstream_status_code,
|
|
failure_class: filter.failure_class,
|
|
response_id: filter.response_id,
|
|
upstream_request_id: filter.upstream_request_id,
|
|
search: filter.search,
|
|
}
|
|
}
|
|
}
|