mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
refactor(dispatch): harden account failover boundaries
Invalidate unavailable accounts in memory before asynchronous persistence, evict account WebSockets, and classify explicit auth, quota, ban, and workspace failures for immediate rotation. Separate account-scoped installation identity from client session semantics, preserve logical IDs across failover, clear opaque account state, remove forced HTTP/1.1 configuration, and document the lifecycle rules. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -103,36 +103,36 @@ pub fn build_codex_request(
|
||||
None => {}
|
||||
}
|
||||
apply_context_header_fields(&mut request, headers);
|
||||
apply_identity_fields(&mut request, headers);
|
||||
apply_session_fields(&mut request, headers);
|
||||
request.client_user_agent = user_agent_from_headers(headers);
|
||||
request
|
||||
}
|
||||
|
||||
fn apply_identity_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) {
|
||||
request.client_conversation_id = identity_context_string(request.body(), "conversation_id")
|
||||
fn apply_session_fields(request: &mut CodexResponsesRequest, headers: &HeaderMap) {
|
||||
request.client_conversation_id = session_context_string(request.body(), "conversation_id")
|
||||
.or_else(|| header_string(headers, "conversation-id"))
|
||||
.or_else(|| header_string(headers, "conversation_id"))
|
||||
.or_else(|| request.prompt_cache_key().map(ToString::to_string));
|
||||
request.client_session_id = identity_context_string(request.body(), "session_id")
|
||||
request.client_session_id = session_context_string(request.body(), "session_id")
|
||||
.or_else(|| header_string(headers, "session-id"))
|
||||
.or_else(|| header_string(headers, "session_id"));
|
||||
request.client_thread_id = identity_context_string(request.body(), "thread_id")
|
||||
request.client_thread_id = session_context_string(request.body(), "thread_id")
|
||||
.or_else(|| header_string(headers, "thread-id"));
|
||||
request.client_request_id = identity_context_string(request.body(), "x-client-request-id")
|
||||
request.client_request_id = session_context_string(request.body(), "x-client-request-id")
|
||||
.or_else(|| header_string(headers, "x-client-request-id"));
|
||||
request.client_turn_id = identity_context_string(request.body(), "turn_id")
|
||||
request.client_turn_id = session_context_string(request.body(), "turn_id")
|
||||
.or_else(|| header_string(headers, "x-codex-turn-id"));
|
||||
request.codex_window_id = request.codex_window_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-window-id")
|
||||
session_context_string(request.body(), "x-codex-window-id")
|
||||
.or_else(|| header_string(headers, "x-codex-window-id"))
|
||||
});
|
||||
request.parent_thread_id = request.parent_thread_id.take().or_else(|| {
|
||||
identity_context_string(request.body(), "x-codex-parent-thread-id")
|
||||
session_context_string(request.body(), "x-codex-parent-thread-id")
|
||||
.or_else(|| header_string(headers, "x-codex-parent-thread-id"))
|
||||
});
|
||||
}
|
||||
|
||||
fn identity_context_string(body: &Map<String, Value>, key: &str) -> Option<String> {
|
||||
fn session_context_string(body: &Map<String, Value>, key: &str) -> Option<String> {
|
||||
body_context_string(body, key).or_else(|| {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
|
||||
@@ -105,7 +105,6 @@ impl BootstrapConfig {
|
||||
redis,
|
||||
runtime,
|
||||
quota,
|
||||
tls,
|
||||
ws_pool,
|
||||
fingerprint,
|
||||
admin,
|
||||
@@ -122,7 +121,6 @@ impl BootstrapConfig {
|
||||
database: DatabaseConfig { url: database.url },
|
||||
redis: RedisConfig { url: redis.url },
|
||||
runtime,
|
||||
tls,
|
||||
ws_pool,
|
||||
fingerprint,
|
||||
admin: AdminConfig {
|
||||
@@ -241,8 +239,6 @@ pub struct AppConfig {
|
||||
pub redis: RedisConfig,
|
||||
/// 本地持久运行目录。
|
||||
pub runtime: RuntimePathsConfig,
|
||||
/// TLS 偏好配置。
|
||||
pub tls: TlsConfig,
|
||||
/// WebSocket 连接池启动设置。
|
||||
pub ws_pool: WebSocketPoolSettings,
|
||||
/// 上游请求指纹默认配置。
|
||||
@@ -344,14 +340,6 @@ pub struct RuntimePathsConfig {
|
||||
pub data_directory: PathBuf,
|
||||
}
|
||||
|
||||
/// TLS/HTTP 协议偏好配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TlsConfig {
|
||||
/// 是否强制 HTTP/1.1。
|
||||
pub force_http11: bool,
|
||||
}
|
||||
|
||||
/// WebSocket 连接池启动设置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -464,7 +452,6 @@ struct FileAppConfig {
|
||||
redis: FileRedisConfig,
|
||||
runtime: RuntimePathsConfig,
|
||||
quota: QuotaConfig,
|
||||
tls: TlsConfig,
|
||||
ws_pool: WebSocketPoolSettings,
|
||||
fingerprint: FingerprintConfig,
|
||||
admin: FileAdminConfig,
|
||||
|
||||
@@ -212,7 +212,7 @@ impl Services {
|
||||
let websocket_initial_event_timeout = (config.ws_pool.initial_event_timeout_ms > 0)
|
||||
.then(|| std::time::Duration::from_millis(config.ws_pool.initial_event_timeout_ms));
|
||||
let client = CodexBackendClient::new(
|
||||
build_reqwest_client(config.tls.force_http11)?,
|
||||
build_reqwest_client()?,
|
||||
config.api.base_url.clone(),
|
||||
fingerprint.clone(),
|
||||
)
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
//! 客户端身份、本地会话索引与账号作用域 wire 身份隔离。
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
infra::identity::AccountPseudonymizer,
|
||||
upstream::openai::protocol::responses::CodexResponsesRequest,
|
||||
};
|
||||
|
||||
const TURN_STATE_METADATA_KEY: &str = "x-codex-turn-state";
|
||||
const TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
|
||||
/// 持久化密钥驱动的账号身份隔离服务。
|
||||
#[derive(Clone)]
|
||||
pub struct AccountIdentityService {
|
||||
@@ -21,25 +26,51 @@ impl std::fmt::Debug for AccountIdentityService {
|
||||
|
||||
/// 单次账号 attempt 的 wire 身份。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountScopedIdentity {
|
||||
pub prompt_cache_key: Option<String>,
|
||||
pub session_id: Option<String>,
|
||||
pub thread_id: Option<String>,
|
||||
pub client_request_id: String,
|
||||
pub turn_id: Option<String>,
|
||||
pub window_id: Option<String>,
|
||||
pub parent_thread_id: Option<String>,
|
||||
pub(in crate::dispatch) struct AccountScopedIdentity {
|
||||
pub installation_id: String,
|
||||
}
|
||||
|
||||
struct IdentityValues<'a> {
|
||||
prompt_cache_key: Option<&'a str>,
|
||||
session_id: Option<&'a str>,
|
||||
thread_id: Option<&'a str>,
|
||||
client_request_id: Option<&'a str>,
|
||||
turn_id: Option<&'a str>,
|
||||
window_id: Option<&'a str>,
|
||||
parent_thread_id: Option<&'a str>,
|
||||
/// 单个请求内账号绑定 opaque 状态的 owner。
|
||||
pub(in crate::dispatch) struct AccountIdentityScope {
|
||||
turn_state_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountIdentityScope {
|
||||
pub(in crate::dispatch) fn new(preferred_account_id: Option<String>) -> Self {
|
||||
Self {
|
||||
turn_state_account_id: preferred_account_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn preserves_turn_state(&mut self, account_id: &str, has_turn_state: bool) -> bool {
|
||||
if !has_turn_state {
|
||||
return false;
|
||||
}
|
||||
let owner = self
|
||||
.turn_state_account_id
|
||||
.get_or_insert_with(|| account_id.to_string());
|
||||
owner == account_id
|
||||
}
|
||||
}
|
||||
|
||||
/// 已按当前账号重建 wire 身份、可安全交给上游 transport 的请求。
|
||||
pub(in crate::dispatch) struct AccountScopedRequest {
|
||||
request: CodexResponsesRequest,
|
||||
identity: AccountScopedIdentity,
|
||||
}
|
||||
|
||||
impl AccountScopedRequest {
|
||||
pub(in crate::dispatch) fn request(&self) -> &CodexResponsesRequest {
|
||||
&self.request
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn identity(&self) -> &AccountScopedIdentity {
|
||||
&self.identity
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn into_request(self) -> CodexResponsesRequest {
|
||||
self.request
|
||||
}
|
||||
}
|
||||
|
||||
impl AccountIdentityService {
|
||||
@@ -63,88 +94,183 @@ impl AccountIdentityService {
|
||||
.map(|anchor| format!("lc_{}", self.pseudonym("local-conversation", None, anchor)));
|
||||
}
|
||||
|
||||
pub fn scope(
|
||||
/// 从请求语义为单次账号 attempt 重建完整 wire 身份。
|
||||
pub(in crate::dispatch) fn scope_request(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
scope: &mut AccountIdentityScope,
|
||||
mut request: CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
self.scope_values(
|
||||
) -> AccountScopedRequest {
|
||||
let identity = AccountScopedIdentity {
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
};
|
||||
let client_metadata_turn_state = client_metadata_string(&request, TURN_STATE_METADATA_KEY);
|
||||
let preserve_turn_state = scope.preserves_turn_state(
|
||||
account_id,
|
||||
proxy_request_id,
|
||||
IdentityValues {
|
||||
prompt_cache_key: request.prompt_cache_key(),
|
||||
session_id: request.client_session_id.as_deref(),
|
||||
thread_id: request.client_thread_id.as_deref(),
|
||||
client_request_id: request.client_request_id.as_deref(),
|
||||
turn_id: request.client_turn_id.as_deref(),
|
||||
window_id: request.codex_window_id.as_deref(),
|
||||
parent_thread_id: request.parent_thread_id.as_deref(),
|
||||
},
|
||||
)
|
||||
request.turn_state.is_some() || client_metadata_turn_state.is_some(),
|
||||
);
|
||||
let turn_state = if preserve_turn_state {
|
||||
request.turn_state.clone()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let client_metadata_turn_state = if preserve_turn_state {
|
||||
client_metadata_turn_state
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let turn_metadata = request
|
||||
.turn_metadata
|
||||
.as_deref()
|
||||
.map(|raw| Self::scope_turn_metadata(raw, &identity));
|
||||
let client_metadata_turn_metadata = client_metadata_string(&request, TURN_METADATA_KEY)
|
||||
.map(|raw| Self::scope_turn_metadata(&raw, &identity));
|
||||
|
||||
apply_identity_to_request(
|
||||
&mut request,
|
||||
&identity,
|
||||
turn_state.as_deref(),
|
||||
turn_metadata.as_deref(),
|
||||
client_metadata_turn_state.as_deref(),
|
||||
client_metadata_turn_metadata.as_deref(),
|
||||
);
|
||||
request.turn_state = turn_state;
|
||||
request.turn_metadata = turn_metadata;
|
||||
|
||||
AccountScopedRequest { request, identity }
|
||||
}
|
||||
|
||||
pub fn scope_auxiliary(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
) -> AccountScopedIdentity {
|
||||
pub(in crate::dispatch) fn scope_auxiliary(&self, account_id: &str) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym("client-request-id", Some(account_id), proxy_request_id)
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
..AccountScopedIdentity::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_values(
|
||||
&self,
|
||||
account_id: &str,
|
||||
proxy_request_id: &str,
|
||||
values: IdentityValues<'_>,
|
||||
) -> AccountScopedIdentity {
|
||||
AccountScopedIdentity {
|
||||
prompt_cache_key: self.scope_optional(
|
||||
"prompt-cache-key",
|
||||
account_id,
|
||||
values.prompt_cache_key,
|
||||
),
|
||||
session_id: self.scope_optional("session-id", account_id, values.session_id),
|
||||
thread_id: self.scope_optional("thread-id", account_id, values.thread_id),
|
||||
client_request_id: format!(
|
||||
"cr_{}",
|
||||
self.pseudonym(
|
||||
"client-request-id",
|
||||
Some(account_id),
|
||||
values.client_request_id.unwrap_or(proxy_request_id),
|
||||
)
|
||||
),
|
||||
turn_id: self.scope_optional("turn-id", account_id, values.turn_id),
|
||||
window_id: self.scope_optional("window-id", account_id, values.window_id),
|
||||
parent_thread_id: self.scope_optional(
|
||||
"parent-thread-id",
|
||||
account_id,
|
||||
values.parent_thread_id,
|
||||
),
|
||||
installation_id: self.pseudonymizer.installation_id(account_id),
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_optional(
|
||||
&self,
|
||||
domain: &str,
|
||||
account_id: &str,
|
||||
value: Option<&str>,
|
||||
) -> Option<String> {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| format!("wi_{}", self.pseudonym(domain, Some(account_id), value)))
|
||||
fn scope_turn_metadata(raw: &str, identity: &AccountScopedIdentity) -> String {
|
||||
let Ok(Value::Object(mut metadata)) = serde_json::from_str::<Value>(raw) else {
|
||||
return raw.to_string();
|
||||
};
|
||||
|
||||
let contains_installation_identity = [
|
||||
"installation_id",
|
||||
"installationId",
|
||||
"x-codex-installation-id",
|
||||
]
|
||||
.into_iter()
|
||||
.any(|key| metadata.contains_key(key));
|
||||
if !contains_installation_identity {
|
||||
return raw.to_string();
|
||||
}
|
||||
|
||||
for key in [
|
||||
"installation_id",
|
||||
"installationId",
|
||||
"x-codex-installation-id",
|
||||
] {
|
||||
replace_existing_metadata_field(&mut metadata, key, Some(&identity.installation_id));
|
||||
}
|
||||
|
||||
serde_json::to_string(&metadata).unwrap_or_else(|_| raw.to_string())
|
||||
}
|
||||
|
||||
fn pseudonym(&self, domain: &str, account_id: Option<&str>, value: &str) -> String {
|
||||
self.pseudonymizer.scoped(domain, account_id, value)
|
||||
}
|
||||
}
|
||||
|
||||
fn client_metadata_string(request: &CodexResponsesRequest, key: &str) -> Option<String> {
|
||||
request
|
||||
.client_metadata()?
|
||||
.as_object()?
|
||||
.get(key)?
|
||||
.as_str()
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn apply_identity_to_request(
|
||||
request: &mut CodexResponsesRequest,
|
||||
identity: &AccountScopedIdentity,
|
||||
turn_state: Option<&str>,
|
||||
turn_metadata: Option<&str>,
|
||||
client_metadata_turn_state: Option<&str>,
|
||||
client_metadata_turn_metadata: Option<&str>,
|
||||
) {
|
||||
for (key, value) in [
|
||||
(
|
||||
"x-codex-installation-id",
|
||||
Some(identity.installation_id.as_str()),
|
||||
),
|
||||
("installation_id", Some(identity.installation_id.as_str())),
|
||||
("installationId", Some(identity.installation_id.as_str())),
|
||||
("turnState", turn_state),
|
||||
("x-codex-turn-state", turn_state),
|
||||
("turnMetadata", turn_metadata),
|
||||
("x-codex-turn-metadata", turn_metadata),
|
||||
] {
|
||||
request.replace_existing_identity_field(key, value);
|
||||
}
|
||||
|
||||
let client_metadata = scoped_client_metadata(
|
||||
request.client_metadata(),
|
||||
identity,
|
||||
client_metadata_turn_state,
|
||||
client_metadata_turn_metadata,
|
||||
);
|
||||
request.set_client_metadata(client_metadata);
|
||||
}
|
||||
|
||||
fn scoped_client_metadata(
|
||||
client_metadata: Option<&Value>,
|
||||
identity: &AccountScopedIdentity,
|
||||
turn_state: Option<&str>,
|
||||
turn_metadata: Option<&str>,
|
||||
) -> Option<Value> {
|
||||
let client_metadata = client_metadata?;
|
||||
let Value::Object(mut metadata) = client_metadata.clone() else {
|
||||
return Some(client_metadata.clone());
|
||||
};
|
||||
|
||||
for (key, value) in [
|
||||
(
|
||||
"x-codex-installation-id",
|
||||
Some(identity.installation_id.as_str()),
|
||||
),
|
||||
(TURN_STATE_METADATA_KEY, turn_state),
|
||||
(TURN_METADATA_KEY, turn_metadata),
|
||||
] {
|
||||
replace_metadata_field(&mut metadata, key, value);
|
||||
}
|
||||
replace_existing_metadata_field(
|
||||
&mut metadata,
|
||||
"installation_id",
|
||||
Some(&identity.installation_id),
|
||||
);
|
||||
replace_existing_metadata_field(
|
||||
&mut metadata,
|
||||
"installationId",
|
||||
Some(&identity.installation_id),
|
||||
);
|
||||
|
||||
(!metadata.is_empty()).then_some(Value::Object(metadata))
|
||||
}
|
||||
|
||||
fn replace_existing_metadata_field(
|
||||
metadata: &mut Map<String, Value>,
|
||||
key: &str,
|
||||
value: Option<&str>,
|
||||
) {
|
||||
if metadata.contains_key(key) {
|
||||
replace_metadata_field(metadata, key, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_metadata_field(metadata: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
metadata.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
metadata.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@ pub mod service;
|
||||
pub mod store;
|
||||
pub mod types;
|
||||
|
||||
pub use identity::{AccountIdentityService, AccountScopedIdentity};
|
||||
pub use identity::AccountIdentityService;
|
||||
pub(in crate::dispatch) use identity::{AccountIdentityScope, AccountScopedRequest};
|
||||
pub use resolve::*;
|
||||
pub use service::*;
|
||||
pub use store::*;
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
controllers::ControllerFailureFact,
|
||||
controllers::{
|
||||
ControllerFailureFact,
|
||||
account_state::{AccountStateEffect, AccountStateEffects},
|
||||
},
|
||||
errors::ClientFailure,
|
||||
failure::exhaustion::{AccountExhaustionRecord, ExhaustedAccountKind},
|
||||
lifecycle::contract::{
|
||||
@@ -11,19 +14,14 @@ use crate::{
|
||||
transport::observation::{UpstreamFailureFacts, UpstreamFailureKind},
|
||||
},
|
||||
fleet::{account::AccountStatus, pool::AccountPoolService},
|
||||
upstream::openai::protocol::responses::ResponsesSseFailure,
|
||||
upstream::openai::{protocol::responses::ResponsesSseFailure, transport::CodexBackendClient},
|
||||
};
|
||||
|
||||
pub(super) struct AccountFailureController;
|
||||
|
||||
pub(super) struct ClassifiedFailure {
|
||||
exhaustion: AccountExhaustionRecord,
|
||||
effect: AccountEffect,
|
||||
}
|
||||
|
||||
enum AccountEffect {
|
||||
None,
|
||||
SetStatus(AccountStatus),
|
||||
effect: Option<AccountStateEffect>,
|
||||
}
|
||||
|
||||
impl AccountFailureController {
|
||||
@@ -41,16 +39,14 @@ impl AccountFailureController {
|
||||
|
||||
pub(super) async fn apply_effect(
|
||||
account_pool: &AccountPoolService,
|
||||
codex: &CodexBackendClient,
|
||||
classified: &ClassifiedFailure,
|
||||
) {
|
||||
let Some(account_id) = classified.exhaustion.account_id.as_deref() else {
|
||||
return;
|
||||
};
|
||||
match &classified.effect {
|
||||
AccountEffect::None => {}
|
||||
AccountEffect::SetStatus(status) => {
|
||||
account_pool.set_status(account_id, *status).await;
|
||||
}
|
||||
if let Some(effect) = &classified.effect {
|
||||
AccountStateEffects::apply(account_pool, codex, account_id, effect).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,6 +110,11 @@ impl AccountFailureController {
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
let error_type = failure
|
||||
.upstream_type
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
let message = failure.message.to_ascii_lowercase();
|
||||
if is_model_unsupported(&code)
|
||||
|| is_model_unsupported(&message)
|
||||
@@ -125,10 +126,11 @@ impl AccountFailureController {
|
||||
{
|
||||
return Some(ClientFailure::new(failure.clone(), 400, false));
|
||||
}
|
||||
if is_auth_failure(&code, &message) {
|
||||
if is_auth_failure(&code, &error_type, &message) {
|
||||
return Some(ClientFailure::new(failure.clone(), 401, false));
|
||||
}
|
||||
if code.contains("forbidden") || code.contains("banned") {
|
||||
if code.contains("forbidden") || code.contains("banned") || error_type == "permission_error"
|
||||
{
|
||||
return Some(ClientFailure::new(failure.clone(), 403, false));
|
||||
}
|
||||
if code.contains("server_overloaded") {
|
||||
@@ -147,7 +149,7 @@ fn classify(observation: &AttemptObservation, account_id: &str) -> Option<Classi
|
||||
ExhaustedAccountKind::UpstreamUnavailable,
|
||||
"upstream response did not include visible output",
|
||||
),
|
||||
effect: AccountEffect::None,
|
||||
effect: None,
|
||||
})
|
||||
}
|
||||
_ => ControllerFailureFact::from_attempt(observation)
|
||||
@@ -168,108 +170,151 @@ fn classify_failure(
|
||||
fn classify_upstream(account_id: &str, facts: &UpstreamFailureFacts) -> Option<ClassifiedFailure> {
|
||||
let status = facts.status_code;
|
||||
let body = facts.body.as_str();
|
||||
let lower = body.to_ascii_lowercase();
|
||||
let (kind, effect, message, status_code) = if (status == Some(403) && !is_html(body))
|
||||
|| (status == Some(402) && is_deactivated_workspace(body))
|
||||
{
|
||||
(
|
||||
ExhaustedAccountKind::Banned,
|
||||
AccountEffect::SetStatus(AccountStatus::Banned),
|
||||
body.to_string(),
|
||||
let code = facts.code.as_deref().unwrap_or_default();
|
||||
let error_type = facts.error_type.as_deref().unwrap_or_default();
|
||||
let message = facts.message.as_str();
|
||||
|
||||
if is_model_unsupported(code) || is_model_unsupported(message) || is_model_unsupported(body) {
|
||||
return Some(model_unsupported_failure(account_id, body));
|
||||
}
|
||||
if let Some(account_status) = explicit_account_status(code, error_type, body) {
|
||||
return Some(account_status_failure(
|
||||
account_id,
|
||||
account_status,
|
||||
body,
|
||||
status,
|
||||
)
|
||||
} else if status == Some(401) {
|
||||
let account_status = if lower.contains("banned") || lower.contains("deactivated") {
|
||||
AccountStatus::Banned
|
||||
} else {
|
||||
AccountStatus::Expired
|
||||
};
|
||||
(
|
||||
if account_status == AccountStatus::Banned {
|
||||
ExhaustedAccountKind::Banned
|
||||
} else {
|
||||
ExhaustedAccountKind::Expired
|
||||
},
|
||||
AccountEffect::SetStatus(account_status),
|
||||
body.to_string(),
|
||||
));
|
||||
}
|
||||
if status == Some(403) && !is_html(body) {
|
||||
return Some(account_status_failure(
|
||||
account_id,
|
||||
AccountStatus::Banned,
|
||||
body,
|
||||
status,
|
||||
)
|
||||
} else if status.is_some_and(|status| {
|
||||
(400..=499).contains(&status) && !matches!(status, 401 | 402 | 403 | 404 | 429)
|
||||
}) && is_model_unsupported(body)
|
||||
{
|
||||
(
|
||||
));
|
||||
}
|
||||
if status == Some(401) {
|
||||
return Some(account_status_failure(
|
||||
account_id,
|
||||
AccountStatus::Expired,
|
||||
body,
|
||||
status,
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn classify_sse(account_id: &str, failure: &ResponsesSseFailure) -> Option<ClassifiedFailure> {
|
||||
let code = failure.upstream_code.as_deref().unwrap_or_default();
|
||||
let error_type = failure.upstream_type.as_deref().unwrap_or_default();
|
||||
if is_model_unsupported(code) || is_model_unsupported(&failure.message) {
|
||||
return Some(model_unsupported_failure(
|
||||
account_id,
|
||||
&crate::dispatch::failure::sse::sse_failure_error_body(failure),
|
||||
));
|
||||
}
|
||||
let account_status =
|
||||
explicit_account_status(code, error_type, &failure.message).or_else(|| {
|
||||
(code.eq_ignore_ascii_case("forbidden") || error_type == "permission_error")
|
||||
.then_some(AccountStatus::Banned)
|
||||
})?;
|
||||
Some(account_status_failure(
|
||||
account_id,
|
||||
account_status,
|
||||
&crate::dispatch::failure::sse::sse_failure_error_body(failure),
|
||||
failure.explicit_status_code,
|
||||
))
|
||||
}
|
||||
|
||||
fn model_unsupported_failure(account_id: &str, message: &str) -> ClassifiedFailure {
|
||||
ClassifiedFailure {
|
||||
exhaustion: AccountExhaustionRecord::new(
|
||||
account_id,
|
||||
ExhaustedAccountKind::ModelUnsupported,
|
||||
AccountEffect::None,
|
||||
body.to_string(),
|
||||
None,
|
||||
)
|
||||
} else {
|
||||
return None;
|
||||
message,
|
||||
),
|
||||
effect: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn account_status_failure(
|
||||
account_id: &str,
|
||||
status: AccountStatus,
|
||||
message: &str,
|
||||
status_code: Option<u16>,
|
||||
) -> ClassifiedFailure {
|
||||
let kind = match status {
|
||||
AccountStatus::Expired => ExhaustedAccountKind::Expired,
|
||||
AccountStatus::Disabled => ExhaustedAccountKind::Disabled,
|
||||
AccountStatus::Banned => ExhaustedAccountKind::Banned,
|
||||
AccountStatus::Active | AccountStatus::QuotaExhausted => {
|
||||
unreachable!("account failure controller only invalidates unavailable statuses")
|
||||
}
|
||||
};
|
||||
let mut exhaustion = AccountExhaustionRecord::new(account_id, kind, message);
|
||||
if let Some(status_code) = status_code {
|
||||
exhaustion = exhaustion.with_status_code(status_code);
|
||||
}
|
||||
Some(ClassifiedFailure { exhaustion, effect })
|
||||
}
|
||||
|
||||
fn classify_sse(account_id: &str, failure: &ResponsesSseFailure) -> Option<ClassifiedFailure> {
|
||||
let code = failure.upstream_code.as_deref().unwrap_or_default();
|
||||
let lower_message = failure.message.to_ascii_lowercase();
|
||||
let (kind, effect, status) = if is_model_unsupported(code)
|
||||
|| is_model_unsupported(&failure.message)
|
||||
{
|
||||
(
|
||||
ExhaustedAccountKind::ModelUnsupported,
|
||||
AccountEffect::None,
|
||||
None,
|
||||
)
|
||||
} else if is_auth_failure(code, &lower_message) {
|
||||
let account_status = if lower_message.contains("banned") || code == "account_deactivated" {
|
||||
AccountStatus::Banned
|
||||
} else {
|
||||
AccountStatus::Expired
|
||||
};
|
||||
(
|
||||
if account_status == AccountStatus::Banned {
|
||||
ExhaustedAccountKind::Banned
|
||||
} else {
|
||||
ExhaustedAccountKind::Expired
|
||||
},
|
||||
AccountEffect::SetStatus(account_status),
|
||||
Some(if account_status == AccountStatus::Banned {
|
||||
403
|
||||
} else {
|
||||
401
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
let mut exhaustion = AccountExhaustionRecord::new(
|
||||
account_id,
|
||||
kind,
|
||||
crate::dispatch::failure::sse::sse_failure_error_body(failure),
|
||||
);
|
||||
if let Some(status) = status {
|
||||
exhaustion = exhaustion.with_status_code(status);
|
||||
ClassifiedFailure {
|
||||
exhaustion,
|
||||
effect: Some(AccountStateEffect::SetStatus(status)),
|
||||
}
|
||||
Some(ClassifiedFailure { exhaustion, effect })
|
||||
}
|
||||
|
||||
fn is_auth_failure(code: &str, message: &str) -> bool {
|
||||
fn explicit_account_status(code: &str, error_type: &str, message: &str) -> Option<AccountStatus> {
|
||||
let code = code.trim().to_ascii_lowercase();
|
||||
let error_type = error_type.trim().to_ascii_lowercase();
|
||||
let message = message.to_ascii_lowercase();
|
||||
if matches!(
|
||||
code.as_str(),
|
||||
"identity_verification_required" | "verification_required"
|
||||
) || message.contains("identity verification is required")
|
||||
{
|
||||
return Some(AccountStatus::Disabled);
|
||||
}
|
||||
if matches!(
|
||||
code.as_str(),
|
||||
"account_banned"
|
||||
| "account_deactivated"
|
||||
| "account_disabled"
|
||||
| "account_suspended"
|
||||
| "deactivated_workspace"
|
||||
| "organization_disabled"
|
||||
| "workspace_deactivated"
|
||||
) || message.contains("account is banned")
|
||||
|| message.contains("account has been banned")
|
||||
|| message.contains("account deactivated")
|
||||
|| message.contains("account has been deactivated")
|
||||
|| message.contains("account disabled")
|
||||
|| message.contains("account has been disabled")
|
||||
|| message.contains("account suspended")
|
||||
|| message.contains("organization has been disabled")
|
||||
|| message.contains("workspace has been deactivated")
|
||||
|| message.contains("deactivated_workspace")
|
||||
{
|
||||
return Some(AccountStatus::Banned);
|
||||
}
|
||||
is_auth_failure(&code, &error_type, &message).then_some(AccountStatus::Expired)
|
||||
}
|
||||
|
||||
fn is_auth_failure(code: &str, error_type: &str, message: &str) -> bool {
|
||||
matches!(
|
||||
code,
|
||||
"token_invalid"
|
||||
| "token_invalidated"
|
||||
| "token_expired"
|
||||
| "token_revoked"
|
||||
| "account_deactivated"
|
||||
| "refresh_token_invalidated"
|
||||
| "unauthorized"
|
||||
| "invalid_api_key"
|
||||
) || message.contains("token revoked")
|
||||
| "authentication_error"
|
||||
) || error_type == "authentication_error"
|
||||
|| message.contains("token revoked")
|
||||
|| message.contains("token invalidated")
|
||||
|| message.contains("token invalid")
|
||||
|| message.contains("token expired")
|
||||
|| message.contains("unauthorized")
|
||||
|| message.contains("invalid api key")
|
||||
}
|
||||
|
||||
fn is_model_unsupported(value: &str) -> bool {
|
||||
@@ -287,12 +332,3 @@ fn is_html(value: &str) -> bool {
|
||||
let value = value.trim_start().to_ascii_lowercase();
|
||||
value.starts_with("<!doctype") || value.starts_with("<html") || value.contains("<html")
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(value: &str) -> bool {
|
||||
serde_json::from_str::<serde_json::Value>(value).is_ok_and(|value| {
|
||||
value
|
||||
.pointer("/detail/code")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some("deactivated_workspace")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
//! 明确账号事实触发的运行时失效与连接驱逐。
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::{
|
||||
fleet::{account::AccountStatus, pool::AccountPoolService},
|
||||
upstream::openai::transport::CodexBackendClient,
|
||||
};
|
||||
|
||||
pub(super) enum AccountStateEffect {
|
||||
SetStatus(AccountStatus),
|
||||
MarkQuotaLimitedUntil(DateTime<Utc>),
|
||||
}
|
||||
|
||||
pub(super) struct AccountStateEffects;
|
||||
|
||||
impl AccountStateEffects {
|
||||
pub(super) async fn apply(
|
||||
account_pool: &AccountPoolService,
|
||||
codex: &CodexBackendClient,
|
||||
account_id: &str,
|
||||
effect: &AccountStateEffect,
|
||||
) {
|
||||
match effect {
|
||||
AccountStateEffect::SetStatus(status) => {
|
||||
account_pool
|
||||
.set_status_immediately(account_id, *status)
|
||||
.await;
|
||||
}
|
||||
AccountStateEffect::MarkQuotaLimitedUntil(until) => {
|
||||
account_pool
|
||||
.mark_quota_limited_until_immediately(account_id, *until)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
codex.evict_websocket_account(account_id).await;
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,10 @@ use tokio::time::timeout;
|
||||
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::{AccountIdentityService, SessionAffinityService},
|
||||
affinity::{
|
||||
AccountIdentityScope, AccountIdentityService, AccountScopedRequest,
|
||||
SessionAffinityService,
|
||||
},
|
||||
errors::{ClientFailure, ResponseDispatchError, upstream_error_set_cookie_headers},
|
||||
lifecycle::trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
|
||||
},
|
||||
@@ -53,6 +56,7 @@ use crate::{
|
||||
};
|
||||
|
||||
mod account_failure;
|
||||
mod account_state;
|
||||
mod affinity;
|
||||
pub(crate) mod cloudflare;
|
||||
pub(crate) mod cyber_policy;
|
||||
@@ -243,6 +247,7 @@ pub(in crate::dispatch) struct ControllerSet {
|
||||
cyber_policy: CyberPolicyController,
|
||||
session_affinity: Arc<SessionAffinityService>,
|
||||
account_pool: Arc<AccountPoolService>,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
cloudflare: CloudflareRecovery,
|
||||
recorder: Arc<Recorder>,
|
||||
}
|
||||
@@ -251,6 +256,7 @@ pub(in crate::dispatch) struct ControllerSet {
|
||||
pub(in crate::dispatch) struct ControllerRequestScope {
|
||||
cyber_policy: CyberPolicyScope,
|
||||
history: HistoryState,
|
||||
identity: AccountIdentityScope,
|
||||
usage: usage::UsageScope,
|
||||
}
|
||||
|
||||
@@ -277,8 +283,13 @@ pub(in crate::dispatch) enum AttemptAccountPreparation {
|
||||
Rejected,
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) struct AttemptRoutePreparationContext<'a> {
|
||||
pub account_identity: &'a AccountIdentityService,
|
||||
pub account_id: &'a str,
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) enum AttemptRoutePreparation {
|
||||
Ready(Box<CodexResponsesRequest>),
|
||||
Ready(Box<AccountScopedRequest>),
|
||||
Unavailable { message: String },
|
||||
}
|
||||
|
||||
@@ -323,12 +334,21 @@ impl ControllerSet {
|
||||
&self,
|
||||
scope: &mut ControllerRequestScope,
|
||||
request: &CodexResponsesRequest,
|
||||
account_id: &str,
|
||||
context: AttemptRoutePreparationContext<'_>,
|
||||
) -> AttemptRoutePreparation {
|
||||
match HistoryController::prepare_attempt(&mut scope.history, request, account_id) {
|
||||
Ok(request) => AttemptRoutePreparation::Ready(Box::new(request)),
|
||||
Err(message) => AttemptRoutePreparation::Unavailable { message },
|
||||
}
|
||||
let request = match HistoryController::prepare_attempt(
|
||||
&mut scope.history,
|
||||
request,
|
||||
context.account_id,
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(message) => return AttemptRoutePreparation::Unavailable { message },
|
||||
};
|
||||
AttemptRoutePreparation::Ready(Box::new(context.account_identity.scope_request(
|
||||
&mut scope.identity,
|
||||
request,
|
||||
context.account_id,
|
||||
)))
|
||||
}
|
||||
|
||||
pub(in crate::dispatch) fn prepare_same_account_retry(
|
||||
@@ -473,6 +493,7 @@ impl ControllerSet {
|
||||
pub(in crate::dispatch) fn new(
|
||||
session_affinity: Arc<crate::dispatch::affinity::SessionAffinityService>,
|
||||
account_pool: Arc<AccountPoolService>,
|
||||
codex: Arc<CodexBackendClient>,
|
||||
cloudflare: CloudflareRecovery,
|
||||
recorder: Arc<Recorder>,
|
||||
) -> Self {
|
||||
@@ -480,6 +501,7 @@ impl ControllerSet {
|
||||
cyber_policy: CyberPolicyController::new(Arc::clone(&session_affinity)),
|
||||
session_affinity,
|
||||
account_pool,
|
||||
codex,
|
||||
cloudflare,
|
||||
recorder,
|
||||
}
|
||||
@@ -508,6 +530,7 @@ impl ControllerSet {
|
||||
scope: ControllerRequestScope {
|
||||
cyber_policy,
|
||||
history,
|
||||
identity: AccountIdentityScope::new(preferred_account_id.clone()),
|
||||
usage,
|
||||
},
|
||||
preferred_account_id,
|
||||
@@ -553,33 +576,32 @@ impl ControllerSet {
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_stream_failure(
|
||||
async fn apply_stream_account_state(&self, classification: &StreamFailureClassification<'_>) {
|
||||
match classification.owner.as_ref() {
|
||||
Some(SharedFailureClassification::AccountFailure(classified)) => {
|
||||
AccountFailureController::apply_effect(&self.account_pool, &self.codex, classified)
|
||||
.await;
|
||||
}
|
||||
Some(SharedFailureClassification::Quota(classified)) => {
|
||||
QuotaController::apply_effect(&self.account_pool, &self.codex, classified).await;
|
||||
}
|
||||
Some(SharedFailureClassification::CyberPolicy(_)) | None => {}
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_stream_policy_failure(
|
||||
&self,
|
||||
scope: &ControllerRequestScope,
|
||||
account_id: &str,
|
||||
classification: &StreamFailureClassification<'_>,
|
||||
) {
|
||||
match classification.owner.as_ref() {
|
||||
Some(SharedFailureClassification::CyberPolicy(_)) => {
|
||||
self.cyber_policy
|
||||
.exclude_account(&scope.cyber_policy, account_id)
|
||||
.await;
|
||||
}
|
||||
Some(SharedFailureClassification::AccountFailure(classified)) => {
|
||||
let _ = best_effort_controller_io(
|
||||
"account_failure.stream",
|
||||
AccountFailureController::apply_effect(&self.account_pool, classified),
|
||||
)
|
||||
if matches!(
|
||||
classification.owner,
|
||||
Some(SharedFailureClassification::CyberPolicy(_))
|
||||
) {
|
||||
self.cyber_policy
|
||||
.exclude_account(&scope.cyber_policy, account_id)
|
||||
.await;
|
||||
}
|
||||
Some(SharedFailureClassification::Quota(classified)) => {
|
||||
let _ = best_effort_controller_io(
|
||||
"quota.stream",
|
||||
QuotaController::apply_effect(&self.account_pool, classified),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -611,6 +633,11 @@ impl ControllerSet {
|
||||
StreamTerminal::Cancelled | StreamTerminal::DownstreamClosed => FinalOutcome::Cancelled,
|
||||
StreamTerminal::Shutdown => FinalOutcome::Shutdown,
|
||||
};
|
||||
if !matches!(outcome, FinalOutcome::Cancelled | FinalOutcome::Shutdown)
|
||||
&& let Some(classification) = stream_failure.as_ref()
|
||||
{
|
||||
self.apply_stream_account_state(classification).await;
|
||||
}
|
||||
let _ = best_effort_controller_io(
|
||||
"cloudflare",
|
||||
cloudflare::CloudflareController::leave_stream(cloudflare::StreamExit {
|
||||
@@ -684,7 +711,7 @@ impl ControllerSet {
|
||||
|
||||
if !matches!(outcome, FinalOutcome::Cancelled | FinalOutcome::Shutdown) {
|
||||
if let Some(classification) = stream_failure.as_ref() {
|
||||
self.apply_stream_failure(&scope, &context.account_id, classification)
|
||||
self.apply_stream_policy_failure(&scope, &context.account_id, classification)
|
||||
.await;
|
||||
} else {
|
||||
self.finalize_stream_success(&scope, &summary.terminal)
|
||||
@@ -730,18 +757,16 @@ impl ControllerSet {
|
||||
}
|
||||
}
|
||||
SharedFailureClassification::AccountFailure(classified) => {
|
||||
let _ = best_effort_controller_io(
|
||||
"account_failure.attempt",
|
||||
AccountFailureController::apply_effect(&self.account_pool, classified),
|
||||
AccountFailureController::apply_effect(
|
||||
&self.account_pool,
|
||||
&self.codex,
|
||||
classified,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
SharedFailureClassification::Quota(classified) => {
|
||||
let _ = best_effort_controller_io(
|
||||
"quota.attempt",
|
||||
QuotaController::apply_effect(&self.account_pool, classified),
|
||||
)
|
||||
.await;
|
||||
QuotaController::apply_effect(&self.account_pool, &self.codex, classified)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
AttemptClassification::Cloudflare(failure) => {
|
||||
|
||||
@@ -6,7 +6,10 @@ use serde_json::Value;
|
||||
use crate::{
|
||||
dispatch::{
|
||||
affinity::AccountIdentityService,
|
||||
controllers::ControllerFailureFact,
|
||||
controllers::{
|
||||
ControllerFailureFact,
|
||||
account_state::{AccountStateEffect, AccountStateEffects},
|
||||
},
|
||||
errors::ClientFailure,
|
||||
failure::exhaustion::{AccountExhaustionRecord, ExhaustedAccountKind},
|
||||
lifecycle::contract::{
|
||||
@@ -55,9 +58,7 @@ impl QuotaController {
|
||||
}
|
||||
|
||||
let account_id = acquired.account.id.clone();
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope_auxiliary(&account_id, context.request_id);
|
||||
let identity = context.account_identity.scope_auxiliary(&account_id);
|
||||
let usage = context
|
||||
.codex
|
||||
.fetch_usage(CodexRequestContext {
|
||||
@@ -75,8 +76,7 @@ impl QuotaController {
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
.await;
|
||||
@@ -132,7 +132,7 @@ impl QuotaController {
|
||||
ExhaustedAccountKind::RateLimited,
|
||||
LIMIT_REACHED_MESSAGE,
|
||||
),
|
||||
effect: QuotaEffect::None,
|
||||
effect: None,
|
||||
});
|
||||
}
|
||||
classify(observation)
|
||||
@@ -160,26 +160,14 @@ impl QuotaController {
|
||||
|
||||
pub(super) async fn apply_effect(
|
||||
account_pool: &AccountPoolService,
|
||||
codex: &CodexBackendClient,
|
||||
classified: &ClassifiedQuotaFailure,
|
||||
) {
|
||||
let Some(account_id) = classified.exhaustion.account_id.as_deref() else {
|
||||
return;
|
||||
};
|
||||
match classified.effect {
|
||||
QuotaEffect::None => {}
|
||||
QuotaEffect::SetExhausted => {
|
||||
account_pool
|
||||
.set_status(
|
||||
account_id,
|
||||
crate::fleet::account::AccountStatus::QuotaExhausted,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
QuotaEffect::MarkLimitedUntil(until) => {
|
||||
account_pool
|
||||
.mark_quota_limited_until(account_id, until)
|
||||
.await;
|
||||
}
|
||||
if let Some(effect) = &classified.effect {
|
||||
AccountStateEffects::apply(account_pool, codex, account_id, effect).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,15 +176,9 @@ impl QuotaController {
|
||||
}
|
||||
}
|
||||
|
||||
enum QuotaEffect {
|
||||
None,
|
||||
SetExhausted,
|
||||
MarkLimitedUntil(chrono::DateTime<Utc>),
|
||||
}
|
||||
|
||||
pub(super) struct ClassifiedQuotaFailure {
|
||||
exhaustion: AccountExhaustionRecord,
|
||||
effect: QuotaEffect,
|
||||
effect: Option<AccountStateEffect>,
|
||||
}
|
||||
|
||||
fn classify(observation: &AttemptObservation) -> Option<ClassifiedQuotaFailure> {
|
||||
@@ -219,29 +201,37 @@ fn classify_upstream(
|
||||
account_id: &str,
|
||||
facts: &UpstreamFailureFacts,
|
||||
) -> Option<ClassifiedQuotaFailure> {
|
||||
if facts.status_code == Some(429) {
|
||||
return Some(rate_limited_failure(
|
||||
let signaled_kind = facts
|
||||
.code
|
||||
.as_deref()
|
||||
.and_then(classify_quota_signal)
|
||||
.or_else(|| facts.error_type.as_deref().and_then(classify_quota_signal))
|
||||
.or_else(|| classify_quota_message(&facts.message));
|
||||
match signaled_kind.or(match facts.status_code {
|
||||
Some(429) => Some(StreamQuotaKind::RateLimited),
|
||||
Some(402) => Some(StreamQuotaKind::QuotaExhausted),
|
||||
_ => None,
|
||||
})? {
|
||||
StreamQuotaKind::RateLimited => Some(rate_limited_failure(
|
||||
account_id,
|
||||
facts.body.clone(),
|
||||
facts.retry_after_seconds,
|
||||
));
|
||||
)),
|
||||
StreamQuotaKind::QuotaExhausted => {
|
||||
Some(quota_exhausted_failure(account_id, facts.body.clone()))
|
||||
}
|
||||
}
|
||||
if facts.status_code == Some(402) {
|
||||
return Some(quota_exhausted_failure(account_id, facts.body.clone()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn classify_stream(
|
||||
account_id: &str,
|
||||
failure: &ResponsesSseFailure,
|
||||
) -> Option<ClassifiedQuotaFailure> {
|
||||
let kind = match failure.explicit_status_code {
|
||||
Some(429) => StreamQuotaKind::RateLimited,
|
||||
Some(402) => StreamQuotaKind::QuotaExhausted,
|
||||
Some(_) => return None,
|
||||
None => classify_stream_signal(failure)?,
|
||||
};
|
||||
let kind = classify_stream_signal(failure).or(match failure.explicit_status_code {
|
||||
Some(429) => Some(StreamQuotaKind::RateLimited),
|
||||
Some(402) => Some(StreamQuotaKind::QuotaExhausted),
|
||||
_ => None,
|
||||
})?;
|
||||
let body = crate::dispatch::failure::sse::sse_failure_error_body(failure);
|
||||
Some(match kind {
|
||||
StreamQuotaKind::RateLimited => {
|
||||
@@ -277,10 +267,15 @@ fn classify_quota_signal(signal: &str) -> Option<StreamQuotaKind> {
|
||||
"usage_limit_reached"
|
||||
| "rate_limit_exceeded"
|
||||
| "rate_limit_reached"
|
||||
| "rate_limit_error" => Some(StreamQuotaKind::RateLimited),
|
||||
"quota_exhausted" | "quota_exceeded" | "payment_required" | "insufficient_quota" => {
|
||||
Some(StreamQuotaKind::QuotaExhausted)
|
||||
}
|
||||
| "rate_limit_error"
|
||||
| "workspace_owner_usage_limit_reached"
|
||||
| "workspace_member_usage_limit_reached" => Some(StreamQuotaKind::RateLimited),
|
||||
"quota_exhausted"
|
||||
| "quota_exceeded"
|
||||
| "payment_required"
|
||||
| "insufficient_quota"
|
||||
| "workspace_owner_credits_depleted"
|
||||
| "workspace_member_credits_depleted" => Some(StreamQuotaKind::QuotaExhausted),
|
||||
signal if signal.starts_with("billing_limit") => Some(StreamQuotaKind::QuotaExhausted),
|
||||
_ => None,
|
||||
}
|
||||
@@ -314,7 +309,9 @@ fn rate_limited_failure(
|
||||
ExhaustedAccountKind::RateLimited,
|
||||
message,
|
||||
),
|
||||
effect: QuotaEffect::MarkLimitedUntil(Utc::now() + Duration::seconds(seconds)),
|
||||
effect: Some(AccountStateEffect::MarkQuotaLimitedUntil(
|
||||
Utc::now() + Duration::seconds(seconds),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,7 +322,9 @@ fn quota_exhausted_failure(account_id: &str, message: String) -> ClassifiedQuota
|
||||
ExhaustedAccountKind::QuotaExhausted,
|
||||
message,
|
||||
),
|
||||
effect: QuotaEffect::SetExhausted,
|
||||
effect: Some(AccountStateEffect::SetStatus(
|
||||
crate::fleet::account::AccountStatus::QuotaExhausted,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ use crate::{
|
||||
affinity::AccountIdentityService,
|
||||
controllers::{
|
||||
AttemptAccountPreparation, AttemptAccountPreparationContext, AttemptRoutePreparation,
|
||||
AttemptUpstreamErrorContext, ControllerRequestScope, ControllerSet,
|
||||
AttemptRoutePreparationContext, AttemptUpstreamErrorContext, ControllerRequestScope,
|
||||
ControllerSet,
|
||||
},
|
||||
errors::{ClientFailure, ResponseDispatchError},
|
||||
failure::exhaustion::AccountExhaustionTracker,
|
||||
@@ -218,10 +219,13 @@ impl<'a> AttemptRunner<'a> {
|
||||
let acquired = prepared_account.lease;
|
||||
let cookie_header = prepared_account.cookie_header;
|
||||
|
||||
let attempt_request = match self.dependencies.controllers.prepare_attempt_route(
|
||||
let account_scoped_request = match self.dependencies.controllers.prepare_attempt_route(
|
||||
controller_scope,
|
||||
request,
|
||||
&account.id,
|
||||
AttemptRoutePreparationContext {
|
||||
account_identity: self.dependencies.account_identity,
|
||||
account_id: &account.id,
|
||||
},
|
||||
) {
|
||||
AttemptRoutePreparation::Ready(request) => *request,
|
||||
AttemptRoutePreparation::Unavailable { message } => {
|
||||
@@ -242,7 +246,6 @@ impl<'a> AttemptRunner<'a> {
|
||||
.start_attempt(&account.id);
|
||||
let upstream_context = AccountUpstreamContext {
|
||||
codex: self.dependencies.codex,
|
||||
account_identity: self.dependencies.account_identity,
|
||||
request_id: self.dependencies.request_id,
|
||||
account: &account,
|
||||
cookie_header: cookie_header.as_deref(),
|
||||
@@ -253,9 +256,13 @@ impl<'a> AttemptRunner<'a> {
|
||||
started_at,
|
||||
tuple_schema,
|
||||
} => {
|
||||
let result =
|
||||
create_response_with_account(upstream_context, &attempt_request, *started_at)
|
||||
.await;
|
||||
let result = create_response_with_account(
|
||||
upstream_context,
|
||||
&account_scoped_request,
|
||||
*started_at,
|
||||
)
|
||||
.await;
|
||||
let attempt_request = account_scoped_request.into_request();
|
||||
acquired.complete().await;
|
||||
self.pending = Some(match result {
|
||||
Ok(response) => {
|
||||
@@ -317,7 +324,9 @@ impl<'a> AttemptRunner<'a> {
|
||||
}
|
||||
AttemptMode::Stream { tuple_schema } => {
|
||||
let result =
|
||||
create_response_stream_with_account(upstream_context, &attempt_request).await;
|
||||
create_response_stream_with_account(upstream_context, &account_scoped_request)
|
||||
.await;
|
||||
let attempt_request = account_scoped_request.into_request();
|
||||
self.pending = Some(match result {
|
||||
Err(error) => {
|
||||
acquired.complete().await;
|
||||
|
||||
@@ -183,6 +183,7 @@ impl ResponseDispatchService {
|
||||
let controllers = ControllerSet::new(
|
||||
Arc::clone(&self.session_affinity),
|
||||
Arc::clone(&self.account_pool),
|
||||
Arc::clone(&self.codex),
|
||||
self.cloudflare.clone(),
|
||||
Arc::clone(&self.recorder),
|
||||
);
|
||||
|
||||
@@ -39,6 +39,7 @@ impl ResponseDispatchService {
|
||||
let controllers = ControllerSet::new(
|
||||
Arc::clone(&self.session_affinity),
|
||||
Arc::clone(&self.account_pool),
|
||||
Arc::clone(&self.codex),
|
||||
self.cloudflare.clone(),
|
||||
Arc::clone(&self.recorder),
|
||||
);
|
||||
|
||||
@@ -3,21 +3,17 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::{
|
||||
dispatch::affinity::AccountIdentityService,
|
||||
dispatch::affinity::AccountScopedRequest,
|
||||
fleet::account::Account,
|
||||
upstream::openai::{
|
||||
protocol::responses::CodexResponsesRequest,
|
||||
transport::{
|
||||
CodexBackendClient, CodexBackendResponse, CodexBackendStreamingResponse,
|
||||
CodexClientError, CodexRequestContext,
|
||||
},
|
||||
upstream::openai::transport::{
|
||||
CodexBackendClient, CodexBackendResponse, CodexBackendStreamingResponse, CodexClientError,
|
||||
CodexRequestContext,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(in crate::dispatch) struct AccountUpstreamContext<'a> {
|
||||
pub codex: &'a CodexBackendClient,
|
||||
pub account_identity: &'a AccountIdentityService,
|
||||
pub request_id: &'a str,
|
||||
pub account: &'a Account,
|
||||
pub cookie_header: Option<&'a str>,
|
||||
@@ -25,35 +21,14 @@ pub(in crate::dispatch) struct AccountUpstreamContext<'a> {
|
||||
|
||||
pub(in crate::dispatch) async fn create_response_with_account(
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request: &AccountScopedRequest,
|
||||
started_at: Instant,
|
||||
) -> Result<CodexBackendResponse, CodexClientError> {
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope(request, &context.account.id, context.request_id);
|
||||
context
|
||||
.codex
|
||||
.create_response_with_pool_account_started_at(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
turn_metadata: request.turn_metadata.as_deref(),
|
||||
beta_features: request.beta_features.as_deref(),
|
||||
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
||||
version: request.version.as_deref(),
|
||||
codex_window_id: identity.window_id.as_deref(),
|
||||
parent_thread_id: identity.parent_thread_id.as_deref(),
|
||||
cookie_header: context.cookie_header,
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: identity.session_id.as_deref(),
|
||||
thread_id: identity.thread_id.as_deref(),
|
||||
prompt_cache_key: identity.prompt_cache_key.as_deref(),
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: identity.turn_id.as_deref(),
|
||||
},
|
||||
request.request(),
|
||||
codex_request_context(context, request),
|
||||
Some(&context.account.id),
|
||||
started_at,
|
||||
)
|
||||
@@ -62,35 +37,40 @@ pub(in crate::dispatch) async fn create_response_with_account(
|
||||
|
||||
pub(in crate::dispatch) async fn create_response_stream_with_account(
|
||||
context: AccountUpstreamContext<'_>,
|
||||
request: &CodexResponsesRequest,
|
||||
request: &AccountScopedRequest,
|
||||
) -> Result<CodexBackendStreamingResponse, CodexClientError> {
|
||||
let identity = context
|
||||
.account_identity
|
||||
.scope(request, &context.account.id, context.request_id);
|
||||
context
|
||||
.codex
|
||||
.create_response_stream_with_pool_account(
|
||||
request,
|
||||
CodexRequestContext {
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
turn_metadata: request.turn_metadata.as_deref(),
|
||||
beta_features: request.beta_features.as_deref(),
|
||||
include_timing_metrics: request.include_timing_metrics.as_deref(),
|
||||
version: request.version.as_deref(),
|
||||
codex_window_id: identity.window_id.as_deref(),
|
||||
parent_thread_id: identity.parent_thread_id.as_deref(),
|
||||
cookie_header: context.cookie_header,
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: identity.session_id.as_deref(),
|
||||
thread_id: identity.thread_id.as_deref(),
|
||||
prompt_cache_key: identity.prompt_cache_key.as_deref(),
|
||||
client_request_id: Some(&identity.client_request_id),
|
||||
turn_id: identity.turn_id.as_deref(),
|
||||
},
|
||||
request.request(),
|
||||
codex_request_context(context, request),
|
||||
Some(&context.account.id),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn codex_request_context<'a>(
|
||||
context: AccountUpstreamContext<'a>,
|
||||
request: &'a AccountScopedRequest,
|
||||
) -> CodexRequestContext<'a> {
|
||||
let request_body = request.request();
|
||||
let identity = request.identity();
|
||||
CodexRequestContext {
|
||||
access_token: &context.account.access_token,
|
||||
account_id: context.account.account_id.as_deref(),
|
||||
request_id: context.request_id,
|
||||
turn_state: request_body.turn_state.as_deref(),
|
||||
turn_metadata: request_body.turn_metadata.as_deref(),
|
||||
beta_features: request_body.beta_features.as_deref(),
|
||||
include_timing_metrics: request_body.include_timing_metrics.as_deref(),
|
||||
version: request_body.version.as_deref(),
|
||||
codex_window_id: request_body.codex_window_id.as_deref(),
|
||||
parent_thread_id: request_body.parent_thread_id.as_deref(),
|
||||
cookie_header: context.cookie_header,
|
||||
installation_id: Some(&identity.installation_id),
|
||||
session_id: request_body.client_session_id.as_deref(),
|
||||
thread_id: request_body.client_thread_id.as_deref(),
|
||||
client_request_id: request_body.client_request_id.as_deref(),
|
||||
turn_id: request_body.client_turn_id.as_deref(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ pub(in crate::dispatch) struct UpstreamFailureFacts {
|
||||
pub kind: UpstreamFailureKind,
|
||||
pub status_code: Option<u16>,
|
||||
pub code: Option<String>,
|
||||
pub error_type: Option<String>,
|
||||
pub message: String,
|
||||
pub body: String,
|
||||
pub retry_after_seconds: Option<u64>,
|
||||
@@ -47,6 +48,7 @@ pub(in crate::dispatch) fn upstream_failure_facts(
|
||||
},
|
||||
status_code: error.status().map(|status| status.as_u16()),
|
||||
code: None,
|
||||
error_type: None,
|
||||
message: error.to_string(),
|
||||
body: error.to_string(),
|
||||
retry_after_seconds: None,
|
||||
@@ -57,11 +59,12 @@ pub(in crate::dispatch) fn upstream_failure_facts(
|
||||
retry_after_seconds,
|
||||
..
|
||||
} => {
|
||||
let (code, message) = error_fields(body);
|
||||
let (code, error_type, message) = error_fields(body);
|
||||
UpstreamFailureFacts {
|
||||
kind: UpstreamFailureKind::HttpStatus,
|
||||
status_code: Some(status.as_u16()),
|
||||
code,
|
||||
error_type,
|
||||
message: message.unwrap_or_else(|| body.clone()),
|
||||
body: body.clone(),
|
||||
retry_after_seconds: *retry_after_seconds,
|
||||
@@ -71,17 +74,16 @@ pub(in crate::dispatch) fn upstream_failure_facts(
|
||||
simple_facts(UpstreamFailureKind::StreamIdle, error)
|
||||
}
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
|
||||
let (body_code, body_type, body_message) = error_fields(&upstream.body);
|
||||
UpstreamFailureFacts {
|
||||
kind: UpstreamFailureKind::WebSocketUpstream,
|
||||
status_code: Some(upstream.status_code),
|
||||
code: upstream
|
||||
.code
|
||||
.clone()
|
||||
.or_else(|| error_fields(&upstream.body).0),
|
||||
code: upstream.code.clone().or(body_code),
|
||||
error_type: upstream.error_type.clone().or(body_type),
|
||||
message: upstream
|
||||
.message
|
||||
.clone()
|
||||
.or_else(|| error_fields(&upstream.body).1)
|
||||
.or(body_message)
|
||||
.unwrap_or_else(|| upstream.body.clone()),
|
||||
body: upstream.body.clone(),
|
||||
retry_after_seconds: upstream.retry_after_seconds,
|
||||
@@ -123,15 +125,16 @@ fn simple_facts(kind: UpstreamFailureKind, error: &CodexClientError) -> Upstream
|
||||
kind,
|
||||
status_code: None,
|
||||
code: None,
|
||||
error_type: None,
|
||||
body: message.clone(),
|
||||
message,
|
||||
retry_after_seconds: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_fields(body: &str) -> (Option<String>, Option<String>) {
|
||||
fn error_fields(body: &str) -> (Option<String>, Option<String>, Option<String>) {
|
||||
let Ok(value) = serde_json::from_str::<Value>(body) else {
|
||||
return (None, None);
|
||||
return (None, None, None);
|
||||
};
|
||||
let error = value
|
||||
.pointer("/response/error")
|
||||
@@ -143,6 +146,11 @@ fn error_fields(body: &str) -> (Option<String>, Option<String>) {
|
||||
.or_else(|| value.get("code"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string),
|
||||
error
|
||||
.get("type")
|
||||
.or_else(|| value.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string),
|
||||
error
|
||||
.get("message")
|
||||
.or_else(|| value.get("message"))
|
||||
|
||||
@@ -208,7 +208,6 @@ impl AccountManageService {
|
||||
installation_id: installation_id.as_deref(),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
@@ -171,7 +171,6 @@ impl AccountManageService {
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
@@ -50,7 +50,6 @@ impl AccountManageService {
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
|
||||
@@ -33,8 +33,11 @@ use crate::upstream::openai::protocol::events::{
|
||||
pub use crate::fleet::scheduler::RotationStrategy;
|
||||
|
||||
mod filters;
|
||||
mod persistence;
|
||||
mod state;
|
||||
|
||||
use persistence::AccountStatePersistence;
|
||||
|
||||
pub use filters::*;
|
||||
pub use state::*;
|
||||
|
||||
@@ -71,6 +74,7 @@ pub struct AccountPoolService {
|
||||
usage_store: Arc<dyn AccountUsageStore>,
|
||||
request_interval: Arc<std::sync::RwLock<StdDuration>>,
|
||||
candidate_changed: Arc<tokio::sync::Notify>,
|
||||
state_persistence: AccountStatePersistence,
|
||||
}
|
||||
|
||||
/// 一次请求对账号池 slot 的唯一所有权凭据。
|
||||
@@ -191,6 +195,7 @@ impl AccountPoolService {
|
||||
options: AccountPoolOptions,
|
||||
request_interval_ms: u64,
|
||||
) -> Self {
|
||||
let state_persistence = AccountStatePersistence::new(Arc::clone(&store));
|
||||
Self {
|
||||
pool: Arc::new(tokio::sync::Mutex::new(AccountPool::with_options(options))),
|
||||
store,
|
||||
@@ -199,6 +204,7 @@ impl AccountPoolService {
|
||||
request_interval_ms,
|
||||
))),
|
||||
candidate_changed: Arc::new(tokio::sync::Notify::new()),
|
||||
state_persistence,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,9 +346,26 @@ impl AccountPoolService {
|
||||
}
|
||||
};
|
||||
let in_memory = self.pool.lock().await.set_status(account_id, status);
|
||||
if in_memory {
|
||||
self.candidate_changed.notify_waiters();
|
||||
}
|
||||
persisted || in_memory
|
||||
}
|
||||
|
||||
/// 先在线性化的内存池中更新账号状态,再将持久化写入后台队列。
|
||||
pub async fn set_status_immediately(&self, account_id: &str, status: AccountStatus) -> bool {
|
||||
let in_memory = {
|
||||
let mut pool = self.pool.lock().await;
|
||||
let in_memory = pool.set_status(account_id, status);
|
||||
self.state_persistence.set_status(account_id, status);
|
||||
in_memory
|
||||
};
|
||||
if in_memory {
|
||||
self.candidate_changed.notify_waiters();
|
||||
}
|
||||
in_memory
|
||||
}
|
||||
|
||||
/// 标记账号 quota 冷却状态。
|
||||
pub async fn mark_quota_limited_until(&self, account_id: &str, until: DateTime<Utc>) -> bool {
|
||||
let persisted = match self.store.mark_quota_limited_until(account_id, until).await {
|
||||
@@ -361,9 +384,31 @@ impl AccountPoolService {
|
||||
.lock()
|
||||
.await
|
||||
.mark_quota_limited_until(account_id, until);
|
||||
if in_memory {
|
||||
self.candidate_changed.notify_waiters();
|
||||
}
|
||||
persisted || in_memory
|
||||
}
|
||||
|
||||
/// 先在线性化的内存池中标记配额冷却,再将持久化写入后台队列。
|
||||
pub async fn mark_quota_limited_until_immediately(
|
||||
&self,
|
||||
account_id: &str,
|
||||
until: DateTime<Utc>,
|
||||
) -> bool {
|
||||
let in_memory = {
|
||||
let mut pool = self.pool.lock().await;
|
||||
let in_memory = pool.mark_quota_limited_until(account_id, until);
|
||||
self.state_persistence
|
||||
.mark_quota_limited_until(account_id, until);
|
||||
in_memory
|
||||
};
|
||||
if in_memory {
|
||||
self.candidate_changed.notify_waiters();
|
||||
}
|
||||
in_memory
|
||||
}
|
||||
|
||||
/// 按请求上下文获取可用账号。
|
||||
pub async fn acquire_with(&self, request: &AccountAcquireRequest) -> Option<AccountLease> {
|
||||
let refresh = self.pool.lock().await.acquire_with_status_refresh(request);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
//! 账号运行状态的有序后台持久化。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::fleet::{account::AccountStatus, store::AccountStore};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct AccountStatePersistence {
|
||||
sender: mpsc::UnboundedSender<AccountStateWrite>,
|
||||
}
|
||||
|
||||
enum AccountStateWrite {
|
||||
SetStatus {
|
||||
account_id: String,
|
||||
status: AccountStatus,
|
||||
},
|
||||
MarkQuotaLimitedUntil {
|
||||
account_id: String,
|
||||
until: DateTime<Utc>,
|
||||
},
|
||||
}
|
||||
|
||||
impl AccountStatePersistence {
|
||||
pub(super) fn new(store: Arc<dyn AccountStore>) -> Self {
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
tokio::spawn(run(receiver, store));
|
||||
Self { sender }
|
||||
}
|
||||
|
||||
pub(super) fn set_status(&self, account_id: &str, status: AccountStatus) {
|
||||
self.enqueue(AccountStateWrite::SetStatus {
|
||||
account_id: account_id.to_owned(),
|
||||
status,
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn mark_quota_limited_until(&self, account_id: &str, until: DateTime<Utc>) {
|
||||
self.enqueue(AccountStateWrite::MarkQuotaLimitedUntil {
|
||||
account_id: account_id.to_owned(),
|
||||
until,
|
||||
});
|
||||
}
|
||||
|
||||
fn enqueue(&self, write: AccountStateWrite) {
|
||||
if self.sender.send(write).is_err() {
|
||||
tracing::error!("Account state persistence worker stopped unexpectedly");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(
|
||||
mut receiver: mpsc::UnboundedReceiver<AccountStateWrite>,
|
||||
store: Arc<dyn AccountStore>,
|
||||
) {
|
||||
while let Some(write) = receiver.recv().await {
|
||||
match write {
|
||||
AccountStateWrite::SetStatus { account_id, status } => {
|
||||
match store.set_status(&account_id, status).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => tracing::warn!(
|
||||
account_id,
|
||||
?status,
|
||||
"Account disappeared before deferred status persistence"
|
||||
),
|
||||
Err(error) => tracing::error!(
|
||||
account_id,
|
||||
?status,
|
||||
error = %error,
|
||||
"Failed to persist deferred account status"
|
||||
),
|
||||
}
|
||||
}
|
||||
AccountStateWrite::MarkQuotaLimitedUntil { account_id, until } => {
|
||||
match store.mark_quota_limited_until(&account_id, until).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => tracing::warn!(
|
||||
account_id,
|
||||
%until,
|
||||
"Account disappeared before deferred quota persistence"
|
||||
),
|
||||
Err(error) => tracing::error!(
|
||||
account_id,
|
||||
%until,
|
||||
error = %error,
|
||||
"Failed to persist deferred quota state"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -176,7 +176,6 @@ impl QuotaRefreshService {
|
||||
installation_id: Some(&installation_id),
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
|
||||
@@ -520,7 +520,14 @@ fn retry_after_seconds_from_rate_limit_message(error: &Value) -> Option<u64> {
|
||||
.get("code")
|
||||
.or_else(|| error.get("type"))
|
||||
.and_then(Value::as_str)?;
|
||||
if code != "rate_limit_exceeded" {
|
||||
if !matches!(
|
||||
code,
|
||||
"rate_limit_exceeded"
|
||||
| "rate_limit_reached"
|
||||
| "usage_limit_reached"
|
||||
| "workspace_owner_usage_limit_reached"
|
||||
| "workspace_member_usage_limit_reached"
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
let message = error.get("message").and_then(Value::as_str)?;
|
||||
|
||||
@@ -724,9 +724,9 @@ impl CodexResponsesRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅在客户端原本提供字符串字段时替换其账号作用域 wire 值。
|
||||
/// 替换客户端原本提供的账号身份字段;无法安全重建时删除该字段。
|
||||
pub(crate) fn replace_existing_identity_field(&mut self, key: &str, value: Option<&str>) {
|
||||
if !self.body.get(key).is_some_and(Value::is_string) {
|
||||
if !self.body.contains_key(key) {
|
||||
return;
|
||||
}
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
|
||||
@@ -57,12 +57,12 @@ const UPSTREAM_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const UPSTREAM_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
|
||||
const X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY: &str =
|
||||
"x-codex-ws-stream-request-start-ms";
|
||||
type ReqwestClientCacheKey = (bool, Option<String>);
|
||||
type ReqwestClientCacheKey = Option<String>;
|
||||
type ReqwestClientCache = Mutex<HashMap<ReqwestClientCacheKey, Client>>;
|
||||
|
||||
/// 构建带缓存的 reqwest Client。若 `force_http11` 为 true 则强制 HTTP/1.1。
|
||||
pub fn build_reqwest_client(force_http11: bool) -> Result<Client, CustomCaError> {
|
||||
let cache_key = (force_http11, custom_ca_env_cache_key());
|
||||
/// 构建带缓存、自动协商 HTTP/2 的 reqwest Client。
|
||||
pub fn build_reqwest_client() -> Result<Client, CustomCaError> {
|
||||
let cache_key = custom_ca_env_cache_key();
|
||||
static CLIENTS: OnceLock<ReqwestClientCache> = OnceLock::new();
|
||||
let cache = CLIENTS.get_or_init(|| Mutex::new(HashMap::new()));
|
||||
if let Some(client) = cache
|
||||
@@ -83,11 +83,6 @@ pub fn build_reqwest_client(force_http11: bool) -> Result<Client, CustomCaError>
|
||||
.brotli(true)
|
||||
.zstd(true)
|
||||
.deflate(true);
|
||||
let builder = if force_http11 {
|
||||
builder.http1_only()
|
||||
} else {
|
||||
builder
|
||||
};
|
||||
let client = build_reqwest_client_with_custom_ca(builder)?;
|
||||
let mut clients = cache
|
||||
.lock()
|
||||
@@ -232,11 +227,11 @@ pub struct CodexRequestContext<'a> {
|
||||
pub access_token: &'a str,
|
||||
/// ChatGPT account id。
|
||||
pub account_id: Option<&'a str>,
|
||||
/// 请求 ID。
|
||||
/// 代理请求 ID。
|
||||
pub request_id: &'a str,
|
||||
/// x-codex-turn-state。
|
||||
/// 当前账号同一 turn 内的 opaque sticky-routing 状态。
|
||||
pub turn_state: Option<&'a str>,
|
||||
/// x-codex-turn-metadata。
|
||||
/// 客户端 turn metadata;其中 installation ID 已按当前账号处理。
|
||||
pub turn_metadata: Option<&'a str>,
|
||||
/// x-codex-beta-features。
|
||||
pub beta_features: Option<&'a str>,
|
||||
@@ -244,23 +239,21 @@ pub struct CodexRequestContext<'a> {
|
||||
pub include_timing_metrics: Option<&'a str>,
|
||||
/// version。
|
||||
pub version: Option<&'a str>,
|
||||
/// x-codex-window-id。
|
||||
/// 客户端 window ID。
|
||||
pub codex_window_id: Option<&'a str>,
|
||||
/// x-codex-parent-thread-id。
|
||||
/// 客户端 parent thread ID。
|
||||
pub parent_thread_id: Option<&'a str>,
|
||||
/// cookie 头。
|
||||
pub cookie_header: Option<&'a str>,
|
||||
/// x-codex-installation-id。
|
||||
/// 当前账号稳定派生的 installation ID。
|
||||
pub installation_id: Option<&'a str>,
|
||||
/// session_id。
|
||||
/// 客户端 session ID。
|
||||
pub session_id: Option<&'a str>,
|
||||
/// thread_id。
|
||||
/// 客户端 thread ID。
|
||||
pub thread_id: Option<&'a str>,
|
||||
/// 账号作用域 prompt cache key。
|
||||
pub prompt_cache_key: Option<&'a str>,
|
||||
/// 账号作用域 client request ID。
|
||||
/// 客户端逻辑 request ID;缺失时使用代理请求 ID。
|
||||
pub client_request_id: Option<&'a str>,
|
||||
/// 账号作用域 turn ID。
|
||||
/// 客户端 turn ID。
|
||||
pub turn_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
@@ -391,7 +384,6 @@ impl CodexModelCatalogClient for CodexBackendClient {
|
||||
installation_id: request.installation_id,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
@@ -517,29 +509,6 @@ fn is_model_entry(value: &Value) -> bool {
|
||||
// Request helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn response_upstream_request(
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> CodexResponsesRequest {
|
||||
let mut upstream = request.clone();
|
||||
if request.prompt_cache_key().is_some() {
|
||||
upstream.set_prompt_cache_key(context.prompt_cache_key.map(ToString::to_string));
|
||||
}
|
||||
for (key, value) in [
|
||||
("session_id", context.session_id),
|
||||
("thread_id", context.thread_id),
|
||||
("turn_id", context.turn_id),
|
||||
("x-client-request-id", context.client_request_id),
|
||||
("x-codex-window-id", context.codex_window_id),
|
||||
("x-codex-parent-thread-id", context.parent_thread_id),
|
||||
("x-codex-installation-id", context.installation_id),
|
||||
] {
|
||||
upstream.replace_existing_identity_field(key, value);
|
||||
}
|
||||
upstream.set_client_metadata(response_client_metadata(request.client_metadata(), context));
|
||||
upstream
|
||||
}
|
||||
|
||||
fn websocket_upstream_request(request: &CodexResponsesRequest) -> CodexResponsesRequest {
|
||||
let mut request = request.clone();
|
||||
stamp_ws_stream_request_start_ms(&mut request);
|
||||
@@ -566,66 +535,6 @@ fn now_unix_timestamp_millis() -> u128 {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn response_client_metadata(
|
||||
client_metadata: Option<&Value>,
|
||||
context: CodexRequestContext<'_>,
|
||||
) -> Option<Value> {
|
||||
// 以客户端原始 client_metadata 为基础(保留 number/bool/object 等所有值类型),
|
||||
// 在其上追加代理自身的上下文字段。
|
||||
let mut metadata = match client_metadata? {
|
||||
Value::Object(input) => input.clone(),
|
||||
value => return Some(value.clone()),
|
||||
};
|
||||
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-installation-id",
|
||||
context.installation_id,
|
||||
);
|
||||
replace_metadata_string(&mut metadata, "session_id", context.session_id);
|
||||
replace_metadata_string(&mut metadata, "thread_id", context.thread_id);
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-client-request-id",
|
||||
context.client_request_id,
|
||||
);
|
||||
replace_metadata_string(&mut metadata, "turn_id", context.turn_id);
|
||||
replace_metadata_string(&mut metadata, "x-codex-window-id", context.codex_window_id);
|
||||
insert_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-turn-metadata",
|
||||
context.turn_metadata,
|
||||
);
|
||||
replace_metadata_string(
|
||||
&mut metadata,
|
||||
"x-codex-parent-thread-id",
|
||||
context.parent_thread_id,
|
||||
);
|
||||
|
||||
if metadata.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Value::Object(metadata))
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_metadata_string(metadata: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||
if let Some(value) = value.filter(|value| !value.trim().is_empty()) {
|
||||
metadata.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_metadata_string(metadata: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||
match value.filter(|value| !value.trim().is_empty()) {
|
||||
Some(value) => {
|
||||
metadata.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
None => {
|
||||
metadata.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn openai_subagent_from_metadata(client_metadata: Option<&Value>) -> Option<String> {
|
||||
let value = client_metadata?
|
||||
.as_object()?
|
||||
|
||||
@@ -167,7 +167,7 @@ impl CodexBackendClient {
|
||||
pool_account_id: Option<&str>,
|
||||
started_at: Instant,
|
||||
) -> CodexClientResult<CodexBackendResponse> {
|
||||
let upstream_request = response_upstream_request(request, context);
|
||||
let upstream_request = request.clone();
|
||||
match transport_for_request(&upstream_request) {
|
||||
CodexTransport::HttpSse => {
|
||||
self.create_response_http_sse(&upstream_request, context, started_at)
|
||||
@@ -222,7 +222,7 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let upstream_request = response_upstream_request(request, context);
|
||||
let upstream_request = request.clone();
|
||||
match transport_for_request(&upstream_request) {
|
||||
CodexTransport::HttpSse => {
|
||||
self.create_response_stream_http_sse(&upstream_request, context)
|
||||
|
||||
@@ -133,6 +133,19 @@ fn config_loader_should_reject_unknown_fields() {
|
||||
assert!(matches!(error, ConfigError::InvalidDocument { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_removed_tls_section() {
|
||||
let yaml = complete_config().replace(
|
||||
" ws_pool:",
|
||||
" tls:\n force_http11: false\n\n ws_pool:",
|
||||
);
|
||||
let directory = write_config(&yaml);
|
||||
|
||||
let error = BootstrapConfig::load_from_path(directory.path().join("config.yaml")).unwrap_err();
|
||||
|
||||
assert!(matches!(error, ConfigError::InvalidDocument { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_missing_explicit_fields() {
|
||||
let yaml = complete_config().replace(" enabled: true\n\n\n# Compose", "\n\n# Compose");
|
||||
|
||||
@@ -240,11 +240,12 @@ fn controller_exit_order_should_be_static_and_effect_only() {
|
||||
assert_appears_in_order(
|
||||
stream_exit,
|
||||
&[
|
||||
"self.apply_stream_account_state",
|
||||
"CloudflareController::leave_stream",
|
||||
"UsageController::leave_stream",
|
||||
"AffinityController::leave_stream",
|
||||
"TelemetryController::leave_stream",
|
||||
"self.apply_stream_failure",
|
||||
"self.apply_stream_policy_failure",
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -677,10 +677,7 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop
|
||||
assert_eq!(upstream_body["service_tier"], "priority");
|
||||
assert_eq!(upstream_body["reasoning"], json!({"effort": "high"}));
|
||||
assert!(upstream_body.get("include").is_none());
|
||||
let prompt_cache_key = upstream_body["prompt_cache_key"]
|
||||
.as_str()
|
||||
.expect("prompt cache key should be account scoped");
|
||||
assert_ne!(prompt_cache_key, "pcache");
|
||||
assert_eq!(upstream_body["prompt_cache_key"], "pcache");
|
||||
assert_eq!(upstream_body["client_metadata"]["safe"], "yes");
|
||||
assert_eq!(upstream_body["client_metadata"]["drop"], 42);
|
||||
assert_eq!(
|
||||
@@ -695,16 +692,16 @@ async fn responses_should_forward_parity_fields_context_headers_and_account_scop
|
||||
assert!(upstream_body["client_metadata"].get("thread_id").is_none());
|
||||
let window_id = upstream_body["client_metadata"]["x-codex-window-id"]
|
||||
.as_str()
|
||||
.expect("window id should be account scoped");
|
||||
assert_ne!(window_id, "window-metadata");
|
||||
.expect("window id should be preserved");
|
||||
assert_eq!(window_id, "window-metadata");
|
||||
assert_eq!(
|
||||
upstream_body["client_metadata"]["x-codex-turn-metadata"],
|
||||
"meta-metadata"
|
||||
);
|
||||
let parent_thread_id = upstream_body["client_metadata"]["x-codex-parent-thread-id"]
|
||||
.as_str()
|
||||
.expect("parent thread id should be account scoped");
|
||||
assert_ne!(parent_thread_id, "parent-metadata");
|
||||
.expect("parent thread id should be preserved");
|
||||
assert_eq!(parent_thread_id, "parent-metadata");
|
||||
assert_eq!(upstream_header("session-id"), None);
|
||||
assert_eq!(upstream_header("thread-id"), None);
|
||||
assert_eq!(upstream_header("session_id"), None);
|
||||
|
||||
@@ -544,6 +544,57 @@ async fn responses_should_mark_expired_after_401_and_fallback() {
|
||||
assert_eq!(account_status.0, "expired");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_disable_identity_verification_account_and_fallback() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-primary"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_json(json!({
|
||||
"error": {
|
||||
"code": "identity_verification_required",
|
||||
"message": "Identity verification is required before using this account"
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-secondary"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_AFTER_403_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (app, api_key, pool, _dir) = test_app_with_two_accounts(server.uri()).await;
|
||||
|
||||
let response = app
|
||||
.oneshot(responses_json_request(
|
||||
&api_key,
|
||||
&json!({
|
||||
"model": "gpt-5.5",
|
||||
"input": [{"role": "user", "content": "Say hello"}],
|
||||
"stream": false,
|
||||
"use_websocket": false
|
||||
}),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json(response).await;
|
||||
let account_status: String = sqlx::query_scalar("select status from accounts where id = $1")
|
||||
.bind("acct_primary")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(body["id"], "resp_after_403");
|
||||
assert_eq!(account_status, "disabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_recover_auth_failure_from_sse_failed_event() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -1168,12 +1219,12 @@ async fn responses_should_disable_account_after_three_cloudflare_path_block_404s
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_fallback_after_http_model_unsupported() {
|
||||
async fn responses_should_fallback_after_http_404_model_unsupported() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-primary"))
|
||||
.respond_with(ResponseTemplate::new(400).set_body_json(json!({
|
||||
.respond_with(ResponseTemplate::new(404).set_body_json(json!({
|
||||
"error": {
|
||||
"code": "model_not_supported",
|
||||
"message": "Model gpt-5.5 is not supported on this account plan"
|
||||
@@ -2401,6 +2452,274 @@ async fn responses_should_rotate_account_after_5xx_without_same_account_retry()
|
||||
assert_eq!(secondary_requests, 1, "requests: {authorizations:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_failover_should_replace_account_identity_and_preserve_session_semantics() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-primary"))
|
||||
.respond_with(ResponseTemplate::new(401).set_body_json(json!({
|
||||
"error": {
|
||||
"type": "authentication_error",
|
||||
"code": "token_invalidated",
|
||||
"message": "token is no longer valid"
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-secondary"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(RESPONSES_AFTER_5XX_RETRY_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let (app, api_key, pool, _dir) = test_app_with_two_accounts(server.uri()).await;
|
||||
let cookie_store = PgCookieStore::new(pool);
|
||||
cookie_store
|
||||
.capture_set_cookie(
|
||||
"acct_primary",
|
||||
"cf_clearance=primary; Domain=.chatgpt.com; Path=/codex",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
cookie_store
|
||||
.capture_set_cookie(
|
||||
"acct_secondary",
|
||||
"cf_clearance=secondary; Domain=.chatgpt.com; Path=/codex",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let turn_metadata = json!({
|
||||
"installation_id": "installation-client",
|
||||
"session_id": "session-client",
|
||||
"thread_id": "thread-client",
|
||||
"turn_id": "turn-client",
|
||||
"window_id": "window-client",
|
||||
"forked_from_thread_id": "forked-client",
|
||||
"parent_thread_id": "parent-client",
|
||||
"request_kind": "turn",
|
||||
"workspace_kind": "local"
|
||||
})
|
||||
.to_string();
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"input": [{"role": "user", "content": "Replay without identity pollution"}],
|
||||
"stream": false,
|
||||
"use_websocket": false,
|
||||
"prompt_cache_key": "cache-client",
|
||||
"conversation_id": "conversation-client",
|
||||
"session_id": "session-client",
|
||||
"thread_id": "thread-client",
|
||||
"turn_id": "turn-client",
|
||||
"x-client-request-id": "request-client",
|
||||
"x-codex-window-id": "window-client",
|
||||
"x-codex-parent-thread-id": "parent-client",
|
||||
"x-codex-installation-id": "installation-client",
|
||||
"turnState": "turn-state-primary",
|
||||
"turnMetadata": turn_metadata,
|
||||
"codexWindowId": "window-client",
|
||||
"parentThreadId": "parent-client",
|
||||
"client_metadata": {
|
||||
"safe": "preserved",
|
||||
"conversation_id": "conversation-client",
|
||||
"session_id": "session-client",
|
||||
"thread_id": "thread-client",
|
||||
"turn_id": "turn-client",
|
||||
"x-client-request-id": "request-client",
|
||||
"x-codex-window-id": "window-client",
|
||||
"x-codex-parent-thread-id": "parent-client",
|
||||
"x-codex-installation-id": "installation-client",
|
||||
"x-codex-turn-state": "turn-state-primary",
|
||||
"x-codex-turn-metadata": turn_metadata
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let responses_requests = requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/codex/responses")
|
||||
.collect::<Vec<_>>();
|
||||
let primary = responses_requests
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|request| {
|
||||
request
|
||||
.headers
|
||||
.get("authorization")
|
||||
.is_some_and(|value| value == "Bearer access-primary")
|
||||
})
|
||||
.unwrap();
|
||||
let secondary = responses_requests
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|request| {
|
||||
request
|
||||
.headers
|
||||
.get("authorization")
|
||||
.is_some_and(|value| value == "Bearer access-secondary")
|
||||
})
|
||||
.unwrap();
|
||||
let primary_body: Value = serde_json::from_slice(&primary.body).unwrap();
|
||||
let secondary_body: Value = serde_json::from_slice(&secondary.body).unwrap();
|
||||
let primary_metadata = &primary_body["client_metadata"];
|
||||
let secondary_metadata = &secondary_body["client_metadata"];
|
||||
|
||||
assert_eq!(responses_requests.len(), 2);
|
||||
assert_eq!(primary.headers["chatgpt-account-id"], "chatgpt-primary");
|
||||
assert_eq!(secondary.headers["chatgpt-account-id"], "chatgpt-secondary");
|
||||
assert_eq!(primary.headers["cookie"], "cf_clearance=primary");
|
||||
assert_eq!(secondary.headers["cookie"], "cf_clearance=secondary");
|
||||
assert_ne!(
|
||||
primary_metadata["x-codex-installation-id"],
|
||||
secondary_metadata["x-codex-installation-id"]
|
||||
);
|
||||
for key in [
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
"x-client-request-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-parent-thread-id",
|
||||
] {
|
||||
assert_eq!(primary_metadata[key], secondary_metadata[key], "key: {key}");
|
||||
}
|
||||
assert_ne!(
|
||||
primary_body["x-codex-installation-id"],
|
||||
secondary_body["x-codex-installation-id"]
|
||||
);
|
||||
for key in [
|
||||
"prompt_cache_key",
|
||||
"conversation_id",
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
"x-client-request-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-parent-thread-id",
|
||||
"codexWindowId",
|
||||
"parentThreadId",
|
||||
] {
|
||||
assert_eq!(primary_body[key], secondary_body[key], "key: {key}");
|
||||
}
|
||||
let primary_turn_metadata: Value =
|
||||
serde_json::from_str(primary_metadata["x-codex-turn-metadata"].as_str().unwrap()).unwrap();
|
||||
let secondary_turn_metadata: Value = serde_json::from_str(
|
||||
secondary_metadata["x-codex-turn-metadata"]
|
||||
.as_str()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_ne!(
|
||||
primary_turn_metadata["installation_id"],
|
||||
secondary_turn_metadata["installation_id"]
|
||||
);
|
||||
for key in [
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
"window_id",
|
||||
"forked_from_thread_id",
|
||||
"parent_thread_id",
|
||||
] {
|
||||
assert_eq!(
|
||||
primary_turn_metadata[key], secondary_turn_metadata[key],
|
||||
"turn metadata key: {key}"
|
||||
);
|
||||
}
|
||||
for (request, body, metadata, turn_metadata) in [
|
||||
(
|
||||
primary,
|
||||
&primary_body,
|
||||
primary_metadata,
|
||||
&primary_turn_metadata,
|
||||
),
|
||||
(
|
||||
secondary,
|
||||
&secondary_body,
|
||||
secondary_metadata,
|
||||
&secondary_turn_metadata,
|
||||
),
|
||||
] {
|
||||
for (header_key, body_key, metadata_key, turn_metadata_key) in [
|
||||
(
|
||||
"x-codex-installation-id",
|
||||
"x-codex-installation-id",
|
||||
"x-codex-installation-id",
|
||||
"installation_id",
|
||||
),
|
||||
("session-id", "session_id", "session_id", "session_id"),
|
||||
("thread-id", "thread_id", "thread_id", "thread_id"),
|
||||
("x-codex-turn-id", "turn_id", "turn_id", "turn_id"),
|
||||
(
|
||||
"x-codex-window-id",
|
||||
"x-codex-window-id",
|
||||
"x-codex-window-id",
|
||||
"window_id",
|
||||
),
|
||||
(
|
||||
"x-codex-parent-thread-id",
|
||||
"x-codex-parent-thread-id",
|
||||
"x-codex-parent-thread-id",
|
||||
"parent_thread_id",
|
||||
),
|
||||
] {
|
||||
let header_value = request.headers[header_key].to_str().unwrap();
|
||||
assert_eq!(body[body_key].as_str(), Some(header_value));
|
||||
assert_eq!(metadata[metadata_key].as_str(), Some(header_value));
|
||||
assert_eq!(
|
||||
turn_metadata[turn_metadata_key].as_str(),
|
||||
Some(header_value)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
body["x-client-request-id"].as_str(),
|
||||
request.headers["x-client-request-id"].to_str().ok()
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["x-client-request-id"].as_str(),
|
||||
request.headers["x-client-request-id"].to_str().ok()
|
||||
);
|
||||
assert_eq!(
|
||||
body["turnMetadata"].as_str(),
|
||||
metadata["x-codex-turn-metadata"].as_str()
|
||||
);
|
||||
assert_eq!(
|
||||
request.headers["x-codex-turn-metadata"].to_str().ok(),
|
||||
metadata["x-codex-turn-metadata"].as_str()
|
||||
);
|
||||
}
|
||||
assert_eq!(primary_body["turnState"], "turn-state-primary");
|
||||
assert_eq!(primary_metadata["x-codex-turn-state"], "turn-state-primary");
|
||||
assert!(secondary_body.get("turnState").is_none());
|
||||
assert!(secondary_metadata.get("x-codex-turn-state").is_none());
|
||||
assert!(secondary.headers.get("x-codex-turn-state").is_none());
|
||||
assert_eq!(secondary_metadata["safe"], "preserved");
|
||||
assert_eq!(secondary_body["input"], primary_body["input"]);
|
||||
assert!(secondary_body.get("previous_response_id").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_traverse_all_configured_strategy_candidates_without_fixed_cap() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -1286,10 +1286,9 @@ async fn responses_websocket_should_prefer_conversation_account_without_previous
|
||||
let (_reused, first_payload, second_payload) = upstream.await.unwrap();
|
||||
|
||||
assert!(body.contains("\"id\":\"resp_conversation_affinity\""));
|
||||
assert!(
|
||||
first_payload["prompt_cache_key"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.starts_with("wi_"))
|
||||
assert_eq!(
|
||||
first_payload["prompt_cache_key"],
|
||||
"conv_conversation_affinity"
|
||||
);
|
||||
assert_eq!(
|
||||
second_payload["prompt_cache_key"],
|
||||
@@ -1646,7 +1645,7 @@ async fn responses_websocket_response_failed_rate_limit_should_preserve_cooldown
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"type": "rate_limit_error",
|
||||
"code": "rate_limit_exceeded",
|
||||
"code": "workspace_owner_usage_limit_reached",
|
||||
"message": "Rate limit reached. Please try again in 11.054s."
|
||||
}
|
||||
}
|
||||
@@ -1715,7 +1714,7 @@ async fn responses_websocket_response_failed_quota_should_retry_fallback_account
|
||||
"response": {
|
||||
"id": "resp_ws_quota_failed",
|
||||
"error": {
|
||||
"code": "insufficient_quota",
|
||||
"code": "workspace_owner_credits_depleted",
|
||||
"message": "quota exhausted"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
use std::sync::Arc;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use chrono::Utc;
|
||||
use chrono::{DateTime, Utc};
|
||||
use codex_proxy_rs::fleet::{
|
||||
account::{Account, AccountStatus},
|
||||
pool::{AccountAcquireRequest, AccountPoolOptions, AccountPoolService, RotationStrategy},
|
||||
store::{AccountStore, PgAccountStore},
|
||||
store::{AccountStore, AccountStoreResult, PgAccountStore},
|
||||
};
|
||||
use codex_proxy_rs::telemetry::account_usage::store::{
|
||||
AccountUsageDelta, AccountUsageSnapshot, AccountUsageStore, AccountUsageStoreError,
|
||||
AccountUsageWindow, PgAccountUsageStore,
|
||||
};
|
||||
use codex_proxy_rs::telemetry::account_usage::store::PgAccountUsageStore;
|
||||
|
||||
use crate::support::storage::init_test_db;
|
||||
|
||||
@@ -81,6 +85,73 @@ async fn runtime_account_pool_should_restore_accounts_by_added_at_and_id() {
|
||||
assert_eq!(acquired.account.id, "acct_a");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upstream_status_should_exclude_account_before_database_write_completes() {
|
||||
let store = Arc::new(BlockingAccountStore::new());
|
||||
let service = AccountPoolService::new(
|
||||
Arc::clone(&store) as Arc<dyn AccountStore>,
|
||||
Arc::new(NoopAccountUsageStore),
|
||||
AccountPoolOptions::default(),
|
||||
0,
|
||||
);
|
||||
service.restore_from_store().await.unwrap();
|
||||
|
||||
let updated = tokio::time::timeout(
|
||||
std::time::Duration::from_millis(250),
|
||||
service.set_status_immediately("acct_primary", AccountStatus::Banned),
|
||||
)
|
||||
.await
|
||||
.expect("runtime invalidation must not wait for PostgreSQL");
|
||||
store.wait_until_write_started().await;
|
||||
let acquired = service
|
||||
.acquire_with(&AccountAcquireRequest::new("gpt-5.5", Utc::now()))
|
||||
.await
|
||||
.expect("secondary account should remain available");
|
||||
|
||||
assert!(updated);
|
||||
assert_eq!(acquired.account.id, "acct_secondary");
|
||||
|
||||
acquired.release_without_usage().await;
|
||||
store.release_write();
|
||||
store
|
||||
.wait_for_write(PersistedAccountState::Status(AccountStatus::Banned))
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upstream_quota_should_exclude_account_before_database_write_completes() {
|
||||
let store = Arc::new(BlockingAccountStore::new());
|
||||
let service = AccountPoolService::new(
|
||||
Arc::clone(&store) as Arc<dyn AccountStore>,
|
||||
Arc::new(NoopAccountUsageStore),
|
||||
AccountPoolOptions::default(),
|
||||
0,
|
||||
);
|
||||
service.restore_from_store().await.unwrap();
|
||||
let cooldown_until = Utc::now() + chrono::Duration::minutes(5);
|
||||
|
||||
let updated = tokio::time::timeout(
|
||||
std::time::Duration::from_millis(250),
|
||||
service.mark_quota_limited_until_immediately("acct_primary", cooldown_until),
|
||||
)
|
||||
.await
|
||||
.expect("runtime quota invalidation must not wait for PostgreSQL");
|
||||
store.wait_until_write_started().await;
|
||||
let acquired = service
|
||||
.acquire_with(&AccountAcquireRequest::new("gpt-5.5", Utc::now()))
|
||||
.await
|
||||
.expect("secondary account should remain available");
|
||||
|
||||
assert!(updated);
|
||||
assert_eq!(acquired.account.id, "acct_secondary");
|
||||
|
||||
acquired.release_without_usage().await;
|
||||
store.release_write();
|
||||
store
|
||||
.wait_for_write(PersistedAccountState::QuotaLimitedUntil(cooldown_until))
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_account_lease_drop_should_release_capacity() {
|
||||
let (pool, _dir) = init_test_db("runtime-account-pool-lease-drop").await;
|
||||
@@ -224,3 +295,157 @@ async fn insert_account(pool: &sqlx::PgPool, id: &str) {
|
||||
.await
|
||||
.expect("account should be inserted");
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
enum PersistedAccountState {
|
||||
Status(AccountStatus),
|
||||
QuotaLimitedUntil(DateTime<Utc>),
|
||||
}
|
||||
|
||||
struct BlockingAccountStore {
|
||||
accounts: Vec<Account>,
|
||||
write_started: tokio::sync::Semaphore,
|
||||
release_write: tokio::sync::Semaphore,
|
||||
persisted: tokio::sync::Mutex<Vec<PersistedAccountState>>,
|
||||
}
|
||||
|
||||
impl BlockingAccountStore {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
accounts: vec![
|
||||
crate::support::accounts::test_account("acct_primary", AccountStatus::Active),
|
||||
crate::support::accounts::test_account("acct_secondary", AccountStatus::Active),
|
||||
],
|
||||
write_started: tokio::sync::Semaphore::new(0),
|
||||
release_write: tokio::sync::Semaphore::new(0),
|
||||
persisted: tokio::sync::Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn persist(&self, state: PersistedAccountState) {
|
||||
self.write_started.add_permits(1);
|
||||
self.release_write
|
||||
.acquire()
|
||||
.await
|
||||
.expect("test persistence gate should remain open")
|
||||
.forget();
|
||||
self.persisted.lock().await.push(state);
|
||||
}
|
||||
|
||||
async fn wait_until_write_started(&self) {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
self.write_started.acquire(),
|
||||
)
|
||||
.await
|
||||
.expect("deferred persistence should start")
|
||||
.expect("test persistence gate should remain open")
|
||||
.forget();
|
||||
}
|
||||
|
||||
fn release_write(&self) {
|
||||
self.release_write.add_permits(1);
|
||||
}
|
||||
|
||||
async fn wait_for_write(&self, expected: PersistedAccountState) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
loop {
|
||||
if self.persisted.lock().await.contains(&expected) {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("deferred account state should be persisted");
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AccountStore for BlockingAccountStore {
|
||||
async fn list_pool_accounts(&self) -> AccountStoreResult<Vec<Account>> {
|
||||
Ok(self.accounts.clone())
|
||||
}
|
||||
|
||||
async fn mark_quota_limited_until(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
cooldown_until: DateTime<Utc>,
|
||||
) -> AccountStoreResult<bool> {
|
||||
self.persist(PersistedAccountState::QuotaLimitedUntil(cooldown_until))
|
||||
.await;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn set_cloudflare_cooldown_until(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_cooldown_until: DateTime<Utc>,
|
||||
) -> AccountStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn set_status(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
status: AccountStatus,
|
||||
) -> AccountStoreResult<bool> {
|
||||
self.persist(PersistedAccountState::Status(status)).await;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn get_quota_json(&self, _account_id: &str) -> AccountStoreResult<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn apply_quota_snapshot(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_quota_json: &str,
|
||||
_limit_reached: bool,
|
||||
_cooldown_until: Option<DateTime<Utc>>,
|
||||
) -> AccountStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn sync_runtime_account_state(&self, _account: &Account) -> AccountStoreResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
struct NoopAccountUsageStore;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AccountUsageStore for NoopAccountUsageStore {
|
||||
async fn snapshots(
|
||||
&self,
|
||||
_account_ids: &[String],
|
||||
) -> Result<HashMap<String, AccountUsageSnapshot>, AccountUsageStoreError> {
|
||||
Ok(HashMap::new())
|
||||
}
|
||||
|
||||
async fn record_usage_delta(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_usage: AccountUsageDelta,
|
||||
) -> Result<(), AccountUsageStoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_runtime_window(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_window: AccountUsageWindow,
|
||||
) -> Result<(), AccountUsageStoreError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_rate_limit_window(
|
||||
&self,
|
||||
_account_id: &str,
|
||||
_reset_at: DateTime<Utc>,
|
||||
_limit_window_seconds: Option<u64>,
|
||||
) -> Result<(), AccountUsageStoreError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use codex_proxy_rs::bootstrap::config::{
|
||||
AdminConfig, ApiConfig, AppConfig, AuthConfig, DatabaseConfig, FileLoggingConfig,
|
||||
LoggingConfig, QuotaConfig, RedisConfig, RuntimePathsConfig, ServerConfig, TelemetryConfig,
|
||||
TlsConfig, WebSocketPoolSettings,
|
||||
WebSocketPoolSettings,
|
||||
};
|
||||
|
||||
pub(crate) fn test_config(database_url: String) -> AppConfig {
|
||||
@@ -36,9 +36,6 @@ pub(crate) fn test_config(database_url: String) -> AppConfig {
|
||||
runtime: RuntimePathsConfig {
|
||||
data_directory: std::path::PathBuf::from(".runtime/test-data"),
|
||||
},
|
||||
tls: TlsConfig {
|
||||
force_http11: false,
|
||||
},
|
||||
ws_pool: WebSocketPoolSettings {
|
||||
enabled: true,
|
||||
max_age_ms: 3_300_000,
|
||||
|
||||
@@ -223,7 +223,7 @@ async fn websocket_execute_response_create_request_should_capture_handshake_head
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_websocket_should_forward_security_chain_headers_and_payload_fields() {
|
||||
async fn codex_backend_client_websocket_should_forward_headers_and_preserve_payload_fields() {
|
||||
let received_headers = Arc::new(Mutex::new(Vec::<(String, String)>::new()));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
@@ -304,7 +304,6 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
installation_id: Some("install-123"),
|
||||
session_id: Some("cp_derived"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: Some("cp_derived"),
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
@@ -327,20 +326,14 @@ async fn codex_backend_client_websocket_should_forward_security_chain_headers_an
|
||||
);
|
||||
let mut stable_metadata = metadata.clone();
|
||||
stable_metadata.remove("x-codex-ws-stream-request-start-ms");
|
||||
assert_eq!(payload["prompt_cache_key"], "cp_derived");
|
||||
// 透明代理:客户端 client_metadata 的非字符串值(ignored_non_string: 42)原样保留,
|
||||
// 代理字段只做追加式合并,不再过滤用户字段。
|
||||
assert_eq!(payload["prompt_cache_key"], "client-thread");
|
||||
// transport 只生成请求头,不把 context 中的身份或会话字段回填进 payload。
|
||||
assert_eq!(
|
||||
Value::Object(stable_metadata),
|
||||
json!({
|
||||
"safe": "yes",
|
||||
"x-openai-subagent": "review",
|
||||
"ignored_non_string": 42,
|
||||
"x-codex-installation-id": "install-123",
|
||||
"session_id": "cp_derived",
|
||||
"x-codex-window-id": "cw_derived",
|
||||
"x-codex-turn-metadata": "{\"thread_source\":\"subagent\"}",
|
||||
"x-codex-parent-thread-id": "parent-456"
|
||||
"ignored_non_string": 42
|
||||
})
|
||||
);
|
||||
|
||||
@@ -468,7 +461,6 @@ async fn codex_backend_client_should_send_desktop_headers_and_capture_response_m
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
@@ -541,7 +533,6 @@ async fn codex_backend_client_should_use_latest_runtime_fingerprint_for_new_requ
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
};
|
||||
@@ -620,7 +611,6 @@ async fn codex_backend_client_usage_should_use_wham_usage_headers() {
|
||||
installation_id: Some("install-1"),
|
||||
session_id: Some("session-1"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
})
|
||||
@@ -789,7 +779,6 @@ async fn codex_backend_client_should_send_http_sse_headers_in_fingerprint_order(
|
||||
installation_id: Some("install-1"),
|
||||
session_id: Some("session-1"),
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
|
||||
@@ -135,7 +135,6 @@ async fn codex_backend_client_should_cap_non_success_error_body_at_one_mib() {
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
@@ -193,7 +192,6 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
@@ -309,7 +307,6 @@ async fn codex_backend_client_should_capture_only_safe_response_metadata() {
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
|
||||
@@ -178,7 +178,6 @@ pub(crate) fn request_context<'a>(
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
}
|
||||
|
||||
@@ -1690,7 +1690,6 @@ async fn codex_backend_client_should_use_websocket_when_previous_response_id_is_
|
||||
installation_id: None,
|
||||
session_id: None,
|
||||
thread_id: None,
|
||||
prompt_cache_key: None,
|
||||
client_request_id: None,
|
||||
turn_id: None,
|
||||
},
|
||||
|
||||
@@ -40,9 +40,6 @@ x-cpr:
|
||||
refresh_interval_minutes: 5
|
||||
skip_exhausted: true
|
||||
|
||||
tls:
|
||||
force_http11: false
|
||||
|
||||
ws_pool:
|
||||
enabled: true
|
||||
max_age_ms: 3300000
|
||||
|
||||
+15
-8
@@ -457,18 +457,25 @@ previous response 按历史来源处理:
|
||||
|
||||
Responses 协议没有按事件序号续传未完成 response 的请求字段。官方 Codex 在 turn 层持有完整会话历史和工具执行状态,收到可重试的断流后会建立新请求重新采样。代理保留这一责任边界:新请求仍进入账号池调度和会话恢复,但不会伪造同一个 response 的续传。
|
||||
|
||||
## 账号作用域身份
|
||||
## 账号身份、安装身份与会话语义
|
||||
|
||||
`identity_hmac_secret` 是实例级 256-bit 持久密钥。服务端用它对以下值做 HMAC 派生:
|
||||
账号切换不能把所有带 `id` 的字段都一起改写。官方 Codex `5bed644` 将 `session_id`、`thread_id`、`turn_id`、`window_id`、parent/forked thread 共同建模为客户端 turn metadata;默认 `prompt_cache_key` 直接取 `session_id`,HTTP/WS 的 `x-client-request-id` 也使用 thread ID。它们表达客户端会话拓扑,不表达登录账号。盲目按账号 HMAC 会让一次换号在上游变成另一条会话,并破坏 prompt cache、父子线程和增量 WebSocket 语义。
|
||||
|
||||
- prompt cache key
|
||||
- session、thread、turn、window 和 parent thread ID
|
||||
- client request ID
|
||||
- installation ID
|
||||
官方客户端把 `installation_id` 持久化在 Codex home 中,登录账号切换时不会自动变化。号池与单客户端不同:多个互不相关的上游账号共享一个代理 installation ID 会形成横向关联。因此工程有意采用更严格的隔离策略,用实例级 `identity_hmac_secret` 和账号 ID 稳定派生 UUID 形态的 installation ID。该偏离是号池风险隔离策略,不是对官方字段语义的误判。
|
||||
|
||||
派生输入包含字段域和账号 ID。同一客户端标识在不同账号上得到不同值;同一账号、同一输入在密钥不变时保持稳定。installation ID 只由账号 ID 和持久密钥派生,因此每个账号有一个稳定值,不在每次请求时随机生成。
|
||||
| 字段或状态 | 语义 | 换号规则 |
|
||||
| --- | --- | --- |
|
||||
| access token、`chatgpt-account-id`、Cookie | 选中账号的认证身份 | 每个 attempt 必须从当前账号重建,禁止客户端值或上一账号值进入下个 attempt |
|
||||
| `x-codex-installation-id`、turn metadata 的 `installation_id` | 安装/设备指纹 | 使用当前账号的稳定派生值;header、body、`client_metadata` 和嵌套 turn metadata 中已有投影必须一致 |
|
||||
| `x-codex-turn-state` | 上游在同一 turn 内签发的 opaque sticky-routing 状态 | 同账号重试可复用;切换账号必须删除,不能伪造新值 |
|
||||
| `previous_response_id` | 上游历史状态句柄 | 只在拥有该状态的账号上原样使用;代理持有完整 transcript 时可删除 ID 后全量重放,否则禁止跨账号尝试 |
|
||||
| `session_id`、`thread_id`/`conversation_id`、`turn_id`、window、parent/forked thread | 客户端逻辑会话拓扑 | 原样透传,不做 HMAC、不按账号改写;transport 只可把同一原值投影到协议要求的 header |
|
||||
| `prompt_cache_key`、`x-client-request-id` | 会话缓存命名空间和客户端请求关联 | 客户端提供时原样透传;缺失的 `x-client-request-id` 才回退代理 request ID |
|
||||
| response/item `id`、`encrypted_content`、工具 `call_id` | 上游输出引用和工具配对语义 | 跨账号全量重放时删除上游 item `id` 与 `encrypted_content`,保留工具配对所需 `call_id` |
|
||||
|
||||
镜像更新或容器重建时必须保留数据目录中的 `identity_hmac_secret`。丢失该文件会为所有账号生成新的身份集合,已有会话亲和也失去连续性。
|
||||
`AccountScopedIdentity` 类型只包含 installation ID;会话字段不进入该类型。`AccountScopedRequest` 只有在当前账号 token/account/Cookie、installation ID 和 opaque turn state 边界处理完成后才能交给 transport。这样“账号身份要替换”和“会话语义要透传”不是依赖注释约定,而是由类型边界分开。
|
||||
|
||||
镜像更新或容器重建时必须保留数据目录中的 `identity_hmac_secret`。丢失该文件会为所有账号生成新的 installation ID,也会改变只在代理内部使用的本地 conversation HMAC 索引;客户端会话 ID 本身不受影响。
|
||||
|
||||
## 数据存储
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Responses 请求生命周期控制器设计
|
||||
|
||||
状态:已实现。`v1/*` 已统一采用 Request → Attempt → Stream 三层洋葱生命周期;不存在新旧双路径或兼容层。
|
||||
状态:Request → Attempt → Stream 三层洋葱生命周期、明确账号事实的运行时秒级失效,以及单一 HTTP 自动协商路径已经实现;不存在新旧双路径或兼容层。本文“Responses WebSocket v2 与 HTTP/2 低延迟传输审计”一节中的预热、快路径预算、熔断与 HTTP/2 PING 仍是尚未落地的传输目标,不能把这些参数误认为当前生产行为。
|
||||
|
||||
本文回答一个具体问题:Responses 代理能否像 Koa 洋葱模型一样,让身份、历史恢复、账号恢复、会话策略、用量和遥测等模块只处理自己的入口与出口,并按固定顺序组合,而不把同一规则散落在请求过程中。
|
||||
|
||||
@@ -77,6 +77,7 @@ backend/src/dispatch/
|
||||
│ ├── history.rs
|
||||
│ ├── quota.rs
|
||||
│ ├── account_failure.rs
|
||||
│ ├── account_state.rs 运行时失效与 WS 驱逐的共享 effect 执行器
|
||||
│ ├── cloudflare.rs
|
||||
│ ├── affinity.rs
|
||||
│ ├── usage.rs
|
||||
@@ -116,8 +117,8 @@ RequestLifecycle
|
||||
enter: normalize/model -> local identity -> history/session plan -> candidate snapshot
|
||||
|
||||
AttemptLifecycle (每个候选账号一次)
|
||||
enter: acquire -> quota verify -> history for account -> interval -> scoped identity -> call
|
||||
leave: normalize observation -> classify -> account effects -> retry/fail decision -> trace
|
||||
enter: acquire -> quota verify -> history for account -> interval -> account boundary -> call
|
||||
leave: normalize observation -> classify -> runtime invalidation -> retry/fail decision -> trace
|
||||
|
||||
StreamLifecycle (响应提交后)
|
||||
enter: canonical decoder -> observers -> downstream encoder
|
||||
@@ -193,6 +194,8 @@ controller 分为两类:
|
||||
- Critical:无法建立正确请求语义,转为 `DispatchFailure` 并触发逆序退出。
|
||||
- BestEffort:记录警告并 fail-open,不阻塞正常请求。
|
||||
|
||||
账号可用性 effect 属于第三种更窄的 `CriticalRoutingState`:它不能改写已经观察到的上游 outcome,但必须在返回换号 decision 前完成内存池失效和 WebSocket 池摘除。PostgreSQL 只是该事实的持久副本,必须后台写入,不能位于换号的等待链上。遥测、Cookie 和普通 usage 持久化仍是 BestEffort,不能与这条关键路径混为一谈。
|
||||
|
||||
同层 controller 不直接调用另一个 controller,也不自行选择下一账号。lifecycle 是唯一解释 decision 的组件。这样模块只拥有规则,编排器只拥有顺序和控制流。
|
||||
|
||||
## 固定顺序
|
||||
@@ -212,7 +215,7 @@ controller 分为两类:
|
||||
2. 必要时验证 quota。
|
||||
3. 为该账号准备 history;不能跨账号时返回 typed failure。
|
||||
4. 等待账号请求间隔。
|
||||
5. 生成账号作用域 identity、Cookie 和上游上下文。
|
||||
5. 从当前账号生成 token/account/Cookie 和稳定 installation ID;会话语义 ID 保持客户端原值。
|
||||
6. 发起上游调用。
|
||||
7. 统一规范化 HTTP、SSE 首帧和 WebSocket 错误为 `AttemptObservation`。
|
||||
8. 按唯一优先级分类 history、session policy、账号隔离和 transport retry。
|
||||
@@ -220,6 +223,52 @@ controller 分为两类:
|
||||
|
||||
classifier 优先级只能存在一处。complete、prefetch 与 committed stream 共同经过同一个 feature failure owner classifier;`cyber_policy` 必须先于通用 403、quota 或 5xx 分类。composition root 只能把不透明的分类结果交还 owner 产生 decision,其他模块不能再次调用 `is_cyber_policy_*` 或自行拼装 Cyber retry。
|
||||
|
||||
### 账号边界与会话 ID 边界
|
||||
|
||||
官方 Codex 源码将 identifier 分为三种完全不同的状态,lifecycle 不能用一个“账号作用域 identity”概念把它们混在一起:
|
||||
|
||||
| 类别 | 字段 | Attempt 规则 |
|
||||
| --- | --- | --- |
|
||||
| 当前账号认证/指纹 | token、`chatgpt-account-id`、Cookie、installation ID | 每次按当前账号重建;installation ID 按账号稳定派生 |
|
||||
| 上游 opaque 状态 | `x-codex-turn-state`、`previous_response_id` | turn state 换号清除;previous ID 只留在原 owner,或在有完整 transcript 时删除并全量重放 |
|
||||
| 客户端会话语义 | session/thread/conversation、turn、window、parent/forked thread、`prompt_cache_key`、`x-client-request-id` | 原样透传;不得 HMAC、随机化或随账号改写 |
|
||||
|
||||
官方客户端将 installation ID 持久化为安装级 UUID,所以它不是登录账号 ID;本工程按账号稳定派生是号池防关联策略。相反,官方 `prompt_cache_key` 默认取 session ID,`x-client-request-id` 使用 thread ID,证明它们属于会话连续性。代理可以把客户端原值投影到 HTTP/WS header,但不能生成第二份账号作用域值。
|
||||
|
||||
`AccountScopedRequest` 是 attempt 到 transport 的提交能力:只有当前账号认证、installation ID、Cookie 和 opaque state 已处理的请求才能调用上游。`AccountScopedIdentity` 被刻意收窄到 installation ID,不保存任何 session/thread/turn/window 值;换号时编译期可见的身份处理入口因此只有一个。
|
||||
|
||||
### 明确账号事实与秒切号边界
|
||||
|
||||
数据库中的 `active` 只是上一次持久化快照,不是当前账号可用性的最终真相。请求已经选中账号后,上游仍可能返回封禁、工作区停用、token 失效、额度耗尽或窗口限流。处理顺序固定为:
|
||||
|
||||
```text
|
||||
typed upstream fact
|
||||
-> feature owner 唯一分类
|
||||
-> 持有账号池 mutex 时线性化内存失效并移除现有 slots
|
||||
-> 将同一状态写入有序 PostgreSQL 后台队列
|
||||
-> 从 WebSocket 池摘除该账号的 idle/busy entries
|
||||
-> OpenAttempt 返回 RetryNextCandidate
|
||||
```
|
||||
|
||||
任何并发请求即使已经冻结了包含该账号的 candidate snapshot,在真正 `acquire_candidate` 时也必须重读运行时状态,因此内存失效完成后不能再租到旧账号。数据库连接池拥塞、行锁或写失败只能影响持久化收敛,不能让当前请求等待,也不能让其他请求继续看到旧 `Active`。账号运行状态后台写入按产生顺序消费,避免同一进程内的失败事实乱序落库。
|
||||
|
||||
账号事实矩阵来自当前工程、官方 Codex error/rate-limit 类型以及 `/home/zyy/桌面/Codes/sub2api` 的 OpenAI failover 实现交叉审计;只吸收明确的 OpenAI/Codex 信号,不复制 sub2api 的多平台、自定义状态码或连续次数产品策略:
|
||||
|
||||
| 上游事实 | owner 与运行时 effect | 当前 attempt |
|
||||
| --- | --- | --- |
|
||||
| `account_deactivated`、`deactivated_workspace`、明确 banned/suspended/organization disabled | `account_failure` → `Banned` + WS 驱逐 | 立即下一账号 |
|
||||
| `identity_verification_required` | `account_failure` → `Disabled` + WS 驱逐 | 立即下一账号 |
|
||||
| 401、`token_invalidated`、`token_revoked`、`refresh_token_invalidated`、`invalid_api_key`、`authentication_error` | `account_failure` → `Expired` + WS 驱逐 | 立即下一账号 |
|
||||
| `insufficient_quota`、`payment_required`、`workspace_*_credits_depleted` 或普通 402 | `quota` → `QuotaExhausted` + WS 驱逐 | 立即下一账号 |
|
||||
| 429、`rate_limit_exceeded`、`usage_limit_reached`、`workspace_*_usage_limit_reached` | `quota` → 带 reset/retry 时间的运行时限流 + WS 驱逐 | 立即下一账号 |
|
||||
| model unavailable/not supported,包括明确的 404 | `account_failure` 只记录 `ModelUnsupported`,不失效整账号 | 立即下一账号 |
|
||||
| connect/timeout、529、5xx | 通用 transport retry;不写账号状态 | 有候选时下一账号 |
|
||||
| Cyber、Cloudflare HTML challenge、previous-response failure | 各自 controller/history owner | 不得被通用 403/404 抢占 |
|
||||
|
||||
同一失败在 HTTP 状态响应、SSE/WS `response.failed` 和 `error` 事件中都消费相同的 `code`、`type`、status、message 与 retry typed facts。transport 只能提取这些字段,不能决定账号状态。普通 `403` 在 Cyber、Cloudflare 和 model classifier 都未命中后才作为账号禁止事实处理。
|
||||
|
||||
`CommittedAttempt` 类型上没有 retry API:如果失败在客户端响应已经提交后才到达,只能在其他 BestEffort finalizer I/O 之前执行同样的内存失效、后台持久化和 WS 驱逐,让后续请求立即避开该账号;绝不能把另一账号的输出拼到当前流。
|
||||
|
||||
### Stream enter/leave
|
||||
|
||||
1. 上游 SSE 或 WebSocket 事件转换为一次性的 `CanonicalResponseChunk` / `CanonicalResponseEvent`。
|
||||
@@ -249,6 +298,7 @@ classifier 优先级只能存在一处。complete、prefetch 与 committed strea
|
||||
- complete 与 stream 入口共享 `enter_request` 和 `AttemptRunner`,不存在各自维护的账号尝试主循环。
|
||||
- `ControllerSet` 是静态 composition root;controller 顺序和 typed hook 在编译期可见,不使用动态 middleware 链。
|
||||
- complete、prefetch 与 committed stream 的公共失败 owner 优先级只在 `ControllerSet::classify_shared_failure` 声明一次;composition root 不实现 Cyber、quota 或账号失败规则。
|
||||
- ban、deactivated workspace、认证失效与 quota owner 共享同一 `account_state` effect 执行器:内存状态先线性化,PostgreSQL 有序后台持久化,WS entry 随即驱逐;controller 不直接等待数据库。
|
||||
- Cyber failure 由 `CyberPolicyController` 返回不透明分类,并由该 owner 唯一构造换号或返回 decision;`ControllerSet` 只分发 effect 与 decision。
|
||||
- `ControllerRequestScope` 只保存 cyber、history、usage 的真实状态;不存在通用 entered-controller marker、容量错误或空 unwind hook。
|
||||
- complete 的 accepted response 归一为 `FinalOutcome` 后由 `finalize_complete` 消费 scope;retry 继续进入下一 attempt,rejection 在 controller effect 已完成后直接返回,不经过空 finalize。
|
||||
@@ -260,10 +310,182 @@ classifier 优先级只能存在一处。complete、prefetch 与 committed strea
|
||||
- 上游 WebSocket pool 保持独立的连接 lease 与回收职责,dispatch lifecycle 只消费其 typed transport outcome。
|
||||
- 测试全部位于 `backend/tests/`;`backend/src/` 中不存在 `#[cfg(test)]`、`#[test]`、`mod tests` 或 test-only helper。
|
||||
|
||||
## Responses WebSocket v2 与 HTTP/2 低延迟传输审计
|
||||
|
||||
状态:审计完成,目标方案已确定,尚未实现。
|
||||
|
||||
本节解决的不是“WebSocket 和 HTTP/SSE 谁在理论上更快”,而是两个直接影响体感的问题:如何让健康连接尽量在请求到来前准备好,以及当 WebSocket 建连异常时,如何避免在换账号或 HTTP fallback 之前白等 15 秒。
|
||||
|
||||
最终结论是双热通道:精确会话 WebSocket 负责 v2 的连接本地状态和增量续接;按 origin 共享的 HTTP/2 连接负责新链和可持久化续链的低延迟后备。两者不能互相冒充,也不能用一个通用“连接可用”布尔值抹掉 continuation 的所有权差异。
|
||||
|
||||
### 审计证据与事实边界
|
||||
|
||||
以下结论来自 2026-07-14 的四类证据,必须区分来源:
|
||||
|
||||
1. 官方 Codex 源码确认:本地官方仓库当时的 `main` 为 `5bed6447998c754d154dbd796517310b8f04d4ce`。OpenAI provider 默认声明支持 WebSocket;旧 `responses_websockets` / `responses_websockets_v2` feature 已标记 Removed,实际传输路径不再依赖这两个开关;握手固定发送 `OpenAI-Beta: responses_websockets=2026-02-06`。
|
||||
2. 官方 Codex 源码确认:会话初始化后会在后台构造稳定的 instructions、tools 和空 input,发送 v2 `response.create` 且设置 `generate=false`,等待 `response.completed` 后把连接和 response ID 交给首个真实 turn。首个 turn 只有在请求属性仍匹配时才使用 warmup response ID;模型、service tier、tools 等属性变化时仍可复用 socket,但必须发送新的完整 create。
|
||||
3. OpenAI 官方文档确认:WebSocket Mode 在活跃 socket 上只保存最新一个 previous-response 的连接本地内存状态;`store=false` / ZDR 下没有持久化 fallback,ID 不在该 socket 缓存时返回 `previous_response_not_found`;单条连接一次只允许一个 in-flight response,不支持 multiplex,连接最长 60 分钟。
|
||||
4. 当前工程源码确认:已经透明透传 `generate=false`,发送 v2 beta header,按 `(base_url, account_id, conversation_id)` 建池,并在 socket 上记录和校验最新 response ID;池内连接默认 25 秒 ping、5 秒探活、55 分钟最大寿命。
|
||||
5. 当前工程源码确认:reqwest Client 已按自定义 CA profile 全局缓存,HTTP/2 可以跨账号共享同一 origin 的连接池;强制 HTTP/1.1 的配置和分支已经删除,但尚未配置 HTTP/2 PING,reqwest 0.12.28 默认会在 90 秒后清理 idle pool。
|
||||
6. 线上与网络探针只能用于校准,不构成协议保证:此前当日生产失败样本中,59 次 WebSocket 超时有 56 次被同账号 HTTP 恢复;OCI 未鉴权探针中,HTTP/2 冷连接完成 TLS 约 32–35ms,WebSocket upgrade 响应约 70–591ms。该探针不包含真实鉴权和模型 TTFT,只说明健康握手通常不需要 15 秒。
|
||||
|
||||
官方依据:
|
||||
|
||||
- [Codex `client.rs` 的 prewarm 契约](https://github.com/openai/codex/blob/5bed6447998c754d154dbd796517310b8f04d4ce/codex-rs/core/src/client.rs#L15-L24)
|
||||
- [Codex 会话启动预热](https://github.com/openai/codex/blob/5bed6447998c754d154dbd796517310b8f04d4ce/codex-rs/core/src/session_startup_prewarm.rs#L183-L224)
|
||||
- [Codex `generate=false` 请求构造与等待完成](https://github.com/openai/codex/blob/5bed6447998c754d154dbd796517310b8f04d4ce/codex-rs/core/src/session_startup_prewarm.rs#L241-L324)
|
||||
- [OpenAI Responses API WebSocket Mode](https://developers.openai.com/api/docs/guides/websocket-mode)
|
||||
- [reqwest 0.12.28 HTTP/2 keep-alive 配置](https://docs.rs/reqwest/0.12.28/reqwest/struct.ClientBuilder.html#method.http2_keep_alive_interval)
|
||||
|
||||
### WebSocket v2 的收益边界
|
||||
|
||||
WebSocket v2 的主要收益不是帧格式比 HTTP/SSE 更快,而是同时消除三类重复工作:
|
||||
|
||||
- socket 持续存在,避免每轮重新建立 TCP/TLS/HTTP 请求通道;健康 HTTP/2 keep-alive 也能覆盖这一部分。
|
||||
- 客户端只发送新增 input 和 `previous_response_id`,不重复上传完整 history、instructions 和 tools。
|
||||
- 上游直接复用当前 socket 上的 connection-local previous-response 状态;HTTP/2 连接复用不能提供这份模型请求状态。
|
||||
|
||||
因此,热 HTTP/2 可以让 fallback 的网络建连接近无感,但不能替代 `store=false` WebSocket 链。OpenAI 官方文档给出的典型收益场景是长工具链,并报告 20 次以上工具调用的 rollout 最高约 40% 端到端加速;这不是所有单轮请求都能获得的固定收益。
|
||||
|
||||
### 当前实现缺口
|
||||
|
||||
当前实现已经具备 v2 透传和池化基础,但还有四个会影响正确性或尾延迟的缺口:
|
||||
|
||||
1. `upstream/openai/transport/client_sse.rs` 只根据是否携带 previous ID 判断 fallback。首个 `generate=false` 没有 previous ID,因此上游 WebSocket 失败后仍可能回退 HTTP。
|
||||
2. `controllers/affinity.rs` 只根据 `store` 记录 continuation scope。HTTP fallback 产生的 `store=false` response 也会被记为 `ConnectionLocal`,但实际没有任何 WebSocket 持有该 ID。
|
||||
3. `WebSocketPreferred` 与 `WebSocketRequired` 的冷建连都可能在前台等待完整 15 秒;池只有 Idle / Busy 等连接结果,没有复用中的单飞 Connecting 状态和短快路径预算。
|
||||
4. HTTP Client 虽然全局缓存并默认协商 HTTP/2,但 90 秒 idle pool timeout 会在长期只走 WebSocket 时主动清掉健康后备连接;TCP keepalive 不能阻止连接池自己驱逐 socket。
|
||||
|
||||
第 1、2 点会形成错误链:
|
||||
|
||||
```text
|
||||
客户端后台发送 generate=false
|
||||
-> 上游 WebSocket 失败
|
||||
-> 代理降级 HTTP,得到 store=false response_id
|
||||
-> 客户端下一轮携带 previous_response_id
|
||||
-> 新 WebSocket 没有该 connection-local 状态
|
||||
-> previous_response_not_found 或被迫全量重放
|
||||
```
|
||||
|
||||
目标实现必须禁止这条链,而不是再增加一次重试掩盖它。
|
||||
|
||||
### 最终传输状态机
|
||||
|
||||
请求首先被规范化为显式 transport requirement,不继续靠 `use_websocket`、`force_http_sse` 和 `previous_response_id.is_some()` 的组合隐式推断:
|
||||
|
||||
| 请求事实 | 传输要求 | 允许行为 |
|
||||
| --- | --- | --- |
|
||||
| 客户端显式 `generate=false` 且 `store=false` | Explicit WebSocket Warmup | 必须使用 WebSocket;允许后台完整建连预算;禁止 HTTP fallback |
|
||||
| `store=false` connection-local previous ID | Exact WebSocket Continuation | 只能租用持有同一最新 ID 的精确 socket;缺失时立即返回 typed unavailable,由 HistoryController 决定全量重放或失败 |
|
||||
| 已确认 persisted previous ID | Persisted Continuation | 热 WebSocket 优先;没有热 socket 时允许使用 HTTP/2 或新 WebSocket hydration |
|
||||
| 外部来源且所有权未知的 previous ID | External Unknown | 只在已选定账号原样尝试一次,不遍历账号碰运气;失败后返回观察到的协议错误 |
|
||||
| 没有 previous ID 的普通新链 | New Chain | 精确热 WebSocket 优先;冷建连只等快路径预算,超时后同账号走热 HTTP/2 |
|
||||
| 客户端显式要求 HTTP | HTTP Required | 直接使用 HTTP,不启动 WebSocket 冷建连 |
|
||||
|
||||
response 完成后的 continuation scope 必须同时由 `store` 和实际 transport 推导:`store=true` 是 Persisted;`store=false + WebSocket completed` 才是 ConnectionLocal;`store=false + HTTP` 必须是 ReplayRequired 或 Unavailable,绝不能伪装成 ConnectionLocal。只有入站 WebSocket 已持有完整 canonical transcript 时才能标记 ReplayRequired,普通 HTTP 客户端没有本地 replay snapshot 时只能标记 Unavailable。
|
||||
|
||||
普通新链的目标时序:
|
||||
|
||||
```text
|
||||
获取账号 lease
|
||||
-> controller 完成账号与 route 准备
|
||||
-> 生成当前账号凭据、installation ID / Cookie / handshake 输入
|
||||
-> 启动精确 WebSocket 单飞建连
|
||||
与 wait_for_request_interval 并行
|
||||
-> 已有热 socket 或 800ms 内完成:发送 response.create
|
||||
-> 800ms 未完成:取消尚未发送 payload 的冷连接,同账号立即走热 HTTP/2
|
||||
-> HTTP/2 也失败或返回明确可换号失败:Attempt lifecycle 选择下一候选账号
|
||||
```
|
||||
|
||||
800ms 是初始快路径预算,不是永久硬编码真理。它来自当前 OCI 探针的健康区间并为鉴权波动保留余量,必须根据 `ws_connect_ms` 的生产 p95/p99 调整。无论以后调成多少,该预算都只能覆盖发送 payload 之前的握手阶段。
|
||||
|
||||
一旦 `response.create` 已可能送达上游,连接断开或首事件超时就是“是否已经开始推理未知”的 ambiguous failure。没有上游幂等保证时,不能为了缩短体感而并发发送同一请求到 HTTP 或其他账号,否则可能重复推理、重复扣费或产生两条不一致输出。现有 20 秒 initial-event guard 与 5 分钟 active-stream idle timeout 必须继续和 pre-send connect budget 分开建模。
|
||||
|
||||
### 官方客户端预热与代理预热
|
||||
|
||||
官方 Codex 客户端已经提供标准预热信号。代理收到 `generate=false` 后应当:
|
||||
|
||||
1. 正常进入 Request / Attempt 洋葱并选定账号,不绕过 quota、history 或账号隔离规则。
|
||||
2. 在该账号、base URL 和本地 conversation identity 下建立 WebSocket。
|
||||
3. 原样发送 `response.create(generate=false)`,等待 `response.completed`。
|
||||
4. 将 socket、最新 warmup response ID 和连接元数据作为一个不可拆分的池条目保存。
|
||||
5. 首个真实请求只有在 previous ID 和请求属性都满足续接条件时,才在该 socket 上发送增量 input。
|
||||
|
||||
代理不得给所有账号盲目建立“空 WebSocket”:握手携带账号凭据、session/thread 会话语义、Cookie 和路由信息,空连接不能安全跨账号或跨 conversation 借用;单 socket 不支持 multiplex,批量空连接还会提前消耗上游连接额度。对不发送标准 warmup 的客户端,只允许在候选账号和会话语义已经确定后做 handshake-only 单飞建连,不得由代理擅自补发 `generate=false`,因为后者可能额外处理完整上下文并引入隐藏成本、状态映射和并发竞态。
|
||||
|
||||
### HTTP/2 热后备
|
||||
|
||||
HTTP/2 连接按 `(scheme, host, port, proxy/TLS profile)` 共享,不按账号建池;Authorization 和账号头属于单个 HTTP/2 stream,同一 origin 的多个账号可以复用一条 TLS/H2 连接。这正适合作为 WebSocket 冷建连失败后的廉价后备。
|
||||
|
||||
工程不再暴露 `tls.force_http11`,也不保留 `http1_only()` 兼容分支;旧 `tls` 配置节点会被 `deny_unknown_fields` 直接拒绝。上游 HTTP 只有一条自动 ALPN 协商路径,主动连接保活优化只针对 HTTP/2,不提供运行时切回“强制 HTTP/1.1”的第二套策略。目标 Client 配置为:
|
||||
|
||||
```rust
|
||||
Client::builder()
|
||||
.pool_idle_timeout(None::<Duration>)
|
||||
.http2_keep_alive_interval(Duration::from_secs(30))
|
||||
.http2_keep_alive_timeout(Duration::from_secs(5))
|
||||
.http2_keep_alive_while_idle(true)
|
||||
```
|
||||
|
||||
有可用账号时,现有启动模型目录刷新已经会在服务启动后建立同 origin HTTP 连接,不需要新增一条“为了预热而预热”的业务请求。目标配置的目的只是让该连接在 WebSocket-only 流量期间不被本地 90 秒 idle timeout 清理,并用 HTTP/2 PING 在后台发现已经失活的链路。
|
||||
|
||||
### 熔断与账号边界
|
||||
|
||||
WebSocket 建连失败通常是 origin、出口网络、代理或 Cloudflare 边缘链路事实,不应直接把账号标为失效。此前生产样本中绝大多数超时能被同账号 HTTP 恢复,因此顺序必须是“同账号换 transport,再由 HTTP 结果决定是否换账号”,而不是“换账号后再重复一次同样的 WebSocket 超时”。
|
||||
|
||||
初始熔断规则:
|
||||
|
||||
- key 为 origin + 出口 route/TLS profile,不按账号 key。
|
||||
- 30 秒窗口内连续 3 次 WebSocket 快路径超时,打开 30 秒。
|
||||
- 熔断期间已有热 WebSocket 仍可使用;普通新链直接走 HTTP/2,不再创建冷 WebSocket。
|
||||
- 30 秒后只允许一个 half-open 单飞探针;成功关闭熔断,失败重新打开。
|
||||
- 401、403、quota、账号停用和套餐限制仍由对应 controller 处理,不能污染 transport 熔断。
|
||||
- `previous_response_not_found` 只驱逐精确 conversation socket 并交给 HistoryController,不打开 origin 熔断。
|
||||
|
||||
这些时间和次数属于 transport policy,只能由 `upstream/openai/transport` 单一 owner 定义和观测;lifecycle 只能消费 `ReusedWebSocket`、`ConnectedWebSocket`、`Http2Fallback`、`ContinuationUnavailable`、`PreSendFailure`、`PostSendAmbiguousFailure` 等 typed outcome。
|
||||
|
||||
### 洋葱边界下的代码归属
|
||||
|
||||
| 责任 | 唯一归属 | 禁止越界 |
|
||||
| --- | --- | --- |
|
||||
| v2 payload、握手、快路径预算、单飞、连接池、origin 熔断 | `upstream/openai/transport/` | lifecycle/controller 不解析 WebSocket 错误字符串,不维护连接状态 |
|
||||
| `store` + 实际 transport 推导 continuation scope | upstream 产出事实,`controllers/history.rs` 解释所有权与重放 | affinity 不能只看 `store` 猜测 scope |
|
||||
| 账号选择、retry next、提交边界 | `dispatch/lifecycle/` | transport 不选择下一个账号,不写账号状态 |
|
||||
| history full replay、previous ID owner、跨账号可恢复性 | `controllers/history.rs` | API 和 WebSocket pool 不自行拼接历史 |
|
||||
| quota、封禁、Cyber 和账号状态 | 各自 classifier controller;`account_state` 只执行通用失效 effect | transport connect failure 不能直接停用账号,effect 执行器不能识别错误语义 |
|
||||
| HTTP/WS 入站解析和最终事件编码 | `api/client/responses/` | API 不决定 upstream transport fallback |
|
||||
|
||||
Attempt lifecycle 只增加一个窄的 transport preparation 调用点:在账号、route、identity 和 Cookie 已确定后启动连接准备,并与账号请求间隔并行;它接收准备结果,不拥有 800ms、30 秒或熔断阈值。这样低延迟优化不会把 WebSocket 规则扩散进 `attempt.rs`,也不会形成第二套 retry 主循环。
|
||||
|
||||
### 可观测性与验收
|
||||
|
||||
必须分别记录选择账号、连接准备、发送请求、首协议事件和首真实输出,不能只用一个总耗时掩盖慢点:
|
||||
|
||||
- `transport_decision`:`ws_reused`、`ws_connected_fast`、`http2_ws_slow`、`http2_breaker_open`、`ws_exact_required`。
|
||||
- `ws_connect_ms`、`transport_decision_wait_ms`、`upstream_headers_ms`、`first_event_ms`、`first_output_ms`。
|
||||
- `http_version`、origin breaker state、WebSocket pool decision、connection reused。
|
||||
- pre-send failure 与 post-send ambiguous failure 必须是不同枚举和不同指标。
|
||||
- `generate=false` warmup 成功率、warmup 被首轮使用率、请求属性变化导致的 warmup miss。
|
||||
|
||||
完成实现必须同时满足:
|
||||
|
||||
- `generate=false` 的 `store=false` 请求在 WebSocket 失败后绝不回退 HTTP,也不记录虚假的 ConnectionLocal scope。
|
||||
- warmup 与首个真实 turn 在属性匹配时使用同一 socket,首轮携带 warmup response ID 和增量 input。
|
||||
- 精确 connection-local socket 缺失时不冷建连碰运气,不遍历账号;只走 HistoryController 的 typed replay/failure。
|
||||
- 普通 `WebSocketPreferred` 冷建连不会在前台等待 15 秒;健康快路径预算耗尽后同账号 HTTP/2 立即接管。
|
||||
- WebSocket transport failure 不更新账号失效状态;只有后续明确的账号事实才能触发账号 controller。
|
||||
- 明确账号事实发生后,即使 PostgreSQL 行写入被阻塞,当前 attempt 仍立即切到下一账号,并发候选租用也不能再次获得旧账号;解除阻塞后状态最终落库。
|
||||
- 账号失效必须同时驱逐该账号的 WebSocket entries;已提交流只能影响后续请求,不能构造 retry。
|
||||
- origin 熔断期间没有冷 WebSocket 建连风暴,已有热连接不受影响。
|
||||
- payload 发送后的 ambiguous failure 不自动并发 hedge 到第二 transport 或账号。
|
||||
- HTTP/2 连接在超过 90 秒的 WebSocket-only 空闲窗口后仍可复用,并能在 PING timeout 后清理失活连接。
|
||||
- 所有测试位于 `backend/tests/`,按 transport、history controller 和 lifecycle contract 边界组织,`backend/src/` 不出现 test-only 代码。
|
||||
|
||||
## 性能约束
|
||||
|
||||
- 独立且都需要的入口查询并行执行,不能把多个 Redis RTT 串行叠加。
|
||||
- best-effort controller 必须有有界超时和明确 fail-open 行为。
|
||||
- 账号可用性内存失效和 WS 池摘除不是数据库 BestEffort I/O;数据库持久化只能进入后台队列,不能占用换号延迟预算。
|
||||
- 逐事件 observer 只做同步、低成本状态更新;Redis、数据库和遥测 I/O 只在必要的 terminal/finalize 阶段执行。
|
||||
- canonical event 同时保留 raw payload,避免为透明转发再次序列化。
|
||||
- 是否引入动态分发以基准和可维护性为依据,不凭感觉优化。
|
||||
|
||||
Reference in New Issue
Block a user