mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-20 08:01:46 +00:00
refactor(openai): simplify account selection and session admission
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! AttemptContext 驱动的 Codex 账号选择与 Redis lease port
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::fmt;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -10,11 +10,11 @@ use gateway_core::account::CredentialRevision;
|
||||
use gateway_core::error::ErrorSource;
|
||||
|
||||
use gateway_core::account::{
|
||||
AccountAffinity, AccountCandidate, AccountCapacitySnapshot, AccountErrorReason,
|
||||
AccountFeedbackStats, AccountModelAccessMode, AccountQuotaSignals, AccountRuntimeSignals,
|
||||
AccountSchedulingBlocker, AccountSelectionContext, AccountSelectionPolicy, AccountSelector,
|
||||
AccountStatus, CredentialState, PreferredAccountSelection, ProviderAccount, ProviderAccountId,
|
||||
QuotaEvidence, RotationStrategy,
|
||||
AccountAffinity, AccountCandidate, AccountCapacitySnapshot, AccountCooldown,
|
||||
AccountCooldownKind, AccountErrorReason, AccountFeedbackStats, AccountModelAccessMode,
|
||||
AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext,
|
||||
AccountSelectionPolicy, AccountSelector, AccountStatus, CredentialState,
|
||||
PreferredAccountSelection, ProviderAccount, ProviderAccountId, QuotaEvidence, RotationStrategy,
|
||||
};
|
||||
use gateway_core::concurrency::{
|
||||
CapacityWait, ConcurrencyQueuePolicy, ConcurrencyWaitQueue, QueueRejection, WaitPriority,
|
||||
@@ -30,7 +30,7 @@ use gateway_core::provider_ports::{
|
||||
ProviderSessionAffinityPort, ProviderSessionBinding, ProviderSessionClaim,
|
||||
ProviderSessionExclusionPort, ProviderSessionExclusions, ProviderStoreError,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
@@ -185,48 +185,63 @@ impl AffinityEscapeReason {
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct AffinitySelection {
|
||||
bound_account: Option<ProviderAccountId>,
|
||||
preferred_account: Option<ProviderAccountId>,
|
||||
escape_reason: Option<AffinityEscapeReason>,
|
||||
enum AffinitySelection<'a> {
|
||||
#[default]
|
||||
Unbound,
|
||||
Preferred(&'a ProviderAccountId),
|
||||
Escaped(&'a ProviderAccountId, AffinityEscapeReason),
|
||||
}
|
||||
|
||||
impl AffinitySelection {
|
||||
fn preferred(account_id: ProviderAccountId) -> Self {
|
||||
Self {
|
||||
bound_account: Some(account_id.clone()),
|
||||
preferred_account: Some(account_id),
|
||||
escape_reason: None,
|
||||
impl<'a> AffinitySelection<'a> {
|
||||
fn for_binding(
|
||||
account: &'a ProviderAccountId,
|
||||
candidates: &[AccountCandidate],
|
||||
now: SystemTime,
|
||||
) -> Self {
|
||||
let status = candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == account)
|
||||
.map(|candidate| {
|
||||
candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.cooldown)
|
||||
.status
|
||||
});
|
||||
match status {
|
||||
Some(AccountStatus::Normal) => Self::Preferred(account),
|
||||
Some(AccountStatus::QuotaExhausted) => {
|
||||
Self::Escaped(account, AffinityEscapeReason::QuotaExhausted)
|
||||
}
|
||||
Some(AccountStatus::RateLimited | AccountStatus::QuotaPaused) => {
|
||||
Self::Escaped(account, AffinityEscapeReason::Cooldown)
|
||||
}
|
||||
Some(AccountStatus::Disabled | AccountStatus::Error) | None => {
|
||||
Self::Escaped(account, AffinityEscapeReason::HardUnavailable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn escaped(account_id: ProviderAccountId, reason: AffinityEscapeReason) -> Self {
|
||||
Self {
|
||||
bound_account: Some(account_id),
|
||||
preferred_account: None,
|
||||
escape_reason: Some(reason),
|
||||
fn bound_account(&self) -> Option<&'a ProviderAccountId> {
|
||||
match self {
|
||||
Self::Preferred(account) | Self::Escaped(account, _) => Some(account),
|
||||
Self::Unbound => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_account(&self) -> Option<&ProviderAccountId> {
|
||||
self.bound_account.as_ref()
|
||||
}
|
||||
|
||||
fn preferred_account(&self) -> Option<&ProviderAccountId> {
|
||||
self.preferred_account.as_ref()
|
||||
fn preferred_account(&self) -> Option<&'a ProviderAccountId> {
|
||||
match self {
|
||||
Self::Preferred(account) => Some(account),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn escape(&mut self, reason: AffinityEscapeReason) {
|
||||
if self.bound_account.is_some() && self.escape_reason.is_none() {
|
||||
self.escape_reason = Some(reason);
|
||||
self.preferred_account = None;
|
||||
if let Self::Preferred(account) = self {
|
||||
*self = Self::Escaped(account, reason);
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_preferred_selection(&mut self, selection: PreferredAccountSelection) {
|
||||
if self.bound_account.is_none() || self.escape_reason.is_some() {
|
||||
return;
|
||||
}
|
||||
match selection {
|
||||
PreferredAccountSelection::Hit => {}
|
||||
PreferredAccountSelection::OverriddenByPolicy => {
|
||||
@@ -253,7 +268,7 @@ impl AffinitySelection {
|
||||
}
|
||||
|
||||
fn observe_lease_busy(&mut self, account_id: &ProviderAccountId) {
|
||||
if self.bound_account.as_ref() == Some(account_id) {
|
||||
if self.preferred_account() == Some(account_id) {
|
||||
self.escape(AffinityEscapeReason::LeaseSaturated);
|
||||
}
|
||||
}
|
||||
@@ -264,14 +279,15 @@ impl AffinitySelection {
|
||||
cache_preference: bool,
|
||||
) -> AffinityTelemetry {
|
||||
AffinityTelemetry {
|
||||
affinity_hit: self.bound_account.as_ref() == Some(selected_account)
|
||||
&& self.escape_reason.is_none(),
|
||||
escape_reason: self.escape_reason,
|
||||
// 仅因缓存偏好而分流时,客户端的轮次状态仍没有已知归属。
|
||||
affinity_hit: self.preferred_account() == Some(selected_account),
|
||||
escape_reason: match self {
|
||||
Self::Escaped(_, reason) => Some(*reason),
|
||||
_ => None,
|
||||
},
|
||||
// 仅因缓存偏好而分流时,客户端的轮次状态仍没有已知归属
|
||||
account_switch: !cache_preference
|
||||
&& self
|
||||
.bound_account
|
||||
.as_ref()
|
||||
.bound_account()
|
||||
.is_some_and(|bound| bound != selected_account),
|
||||
}
|
||||
}
|
||||
@@ -556,6 +572,123 @@ impl CodexCredentialSelector {
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_candidate_accounts(
|
||||
&self,
|
||||
attempt: &AttemptContext,
|
||||
model: CodexSelectionModel<'_>,
|
||||
adapter_model: Option<&UpstreamModelId>,
|
||||
only_account: Option<&ProviderAccountId>,
|
||||
) -> Result<Vec<ProviderAccount>, CredentialSelectionError> {
|
||||
let diagnostic = attempt.account_eligibility().bypasses_local_eligibility();
|
||||
// 固定归属或已有完整投影时只重查目标账号,诊断允许读取停用账号
|
||||
if let Some(id) = only_account {
|
||||
let account = self
|
||||
.repository
|
||||
.store()
|
||||
.get_account(id)
|
||||
.await
|
||||
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?;
|
||||
return Ok(account
|
||||
.filter(|account| diagnostic || account.enabled())
|
||||
.into_iter()
|
||||
.collect());
|
||||
}
|
||||
if diagnostic {
|
||||
return Ok(self.repository.list_for_provider().await?);
|
||||
}
|
||||
let account_ids: Vec<_> = attempt
|
||||
.account_scope()
|
||||
.into_iter()
|
||||
.flat_map(|scope| scope.allowed_account_ids().cloned())
|
||||
.collect();
|
||||
let omissions = match (model, attempt.account_scope()) {
|
||||
(CodexSelectionModel::Requested(model), Some(scope)) => {
|
||||
self.catalog.omissions(model, &account_ids, |id| {
|
||||
scope.directory().account(id).is_some_and(|account| {
|
||||
account.model_access().mode() == AccountModelAccessMode::Allowlist
|
||||
}) || adapter_model.is_some_and(|model| {
|
||||
!matches!(
|
||||
attempt.upstream_adapter_for_account(&self.provider_kind, model, id),
|
||||
Ok(None)
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let Some(filter) = omissions else {
|
||||
return Ok(self.repository.list_for_accounts(&account_ids).await?);
|
||||
};
|
||||
let candidates = self
|
||||
.repository
|
||||
.store()
|
||||
.list_catalog_candidates(&self.provider_kind, &account_ids, &filter)
|
||||
.await
|
||||
.map_err(CredentialRepositoryError::from)?;
|
||||
if attempt.trace().is_enabled() {
|
||||
attempt.trace().record(
|
||||
"account.catalog_candidates",
|
||||
serde_json::json!({
|
||||
"scopedCount": account_ids.len(),
|
||||
"candidateCount": candidates.len(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
async fn load_candidate_cooldowns(
|
||||
&self,
|
||||
accounts: &[ProviderAccount],
|
||||
account_ids: &[ProviderAccountId],
|
||||
native_accounts: &BTreeSet<ProviderAccountId>,
|
||||
model: CodexSelectionModel<'_>,
|
||||
) -> HashMap<ProviderAccountId, AccountCooldown> {
|
||||
let native = accounts
|
||||
.iter()
|
||||
.filter(|account| native_accounts.contains(account.id()));
|
||||
let model_cooldowns = async {
|
||||
let CodexSelectionModel::Requested(model) = model else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(model) = UpstreamModelId::new(model) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let ids: Vec<_> = native.clone().map(|account| account.id().clone()).collect();
|
||||
self.cooldowns
|
||||
.read_scoped_many(&ids, &ProviderCooldownScope::upstream_model(model))
|
||||
.await
|
||||
};
|
||||
let (account_cooldowns, model_cooldowns) =
|
||||
futures::join!(self.cooldowns.read_many(account_ids), model_cooldowns);
|
||||
let mut active: HashMap<_, _> = account_ids
|
||||
.iter()
|
||||
.zip(account_cooldowns)
|
||||
.filter_map(|(id, cooldown)| {
|
||||
let state = cooldown.ok().flatten()?.scheduling_state();
|
||||
state
|
||||
.is_active(SystemTime::now())
|
||||
.then(|| (id.clone(), state))
|
||||
})
|
||||
.collect();
|
||||
for (account, cooldown) in native.zip(model_cooldowns) {
|
||||
if let Some(cooldown) = cooldown.ok().flatten().filter(|cooldown| {
|
||||
cooldown.credential_revision() == account.revision()
|
||||
&& cooldown.until() > SystemTime::now()
|
||||
}) {
|
||||
// 两种冷却取较晚期限,保留账号级冷却的恢复探测语义
|
||||
active
|
||||
.entry(account.id().clone())
|
||||
.and_modify(|state| state.until = state.until.max(cooldown.until()))
|
||||
.or_insert(AccountCooldown {
|
||||
until: cooldown.until(),
|
||||
kind: AccountCooldownKind::RateLimit,
|
||||
});
|
||||
}
|
||||
}
|
||||
active
|
||||
}
|
||||
|
||||
async fn select_inner_loop(
|
||||
&self,
|
||||
request: &CredentialSelectionInput<'_>,
|
||||
@@ -567,7 +700,7 @@ impl CodexCredentialSelector {
|
||||
) -> Result<CodexCredentialLease, CredentialSelectionError> {
|
||||
let adapter_model = match model {
|
||||
CodexSelectionModel::Requested(model) if request.managed_upstream => Some(
|
||||
gateway_core::routing::UpstreamModelId::new(model)
|
||||
UpstreamModelId::new(model)
|
||||
.map_err(|_| CredentialSelectionError::NoEligibleCredential)?,
|
||||
),
|
||||
_ => None,
|
||||
@@ -730,64 +863,18 @@ impl CodexCredentialSelector {
|
||||
.as_ref()
|
||||
.and_then(|_| binding.as_ref().map(ProviderSessionBinding::account_id));
|
||||
let projection_started = std::time::Instant::now();
|
||||
// 固定归属或已有完整投影的会话只查询目标账号,随后实时复核其资格和租约
|
||||
// 常规调度排除停用账号,管理端诊断允许验证其真实上游状态
|
||||
let accounts = if let Some(pinned) = pinned_account
|
||||
let only_account = pinned_account
|
||||
.as_ref()
|
||||
.or(queued_account.as_ref())
|
||||
.or(projected_account)
|
||||
{
|
||||
self.repository
|
||||
.store()
|
||||
.get_account(pinned)
|
||||
.await
|
||||
.map_err(|source| CredentialSelectionError::Store(ErrorSource::new(source)))?
|
||||
.filter(|account| diagnostic || account.enabled())
|
||||
.into_iter()
|
||||
.collect()
|
||||
} else if diagnostic {
|
||||
self.repository.list_for_provider().await?
|
||||
} else {
|
||||
let account_ids = request
|
||||
.attempt
|
||||
.account_scope()
|
||||
.into_iter()
|
||||
.flat_map(|scope| scope.allowed_account_ids().cloned())
|
||||
.collect::<Vec<_>>();
|
||||
let omissions = match (model, request.attempt.account_scope()) {
|
||||
(CodexSelectionModel::Requested(model), Some(scope)) => {
|
||||
self.catalog.omissions(model, &account_ids, |id| {
|
||||
scope.directory().account(id).is_some_and(|a| {
|
||||
a.model_access().mode() == AccountModelAccessMode::Allowlist
|
||||
}) || adapter_model.as_ref().is_some_and(|m| {
|
||||
!matches!(
|
||||
request.attempt.upstream_adapter_for_account(
|
||||
&self.provider_kind,
|
||||
m,
|
||||
id
|
||||
),
|
||||
Ok(None)
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Some(filter) = omissions {
|
||||
let candidates = self
|
||||
.repository
|
||||
.store()
|
||||
.list_catalog_candidates(&self.provider_kind, &account_ids, &filter)
|
||||
.await
|
||||
.map_err(CredentialRepositoryError::from)?;
|
||||
if request.attempt.trace().is_enabled() {
|
||||
request.attempt.trace().record("account.catalog_candidates",serde_json::json!({"scopedCount":account_ids.len(),"candidateCount":candidates.len()}));
|
||||
}
|
||||
candidates
|
||||
} else {
|
||||
self.repository.list_for_accounts(&account_ids).await?
|
||||
}
|
||||
};
|
||||
.or(projected_account);
|
||||
let accounts = self
|
||||
.load_candidate_accounts(
|
||||
request.attempt,
|
||||
model,
|
||||
adapter_model.as_ref(),
|
||||
only_account,
|
||||
)
|
||||
.await?;
|
||||
let mut model_access_rejected = 0_usize;
|
||||
let accounts = accounts
|
||||
.into_iter()
|
||||
@@ -822,7 +909,7 @@ impl CodexCredentialSelector {
|
||||
_ => None,
|
||||
};
|
||||
let mut catalog_rejected = 0_usize;
|
||||
let mut native_accounts = std::collections::BTreeSet::new();
|
||||
let mut native_accounts = BTreeSet::new();
|
||||
for account in accounts {
|
||||
let adapter = if let Some(model) = adapter_model.as_ref() {
|
||||
match request.attempt.upstream_adapter_for_account(
|
||||
@@ -943,65 +1030,12 @@ impl CodexCredentialSelector {
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
.collect::<Vec<_>>();
|
||||
let mut rate_limits = HashMap::with_capacity(accounts.len());
|
||||
let model_scope = match model {
|
||||
CodexSelectionModel::Requested(model) => {
|
||||
gateway_core::routing::UpstreamModelId::new(model)
|
||||
.ok()
|
||||
.map(ProviderCooldownScope::upstream_model)
|
||||
}
|
||||
_ => None,
|
||||
let rate_limits = if diagnostic {
|
||||
HashMap::new()
|
||||
} else {
|
||||
self.load_candidate_cooldowns(&accounts, &account_ids, &native_accounts, model)
|
||||
.await
|
||||
};
|
||||
if !diagnostic {
|
||||
let model_account_ids = account_ids
|
||||
.iter()
|
||||
.filter(|id| native_accounts.contains(*id))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let model_cooldowns = async {
|
||||
match model_scope.as_ref() {
|
||||
Some(scope) => {
|
||||
self.cooldowns
|
||||
.read_scoped_many(&model_account_ids, scope)
|
||||
.await
|
||||
}
|
||||
None => Vec::new(),
|
||||
}
|
||||
};
|
||||
let (account_cooldowns, model_cooldowns) =
|
||||
futures::join!(self.cooldowns.read_many(&account_ids), model_cooldowns,);
|
||||
// 存储端有界批量读取,单项失败沿用未知冷却;待探测冻结由端口事实判断
|
||||
for (id, cooldown) in account_ids.iter().zip(account_cooldowns) {
|
||||
let state = cooldown
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|value| value.scheduling_state());
|
||||
rate_limits.insert(
|
||||
id.clone(),
|
||||
state.filter(|state| state.is_active(SystemTime::now())),
|
||||
);
|
||||
}
|
||||
for (account, cooldown) in accounts
|
||||
.iter()
|
||||
.filter(|account| native_accounts.contains(account.id()))
|
||||
.zip(model_cooldowns)
|
||||
{
|
||||
if let Some(cooldown) = cooldown.ok().flatten().filter(|cooldown| {
|
||||
cooldown.credential_revision() == account.revision()
|
||||
&& cooldown.until() > SystemTime::now()
|
||||
}) {
|
||||
// 两种冷却取较晚期限,账号保护保留其恢复探测语义
|
||||
let until = rate_limits
|
||||
.entry(account.id().clone())
|
||||
.or_default()
|
||||
.get_or_insert(gateway_core::account::AccountCooldown {
|
||||
until: cooldown.until(),
|
||||
kind: gateway_core::account::AccountCooldownKind::RateLimit,
|
||||
});
|
||||
until.until = until.until.max(cooldown.until());
|
||||
}
|
||||
}
|
||||
}
|
||||
let policy = request.attempt.account_selection_policy();
|
||||
let client_key = request.attempt.client_api_key_ref();
|
||||
let use_assignment_cursor = policy.strategy() == RotationStrategy::RoundRobin
|
||||
@@ -1068,7 +1102,7 @@ impl CodexCredentialSelector {
|
||||
first_output_latency_ms: None,
|
||||
})
|
||||
.with_provider_quota(quota)
|
||||
.with_rate_limit(rate_limits.get(account.id()).copied().flatten())
|
||||
.with_rate_limit(rate_limits.get(account.id()).copied())
|
||||
.with_runtime_health(health.0, health.1);
|
||||
AccountCandidate {
|
||||
account,
|
||||
@@ -1107,8 +1141,8 @@ impl CodexCredentialSelector {
|
||||
.as_ref()
|
||||
.filter(|_| affinity_mode != AccountAffinity::Relaxed)
|
||||
.map_or_else(AffinitySelection::default, |binding| {
|
||||
affinity_selection_for_bound_account(
|
||||
binding.account_id().clone(),
|
||||
AffinitySelection::for_binding(
|
||||
binding.account_id(),
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
@@ -1410,17 +1444,13 @@ impl CodexCredentialSelector {
|
||||
};
|
||||
affinity.observe_preferred_selection(selection.preferred());
|
||||
let selected = selection.candidate();
|
||||
let account = candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == selected.account.id())
|
||||
.map(|candidate| candidate.account.clone())
|
||||
.ok_or(CredentialSelectionError::InvalidCredential(None))?;
|
||||
let account = &selected.account;
|
||||
// 凭据、状态或额度结论的并发更新会使账号快照失效,必须重新选号并校验资格;
|
||||
// 仅额度观测时刻前进时沿用最新快照
|
||||
// 在占用租约前完成校验,避免重读被自己的异步释放挡住
|
||||
let (account, runtime) = match self
|
||||
.repository
|
||||
.reload_selected_credential(&account)
|
||||
.reload_selected_credential(account)
|
||||
.await
|
||||
{
|
||||
Ok(reloaded) => reloaded,
|
||||
@@ -1429,7 +1459,7 @@ impl CodexCredentialSelector {
|
||||
self.scheduling_projection.invalidate(key);
|
||||
}
|
||||
refresh_candidates = true;
|
||||
retry_account_snapshot(request.attempt, &account, &mut snapshot_retries)?;
|
||||
retry_account_snapshot(request.attempt, account, &mut snapshot_retries)?;
|
||||
continue 'capacity;
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
@@ -1445,7 +1475,7 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
let allows_account_state_mutation =
|
||||
eligibility.allows_account_state_mutation(account.enabled());
|
||||
match self
|
||||
let guard = match self
|
||||
.leases
|
||||
.try_acquire(ProviderLeaseRequest::Scheduling(
|
||||
ProviderSchedulingLeaseRequest::new(
|
||||
@@ -1474,167 +1504,117 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
shortest_retry = minimum_duration(shortest_retry, retry_after);
|
||||
busy_accounts.insert(account.id().clone());
|
||||
continue;
|
||||
}
|
||||
ProviderLeaseAcquisition::Acquired(guard) => {
|
||||
let mut binding_account = account.id();
|
||||
if affinity_mode == AccountAffinity::Preferred
|
||||
&& let Some(primary) = binding.as_ref()
|
||||
&& (primary.account_id() == account.id()
|
||||
|| pinned_account.is_some()
|
||||
|| !self
|
||||
.preferred_primary_needs_migration(
|
||||
primary,
|
||||
queue_policy.timeout,
|
||||
)
|
||||
.await?)
|
||||
{
|
||||
binding_account = primary.account_id();
|
||||
}
|
||||
let claim = binding_key.map(|key| ProviderSessionClaim {
|
||||
key,
|
||||
expected: binding.as_ref(),
|
||||
account: binding_account,
|
||||
ttl: policy.openai_session_affinity_ttl(),
|
||||
});
|
||||
// 插件委托与普通内置选号共享轮询,显式选择、亲和命中及等待不消耗位置
|
||||
let admission = match assignment_cursor {
|
||||
Some(cursor)
|
||||
if !selection.is_policy_choice()
|
||||
&& selection.preferred() != PreferredAccountSelection::Hit =>
|
||||
{
|
||||
Some(ProviderAccountAdmission::RoundRobin {
|
||||
client_key,
|
||||
cursor,
|
||||
session: claim,
|
||||
})
|
||||
}
|
||||
_ => claim.map(ProviderAccountAdmission::Session),
|
||||
};
|
||||
if let Some(admission) = admission {
|
||||
let admitted = match self.admit_account(admission).await {
|
||||
Ok(false) => false,
|
||||
Err(error)
|
||||
if cache_preference
|
||||
&& affinity_mode == AccountAffinity::Preferred
|
||||
&& admission.session().is_some() =>
|
||||
{
|
||||
tracing::warn!(%error, "OpenAI cache preference update unavailable");
|
||||
// 缓存偏好允许丢失,实际的轮询分配仍须占用该位置
|
||||
match admission {
|
||||
ProviderAccountAdmission::RoundRobin {
|
||||
client_key,
|
||||
cursor,
|
||||
..
|
||||
} => {
|
||||
self.admit_account(
|
||||
ProviderAccountAdmission::RoundRobin {
|
||||
client_key,
|
||||
cursor,
|
||||
session: None,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
ProviderAccountAdmission::Session(_) => true,
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
Ok(true) => {
|
||||
if let Some(reason) = queue_rejection
|
||||
&& let Some(previous) = binding.as_ref().filter(|binding| {
|
||||
binding.account_id() != binding_account
|
||||
})
|
||||
{
|
||||
tracing::info!(
|
||||
request_id = %request.attempt.request_id(),
|
||||
attempt_index = request.attempt.attempt_index().get(),
|
||||
reason = %reason,
|
||||
from_account_id = %previous.account_id(),
|
||||
to_account_id = %binding_account,
|
||||
"OpenAI 账号排队失败后已迁移绑定"
|
||||
);
|
||||
request.attempt.trace().record_evidence(
|
||||
"account.binding.migrated",
|
||||
serde_json::json!({
|
||||
"reason": reason.to_string(),
|
||||
"fromAccountId": previous.account_id().as_str(),
|
||||
"toAccountId": binding_account.as_str(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
};
|
||||
if !admitted {
|
||||
guard.release().await?;
|
||||
if request.attempt.deadline().is_elapsed() {
|
||||
return Err(CredentialSelectionError::AccountSnapshotChanged);
|
||||
}
|
||||
continue 'capacity;
|
||||
}
|
||||
}
|
||||
let affinity_telemetry = affinity.telemetry(account.id(), cache_preference);
|
||||
let affinity_observation = request.session_affinity_observation;
|
||||
tracing::info!(
|
||||
request_id = %request.attempt.request_id(),
|
||||
attempt_index = request.attempt.attempt_index().get(),
|
||||
rotation_strategy = policy.strategy().as_str(),
|
||||
account_id = %account.id(),
|
||||
affinity_hit = affinity_telemetry.affinity_hit,
|
||||
escape_reason = affinity_telemetry
|
||||
.escape_reason
|
||||
.map_or("", AffinityEscapeReason::as_str),
|
||||
account_switch = affinity_telemetry.account_switch,
|
||||
affinity_key_hash = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::key_hash),
|
||||
affinity_anchor_source = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::anchor_source),
|
||||
affinity_anchor = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::anchor),
|
||||
session_id = affinity_observation
|
||||
.and_then(CodexSessionAffinity::session_id)
|
||||
.unwrap_or(""),
|
||||
session_id_present = affinity_observation
|
||||
.is_some_and(CodexSessionAffinity::session_id_present),
|
||||
"OpenAI account selected"
|
||||
);
|
||||
let cookies = runtime
|
||||
.cookies
|
||||
.into_iter()
|
||||
.filter(|cookie| {
|
||||
cookie
|
||||
.expires_at
|
||||
.is_none_or(|expires| expires > chrono::Utc::now())
|
||||
&& self.cookie_policy.may_replay(
|
||||
request.request_url,
|
||||
&cookie.domain,
|
||||
&cookie.path,
|
||||
cookie.host_only,
|
||||
cookie.secure,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
if !waiting.elapsed().is_zero() {
|
||||
request.attempt.trace().record(
|
||||
"account.queue.acquired",
|
||||
serde_json::json!({"waitMs": waiting.elapsed().as_millis() as u64}),
|
||||
);
|
||||
}
|
||||
return Ok(CodexCredentialLease {
|
||||
installation_id: runtime.installation_id,
|
||||
transport: runtime.transport,
|
||||
account,
|
||||
authentication: runtime.authentication,
|
||||
cookies,
|
||||
cyber_policy_scope,
|
||||
allows_account_state_mutation,
|
||||
affinity_telemetry,
|
||||
admitted_session: binding_key.filter(|_| !cache_preference).cloned(),
|
||||
capacity: capacity.map(AccountCapacitySnapshot::with_acquired_request),
|
||||
_guard: guard,
|
||||
});
|
||||
}
|
||||
ProviderLeaseAcquisition::Acquired(guard) => guard,
|
||||
};
|
||||
let mut binding_account = account.id();
|
||||
if affinity_mode == AccountAffinity::Preferred
|
||||
&& let Some(primary) = binding.as_ref()
|
||||
&& (primary.account_id() == account.id()
|
||||
|| pinned_account.is_some()
|
||||
|| !self
|
||||
.preferred_primary_needs_migration(primary, queue_policy.timeout)
|
||||
.await?)
|
||||
{
|
||||
binding_account = primary.account_id();
|
||||
}
|
||||
let claim = binding_key.map(|key| ProviderSessionClaim {
|
||||
key,
|
||||
expected: binding.as_ref(),
|
||||
account: binding_account,
|
||||
ttl: policy.openai_session_affinity_ttl(),
|
||||
});
|
||||
// 插件委托与普通内置选号共享轮询,显式选择、亲和命中及等待不消耗位置
|
||||
let admission = match assignment_cursor {
|
||||
Some(cursor)
|
||||
if !selection.is_policy_choice()
|
||||
&& selection.preferred() != PreferredAccountSelection::Hit =>
|
||||
{
|
||||
Some(ProviderAccountAdmission::RoundRobin {
|
||||
client_key,
|
||||
cursor,
|
||||
session: claim,
|
||||
})
|
||||
}
|
||||
_ => claim.map(ProviderAccountAdmission::Session),
|
||||
};
|
||||
if let Some(admission) = admission
|
||||
&& !self
|
||||
.admit_selected_account(
|
||||
admission,
|
||||
cache_preference && affinity_mode == AccountAffinity::Preferred,
|
||||
request.attempt,
|
||||
queue_rejection,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
guard.release().await?;
|
||||
if request.attempt.deadline().is_elapsed() {
|
||||
return Err(CredentialSelectionError::AccountSnapshotChanged);
|
||||
}
|
||||
continue 'capacity;
|
||||
}
|
||||
let affinity_telemetry = affinity.telemetry(account.id(), cache_preference);
|
||||
let affinity_observation = request.session_affinity_observation;
|
||||
tracing::info!(
|
||||
request_id = %request.attempt.request_id(),
|
||||
attempt_index = request.attempt.attempt_index().get(),
|
||||
rotation_strategy = policy.strategy().as_str(),
|
||||
account_id = %account.id(),
|
||||
affinity_hit = affinity_telemetry.affinity_hit,
|
||||
escape_reason = affinity_telemetry
|
||||
.escape_reason
|
||||
.map_or("", AffinityEscapeReason::as_str),
|
||||
account_switch = affinity_telemetry.account_switch,
|
||||
affinity_key_hash = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::key_hash),
|
||||
affinity_anchor_source = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::anchor_source),
|
||||
affinity_anchor = affinity_observation
|
||||
.map_or("", CodexSessionAffinity::anchor),
|
||||
session_id = affinity_observation
|
||||
.and_then(CodexSessionAffinity::session_id)
|
||||
.unwrap_or(""),
|
||||
session_id_present = affinity_observation
|
||||
.is_some_and(CodexSessionAffinity::session_id_present),
|
||||
"OpenAI account selected"
|
||||
);
|
||||
let cookies = runtime
|
||||
.cookies
|
||||
.into_iter()
|
||||
.filter(|cookie| {
|
||||
cookie
|
||||
.expires_at
|
||||
.is_none_or(|expires| expires > chrono::Utc::now())
|
||||
&& self.cookie_policy.may_replay(
|
||||
request.request_url,
|
||||
&cookie.domain,
|
||||
&cookie.path,
|
||||
cookie.host_only,
|
||||
cookie.secure,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
if !waiting.elapsed().is_zero() {
|
||||
request.attempt.trace().record(
|
||||
"account.queue.acquired",
|
||||
serde_json::json!({"waitMs": waiting.elapsed().as_millis() as u64}),
|
||||
);
|
||||
}
|
||||
return Ok(CodexCredentialLease {
|
||||
installation_id: runtime.installation_id,
|
||||
transport: runtime.transport,
|
||||
account,
|
||||
authentication: runtime.authentication,
|
||||
cookies,
|
||||
cyber_policy_scope,
|
||||
allows_account_state_mutation,
|
||||
affinity_telemetry,
|
||||
admitted_session: binding_key.filter(|_| !cache_preference).cloned(),
|
||||
capacity: capacity.map(AccountCapacitySnapshot::with_acquired_request),
|
||||
_guard: guard,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1717,6 +1697,60 @@ impl CodexCredentialSelector {
|
||||
.map_err(binding_store_error)
|
||||
}
|
||||
|
||||
async fn admit_selected_account(
|
||||
&self,
|
||||
admission: ProviderAccountAdmission<'_>,
|
||||
optional_cache_binding: bool,
|
||||
attempt: &AttemptContext,
|
||||
queue_rejection: Option<QueueRejection>,
|
||||
) -> Result<bool, CredentialSelectionError> {
|
||||
let claim = admission.session();
|
||||
match self.admit_account(admission).await {
|
||||
Ok(false) => return Ok(false),
|
||||
Err(error) if optional_cache_binding && claim.is_some() => {
|
||||
tracing::warn!(%error, "OpenAI cache preference update unavailable");
|
||||
// 缓存偏好允许丢失,实际的轮询分配仍须占用该位置
|
||||
return match admission {
|
||||
ProviderAccountAdmission::RoundRobin {
|
||||
client_key, cursor, ..
|
||||
} => {
|
||||
self.admit_account(ProviderAccountAdmission::RoundRobin {
|
||||
client_key,
|
||||
cursor,
|
||||
session: None,
|
||||
})
|
||||
.await
|
||||
}
|
||||
ProviderAccountAdmission::Session(_) => Ok(true),
|
||||
};
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
Ok(true) => {}
|
||||
}
|
||||
if let (Some(reason), Some(claim)) = (queue_rejection, claim)
|
||||
&& let Some(previous) = claim.expected
|
||||
&& previous.account_id() != claim.account
|
||||
{
|
||||
tracing::info!(
|
||||
request_id = %attempt.request_id(),
|
||||
attempt_index = attempt.attempt_index().get(),
|
||||
reason = %reason,
|
||||
from_account_id = %previous.account_id(),
|
||||
to_account_id = %claim.account,
|
||||
"OpenAI 账号排队失败后已迁移绑定"
|
||||
);
|
||||
attempt.trace().record_evidence(
|
||||
"account.binding.migrated",
|
||||
serde_json::json!({
|
||||
"reason": reason.to_string(),
|
||||
"fromAccountId": previous.account_id().as_str(),
|
||||
"toAccountId": claim.account.as_str(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn admit_account(
|
||||
&self,
|
||||
admission: ProviderAccountAdmission<'_>,
|
||||
@@ -2194,49 +2228,6 @@ fn binding_store_error(
|
||||
)))
|
||||
}
|
||||
|
||||
fn affinity_selection_for_bound_account(
|
||||
account_id: ProviderAccountId,
|
||||
candidates: &[AccountCandidate],
|
||||
now: SystemTime,
|
||||
) -> AffinitySelection {
|
||||
let Some(candidate) = candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == &account_id)
|
||||
else {
|
||||
return AffinitySelection::escaped(account_id, AffinityEscapeReason::HardUnavailable);
|
||||
};
|
||||
match candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.cooldown)
|
||||
.status
|
||||
{
|
||||
AccountStatus::Normal => AffinitySelection::preferred(account_id),
|
||||
AccountStatus::QuotaPaused
|
||||
| AccountStatus::QuotaExhausted
|
||||
| AccountStatus::RateLimited
|
||||
| AccountStatus::Disabled
|
||||
| AccountStatus::Error => {
|
||||
AffinitySelection::escaped(account_id, affinity_unavailable_reason(candidate, now))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn affinity_unavailable_reason(
|
||||
candidate: &AccountCandidate,
|
||||
now: SystemTime,
|
||||
) -> AffinityEscapeReason {
|
||||
match candidate
|
||||
.account
|
||||
.status_projection(now, candidate.signals.cooldown)
|
||||
.status
|
||||
{
|
||||
AccountStatus::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountStatus::RateLimited | AccountStatus::QuotaPaused => AffinityEscapeReason::Cooldown,
|
||||
AccountStatus::Normal => AffinityEscapeReason::SelectionInvariant,
|
||||
AccountStatus::Disabled | AccountStatus::Error => AffinityEscapeReason::HardUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
fn active_risk_recovery<'a>(
|
||||
recovery: &'a mut HashMap<String, RiskRecoveryState>,
|
||||
account_id: &str,
|
||||
|
||||
Reference in New Issue
Block a user