mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-06 08:06:19 +00:00
feat: add account scheduling controls and client profile options (#339)
Stagger credential refresh and quota observations, bound each request to three account rotations, and expose terminal and version lag configuration. Retain main's bounded refresh 401 recovery policy. Version lag accepts integers from 1 to 10 or an empty value. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
committed by
GitHub
co-authored by
Codex
parent
22acbd1d10
commit
0ce81eea1d
@@ -334,6 +334,8 @@ where
|
||||
attempts: 0,
|
||||
websocket_observation_sequence: 0,
|
||||
routing_attempts: 0,
|
||||
last_attempt_account: None,
|
||||
account_rotations: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
@@ -455,6 +457,15 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
websocket_observation_sequence: u64,
|
||||
/// 路由预算只统计正常选号/账号恢复,不被 Provider-owned 传输预算消耗
|
||||
routing_attempts: u32,
|
||||
/// 上一 attempt 实际选中的账号;与选号成功的 attempt 一一同步更新。
|
||||
/// 旧 CurrentAttempt 在丢弃时被 drop、excluded_accounts 无序、持久化记录异步
|
||||
/// best-effort,都不能还原「上一账号」,因此由本字段单独记账。
|
||||
last_attempt_account: Option<crate::account::ProviderAccountId>,
|
||||
/// 已发生的换号次数:选中账号与上一 attempt 不同的路由 attempt 计一次。
|
||||
/// 首个 attempt 不计;同账号钉选重试(瞬态退避、传输恢复、凭据恢复重放、
|
||||
/// continuation 精确重连)不消耗。预算耗尽后所有必然换号的重试门关闭,
|
||||
/// 换号深度由 [`crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS`] 封顶。
|
||||
account_rotations: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::account::ProviderAccountId>,
|
||||
@@ -1109,6 +1120,9 @@ where
|
||||
self.continuation_attempt,
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny
|
||||
)
|
||||
// 跨 Provider 候选推进必然换号(账号行按 Provider 隔离),
|
||||
// 预算耗尽后不再推进,交回容量类失败的原有终态语义。
|
||||
&& self.account_rotations < crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS
|
||||
&& self.advance_provider_candidate();
|
||||
let retryable = self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
@@ -1275,6 +1289,18 @@ where
|
||||
.await?;
|
||||
return Err(EngineError::ContinuationPinMismatch);
|
||||
}
|
||||
// 换号预算按实际选中账号记账:与上一 attempt 选中账号不同即消耗一次。决策门
|
||||
// (重试分类、候选推进、continuation 排除臂)已拦截预算耗尽后的必然换号,
|
||||
// 这里是统一事实源,覆盖 Provider 自主换号(如 replay owner 重放)等路径;
|
||||
// 首 attempt 与同账号钉选重试不计数。
|
||||
if self.last_attempt_account.as_ref() != Some(metadata.provider_account_id())
|
||||
&& self
|
||||
.last_attempt_account
|
||||
.replace(metadata.provider_account_id().clone())
|
||||
.is_some()
|
||||
{
|
||||
self.account_rotations = self.account_rotations.saturating_add(1);
|
||||
}
|
||||
if self.account_state_owner.is_none() {
|
||||
self.account_state_owner = Some(ProviderAccountStateOwner::new(
|
||||
metadata.provider().clone(),
|
||||
@@ -1561,12 +1587,22 @@ where
|
||||
&& !self
|
||||
.credential_recovery_attempted_accounts
|
||||
.contains(current.metadata.provider_account_id());
|
||||
// 排除当前账号的重选必然换号:ordinary 重选与显式 AccountRotation 标记都落入
|
||||
// 同一排除分支。换号预算只在这里消耗;同账号分支(瞬态退避、传输恢复、
|
||||
// 凭据恢复重放、continuation 精确重连)不消耗。不能直接把预算门写进
|
||||
// ordinary_retry:它是同账号瞬态退避的前置条件,会被连带误伤
|
||||
let rotation_retry = !continuation_retry
|
||||
&& !same_account_retry
|
||||
&& transient_retry.is_none()
|
||||
&& transport_recovery.is_none()
|
||||
&& (ordinary_retry || account_rotation_retry)
|
||||
&& self.account_rotations < crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS;
|
||||
let retryable = !error.retry_is_prohibited()
|
||||
&& (continuation_retry
|
||||
|| same_account_retry
|
||||
|| ordinary_retry
|
||||
|| account_rotation_retry
|
||||
|| transport_recovery.is_some());
|
||||
|| transient_retry.is_some()
|
||||
|| transport_recovery.is_some()
|
||||
|| rotation_retry);
|
||||
|
||||
let retryable = match self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
@@ -1602,6 +1638,7 @@ where
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
"connectionRetry": connection_retry_requested,
|
||||
"connectionRetries": self.connection_retries,
|
||||
"connectionBudgetRemainingMs": self.connection_budget.remaining().map(duration_ms),
|
||||
@@ -1799,6 +1836,13 @@ where
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// 两个排除臂都必然换号;预算耗尽后不再排除当前账号做跨账号续写重放,
|
||||
// 落回不可重试路径以原始上游错误终态。
|
||||
ContinuationAttempt::ReplayOwner | ContinuationAttempt::ReplayAny
|
||||
if self.account_rotations >= crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS =>
|
||||
{
|
||||
return false;
|
||||
}
|
||||
ContinuationAttempt::ReplayOwner => {
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts
|
||||
@@ -1846,6 +1890,11 @@ where
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// Native 期间所有 attempt 都在 pin 账号上,到这里预算必未耗尽;
|
||||
// 与其余换号路径保持同一预算门,防止状态机演化后破坏换号上限。
|
||||
if self.account_rotations >= crate::routing::MAX_ACCOUNT_ROTATION_ATTEMPTS {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
|
||||
@@ -886,6 +886,49 @@ impl ProviderRefreshPolicy {
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 按账号派生 `[0, margin]` 内的整秒错峰偏移。
|
||||
///
|
||||
/// 网关以固定周期扫描到期账号,到期时刻相同的账号会在同一轮齐刷,
|
||||
/// 在 auth.openai.com 侧形成单 IP 批量刷新特征。偏移从账号 ID 的
|
||||
/// 稳定哈希派生(与退避扰动同源、独立 salt),跨扫描与重启保持不变;
|
||||
/// 只会增大有效提前量,不会比配置的 margin 更贴近过期时刻。
|
||||
#[must_use]
|
||||
pub fn refresh_stagger(self, account_id: &ProviderAccountId) -> Duration {
|
||||
// margin 以整秒配置;秒级粒度在默认 300s 下对应约 10 个扫描槽位。
|
||||
let bound = u32::try_from(self.margin.as_secs()).unwrap_or(u32::MAX);
|
||||
Duration::from_secs(u64::from(stable_factor(
|
||||
account_id.as_str(),
|
||||
"refresh-stagger",
|
||||
0,
|
||||
bound,
|
||||
)))
|
||||
}
|
||||
|
||||
/// 含账号错峰偏移的到期判定;扫描路径先按 [`Self::staggered_refresh_bound`]
|
||||
/// 取回候选超集,再用它在内存中收窄到本轮真正到期的账号。
|
||||
#[must_use]
|
||||
pub fn is_refresh_due_staggered(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> bool {
|
||||
let staggered_margin = self.margin.saturating_add(self.refresh_stagger(account_id));
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= staggered_margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 错峰候选窗口上界;覆盖 margin 与最大偏移之和。
|
||||
///
|
||||
/// 与 [`Self::refresh_stagger`] 的值域同址维护:偏移上限为 margin,
|
||||
/// 因此 `2 × margin` 必然覆盖最大有效提前量。
|
||||
#[must_use]
|
||||
pub fn staggered_refresh_bound(self) -> Duration {
|
||||
self.margin.saturating_mul(2)
|
||||
}
|
||||
}
|
||||
|
||||
/// 指数退避基准延迟;attempt=1 即为该值
|
||||
|
||||
@@ -26,6 +26,10 @@ use crate::validation::{IdentifierError, RoutingError, validate_text};
|
||||
|
||||
const MAX_REQUEST_ATTEMPTS: u32 = 32;
|
||||
|
||||
/// 单请求内允许的换号次数上限,选中账号与上一 attempt 不同的路由 attempt 记一次
|
||||
/// 初始账号和同账号重试不计入换号次数,所有尝试仍受总路由预算限制
|
||||
pub(crate) const MAX_ACCOUNT_ROTATION_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// 客户端请求中的模型名称
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
|
||||
@@ -292,6 +292,7 @@ enum Script {
|
||||
}
|
||||
|
||||
struct ScriptedProvider {
|
||||
provider_name: &'static str,
|
||||
profile_generation: AtomicUsize,
|
||||
default_profile_calls: AtomicUsize,
|
||||
default_profile: Mutex<Option<gateway_core::account::OpaqueProviderData>>,
|
||||
@@ -311,7 +312,13 @@ impl Drop for TrackedLease {
|
||||
|
||||
impl ScriptedProvider {
|
||||
fn new(scripts: Vec<Script>) -> Self {
|
||||
Self::named("openai", scripts)
|
||||
}
|
||||
|
||||
/// 双 Provider 计划需要第二个 Provider 实例扮演跨候选推进的目标。
|
||||
fn named(provider_name: &'static str, scripts: Vec<Script>) -> Self {
|
||||
Self {
|
||||
provider_name,
|
||||
profile_generation: AtomicUsize::new(1),
|
||||
default_profile_calls: AtomicUsize::new(0),
|
||||
default_profile: Mutex::new(None),
|
||||
@@ -345,7 +352,7 @@ impl Provider for ScriptedProvider {
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
self.provider_name
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
@@ -714,6 +721,63 @@ fn plan_with_profiles(
|
||||
.expect("routing plan")
|
||||
}
|
||||
|
||||
/// 双 Provider 路由计划:候选顺序为 openai、xai,不锁定 required_provider,
|
||||
/// 用于验证跨 Provider 候选推进受换号预算约束。
|
||||
fn dual_provider_plan(operation: &Operation) -> RoutingPlan {
|
||||
let openai = ProviderKind::new("openai").expect("provider");
|
||||
let xai = ProviderKind::new("xai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
let entries = [
|
||||
(openai.clone(), "acct_openai_first"),
|
||||
(openai.clone(), "acct_openai_second"),
|
||||
(openai.clone(), "acct_openai_third"),
|
||||
(openai.clone(), "acct_openai_fourth"),
|
||||
(xai.clone(), "acct_xai_first"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(provider, id)| {
|
||||
(
|
||||
ProviderAccountId::new(id).expect("account"),
|
||||
RuntimeAccount::new(provider, BTreeSet::new()),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let directory = Arc::new(RuntimeAccountDirectory::new(entries));
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
gateway_core::settings::SettingsValues::new(2, 50, "smart", Default::default(), None, None),
|
||||
vec![openai.clone(), xai.clone()],
|
||||
vec![
|
||||
ProviderModel::new(
|
||||
openai,
|
||||
UpstreamModelId::new("gpt-5").expect("upstream model"),
|
||||
capabilities.clone(),
|
||||
),
|
||||
ProviderModel::new(
|
||||
xai,
|
||||
UpstreamModelId::new("gpt-5").expect("upstream model"),
|
||||
capabilities,
|
||||
),
|
||||
],
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.with_account_directory(Arc::clone(&directory));
|
||||
snapshot
|
||||
.plan(
|
||||
&public_model,
|
||||
operation,
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
directory,
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)),
|
||||
&RoutingContext::default(),
|
||||
)
|
||||
.expect("routing plan")
|
||||
}
|
||||
|
||||
fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest {
|
||||
let client_key = ClientApiKeyId::new("key_client_1").expect("client key id");
|
||||
NewModelRequest {
|
||||
@@ -5062,3 +5126,332 @@ fn connection_failure_after_a_replay_safe_rejection_keeps_existing_account_rotat
|
||||
.contains(&ProviderAccountId::new("acct_second").unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_rotation_attempts_stop_at_the_independent_rotation_budget() {
|
||||
// 四个账号依次交付前换号重试,第 4 次换号被独立预算拦下
|
||||
// 不再重放到第 5 个账号,以最后账号上的原始上游错误终态
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_pre_delivery_retry())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_other",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_tool",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
// 预算内不应触达第 5 个账号,保留成功脚本以暴露超限重放
|
||||
Script::Stream {
|
||||
account_id: "acct_wrong",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("rotation budget must stop the fifth account");
|
||||
let gateway_core::engine::EngineError::Provider(error) = error else {
|
||||
panic!("expected provider error")
|
||||
};
|
||||
assert_eq!(error.kind(), ProviderErrorKind::RateLimited);
|
||||
assert_eq!(error.upstream_status(), Some(429));
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 4, "the fifth account must not be replayed");
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 4);
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 4);
|
||||
assert_eq!(state.finalizations[0].upstream_status_code, Some(429));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_account_retries_do_not_consume_the_account_rotation_budget() {
|
||||
// 同账号瞬态退避不消耗换号预算,三轮瞬态预算耗尽后仍允许换号
|
||||
// 第四个账号成功完成请求,验证 3 次换号预算未被同账号重试占用
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let mut scripts = (0..4)
|
||||
.map(|_| Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(transient_rejection())],
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
scripts.extend((0..4).map(|_| Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: vec![Err(transient_rejection())],
|
||||
}));
|
||||
scripts.extend((0..4).map(|_| Script::Stream {
|
||||
account_id: "acct_other",
|
||||
items: vec![Err(transient_rejection())],
|
||||
}));
|
||||
scripts.push(Script::Stream {
|
||||
account_id: "acct_tool",
|
||||
items: complete_stream(None),
|
||||
});
|
||||
let (coordinator, store, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("fourth account succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 13);
|
||||
let first = ProviderAccountId::new("acct_first").expect("account");
|
||||
let second = ProviderAccountId::new("acct_second").expect("account");
|
||||
let third = ProviderAccountId::new("acct_other").expect("account");
|
||||
// 每个账号的 3 次同账号退避都钉选原账号,只有预算耗尽后的失败换号。
|
||||
for context in &contexts[1..4] {
|
||||
assert_eq!(context.required_account(), Some(&first));
|
||||
}
|
||||
for context in &contexts[5..8] {
|
||||
assert_eq!(context.required_account(), Some(&second));
|
||||
}
|
||||
for context in &contexts[9..12] {
|
||||
assert_eq!(context.required_account(), Some(&third));
|
||||
}
|
||||
assert_eq!(contexts[4].required_account(), None);
|
||||
assert!(contexts[4].excluded_accounts().contains(&first));
|
||||
assert_eq!(contexts[8].required_account(), None);
|
||||
assert!(contexts[8].excluded_accounts().contains(&first));
|
||||
assert!(contexts[8].excluded_accounts().contains(&second));
|
||||
assert_eq!(contexts[12].required_account(), None);
|
||||
assert!(contexts[12].excluded_accounts().contains(&first));
|
||||
assert!(contexts[12].excluded_accounts().contains(&second));
|
||||
assert!(contexts[12].excluded_accounts().contains(&third));
|
||||
assert_eq!(
|
||||
store.state.lock().expect("store").finalizations[0].attempt_count,
|
||||
13
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuation_replay_rotation_stops_at_the_account_rotation_budget() {
|
||||
// native → replay owner → replay any 的排除臂也受换号预算约束:
|
||||
// 第三次换号后不再排除当前账号续写换号,以原始上游错误终态
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let mut payload = Map::new();
|
||||
payload.insert("transcript".to_owned(), Value::Array(Vec::new()));
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", payload).expect("provider session state"),
|
||||
));
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![
|
||||
Ok(GatewayEvent::Started(ResponseMeta::new(
|
||||
"response-native",
|
||||
"gpt-5",
|
||||
))),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_continuation_recovery_disposition(
|
||||
ContinuationRecoveryDisposition::ProviderReplayAllowed,
|
||||
)
|
||||
.with_replay_safe(),
|
||||
),
|
||||
],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_tool",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_other",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_wrong",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("rotation budget must stop continuation replay");
|
||||
let gateway_core::engine::EngineError::Provider(error) = error else {
|
||||
panic!("expected provider error")
|
||||
};
|
||||
assert_eq!(error.kind(), ProviderErrorKind::RateLimited);
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 4, "the fifth account must not be replayed");
|
||||
assert_eq!(
|
||||
contexts[2].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[3].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_candidate_advance_stops_at_the_account_rotation_budget() {
|
||||
// openai 上第三次换号耗尽预算后,空选路不再推进到 xai
|
||||
// 沿用容量失败的原有终态语义,返回最后一个可重试上游错误
|
||||
let operation = generate_operation();
|
||||
let route_plan = dual_provider_plan(&operation);
|
||||
let store = Arc::new(FakeStore::default());
|
||||
let openai = Arc::new(ScriptedProvider::new(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_openai_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_openai_second",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_openai_third",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_openai_fourth",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::UpstreamCapacityUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_replay_safe()
|
||||
.with_transient_retry(
|
||||
NonZeroU32::new(3).expect("retry budget"),
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
))],
|
||||
},
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
]));
|
||||
let xai = Arc::new(ScriptedProvider::named(
|
||||
"xai",
|
||||
vec![Script::Stream {
|
||||
account_id: "acct_xai_first",
|
||||
items: complete_stream(None),
|
||||
}],
|
||||
));
|
||||
let mut registry = ProviderRegistry::builder();
|
||||
registry.register(openai.clone()).expect("register openai");
|
||||
registry.register(xai.clone()).expect("register xai");
|
||||
let engine = GatewayEngine::new(store.clone(), registry.build());
|
||||
let coordinator = AttemptCoordinator::new(engine);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("candidate advance must stop at the rotation budget");
|
||||
let gateway_core::engine::EngineError::Provider(error) = error else {
|
||||
panic!("expected provider error")
|
||||
};
|
||||
assert_eq!(error.kind(), ProviderErrorKind::UpstreamCapacityUnavailable);
|
||||
assert_eq!(
|
||||
openai.contexts.lock().expect("contexts lock").len(),
|
||||
5,
|
||||
"the fourth stream failure and the empty selection still run"
|
||||
);
|
||||
assert!(
|
||||
xai.contexts.lock().expect("contexts lock").is_empty(),
|
||||
"the rotation budget must block the provider candidate advance"
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 4);
|
||||
assert_eq!(state.finalizations.len(), 1);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 4);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 验证 Provider 协调端口的敏感值保护、有效期与刷新边界
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
@@ -84,6 +84,113 @@ fn refresh_policy_should_mark_expired_tokens_due() {
|
||||
assert!(policy.is_refresh_due(observed_at - Duration::from_secs(1), observed_at));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_stagger_is_stable_and_bounded_by_the_margin() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(300),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
)
|
||||
.expect("valid policy");
|
||||
let account = ProviderAccountId::new("acct_stagger").expect("valid account");
|
||||
|
||||
let first = policy.refresh_stagger(&account);
|
||||
for _ in 0..10 {
|
||||
assert_eq!(policy.refresh_stagger(&account), first);
|
||||
}
|
||||
assert!(first <= policy.margin());
|
||||
// 偏移随 margin 缩放,不超出新窗口。
|
||||
let widened = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
)
|
||||
.expect("valid policy");
|
||||
assert!(widened.refresh_stagger(&account) <= widened.margin());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_stagger_should_spread_accounts_across_the_margin() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(300),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
)
|
||||
.expect("valid policy");
|
||||
let staggers = (0..64)
|
||||
.map(|index| {
|
||||
let account =
|
||||
ProviderAccountId::new(format!("acct_spread_{index}")).expect("valid account");
|
||||
let stagger = policy.refresh_stagger(&account);
|
||||
assert!(stagger <= policy.margin());
|
||||
stagger
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// 确定性哈希下的分散性:64 个账号在 301 个可能取值中覆盖足够多的档位。
|
||||
let distinct = staggers.iter().collect::<BTreeSet<_>>().len();
|
||||
assert!(
|
||||
distinct >= 24,
|
||||
"stagger values are too concentrated: {staggers:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_policy_staggered_due_boundaries() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(300),
|
||||
NonZeroU32::new(2).expect("positive concurrency"),
|
||||
)
|
||||
.expect("valid policy");
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
|
||||
for index in 0..8 {
|
||||
let account =
|
||||
ProviderAccountId::new(format!("acct_boundary_{index}")).expect("valid account");
|
||||
// 恰好 margin:任何偏移下都到期;2×margin+1 秒:任何偏移下都未到期。
|
||||
assert!(policy.is_refresh_due_staggered(
|
||||
&account,
|
||||
observed_at + Duration::from_secs(300),
|
||||
observed_at,
|
||||
));
|
||||
assert!(!policy.is_refresh_due_staggered(
|
||||
&account,
|
||||
observed_at + Duration::from_secs(601),
|
||||
observed_at,
|
||||
));
|
||||
// 已过期账号不受偏移影响,恒到期。
|
||||
assert!(policy.is_refresh_due_staggered(
|
||||
&account,
|
||||
observed_at - Duration::from_secs(1),
|
||||
observed_at,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_policy_staggered_due_matches_margin_plus_stagger() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(300),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
)
|
||||
.expect("valid policy");
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
|
||||
for index in 0..4 {
|
||||
let account =
|
||||
ProviderAccountId::new(format!("acct_compose_{index}")).expect("valid account");
|
||||
let staggered_margin = policy.margin() + policy.refresh_stagger(&account);
|
||||
for remaining_secs in [299, 300, 301, 400, 450, 500, 599, 600, 601] {
|
||||
assert_eq!(
|
||||
policy.is_refresh_due_staggered(
|
||||
&account,
|
||||
observed_at + Duration::from_secs(remaining_secs),
|
||||
observed_at,
|
||||
),
|
||||
Duration::from_secs(remaining_secs) <= staggered_margin,
|
||||
"remaining {remaining_secs}s"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
let account = ProviderAccountId::new("acct_fixture").expect("valid account");
|
||||
|
||||
@@ -268,7 +268,9 @@ async fn plugin_migration_preserves_main_settings_and_accepts_new_installations(
|
||||
return;
|
||||
};
|
||||
initialize_revision(&database).await;
|
||||
sqlx::query("update runtime_settings set max_concurrent_per_account=0 where id=1")
|
||||
// 0022 会把仍为旧默认 3600 的 margin 收敛到 300;预设自定义值,
|
||||
// 让本测试继续验证「所有已自定义设置都原样保留」。
|
||||
sqlx::query("update runtime_settings set max_concurrent_per_account=0, refresh_margin_seconds=1800 where id=1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -67,7 +67,8 @@ async fn smart_settings_upgrade_preserves_selection_and_publishes_custom_config(
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(before.rotation_strategy, "sticky");
|
||||
assert_eq!(before.refresh_margin_seconds, 3600);
|
||||
// 0022 将仍为旧默认 3600 的行迁移到 300;管理员自定义值才会原样保留。
|
||||
assert_eq!(before.refresh_margin_seconds, 300);
|
||||
assert_eq!(before.smart_scheduling, SmartSchedulingConfig::default());
|
||||
let mut update = settings_with_margin(3600);
|
||||
update.smart_scheduling =
|
||||
@@ -171,6 +172,56 @@ async fn unlimited_default_account_concurrency_round_trips_without_relaxing_othe
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migrations_should_narrow_the_default_refresh_margin_to_the_codex_baseline() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_default").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
// 新装实例:0001 种子行携带旧默认 3600,0022 数据迁移统一收敛到 300。
|
||||
let settings = repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("load settings");
|
||||
assert_eq!(settings.refresh_margin_seconds, 300);
|
||||
// 列默认值同步收窄,重建行不会回退到旧值;目录查询限定本测试 schema,
|
||||
// 同库并行测试里停留在旧迁移版本的 schema 仍保留 3600 旧默认,不能被读到。
|
||||
let column_default: String = sqlx::query_scalar(
|
||||
"select column_default from information_schema.columns \
|
||||
where table_schema = current_schema() \
|
||||
and table_name = 'runtime_settings' \
|
||||
and column_name = 'refresh_margin_seconds'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read column default");
|
||||
assert_eq!(column_default, "300");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_margin_migration_should_preserve_admin_customized_values() {
|
||||
// 升级语义:0021 之前部署的实例经 0022 迁移后,仅旧默认 3600 收敛到 300,
|
||||
// 管理员自定义值原样保留。
|
||||
let Some(database) = TestDatabase::create_through("refresh_margin_custom", 21).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("update runtime_settings set refresh_margin_seconds = 1800 where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("persist customized margin");
|
||||
super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.await
|
||||
.expect("apply remaining migrations");
|
||||
let settings = PgRuntimeSettingsRepository::new(database.pool.clone())
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("load settings");
|
||||
assert_eq!(settings.refresh_margin_seconds, 1_800);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_reject_invalid_model_mapping() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
|
||||
@@ -51,8 +51,9 @@ pub use profile_statistics::{
|
||||
};
|
||||
pub use quota::{
|
||||
CodexAccountQuotaSnapshot, CodexCredentialQuotaError, CodexCredentialQuotaService,
|
||||
CodexQuotaFact, CodexQuotaRefreshPolicy, CodexQuotaSyncSummary, CodexQuotaWindow,
|
||||
CodexQuotaWindowKind, CodexQuotaWindowRole, CodexResetCreditsError, parse_codex_quota_usage,
|
||||
CodexInitialSyncDelays, CodexQuotaFact, CodexQuotaRefreshPolicy, CodexQuotaSyncSummary,
|
||||
CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole, CodexResetCreditsError,
|
||||
parse_codex_quota_usage,
|
||||
};
|
||||
pub use refresh::{
|
||||
CodexCredentialRefreshError, CodexCredentialRefreshOutcome, CodexCredentialRefreshService,
|
||||
|
||||
@@ -66,7 +66,11 @@ const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
|
||||
const PERIODIC_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const QUOTA_RESET_GRACE: Duration = Duration::from_secs(2 * 60);
|
||||
/// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限
|
||||
/// 随机启动延迟已把候选摊到多轮,该上限仅作单轮安全边界
|
||||
const INITIAL_QUOTA_SYNC_BATCH: usize = 100;
|
||||
/// 新账号首查(含失败重排)随机启动延迟的上限;摊开导入批量的 /usage 突发
|
||||
/// 导入即时可见额度由导入流程的每账号后台读取负责,worker 只兜底失败者
|
||||
const INITIAL_QUOTA_SYNC_MAX_START_DELAY: Duration = Duration::from_secs(10 * 60);
|
||||
// 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动
|
||||
const QUOTA_FETCH_5XX_MAX_RETRIES: u32 = 2;
|
||||
const QUOTA_FETCH_5XX_BASE_DELAY: Duration = Duration::from_secs(1);
|
||||
@@ -312,8 +316,14 @@ pub struct CodexCredentialQuotaService {
|
||||
freeze_policy_cache: Mutex<Option<(ProviderFreezePolicy, Instant)>>,
|
||||
scheduling: CodexQuotaSchedulingProjection,
|
||||
reset_consume_locks: Mutex<HashMap<ProviderAccountId, Arc<Mutex<()>>>>,
|
||||
/// 首查随机延迟采样器;测试注入固定值获得确定性节奏。
|
||||
initial_sync_delays: CodexInitialSyncDelays,
|
||||
}
|
||||
|
||||
/// 首查随机延迟采样器:每次调用为一个账号返回独立随机延迟
|
||||
#[doc(hidden)]
|
||||
pub type CodexInitialSyncDelays = Arc<dyn Fn() -> Duration + Send + Sync>;
|
||||
|
||||
/// 冻结策略缓存活跃期;过期后下一次容量错误重新读取运行时设置
|
||||
const FREEZE_POLICY_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
@@ -334,6 +344,7 @@ struct CodexQuotaProjectionState {
|
||||
next_version: u64,
|
||||
entries: BTreeMap<ProviderAccountId, CodexQuotaSchedulingEntry>,
|
||||
last_periodic_refresh_at: BTreeMap<ProviderAccountId, CodexQuotaRefreshAttempt>,
|
||||
initial_refresh_not_before: BTreeMap<ProviderAccountId, Instant>,
|
||||
}
|
||||
|
||||
struct CodexQuotaRefreshAttempt {
|
||||
@@ -392,6 +403,7 @@ impl CodexQuotaSchedulingProjection {
|
||||
for account_id in account_ids {
|
||||
state.entries.remove(account_id);
|
||||
state.last_periodic_refresh_at.remove(account_id);
|
||||
state.initial_refresh_not_before.remove(account_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -555,6 +567,77 @@ impl CodexQuotaSchedulingProjection {
|
||||
}
|
||||
reserved
|
||||
}
|
||||
|
||||
/// 为尚无 quota 快照的账号分配随机首查时点,并返回本轮已到期的候选。
|
||||
///
|
||||
/// `quota_observed_at` 为空代表首次异步观察尚未成功;不另建同步状态表。
|
||||
/// 首次进入候选的账号抽一次随机启动延迟记为到期时刻,避免新导入账号池
|
||||
/// 在 worker 唤醒边界形成批量 `/usage` 突发;到期时刻跨轮保留,零延迟
|
||||
/// (含随机源退化)时保持首轮即查的旧行为。单轮数量仍受
|
||||
/// `INITIAL_QUOTA_SYNC_BATCH` 上限约束。
|
||||
fn reserve_initial_refreshes(
|
||||
&self,
|
||||
accounts: &[ProviderAccount],
|
||||
observed_ids: &BTreeSet<ProviderAccountId>,
|
||||
now: SystemTime,
|
||||
draw_start_delay: &dyn Fn() -> Duration,
|
||||
) -> Vec<ProviderAccount> {
|
||||
let candidates = accounts
|
||||
.iter()
|
||||
.filter(|account| {
|
||||
!observed_ids.contains(account.id()) && eligible_initial_quota_sync(account, now)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_ids = candidates
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let monotonic_now = Instant::now();
|
||||
let mut state = self
|
||||
.state
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// 已产生快照或离开候选集的账号不再保留首查状态。
|
||||
state
|
||||
.initial_refresh_not_before
|
||||
.retain(|account_id, _| candidate_ids.contains(account_id));
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|account| {
|
||||
let due_at = state
|
||||
.initial_refresh_not_before
|
||||
.entry(account.id().clone())
|
||||
.or_insert_with(|| monotonic_now + draw_start_delay());
|
||||
*due_at <= monotonic_now
|
||||
})
|
||||
.take(INITIAL_QUOTA_SYNC_BATCH)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 首查尝试后仍无快照的账号重抽随机延迟,避免失败账号回到固定 30s 重试节奏。
|
||||
///
|
||||
/// 重排同时随机化每轮的候选集合,避免持续失败时按账号顺序只重试同一批。
|
||||
fn defer_failed_initial_refreshes(
|
||||
&self,
|
||||
account_ids: &BTreeSet<ProviderAccountId>,
|
||||
draw_start_delay: &dyn Fn() -> Duration,
|
||||
) {
|
||||
if account_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
let now = Instant::now();
|
||||
let mut state = self
|
||||
.state
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for account_id in account_ids {
|
||||
// 账号是否仍在首查路径由下一轮 reserve 按候选集剪枝,这里无需区分。
|
||||
state
|
||||
.initial_refresh_not_before
|
||||
.insert(account_id.clone(), now + draw_start_delay());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_refresh_candidate(
|
||||
@@ -649,9 +732,23 @@ impl CodexCredentialQuotaService {
|
||||
freeze_policy_cache: Mutex::new(None),
|
||||
scheduling: CodexQuotaSchedulingProjection::default(),
|
||||
reset_consume_locks: Mutex::new(HashMap::new()),
|
||||
initial_sync_delays: Arc::new(|| {
|
||||
crate::jitter::uniform_delay(
|
||||
crate::jitter::random_u64(),
|
||||
INITIAL_QUOTA_SYNC_MAX_START_DELAY,
|
||||
)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// 注入首查随机延迟采样器(测试用确定性节奏);生产默认均匀
|
||||
/// `[0, INITIAL_QUOTA_SYNC_MAX_START_DELAY)`,零延迟等价旧行为
|
||||
#[doc(hidden)]
|
||||
pub fn with_initial_sync_delays(mut self, delays: CodexInitialSyncDelays) -> Self {
|
||||
self.initial_sync_delays = delays;
|
||||
self
|
||||
}
|
||||
|
||||
/// 读取容量熔断策略(带短 TTL 缓存);读取失败退化为关闭,熔断不得放大失败
|
||||
async fn freeze_policy(&self) -> ProviderFreezePolicy {
|
||||
{
|
||||
@@ -949,7 +1046,15 @@ impl CodexCredentialQuotaService {
|
||||
.into_iter()
|
||||
.map(|observation| observation.account_id)
|
||||
.collect::<BTreeSet<_>>();
|
||||
let initial = Self::initial_quota_sync_accounts(&accounts, &observed_ids, now);
|
||||
let initial = self.scheduling.reserve_initial_refreshes(
|
||||
&accounts,
|
||||
&observed_ids,
|
||||
now,
|
||||
self.initial_sync_delays.as_ref(),
|
||||
);
|
||||
// 本轮首查后仍无快照的账号需要重排随机延迟。
|
||||
let mut pending_initial: BTreeSet<ProviderAccountId> =
|
||||
initial.iter().map(|account| account.id().clone()).collect();
|
||||
let periodic =
|
||||
self.scheduling
|
||||
.reserve_periodic_refreshes(accounts, &observed_snapshots, now);
|
||||
@@ -975,16 +1080,21 @@ impl CodexCredentialQuotaService {
|
||||
match self.fetch_usage(&client, &account).await {
|
||||
Ok(FetchedCodexQuota { account, value }) => {
|
||||
// 单账号解析或落库失败只影响该账号;其余账号继续同步
|
||||
if let Err(error) = self
|
||||
match self
|
||||
.apply_fetched_quota(&account, &value, observed_at, &mut summary)
|
||||
.await
|
||||
{
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI quota synchronization skipped one account"
|
||||
);
|
||||
Ok(()) => {
|
||||
pending_initial.remove(account.id());
|
||||
}
|
||||
Err(error) => {
|
||||
summary.transient += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI quota synchronization skipped one account"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(CodexQuotaFetchError::InvalidCredential) => {
|
||||
@@ -1025,6 +1135,9 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
}
|
||||
}
|
||||
// 首查失败或未能落库的账号重排随机延迟;已成功落库的账号由快照事实自然退出首查路径。
|
||||
self.scheduling
|
||||
.defer_failed_initial_refreshes(&pending_initial, self.initial_sync_delays.as_ref());
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
@@ -1220,22 +1333,6 @@ impl CodexCredentialQuotaService {
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// `quota_observed_at` 为空代表首次异步观察尚未成功;不另建同步状态表
|
||||
fn initial_quota_sync_accounts(
|
||||
accounts: &[ProviderAccount],
|
||||
observed_ids: &BTreeSet<ProviderAccountId>,
|
||||
now: SystemTime,
|
||||
) -> Vec<ProviderAccount> {
|
||||
accounts
|
||||
.iter()
|
||||
.filter(|account| {
|
||||
!observed_ids.contains(account.id()) && eligible_initial_quota_sync(account, now)
|
||||
})
|
||||
.take(INITIAL_QUOTA_SYNC_BATCH)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 解析并 revision-fenced 落库单账号的 Provider quota JSON
|
||||
async fn apply_fetched_quota(
|
||||
&self,
|
||||
|
||||
@@ -397,9 +397,12 @@ impl CodexCredentialRefreshService {
|
||||
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?;
|
||||
let limit = NonZeroU32::new(MAX_REFRESH_BATCH)
|
||||
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
|
||||
// 查询窗口按错峰上界(2×margin)放宽,先取回候选超集,
|
||||
// 再在内存中按账号稳定偏移收窄;存储层谓词保持通用窗口语义。
|
||||
let query = ProviderRefreshQuery::new(
|
||||
provider,
|
||||
now.checked_add(policy.margin()).unwrap_or(now),
|
||||
now.checked_add(policy.staggered_refresh_bound())
|
||||
.unwrap_or(now),
|
||||
now.checked_sub(REFRESH_RECOVERY_WINDOW)
|
||||
.unwrap_or(SystemTime::UNIX_EPOCH),
|
||||
now,
|
||||
@@ -410,6 +413,18 @@ impl CodexCredentialRefreshService {
|
||||
let mut due = Vec::with_capacity(candidates.len());
|
||||
let mut failures = Vec::new();
|
||||
for loaded in candidates {
|
||||
// 按账号稳定错峰偏移收窄到本轮真正到期的账号,避免同批到期账号
|
||||
// 同一轮齐刷;已过期账号 remaining 为负、偏移判定恒真,恢复窗口
|
||||
// 的强制刷新不受错峰影响。无到期时间的账号不会成为候选。
|
||||
if !loaded
|
||||
.account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| {
|
||||
policy.is_refresh_due_staggered(loaded.account.id(), expires_at, now)
|
||||
})
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let account_id = loaded.account.id().to_string();
|
||||
match self.repository.decode_runtime_credential(&loaded) {
|
||||
Ok(runtime)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
//! usage 观测节奏的共享随机源与抖动映射。
|
||||
//!
|
||||
//! 多账号池化代理的固定周期请求(整点目录轮询、固定延迟补查、批量首查)
|
||||
//! 会形成可识别的共享特征;官方 codex 客户端不存在任何周期 usage 轮询。
|
||||
//! 本模块集中提供观测节奏的随机化策略:周期任务用单侧附加抖动
|
||||
//! `[0, base + jitter)`(k8s `wait.Jitter`、systemd `RandomizedDelaySec`
|
||||
//! 同款惯例,不缩短基础周期、不增加请求速率),重试类延迟用小范围均匀
|
||||
//! 随机替换固定值。所有映射都是纯函数,`None`(系统随机源不可用)时
|
||||
//! 退化为旧的固定基准,与 `websocket_retry_backoff` 的降级风格一致。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// 从系统随机源读取一个 u64 样本;失败返回 `None`,由调用方退化为固定基准。
|
||||
pub(crate) fn random_u64() -> Option<u64> {
|
||||
let mut bytes = [0_u8; 8];
|
||||
getrandom::fill(&mut bytes)
|
||||
.ok()
|
||||
.map(|()| u64::from_le_bytes(bytes))
|
||||
}
|
||||
|
||||
/// 把随机样本映射为 `[0, max)` 内的均匀时长。
|
||||
///
|
||||
/// `max` 为零或随机源不可用时返回零,即不引入延迟(保持旧行为)。
|
||||
#[doc(hidden)]
|
||||
pub fn uniform_delay(sample: Option<u64>, max: Duration) -> Duration {
|
||||
let Some(sample) = sample else {
|
||||
return Duration::ZERO;
|
||||
};
|
||||
if max.is_zero() {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
let nanos = (u128::from(sample) % max.as_nanos()).min(u64::MAX as u128);
|
||||
Duration::from_nanos(nanos as u64)
|
||||
}
|
||||
|
||||
/// 额度失败补查的基准延迟;随机源不可用时退回该固定值。
|
||||
const QUOTA_FAILURE_REFRESH_DELAY: Duration = Duration::from_secs(2);
|
||||
/// 补查随机区间的下限(含)。
|
||||
const QUOTA_FAILURE_REFRESH_MIN_DELAY: Duration = Duration::from_secs(1);
|
||||
/// 补查随机区间的上限(不含),即基准的 +50%。
|
||||
const QUOTA_FAILURE_REFRESH_MAX_DELAY: Duration = Duration::from_secs(3);
|
||||
|
||||
/// 额度拒绝后补查 usage 的随机延迟:基准 2s ±50%,即 `[1s, 3s)`。
|
||||
///
|
||||
/// 固定 2 秒会让同时受限的多账号补查完全同步;区间保持在上游结算窗口
|
||||
/// 附近,避免显著拖慢展示基线。随机源不可用时退回固定基准。
|
||||
#[doc(hidden)]
|
||||
pub fn quota_failure_refresh_delay(sample: Option<u64>) -> Duration {
|
||||
match sample {
|
||||
Some(sample) => {
|
||||
// 跨度是「上限 − 下限」而非「基准 − 下限」,否则区间上半段永远取不到。
|
||||
QUOTA_FAILURE_REFRESH_MIN_DELAY
|
||||
+ uniform_delay(
|
||||
Some(sample),
|
||||
QUOTA_FAILURE_REFRESH_MAX_DELAY - QUOTA_FAILURE_REFRESH_MIN_DELAY,
|
||||
)
|
||||
}
|
||||
None => QUOTA_FAILURE_REFRESH_DELAY,
|
||||
}
|
||||
}
|
||||
|
||||
/// catalog 周期刷新的单侧抖动比例(20%)。
|
||||
const CATALOG_REFRESH_JITTER_PER_MILLE: u128 = 200;
|
||||
|
||||
/// catalog 周期刷新的附加抖动:`[0, 20%·interval)` 单侧均匀。
|
||||
///
|
||||
/// 只追加不缩短:有效周期在配置周期之上随机浮动,打破固定整点节奏,
|
||||
/// 同时不提高刷新频率。随机源不可用或周期过小时不抖动。
|
||||
#[doc(hidden)]
|
||||
pub fn catalog_refresh_jitter(sample: Option<u64>, interval: Duration) -> Duration {
|
||||
let Some(sample) = sample else {
|
||||
return Duration::ZERO;
|
||||
};
|
||||
let max_nanos = interval.as_nanos() * CATALOG_REFRESH_JITTER_PER_MILLE / 1_000;
|
||||
if max_nanos == 0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
let nanos = (u128::from(sample) % max_nanos).min(u64::MAX as u128);
|
||||
Duration::from_nanos(nanos as u64)
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod admin;
|
||||
pub mod config;
|
||||
mod jitter;
|
||||
mod provider;
|
||||
mod session_transport;
|
||||
|
||||
@@ -28,6 +29,8 @@ use crate::transport::profile::{
|
||||
use crate::transport::{CodexWebSocketPool, build_reqwest_client};
|
||||
|
||||
pub use config::{OpenAiConfig, OpenAiConfigError};
|
||||
#[doc(hidden)]
|
||||
pub use jitter::{catalog_refresh_jitter, quota_failure_refresh_delay, uniform_delay};
|
||||
pub use provider::{
|
||||
CodexProvider, CodexProviderConfigError, CodexProviderTransport, OFFICIAL_CODEX_BASE_PATH,
|
||||
OFFICIAL_CODEX_BASE_URL, openai_failure_affects_account_score,
|
||||
|
||||
@@ -447,11 +447,15 @@ pub(super) fn schedule_authoritative_quota_refresh_after_failure(
|
||||
) {
|
||||
// 限额错误可以确认账号状态,但 Responses 事件中的 used_percent 可能仍停在上一结算点
|
||||
// usage 快照在后台补齐展示基线,不得撤销同一轮真实失败,也不能阻塞原始响应
|
||||
// 先等待 2 秒让上游结算,再查询;5 秒仅限制实际查询自身
|
||||
// 随机等待 1–3 秒让上游结算再查询,避免多账号同时受限时补查完全同步;
|
||||
// 仍保持单次语义(一次 sleep + 一次查询),5 秒仅限制实际查询自身
|
||||
let quota = Arc::clone(quota);
|
||||
let account_id = account.id().clone();
|
||||
drop(tokio::spawn(async move {
|
||||
tokio::time::sleep(QUOTA_FAILURE_REFRESH_DELAY).await;
|
||||
tokio::time::sleep(crate::jitter::quota_failure_refresh_delay(
|
||||
crate::jitter::random_u64(),
|
||||
))
|
||||
.await;
|
||||
match tokio::time::timeout(
|
||||
QUOTA_FAILURE_REFRESH_TIMEOUT,
|
||||
quota.refresh_account_after_failure(&account_id),
|
||||
|
||||
@@ -131,8 +131,7 @@ const MAX_STREAM_PREFETCH_BYTES: usize = 128 * 1024;
|
||||
/// 短暂保留 response.created 等结构事件,让随后到达的明确拒绝可以无感换号;
|
||||
/// 到期即放行,避免模型长时间思考时让客户端一直收不到首事件
|
||||
const STREAM_REPLAY_GRACE: Duration = Duration::from_millis(2_500);
|
||||
// 额度拒绝后先给上游额度结算留出时间,再以受限时长同步 usage 快照
|
||||
const QUOTA_FAILURE_REFRESH_DELAY: Duration = Duration::from_secs(2);
|
||||
// 额度拒绝后的 usage 补查自身执行时长上限;结算等待的随机延迟见 jitter 模块
|
||||
const QUOTA_FAILURE_REFRESH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
pub const OFFICIAL_CODEX_BASE_PATH: &str = "/backend-api";
|
||||
pub const OFFICIAL_CODEX_BASE_URL: &str = "https://chatgpt.com/backend-api";
|
||||
|
||||
@@ -39,6 +39,7 @@ pub(crate) fn worker_contributions(
|
||||
WorkerId::try_new(WorkerKind::QuotaCatalogHealth, DESKTOP_RELEASE_WORKER_OWNER)?;
|
||||
let cli_release_id = WorkerId::try_new(WorkerKind::QuotaCatalogHealth, "openai-cli-release")?;
|
||||
let warmup_id = WorkerId::try_new(WorkerKind::QuotaCatalogHealth, WARMUP_WORKER_OWNER)?;
|
||||
let catalog_interval = quota_refresh_policy.interval();
|
||||
let mut contributions = Vec::new();
|
||||
if oauth_refresh_enabled {
|
||||
contributions.push(WorkerContribution::Registration(scheduled_registration(
|
||||
@@ -79,9 +80,10 @@ pub(crate) fn worker_contributions(
|
||||
)?),
|
||||
WorkerContribution::Registration(scheduled_registration(
|
||||
catalog_id,
|
||||
quota_refresh_policy.interval(),
|
||||
catalog_interval,
|
||||
Box::new(OpenAiCatalogTask {
|
||||
catalog: Arc::clone(&catalog),
|
||||
interval: catalog_interval,
|
||||
}),
|
||||
)?),
|
||||
WorkerContribution::Registration(WorkerRegistration::try_new(
|
||||
@@ -207,6 +209,8 @@ pub(super) struct OpenAiQuotaTask {
|
||||
|
||||
pub(super) struct OpenAiCatalogTask {
|
||||
catalog: Arc<CodexCredentialCatalogService>,
|
||||
/// 配置的目录刷新周期;用于成功周期尾部的随机抖动。
|
||||
interval: Duration,
|
||||
}
|
||||
|
||||
pub(super) struct OpenAiCatalogEtagTask {
|
||||
@@ -273,7 +277,7 @@ impl ScheduledTask for OpenAiCatalogTask {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
match self.catalog.refresh_catalogs().await {
|
||||
let result = match self.catalog.refresh_catalogs().await {
|
||||
Ok(_) | Err(CodexCredentialCatalogError::NoEligibleCredential) => Ok(()),
|
||||
Err(error) => {
|
||||
tracing::warn!(error = %error, "OpenAI model catalog refresh failed");
|
||||
@@ -281,7 +285,21 @@ impl ScheduledTask for OpenAiCatalogTask {
|
||||
"OpenAI model catalog synchronization failed",
|
||||
))
|
||||
}
|
||||
};
|
||||
// 成功周期尾部追加 [0, 20%) 单侧随机抖动,打破固定周期轮询特征;
|
||||
// 放在周期尾部避免延迟冷启动首轮刷新,失败路径交给宿主退避不叠加。
|
||||
// 抖动期间 leader lease 由宿主监督循环并发续租,不会超时。
|
||||
if result.is_ok() {
|
||||
let jitter = crate::jitter::catalog_refresh_jitter(
|
||||
crate::jitter::random_u64(),
|
||||
self.interval,
|
||||
);
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => {},
|
||||
() = tokio::time::sleep(jitter) => {},
|
||||
}
|
||||
}
|
||||
result
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Codex Desktop 上游请求画像
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -155,13 +155,33 @@ pub struct CodexWireProfileState {
|
||||
|
||||
type ClientReleases = BTreeMap<(ClientKind, ClientPlatform, String), ClientReleaseObservation>;
|
||||
|
||||
/// 每渠道保留的发布观察深度:当前版本加上全部滞后档位。
|
||||
const RELEASE_HISTORY_CAPACITY: usize = selection::MAX_VERSION_LAG as usize + 1;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct ClientReleaseObservation {
|
||||
release: Option<ClientRelease>,
|
||||
/// 本渠道倒序观察到的发布序列(含当前版本),仅进程内存;重启后从种子与缓存重新积累。
|
||||
history: VecDeque<ClientRelease>,
|
||||
checked_at: Option<DateTime<Utc>>,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
/// 观察到的发布进入历史:同版本刷新队首,新版本压入队首;超出深度丢弃最旧一项。
|
||||
fn observe_release(history: &mut VecDeque<ClientRelease>, release: ClientRelease) {
|
||||
if history
|
||||
.front()
|
||||
.is_some_and(|front| front.same_identity(&release))
|
||||
{
|
||||
history[0] = release;
|
||||
} else {
|
||||
history.push_front(release);
|
||||
}
|
||||
while history.len() > RELEASE_HISTORY_CAPACITY {
|
||||
history.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexWireProfileState {
|
||||
/// 从启动画像创建运行时状态
|
||||
pub fn new(profile: CodexWireProfile) -> Self {
|
||||
@@ -169,6 +189,20 @@ impl CodexWireProfileState {
|
||||
profile: Arc::new(RwLock::new(profile)),
|
||||
releases: Arc::default(),
|
||||
};
|
||||
// macOS arm64 的当前发布来自初始画像(默认快照或恢复缓存);补齐观察表项,
|
||||
// 让滞后档位在首次 appcast 检查前也有历史可解析。
|
||||
let initial = state.snapshot();
|
||||
state.seed_client_release(
|
||||
ClientKind::Desktop,
|
||||
ClientPlatform::Macos,
|
||||
"arm64",
|
||||
ClientRelease {
|
||||
codex_version: initial.codex_version,
|
||||
desktop_version: Some(initial.desktop_version),
|
||||
desktop_build: Some(initial.desktop_build),
|
||||
verified_at: Some(initial.verified_at),
|
||||
},
|
||||
);
|
||||
cli_release::seed_releases(&state);
|
||||
platform_release::seed_releases(&state);
|
||||
state
|
||||
@@ -216,6 +250,29 @@ impl CodexWireProfileState {
|
||||
.and_then(|state| state.release.clone())
|
||||
}
|
||||
|
||||
/// 按滞后档位解析发布:取观察历史的第 `lag` 项,历史不足时回退最旧一项。
|
||||
///
|
||||
/// 滞后档位不引入新的失败模式:历史缺失时调用方回退到当前发布。
|
||||
pub fn lagged_client_release(
|
||||
&self,
|
||||
client: ClientKind,
|
||||
platform: ClientPlatform,
|
||||
arch: &str,
|
||||
lag: u32,
|
||||
) -> Option<ClientRelease> {
|
||||
self.releases
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&(client, platform, arch.to_owned()))
|
||||
.and_then(|state| {
|
||||
state
|
||||
.history
|
||||
.get(lag as usize)
|
||||
.or_else(|| state.history.back())
|
||||
.cloned()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn client_release_status(
|
||||
&self,
|
||||
client: ClientKind,
|
||||
@@ -238,12 +295,16 @@ impl CodexWireProfileState {
|
||||
arch: &str,
|
||||
release: ClientRelease,
|
||||
) {
|
||||
self.releases
|
||||
let mut states = self
|
||||
.releases
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let state = states
|
||||
.entry((client, platform, arch.to_owned()))
|
||||
.or_default()
|
||||
.release = Some(release);
|
||||
.or_default();
|
||||
// 种子与缓存恢复同样构成观察:更晚恢复的版本会排在种子之前,形成启动历史。
|
||||
observe_release(&mut state.history, release.clone());
|
||||
state.release = Some(release);
|
||||
}
|
||||
|
||||
pub(crate) fn record_client_release(
|
||||
@@ -263,9 +324,11 @@ impl CodexWireProfileState {
|
||||
state.checked_at = Some(Utc::now());
|
||||
match result {
|
||||
Ok(release) => {
|
||||
state.release = Some(release);
|
||||
state.release = Some(release.clone());
|
||||
state.error = None;
|
||||
observe_release(&mut state.history, release);
|
||||
}
|
||||
// 失败观察只更新状态,不改动发布历史。
|
||||
Err(error) => state.error = Some(error),
|
||||
}
|
||||
}
|
||||
@@ -569,6 +632,20 @@ impl CodexDesktopReleaseService {
|
||||
) -> Self {
|
||||
if let Some(verified) = verified.as_ref() {
|
||||
profile.update_bundled_release(verified);
|
||||
// 缓存恢复与 CLI/平台渠道一样构成发布观察:只更新画像字段会让恢复版本
|
||||
// 在首次检查发现新版时被跳过,滞后档位落回启动种子。这里补写进
|
||||
// releases 历史,使「启动种子 → 恢复版本 → 新发现版本」形成完整序列。
|
||||
profile.seed_client_release(
|
||||
ClientKind::Desktop,
|
||||
ClientPlatform::Macos,
|
||||
"arm64",
|
||||
ClientRelease {
|
||||
codex_version: verified.codex_version.clone(),
|
||||
desktop_version: Some(verified.desktop_version.clone()),
|
||||
desktop_build: Some(verified.desktop_build.clone()),
|
||||
verified_at: Some(verified.verified_at),
|
||||
},
|
||||
);
|
||||
}
|
||||
Self {
|
||||
transport,
|
||||
|
||||
@@ -71,6 +71,9 @@ pub enum VersionMode {
|
||||
Fixed,
|
||||
}
|
||||
|
||||
/// `latest` 模式允许的最大滞后数量,同时限制发布历史的保留深度
|
||||
pub const MAX_VERSION_LAG: u32 = 10;
|
||||
|
||||
/// 空的可选字段表示使用对应预设参数;Key 覆盖始终是一份完整选择
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -78,6 +81,8 @@ pub struct ClientProfileSelection {
|
||||
pub client: ClientKind,
|
||||
pub platform: ClientPlatform,
|
||||
pub version_mode: VersionMode,
|
||||
/// `latest` 模式可选:跟随官方发布但滞后 N 个已观察版本。
|
||||
pub version_lag: Option<u32>,
|
||||
pub cli_entry: Option<CliEntry>,
|
||||
pub originator: Option<String>,
|
||||
pub os_type: Option<String>,
|
||||
@@ -95,6 +100,7 @@ impl Default for ClientProfileSelection {
|
||||
client: ClientKind::Desktop,
|
||||
platform: ClientPlatform::Macos,
|
||||
version_mode: VersionMode::Latest,
|
||||
version_lag: None,
|
||||
cli_entry: None,
|
||||
originator: None,
|
||||
os_type: None,
|
||||
@@ -163,7 +169,19 @@ impl ClientProfileSelection {
|
||||
{
|
||||
return Err(ClientProfileError::Invalid);
|
||||
}
|
||||
// 滞后档位只对 latest 有意义;与 fixed 版本字段同样按混用拒绝。
|
||||
VersionMode::Latest => {
|
||||
if self
|
||||
.version_lag
|
||||
.is_some_and(|lag| !(1..=MAX_VERSION_LAG).contains(&lag))
|
||||
{
|
||||
return Err(ClientProfileError::Invalid);
|
||||
}
|
||||
}
|
||||
VersionMode::Fixed => {
|
||||
if self.version_lag.is_some() {
|
||||
return Err(ClientProfileError::Invalid);
|
||||
}
|
||||
let version = self
|
||||
.codex_version
|
||||
.as_deref()
|
||||
@@ -188,7 +206,6 @@ impl ClientProfileSelection {
|
||||
}
|
||||
}
|
||||
}
|
||||
VersionMode::Latest => {}
|
||||
}
|
||||
if self.client == ClientKind::Cli
|
||||
&& (self.desktop_version.is_some() || self.desktop_build.is_some())
|
||||
@@ -204,9 +221,23 @@ impl ClientProfileSelection {
|
||||
) -> Result<CodexWireProfile, ClientProfileError> {
|
||||
self.validate()?;
|
||||
let release = match self.version_mode {
|
||||
VersionMode::Latest => state
|
||||
.client_release(self.client, self.platform, self.architecture())
|
||||
.ok_or(ClientProfileError::ReleaseUnavailable)?,
|
||||
VersionMode::Latest => {
|
||||
let latest = state
|
||||
.client_release(self.client, self.platform, self.architecture())
|
||||
.ok_or(ClientProfileError::ReleaseUnavailable)?;
|
||||
// 滞后档位取已观察发布序列的第 N 项;序列不足时回退最旧版本,
|
||||
// 滞后配置不能在观察积累期变成请求失败。
|
||||
self.version_lag
|
||||
.and_then(|lag| {
|
||||
state.lagged_client_release(
|
||||
self.client,
|
||||
self.platform,
|
||||
self.architecture(),
|
||||
lag,
|
||||
)
|
||||
})
|
||||
.unwrap_or(latest)
|
||||
}
|
||||
VersionMode::Fixed => ClientRelease {
|
||||
codex_version: self
|
||||
.codex_version
|
||||
@@ -267,6 +298,15 @@ pub struct ClientRelease {
|
||||
pub verified_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
impl ClientRelease {
|
||||
/// 版本同一性判断,忽略核验时间;发布历史按版本元组判重。
|
||||
pub(super) fn same_identity(&self, other: &ClientRelease) -> bool {
|
||||
self.codex_version == other.codex_version
|
||||
&& self.desktop_version == other.desktop_version
|
||||
&& self.desktop_build == other.desktop_build
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexWireProfileState {
|
||||
pub fn preview_selection(
|
||||
&self,
|
||||
@@ -297,6 +337,7 @@ impl CodexWireProfileState {
|
||||
"desktopBuild": (profile.client_kind == ClientKind::Desktop).then_some(&profile.desktop_build),
|
||||
"userAgent": profile.user_agent(),
|
||||
"versionSource": if selection.version_mode == VersionMode::Fixed { "custom" } else { "official" },
|
||||
"versionLag": selection.version_lag,
|
||||
"verifiedAt": (profile.verified_at != DateTime::UNIX_EPOCH).then_some(profile.verified_at),
|
||||
"checkedAt": status.0,
|
||||
"error": status.1,
|
||||
@@ -325,7 +366,7 @@ impl CodexWireProfileState {
|
||||
}));
|
||||
}
|
||||
}
|
||||
object(&json!({ "presets": presets }))
|
||||
object(&json!({ "presets": presets, "maxVersionLag": MAX_VERSION_LAG }))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
//! 首查随机启动延迟与失败重排的确定性边界(通过注入延迟采样器驱动)。
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use provider_openai::credential::CodexInitialSyncDelays;
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn usage_ok() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"rate_limit": {"allowed": true, "primary_window": {"used_percent": 10}}
|
||||
})
|
||||
}
|
||||
|
||||
fn zero_delays() -> CodexInitialSyncDelays {
|
||||
Arc::new(|| Duration::ZERO)
|
||||
}
|
||||
|
||||
fn fixed_delays(delay: Duration) -> CodexInitialSyncDelays {
|
||||
Arc::new(move || delay)
|
||||
}
|
||||
|
||||
/// 按给定顺序逐个返回脚本化延迟,耗尽后退化为零延迟。
|
||||
fn scripted_delays(delays: Vec<Duration>) -> CodexInitialSyncDelays {
|
||||
let queue = Arc::new(Mutex::new(VecDeque::from(delays)));
|
||||
Arc::new(move || {
|
||||
queue
|
||||
.lock()
|
||||
.expect("delay queue")
|
||||
.pop_front()
|
||||
.unwrap_or(Duration::ZERO)
|
||||
})
|
||||
}
|
||||
|
||||
async fn mount_usage(server: &MockServer, status: u16) {
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(status).set_body_json(usage_ok()))
|
||||
.mount(server)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn usage_requests(server: &MockServer) -> usize {
|
||||
server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("requests")
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/api/codex/usage")
|
||||
.count()
|
||||
}
|
||||
|
||||
fn local_service(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
server: &MockServer,
|
||||
delays: CodexInitialSyncDelays,
|
||||
) -> CodexCredentialQuotaService {
|
||||
quota_service_with_initial_delays(
|
||||
store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
delays,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_initial_delay_keeps_immediate_first_observation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_initial_zero").await;
|
||||
let server = MockServer::start().await;
|
||||
mount_usage(&server, 200).await;
|
||||
let service = local_service(&store, &server, zero_delays());
|
||||
|
||||
assert_eq!(
|
||||
service.synchronize().await.expect("first cycle").updated,
|
||||
1,
|
||||
"零延迟必须保持首轮即查的旧行为"
|
||||
);
|
||||
// 已产生快照后退出首查路径,正常账号也不满足周期复核条件。
|
||||
service.synchronize().await.expect("second cycle");
|
||||
assert_eq!(usage_requests(&server).await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deferred_initial_delay_holds_back_first_observation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_initial_deferred").await;
|
||||
let server = MockServer::start().await;
|
||||
mount_usage(&server, 200).await;
|
||||
let service = local_service(&store, &server, fixed_delays(Duration::from_secs(3600)));
|
||||
|
||||
for cycle in 0..2 {
|
||||
assert_eq!(
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("deferred cycle {cycle} failed"))
|
||||
.updated,
|
||||
0,
|
||||
"随机延迟未到期前不得发起首查"
|
||||
);
|
||||
}
|
||||
assert_eq!(usage_requests(&server).await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn per_account_delays_are_drawn_independently() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_initial_first").await;
|
||||
create_account(&store, "acct_initial_second").await;
|
||||
let server = MockServer::start().await;
|
||||
mount_usage(&server, 200).await;
|
||||
// 首个候选账号零延迟立即观察,次个账号延迟 1 小时。
|
||||
let service = local_service(
|
||||
&store,
|
||||
&server,
|
||||
scripted_delays(vec![Duration::ZERO, Duration::from_secs(3600)]),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
service.synchronize().await.expect("first cycle").updated,
|
||||
1,
|
||||
"只有延迟到期的账号参与本轮首查"
|
||||
);
|
||||
service.synchronize().await.expect("second cycle");
|
||||
assert_eq!(usage_requests(&server).await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn initial_sync_batch_cap_bounds_first_round() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
for index in 0..150 {
|
||||
create_account(&store, &format!("acct_initial_cap_{index:03}")).await;
|
||||
}
|
||||
let server = MockServer::start().await;
|
||||
mount_usage(&server, 200).await;
|
||||
let service = local_service(&store, &server, zero_delays());
|
||||
|
||||
assert_eq!(
|
||||
service.synchronize().await.expect("first round").updated,
|
||||
100,
|
||||
"单轮首查仍受批量上限约束"
|
||||
);
|
||||
assert_eq!(
|
||||
service.synchronize().await.expect("second round").updated,
|
||||
50,
|
||||
"剩余候选在后续轮次补齐"
|
||||
);
|
||||
assert_eq!(usage_requests(&server).await, 150);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_initial_attempt_is_rearmed_with_fresh_delay() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_initial_retry").await;
|
||||
let server = MockServer::start().await;
|
||||
// 429 属非 5xx 的瞬时上游拒绝,不触发 usage 拉取自身的退避重试。
|
||||
mount_usage(&server, 429).await;
|
||||
// 首查分配零延迟(立即观察),失败后重排 1 小时。
|
||||
let service = local_service(
|
||||
&store,
|
||||
&server,
|
||||
scripted_delays(vec![Duration::ZERO, Duration::from_secs(3600)]),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("failing first attempt")
|
||||
.transient,
|
||||
1
|
||||
);
|
||||
assert_eq!(usage_requests(&server).await, 1);
|
||||
service.synchronize().await.expect("rearmed cycle");
|
||||
assert_eq!(
|
||||
usage_requests(&server).await,
|
||||
1,
|
||||
"失败重排的随机延迟未到期,不得回到固定 30s 节奏重复请求"
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
//! OpenAI 额度事实边界与展示快照回归
|
||||
|
||||
mod capacity_freeze;
|
||||
mod initial_sync;
|
||||
mod recovery;
|
||||
mod refresh_timing;
|
||||
mod scheduling;
|
||||
@@ -10,7 +11,7 @@ mod subscription;
|
||||
mod warmup;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use gateway_core::account::{
|
||||
@@ -64,6 +65,17 @@ pub(super) fn quota_service_with_base_url(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
http: reqwest::Client,
|
||||
base_url: String,
|
||||
) -> CodexCredentialQuotaService {
|
||||
// 首查随机启动延迟只在 initial_sync 专项测试中注入非零值;
|
||||
// 其余测试固定为零,保持「首轮即查」的确定性旧行为。
|
||||
quota_service_with_initial_delays(store, http, base_url, Arc::new(|| Duration::ZERO))
|
||||
}
|
||||
|
||||
pub(super) fn quota_service_with_initial_delays(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
http: reqwest::Client,
|
||||
base_url: String,
|
||||
delays: provider_openai::credential::CodexInitialSyncDelays,
|
||||
) -> CodexCredentialQuotaService {
|
||||
CodexCredentialQuotaService::new(
|
||||
store.repository(),
|
||||
@@ -74,6 +86,7 @@ pub(super) fn quota_service_with_base_url(
|
||||
Arc::new(crate::support::TestLeaseCoordinator::default()),
|
||||
crate::support::runtime_policy(),
|
||||
)
|
||||
.with_initial_sync_delays(delays)
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, account_id: &str) {
|
||||
|
||||
@@ -387,6 +387,50 @@ async fn scheduled_refresh_uses_the_current_margin_without_persisting_a_normal_s
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduled_refresh_staggers_accounts_sharing_the_same_expiry() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let policy = MutableRuntimePolicy::new(Duration::from_secs(5 * 60));
|
||||
let refresher = SingleUseRefresher::new();
|
||||
let service = refresh_service(&store, Arc::clone(&refresher), policy);
|
||||
// margin=300s:寻找一个错峰偏移覆盖到 exp、一个未覆盖的账号,两者共享同一到期时刻。
|
||||
let policy_value = refresh_policy(Duration::from_secs(5 * 60));
|
||||
let mut covered = None;
|
||||
let mut pending = None;
|
||||
for index in 0..64 {
|
||||
let candidate =
|
||||
ProviderAccountId::new(format!("acct_stagger_{index}")).expect("valid account");
|
||||
let stagger = policy_value.refresh_stagger(&candidate).as_secs();
|
||||
if stagger >= 150 && covered.is_none() {
|
||||
covered = Some(candidate.clone());
|
||||
}
|
||||
if stagger < 60 && pending.is_none() {
|
||||
pending = Some(candidate);
|
||||
}
|
||||
}
|
||||
let covered = covered.expect("an account covered by its stagger");
|
||||
let pending = pending.expect("an account not yet covered by its stagger");
|
||||
// exp = now+400s:covered 的有效提前量 ≥ 450s(到期);pending 的 ≤ 360s,
|
||||
// 即使种子与扫描之间存在秒级耗时也不会提前触发。
|
||||
let expires_at = SystemTime::now()
|
||||
.checked_add(Duration::from_secs(400))
|
||||
.expect("test expiry");
|
||||
seed_refreshable_account(&store, covered.as_str(), expires_at, None).await;
|
||||
seed_refreshable_account(&store, pending.as_str(), expires_at, None).await;
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("refresh cycle");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Refreshed { account_id, .. }] if account_id == covered.as_str()
|
||||
));
|
||||
assert_eq!(refresher.calls(), 1);
|
||||
// 未到错峰窗口的账号本轮零接触:不刷新、也不落退避计划。
|
||||
let pending_account = store.account(pending.as_str()).expect("seeded account");
|
||||
assert!(pending_account.next_refresh_at().is_none());
|
||||
assert_eq!(pending_account.credential_state(), CredentialState::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduled_refresh_rotates_tokens_while_quota_is_exhausted() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
//! usage 观测抖动映射的确定性边界。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use provider_openai::{catalog_refresh_jitter, quota_failure_refresh_delay, uniform_delay};
|
||||
|
||||
#[test]
|
||||
fn uniform_delay_maps_samples_into_half_open_range() {
|
||||
let max = Duration::from_secs(600);
|
||||
assert_eq!(
|
||||
uniform_delay(None, max),
|
||||
Duration::ZERO,
|
||||
"随机源退化时不延迟"
|
||||
);
|
||||
assert_eq!(uniform_delay(Some(0), max), Duration::ZERO);
|
||||
assert_eq!(
|
||||
uniform_delay(Some(u64::MAX), Duration::ZERO),
|
||||
Duration::ZERO,
|
||||
"零上限不产生延迟"
|
||||
);
|
||||
// 样本按 [0, max) 取模映射,边界样本必须落在区间内。
|
||||
for sample in [1, 1_000, u64::MAX / 3, u64::MAX / 2, u64::MAX - 1, u64::MAX] {
|
||||
let delay = uniform_delay(Some(sample), max);
|
||||
assert!(delay < max, "sample {sample} 超出上限: {delay:?}");
|
||||
}
|
||||
assert_eq!(
|
||||
uniform_delay(Some(u64::MAX), max).as_nanos(),
|
||||
u128::from(u64::MAX) % max.as_nanos()
|
||||
);
|
||||
// 固定样本集合应产生分散的延迟,覆盖区间的不同位置。
|
||||
let delays: Vec<_> = (0..8_u64)
|
||||
.map(|index| uniform_delay(Some(index * (u64::MAX / 8)), max))
|
||||
.collect();
|
||||
assert!(
|
||||
delays.iter().any(|delay| *delay < max / 2),
|
||||
"样本集合应覆盖区间下半段"
|
||||
);
|
||||
assert!(
|
||||
delays.iter().any(|delay| *delay >= max / 2),
|
||||
"样本集合应覆盖区间上半段"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_failure_refresh_delay_stays_within_settlement_window() {
|
||||
assert_eq!(
|
||||
quota_failure_refresh_delay(None),
|
||||
Duration::from_secs(2),
|
||||
"随机源退化时回到固定 2s 基准"
|
||||
);
|
||||
let delays: Vec<_> = [1, 1_000, u64::MAX / 3, u64::MAX / 2, u64::MAX - 1, u64::MAX]
|
||||
.into_iter()
|
||||
.map(|sample| (sample, quota_failure_refresh_delay(Some(sample))))
|
||||
.collect();
|
||||
for (sample, delay) in &delays {
|
||||
assert!(delay >= &Duration::from_secs(1), "sample {sample} 低于 1s");
|
||||
assert!(delay < &Duration::from_secs(3), "sample {sample} 达到 3s");
|
||||
}
|
||||
assert_eq!(
|
||||
quota_failure_refresh_delay(Some(0)),
|
||||
Duration::from_secs(1),
|
||||
"零样本映射到下界,保持确定性"
|
||||
);
|
||||
// 跨度若误用「基准 − 下限」,该样本只会映射到 1.5s;
|
||||
// 按 [1s, 3s) 全区间均匀映射应落在上半区间的 2.5s。
|
||||
assert_eq!(
|
||||
quota_failure_refresh_delay(Some(1_500_000_000)),
|
||||
Duration::from_millis(2_500),
|
||||
"样本 1_500_000_000 应映射到 2.5s"
|
||||
);
|
||||
// 固定样本集合需同时覆盖上下半区间,防止区间被静默收窄。
|
||||
assert!(
|
||||
delays
|
||||
.iter()
|
||||
.any(|(_, delay)| *delay < Duration::from_secs(2)),
|
||||
"样本集合应覆盖区间下半段 [1s, 2s)"
|
||||
);
|
||||
assert!(
|
||||
delays
|
||||
.iter()
|
||||
.any(|(_, delay)| *delay >= Duration::from_secs(2)),
|
||||
"样本集合应覆盖区间上半段 [2s, 3s)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_refresh_jitter_is_single_sided_and_bounded() {
|
||||
let interval = Duration::from_secs(15 * 60);
|
||||
assert_eq!(
|
||||
catalog_refresh_jitter(None, interval),
|
||||
Duration::ZERO,
|
||||
"随机源退化时不抖动"
|
||||
);
|
||||
assert_eq!(catalog_refresh_jitter(Some(0), interval), Duration::ZERO);
|
||||
assert_eq!(
|
||||
catalog_refresh_jitter(Some(u64::MAX), Duration::ZERO),
|
||||
Duration::ZERO,
|
||||
"周期过小时不抖动"
|
||||
);
|
||||
for sample in [1, 1_000, u64::MAX / 3, u64::MAX / 2, u64::MAX - 1, u64::MAX] {
|
||||
let jitter = catalog_refresh_jitter(Some(sample), interval);
|
||||
assert!(
|
||||
jitter < interval / 5,
|
||||
"sample {sample} 抖动超过周期的 20%: {jitter:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
catalog_refresh_jitter(Some(u64::MAX), interval).as_nanos(),
|
||||
u128::from(u64::MAX) % (interval.as_nanos() / 5)
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
mod admin;
|
||||
mod config;
|
||||
mod credential;
|
||||
mod jitter;
|
||||
mod provider;
|
||||
mod support;
|
||||
mod transport;
|
||||
|
||||
@@ -116,6 +116,16 @@ async fn official_cli_cache_updates_entry_headers_together_and_preserves_frozen_
|
||||
);
|
||||
assert_eq!(frozen.codex_version, "0.155.0");
|
||||
|
||||
// 缓存恢复与启动种子构成观察历史,滞后档位据此钉住上一版。
|
||||
let lagged = ClientProfileSelection {
|
||||
version_lag: Some(1),
|
||||
..automatic.clone()
|
||||
};
|
||||
assert_eq!(
|
||||
lagged.resolve(&state).unwrap().user_agent(),
|
||||
"codex-tui/0.155.0 (Alpine Linux 3.24.1; x86_64) xterm-256color (codex-tui; 0.155.0)"
|
||||
);
|
||||
|
||||
let restarted = CodexWireProfileState::new(super::wire_profile());
|
||||
CliReleaseService::new(provider, restarted.clone(), cache)
|
||||
.unwrap()
|
||||
|
||||
@@ -330,6 +330,133 @@ async fn cache_failure_should_not_publish_a_partially_verified_profile() {
|
||||
assert!(service.status().snapshot().latest.is_none());
|
||||
}
|
||||
|
||||
/// 滞后档位按发布观察序列解析:启动种子、历次核验发布与失败观察共同构成历史。
|
||||
#[tokio::test]
|
||||
async fn lagged_profiles_track_observed_release_history() {
|
||||
use provider_openai::transport::profile::selection::ClientProfileSelection;
|
||||
|
||||
let state = CodexWireProfileState::new(wire_profile());
|
||||
let transport = Arc::new(ReleaseTransport::new(
|
||||
[
|
||||
Ok(release("26.810.41047", "6570")),
|
||||
Ok(release("26.820.41131", "6700")),
|
||||
Err(CodexDesktopReleaseError::InvalidDocument),
|
||||
],
|
||||
[Ok("0.148.0-alpha.9".to_owned()), Ok("0.149.0".to_owned())],
|
||||
));
|
||||
let service = service(
|
||||
state.clone(),
|
||||
transport,
|
||||
Arc::new(ArtifactProfiles::default()),
|
||||
None,
|
||||
);
|
||||
let selection = |lag: Option<u32>| ClientProfileSelection {
|
||||
version_lag: lag,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 启动种子来自初始画像;首次检查前档位回退到该版本而不是失败。
|
||||
assert_eq!(
|
||||
selection(Some(8)).resolve(&state).unwrap().codex_version,
|
||||
"0.147.0-alpha.6.6"
|
||||
);
|
||||
|
||||
service.refresh().await.expect("first release refresh");
|
||||
service.refresh().await.expect("second release refresh");
|
||||
|
||||
let latest = selection(None).resolve(&state).unwrap();
|
||||
assert_eq!(latest.codex_version, "0.149.0");
|
||||
assert_eq!(latest.desktop_version, "26.820.41131");
|
||||
// 滞后一档是上一次核验发布,滞后两档落到启动种子。
|
||||
let lagged = selection(Some(1)).resolve(&state).unwrap();
|
||||
assert_eq!(lagged.codex_version, "0.148.0-alpha.9");
|
||||
assert_eq!(lagged.desktop_version, "26.810.41047");
|
||||
assert_eq!(
|
||||
lagged.user_agent(),
|
||||
"Codex Desktop/0.148.0-alpha.9 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.810.41047)"
|
||||
);
|
||||
assert_eq!(
|
||||
selection(Some(2)).resolve(&state).unwrap().codex_version,
|
||||
"0.147.0-alpha.6.6"
|
||||
);
|
||||
// 观察序列不足档位深度时回退最旧已核验版本,不产生新的失败模式。
|
||||
assert_eq!(
|
||||
selection(Some(8)).resolve(&state).unwrap().codex_version,
|
||||
"0.147.0-alpha.6.6"
|
||||
);
|
||||
|
||||
// 失败观察只更新状态,发布历史保持不变。
|
||||
service
|
||||
.refresh()
|
||||
.await
|
||||
.expect_err("third release refresh should fail");
|
||||
assert_eq!(
|
||||
selection(Some(1)).resolve(&state).unwrap().codex_version,
|
||||
"0.148.0-alpha.9"
|
||||
);
|
||||
}
|
||||
|
||||
/// 重启后恢复缓存、首次检查即发现新版:恢复版本必须进入发布历史,
|
||||
/// 滞后一档不能跳回启动种子(PR #339 审查反馈的触发序列)。
|
||||
#[tokio::test]
|
||||
async fn restored_cache_release_should_stay_in_version_lag_history() {
|
||||
use provider_openai::transport::profile::selection::ClientProfileSelection;
|
||||
|
||||
let state = CodexWireProfileState::new(wire_profile());
|
||||
let restored = CodexBundledReleaseProfile {
|
||||
codex_version: "0.148.0-alpha.9".to_owned(),
|
||||
desktop_version: "26.810.41047".to_owned(),
|
||||
desktop_build: "6570".to_owned(),
|
||||
verified_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 20, 0, 0, 0)
|
||||
.single()
|
||||
.expect("valid fixture time"),
|
||||
};
|
||||
let transport = Arc::new(ReleaseTransport::new(
|
||||
[Ok(release("26.820.41131", "6700"))],
|
||||
[Ok("0.149.0".to_owned())],
|
||||
));
|
||||
let service = service(
|
||||
state.clone(),
|
||||
transport,
|
||||
Arc::new(ArtifactProfiles::default()),
|
||||
Some(restored),
|
||||
);
|
||||
let selection = |lag: Option<u32>| ClientProfileSelection {
|
||||
version_lag: lag,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// 恢复版本进入历史队首:最新档位取恢复后的画像,滞后一档解析到启动种子。
|
||||
assert_eq!(
|
||||
selection(None).resolve(&state).unwrap().codex_version,
|
||||
"0.148.0-alpha.9"
|
||||
);
|
||||
assert_eq!(
|
||||
selection(Some(1)).resolve(&state).unwrap().codex_version,
|
||||
"0.147.0-alpha.6.6"
|
||||
);
|
||||
|
||||
service.refresh().await.expect("first release refresh");
|
||||
|
||||
let latest = selection(None).resolve(&state).unwrap();
|
||||
assert_eq!(latest.codex_version, "0.149.0");
|
||||
assert_eq!(latest.desktop_version, "26.820.41131");
|
||||
// 滞后一档是缓存恢复的版本 B,而不是启动种子 A。
|
||||
let lagged = selection(Some(1)).resolve(&state).unwrap();
|
||||
assert_eq!(lagged.codex_version, "0.148.0-alpha.9");
|
||||
assert_eq!(lagged.desktop_version, "26.810.41047");
|
||||
assert_eq!(
|
||||
lagged.user_agent(),
|
||||
"Codex Desktop/0.148.0-alpha.9 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.810.41047)"
|
||||
);
|
||||
// 滞后两档落到启动种子,恢复观察与启动种子共同构成启动历史。
|
||||
assert_eq!(
|
||||
selection(Some(2)).resolve(&state).unwrap().codex_version,
|
||||
"0.147.0-alpha.6.6"
|
||||
);
|
||||
}
|
||||
|
||||
fn service(
|
||||
state: CodexWireProfileState,
|
||||
transport: Arc<ReleaseTransport>,
|
||||
|
||||
@@ -104,6 +104,11 @@ fn invalid_headers_and_mixed_version_modes_are_rejected() {
|
||||
json!({"versionMode":"fixed", "codexVersion":"latest"}),
|
||||
json!({"client":"desktop", "versionMode":"fixed", "codexVersion":"0.153.4", "desktopVersion":"1.preview", "desktopBuild":"8109"}),
|
||||
json!({"client":"desktop", "versionMode":"fixed", "codexVersion":"0.153.4", "desktopVersion":"26.901.51231", "desktopBuild":"build"}),
|
||||
json!({"versionLag":0}),
|
||||
json!({"versionLag":11}),
|
||||
json!({"versionLag":-1}),
|
||||
json!({"versionLag":1.5}),
|
||||
json!({"versionMode":"fixed", "codexVersion":"0.120.0", "versionLag":1}),
|
||||
json!({"extraHeader":"secret"}),
|
||||
] {
|
||||
let mut fields = baseline.as_object().unwrap().clone();
|
||||
@@ -112,6 +117,39 @@ fn invalid_headers_and_mixed_version_modes_are_rejected() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_lag_round_trips_and_falls_back_to_the_oldest_observed_release() {
|
||||
let state = CodexWireProfileState::new(super::wire_profile());
|
||||
let baseline = json!({"client":"cli", "platform":"linux", "versionMode":"latest"});
|
||||
let parse = |fields: serde_json::Map<String, serde_json::Value>| {
|
||||
ClientProfileSelection::parse(&OpaqueProviderData::new(fields)).unwrap()
|
||||
};
|
||||
let mut fields = baseline.as_object().unwrap().clone();
|
||||
fields.insert("versionLag".to_owned(), json!(null));
|
||||
assert_eq!(parse(fields).version_lag, None);
|
||||
|
||||
let mut fields = baseline.as_object().unwrap().clone();
|
||||
fields.insert("versionLag".to_owned(), json!(10));
|
||||
let lagged = parse(fields);
|
||||
assert_eq!(lagged.version_lag, Some(10));
|
||||
// 观察历史只有启动种子时,最大档位回退到最旧已核验版本而不是失败。
|
||||
let latest = parse(baseline.as_object().unwrap().clone())
|
||||
.resolve(&state)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
lagged.resolve(&state).unwrap().codex_version,
|
||||
latest.codex_version
|
||||
);
|
||||
|
||||
let preview = state
|
||||
.preview_selection(&lagged.document().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(preview.expose_to_provider()["versionLag"], 10);
|
||||
assert_eq!(preview.expose_to_provider()["versionSource"], "official");
|
||||
let options = state.selection_options().unwrap();
|
||||
assert_eq!(options.expose_to_provider()["maxVersionLag"], 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_entry_suffix_keeps_its_name_when_originator_is_overridden() {
|
||||
let state = CodexWireProfileState::new(super::wire_profile());
|
||||
|
||||
@@ -19,3 +19,4 @@ fcfae912ddefd54551ebf8e7e733442519d2862d2977134d548ebc1e75c1dc34 0017_unlimited
|
||||
40a4d4b16e6ec2c2c659980d416b089fc170acd2255aaaf751c4a04c3790a941 0019_plugin_resources.sql
|
||||
e8c1339166ac02fb2d4a077424ee5f64f6dc20eb4d3f8ec4fcb09c0f2a0e61f6 0020_plugin_trust.sql
|
||||
22faf2646168275150b1786c0f463f9ff2ca52463a272479e4fcc6f46c8cfa37 0021_runtime_scheduling.sql
|
||||
70f199c636253cd3020b87d606e45c2acb411763e18b95b5c020984ea69f3a3e 0022_narrow_refresh_margin.sql
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- OAuth 刷新提前量收窄到官方 codex 客户端基线(exp 前 5 分钟,manager.rs
|
||||
-- CHATGPT_ACCESS_TOKEN_REFRESH_WINDOW_MINUTES=5);仍为旧默认 3600 的已部署
|
||||
-- 实例一并迁移,管理员自定义值保持不变。
|
||||
alter table runtime_settings
|
||||
alter column refresh_margin_seconds set default 300;
|
||||
|
||||
update runtime_settings
|
||||
set refresh_margin_seconds = 300
|
||||
where id = 1
|
||||
and refresh_margin_seconds = 3600;
|
||||
+13
-2
@@ -1418,7 +1418,7 @@ models.dev 同步只导入可表示为当前文本 Token 计价的 OpenAI/xAI
|
||||
`openaiClientProfile` 与 `xaiClientProfile` 是同一映射的兼容字段:省略保留,不能显式提交 `null`;
|
||||
与通用字段同时提供时必须一致,否则整次更新拒绝。读取响应从映射派生这两个字段,不维护平行状态
|
||||
|
||||
直接读取 `openai` 或 `xai` 的选项与预览。OpenAI 选项返回六个 `presets`,xAI 返回 `defaults`;
|
||||
直接读取 `openai` 或 `xai` 的选项与预览。OpenAI 选项返回六个 `presets` 与 `maxVersionLag`,xAI 返回 `defaults`;
|
||||
响应均包含 `globalConfiguration`
|
||||
|
||||
### OpenAI 上游客户端身份
|
||||
@@ -1441,6 +1441,7 @@ UA 须为 1 至 4096 字节的单行可见 ASCII 文本,首尾不能含空白
|
||||
| `client` | 必填,`desktop` 或 `cli` |
|
||||
| `platform` | 必填,`macos`、`linux` 或 `windows` |
|
||||
| `versionMode` | 必填,`latest` 或 `fixed` |
|
||||
| `versionLag` | 仅 `latest` 可选的滞后数量,1~10 的整数;`fixed` 必须省略或为 `null` |
|
||||
| `cliEntry` | CLI 可选 `tui` 或 `exec`,省略或 `null` 保留 Core 默认身份;Desktop 不接受此字段 |
|
||||
| `originator`、`osType`、`osVersion`、`arch`、`terminal` | 可选自定义参数,非空、最多 128 字节;只接受可见 ASCII,不能包含括号、分号、反斜杠及首尾空白 |
|
||||
| `codexVersion` | `fixed` 必填的 Core SemVer;`latest` 必须省略或为 `null` |
|
||||
@@ -1454,6 +1455,16 @@ TUI 默认标识为 `codex-tui`,Exec 为 `codex_exec`,入口后缀使用同
|
||||
`originator` 覆盖只更改产品名前缀和配套头,后缀仍表示所选入口。省略 `osType` 使用平台名称;
|
||||
自定义运行环境在自动更新时保持不变。未指定 `cliEntry` 时使用 `codex_cli_rs` 默认值且不添加入口后缀
|
||||
|
||||
`terminal` 填写官方客户端的真实终端标记,如 `iTerm.app/3.5.0`、`vscode/1.99.0`、`Apple_Terminal`、
|
||||
`Ghostty`、`WindowsTerminal`、`tmux-256color`、`xterm-256color`。省略时使用 `unknown`:
|
||||
官方 Desktop 由 GUI 启动、无终端环境变量,其 UA 中的终端标记本就是 `unknown`,
|
||||
因此 Desktop 预设保持默认即可;CLI 预设建议配置真实值,多 Key 可通过各自的画像覆盖差异化
|
||||
|
||||
`versionLag` 让 `latest` 模式跟随官方发布但滞后 N 个已观察版本采用,省略时采用最新版本。
|
||||
档位基于网关观察到的发布序列解析:启动种子与历次 24 小时检查的核验版本按时间倒序构成历史,
|
||||
取第 N 项;序列不足时回退最旧已核验版本,不会因此解析失败。历史仅保留在进程内,
|
||||
重启后从启动种子与制品缓存重新积累
|
||||
|
||||
六套预设均支持自动更新:macOS Desktop 支持 arm64,Windows/Linux Desktop 及三套 CLI 支持 arm64、x86_64。
|
||||
预设接口的 `automaticAvailable`、`reason` 表示当前组合的可用性;自定义架构可能使自动解析不可用。
|
||||
每 24 小时后台检查官方稳定发布,失败保留同组合上次有效版本;固定值不受后台更新影响。
|
||||
@@ -1461,7 +1472,7 @@ Desktop 的应用版本、Core 和构建号来自同一平台、架构的官方
|
||||
Windows/Linux 通过 ETag 检查更新,未变化时复用已核验版本;CLI 依据官方 npm 稳定标签和对应平台依赖
|
||||
|
||||
预览返回 `configuration`、`source`(`global` / `override`)、`userAgent`、解析后的环境和版本字段,
|
||||
以及 `versionSource`(`official` / `custom`)、`verifiedAt`、`checkedAt`、`error`。
|
||||
以及 `versionSource`(`official` / `custom`)、`versionLag`、`verifiedAt`、`checkedAt`、`error`。
|
||||
自定义预览中的 `recognized` 表示是否识别出配套请求头。
|
||||
`verifiedAt` 只表示版本资料核验,不能代表自定义运行环境或 TLS 已核验;固定版本返回 `null`。
|
||||
完整自定义配置不携带官方制品核验时间。
|
||||
|
||||
@@ -369,6 +369,9 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别 Codex Desktop/
|
||||
重试与提交边界。具体错误合同见 [数据面接口](api.md#3-openai-数据面与模型目录)
|
||||
- Provider 可将明确容量拒绝标记为有界同账号退避,Core 在既有安全重放边界内执行,按账号维护请求内
|
||||
预算,耗尽后复用普通换号路径。该退避消耗总路由预算,与 WS 传输恢复、OAuth 刷新及账号额度冷却分开
|
||||
- 换号有独立的请求内预算:选中账号与上一 attempt 不同的路由 attempt 至多 3 次,跨 Provider 候选
|
||||
推进与 continuation 排除重放同样受限;预算耗尽后不再换号,以最后一个原始上游错误终态。同账号重试
|
||||
与总路由预算不受影响
|
||||
- Provider 显式标记的终止拒绝禁止自动重试与传输回退,优先于未发送状态和安全重放证明
|
||||
- 跨 Provider 只在账号范围和能力都允许,且请求尚未到达上游或已被证明可安全重放时发生
|
||||
- 可恢复观测写入失败不能替换已经确定的客户端协议结果
|
||||
@@ -717,6 +720,9 @@ credential 与 quota 是两组独立事实:credential refresh 不等于 quota
|
||||
- OpenAI 支持 OAuth、AT/RT、PAT 和上游 API Key。OAuth 身份来自官方 JWT claims,PAT 经官方身份接口验证,
|
||||
不信任导入文档顶层身份字段。RT-only 导入先换取 AT;AT-only、PAT 与 API Key 不参加 OAuth 自动续期。
|
||||
输入形态和适用操作见 [账号能力与导入](api.md#账号能力导入与-oauth)
|
||||
- OAuth 自动续期按运行时设置的提前量触发,默认 300 秒对齐官方客户端 exp 前 5 分钟的刷新窗口。
|
||||
每个账号的有效提前量在 [margin, 2×margin] 内由账号 ID 派生稳定错峰偏移,减少同一时刻
|
||||
到期账号的集中刷新;恢复窗口内的强制刷新不受偏移影响
|
||||
- xAI 使用 OAuth session;API Key 不是受支持的账号 credential。刷新额度时同步查询官方实时订阅,
|
||||
只把套餐事实写入现有 quota JSON。明确无付费订阅的个人账号显示 Free;查询失败、缺失字段或
|
||||
团队身份不推断为 Free,订阅查询失败不影响额度观测
|
||||
|
||||
@@ -9,6 +9,7 @@ export interface PresetClientProfileSelection {
|
||||
client: 'desktop' | 'cli'
|
||||
platform: 'macos' | 'linux' | 'windows'
|
||||
versionMode: 'latest' | 'fixed'
|
||||
versionLag?: number | null
|
||||
cliEntry?: 'tui' | 'exec' | null
|
||||
osType?: string | null
|
||||
originator: string | null
|
||||
@@ -32,6 +33,7 @@ export type ClientProfileSelection = PresetClientProfileSelection | CustomClient
|
||||
export interface ClientProfileOptions {
|
||||
presets: ClientProfilePreset[]
|
||||
globalConfiguration: ClientProfileSelection
|
||||
maxVersionLag: number
|
||||
}
|
||||
|
||||
export interface ClientProfilePreview {
|
||||
@@ -47,6 +49,7 @@ export interface ClientProfilePreview {
|
||||
desktopBuild: string | null
|
||||
userAgent: string
|
||||
versionSource: 'official' | 'custom'
|
||||
versionLag: number | null
|
||||
recognized?: boolean
|
||||
verifiedAt: string | null
|
||||
verifiedAtDisplay: string | null
|
||||
|
||||
@@ -121,6 +121,7 @@ onMounted(() => load())
|
||||
:previewing="previewing"
|
||||
:error="previewError"
|
||||
:disabled="disabled || loading"
|
||||
:max-version-lag="options?.maxVersionLag"
|
||||
:aria-busy="loading || undefined"
|
||||
@update:model-value="model = $event"
|
||||
/>
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
<script setup lang="ts">
|
||||
import type { ClientProfilePreset, ClientProfilePreview, ClientProfileSelection, CustomClientProfileSelection, PresetClientProfileSelection } from '@/api/modules/client-profiles'
|
||||
import { BaseFormItem, BaseSelect, BaseTextarea } from '@codex-proxy/ui'
|
||||
import { BaseFormItem, BaseInput, BaseSelect, BaseTextarea } from '@codex-proxy/ui'
|
||||
import { computed, shallowRef } from 'vue'
|
||||
|
||||
const props = defineProps<{
|
||||
const props = withDefaults(defineProps<{
|
||||
presets: ClientProfilePreset[]
|
||||
preview?: ClientProfilePreview
|
||||
previewing: boolean
|
||||
error: string
|
||||
disabled: boolean
|
||||
}>()
|
||||
maxVersionLag?: number
|
||||
}>(), {
|
||||
maxVersionLag: 10,
|
||||
})
|
||||
const model = defineModel<ClientProfileSelection | null>({ required: true })
|
||||
const presetDraft = shallowRef<PresetClientProfileSelection>()
|
||||
const customDraft = shallowRef<CustomClientProfileSelection>()
|
||||
@@ -65,6 +68,26 @@ const versionMode = computed({
|
||||
}
|
||||
},
|
||||
})
|
||||
const versionLag = computed({
|
||||
get: () => preset.value?.versionLag == null ? '' : String(preset.value.versionLag),
|
||||
set: (value: string) => {
|
||||
if (preset.value)
|
||||
model.value = { ...preset.value, versionLag: value === '' ? null : Number(value) }
|
||||
},
|
||||
})
|
||||
const versionLagError = computed(() => {
|
||||
const value = preset.value?.versionLag
|
||||
return value != null && (!Number.isInteger(value) || value < 1 || value > props.maxVersionLag)
|
||||
? `请输入 1~${props.maxVersionLag} 的正整数`
|
||||
: ''
|
||||
})
|
||||
const terminal = computed({
|
||||
get: () => preset.value?.terminal ?? '',
|
||||
set: (value: string) => {
|
||||
if (preset.value)
|
||||
model.value = { ...preset.value, terminal: value === '' ? null : value }
|
||||
},
|
||||
})
|
||||
const userAgent = computed({
|
||||
get: () => model.value?.mode === 'custom' ? model.value.userAgent : props.preview?.userAgent ?? '',
|
||||
set: (value: string) => {
|
||||
@@ -115,6 +138,33 @@ const userAgent = computed({
|
||||
<p v-if="!custom && currentPreset?.reason" class="m-0 text-cp-sm text-cp-text-secondary">
|
||||
{{ currentPreset.reason }}
|
||||
</p>
|
||||
<div v-if="!custom" class="grid gap-4 sm:grid-cols-2">
|
||||
<BaseFormItem label="版本滞后" :error="versionLagError">
|
||||
<BaseInput
|
||||
v-model="versionLag"
|
||||
aria-label="版本滞后"
|
||||
type="number"
|
||||
inputmode="numeric"
|
||||
min="1"
|
||||
:max="maxVersionLag"
|
||||
step="1"
|
||||
:placeholder="`滞后 1~${maxVersionLag} 个版本,留空不滞后`"
|
||||
:disabled="disabled"
|
||||
/>
|
||||
</BaseFormItem>
|
||||
<BaseFormItem label="终端标识">
|
||||
<BaseInput
|
||||
v-model="terminal"
|
||||
:disabled="disabled"
|
||||
aria-label="终端标识"
|
||||
placeholder="留空使用 unknown"
|
||||
maxlength="128"
|
||||
spellcheck="false"
|
||||
autocomplete="off"
|
||||
class="font-mono"
|
||||
/>
|
||||
</BaseFormItem>
|
||||
</div>
|
||||
<template v-if="custom">
|
||||
<BaseTextarea
|
||||
v-model="userAgent"
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { ClientProfilePreview } from '@/api/modules/client-profiles'
|
||||
import { BaseSkeleton } from '@codex-proxy/ui'
|
||||
|
||||
defineProps<{
|
||||
preview?: Pick<ClientProfilePreview, 'userAgent' | 'versionSource' | 'checkedAtDisplay' | 'error'>
|
||||
preview?: Pick<ClientProfilePreview, 'userAgent' | 'versionSource' | 'checkedAtDisplay' | 'error'> & { versionLag?: number | null }
|
||||
previewing: boolean
|
||||
needsVersionInput: boolean
|
||||
error: string
|
||||
@@ -36,6 +36,9 @@ defineProps<{
|
||||
<p class="m-0 text-cp-xs text-cp-text-tertiary">
|
||||
{{ preview.versionSource === 'custom' ? '固定身份' : '自动更新' }}
|
||||
<template v-if="preview.versionSource === 'official'">
|
||||
<template v-if="preview.versionLag">
|
||||
· 滞后 {{ preview.versionLag }} 版
|
||||
</template>
|
||||
· {{ preview.checkedAtDisplay ? `检查于 ${preview.checkedAtDisplay}` : '待检查' }}
|
||||
</template>
|
||||
</p>
|
||||
|
||||
Reference in New Issue
Block a user