diff --git a/backend/crates/gateway-admin/src/use_case/accounts.rs b/backend/crates/gateway-admin/src/use_case/accounts.rs index 2d0620369..f49adcd86 100644 --- a/backend/crates/gateway-admin/src/use_case/accounts.rs +++ b/backend/crates/gateway-admin/src/use_case/accounts.rs @@ -269,6 +269,11 @@ impl DefaultAccountsService { }) .await .map_err(|error| map_provider_error(error, "provider quota"))?; + let stored = if refresh_quota { + self.load_account(account_id).await? + } else { + stored + }; self.attach_quota_local_usage( std::slice::from_ref(&stored), std::slice::from_mut(&mut quota), diff --git a/backend/crates/gateway-admin/tests/use_case/accounts.rs b/backend/crates/gateway-admin/tests/use_case/accounts.rs index fbd52a81a..6f5f35e9c 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts.rs @@ -72,6 +72,7 @@ pub(super) struct FakeProviderAdmin { import_account_ids: Mutex>, quota_requests: Mutex>, quota: Mutex, + quota_refresh_account: Mutex, AccountRecord)>>, current_credential_revision: Mutex, reset_credit_commands: Mutex>, } @@ -89,6 +90,7 @@ impl FakeProviderAdmin { import_account_ids: Mutex::new(vec!["acct_prepared".to_owned()]), quota_requests: Mutex::new(Vec::new()), quota: Mutex::new(empty_quota()), + quota_refresh_account: Mutex::new(None), current_credential_revision: Mutex::new(revision(1)), reset_credit_commands: Mutex::new(Vec::new()), }) @@ -346,6 +348,14 @@ impl ProviderAdmin for FakeProviderAdmin { ) -> Result { if request.refresh { self.record("provider.quota"); + if let Some((store, account)) = self + .quota_refresh_account + .lock() + .expect("quota refresh account") + .take() + { + store.set_accounts(vec![account]); + } } self.quota_requests .lock() @@ -1098,6 +1108,47 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() { assert_eq!(store.audit_requests(), ["refresh-request"]); } +#[tokio::test] +async fn accounts_quota_refresh_should_return_the_updated_account_status() { + let events = events(); + let provider = FakeProviderAdmin::new("openai", events.clone()); + let mut account = account_record("openai"); + account.quota = QuotaState::exhausted( + QuotaEvidence::UsageLimitReached, + std::time::SystemTime::now(), + None, + ); + let store = FakeAccountStore::with_account(account.clone(), events.clone()); + account.quota = QuotaState::allowed(std::time::SystemTime::now()); + *provider + .quota_refresh_account + .lock() + .expect("quota refresh account") = Some((store.clone(), account)); + + let result = accounts_service(provider, store) + .await + .accounts() + .quota( + &ProviderAccountId::new("acct_test").expect("account ID"), + true, + ) + .await + .expect("refresh quota"); + + assert_eq!( + result.account.quota.access(), + gateway_core::account::QuotaAccessState::Allowed + ); + assert_eq!( + result.projection.status, + gateway_admin::model::accounts::AccountStatus::Normal + ); + assert_eq!( + recorded(&events), + ["store.load_account", "provider.quota", "store.load_account"] + ); +} + #[tokio::test] async fn accounts_recover_should_commit_facts_then_return_normal_account() { let events = events(); diff --git a/backend/crates/providers/openai/src/admin.rs b/backend/crates/providers/openai/src/admin.rs index 75ec75619..92b209e73 100644 --- a/backend/crates/providers/openai/src/admin.rs +++ b/backend/crates/providers/openai/src/admin.rs @@ -955,13 +955,7 @@ fn codex_quota_window_label( match kind { CodexQuotaWindowKind::Monthly => "月额度".to_owned(), CodexQuotaWindowKind::Weekly => "周额度".to_owned(), - CodexQuotaWindowKind::ShortTerm => { - if window_seconds.is_some_and(|seconds| seconds > 86_400) { - "周额度".to_owned() - } else { - "5小时额度".to_owned() - } - } + CodexQuotaWindowKind::ShortTerm => "5小时额度".to_owned(), CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds, role), } } diff --git a/backend/crates/providers/openai/src/credential/quota/mod.rs b/backend/crates/providers/openai/src/credential/quota/mod.rs index 0e936a6bd..512e45d2a 100644 --- a/backend/crates/providers/openai/src/credential/quota/mod.rs +++ b/backend/crates/providers/openai/src/credential/quota/mod.rs @@ -3,9 +3,11 @@ //! - [`document`]:上游 `/usage` 输入到多桶 map 的单向规范化。 //! - [`snapshot`]:快照/窗口解析、聚合、O(1) 滚动与调度信号。 //! - [`evidence`]:额度接口错误到凭据/额度事实的分类。 +//! - [`recovery`]:各额度窗口独立的恢复基准与访问结论。 mod document; pub(crate) mod evidence; +mod recovery; pub(crate) mod snapshot; pub use snapshot::{ @@ -46,6 +48,7 @@ use document::{ canonicalize_rate_limit_document, }; use evidence::{QuotaEndpointFailure, classify_quota_endpoint_failure}; +use recovery::{RECOVERY_FIELD, reconcile_refresh}; use snapshot::{ parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation, scheduling_signals_from_snapshot, @@ -56,7 +59,6 @@ pub(crate) const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60); const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5); const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(30 * 60); const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60); -const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0; /// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限。 const INITIAL_QUOTA_SYNC_BATCH: usize = 100; // 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动。 @@ -775,13 +777,14 @@ impl CodexCredentialQuotaService { observed_at: SystemTime, summary: &mut CodexQuotaSyncSummary, ) -> Result<(), CodexCredentialQuotaError> { - let object = normalize_quota_window_placeholders( + let mut object = normalize_quota_window_placeholders( value .as_object() .cloned() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?, ); - let snapshot = parse_account_quota_snapshot( + object.remove(RECOVERY_FIELD); + let mut snapshot = parse_account_quota_snapshot( account.id().clone(), account.revision(), observed_at, @@ -792,33 +795,12 @@ impl CodexCredentialQuotaService { } else { None }; - if account.quota().is_exhausted() - && !quota_refresh_allows_recovery(account, &snapshot, previous.as_ref()) - { - let outcome = self - .store - .touch_quota_observation(QuotaObservationTouch { - account_id: account.id().clone(), - expected_revision: account.revision(), - observed_at, - }) - .await?; - if outcome == QuotaWriteOutcome::Conflict { - summary.stale += 1; - return Ok(()); - } - if let Some(previous) = previous { - self.scheduling - .observe(&previous.with_observed_at(observed_at)); - } - summary.exhausted += 1; - return Ok(()); - } - let state = if account.quota().is_exhausted() { - QuotaState::allowed(observed_at) - } else { - account.quota().merge_observation(snapshot.quota()) - }; + let state = reconcile_refresh( + account.quota(), + &mut snapshot, + previous.as_ref(), + &mut object, + )?; let snapshot = snapshot.with_quota_state(state); let outcome = self .store @@ -1086,13 +1068,14 @@ impl CodexCredentialQuotaService { }); } }; - let object = normalize_quota_window_placeholders( + let mut object = normalize_quota_window_placeholders( value .as_object() .cloned() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?, ); - let snapshot = parse_account_quota_snapshot( + object.remove(RECOVERY_FIELD); + let mut snapshot = parse_account_quota_snapshot( account.id().clone(), account.revision(), observed_at, @@ -1103,7 +1086,9 @@ impl CodexCredentialQuotaService { } else { None }; - if !quota_refresh_allows_recovery(&account, &snapshot, previous.as_ref()) { + if account.quota().is_exhausted() + && matches!(authority, QuotaRefreshAuthority::PreserveAccess) + { if self .store .touch_quota_observation(QuotaObservationTouch { @@ -1127,15 +1112,14 @@ impl CodexCredentialQuotaService { .with_observed_at(observed_at), }); } - let state = if account.quota().is_exhausted() { - QuotaState::allowed(observed_at) - } else { - match authority { - QuotaRefreshAuthority::ObserveAccess => { - account.quota().merge_observation(snapshot.quota()) - } - QuotaRefreshAuthority::PreserveAccess => account.quota(), - } + let state = match authority { + QuotaRefreshAuthority::ObserveAccess => reconcile_refresh( + account.quota(), + &mut snapshot, + previous.as_ref(), + &mut object, + )?, + QuotaRefreshAuthority::PreserveAccess => account.quota(), }; let snapshot = snapshot.with_quota_state(state); if self @@ -1315,44 +1299,6 @@ fn reset_credit_response_was_explicit_rejection( !status.is_success() && diagnostics.status_code == Some(status.as_u16()) } -fn quota_refresh_allows_recovery( - account: &ProviderAccount, - refreshed: &CodexAccountQuotaSnapshot, - previous: Option<&CodexAccountQuotaSnapshot>, -) -> bool { - if !account.quota().is_exhausted() { - return true; - } - if refreshed.quota().is_exhausted() { - return false; - } - let previous_reset_at = account.quota().reset_at().or_else(|| { - previous? - .windows() - .iter() - .find(|window| { - window.source() == "codex" && window.role() == CodexQuotaWindowRole::Primary - })? - .reset_at() - .map(SystemTime::from) - }); - let Some(previous_reset_at) = previous_reset_at else { - return false; - }; - let Some(primary) = refreshed.windows().iter().find(|window| { - window.source() == "codex" && window.role() == CodexQuotaWindowRole::Primary - }) else { - return false; - }; - let Some(refreshed_reset_at) = primary.reset_at().map(SystemTime::from) else { - return false; - }; - let Some(used_percent) = primary.used_percent() else { - return false; - }; - refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT -} - async fn fetch_usage_once( client: &CodexBackendClient, prepared: &PreparedCodexRuntimeCredential, diff --git a/backend/crates/providers/openai/src/credential/quota/recovery.rs b/backend/crates/providers/openai/src/credential/quota/recovery.rs new file mode 100644 index 000000000..9b98a17f4 --- /dev/null +++ b/backend/crates/providers/openai/src/credential/quota/recovery.rs @@ -0,0 +1,143 @@ +//! 将耗尽窗口的恢复基准与最新用量分开持久化,允许各窗口分批恢复。 + +use std::collections::BTreeMap; +use std::time::SystemTime; + +use chrono::{DateTime, Utc}; +use gateway_core::account::{QuotaEvidence, QuotaState}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use super::{CodexAccountQuotaSnapshot, CodexCredentialQuotaError, CodexQuotaWindow}; + +pub(super) const RECOVERY_FIELD: &str = "_quota_recovery"; +const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(super) struct QuotaRecovery { + // 绑定本次耗尽事实,避免真实推理成功后再次耗尽时复用上次的恢复进度。 + exhausted_at_micros: i64, + pending: BTreeMap>>, +} + +impl QuotaRecovery { + fn new(state: QuotaState, baseline: &CodexAccountQuotaSnapshot) -> Self { + let mut pending = BTreeMap::new(); + for window in baseline.windows().iter().filter(|w| w.is_account_wide()) { + if window.limit_reached() { + pending.insert(window.key().to_owned(), window.reset_at()); + } + } + // 旧版从 primary 派生的账号 reset 不能覆盖明确的周窗口耗尽事实。 + // 真实 usage_limit_reached 错误则可以补充尚未反映到百分比的耗尽窗口。 + if pending.is_empty() || state.evidence() == Some(QuotaEvidence::UsageLimitReached) { + for window in baseline.windows().iter().filter(|w| w.is_account_wide()) { + // 错误与 usage 的秒级 reset 可能相差一秒;只用它识别窗口, + // 后续推进比较始终使用该窗口自己的 reset。 + let matches_error_reset = window.reset_at().zip(state.reset_at()).is_some_and( + |(window_reset, error_reset)| { + (window_reset.timestamp() - DateTime::::from(error_reset).timestamp()) + .abs() + <= 1 + }, + ); + if matches_error_reset { + pending.insert(window.key().to_owned(), window.reset_at()); + } + } + } + // 上游未指出耗尽窗口时保留所有候选,不能任意把错误归给 primary。 + if pending.is_empty() { + pending.extend( + baseline + .windows() + .iter() + .filter(|w| w.is_account_wide()) + .map(|w| (w.key().to_owned(), w.reset_at())), + ); + } + Self { + exhausted_at_micros: exhaustion_time(state), + pending, + } + } + + fn observe(&mut self, refreshed: &CodexAccountQuotaSnapshot) -> bool { + let had_baseline = !self.pending.is_empty(); + for window in refreshed.windows().iter().filter(|w| w.is_account_wide()) { + if let Some(previous_reset) = self.pending.get_mut(window.key()) { + if window_reset_recovered(window, *previous_reset) { + self.pending.remove(window.key()); + } else if previous_reset.is_none() { + *previous_reset = window.reset_at(); + } + } else if window.limit_reached() { + self.pending + .insert(window.key().to_owned(), window.reset_at()); + } + } + had_baseline && self.pending.is_empty() + } +} + +fn exhaustion_time(state: QuotaState) -> i64 { + DateTime::::from(state.observed_at().unwrap_or(SystemTime::UNIX_EPOCH)).timestamp_micros() +} + +fn window_reset_recovered(window: &CodexQuotaWindow, previous: Option>) -> bool { + previous + .zip(window.reset_at()) + .is_some_and(|(previous, current)| current > previous) + && window + .used_percent() + .is_some_and(|used| used < RESET_RECOVERY_MAX_USED_PERCENT) +} + +/// 统一手动刷新和 worker 的恢复规则;上游 allowed 不参与已耗尽窗口的解除判断。 +pub(super) fn reconcile_refresh( + current: QuotaState, + refreshed: &mut CodexAccountQuotaSnapshot, + previous: Option<&CodexAccountQuotaSnapshot>, + document: &mut Map, +) -> Result { + document.remove(RECOVERY_FIELD); + let state = if current.is_exhausted() { + current + } else { + current.merge_observation(refreshed.quota()) + }; + if !state.is_exhausted() { + return Ok(state); + } + let mut recovery = previous + .and_then(|previous| previous.recovery.as_ref()) + .filter(|recovery| recovery.exhausted_at_micros == exhaustion_time(state)) + .cloned() + .unwrap_or_else(|| QuotaRecovery::new(state, previous.unwrap_or(refreshed))); + if recovery.pending.is_empty() { + recovery = QuotaRecovery::new(state, refreshed); + } + if current.is_exhausted() && recovery.observe(refreshed) { + return Ok(QuotaState::allowed(refreshed.observed_at())); + } + let reset_at = recovery + .pending + .values() + .filter_map(|reset| *reset) + .min() + .map(SystemTime::from) + .or(state.reset_at()); + document.insert( + RECOVERY_FIELD.to_owned(), + serde_json::to_value(&recovery) + .map_err(|_| CodexCredentialQuotaError::InvalidCredentialData)?, + ); + refreshed.recovery = Some(recovery); + Ok(QuotaState::exhausted( + state + .evidence() + .unwrap_or(QuotaEvidence::AccountLimitReached), + state.observed_at().unwrap_or(refreshed.observed_at()), + reset_at, + )) +} diff --git a/backend/crates/providers/openai/src/credential/quota/snapshot.rs b/backend/crates/providers/openai/src/credential/quota/snapshot.rs index df9d44071..dfccd3353 100644 --- a/backend/crates/providers/openai/src/credential/quota/snapshot.rs +++ b/backend/crates/providers/openai/src/credential/quota/snapshot.rs @@ -15,6 +15,7 @@ use serde_json::{Map, Value}; use super::document::{ DEFAULT_CODEX_LIMIT_ID, RATE_LIMITS_BY_LIMIT_ID, canonicalize_rate_limit_document, }; +use super::recovery::{QuotaRecovery, RECOVERY_FIELD}; use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -102,8 +103,8 @@ impl CodexQuotaWindow { /// 该窗口是否已触顶(`limit_reached` 或 `used_percent >= 100`)。 #[must_use] - pub const fn limit_reached(&self) -> bool { - self.limit_reached + pub fn limit_reached(&self) -> bool { + self.limit_reached || self.used_percent.is_some_and(|used| used >= 100.0) } } @@ -115,6 +116,7 @@ pub struct CodexAccountQuotaSnapshot { fact: CodexQuotaFact, quota: QuotaState, windows: Vec, + pub(super) recovery: Option, } impl CodexAccountQuotaSnapshot { @@ -300,11 +302,9 @@ pub(crate) fn parse_account_quota_snapshot( ) -> Result { let object = canonical_quota_object(usage)?; let fact = parse_codex_quota_object(&object)?; - let quota = authoritative_quota_state(&object, observed_at)?; let mut windows = Vec::new(); for limit in canonical_rate_limits(&object)? { parse_rate_limit_windows( - &limit.key, &limit.source, limit.account_wide, limit.limit_name.as_deref(), @@ -317,6 +317,12 @@ pub(crate) fn parse_account_quota_snapshot( .cmp("a_source_order(right.source())) .then_with(|| left.key.cmp(&right.key)) }); + let quota = authoritative_quota_state(&object, &windows, observed_at)?; + let recovery = object + .get(RECOVERY_FIELD) + .map(|value| serde_json::from_value(value.clone())) + .transpose() + .map_err(|_| CodexCredentialQuotaError::InvalidCredentialData)?; Ok(CodexAccountQuotaSnapshot { account_id, credential_revision, @@ -324,6 +330,7 @@ pub(crate) fn parse_account_quota_snapshot( fact, quota, windows, + recovery, }) } @@ -338,6 +345,7 @@ fn quota_source_order(source: &str) -> u8 { /// 默认 `codex` 桶是唯一可改变账号额度访问结论的 quota 响应字段。 fn authoritative_quota_state( usage: &Map, + windows: &[CodexQuotaWindow], observed_at: SystemTime, ) -> Result { let Some(rate_limit) = usage @@ -355,37 +363,28 @@ fn authoritative_quota_state( .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; let allowed = optional_bool(rate_limit, "allowed")?; let top_level_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false); - let primary_window = rate_limit - .get("primary_window") - .filter(|value| !value.is_null()) - .map(|value| { - value - .as_object() - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) + let window_reached = windows + .iter() + .any(|w| w.is_account_wide() && w.limit_reached); + let reset_at = windows + .iter() + .filter(|w| w.is_account_wide() && w.limit_reached()) + .filter_map(CodexQuotaWindow::reset_at) + .min() + .or_else(|| { + windows + .iter() + .filter(|w| w.is_account_wide()) + .filter_map(CodexQuotaWindow::reset_at) + .min() }) - .transpose()?; - let primary_reached = primary_window - .map(|window| optional_bool(window, "limit_reached")) - .transpose()? - .flatten() - .unwrap_or(false); - let reset_at = primary_window - .and_then(|window| window.get("reset_at")) - .map(|value| { - value - .as_i64() - .filter(|value| *value > 0) - .and_then(|value| DateTime::::from_timestamp(value, 0)) - .map(SystemTime::from) - .ok_or(CodexCredentialQuotaError::InvalidCredentialData) - }) - .transpose()?; + .map(SystemTime::from); if allowed == Some(true) { - if top_level_reached || primary_reached { + if top_level_reached || window_reached { tracing::warn!( top_level_reached, - primary_reached, + window_reached, "OpenAI quota returned contradictory access facts; explicit allowed=true wins" ); } @@ -398,7 +397,7 @@ fn authoritative_quota_state( reset_at, )); } - if top_level_reached || primary_reached { + if top_level_reached || window_reached { return Ok(QuotaState::exhausted( QuotaEvidence::AccountLimitReached, observed_at, @@ -408,9 +407,8 @@ fn authoritative_quota_state( Ok(QuotaState::observed_unknown(observed_at)) } -/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。 +/// 一个规范化后的限流桶,source 是上游的稳定 limit_id。 struct CanonicalRateLimit<'a> { - key: String, source: String, account_wide: bool, limit_name: Option, @@ -441,7 +439,6 @@ fn canonical_rate_limits( .and_then(Value::as_str) .map(str::to_owned); limits.push(CanonicalRateLimit { - key: source.clone(), account_wide: is_codex_limit_id(&source), source, limit_name, @@ -456,7 +453,6 @@ fn is_codex_limit_id(value: &str) -> bool { } fn parse_rate_limit_windows( - key_source: &str, source: &str, account_wide: bool, limit_name: Option<&str>, @@ -466,9 +462,7 @@ fn parse_rate_limit_windows( let object = value .as_object() .ok_or(CodexCredentialQuotaError::InvalidCredentialData)?; - let allowed = optional_bool(object, "allowed")?; - let top_level_reached = allowed != Some(true) - && (optional_bool(object, "limit_reached")?.unwrap_or(false) || allowed == Some(false)); + let start = output.len(); for (name, role) in [ ("primary_window", CodexQuotaWindowRole::Primary), ("secondary_window", CodexQuotaWindowRole::Secondary), @@ -506,15 +500,16 @@ fn parse_rate_limit_windows( .ok_or(CodexCredentialQuotaError::InvalidCredentialData) }) .transpose()?; - let limit_reached = allowed != Some(true) - && (top_level_reached - || window - .get("limit_reached") - .and_then(Value::as_bool) - .unwrap_or(false)); + let limit_reached = optional_bool(window, "limit_reached")?.unwrap_or(false); let kind = quota_window_kind(window_seconds); + // primary/secondary 是协议槽位,不代表固定周期。窗口身份使用上游 + // limit_id 和实际时长,5h/周/月分类只参与展示。 + let key = match window_seconds { + Some(seconds) => format!("{source}:{seconds}s"), + None => format!("{source}:{name}"), + }; output.push(CodexQuotaWindow { - key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)), + key, source: source.to_owned(), account_wide, limit_name: limit_name.map(str::to_owned), @@ -526,6 +521,13 @@ fn parse_rate_limit_windows( limit_reached, }); } + // 同桶同周期无法仅凭时长区分,保留官方槽位,避免合并两个独立限制。 + if let [primary, secondary] = &mut output[start..] + && primary.key == secondary.key + { + primary.key.push_str(":primary_window"); + secondary.key.push_str(":secondary_window"); + } Ok(()) } @@ -553,41 +555,6 @@ const fn quota_window_kind(seconds: Option) -> CodexQuotaWindowKind { } } -fn quota_key(value: &str) -> String { - let mut key = value - .chars() - .map(|character| { - if character.is_ascii_alphanumeric() { - character.to_ascii_lowercase() - } else { - '-' - } - }) - .collect::(); - while key.contains("--") { - key = key.replace("--", "-"); - } - let key = key.trim_matches('-'); - if key.is_empty() { - "quota".to_owned() - } else { - key.to_owned() - } -} - -const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind) -> &'static str { - match kind { - CodexQuotaWindowKind::ShortTerm => "five-hour", - CodexQuotaWindowKind::Weekly => "weekly", - CodexQuotaWindowKind::Monthly => "monthly", - CodexQuotaWindowKind::Other => match role { - CodexQuotaWindowRole::Primary => "primary", - CodexQuotaWindowRole::Secondary => "secondary", - CodexQuotaWindowRole::Monthly => "monthly", - }, - } -} - #[derive(Default)] struct QuotaAggregate { recognized: bool, diff --git a/backend/crates/providers/openai/tests/admin.rs b/backend/crates/providers/openai/tests/admin.rs index 30676f356..2af4e64eb 100644 --- a/backend/crates/providers/openai/tests/admin.rs +++ b/backend/crates/providers/openai/tests/admin.rs @@ -737,7 +737,10 @@ async fn openai_admin_provider_projects_official_codex_quota_and_independent_buc let secondary = quota .windows .iter() - .find(|window| window.key == "codex-secondary") + .find(|window| { + window.limit_id.as_deref() == Some("codex") + && window.role == Some(ProviderQuotaWindowRole::Secondary) + }) .expect("core secondary quota"); assert_eq!(secondary.label, "次级额度"); assert_eq!(secondary.used_percent, Some(88.0)); diff --git a/backend/crates/providers/openai/tests/credential/quota/mod.rs b/backend/crates/providers/openai/tests/credential/quota/mod.rs index 863a63ef4..48dcc25f2 100644 --- a/backend/crates/providers/openai/tests/credential/quota/mod.rs +++ b/backend/crates/providers/openai/tests/credential/quota/mod.rs @@ -1,6 +1,7 @@ //! OpenAI 额度事实边界与展示快照回归。 mod recovery; +mod slots; mod snapshot; use std::sync::Arc; diff --git a/backend/crates/providers/openai/tests/credential/quota/recovery.rs b/backend/crates/providers/openai/tests/credential/quota/recovery.rs index a3f7c9bef..f3490c6b9 100644 --- a/backend/crates/providers/openai/tests/credential/quota/recovery.rs +++ b/backend/crates/providers/openai/tests/credential/quota/recovery.rs @@ -75,7 +75,7 @@ async fn quota_refresh_updates_access_fact_without_recovering_credential_error() } #[tokio::test] -async fn exhausted_quota_worker_only_touches_refresh_time_when_recovery_is_rejected() { +async fn exhausted_quota_worker_updates_usage_without_unlocking_at_high_usage() { let store = Arc::new(MemoryAccountStore::default()); let account_id = "acct_worker_quota_recovery_gate"; create_account(&store, account_id).await; @@ -124,14 +124,14 @@ async fn exhausted_quota_worker_only_touches_refresh_time_when_recovery_is_rejec assert_eq!(summary.exhausted, 1); assert_eq!(summary.updated, 0); - assert_eq!(store.quota_json(account_id), Some(old_quota)); + assert_ne!(store.quota_json(account_id), Some(old_quota)); let refreshed = service .read_account(account.id()) .await .expect("read refreshed quota") .expect("refreshed quota snapshot"); assert!(refreshed.observed_at() > old_snapshot.observed_at()); - assert_eq!(refreshed.fact(), old_snapshot.fact()); + assert_eq!(refreshed.fact().remaining_percent(), Some(2)); assert_eq!(refreshed.quota().access(), QuotaAccessState::Exhausted); assert_eq!( store @@ -146,14 +146,7 @@ async fn exhausted_quota_worker_only_touches_refresh_time_when_recovery_is_rejec #[tokio::test] async fn manual_quota_refresh_only_recovers_after_reset_advances_below_ten_percent() { for (suffix, allowed, limit_reached, reset_at, used_percent, recovered) in [ - ( - "still_exhausted", - false, - true, - 1_900_003_600_i64, - 0_u8, - false, - ), + ("stale_denial", false, true, 1_900_003_600_i64, 0_u8, true), ("same_reset", true, false, 1_900_000_000_i64, 0_u8, false), ("exactly_ten", true, false, 1_900_003_600_i64, 10_u8, false), ("below_ten", true, false, 1_900_003_600_i64, 9_u8, true), @@ -207,17 +200,13 @@ async fn manual_quota_refresh_only_recovers_after_reset_advances_below_ten_perce .expect("refresh exhausted quota"); let current = store.account(&account_id).expect("refreshed account"); - if recovered { - assert_eq!(snapshot.fact().remaining_percent(), Some(91)); - assert_eq!(snapshot.quota().access(), QuotaAccessState::Allowed); - assert_ne!(store.quota_json(&account_id), Some(old_quota)); - assert_eq!(current.quota().access(), QuotaAccessState::Allowed); - } else { - assert_eq!(snapshot.fact(), old_snapshot.fact()); - assert_eq!(snapshot.quota().access(), QuotaAccessState::Exhausted); - assert_eq!(store.quota_json(&account_id), Some(old_quota)); - assert_eq!(current.quota().access(), QuotaAccessState::Exhausted); - } + assert_eq!( + snapshot.fact().remaining_percent(), + Some(100 - used_percent) + ); + assert_ne!(store.quota_json(&account_id), Some(old_quota)); + assert_eq!(snapshot.quota().is_exhausted(), !recovered, "{suffix}"); + assert_eq!(current.quota().is_exhausted(), !recovered, "{suffix}"); assert!(snapshot.observed_at() > old_snapshot.observed_at()); } } diff --git a/backend/crates/providers/openai/tests/credential/quota/slots.rs b/backend/crates/providers/openai/tests/credential/quota/slots.rs new file mode 100644 index 000000000..d9d46e594 --- /dev/null +++ b/backend/crates/providers/openai/tests/credential/quota/slots.rs @@ -0,0 +1,574 @@ +//! 各额度槽独立恢复,跨刷新保留未解除的限制。 + +use super::*; + +const SHORT_RESET: i64 = 1_700_000_000; +const WEEK_RESET: i64 = SHORT_RESET + 604_800; + +fn usage(short: (u64, i64), weekly: (u64, i64)) -> serde_json::Value { + json!({"rate_limit": { + "primary_window": {"used_percent": short.0, "reset_at": short.1, "limit_window_seconds": 18_000}, + "secondary_window": {"used_percent": weekly.0, "reset_at": weekly.1, "limit_window_seconds": 604_800} + }}) +} + +async fn mount_usage(server: &MockServer, value: serde_json::Value) { + server.reset().await; + Mock::given(method("GET")) + .and(path("/api/codex/usage")) + .respond_with(ResponseTemplate::new(200).set_body_json(value)) + .mount(server) + .await; +} + +#[tokio::test] +async fn recovery_checks_each_exhausted_slot_in_worker_and_manual_refresh() { + for (name, old_short, old_week, short, week, allowed, recovered) in [ + ( + "short_only", + 100, + 45, + (9, SHORT_RESET + 18_000), + (45, WEEK_RESET), + false, + true, + ), + ( + "weekly_only", + 88, + 100, + (88, SHORT_RESET), + (9, WEEK_RESET + 604_800), + false, + true, + ), + ( + "short_cannot_unlock_week", + 100, + 100, + (0, SHORT_RESET + 18_000), + (100, WEEK_RESET), + true, + false, + ), + ( + "week_cannot_unlock_short", + 100, + 100, + (100, SHORT_RESET), + (0, WEEK_RESET + 604_800), + true, + false, + ), + ( + "both_reset", + 100, + 100, + (0, SHORT_RESET + 18_000), + (9, WEEK_RESET + 604_800), + false, + true, + ), + ( + "same_reset", + 100, + 45, + (0, SHORT_RESET), + (45, WEEK_RESET), + true, + false, + ), + ( + "backward_reset", + 100, + 45, + (0, SHORT_RESET - 1), + (45, WEEK_RESET), + true, + false, + ), + ( + "exactly_ten", + 100, + 45, + (10, SHORT_RESET + 18_000), + (45, WEEK_RESET), + true, + false, + ), + ( + "high_usage", + 100, + 45, + (98, SHORT_RESET + 18_000), + (45, WEEK_RESET), + true, + false, + ), + ( + "week_now_full", + 100, + 45, + (0, SHORT_RESET + 18_000), + (100, WEEK_RESET), + true, + false, + ), + ] { + let name = format!("acct_{name}"); + for worker in [false, true] { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, &name).await; + let account = store.account(&name).expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((old_short, SHORT_RESET), (old_week, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + mount_usage(&server, old).await; + service + .refresh_account(account.id()) + .await + .expect("seed exhausted quota"); + let mut new = usage(short, week); + new["rate_limit"]["allowed"] = json!(allowed); + mount_usage(&server, new).await; + if worker { + let summary = service.synchronize().await.expect("worker refresh"); + assert_eq!(summary.updated, u64::from(recovered), "{name}"); + assert_eq!(summary.exhausted, u64::from(!recovered), "{name}"); + } else { + service + .refresh_account(account.id()) + .await + .expect("manual refresh"); + } + assert_eq!( + store + .account(&name) + .expect("updated account") + .quota() + .is_exhausted(), + !recovered, + "{name}, worker={worker}" + ); + let snapshot = service + .read_account(account.id()) + .await + .expect("read quota") + .expect("snapshot"); + for (seconds, expected) in [(18_000, short.0), (604_800, week.0)] { + let slot = snapshot + .windows() + .iter() + .find(|w| w.window_seconds() == Some(seconds)) + .expect("slot"); + assert_eq!( + slot.used_percent(), + Some(expected as f64), + "latest usage: {name}" + ); + } + } + } +} + +#[tokio::test] +async fn partial_recovery_survives_restart_missing_windows_and_changed_roles() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_partial").await; + let account = store.account("acct_partial").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((100, SHORT_RESET), (100, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + mount_usage(&server, old).await; + service + .refresh_account(account.id()) + .await + .expect("seed exhausted quota"); + mount_usage(&server, usage((1, SHORT_RESET + 18_000), (100, WEEK_RESET))).await; + let partial = service + .refresh_account(account.id()) + .await + .expect("recover 5h only"); + assert!(partial.quota().is_exhausted()); + assert_eq!( + partial.quota().reset_at(), + Some(std::time::UNIX_EPOCH + std::time::Duration::from_secs(WEEK_RESET as u64)) + ); + drop(service); + let restarted = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut missing = usage((50, SHORT_RESET + 18_000), (0, WEEK_RESET + 604_800)); + let weekly = missing["rate_limit"] + .as_object_mut() + .expect("limit") + .remove("secondary_window") + .expect("weekly"); + missing["rate_limit"]["allowed"] = json!(true); + missing["additional_rate_limits"] = + json!([{"limit_name": "separate_model", "rate_limit": {"primary_window": weekly}}]); + mount_usage(&server, missing).await; + assert!( + restarted + .refresh_account(account.id()) + .await + .expect("missing weekly") + .quota() + .is_exhausted() + ); + let mut new = usage((50, SHORT_RESET + 18_000), (9, WEEK_RESET + 604_800)); + let limit = new["rate_limit"].as_object_mut().expect("limit"); + let short = limit.remove("primary_window").expect("short"); + let weekly = limit.remove("secondary_window").expect("weekly"); + limit.insert("primary_window".into(), weekly); + limit.insert("secondary_window".into(), short); + mount_usage(&server, new).await; + assert_eq!( + restarted + .refresh_account(account.id()) + .await + .expect("weekly recovered") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn weekly_error_reset_identifies_the_slot_before_display_reaches_one_hundred() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_weekly_error").await; + let account = store.account("acct_weekly_error").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((88, SHORT_RESET), (99, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(true); + mount_usage(&server, old).await; + service + .refresh_account(account.id()) + .await + .expect("seed allowed quota"); + persist_quota_state( + &store, + &account, + exhausted_quota(Some( + std::time::UNIX_EPOCH + std::time::Duration::from_secs((WEEK_RESET - 1) as u64), + )), + ) + .await; + mount_usage(&server, usage((88, SHORT_RESET), (0, WEEK_RESET + 604_800))).await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("weekly recovered") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn recovery_retains_reset_baseline_until_usage_is_below_ten() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_baseline").await; + let account = store.account("acct_baseline").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((100, SHORT_RESET), (45, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + mount_usage(&server, old).await; + service + .refresh_account(account.id()) + .await + .expect("seed exhausted quota"); + for used in [98, 10, 9] { + mount_usage( + &server, + usage((used, SHORT_RESET + 18_000), (45, WEEK_RESET)), + ) + .await; + let snapshot = service + .refresh_account(account.id()) + .await + .expect("refresh quota"); + assert_eq!(snapshot.quota().is_exhausted(), used >= 10); + } +} + +#[tokio::test] +async fn secondary_window_limit_signal_does_not_mark_the_primary_window_exhausted() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_secondary_signal").await; + let account = store.account("acct_secondary_signal").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((88, SHORT_RESET), (99, WEEK_RESET)); + old["rate_limit"]["secondary_window"]["limit_reached"] = json!(true); + mount_usage(&server, old).await; + let snapshot = service + .refresh_account(account.id()) + .await + .expect("observe weekly exhaustion"); + assert!(snapshot.quota().is_exhausted()); + assert_eq!( + snapshot.quota().reset_at(), + Some(std::time::UNIX_EPOCH + std::time::Duration::from_secs(WEEK_RESET as u64)) + ); + assert!( + !snapshot + .windows() + .iter() + .find(|w| w.window_seconds() == Some(18_000)) + .expect("5h slot") + .limit_reached() + ); + mount_usage(&server, usage((88, SHORT_RESET), (0, WEEK_RESET + 604_800))).await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("weekly recovered") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn a_new_exhaustion_cannot_reuse_partial_recovery_from_the_previous_failure() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_new_exhaustion").await; + let account = store.account("acct_new_exhaustion").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((100, SHORT_RESET), (100, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + mount_usage(&server, old).await; + service + .refresh_account(account.id()) + .await + .expect("seed exhausted quota"); + mount_usage(&server, usage((0, SHORT_RESET + 18_000), (100, WEEK_RESET))).await; + service + .refresh_account(account.id()) + .await + .expect("recover short slot"); + persist_quota_state(&store, &account, QuotaState::allowed(SystemTime::now())).await; + persist_quota_state( + &store, + &account, + exhausted_quota(Some( + std::time::UNIX_EPOCH + std::time::Duration::from_secs((SHORT_RESET + 18_000) as u64), + )), + ) + .await; + mount_usage( + &server, + usage((0, SHORT_RESET + 18_000), (0, WEEK_RESET + 604_800)), + ) + .await; + assert!( + service + .refresh_account(account.id()) + .await + .expect("same short reset cannot recover new failure") + .quota() + .is_exhausted() + ); + mount_usage( + &server, + usage((0, SHORT_RESET + 36_000), (50, WEEK_RESET + 604_800)), + ) + .await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("new short reset") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn stored_weekly_exhaustion_ignores_the_old_primary_only_account_reset() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_old_weekly").await; + let account = store.account("acct_old_weekly").expect("account"); + let observed_at = SystemTime::now(); + let mut old = usage((88, SHORT_RESET), (100, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + store + .compare_and_swap_quota(gateway_core::account::QuotaObservation { + account_id: account.id().clone(), + expected_revision: account.revision(), + quota: gateway_core::account::OpaqueProviderData::new( + old.as_object().expect("document").clone(), + ), + observed_at, + state: QuotaState::exhausted( + QuotaEvidence::ProviderDenied, + observed_at, + Some(std::time::UNIX_EPOCH + std::time::Duration::from_secs(SHORT_RESET as u64)), + ), + }) + .await + .expect("seed old persisted quota"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + mount_usage(&server, usage((88, SHORT_RESET), (0, WEEK_RESET + 604_800))).await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("recover stored weekly exhaustion") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn an_unknown_slot_establishes_a_baseline_before_it_can_recover() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_unknown_slot").await; + let account = store.account("acct_unknown_slot").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + mount_usage(&server, json!({"rate_limit": {"allowed": false}})).await; + service + .refresh_account(account.id()) + .await + .expect("exhaustion without windows"); + for (window, recovered) in [ + (json!({"used_percent": 0}), false), + (json!({"used_percent": 0, "reset_at": SHORT_RESET}), false), + (json!({"reset_at": SHORT_RESET + 18_000}), false), + ( + json!({"used_percent": 9, "reset_at": SHORT_RESET + 18_000}), + true, + ), + ] { + mount_usage( + &server, + json!({"rate_limit": {"allowed": true, "primary_window": window}}), + ) + .await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("observe window") + .quota() + .is_exhausted(), + !recovered + ); + } +} + +#[tokio::test] +async fn monthly_and_arbitrary_periods_use_the_same_recovery_rule() { + for seconds in [18_000, 86_400, 604_800, 864_000, 2_592_000, 7_776_000] { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_generic_period").await; + let account = store.account("acct_generic_period").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + for (used, reset, recovered) in [ + (100, SHORT_RESET, false), + (0, SHORT_RESET, false), + (10, SHORT_RESET + seconds, false), + (1, SHORT_RESET + seconds, true), + ] { + mount_usage(&server, json!({"rate_limit": { + "allowed": false, + "primary_window": {"used_percent": used, "reset_at": reset, "limit_window_seconds": seconds}, + "secondary_window": null + }})).await; + let snapshot = service + .refresh_account(account.id()) + .await + .expect("refresh quota"); + assert_eq!( + snapshot.quota().is_exhausted(), + !recovered, + "period={seconds}, used={used}" + ); + } + } +} + +#[tokio::test] +async fn arbitrary_periods_keep_their_identity_when_roles_change() { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, "acct_generic_roles").await; + let account = store.account("acct_generic_roles").expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((100, SHORT_RESET), (50, WEEK_RESET)); + old["rate_limit"]["allowed"] = json!(false); + old["rate_limit"]["primary_window"]["limit_window_seconds"] = json!(864_000); + old["rate_limit"]["secondary_window"]["limit_window_seconds"] = json!(172_800); + mount_usage(&server, old.clone()).await; + service + .refresh_account(account.id()) + .await + .expect("seed 10-day exhaustion"); + let mut new = old.clone(); + new["rate_limit"]["primary_window"] = old["rate_limit"]["secondary_window"].clone(); + new["rate_limit"]["secondary_window"] = old["rate_limit"]["primary_window"].clone(); + new["rate_limit"]["secondary_window"]["used_percent"] = json!(9); + new["rate_limit"]["secondary_window"]["reset_at"] = json!(SHORT_RESET + 864_000); + mount_usage(&server, new).await; + assert_eq!( + service + .refresh_account(account.id()) + .await + .expect("recover 10-day slot") + .quota() + .access(), + QuotaAccessState::Allowed + ); +} + +#[tokio::test] +async fn windows_with_the_same_display_kind_or_duration_do_not_share_recovery() { + for (second_period, name) in [ + (18_900, "acct_similar_periods"), + (18_000, "acct_equal_periods"), + ] { + let store = Arc::new(MemoryAccountStore::default()); + create_account(&store, name).await; + let account = store.account(name).expect("account"); + let server = MockServer::start().await; + let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri()); + let mut old = usage((100, SHORT_RESET), (100, SHORT_RESET + 100)); + old["rate_limit"]["allowed"] = json!(false); + old["rate_limit"]["secondary_window"]["limit_window_seconds"] = json!(second_period); + mount_usage(&server, old.clone()).await; + let snapshot = service + .refresh_account(account.id()) + .await + .expect("seed independent exhausted slots"); + assert_ne!( + snapshot.windows()[0].key(), + snapshot.windows()[1].key(), + "{name}" + ); + old["rate_limit"]["secondary_window"]["used_percent"] = json!(0); + old["rate_limit"]["secondary_window"]["reset_at"] = + json!(SHORT_RESET + 100 + second_period); + mount_usage(&server, old).await; + assert!( + service + .refresh_account(account.id()) + .await + .expect("recover secondary only") + .quota() + .is_exhausted(), + "{name}" + ); + } +} diff --git a/backend/crates/providers/openai/tests/provider/contract.rs b/backend/crates/providers/openai/tests/provider/contract.rs index 2e0bbab7f..824e3d9f1 100644 --- a/backend/crates/providers/openai/tests/provider/contract.rs +++ b/backend/crates/providers/openai/tests/provider/contract.rs @@ -3690,8 +3690,8 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() Mock::given(method("GET")) .and(path("/api/codex/usage")) .and(header("authorization", format!("Bearer at-{account_id}"))) - // 生产环境会短暂返回互相矛盾的快照:98/99% 且布尔位仍声称可用。 - // 后台同步不能覆盖真实请求已确认的额度耗尽。 + // 失败后的补查只更新观察时间,即使响应满足恢复条件, + // 也不能覆盖本次推理刚确认的耗尽;恢复由独立的主动刷新判断。 .respond_with( ResponseTemplate::new(200) // 验证后台 usage 同步不能把原始的额度错误响应拖到查询完成之后。 @@ -3700,7 +3700,7 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() "rate_limit": { "allowed": true, "limit_reached": false, - "primary_window": {"used_percent": 99, "reset_at": reset_at}, + "primary_window": {"used_percent": 0, "reset_at": reset_at + 18_000}, "secondary_window": {"used_percent": 0} } })), @@ -3781,10 +3781,13 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() timeout(Duration::from_secs(5), async { loop { - let requests = server.received_requests().await.expect("received requests"); - if requests - .iter() - .any(|request| request.url.path() == "/api/codex/usage") + let observations = store + .get_quotas(&[account.id().clone()]) + .await + .expect("quota observations"); + if observations + .first() + .is_some_and(|observation| observation.observed_at > projected_observed_at) { break; } @@ -3792,7 +3795,7 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() } }) .await - .expect("background usage refresh must run"); + .expect("background usage refresh must complete"); let observation = store .get_quotas(&[account.id().clone()]) .await @@ -3800,7 +3803,7 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() .into_iter() .next() .expect("authoritative quota projection"); - assert_eq!(observation.observed_at, projected_observed_at); + assert!(observation.observed_at > projected_observed_at); let quota = Value::Object(observation.quota.into_inner()); assert_eq!( quota @@ -3836,7 +3839,7 @@ async fn official_usage_limit_failure_persists_fact_without_fabricating_usage() .quota() .access(), QuotaAccessState::Exhausted, - "a contradictory full usage snapshot must not unlock a confirmed exhausted account" + "a failure follow-up must preserve access even when the reset recovery condition is met" ); } diff --git a/backend/crates/providers/openai/tests/transport/profile/mod.rs b/backend/crates/providers/openai/tests/transport/profile/mod.rs index 7ca7e15a9..e9b879709 100644 --- a/backend/crates/providers/openai/tests/transport/profile/mod.rs +++ b/backend/crates/providers/openai/tests/transport/profile/mod.rs @@ -12,8 +12,7 @@ use gateway_core::routing::ProviderKind; use provider_openai::transport::profile::{ CodexArtifactProfileCache, CodexBundledReleaseProfile, CodexDesktopRelease, CodexDesktopReleaseError, CodexDesktopReleaseService, CodexDesktopReleaseTransport, - CodexWireProfile, CodexWireProfileState, OfficialCodexDesktopReleaseTransport, - parse_desktop_release, + CodexWireProfile, CodexWireProfileState, parse_desktop_release, }; mod desktop_artifact; @@ -319,23 +318,6 @@ async fn cache_failure_should_not_publish_a_partially_verified_profile() { assert!(service.status().snapshot().latest.is_none()); } -#[tokio::test] -#[ignore = "downloads bounded ranges from the current official macOS artifact"] -async fn official_desktop_artifact_should_expose_its_bundled_core_version() { - let transport = OfficialCodexDesktopReleaseTransport::new().expect("official transport"); - let release = transport.fetch().await.expect("official appcast release"); - let core_version = transport - .fetch_bundled_core_version(&release) - .await - .expect("bundled Core version"); - - semver::Version::parse(&core_version).expect("semantic bundled Core version"); - eprintln!( - "official Desktop {} (build {}) bundles Core {core_version}", - release.version, release.build - ); -} - fn service( state: CodexWireProfileState, transport: Arc,