diff --git a/backend/crates/gateway-admin/src/model/quota_forecast_sampling.rs b/backend/crates/gateway-admin/src/model/quota_forecast_sampling.rs index 3bb000b94..f5107d715 100644 --- a/backend/crates/gateway-admin/src/model/quota_forecast_sampling.rs +++ b/backend/crates/gateway-admin/src/model/quota_forecast_sampling.rs @@ -8,6 +8,7 @@ use super::provider_credentials::ProviderDocument; pub const MAX_FORECAST_HISTORY_POINTS: usize = 128; const MIN_BLOCK_PERCENT: f64 = 5.0; const RECENT_BLOCKS: usize = 3; +const MAX_PERCENT_REVERSAL: f64 = 1.0; #[derive(Debug, Clone, Default, PartialEq)] pub struct QuotaForecastUsage { @@ -80,11 +81,55 @@ pub struct QuotaForecastObservation { #[derive(Debug, Clone)] pub struct QuotaForecastPoint { + pub started_at: DateTime, pub observed_at: DateTime, + pub reset_at: DateTime, pub used_percent: f64, pub usage: QuotaForecastUsage, } +impl QuotaForecastPoint { + fn follows(&self, previous: &Self) -> bool { + previous.observed_at <= self.started_at && previous.observed_at < self.observed_at + } +} + +/// 当前窗口内仍有旧周期记录时,只在旧读数已到期、用量回落且新读数得到印证后重设累计 +fn cycle_boundary( + history: &[QuotaForecastPoint], + current: &QuotaForecastPoint, +) -> (Option, bool) { + let mut boundary = None; + let mut awaiting_confirmation = false; + for (index, pair) in history.windows(2).enumerate() { + let (older, previous) = (&pair[0], &pair[1]); + let first_index = index + 2; + let first = history.get(first_index).unwrap_or(current); + let before_reset = older.used_percent.min(previous.used_percent); + if !previous.follows(older) + || !first.follows(previous) + || older.reset_at <= previous.observed_at + || older.reset_at > first.started_at + || previous.reset_at > first.started_at + || before_reset - first.used_percent <= MAX_PERCENT_REVERSAL + { + continue; + } + // 快照可能复用请求读数;只有另一条在新窗口内独立完成的请求能确认换段 + let confirmation = history + .get(first_index + 1..) + .and_then(|later| later.iter().find(|next| next.follows(first))); + awaiting_confirmation = confirmation.is_none(); + if confirmation.is_some_and(|next| { + next.observed_at < first.reset_at + && before_reset - next.used_percent > MAX_PERCENT_REVERSAL + }) { + boundary = Some(first_index); + } + } + (boundary, awaiting_confirmation) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum QuotaForecastMethod { Cumulative, @@ -117,7 +162,7 @@ pub fn select_forecast_sample( current: QuotaForecastPoint, mut history: Vec, pending_request_count: u64, - interrupted: bool, + plan_changed: bool, ) -> QuotaForecastSample { history.retain(|point| { start_at <= point.observed_at @@ -126,6 +171,9 @@ pub fn select_forecast_sample( && (0.0..=100.0).contains(&point.used_percent) }); history.sort_by_key(|point| point.observed_at); + let (boundary, awaiting_confirmation) = cycle_boundary(&history, ¤t); + let history = &history[boundary.unwrap_or(0)..]; + let interrupted = plan_changed || boundary.is_some(); let mut sample = QuotaForecastSample { key, method: QuotaForecastMethod::Cumulative, @@ -136,23 +184,35 @@ pub fn select_forecast_sample( block_count: 0, observation_count: history.len(), pending_request_count, - discontinuous: interrupted, + discontinuous: interrupted || awaiting_confirmation, cycle_usage: current.usage.clone(), usage: current.usage.clone(), }; - history.push(current); - // 重置边界不在查询起点时,从首个新观测重新计数,避免把旧周期累计带入总量 - let cycle_baseline = interrupted.then(|| &history[0]); + if interrupted { + let baseline = history.first().unwrap_or(¤t); + let Some(usage) = current.usage.difference(&baseline.usage) else { + return sample; + }; + sample.cycle_usage = usage; + } + if awaiting_confirmation { + return sample; + } let mut anchors: Vec<&QuotaForecastPoint> = Vec::new(); let mut high_water: Option<&QuotaForecastPoint> = None; - for point in &history { + for point in history.iter().chain(std::iter::once(¤t)) { + // 截至快照已过期的读数不能用于预测剩余量,窗口累计仍由上面的周期证据决定 + if point.reset_at <= current.observed_at { + sample.discontinuous = true; + continue; + } if let Some(previous) = high_water { if point.observed_at <= previous.observed_at { continue; } if point.used_percent < previous.used_percent { // 一个百分点内的回落可能是量化/并发乱序;更大的回落不能跨越 - if previous.used_percent - point.used_percent <= 1.0 { + if previous.used_percent - point.used_percent <= MAX_PERCENT_REVERSAL { continue; } // 同窗口回落不能证明周期重置,只重建采样并保留已记录累计 @@ -168,16 +228,6 @@ pub fn select_forecast_sample( anchors.push(point); } } - let Some(current) = history.last() else { - return sample; - }; - if let Some(baseline) = cycle_baseline { - let Some(usage) = current.usage.difference(&baseline.usage) else { - sample.discontinuous = true; - return sample; - }; - sample.cycle_usage = usage; - } if anchors.len() < 2 { return sample; } diff --git a/backend/crates/gateway-admin/src/use_case/accounts/mod.rs b/backend/crates/gateway-admin/src/use_case/accounts/mod.rs index e68a2e3aa..21b0d5011 100644 --- a/backend/crates/gateway-admin/src/use_case/accounts/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/accounts/mod.rs @@ -1057,6 +1057,8 @@ impl AccountsService for DefaultAccountsService { ) else { continue; }; + let reset_at = query.range.end; + let window_duration = reset_at - query.range.start; query.range.start = query.range.start.max(stored.account.created_at); if query.range.start >= observed || observed > now @@ -1066,7 +1068,6 @@ impl AccountsService for DefaultAccountsService { { continue; } - let reset_at = query.range.end; query.range.end = observed; let history = self .accounts @@ -1074,27 +1075,37 @@ impl AccountsService for DefaultAccountsService { .await .map_err(|error| map_store_error(error, "forecast paired usage"))?; let mut points = Vec::new(); - let mut interrupted = false; + let mut plan_changed = false; for point in history.points { let Some(fact) = provider.quota_forecast_observation(&point.provider_observation, window) else { continue; }; - let same_plan = quota - .plan_type - .as_deref() - .zip(fact.plan_type.as_deref()) - .is_some_and(|(current, previous)| current.eq_ignore_ascii_case(previous)); - // 仅容纳已观测到的秒级量化抖动,不用宽时间容差合并实际重置 - // 不匹配的段截断基线;之后的有效观测可以重新积累 - if !same_plan || (fact.reset_at - reset_at).abs() > Duration::seconds(2) { + let (Some(current_plan), Some(previous_plan)) = + (quota.plan_type.as_deref(), fact.plan_type.as_deref()) + else { + continue; + }; + let Some(start) = fact.reset_at.checked_sub_signed(window_duration) else { + continue; + }; + if point.completed_at >= observed + || point.started_at < query.range.start + || point.completed_at < start + || point.completed_at >= fact.reset_at + { + continue; + } + if !current_plan.eq_ignore_ascii_case(previous_plan) { points.clear(); - interrupted = true; + plan_changed = true; continue; } points.push(QuotaForecastPoint { + started_at: point.started_at, observed_at: point.completed_at, + reset_at: fact.reset_at, used_percent: fact.used_percent, usage: point.usage, }); @@ -1103,13 +1114,15 @@ impl AccountsService for DefaultAccountsService { window.key.clone(), query.range.start, QuotaForecastPoint { + started_at: observed, observed_at: observed, + reset_at, used_percent: percent, usage: history.usage, }, points, history.pending_request_count, - interrupted, + plan_changed, ); samples.push(sample); } diff --git a/backend/crates/gateway-admin/tests/model/quota_forecast_sampling.rs b/backend/crates/gateway-admin/tests/model/quota_forecast_sampling.rs index 4d9d461ae..a0d3e2476 100644 --- a/backend/crates/gateway-admin/tests/model/quota_forecast_sampling.rs +++ b/backend/crates/gateway-admin/tests/model/quota_forecast_sampling.rs @@ -12,7 +12,9 @@ fn time(minutes: i64) -> DateTime { fn point(minute: i64, percent: f64, tokens: u64) -> QuotaForecastPoint { QuotaForecastPoint { + started_at: time(minute) - Duration::seconds(1), observed_at: time(minute), + reset_at: time(7 * 24 * 60), used_percent: percent, usage: QuotaForecastUsage { request_count: minute as u64, @@ -188,7 +190,41 @@ fn repeated_same_window_recoveries_preserve_recorded_cycle_usage() { } #[test] -fn changed_window_restarts_cycle_totals_at_first_matching_observation() { +fn an_expired_deadline_alone_cannot_cut_totals_or_anchor_remaining_usage() { + let mut older = point(1, 15.0, 100); + older.reset_at = time(3); + let mut previous = point(2, 20.0, 200); + previous.reset_at = time(3) + Duration::seconds(4); + let first = point(4, 25.0, 700); + let sample = select(vec![older, previous, first], point(6, 35.0, 900)); + assert_eq!(sample.cycle_usage.tokens, 900); + assert_eq!(sample.baseline_percent, 25.0); + assert_eq!(sample.usage.tokens, 200); +} + +#[test] +fn copied_concurrent_or_contradictory_readings_cannot_confirm_a_recovery() { + let mut older = point(1, 75.0, 100); + older.reset_at = time(3); + let mut previous = point(2, 85.0, 200); + previous.reset_at = time(3); + let first = point(4, 5.0, 300); + let current = point(6, 40.0, 900); + let mut overlapping = point(5, 20.0, 400); + overlapping.started_at = first.started_at; + for extra in [None, Some(overlapping), Some(current.clone())] { + let mut points = vec![older.clone(), previous.clone(), first.clone()]; + points.extend(extra); + let sample = select(points, current.clone()); + assert_eq!(sample.cycle_usage.tokens, 900); + assert!(sample.discontinuous); + } + let sample = select(vec![older, previous, first, point(5, 86.0, 400)], current); + assert_eq!(sample.cycle_usage.tokens, 900); +} + +#[test] +fn changed_plan_restarts_cycle_totals_at_first_matching_observation() { let sample = select_forecast_sample( "week".to_owned(), time(0), diff --git a/backend/crates/gateway-admin/tests/use_case/accounts/mod.rs b/backend/crates/gateway-admin/tests/use_case/accounts/mod.rs index 07dcd7a97..6d46cf709 100644 --- a/backend/crates/gateway-admin/tests/use_case/accounts/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/accounts/mod.rs @@ -2576,7 +2576,9 @@ async fn quota_forecast_same_window_reversal_keeps_recorded_usage_in_the_report( usage: usage(index as u64 + 1, tokens), provider_observation: ProviderDocument::new(OpaqueProviderData::new( serde_json::json!({ - "percent": percent, "reset": reset.to_rfc3339(), "plan": "pro", + "percent": percent, + "reset": (reset + TimeDelta::seconds(17 * index as i64)).to_rfc3339(), + "plan": "pro", }) .as_object() .unwrap() @@ -2605,14 +2607,14 @@ async fn quota_forecast_same_window_reversal_keeps_recorded_usage_in_the_report( } #[tokio::test] -async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_changed_bucket() { +async fn quota_forecast_keeps_cycle_totals_through_reset_jitter_and_restarts_after_a_reset() { use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistoryPoint; let now = Utc::now(); let observed = now - TimeDelta::minutes(1); let reset = now + TimeDelta::days(1); let added = now - TimeDelta::hours(5); let provider = FakeProviderAdmin::new("openai", events()); - provider.set_quota(ProviderQuota { + let mut quota = ProviderQuota { upstream: None, credits: None, plan_type: Some("pro".to_owned()), @@ -2627,67 +2629,102 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_ role: None, local_usage_attribution: QuotaLocalUsageAttribution::AccountWide, window_seconds: Some(604_800), - used_percent: Some(40.0), + used_percent: Some(65.0), reset_at: Some(reset), limit_reached: false, local_usage: None, provider_data: None, }], ..empty_quota() - }); + }; let mut account = account_record("openai"); account.created_at = added; let store = FakeAccountStore::with_account(account, events()); - let make_point = |hours, percent, tokens, delta| { - QuotaForecastHistoryPoint { - started_at: now - TimeDelta::hours(hours) - TimeDelta::seconds(10), - completed_at: now - TimeDelta::hours(hours), - usage: QuotaForecastUsage { request_count: tokens / 100, tokens, ..Default::default() }, - provider_observation: ProviderDocument::new(OpaqueProviderData::new(serde_json::json!({ - "percent": percent, "reset": (reset + TimeDelta::seconds(delta)).to_rfc3339(), "plan": "pro", - }).as_object().unwrap().clone())), - } + let usage = |tokens| QuotaForecastUsage { + request_count: tokens / 100, + tokens, + known_cost_count: tokens / 100, + usd: tokens as f64 / 100.0, + ..Default::default() }; + let point_with_plan = |hours, percent, tokens, delta, plan: Option<&str>| { + let observation = serde_json::json!({ + "percent": percent, + "reset": (reset + TimeDelta::seconds(delta)).to_rfc3339(), + "plan": plan, + }); + QuotaForecastHistoryPoint { + started_at: now - TimeDelta::hours(hours) - TimeDelta::seconds(10), + completed_at: now - TimeDelta::hours(hours), + usage: usage(tokens), + provider_observation: ProviderDocument::new(OpaqueProviderData::new( + observation.as_object().unwrap().clone(), + )), + } + }; + let make_point = + |hours, percent, tokens, delta| point_with_plan(hours, percent, tokens, delta, Some("pro")); *store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory { points: vec![ - make_point(3, 20.0, 1_000, 1), - make_point(2, 30.0, 1_500, -1), + make_point(4, 20.0, 20_000, 0), + make_point(3, 58.0, 62_000, 17), + make_point(2, 58.0, 62_500, 0), ], - usage: QuotaForecastUsage { - request_count: 20, - tokens: 2_000, - ..Default::default() - }, + usage: usage(70_600), pending_request_count: 1, }; - let services = accounts_service(provider, store.clone()).await; - let result = services - .accounts() - .quota_forecast(&ProviderAccountId::new("acct_test").unwrap()) - .await - .unwrap(); - assert!(result.forecasts[0].unavailable_reason.is_none()); - assert_eq!(result.forecasts[0].estimated_tokens, Some(5_000)); + let services = accounts_service(provider.clone(), store.clone()).await; + let account_id = ProviderAccountId::new("acct_test").unwrap(); + let read_report = async || { + services + .accounts() + .quota_forecast(&account_id) + .await + .unwrap() + }; + // 回放 #759 的金额和百分比,历史与快照各自报告的到期时间偏移时仍应保留整段累计 + for offset in [0, 3, 6, 7, 17, -23] { + quota.windows[0].reset_at = Some(reset + TimeDelta::seconds(offset)); + provider.set_quota(quota.clone()); + let report = read_report().await; + let week = &report.forecasts[0]; + let source = week.source.as_ref().unwrap(); + assert_eq!(source.tokens, Some(70_600)); + assert_eq!(source.usd, Some(706.0)); + assert_eq!(week.estimated_tokens, Some(109_956)); + assert!((week.estimated_usd.unwrap() - (706.0 + 506.0 * 35.0 / 45.0)).abs() < 1e-9); + assert!(week.unavailable_reason.is_none()); + } assert_eq!(store.quota_window_queries()[0].range.start, added); store .quota_forecast_history .lock() .unwrap() .points - .push(make_point(1, 35.0, 1_700, 3_600)); - let result = services - .accounts() - .quota_forecast(&ProviderAccountId::new("acct_test").unwrap()) - .await - .unwrap(); - assert!(result.forecasts[0].estimated_tokens.is_none()); - assert!( - result.forecasts[0] - .unavailable_reason - .unwrap() - .contains("不连续") + .push(make_point(1, 62.0, 66_000, 3_600)); + let result = read_report().await; + assert_eq!( + result.forecasts[0].source.as_ref().unwrap().tokens, + Some(70_600) ); - // 新额度段已有足够观测后恢复预测,但总量不能带回重置前的累计用量 + assert!(result.forecasts[0].estimated_tokens.is_some()); + // 两份旧读数的截止时刻均已过去,新请求开始于旧窗口结束后 + quota.windows[0].used_percent = Some(40.0); + provider.set_quota(quota.clone()); + *store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory { + points: vec![ + make_point(4, 75.0, 1_000, -90_000), + make_point(2, 85.0, 1_500, -90_021), + ], + usage: usage(2_000), + ..Default::default() + }; + let waiting = read_report().await; + assert_eq!( + waiting.forecasts[0].source.as_ref().unwrap().tokens, + Some(2_000) + ); + assert!(waiting.forecasts[0].estimated_tokens.is_none()); let mut first = make_point(1, 5.0, 1_750, 0); first.completed_at = now - TimeDelta::minutes(45); first.started_at = first.completed_at - TimeDelta::seconds(10); @@ -2700,15 +2737,42 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_ .unwrap() .points .extend([first, next]); - let result = services - .accounts() - .quota_forecast(&ProviderAccountId::new("acct_test").unwrap()) - .await - .unwrap(); + let result = read_report().await; let cycle = &result.forecasts[0]; assert!(cycle.unavailable_reason.is_none()); assert_eq!(cycle.source.as_ref().unwrap().tokens, Some(250)); assert_eq!(cycle.estimated_tokens, Some(679)); + + for (plan, expected_tokens) in [(None, 2_000), (Some("plus"), 500)] { + store.quota_forecast_history.lock().unwrap().points = vec![ + make_point(4, 20.0, 1_000, 0), + point_with_plan(3, 25.0, 1_200, 0, plan), + make_point(2, 30.0, 1_500, 0), + ]; + let report = read_report().await; + assert_eq!( + report.forecasts[0].source.as_ref().unwrap().tokens, + Some(expected_tokens) + ); + } + + // 上游提前给出新的七天窗口时,直接从新起点查询,首条配对样本前的用量也留在累计中 + let new_start = now - TimeDelta::hours(1); + quota.windows[0].reset_at = Some(new_start + TimeDelta::days(7)); + provider.set_quota(quota); + *store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory { + usage: usage(400), + ..Default::default() + }; + let report = read_report().await; + assert_eq!( + store.quota_window_queries().last().unwrap().range.start, + new_start + ); + assert_eq!( + report.forecasts[0].source.as_ref().unwrap().tokens, + Some(400) + ); } #[tokio::test] diff --git a/docs/api.md b/docs/api.md index fab7d15be..0408ff8ce 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1855,7 +1855,7 @@ OpenAI 授权码交换与令牌刷新请求的连接超时为 30 秒,单次上 样本仅使用当前额度窗口内、账号加入后开始且截至观测时间已完整交付的请求,缓存命中不重复计入输入 -重置或套餐变化后重新积累有效进度,中途加入的账号可在新基线后采样,无需等待下一次重置,职责与断点处理见[额度预测架构](architecture.md#额度预测) +重置或套餐变化后重新积累有效进度,中途加入的账号可在新基线后采样。单条历史读数的重置时间偏移不会截断已记录用量。疑似跨周期且仍待确认时,接口保留来源用量并暂停估算,详见[额度预测架构](architecture.md#额度预测) 预测不参与调度或账单结算,站外消耗、日志清理、异步延迟、未成功交付的上游消耗和模型组合变化均可能造成误差 diff --git a/docs/architecture.md b/docs/architecture.md index 780f5e648..4ad22c2da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1165,7 +1165,9 @@ Admin 按卡片与所选指标向 Store 请求计数、金额或分布,缓存 - 同一额度窗口内,每累计至少 5 个百分点形成一段,使用最近 3 个完整段及尾部计算合并比值,不平均逐请求的小分母比值 - 重复读数不增加段数,大于 1 个百分点的额度回落、累计计数倒退或窗口身份变化会中断采样 - 同窗口回落保留已记录累计;触顶后恢复可用不代表新周期,不能据此重设累计基线 -- OpenAI 的窗口匹配复用限流解析器,核对桶、槽位、时长和明确套餐,只容许最多 2 秒的重置时间抖动 +- 本周期累计从当前快照的窗口起点查询。OpenAI 历史读数须匹配额度桶、槽位、时长和明确套餐。重置时间的偏移本身不触发换段,已过期的读数不参与剩余量估算 +- 当前窗口覆盖旧读数时,要有两次独立请求报告的到期时刻均已过去,且随后两次独立请求都显示已用比例回落超过 1 个百分点,才从新段首条读数重新累计。等待第二条读数期间保留累计并暂停估算 +- 跳过缺少套餐的历史读数,确认套餐变化后从新段重新累计 - 重置后从新段重新累计,无法精确归属的用量不补算;只有账号在窗口开始前加入且无已知断点时,才允许以窗口累计量替代增量采样 - 历史点抽样不缩减区间累计用量,同一完成时间共用同一累计值