Compare commits

...
3 Commits
Author SHA1 Message Date
Wei HUandCodex d1be306f03 test(api): isolate virtual time from idle WebSocket I/O (#761)
The idle expiry test reads a real TCP socket with a Tokio timeout while
the clock is paused. Auto-advance can reach the read deadline while the
socket is still waiting for readiness.

Follow the adjacent connection lifecycle test and pause only for the
60-minute jump. Use a running clock for the handshake and frame read.
Keep the exact limit error and zero-execution assertions.

Co-authored-by: Codex <noreply@openai.com>
2026-11-21 22:51:47 +08:00
aa34d5291d fix(accounts): preserve quota forecast totals through reset time jitter (#760)
* fix(accounts): preserve quota forecast totals through reset time jitter

Co-authored-by: Codex <noreply@openai.com>

* fix(accounts): confirm quota cycle changes before trimming usage

Co-authored-by: Codex <noreply@openai.com>

---------

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
2026-11-21 21:52:12 +08:00
github-actions[bot] 56d3078ee8 chore(release): update release channels to 3.54.2 2026-11-21 12:52:38 +00:00
8 changed files with 252 additions and 84 deletions
@@ -8,6 +8,7 @@ use super::provider_credentials::ProviderDocument;
pub const MAX_FORECAST_HISTORY_POINTS: usize = 128;
const MIN_BLOCK_PERCENT: f64 = 5.0;
const RECENT_BLOCKS: usize = 3;
const MAX_PERCENT_REVERSAL: f64 = 1.0;
#[derive(Debug, Clone, Default, PartialEq)]
pub struct QuotaForecastUsage {
@@ -80,11 +81,55 @@ pub struct QuotaForecastObservation {
#[derive(Debug, Clone)]
pub struct QuotaForecastPoint {
pub started_at: DateTime<Utc>,
pub observed_at: DateTime<Utc>,
pub reset_at: DateTime<Utc>,
pub used_percent: f64,
pub usage: QuotaForecastUsage,
}
impl QuotaForecastPoint {
fn follows(&self, previous: &Self) -> bool {
previous.observed_at <= self.started_at && previous.observed_at < self.observed_at
}
}
/// 当前窗口内仍有旧周期记录时,只在旧读数已到期、用量回落且新读数得到印证后重设累计
fn cycle_boundary(
history: &[QuotaForecastPoint],
current: &QuotaForecastPoint,
) -> (Option<usize>, bool) {
let mut boundary = None;
let mut awaiting_confirmation = false;
for (index, pair) in history.windows(2).enumerate() {
let (older, previous) = (&pair[0], &pair[1]);
let first_index = index + 2;
let first = history.get(first_index).unwrap_or(current);
let before_reset = older.used_percent.min(previous.used_percent);
if !previous.follows(older)
|| !first.follows(previous)
|| older.reset_at <= previous.observed_at
|| older.reset_at > first.started_at
|| previous.reset_at > first.started_at
|| before_reset - first.used_percent <= MAX_PERCENT_REVERSAL
{
continue;
}
// 快照可能复用请求读数;只有另一条在新窗口内独立完成的请求能确认换段
let confirmation = history
.get(first_index + 1..)
.and_then(|later| later.iter().find(|next| next.follows(first)));
awaiting_confirmation = confirmation.is_none();
if confirmation.is_some_and(|next| {
next.observed_at < first.reset_at
&& before_reset - next.used_percent > MAX_PERCENT_REVERSAL
}) {
boundary = Some(first_index);
}
}
(boundary, awaiting_confirmation)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuotaForecastMethod {
Cumulative,
@@ -117,7 +162,7 @@ pub fn select_forecast_sample(
current: QuotaForecastPoint,
mut history: Vec<QuotaForecastPoint>,
pending_request_count: u64,
interrupted: bool,
plan_changed: bool,
) -> QuotaForecastSample {
history.retain(|point| {
start_at <= point.observed_at
@@ -126,6 +171,9 @@ pub fn select_forecast_sample(
&& (0.0..=100.0).contains(&point.used_percent)
});
history.sort_by_key(|point| point.observed_at);
let (boundary, awaiting_confirmation) = cycle_boundary(&history, &current);
let history = &history[boundary.unwrap_or(0)..];
let interrupted = plan_changed || boundary.is_some();
let mut sample = QuotaForecastSample {
key,
method: QuotaForecastMethod::Cumulative,
@@ -136,23 +184,35 @@ pub fn select_forecast_sample(
block_count: 0,
observation_count: history.len(),
pending_request_count,
discontinuous: interrupted,
discontinuous: interrupted || awaiting_confirmation,
cycle_usage: current.usage.clone(),
usage: current.usage.clone(),
};
history.push(current);
// 重置边界不在查询起点时,从首个新观测重新计数,避免把旧周期累计带入总量
let cycle_baseline = interrupted.then(|| &history[0]);
if interrupted {
let baseline = history.first().unwrap_or(&current);
let Some(usage) = current.usage.difference(&baseline.usage) else {
return sample;
};
sample.cycle_usage = usage;
}
if awaiting_confirmation {
return sample;
}
let mut anchors: Vec<&QuotaForecastPoint> = Vec::new();
let mut high_water: Option<&QuotaForecastPoint> = None;
for point in &history {
for point in history.iter().chain(std::iter::once(&current)) {
// 截至快照已过期的读数不能用于预测剩余量,窗口累计仍由上面的周期证据决定
if point.reset_at <= current.observed_at {
sample.discontinuous = true;
continue;
}
if let Some(previous) = high_water {
if point.observed_at <= previous.observed_at {
continue;
}
if point.used_percent < previous.used_percent {
// 一个百分点内的回落可能是量化/并发乱序;更大的回落不能跨越
if previous.used_percent - point.used_percent <= 1.0 {
if previous.used_percent - point.used_percent <= MAX_PERCENT_REVERSAL {
continue;
}
// 同窗口回落不能证明周期重置,只重建采样并保留已记录累计
@@ -168,16 +228,6 @@ pub fn select_forecast_sample(
anchors.push(point);
}
}
let Some(current) = history.last() else {
return sample;
};
if let Some(baseline) = cycle_baseline {
let Some(usage) = current.usage.difference(&baseline.usage) else {
sample.discontinuous = true;
return sample;
};
sample.cycle_usage = usage;
}
if anchors.len() < 2 {
return sample;
}
@@ -1057,6 +1057,8 @@ impl AccountsService for DefaultAccountsService {
) else {
continue;
};
let reset_at = query.range.end;
let window_duration = reset_at - query.range.start;
query.range.start = query.range.start.max(stored.account.created_at);
if query.range.start >= observed
|| observed > now
@@ -1066,7 +1068,6 @@ impl AccountsService for DefaultAccountsService {
{
continue;
}
let reset_at = query.range.end;
query.range.end = observed;
let history = self
.accounts
@@ -1074,27 +1075,37 @@ impl AccountsService for DefaultAccountsService {
.await
.map_err(|error| map_store_error(error, "forecast paired usage"))?;
let mut points = Vec::new();
let mut interrupted = false;
let mut plan_changed = false;
for point in history.points {
let Some(fact) =
provider.quota_forecast_observation(&point.provider_observation, window)
else {
continue;
};
let same_plan = quota
.plan_type
.as_deref()
.zip(fact.plan_type.as_deref())
.is_some_and(|(current, previous)| current.eq_ignore_ascii_case(previous));
// 仅容纳已观测到的秒级量化抖动,不用宽时间容差合并实际重置
// 不匹配的段截断基线;之后的有效观测可以重新积累
if !same_plan || (fact.reset_at - reset_at).abs() > Duration::seconds(2) {
let (Some(current_plan), Some(previous_plan)) =
(quota.plan_type.as_deref(), fact.plan_type.as_deref())
else {
continue;
};
let Some(start) = fact.reset_at.checked_sub_signed(window_duration) else {
continue;
};
if point.completed_at >= observed
|| point.started_at < query.range.start
|| point.completed_at < start
|| point.completed_at >= fact.reset_at
{
continue;
}
if !current_plan.eq_ignore_ascii_case(previous_plan) {
points.clear();
interrupted = true;
plan_changed = true;
continue;
}
points.push(QuotaForecastPoint {
started_at: point.started_at,
observed_at: point.completed_at,
reset_at: fact.reset_at,
used_percent: fact.used_percent,
usage: point.usage,
});
@@ -1103,13 +1114,15 @@ impl AccountsService for DefaultAccountsService {
window.key.clone(),
query.range.start,
QuotaForecastPoint {
started_at: observed,
observed_at: observed,
reset_at,
used_percent: percent,
usage: history.usage,
},
points,
history.pending_request_count,
interrupted,
plan_changed,
);
samples.push(sample);
}
@@ -12,7 +12,9 @@ fn time(minutes: i64) -> DateTime<Utc> {
fn point(minute: i64, percent: f64, tokens: u64) -> QuotaForecastPoint {
QuotaForecastPoint {
started_at: time(minute) - Duration::seconds(1),
observed_at: time(minute),
reset_at: time(7 * 24 * 60),
used_percent: percent,
usage: QuotaForecastUsage {
request_count: minute as u64,
@@ -188,7 +190,41 @@ fn repeated_same_window_recoveries_preserve_recorded_cycle_usage() {
}
#[test]
fn changed_window_restarts_cycle_totals_at_first_matching_observation() {
fn an_expired_deadline_alone_cannot_cut_totals_or_anchor_remaining_usage() {
let mut older = point(1, 15.0, 100);
older.reset_at = time(3);
let mut previous = point(2, 20.0, 200);
previous.reset_at = time(3) + Duration::seconds(4);
let first = point(4, 25.0, 700);
let sample = select(vec![older, previous, first], point(6, 35.0, 900));
assert_eq!(sample.cycle_usage.tokens, 900);
assert_eq!(sample.baseline_percent, 25.0);
assert_eq!(sample.usage.tokens, 200);
}
#[test]
fn copied_concurrent_or_contradictory_readings_cannot_confirm_a_recovery() {
let mut older = point(1, 75.0, 100);
older.reset_at = time(3);
let mut previous = point(2, 85.0, 200);
previous.reset_at = time(3);
let first = point(4, 5.0, 300);
let current = point(6, 40.0, 900);
let mut overlapping = point(5, 20.0, 400);
overlapping.started_at = first.started_at;
for extra in [None, Some(overlapping), Some(current.clone())] {
let mut points = vec![older.clone(), previous.clone(), first.clone()];
points.extend(extra);
let sample = select(points, current.clone());
assert_eq!(sample.cycle_usage.tokens, 900);
assert!(sample.discontinuous);
}
let sample = select(vec![older, previous, first, point(5, 86.0, 400)], current);
assert_eq!(sample.cycle_usage.tokens, 900);
}
#[test]
fn changed_plan_restarts_cycle_totals_at_first_matching_observation() {
let sample = select_forecast_sample(
"week".to_owned(),
time(0),
@@ -2576,7 +2576,9 @@ async fn quota_forecast_same_window_reversal_keeps_recorded_usage_in_the_report(
usage: usage(index as u64 + 1, tokens),
provider_observation: ProviderDocument::new(OpaqueProviderData::new(
serde_json::json!({
"percent": percent, "reset": reset.to_rfc3339(), "plan": "pro",
"percent": percent,
"reset": (reset + TimeDelta::seconds(17 * index as i64)).to_rfc3339(),
"plan": "pro",
})
.as_object()
.unwrap()
@@ -2605,14 +2607,14 @@ async fn quota_forecast_same_window_reversal_keeps_recorded_usage_in_the_report(
}
#[tokio::test]
async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_changed_bucket() {
async fn quota_forecast_keeps_cycle_totals_through_reset_jitter_and_restarts_after_a_reset() {
use gateway_admin::model::quota_forecast_sampling::QuotaForecastHistoryPoint;
let now = Utc::now();
let observed = now - TimeDelta::minutes(1);
let reset = now + TimeDelta::days(1);
let added = now - TimeDelta::hours(5);
let provider = FakeProviderAdmin::new("openai", events());
provider.set_quota(ProviderQuota {
let mut quota = ProviderQuota {
upstream: None,
credits: None,
plan_type: Some("pro".to_owned()),
@@ -2627,67 +2629,102 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_
role: None,
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
window_seconds: Some(604_800),
used_percent: Some(40.0),
used_percent: Some(65.0),
reset_at: Some(reset),
limit_reached: false,
local_usage: None,
provider_data: None,
}],
..empty_quota()
});
};
let mut account = account_record("openai");
account.created_at = added;
let store = FakeAccountStore::with_account(account, events());
let make_point = |hours, percent, tokens, delta| {
QuotaForecastHistoryPoint {
started_at: now - TimeDelta::hours(hours) - TimeDelta::seconds(10),
completed_at: now - TimeDelta::hours(hours),
usage: QuotaForecastUsage { request_count: tokens / 100, tokens, ..Default::default() },
provider_observation: ProviderDocument::new(OpaqueProviderData::new(serde_json::json!({
"percent": percent, "reset": (reset + TimeDelta::seconds(delta)).to_rfc3339(), "plan": "pro",
}).as_object().unwrap().clone())),
}
let usage = |tokens| QuotaForecastUsage {
request_count: tokens / 100,
tokens,
known_cost_count: tokens / 100,
usd: tokens as f64 / 100.0,
..Default::default()
};
let point_with_plan = |hours, percent, tokens, delta, plan: Option<&str>| {
let observation = serde_json::json!({
"percent": percent,
"reset": (reset + TimeDelta::seconds(delta)).to_rfc3339(),
"plan": plan,
});
QuotaForecastHistoryPoint {
started_at: now - TimeDelta::hours(hours) - TimeDelta::seconds(10),
completed_at: now - TimeDelta::hours(hours),
usage: usage(tokens),
provider_observation: ProviderDocument::new(OpaqueProviderData::new(
observation.as_object().unwrap().clone(),
)),
}
};
let make_point =
|hours, percent, tokens, delta| point_with_plan(hours, percent, tokens, delta, Some("pro"));
*store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory {
points: vec![
make_point(3, 20.0, 1_000, 1),
make_point(2, 30.0, 1_500, -1),
make_point(4, 20.0, 20_000, 0),
make_point(3, 58.0, 62_000, 17),
make_point(2, 58.0, 62_500, 0),
],
usage: QuotaForecastUsage {
request_count: 20,
tokens: 2_000,
..Default::default()
},
usage: usage(70_600),
pending_request_count: 1,
};
let services = accounts_service(provider, store.clone()).await;
let result = services
.accounts()
.quota_forecast(&ProviderAccountId::new("acct_test").unwrap())
.await
.unwrap();
assert!(result.forecasts[0].unavailable_reason.is_none());
assert_eq!(result.forecasts[0].estimated_tokens, Some(5_000));
let services = accounts_service(provider.clone(), store.clone()).await;
let account_id = ProviderAccountId::new("acct_test").unwrap();
let read_report = async || {
services
.accounts()
.quota_forecast(&account_id)
.await
.unwrap()
};
// 回放 #759 的金额和百分比,历史与快照各自报告的到期时间偏移时仍应保留整段累计
for offset in [0, 3, 6, 7, 17, -23] {
quota.windows[0].reset_at = Some(reset + TimeDelta::seconds(offset));
provider.set_quota(quota.clone());
let report = read_report().await;
let week = &report.forecasts[0];
let source = week.source.as_ref().unwrap();
assert_eq!(source.tokens, Some(70_600));
assert_eq!(source.usd, Some(706.0));
assert_eq!(week.estimated_tokens, Some(109_956));
assert!((week.estimated_usd.unwrap() - (706.0 + 506.0 * 35.0 / 45.0)).abs() < 1e-9);
assert!(week.unavailable_reason.is_none());
}
assert_eq!(store.quota_window_queries()[0].range.start, added);
store
.quota_forecast_history
.lock()
.unwrap()
.points
.push(make_point(1, 35.0, 1_700, 3_600));
let result = services
.accounts()
.quota_forecast(&ProviderAccountId::new("acct_test").unwrap())
.await
.unwrap();
assert!(result.forecasts[0].estimated_tokens.is_none());
assert!(
result.forecasts[0]
.unavailable_reason
.unwrap()
.contains("不连续")
.push(make_point(1, 62.0, 66_000, 3_600));
let result = read_report().await;
assert_eq!(
result.forecasts[0].source.as_ref().unwrap().tokens,
Some(70_600)
);
// 新额度段已有足够观测后恢复预测,但总量不能带回重置前的累计用量
assert!(result.forecasts[0].estimated_tokens.is_some());
// 两份旧读数的截止时刻均已过去,新请求开始于旧窗口结束后
quota.windows[0].used_percent = Some(40.0);
provider.set_quota(quota.clone());
*store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory {
points: vec![
make_point(4, 75.0, 1_000, -90_000),
make_point(2, 85.0, 1_500, -90_021),
],
usage: usage(2_000),
..Default::default()
};
let waiting = read_report().await;
assert_eq!(
waiting.forecasts[0].source.as_ref().unwrap().tokens,
Some(2_000)
);
assert!(waiting.forecasts[0].estimated_tokens.is_none());
let mut first = make_point(1, 5.0, 1_750, 0);
first.completed_at = now - TimeDelta::minutes(45);
first.started_at = first.completed_at - TimeDelta::seconds(10);
@@ -2700,15 +2737,42 @@ async fn quota_forecast_mid_cycle_sampling_accepts_small_reset_jitter_but_not_a_
.unwrap()
.points
.extend([first, next]);
let result = services
.accounts()
.quota_forecast(&ProviderAccountId::new("acct_test").unwrap())
.await
.unwrap();
let result = read_report().await;
let cycle = &result.forecasts[0];
assert!(cycle.unavailable_reason.is_none());
assert_eq!(cycle.source.as_ref().unwrap().tokens, Some(250));
assert_eq!(cycle.estimated_tokens, Some(679));
for (plan, expected_tokens) in [(None, 2_000), (Some("plus"), 500)] {
store.quota_forecast_history.lock().unwrap().points = vec![
make_point(4, 20.0, 1_000, 0),
point_with_plan(3, 25.0, 1_200, 0, plan),
make_point(2, 30.0, 1_500, 0),
];
let report = read_report().await;
assert_eq!(
report.forecasts[0].source.as_ref().unwrap().tokens,
Some(expected_tokens)
);
}
// 上游提前给出新的七天窗口时,直接从新起点查询,首条配对样本前的用量也留在累计中
let new_start = now - TimeDelta::hours(1);
quota.windows[0].reset_at = Some(new_start + TimeDelta::days(7));
provider.set_quota(quota);
*store.quota_forecast_history.lock().unwrap() = QuotaForecastHistory {
usage: usage(400),
..Default::default()
};
let report = read_report().await;
assert_eq!(
store.quota_window_queries().last().unwrap().range.start,
new_start
);
assert_eq!(
report.forecasts[0].source.as_ref().unwrap().tokens,
Some(400)
);
}
#[tokio::test]
@@ -643,7 +643,7 @@ async fn dropping_the_connection_aborts_and_drops_the_socket_owner() {
assert!(cancellation.is_cancelled());
}
#[tokio::test(start_paused = true)]
#[tokio::test]
async fn idle_connection_reaches_the_official_limit_without_starting_an_execution() {
let trace = Arc::new(AtomicFailureTrace::default());
let execution = Arc::new(AtomicFailureExecution {
@@ -672,8 +672,11 @@ async fn idle_connection_reaches_the_official_limit_without_starting_an_executio
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
// 真实 socket 读写期间使用实时时钟,避免 I/O 等待自动耗尽虚拟期限
tokio::time::pause();
tokio::time::advance(Duration::from_secs(60 * 60)).await;
tokio::task::yield_now().await;
tokio::time::resume();
let text = loop {
let message = tokio::time::timeout(Duration::from_secs(1), socket.next())
.await
+1 -1
View File
@@ -1855,7 +1855,7 @@ OpenAI 授权码交换与令牌刷新请求的连接超时为 30 秒,单次上
样本仅使用当前额度窗口内、账号加入后开始且截至观测时间已完整交付的请求,缓存命中不重复计入输入
重置或套餐变化后重新积累有效进度,中途加入的账号可在新基线后采样,无需等待下一次重置,职责与断点处理见[额度预测架构](architecture.md#额度预测)
重置或套餐变化后重新积累有效进度,中途加入的账号可在新基线后采样。单条历史读数的重置时间偏移不会截断已记录用量。疑似跨周期且仍待确认时,接口保留来源用量并暂停估算,详见[额度预测架构](architecture.md#额度预测)
预测不参与调度或账单结算,站外消耗、日志清理、异步延迟、未成功交付的上游消耗和模型组合变化均可能造成误差
+3 -1
View File
@@ -1165,7 +1165,9 @@ Admin 按卡片与所选指标向 Store 请求计数、金额或分布,缓存
- 同一额度窗口内,每累计至少 5 个百分点形成一段,使用最近 3 个完整段及尾部计算合并比值,不平均逐请求的小分母比值
- 重复读数不增加段数,大于 1 个百分点的额度回落、累计计数倒退或窗口身份变化会中断采样
- 同窗口回落保留已记录累计;触顶后恢复可用不代表新周期,不能据此重设累计基线
- OpenAI 的窗口匹配复用限流解析器,核对桶、槽位、时长和明确套餐,只容许最多 2 秒的重置时间抖动
- 本周期累计从当前快照的窗口起点查询。OpenAI 历史读数须匹配额度桶、槽位、时长和明确套餐。重置时间的偏移本身不触发换段,已过期的读数不参与剩余量估算
- 当前窗口覆盖旧读数时,要有两次独立请求报告的到期时刻均已过去,且随后两次独立请求都显示已用比例回落超过 1 个百分点,才从新段首条读数重新累计。等待第二条读数期间保留累计并暂停估算
- 跳过缺少套餐的历史读数,确认套餐变化后从新段重新累计
- 重置后从新段重新累计,无法精确归属的用量不补算;只有账号在窗口开始前加入且无已知断点时,才允许以窗口累计量替代增量采样
- 历史点抽样不缩减区间累计用量,同一完成时间共用同一累计值
+4 -4
View File
@@ -1,7 +1,7 @@
version: 3.54.2
published:
'3':
stable: 3.54.1
rc: 3.54.1
beta: 3.54.1
alpha: 3.54.1
stable: 3.54.2
rc: 3.54.2
beta: 3.54.2
alpha: 3.54.2