mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-18 16:06:47 +00:00
feat(openai): simplify account warmup scheduling
Use a single five-field Cron expression and allow lightweight warmup when quota windows lack active evidence, without extra quota refreshes. Reset saved schedules and disable warmup in migration 0040 while keeping the configured model. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Generated
+1
@@ -1773,6 +1773,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"cron",
|
||||
"event-listener",
|
||||
"futures",
|
||||
"futures-timer",
|
||||
|
||||
@@ -70,7 +70,7 @@ pub struct RuntimeSettingsValues {
|
||||
pub audit_retention_days: u32,
|
||||
pub account_error_protection: gateway_core::account::AccountErrorProtection,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_cron: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
pub reset_credit_lead_minutes: u32,
|
||||
pub reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
@@ -233,7 +233,7 @@ impl RuntimeSettingsValues {
|
||||
self.account_error_protection.validate()?;
|
||||
gateway_core::provider_ports::ProviderWarmupPolicy::validate_values(
|
||||
self.account_warmup_enabled,
|
||||
&self.account_warmup_schedule_time,
|
||||
&self.account_warmup_cron,
|
||||
self.account_warmup_model.as_deref(),
|
||||
)?;
|
||||
Ok(())
|
||||
|
||||
@@ -1183,7 +1183,7 @@ impl SettingsStore for StaticSettingsStore {
|
||||
audit_retention_days: 30,
|
||||
account_error_protection: gateway_core::account::AccountErrorProtection::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: self
|
||||
|
||||
@@ -73,7 +73,7 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
|
||||
..Default::default()
|
||||
},
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
|
||||
@@ -1437,7 +1437,7 @@ impl SettingsStore for FixtureSettingsStore {
|
||||
audit_retention_days: 30,
|
||||
account_error_protection: gateway_core::account::AccountErrorProtection::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
|
||||
@@ -351,7 +351,7 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
|
||||
account_error_protection:
|
||||
gateway_core::account::AccountErrorProtection::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
|
||||
@@ -61,7 +61,7 @@ pub struct RuntimeSettingsFields {
|
||||
pub audit_retention_days: u64,
|
||||
pub account_error_protection: gateway_core::account::AccountErrorProtection,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_cron: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
pub reset_credit_lead_minutes: u32,
|
||||
pub reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
@@ -773,7 +773,7 @@ impl RuntimeSettingsFields {
|
||||
.map_err(|_| WireValidationError::new("settingsAuditRetentionOverflow"))?,
|
||||
account_error_protection: self.account_error_protection,
|
||||
account_warmup_enabled: self.account_warmup_enabled,
|
||||
account_warmup_schedule_time: self.account_warmup_schedule_time,
|
||||
account_warmup_cron: self.account_warmup_cron,
|
||||
account_warmup_model: self.account_warmup_model,
|
||||
reset_credit_lead_minutes: self.reset_credit_lead_minutes,
|
||||
reset_credit_max_weekly_remaining_percent: self
|
||||
@@ -812,7 +812,7 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
account_error_protection: settings.account_error_protection,
|
||||
account_warmup_enabled: settings.account_warmup_enabled,
|
||||
account_warmup_schedule_time: settings.account_warmup_schedule_time,
|
||||
account_warmup_cron: settings.account_warmup_cron,
|
||||
account_warmup_model: settings.account_warmup_model,
|
||||
reset_credit_lead_minutes: settings.reset_credit_lead_minutes,
|
||||
reset_credit_max_weekly_remaining_percent: settings
|
||||
@@ -828,7 +828,7 @@ fn settings_validation_error(field: &'static str) -> WireValidationError {
|
||||
"account_warmup_model" => "accountWarmupModel",
|
||||
"reset_credit_lead_minutes" => "resetCreditLeadMinutes",
|
||||
"reset_credit_max_weekly_remaining_percent" => "resetCreditMaxWeeklyRemainingPercent",
|
||||
"account_warmup_schedule_time" => "accountWarmupScheduleTime",
|
||||
"account_warmup_cron" => "accountWarmupCron",
|
||||
"audit_retention_days" => "auditRetentionDays",
|
||||
"key_queue_timeout_seconds" => "keyQueueTimeoutSeconds",
|
||||
"account_queue_timeout_seconds" => "accountQueueTimeoutSeconds",
|
||||
|
||||
@@ -671,7 +671,7 @@ impl SettingsStore for MemorySettingsStore {
|
||||
audit_retention_days: command.values.audit_retention_days,
|
||||
account_error_protection: command.values.account_error_protection,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
@@ -1914,7 +1914,7 @@ fn test_runtime_settings() -> RuntimeSettings {
|
||||
audit_retention_days: 90,
|
||||
account_error_protection: Default::default(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
|
||||
@@ -70,7 +70,7 @@ fn update_body() -> Value {
|
||||
"auditRetentionDays": 91,
|
||||
"accountErrorProtection": gateway_core::account::AccountErrorProtection::default(),
|
||||
"accountWarmupEnabled": false,
|
||||
"accountWarmupScheduleTime": "08:00",
|
||||
"accountWarmupCron": "0 8 * * *",
|
||||
"accountWarmupModel": null,
|
||||
"resetCreditLeadMinutes": 10,
|
||||
"resetCreditMaxWeeklyRemainingPercent": null
|
||||
@@ -402,6 +402,33 @@ fn settings_request_accepts_unlimited_default_account_concurrency() {
|
||||
request.validate().expect("zero means unlimited");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_validates_cron_and_allows_disabled_empty_schedule() {
|
||||
for (enabled, cron, valid) in [
|
||||
(false, "", true),
|
||||
(true, "", false),
|
||||
(false, "08:00", false),
|
||||
(true, "0 8 * * MON-FRI", true),
|
||||
(true, "60 8 * * *", false),
|
||||
(true, "0 8 * * *\n30 13 * * *", false),
|
||||
] {
|
||||
let mut body = update_body();
|
||||
body["accountWarmupEnabled"] = json!(enabled);
|
||||
body["accountWarmupCron"] = json!(cron);
|
||||
body["accountWarmupModel"] = json!("test-model");
|
||||
let request: UpdateRuntimeSettingsRequest = serde_json::from_value(body).unwrap();
|
||||
let result = request.validate();
|
||||
assert_eq!(result.is_ok(), valid, "{enabled}: {cron:?}");
|
||||
if let Err(error) = result {
|
||||
assert_eq!(error.field(), "accountWarmupCron");
|
||||
}
|
||||
}
|
||||
let mut legacy = update_body();
|
||||
legacy.as_object_mut().unwrap().remove("accountWarmupCron");
|
||||
legacy["accountWarmupScheduleTime"] = json!("08:00");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(legacy).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_requires_model_when_warmup_is_enabled() {
|
||||
let mut body = update_body();
|
||||
|
||||
@@ -17,6 +17,7 @@ base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
chrono-tz = { version = "0.10", features = ["serde"] }
|
||||
cron = "0.17"
|
||||
event-listener = "5.4.2"
|
||||
futures = "0.3.32"
|
||||
http = "1.5.0"
|
||||
|
||||
@@ -28,9 +28,7 @@ pub use cooldown::{
|
||||
mod refresh;
|
||||
pub use refresh::{ProviderRefreshPolicy, provider_refresh_backoff_at, provider_refresh_retry_at};
|
||||
mod runtime_policy;
|
||||
pub use runtime_policy::{
|
||||
ProviderRuntimePolicyPort, ProviderWarmupPolicy, valid_warmup_schedule_time,
|
||||
};
|
||||
pub use runtime_policy::{ProviderRuntimePolicyPort, ProviderWarmupPolicy, valid_warmup_cron};
|
||||
mod oauth;
|
||||
pub use oauth::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
|
||||
@@ -52,39 +52,28 @@ pub trait ProviderRuntimePolicyPort: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验每日预激活时间格式,如 "08:00" 或 "08:00,13:00"
|
||||
/// 校验五段 Cron 预热计划
|
||||
#[must_use]
|
||||
pub fn valid_warmup_schedule_time(value: &str) -> bool {
|
||||
if value.is_empty()
|
||||
|| value.len() > 255
|
||||
pub fn valid_warmup_cron(value: &str) -> bool {
|
||||
parse_warmup_cron(value).is_some()
|
||||
}
|
||||
|
||||
fn parse_warmup_cron(value: &str) -> Option<cron::Schedule> {
|
||||
if value.len() > 255
|
||||
|| value != value.trim()
|
||||
|| value.chars().any(char::is_control)
|
||||
|| value.split_whitespace().count() != 5
|
||||
{
|
||||
return false;
|
||||
return None;
|
||||
}
|
||||
for part in value.split(',') {
|
||||
let bytes = part.as_bytes();
|
||||
if bytes.len() != 5 || bytes[2] != b':' {
|
||||
return false;
|
||||
}
|
||||
let Ok(hour) = part[0..2].parse::<u32>() else {
|
||||
return false;
|
||||
};
|
||||
let Ok(minute) = part[3..5].parse::<u32>() else {
|
||||
return false;
|
||||
};
|
||||
if hour > 23 || minute > 59 {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
format!("0 {value}").parse().ok()
|
||||
}
|
||||
|
||||
/// 账号模型预激活(预热)策略;来源于 `runtime_settings`
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderWarmupPolicy {
|
||||
enabled: bool,
|
||||
schedule_time: String,
|
||||
cron: String,
|
||||
model: Option<String>,
|
||||
}
|
||||
|
||||
@@ -92,11 +81,11 @@ impl ProviderWarmupPolicy {
|
||||
/// 保存设置与创建预热策略使用相同的时间表和模型约束
|
||||
pub fn validate_values(
|
||||
enabled: bool,
|
||||
schedule_time: &str,
|
||||
cron: &str,
|
||||
model: Option<&str>,
|
||||
) -> Result<(), &'static str> {
|
||||
if !valid_warmup_schedule_time(schedule_time) {
|
||||
return Err("account_warmup_schedule_time");
|
||||
if (enabled || !cron.is_empty()) && !valid_warmup_cron(cron) {
|
||||
return Err("account_warmup_cron");
|
||||
}
|
||||
if (enabled && model.is_none()) || !valid_optional_probe_model(model) {
|
||||
return Err("account_warmup_model");
|
||||
@@ -106,10 +95,10 @@ impl ProviderWarmupPolicy {
|
||||
|
||||
pub fn try_new(
|
||||
enabled: bool,
|
||||
schedule_time: String,
|
||||
cron: String,
|
||||
model: Option<String>,
|
||||
) -> Result<Self, ProviderStoreError> {
|
||||
if Self::validate_values(enabled, &schedule_time, model.as_deref()).is_err() {
|
||||
if Self::validate_values(enabled, &cron, model.as_deref()).is_err() {
|
||||
return Err(ProviderStoreError::new(
|
||||
ProviderStoreErrorKind::InvalidData,
|
||||
"validate warmup policy",
|
||||
@@ -117,7 +106,7 @@ impl ProviderWarmupPolicy {
|
||||
}
|
||||
Ok(Self {
|
||||
enabled,
|
||||
schedule_time,
|
||||
cron,
|
||||
model,
|
||||
})
|
||||
}
|
||||
@@ -126,7 +115,7 @@ impl ProviderWarmupPolicy {
|
||||
pub const fn disabled() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
schedule_time: String::new(),
|
||||
cron: String::new(),
|
||||
model: None,
|
||||
}
|
||||
}
|
||||
@@ -137,8 +126,8 @@ impl ProviderWarmupPolicy {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn schedule_time(&self) -> &str {
|
||||
&self.schedule_time
|
||||
pub fn cron(&self) -> &str {
|
||||
&self.cron
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -146,19 +135,10 @@ impl ProviderWarmupPolicy {
|
||||
self.model.as_deref()
|
||||
}
|
||||
|
||||
/// 解析每日时间列表,返回如 `vec![(8, 0)]`
|
||||
/// 匹配部署时区中的整分钟时刻,时区重叠及执行去重由 Worker 与存储端口负责
|
||||
#[must_use]
|
||||
pub fn scheduled_times(&self) -> Vec<(u32, u32)> {
|
||||
self.schedule_time
|
||||
.split(',')
|
||||
.filter_map(|part| {
|
||||
let part = part.trim();
|
||||
let mut iter = part.split(':');
|
||||
let hour = iter.next()?.parse::<u32>().ok()?;
|
||||
let minute = iter.next()?.parse::<u32>().ok()?;
|
||||
Some((hour, minute))
|
||||
})
|
||||
.collect()
|
||||
pub fn matches_slot(&self, slot: chrono::NaiveDateTime) -> bool {
|
||||
parse_warmup_cron(&self.cron).is_some_and(|schedule| schedule.includes(slot.and_utc()))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -221,30 +221,72 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_schedule_time_and_policy_validation() {
|
||||
use gateway_core::provider_ports::{ProviderWarmupPolicy, valid_warmup_schedule_time};
|
||||
|
||||
assert!(valid_warmup_schedule_time("08:00"));
|
||||
assert!(valid_warmup_schedule_time("08:00,13:00"));
|
||||
assert!(valid_warmup_schedule_time("00:00,23:59"));
|
||||
assert!(!valid_warmup_schedule_time(""));
|
||||
assert!(!valid_warmup_schedule_time("8:00"));
|
||||
assert!(!valid_warmup_schedule_time("24:00"));
|
||||
assert!(!valid_warmup_schedule_time("08:60"));
|
||||
assert!(!valid_warmup_schedule_time("08:00,"));
|
||||
assert!(!valid_warmup_schedule_time("08:00, 13:00"));
|
||||
fn warmup_cron_validation_and_calendar_matching() {
|
||||
use chrono::NaiveDate;
|
||||
use gateway_core::provider_ports::{ProviderWarmupPolicy, valid_warmup_cron};
|
||||
|
||||
for value in [
|
||||
"0 8 * * *",
|
||||
"*/15 8-18 * * MON-FRI",
|
||||
"0 8,13 * * MON-FRI",
|
||||
"0 0 29 FEB *",
|
||||
] {
|
||||
assert!(valid_warmup_cron(value), "{value}");
|
||||
}
|
||||
for value in [
|
||||
"",
|
||||
"08:00",
|
||||
"08:00,13:00",
|
||||
"60 8 * * *",
|
||||
"0 24 * * *",
|
||||
"0 8 * *",
|
||||
"0 0 8 * * *",
|
||||
"0 8 * * * *",
|
||||
"@daily",
|
||||
"0 8 * * *\n30 13 * * *",
|
||||
"0 8 * * *\r\n30 13 * * *",
|
||||
"0 8 * * *\t",
|
||||
" 0 8 * * *",
|
||||
"0 8 * * *\n",
|
||||
] {
|
||||
assert!(!valid_warmup_cron(value), "{value:?}");
|
||||
}
|
||||
let policy = ProviderWarmupPolicy::try_new(
|
||||
true,
|
||||
"08:00,13:00".to_owned(),
|
||||
"0 8,13 * * MON-FRI".to_owned(),
|
||||
Some("test-model".to_owned()),
|
||||
)
|
||||
.expect("valid warmup policy");
|
||||
.unwrap();
|
||||
assert!(policy.enabled());
|
||||
assert_eq!(policy.schedule_time(), "08:00,13:00");
|
||||
assert_eq!(policy.cron(), "0 8,13 * * MON-FRI");
|
||||
assert_eq!(policy.model(), Some("test-model"));
|
||||
assert_eq!(policy.scheduled_times(), vec![(8, 0), (13, 0)]);
|
||||
|
||||
for (day, hour, minute, expected) in [
|
||||
(9, 8, 0, true),
|
||||
(9, 13, 0, true),
|
||||
(9, 8, 30, false),
|
||||
(9, 13, 30, false),
|
||||
(14, 8, 0, false),
|
||||
(14, 13, 30, false),
|
||||
] {
|
||||
let slot = NaiveDate::from_ymd_opt(2026, 11, day)
|
||||
.unwrap()
|
||||
.and_hms_opt(hour, minute, 0)
|
||||
.unwrap();
|
||||
assert_eq!(policy.matches_slot(slot), expected, "{slot}");
|
||||
}
|
||||
assert!(ProviderWarmupPolicy::try_new(false, String::new(), None).is_ok());
|
||||
assert!(
|
||||
ProviderWarmupPolicy::try_new(true, String::new(), Some("test-model".to_owned())).is_err()
|
||||
);
|
||||
assert!(ProviderWarmupPolicy::try_new(false, "08:00".to_owned(), None).is_err());
|
||||
assert!(!valid_warmup_cron(&format!("0{}8 * * *", " ".repeat(256))));
|
||||
let leap = ProviderWarmupPolicy::try_new(false, "0 0 29 FEB *".to_owned(), None).unwrap();
|
||||
let slot = NaiveDate::from_ymd_opt(2028, 2, 29)
|
||||
.unwrap()
|
||||
.and_hms_opt(0, 0, 0)
|
||||
.unwrap();
|
||||
assert!(leap.matches_slot(slot));
|
||||
assert!(!leap.matches_slot(slot + chrono::Duration::days(1)));
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "invalid".to_owned(), None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "08:00".to_owned(), None).is_err());
|
||||
assert!(ProviderWarmupPolicy::try_new(true, "0 8 * * *".to_owned(), None).is_err());
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ pub struct RuntimeSettings {
|
||||
pub audit_retention_days: u32,
|
||||
pub account_error_protection: AccountErrorProtection,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_cron: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
pub reset_credit_lead_minutes: u32,
|
||||
pub reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
@@ -199,7 +199,7 @@ pub struct ReplaceRuntimeSettings {
|
||||
pub audit_retention_days: u32,
|
||||
pub account_error_protection: AccountErrorProtection,
|
||||
pub account_warmup_enabled: bool,
|
||||
pub account_warmup_schedule_time: String,
|
||||
pub account_warmup_cron: String,
|
||||
pub account_warmup_model: Option<String>,
|
||||
pub reset_credit_lead_minutes: u32,
|
||||
pub reset_credit_max_weekly_remaining_percent: Option<u32>,
|
||||
@@ -399,7 +399,7 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
|
||||
audit_retention_days: settings.audit_retention_days,
|
||||
account_error_protection: settings.account_error_protection,
|
||||
account_warmup_enabled: settings.account_warmup_enabled,
|
||||
account_warmup_schedule_time: settings.account_warmup_schedule_time,
|
||||
account_warmup_cron: settings.account_warmup_cron,
|
||||
account_warmup_model: settings.account_warmup_model,
|
||||
reset_credit_lead_minutes: settings.reset_credit_lead_minutes,
|
||||
reset_credit_max_weekly_remaining_percent: settings
|
||||
|
||||
@@ -157,7 +157,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
account_warmup_enabled, account_warmup_cron, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
@@ -296,7 +296,7 @@ impl ProviderRuntimePolicyPort for PgRuntimeSettingsRepository {
|
||||
.map_err(|source| crate::provider_unavailable("load warmup policy", source))?;
|
||||
ProviderWarmupPolicy::try_new(
|
||||
settings.values.account_warmup_enabled,
|
||||
settings.values.account_warmup_schedule_time,
|
||||
settings.values.account_warmup_cron,
|
||||
settings.values.account_warmup_model,
|
||||
)
|
||||
})
|
||||
@@ -313,7 +313,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
|
||||
audit_retention_days, min_codex_desktop_version,
|
||||
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
|
||||
account_error_protection_json, account_error_protection_revision,
|
||||
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
account_warmup_enabled, account_warmup_cron, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
|
||||
from runtime_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
@@ -381,7 +381,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
responses_max_decompressed_body_bytes = $17,
|
||||
provider_request_profiles_json = (provider_request_profiles_json - $18::text[]) || $19::jsonb,
|
||||
account_warmup_enabled = $20,
|
||||
account_warmup_schedule_time = $21,
|
||||
account_warmup_cron = $21,
|
||||
account_warmup_model = $22,
|
||||
smart_scheduling_json = $23,
|
||||
openai_guardian_reserved_concurrency = $24,
|
||||
@@ -426,7 +426,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
.bind(request_profile_deletions)
|
||||
.bind(sqlx::types::Json(request_profile_updates))
|
||||
.bind(update.values.account_warmup_enabled)
|
||||
.bind(&update.values.account_warmup_schedule_time)
|
||||
.bind(&update.values.account_warmup_cron)
|
||||
.bind(update.values.account_warmup_model.as_deref())
|
||||
.bind(sqlx::types::Json(update.values.smart_scheduling))
|
||||
.bind(i64::from(update.values.openai_guardian_reserved_concurrency))
|
||||
@@ -537,7 +537,7 @@ struct RuntimeSettingsRow {
|
||||
account_error_protection_revision: i64,
|
||||
account_error_protection_json: sqlx::types::Json<gateway_core::account::AccountErrorProtection>,
|
||||
account_warmup_enabled: bool,
|
||||
account_warmup_schedule_time: String,
|
||||
account_warmup_cron: String,
|
||||
account_warmup_model: Option<String>,
|
||||
reset_credit_lead_minutes: i32,
|
||||
reset_credit_max_weekly_remaining_percent: Option<i32>,
|
||||
@@ -606,7 +606,7 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
|
||||
)?,
|
||||
account_error_protection: row.account_error_protection_json.0,
|
||||
account_warmup_enabled: row.account_warmup_enabled,
|
||||
account_warmup_schedule_time: row.account_warmup_schedule_time,
|
||||
account_warmup_cron: row.account_warmup_cron,
|
||||
account_warmup_model: row.account_warmup_model,
|
||||
reset_credit_lead_minutes: u32::try_from(row.reset_credit_lead_minutes)
|
||||
.map_err(|_| invalid_numeric())?,
|
||||
|
||||
@@ -47,7 +47,7 @@ pub(super) fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettin
|
||||
audit_retention_days: 90,
|
||||
account_error_protection: enabled_protection(),
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_cron: "0 8 * * *".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
@@ -180,7 +180,7 @@ async fn warmup_settings_round_trip_with_database_constraint() {
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let mut update = settings_with_margin(3_600);
|
||||
update.values.account_warmup_enabled = true;
|
||||
update.values.account_warmup_schedule_time = "08:00,13:00".to_owned();
|
||||
update.values.account_warmup_cron = "0 8,13 * * MON-FRI".to_owned();
|
||||
update.values.account_warmup_model = Some("test-model".to_owned());
|
||||
repository
|
||||
.update_runtime_settings(update)
|
||||
@@ -192,7 +192,7 @@ async fn warmup_settings_round_trip_with_database_constraint() {
|
||||
.await
|
||||
.expect("load warmup settings");
|
||||
assert!(settings.values.account_warmup_enabled);
|
||||
assert_eq!(settings.values.account_warmup_schedule_time, "08:00,13:00");
|
||||
assert_eq!(settings.values.account_warmup_cron, "0 8,13 * * MON-FRI");
|
||||
assert_eq!(
|
||||
settings.values.account_warmup_model.as_deref(),
|
||||
Some("test-model")
|
||||
|
||||
@@ -52,7 +52,6 @@ pub struct CodexQuotaWindow {
|
||||
window_seconds: Option<u64>,
|
||||
used_percent: Option<f64>,
|
||||
reset_at: Option<DateTime<Utc>>,
|
||||
reset_after_seconds: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
@@ -103,11 +102,6 @@ impl CodexQuotaWindow {
|
||||
self.reset_at
|
||||
}
|
||||
|
||||
/// 上游观察时的剩余秒数,不能用当前时间重新推算空闲窗口
|
||||
pub(super) const fn reset_after_seconds(&self) -> Option<u64> {
|
||||
self.reset_after_seconds
|
||||
}
|
||||
|
||||
/// 该窗口是否已触顶(`limit_reached` 或 `used_percent >= 100`)
|
||||
#[must_use]
|
||||
pub fn limit_reached(&self) -> bool {
|
||||
@@ -579,14 +573,6 @@ fn parse_rate_limit_windows(
|
||||
})
|
||||
.transpose()?;
|
||||
let limit_reached = optional_bool(window, "limit_reached")?.unwrap_or(false);
|
||||
let reset_after_seconds = window
|
||||
.get("reset_after_seconds")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_u64()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?;
|
||||
let kind = quota_window_kind(window_seconds);
|
||||
// primary/secondary 是协议槽位,不代表固定周期
|
||||
// 窗口身份使用上游
|
||||
@@ -605,7 +591,6 @@ fn parse_rate_limit_windows(
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
reset_after_seconds,
|
||||
limit_reached,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,14 +2,10 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
// 上游倒计时按整秒返回,跨过一个完整秒后再区分刚激活与空闲满窗口
|
||||
const WINDOW_CONFIRM_DELAY: Duration = Duration::from_secs(2);
|
||||
|
||||
enum WarmupDecision<'a> {
|
||||
Ready,
|
||||
Active(&'a CodexQuotaWindow),
|
||||
Exhausted(Option<&'a CodexQuotaWindow>),
|
||||
Verify(&'a CodexQuotaWindow),
|
||||
}
|
||||
|
||||
fn warmup_account_is_available(account: &ProviderAccount) -> bool {
|
||||
@@ -21,11 +17,10 @@ fn warmup_account_is_available(account: &ProviderAccount) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn warmup_decision<'a>(
|
||||
snapshot: &'a CodexAccountQuotaSnapshot,
|
||||
previous: Option<&CodexAccountQuotaSnapshot>,
|
||||
fn warmup_decision(
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> WarmupDecision<'a> {
|
||||
) -> WarmupDecision<'_> {
|
||||
if snapshot.quota().is_exhausted() {
|
||||
return WarmupDecision::Exhausted(None);
|
||||
}
|
||||
@@ -37,49 +32,19 @@ fn warmup_decision<'a>(
|
||||
return WarmupDecision::Exhausted(Some(window));
|
||||
}
|
||||
|
||||
let mut uncertain = None;
|
||||
for window in snapshot.windows().iter().filter(|window| {
|
||||
// 0% 可能取整,也可能属于空闲窗口;倒计时差值不能证明活跃
|
||||
// 对缺少明确活跃证据的账号允许一次轻量预热
|
||||
if let Some(window) = snapshot.windows().iter().find(|window| {
|
||||
window.is_account_wide()
|
||||
&& window.kind() == CodexQuotaWindowKind::ShortTerm
|
||||
&& window.used_percent().is_some_and(|used| used > 0.0)
|
||||
&& window
|
||||
.reset_at()
|
||||
.is_some_and(|reset| reset > now + chrono::Duration::minutes(30))
|
||||
}) {
|
||||
// 0% 可能只是取整;已经开始递减的原始倒计时仍能证明窗口已激活
|
||||
if window.used_percent().is_some_and(|used| used > 0.0)
|
||||
|| window
|
||||
.reset_after_seconds()
|
||||
.zip(window.window_seconds())
|
||||
.is_some_and(|(remaining, total)| remaining > 0 && remaining < total)
|
||||
{
|
||||
return WarmupDecision::Active(window);
|
||||
}
|
||||
// 连续两次实时查询仍返回完整窗口且 reset 向后移动,才确认空闲
|
||||
// 不以快照级 observed_at 推算,其他额度桶的被动更新也会推进该时间
|
||||
let confirmed_idle = idle_full_window(window)
|
||||
&& previous.is_some_and(|previous| {
|
||||
previous.windows().iter().any(|prior| {
|
||||
prior.key() == window.key()
|
||||
&& idle_full_window(prior)
|
||||
&& prior
|
||||
.reset_at()
|
||||
.zip(window.reset_at())
|
||||
.is_some_and(|(before, after)| after > before)
|
||||
})
|
||||
});
|
||||
if !confirmed_idle {
|
||||
uncertain = Some(window);
|
||||
}
|
||||
return WarmupDecision::Active(window);
|
||||
}
|
||||
uncertain.map_or(WarmupDecision::Ready, WarmupDecision::Verify)
|
||||
}
|
||||
|
||||
fn idle_full_window(window: &CodexQuotaWindow) -> bool {
|
||||
window.used_percent() == Some(0.0)
|
||||
&& window
|
||||
.reset_after_seconds()
|
||||
.zip(window.window_seconds())
|
||||
.is_some_and(|(remaining, total)| remaining == total)
|
||||
WarmupDecision::Ready
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
@@ -167,7 +132,7 @@ impl CodexCredentialQuotaService {
|
||||
.map(|account| account.id().clone())
|
||||
.collect::<Vec<_>>();
|
||||
let observed = self.store.get_quotas(&account_ids).await?;
|
||||
let mut observed_snapshots = observed
|
||||
let observed_snapshots = observed
|
||||
.iter()
|
||||
.filter_map(|obs| {
|
||||
quota_snapshot_from_observation(obs)
|
||||
@@ -181,7 +146,7 @@ impl CodexCredentialQuotaService {
|
||||
self.profile.clone(),
|
||||
);
|
||||
|
||||
'accounts: for account in accounts {
|
||||
for account in accounts {
|
||||
if account.quota().is_exhausted() {
|
||||
summary.skipped_exhausted += 1;
|
||||
tracing::info!(
|
||||
@@ -193,96 +158,36 @@ impl CodexCredentialQuotaService {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let mut snapshot = observed_snapshots
|
||||
.remove(account.id())
|
||||
.filter(|snapshot| snapshot.credential_revision() == account.revision());
|
||||
let mut previous = None;
|
||||
let mut refreshes = 0;
|
||||
while let Some(current) = snapshot.as_ref() {
|
||||
let (window, reason) =
|
||||
match warmup_decision(current, previous.as_ref(), chrono::Utc::now()) {
|
||||
WarmupDecision::Ready => break,
|
||||
WarmupDecision::Active(window) => {
|
||||
summary.skipped_active += 1;
|
||||
(Some(window), "active_short_window")
|
||||
}
|
||||
WarmupDecision::Exhausted(window) => {
|
||||
summary.skipped_exhausted += 1;
|
||||
(window, "quota_exhausted")
|
||||
}
|
||||
WarmupDecision::Verify(window) if refreshes == 2 => {
|
||||
summary.failed += 1;
|
||||
(Some(window), "window_activity_unconfirmed")
|
||||
}
|
||||
WarmupDecision::Verify(_) => {
|
||||
// 持久化快照不作为两次实时查询的首份证据
|
||||
if refreshes == 1 {
|
||||
previous = snapshot.take();
|
||||
tokio::time::sleep(WINDOW_CONFIRM_DELAY).await;
|
||||
}
|
||||
match tokio::time::timeout(
|
||||
WARMUP_REQUEST_TIMEOUT,
|
||||
self.refresh_account(account.id()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(refreshed))
|
||||
if refreshed.credential_revision() == account.revision() =>
|
||||
{
|
||||
snapshot = Some(refreshed);
|
||||
refreshes += 1;
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason = "quota_refresh_failed",
|
||||
"OpenAI account warmup quota verification failed"
|
||||
);
|
||||
continue 'accounts;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
observed_at = ?chrono::DateTime::<chrono::Utc>::from(current.observed_at()),
|
||||
window = window.map(CodexQuotaWindow::key),
|
||||
window_seconds = window.and_then(CodexQuotaWindow::window_seconds),
|
||||
used_percent = window.and_then(CodexQuotaWindow::used_percent),
|
||||
reset_at = ?window.and_then(CodexQuotaWindow::reset_at),
|
||||
reset_after_seconds = window.and_then(CodexQuotaWindow::reset_after_seconds),
|
||||
reason,
|
||||
"OpenAI account warmup skipped"
|
||||
);
|
||||
continue 'accounts;
|
||||
if let Some(snapshot) = observed_snapshots
|
||||
.get(account.id())
|
||||
.filter(|snapshot| snapshot.credential_revision() == account.revision())
|
||||
{
|
||||
let skipped = match warmup_decision(snapshot, chrono::Utc::now()) {
|
||||
WarmupDecision::Ready => None,
|
||||
WarmupDecision::Active(window) => {
|
||||
summary.skipped_active += 1;
|
||||
Some((Some(window), "active_short_window"))
|
||||
}
|
||||
WarmupDecision::Exhausted(window) => {
|
||||
summary.skipped_exhausted += 1;
|
||||
Some((window, "quota_exhausted"))
|
||||
}
|
||||
};
|
||||
if let Some((window, reason)) = skipped {
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
observed_at = ?chrono::DateTime::<chrono::Utc>::from(snapshot.observed_at()),
|
||||
window = window.map(CodexQuotaWindow::key),
|
||||
window_seconds = window.and_then(CodexQuotaWindow::window_seconds),
|
||||
used_percent = window.and_then(CodexQuotaWindow::used_percent),
|
||||
reset_at = ?window.and_then(CodexQuotaWindow::reset_at),
|
||||
reason,
|
||||
"OpenAI account warmup skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// 回查会更新额度与套餐;凭据加载仍核对最新账号的完整快照
|
||||
let account = if refreshes > 0 {
|
||||
match self.store.get_account(account.id()).await {
|
||||
Ok(Some(current))
|
||||
if current.provider() == account.provider()
|
||||
&& current.revision() == account.revision()
|
||||
&& warmup_account_is_available(¤t)
|
||||
&& !current.quota().is_exhausted() =>
|
||||
{
|
||||
current
|
||||
}
|
||||
_ => {
|
||||
summary.failed += 1;
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
reason = "account_changed_after_verification",
|
||||
"OpenAI account warmup skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
account
|
||||
};
|
||||
let credential = match self.repository.load_runtime_credential(&account).await {
|
||||
Ok(cred) => cred,
|
||||
Err(_) => {
|
||||
|
||||
@@ -393,23 +393,17 @@ impl ScheduledTask for OpenAiWarmupTask {
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let hour = local_now.time().hour();
|
||||
let minute = local_now.time().minute();
|
||||
let scheduled_times = policy.scheduled_times();
|
||||
let matched = scheduled_times
|
||||
.iter()
|
||||
.any(|&(h, m)| h == hour && m == minute);
|
||||
if !matched {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(model) = policy.model() else {
|
||||
return Err(WorkerTaskError::safe("OpenAI warmup model is missing"));
|
||||
};
|
||||
let slot = local_now
|
||||
.naive_local()
|
||||
.with_second(0)
|
||||
.and_then(|value| value.with_nanosecond(0))
|
||||
.ok_or_else(|| WorkerTaskError::safe("invalid warmup slot"))?;
|
||||
if !policy.matches_slot(slot) {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(model) = policy.model() else {
|
||||
return Err(WorkerTaskError::safe("OpenAI warmup model is missing"));
|
||||
};
|
||||
if !self
|
||||
.quota
|
||||
.runtime_policy()
|
||||
@@ -421,7 +415,7 @@ impl ScheduledTask for OpenAiWarmupTask {
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(hour, minute, model, "OpenAI account warmup cycle started");
|
||||
tracing::info!(%slot, model, "OpenAI account warmup cycle started");
|
||||
let outcome = tokio::select! {
|
||||
() = context.cancellation().cancelled() => return Ok(()),
|
||||
outcome = self.quota.execute_warmup(model) => outcome,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证额度预热的窗口识别、实时复核、SSE 终态与凭据边界
|
||||
//! 验证额度预热的窗口判断、SSE 终态与凭据边界
|
||||
|
||||
use super::*;
|
||||
use gateway_core::account::{OpaqueProviderData, QuotaObservation, QuotaWriteOutcome};
|
||||
@@ -58,7 +58,10 @@ async fn mount_completion(server: &MockServer, expected: u64) {
|
||||
#[tokio::test]
|
||||
async fn warmup_uses_account_windows_and_preserves_the_reset_threshold() {
|
||||
for (name, used, remaining, until_reset, named, warmed) in [
|
||||
("acct_zero_active", 0, Some(17_999), 17_999, false, false),
|
||||
("acct_zero_full", 0, Some(18_000), 18_000, false, true),
|
||||
("acct_zero_near_full", 0, Some(17_999), 17_999, false, true),
|
||||
("acct_zero_lower", 0, Some(12_000), 12_000, false, true),
|
||||
("acct_zero_missing_countdown", 0, None, 14_400, false, true),
|
||||
("acct_passive_active", 1, None, 14_400, false, false),
|
||||
("acct_near_reset", 1, Some(1_800), 1_800, false, true),
|
||||
("acct_expired", 1, Some(3_600), -1, false, true),
|
||||
@@ -112,202 +115,40 @@ async fn warmup_uses_account_windows_and_preserves_the_reset_threshold() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_confirms_idle_full_windows_even_when_the_saved_snapshot_is_old() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_idle").await;
|
||||
seed_quota(
|
||||
&store,
|
||||
"acct_idle",
|
||||
usage(short_window(
|
||||
0,
|
||||
Some(18_000),
|
||||
Utc::now().timestamp() + 14_400,
|
||||
)),
|
||||
SystemTime::now() - Duration::from_secs(3_600),
|
||||
)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(|_: &wiremock::Request| {
|
||||
let mut quota = usage(short_window(
|
||||
0,
|
||||
Some(18_000),
|
||||
Utc::now().timestamp() + 18_000,
|
||||
));
|
||||
quota["plan_type"] = json!("team");
|
||||
ResponseTemplate::new(200).set_body_json(quota)
|
||||
})
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 1).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_active, summary.failed),
|
||||
(1, 0, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_refreshes_zero_percent_passive_windows_before_skipping() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_zero").await;
|
||||
let reset = Utc::now().timestamp() + 14_400;
|
||||
seed_quota(
|
||||
&store,
|
||||
"acct_passive_zero",
|
||||
usage(short_window(0, None, reset)),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(usage(short_window(
|
||||
0,
|
||||
Some(14_400),
|
||||
reset,
|
||||
))))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 0).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_active, summary.failed),
|
||||
(0, 1, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_rechecks_a_just_activated_full_zero_percent_window() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_just_active").await;
|
||||
let reset = Utc::now().timestamp() + 18_000;
|
||||
let full = usage(short_window(0, Some(18_000), reset));
|
||||
seed_quota(&store, "acct_just_active", full.clone(), SystemTime::now()).await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(full))
|
||||
.with_priority(1)
|
||||
.up_to_n_times(1)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(usage(short_window(
|
||||
0,
|
||||
Some(17_998),
|
||||
reset,
|
||||
))))
|
||||
.with_priority(2)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 0).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_active, summary.failed),
|
||||
(0, 1, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_quota_verification_failure_does_not_abort_other_accounts() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_a_uncertain").await;
|
||||
create_account(&store, "acct_b_ready").await;
|
||||
seed_quota(
|
||||
&store,
|
||||
"acct_a_uncertain",
|
||||
usage(short_window(0, None, Utc::now().timestamp() + 14_400)),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(400))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 1).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_active, summary.failed),
|
||||
(1, 0, 1)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_respects_exhaustion_discovered_by_quota_verification() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_exhausted").await;
|
||||
let reset = Utc::now().timestamp() + 14_400;
|
||||
seed_quota(
|
||||
&store,
|
||||
"acct_exhausted",
|
||||
usage(short_window(0, None, reset)),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await;
|
||||
let mut exhausted = usage(short_window(100, Some(14_400), reset));
|
||||
exhausted["rate_limit"]["allowed"] = json!(false);
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(exhausted))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 0).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_exhausted, summary.failed),
|
||||
(0, 1, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_bounds_quota_verification_when_activity_remains_unknown() {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_unknown").await;
|
||||
let quota = usage(short_window(0, None, Utc::now().timestamp() + 14_400));
|
||||
seed_quota(&store, "acct_unknown", quota.clone(), SystemTime::now()).await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(quota))
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
mount_completion(&server, 0).await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(summary.warmed_up, summary.skipped_active, summary.failed),
|
||||
(0, 0, 1)
|
||||
);
|
||||
async fn warmup_skips_exhausted_account_windows_without_refreshing_quota() {
|
||||
for exhausted in [true, false] {
|
||||
let server = MockServer::start().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_exhausted").await;
|
||||
let account = store.account("acct_exhausted").unwrap();
|
||||
let now = SystemTime::now();
|
||||
let quota = json!({"rate_limit": {"allowed": true, "secondary_window": {
|
||||
"used_percent": 100, "limit_window_seconds": 604_800,
|
||||
"reset_at": Utc::now().timestamp() + 604_800,
|
||||
}}});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
expected_connection: None,
|
||||
plan_type: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: OpaqueProviderData::new(quota.as_object().unwrap().clone()),
|
||||
observed_at: now,
|
||||
state: if exhausted {
|
||||
QuotaState::exhausted(QuotaEvidence::UsageLimitReached, now, None)
|
||||
} else {
|
||||
QuotaState::allowed(now)
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
let summary = service.execute_warmup("test-model").await.unwrap();
|
||||
assert_eq!(summary.skipped_exhausted, 1);
|
||||
assert_eq!(summary.warmed_up, 0);
|
||||
assert_eq!(summary.failed, 0);
|
||||
assert!(server.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -37,3 +37,4 @@ d05442ee4cb85db8131e015dbed9c6ad988f93306abcda34647317674857ee44 0036_account_d
|
||||
b215ea90d5c8753aeb8192b17ffe65f7a95c1d620aaeaf9fc4337d8cf6f2634e 0037_retry_after_wait_limit.sql
|
||||
321f4a09367717eb13c3cf5dbfd1f4b014e31c615a74bf6e3b09b8e810d87198 0038_preserve_client_key_ledger.sql
|
||||
a6b4e81d38e5d70776a75eba22749354573ebcb186160ff9eb45fd91157b3e0d 0039_queue_timeouts_quota_credits_and_installation_seed.sql
|
||||
13db1dccac37cc20a716a605934e281a71e75c1aa626a3d7e6328f7a0ef3750d 0040_account_warmup_cron.sql
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- 清空旧时间计划并关闭预热,重新启用时由管理员填写 Cron
|
||||
update runtime_settings set account_warmup_enabled = false;
|
||||
|
||||
alter table runtime_settings
|
||||
drop column account_warmup_schedule_time,
|
||||
add column account_warmup_cron text not null default '' check (
|
||||
octet_length(account_warmup_cron) <= 255
|
||||
and account_warmup_cron = btrim(account_warmup_cron)
|
||||
and account_warmup_cron !~ '[[:cntrl:]]'
|
||||
and (not account_warmup_enabled or account_warmup_cron <> '')
|
||||
);
|
||||
+10
-7
@@ -2016,7 +2016,7 @@ HTTP 返回 `429`,`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly
|
||||
| [客户端门禁](#1-鉴权与公共约定) | `minCodexDesktopVersion`、`minCodexCliVersion` |
|
||||
| [历史保留](#历史保留) | `usageRetentionDays`、`opsEventRetentionDays`、`auditRetentionDays` |
|
||||
| [账号错误保护](#账号错误保护) | `accountErrorProtection` |
|
||||
| [账号预热](#账号预热) | `accountWarmupEnabled`、`accountWarmupScheduleTime`、`accountWarmupModel` |
|
||||
| [账号预热](#账号预热) | `accountWarmupEnabled`、`accountWarmupCron`、`accountWarmupModel` |
|
||||
| [主动额度重置卡](#主动额度重置卡) | `resetCreditLeadMinutes`、`resetCreditMaxWeeklyRemainingPercent` |
|
||||
|
||||
### 历史保留
|
||||
@@ -2032,12 +2032,15 @@ HTTP 返回 `429`,`error.code` 为 `key_daily_budget_exceeded` 或 `key_weekly
|
||||
|
||||
### 账号预热
|
||||
|
||||
定时账号预热默认关闭。`accountWarmupScheduleTime` 使用部署时区中的 `HH:MM`,
|
||||
多个时段以逗号分隔,默认 `08:00`;`accountWarmupModel` 默认 `null`,开启前必须显式选择模型。
|
||||
任务面向可用的 Codex OAuth 账号,按账号级额度桶判断,跳过额度耗尽及已确认活跃、距离重置仍超过 30 分钟的五小时窗口。
|
||||
0% 不代表窗口空闲;已用额度或开始递减的上游倒计时用于确认活跃。距重置超过 30 分钟但缺少活跃证据时,
|
||||
最多回查两次,两次实时查询间隔 2 秒;0% 完整窗口连续随查询后移才视为空闲。
|
||||
回查失败或仍无法确认时计为失败,不发送预热。
|
||||
定时账号预热默认关闭。`accountWarmupCron` 使用部署时区的五段 Cron 表达式,
|
||||
顺序为分、时、日、月、星期,总长度不超过 255 字节,默认空字符串,关闭时允许留空。
|
||||
支持列表、范围和步长,例如 `0 8,13,19 * * *` 每天执行三次,`0 8 * * MON-FRI` 在工作日 08:00 执行。
|
||||
星期可用 `SUN` 至 `SAT` 或数字 1 至 7(日到六),同时指定日与星期时两者均须满足。
|
||||
同一分钟只领取一次;`accountWarmupModel` 默认 `null`,开启前必须填写有效表达式并显式选择模型。
|
||||
|
||||
任务面向可用的 Codex OAuth 账号,按账号级额度桶判断,跳过额度耗尽,以及已用比例大于 0 且距离重置仍超过 30 分钟的五小时窗口。
|
||||
0% 或缺少活跃证据时允许一次轻量预热,不通过倒计时差值判断活跃,也不额外回查额度。
|
||||
因此已激活但显示 0% 的账号也可能收到预热请求。
|
||||
不存在的本地时刻跳过,重复时刻只执行较早一次;执行进度跨重启保留,时钟回拨不补跑已领取时刻之前的时段。
|
||||
只有收到响应成功终态才记为预热成功;预热不计入客户端业务用量
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ export interface RuntimeSettings {
|
||||
auditRetentionDays: number
|
||||
accountErrorProtection: AccountErrorProtection
|
||||
accountWarmupEnabled: boolean
|
||||
accountWarmupScheduleTime: string
|
||||
accountWarmupCron: string
|
||||
accountWarmupModel: string | null
|
||||
resetCreditLeadMinutes: number
|
||||
resetCreditMaxWeeklyRemainingPercent: number | null
|
||||
|
||||
@@ -198,15 +198,15 @@
|
||||
"warmup": {
|
||||
"title": "Account warmup",
|
||||
"description": "Try to start quota windows for idle Codex OAuth accounts",
|
||||
"enableDailyWarmup": "Enable daily warmup",
|
||||
"executionTime": "Execution time",
|
||||
"timesPlaceholder": "HH:MM, separate times with commas",
|
||||
"scheduleLabel": "Daily warmup times",
|
||||
"enableWarmup": "Enable warmup",
|
||||
"cronExpression": "Cron expression",
|
||||
"cronHelp": "Five fields: minute hour day month weekday",
|
||||
"modelLabel": "Warmup model",
|
||||
"modelHelp": "Required when enabled, use an available model",
|
||||
"modelPlaceholder": "Enter model name",
|
||||
"helpTitle": "About account warmup",
|
||||
"help": "Send one request to each idle Codex OAuth account at the scheduled times, starting a quota window is not guaranteed"
|
||||
"help": "Sends a lightweight request to available Codex OAuth accounts, skipping exhausted quotas and confirmed active windows, accounts at 0% may be warmed up, activation is not guaranteed",
|
||||
"cronExample": "Uses the deployment time zone, for example 0 8 * * MON-FRI runs at 08:00 on weekdays, at most once per minute"
|
||||
},
|
||||
"adminApiKey": {
|
||||
"adminApiKey": "Admin API key",
|
||||
@@ -614,7 +614,7 @@
|
||||
"invalidRetention": "History retention must be an integer from 1 to 31 days",
|
||||
"invalidQueueSettings": "Queue capacity must be an integer from 0 to 1,000 and timeout from 1 to 120 seconds",
|
||||
"invalidProbeModel": "The probe model must be at most 128 characters without control characters",
|
||||
"invalidWarmupTimes": "Use HH:MM for warmup times, for example 08:00 or 08:00,13:00",
|
||||
"invalidWarmupCron": "Enter a valid Cron expression",
|
||||
"warmupModelRequired": "Select a model to enable warmup",
|
||||
"invalidWarmupModel": "Warmup model name must not exceed 128 characters",
|
||||
"settingsSaved": "Settings saved",
|
||||
|
||||
@@ -198,15 +198,15 @@
|
||||
"warmup": {
|
||||
"title": "账号预热",
|
||||
"description": "尝试启动空闲 Codex OAuth 账号的额度窗口",
|
||||
"enableDailyWarmup": "启用每日预热",
|
||||
"executionTime": "执行时间",
|
||||
"timesPlaceholder": "HH:MM,多个时间用逗号分隔",
|
||||
"scheduleLabel": "每日预热时间",
|
||||
"enableWarmup": "启用预热",
|
||||
"cronExpression": "Cron 表达式",
|
||||
"cronHelp": "五段表达式:分 时 日 月 星期",
|
||||
"modelLabel": "预热模型",
|
||||
"modelHelp": "启用时必填,需为账号可用模型",
|
||||
"modelPlaceholder": "输入模型名称",
|
||||
"helpTitle": "账号预热说明",
|
||||
"help": "在设定时间向每个空闲的 Codex OAuth 账号发送一次请求,不保证额度窗口成功启动"
|
||||
"help": "向可用的 Codex OAuth 账号发送轻量请求,跳过额度耗尽和已确认活跃的窗口,0% 账号允许预热,不保证窗口成功启动",
|
||||
"cronExample": "按部署时区执行,例如 0 8 * * MON-FRI 表示工作日 08:00,同一分钟只执行一次"
|
||||
},
|
||||
"adminApiKey": {
|
||||
"adminApiKey": "管理员 API密钥",
|
||||
@@ -614,7 +614,7 @@
|
||||
"invalidRetention": "历史保留天数应为 1–31 的整数",
|
||||
"invalidQueueSettings": "队列容量应为 0~1000 的整数,排队超时应为 1~120 秒的整数",
|
||||
"invalidProbeModel": "探测模型最多 128 个字符,不能包含控制字符",
|
||||
"invalidWarmupTimes": "预激活时间格式无效,请输入 HH:MM 格式(如 08:00 或 08:00,13:00)",
|
||||
"invalidWarmupCron": "请输入有效的 Cron 表达式",
|
||||
"warmupModelRequired": "启用预激活时请选择模型",
|
||||
"invalidWarmupModel": "预激活模型名称不能超过 128 个字符",
|
||||
"settingsSaved": "设置已保存",
|
||||
|
||||
@@ -198,15 +198,15 @@
|
||||
"warmup": {
|
||||
"title": "賬號預熱",
|
||||
"description": "嘗試啓動空閒 Codex OAuth 賬號的額度窗口",
|
||||
"enableDailyWarmup": "啓用每日預熱",
|
||||
"executionTime": "執行時間",
|
||||
"timesPlaceholder": "HH:MM,多個時間用逗號分隔",
|
||||
"scheduleLabel": "每日預熱時間",
|
||||
"enableWarmup": "啟用預熱",
|
||||
"cronExpression": "Cron 表達式",
|
||||
"cronHelp": "五段表達式:分 時 日 月 星期",
|
||||
"modelLabel": "預熱模型",
|
||||
"modelHelp": "啓用時必填,需爲賬號可用模型",
|
||||
"modelPlaceholder": "輸入模型名稱",
|
||||
"helpTitle": "賬號預熱說明",
|
||||
"help": "在設定時間向每個空閒的 Codex OAuth 賬號發送一次請求,不保證額度窗口成功啓動"
|
||||
"help": "向可用的 Codex OAuth 帳號傳送輕量請求,略過額度耗盡和已確認活躍的視窗,0% 帳號允許預熱,不保證視窗成功啟動",
|
||||
"cronExample": "按部署時區執行,例如 0 8 * * MON-FRI 表示工作日 08:00,同一分鐘只執行一次"
|
||||
},
|
||||
"adminApiKey": {
|
||||
"adminApiKey": "管理員 API密鑰",
|
||||
@@ -614,7 +614,7 @@
|
||||
"invalidRetention": "歷史保留天數應爲 1–31 的整數",
|
||||
"invalidQueueSettings": "隊列容量應爲 0~1000 的整數,排隊超時應爲 1~120 秒的整數",
|
||||
"invalidProbeModel": "探測模型最多 128 個字符,不能包含控制字符",
|
||||
"invalidWarmupTimes": "預激活時間格式無效,請輸入 HH:MM 格式(如 08:00 或 08:00,13:00)",
|
||||
"invalidWarmupCron": "請輸入有效的 Cron 表達式",
|
||||
"warmupModelRequired": "啓用預激活時請選擇模型",
|
||||
"invalidWarmupModel": "預激活模型名稱不能超過 128 個字符",
|
||||
"settingsSaved": "設置已保存",
|
||||
|
||||
@@ -7,7 +7,7 @@ import { useI18n } from 'vue-i18n'
|
||||
const { t } = useI18n({ useScope: 'global' })
|
||||
|
||||
const enabled = defineModel<boolean>('enabled', { required: true })
|
||||
const scheduleTime = defineModel<string>('scheduleTime', { required: true })
|
||||
const cron = defineModel<string>('cron', { required: true })
|
||||
const model = defineModel<string>('model', { required: true })
|
||||
</script>
|
||||
|
||||
@@ -24,9 +24,10 @@ const model = defineModel<string>('model', { required: true })
|
||||
<CircleAlert class="size-3.5" aria-hidden="true" />
|
||||
</ZIconButton>
|
||||
</template>
|
||||
<p class="max-w-72 px-3 py-2 text-z-sm font-regular leading-relaxed text-z-text-secondary">
|
||||
{{ t('settings.warmup.help') }}
|
||||
</p>
|
||||
<div class="max-w-80 space-y-2 px-3 py-2 text-z-sm font-regular leading-relaxed text-z-text-secondary">
|
||||
<p>{{ t('settings.warmup.help') }}</p>
|
||||
<p>{{ t('settings.warmup.cronExample') }}</p>
|
||||
</div>
|
||||
</ZPopover>
|
||||
</span>
|
||||
</template>
|
||||
@@ -34,20 +35,23 @@ const model = defineModel<string>('model', { required: true })
|
||||
<ZSwitch
|
||||
v-model="enabled"
|
||||
class="col-span-full justify-self-start"
|
||||
:label="t('settings.warmup.enableDailyWarmup')"
|
||||
:label="t('settings.warmup.enableWarmup')"
|
||||
/>
|
||||
|
||||
<ZFormItem
|
||||
:show-message="Boolean(scheduleTime.trim())"
|
||||
prop="accountWarmupScheduleTime"
|
||||
:label="t('settings.warmup.executionTime')"
|
||||
:description="t('settings.warmup.timesPlaceholder')"
|
||||
:show-message="enabled"
|
||||
prop="accountWarmupCron"
|
||||
:label="t('settings.warmup.cronExpression')"
|
||||
:description="t('settings.warmup.cronHelp')"
|
||||
:required="enabled"
|
||||
>
|
||||
<ZInput
|
||||
v-model="scheduleTime"
|
||||
v-model="cron"
|
||||
:disabled="!enabled"
|
||||
:aria-label="t('settings.warmup.scheduleLabel')"
|
||||
placeholder="08:00"
|
||||
:aria-label="t('settings.warmup.cronExpression')"
|
||||
placeholder="0 8 * * *"
|
||||
maxlength="255"
|
||||
spellcheck="false"
|
||||
>
|
||||
<template #prefix>
|
||||
<Clock class="size-4" />
|
||||
|
||||
@@ -71,7 +71,7 @@ export function useSettingsForm() {
|
||||
probeModel: '',
|
||||
} as ProtectionDraft,
|
||||
accountWarmupEnabled: false,
|
||||
accountWarmupScheduleTime: '08:00',
|
||||
accountWarmupCron: '',
|
||||
accountWarmupModel: '',
|
||||
resetCreditLeadMinutes: 10 as number | null,
|
||||
resetCreditMaxWeeklyRemainingPercent: null as number | null,
|
||||
@@ -161,7 +161,7 @@ export function useSettingsForm() {
|
||||
probeModel: data.accountErrorProtection.probeModel ?? '',
|
||||
}
|
||||
form.accountWarmupEnabled = data.accountWarmupEnabled
|
||||
form.accountWarmupScheduleTime = data.accountWarmupScheduleTime ?? '08:00'
|
||||
form.accountWarmupCron = data.accountWarmupCron ?? ''
|
||||
form.accountWarmupModel = data.accountWarmupModel ?? ''
|
||||
form.resetCreditLeadMinutes = data.resetCreditLeadMinutes
|
||||
form.resetCreditMaxWeeklyRemainingPercent = data.resetCreditMaxWeeklyRemainingPercent
|
||||
@@ -292,7 +292,7 @@ export function useSettingsForm() {
|
||||
probeModel: trimProtectionText(form.accountErrorProtection.probeModel) || null,
|
||||
},
|
||||
accountWarmupEnabled: form.accountWarmupEnabled,
|
||||
accountWarmupScheduleTime: form.accountWarmupScheduleTime.trim(),
|
||||
accountWarmupCron: form.accountWarmupCron.trim(),
|
||||
accountWarmupModel: form.accountWarmupModel.trim() || null,
|
||||
resetCreditLeadMinutes: resetCreditLeadMinutes!,
|
||||
resetCreditMaxWeeklyRemainingPercent,
|
||||
|
||||
@@ -50,12 +50,13 @@ export function useSettingsFormRules(form: () => SettingsForm) {
|
||||
'usageRetentionDays': integer(1, 31, t('settings.form.invalidRetention')),
|
||||
'minCodexDesktopVersion': version(),
|
||||
'minCodexCliVersion': version(),
|
||||
'accountWarmupScheduleTime': {
|
||||
required: true,
|
||||
pattern: /^(?:[01]\d|2[0-3]):[0-5]\d(?:,(?:[01]\d|2[0-3]):[0-5]\d)*$/,
|
||||
'accountWarmupCron': {
|
||||
required: form().accountWarmupEnabled,
|
||||
pattern: /^\S+(?: +\S+){4}$/,
|
||||
max: 255,
|
||||
transform: value => value.trim(),
|
||||
trigger: ['blur', 'change'],
|
||||
message: t('settings.form.invalidWarmupTimes'),
|
||||
message: t('settings.form.invalidWarmupCron'),
|
||||
},
|
||||
'accountWarmupModel': [
|
||||
{
|
||||
|
||||
@@ -220,7 +220,7 @@ watch(section, (value) => {
|
||||
<AccountErrorProtectionCard v-model="form.accountErrorProtection" :disabled="disabled" />
|
||||
<AccountWarmupCard
|
||||
v-model:enabled="form.accountWarmupEnabled"
|
||||
v-model:schedule-time="form.accountWarmupScheduleTime"
|
||||
v-model:cron="form.accountWarmupCron"
|
||||
v-model:model="form.accountWarmupModel"
|
||||
/>
|
||||
<ResetCreditCard
|
||||
|
||||
Reference in New Issue
Block a user