mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-13 08:06:41 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55a88f5f6b | ||
|
|
5d90a92582 | ||
|
|
93abe8efcd | ||
|
|
f4b75c42ed | ||
|
|
36ff038c78 | ||
|
|
545009590c | ||
|
|
5b7b9b21ee | ||
|
|
44a2a87b69 |
@@ -13,8 +13,9 @@ use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::Subscriber
|
||||
use crate::config::LoggingConfig;
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
/// OpenAI OAuth 恢复记录含原始 AT/RT,文件日志开启时不能被常规级别过滤掉。
|
||||
const OPENAI_OAUTH_RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
|
||||
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
@@ -36,7 +37,8 @@ pub enum LogError {
|
||||
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = file_filter(&directive)?;
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
@@ -49,25 +51,37 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
guards.push(guard);
|
||||
writer
|
||||
});
|
||||
let file_writer = if config.file.enabled {
|
||||
let (file_writer, recovery_file_writer) = if config.file.enabled {
|
||||
let maximum_bytes = config
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
let file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (writer, guard) = NonBlockingBuilder::default()
|
||||
let recovery_file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (file_writer, file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-file")
|
||||
.finish(writer);
|
||||
guards.push(guard);
|
||||
Some(writer)
|
||||
.finish(file_writer);
|
||||
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-oauth-recovery-file")
|
||||
.finish(recovery_file_writer);
|
||||
guards.push(file_guard);
|
||||
guards.push(recovery_file_guard);
|
||||
(Some(file_writer), Some(recovery_file_writer))
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
@@ -91,23 +105,43 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
.with_span_list(true)
|
||||
.with_filter(file_filter)
|
||||
});
|
||||
let recovery_file_layer = recovery_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard { _writers: guards })
|
||||
}
|
||||
|
||||
fn file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OPENAI_OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OpenAI OAuth recovery log directive is valid");
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
directive.to_owned()
|
||||
@@ -118,6 +152,7 @@ fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> Str
|
||||
|
||||
struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -130,17 +165,20 @@ struct RotatingLogWriter {
|
||||
impl RotatingLogWriter {
|
||||
fn open(
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = Utc::now().date_naive();
|
||||
cleanup_log_files(&directory, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) = writable_log_segment(&directory, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, date, segment)?;
|
||||
cleanup_log_files(&directory, file_prefix, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) =
|
||||
writable_log_segment(&directory, file_prefix, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, file_prefix, date, segment)?;
|
||||
Ok(Self {
|
||||
directory,
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
maximum_files,
|
||||
@@ -168,10 +206,11 @@ impl RotatingLogWriter {
|
||||
} else {
|
||||
self.segment = self.segment.saturating_add(1);
|
||||
}
|
||||
self.file = open_log_segment(&self.directory, self.date, self.segment)?;
|
||||
self.file = open_log_segment(&self.directory, self.file_prefix, self.date, self.segment)?;
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
cleanup_log_files(
|
||||
&self.directory,
|
||||
self.file_prefix,
|
||||
self.date,
|
||||
self.retention_days,
|
||||
self.maximum_files,
|
||||
@@ -203,10 +242,11 @@ struct ManagedLogFile {
|
||||
|
||||
fn writable_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
maximum_bytes: u64,
|
||||
) -> io::Result<(usize, u64)> {
|
||||
let latest = managed_log_files(directory)?
|
||||
let latest = managed_log_files(directory, file_prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| entry.segment);
|
||||
@@ -221,26 +261,31 @@ fn writable_log_segment(
|
||||
}
|
||||
}
|
||||
|
||||
fn open_log_segment(directory: &Path, date: chrono::NaiveDate, segment: usize) -> io::Result<File> {
|
||||
fn open_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(directory.join(log_file_name(date, segment)))
|
||||
.open(directory.join(log_file_name(file_prefix, date, segment)))
|
||||
}
|
||||
|
||||
fn log_file_name(date: chrono::NaiveDate, segment: usize) -> String {
|
||||
fn log_file_name(file_prefix: &str, date: chrono::NaiveDate, segment: usize) -> String {
|
||||
if segment == 0 {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.log")
|
||||
format!("{file_prefix}.{date}.log")
|
||||
} else {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.{segment}.log")
|
||||
format!("{file_prefix}.{date}.{segment}.log")
|
||||
}
|
||||
}
|
||||
|
||||
fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
fn managed_log_files(directory: &Path, file_prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
|
||||
Ok(fs::read_dir(directory)?
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref()).map(
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref(), file_prefix).map(
|
||||
|(date, segment)| ManagedLogFile {
|
||||
date,
|
||||
segment,
|
||||
@@ -251,9 +296,9 @@ fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
fn parse_log_file_name(name: &str, file_prefix: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
let body = name
|
||||
.strip_prefix(&format!("{LOG_FILE_PREFIX}."))?
|
||||
.strip_prefix(&format!("{file_prefix}."))?
|
||||
.strip_suffix(".log")?;
|
||||
let (date, segment) = body.split_once('.').map_or((body, 0), |(date, segment)| {
|
||||
(date, segment.parse().ok().unwrap_or(usize::MAX))
|
||||
@@ -268,6 +313,7 @@ fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
today: chrono::NaiveDate,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -277,7 +323,7 @@ fn cleanup_log_files(
|
||||
.checked_sub_signed(chrono::Duration::days(retention_offset))
|
||||
.unwrap_or(chrono::NaiveDate::MIN);
|
||||
let mut retained = Vec::new();
|
||||
for entry in managed_log_files(directory)? {
|
||||
for entry in managed_log_files(directory, file_prefix)? {
|
||||
if entry.date < cutoff {
|
||||
fs::remove_file(entry.path)?;
|
||||
} else {
|
||||
|
||||
@@ -8,8 +8,13 @@ use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
|
||||
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
|
||||
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_sink() {
|
||||
@@ -42,7 +47,7 @@ fn logging_requires_at_least_one_sink() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_file_logging_overrides_global_log_level() {
|
||||
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
write_oauth_recovery_log(PathBuf::from(
|
||||
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
|
||||
@@ -54,12 +59,15 @@ fn oauth_recovery_file_logging_overrides_global_log_level() {
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_file_logging_overrides_global_log_level",
|
||||
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off,openai_oauth_recovery=off")
|
||||
.env(
|
||||
"RUST_LOG",
|
||||
"off,logging_test_application=info,oauth_recovery=off",
|
||||
)
|
||||
.output()
|
||||
.expect("run logging child process");
|
||||
|
||||
@@ -70,9 +78,18 @@ fn oauth_recovery_file_logging_overrides_global_log_level() {
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
|
||||
let log = read_log_directory(directory.path());
|
||||
assert!(log.contains(OAUTH_RECOVERY_LOG_TARGET));
|
||||
assert!(log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
let application_log = read_log_file_set(directory.path(), APPLICATION_LOG_FILE_PREFIX);
|
||||
assert!(application_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(application_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
|
||||
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
|
||||
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(recovery_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
|
||||
}
|
||||
|
||||
fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
@@ -104,23 +121,39 @@ fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
|
||||
tracing::info!(
|
||||
target: APPLICATION_LOG_TARGET,
|
||||
marker = APPLICATION_LOG_MARKER,
|
||||
"application file test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OpenAI OAuth recovery test record"
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
fn read_log_directory(directory: &Path) -> String {
|
||||
fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
let mut files = fs::read_dir(directory)
|
||||
.expect("read log directory")
|
||||
.map(|entry| entry.expect("read log entry").path())
|
||||
.filter(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
files.sort();
|
||||
|
||||
assert!(!files.is_empty(), "expected log file set {file_prefix}");
|
||||
files
|
||||
.into_iter()
|
||||
.map(|path| fs::read_to_string(path).expect("read log file"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn json_target_field(target: &str) -> String {
|
||||
format!(r#""target":"{target}""#)
|
||||
}
|
||||
|
||||
@@ -315,6 +315,26 @@ pub fn is_rate_limit_header_name(name: &str) -> bool {
|
||||
|| rate_limit_id_from_header_name(&normalized).is_some()
|
||||
}
|
||||
|
||||
/// 判断响应头是否为仅供代理本地观测的 Codex 额度信号。
|
||||
///
|
||||
/// 这类头会让下游官方 Codex 客户端显示账户额度提示,因此不能穿过账号隔离边界。
|
||||
/// 通用限流头(例如 `retry-after`)和非额度的 Codex 头仍允许下发。
|
||||
pub fn is_codex_quota_header_name(name: &str) -> bool {
|
||||
let normalized = name.trim().to_ascii_lowercase();
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"x-codex-credits-has-credits"
|
||||
| "x-codex-credits-unlimited"
|
||||
| "x-codex-credits-balance"
|
||||
| "x-codex-active-limit"
|
||||
| "x-codex-plan-type"
|
||||
| "x-codex-promo-message"
|
||||
| "x-codex-rate-limit-reached-type"
|
||||
| "x-codex-primary-over-secondary-limit-percent"
|
||||
) || rate_limit_id_from_header_name(&normalized)
|
||||
.is_some_and(|limit_id| limit_id == "codex" || limit_id.starts_with("codex_"))
|
||||
}
|
||||
|
||||
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
|
||||
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
if value.get("type").and_then(Value::as_str) != Some("codex.rate_limits") {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use gateway_protocol::openai::events::{
|
||||
RateLimitWindow, TokenUsage, extract_sse_usage, extract_usage, is_rate_limit_header_name,
|
||||
parse_rate_limit_headers, parse_rate_limits_event, rate_limits_to_header_pairs,
|
||||
retry_after_seconds_from_body,
|
||||
RateLimitWindow, TokenUsage, extract_sse_usage, extract_usage, is_codex_quota_header_name,
|
||||
is_rate_limit_header_name, parse_rate_limit_headers, parse_rate_limits_event,
|
||||
rate_limits_to_header_pairs, retry_after_seconds_from_body,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -537,3 +537,21 @@ fn rate_limit_header_filter_should_accept_only_supported_domain_headers() {
|
||||
[true, true, true, false, false]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_quota_header_filter_should_only_match_client_quota_signals() {
|
||||
assert_eq!(
|
||||
[
|
||||
"X-CoDeX-Primary-Used-Percent",
|
||||
"x-codex-code-review-primary-reset-at",
|
||||
"x-codex-credits-balance",
|
||||
"x-codex-primary-over-secondary-limit-percent",
|
||||
"retry-after",
|
||||
"x-ratelimit-remaining",
|
||||
"x-codex-safety-buffering-enabled",
|
||||
"x-codex-turn-state",
|
||||
]
|
||||
.map(is_codex_quota_header_name),
|
||||
[true, true, true, true, false, false, false, false]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -36,7 +36,10 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext
|
||||
|
||||
use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential};
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use recovery::{quota_state_reason, quota_state_transition, quota_success_state};
|
||||
use recovery::{
|
||||
authoritative_quota_snapshot_state, quota_state_reason, quota_state_transition,
|
||||
quota_success_state,
|
||||
};
|
||||
use snapshot::{
|
||||
parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation,
|
||||
scheduling_signals_from_snapshot,
|
||||
@@ -397,7 +400,7 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 写入新快照前读取已确认耗尽窗口的 reset,用于判断窗口是否真的推进。
|
||||
/// 写入新快照前读取 core primary 已确认耗尽窗口的 reset,用于判断该窗口是否真的推进。
|
||||
async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option<SystemTime> {
|
||||
let observations = self
|
||||
.store
|
||||
@@ -409,7 +412,10 @@ impl CodexCredentialQuotaService {
|
||||
&& observation.expected_revision == account.revision()
|
||||
})?;
|
||||
let snapshot = quota_snapshot_from_observation(&observation)?;
|
||||
snapshot.fact().resets_at().map(SystemTime::from)
|
||||
snapshot
|
||||
.core_primary_window()
|
||||
.and_then(|window| window.reset_at())
|
||||
.map(SystemTime::from)
|
||||
}
|
||||
|
||||
/// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。
|
||||
@@ -633,7 +639,6 @@ impl CodexCredentialQuotaService {
|
||||
observed_at,
|
||||
value,
|
||||
)?;
|
||||
let refreshed_fact = refreshed_snapshot.fact();
|
||||
let previous_reset_at = self.previous_confirmed_reset(account).await;
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
@@ -642,12 +647,16 @@ impl CodexCredentialQuotaService {
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(account.availability(), refreshed_fact, previous_reset_at)
|
||||
.is_none()
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
@@ -687,9 +696,11 @@ impl CodexCredentialQuotaService {
|
||||
} else {
|
||||
summary.updated += 1;
|
||||
}
|
||||
if let Some(availability) =
|
||||
quota_success_state(current.availability(), refreshed_fact, previous_reset_at)
|
||||
{
|
||||
if let Some(availability) = authoritative_quota_snapshot_state(
|
||||
current.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
) {
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
@@ -755,13 +766,18 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
let quota = merge_passive_quota(existing, &rate_limits);
|
||||
let observed_at = SystemTime::now();
|
||||
let refreshed_fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota.clone()),
|
||||
)?;
|
||||
let quota = if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
// 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照
|
||||
// 证明旧 reset 已跨过且新窗口已经推进。
|
||||
confirmed_exhaustion_projection(
|
||||
quota,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
quota
|
||||
@@ -957,7 +973,6 @@ impl CodexCredentialQuotaService {
|
||||
observed_at,
|
||||
&value,
|
||||
)?;
|
||||
let refreshed_fact = refreshed_snapshot.fact();
|
||||
let previous_reset_at = self.previous_confirmed_reset(&account).await;
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
@@ -966,12 +981,16 @@ impl CodexCredentialQuotaService {
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(account.availability(), refreshed_fact, previous_reset_at)
|
||||
.is_none()
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, &refreshed_snapshot),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
@@ -1005,9 +1024,11 @@ impl CodexCredentialQuotaService {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
self.scheduling.observe(&snapshot);
|
||||
if let Some(availability) =
|
||||
quota_success_state(current.availability(), refreshed_fact, previous_reset_at)
|
||||
{
|
||||
if let Some(availability) = authoritative_quota_snapshot_state(
|
||||
current.availability(),
|
||||
&refreshed_snapshot,
|
||||
previous_reset_at,
|
||||
) {
|
||||
self.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
.await?;
|
||||
@@ -1314,11 +1335,16 @@ fn confirmed_exhaustion_projection(
|
||||
/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。
|
||||
fn confirmed_exhaustion_reset_at(
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
refreshed_fact: CodexQuotaFact,
|
||||
refreshed_snapshot: &CodexAccountQuotaSnapshot,
|
||||
) -> Option<i64> {
|
||||
let refreshed_reset_at = refreshed_fact.resets_at().map(SystemTime::from);
|
||||
let refreshed_reset_at = refreshed_snapshot
|
||||
.core_primary_window()
|
||||
.and_then(|window| window.reset_at())
|
||||
.map(SystemTime::from);
|
||||
let reset_at = match (previous_reset_at, refreshed_reset_at) {
|
||||
(Some(previous), Some(refreshed)) if refreshed_fact.exhausted() && refreshed > previous => {
|
||||
(Some(previous), Some(refreshed))
|
||||
if refreshed_snapshot.fact().exhausted() && refreshed > previous =>
|
||||
{
|
||||
refreshed
|
||||
}
|
||||
(Some(previous), _) => previous,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! 确认额度耗尽后的恢复证据与 availability 转换。
|
||||
//! 权威额度快照的 availability 转换。
|
||||
//!
|
||||
//! `QuotaExhausted -> Ready` 只能由同一 credential revision 的权威 usage 快照证明:
|
||||
//! 新快照未耗尽,且 reset 已推进到下一窗口。
|
||||
//! 已触顶的 core primary 会立即把可调度账号标为 `QuotaExhausted`;
|
||||
//! `QuotaExhausted -> Ready` 则只能由同一 credential revision 的权威 usage 快照证明:
|
||||
//! 新快照未耗尽、core primary reset 已推进到下一窗口,且该窗口用量低于 10%。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
@@ -9,25 +10,55 @@ use gateway_core::engine::credential::AccountAvailability;
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::snapshot::CodexQuotaFact;
|
||||
use super::snapshot::CodexAccountQuotaSnapshot;
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
const RESET_RECOVERY_MAX_USED_PERCENT: f64 = 10.0;
|
||||
|
||||
/// 基于成功的权威 usage 快照收敛可调度账号的额度状态。
|
||||
///
|
||||
/// 只接受上游明确拒绝使用且 core primary 触顶的证据;secondary、附加桶、
|
||||
/// credits 与 spend control 都不能单独把整个账号标成额度耗尽。
|
||||
pub(crate) fn authoritative_quota_snapshot_state(
|
||||
current: AccountAvailability,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
match current {
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown
|
||||
if snapshot.authoritative_core_primary_exhausted() =>
|
||||
{
|
||||
Some(AccountAvailability::QuotaExhausted)
|
||||
}
|
||||
AccountAvailability::QuotaExhausted => {
|
||||
quota_success_state(current, snapshot, previous_reset_at)
|
||||
}
|
||||
AccountAvailability::Ready | AccountAvailability::Unknown => None,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独覆盖
|
||||
/// 一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
|
||||
///
|
||||
/// `previous_reset_at` 是写入新快照前读取的已确认耗尽窗口。恢复只要求权威快照的 reset
|
||||
/// 指向比旧窗口更晚的新周期,不等待旧 reset 的墙钟时间到达。
|
||||
/// `previous_reset_at` 是写入新快照前读取的 core primary 已确认耗尽窗口。上游可能只
|
||||
/// 调整 reset 时间而尚未重置额度,因此还要求同一新鲜 primary 窗口的用量低于 10%。
|
||||
pub(crate) fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
fact: CodexQuotaFact,
|
||||
snapshot: &CodexAccountQuotaSnapshot,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
) -> Option<AccountAvailability> {
|
||||
if current != AccountAvailability::QuotaExhausted || fact.exhausted() {
|
||||
if current != AccountAvailability::QuotaExhausted || snapshot.fact().exhausted() {
|
||||
return None;
|
||||
}
|
||||
let previous_reset_at = previous_reset_at?;
|
||||
let refreshed_reset_at = fact.resets_at().map(SystemTime::from)?;
|
||||
(refreshed_reset_at > previous_reset_at).then_some(AccountAvailability::Ready)
|
||||
let primary_window = snapshot.core_primary_window()?;
|
||||
let refreshed_reset_at = primary_window.reset_at().map(SystemTime::from)?;
|
||||
let used_percent = primary_window.used_percent()?;
|
||||
(refreshed_reset_at > previous_reset_at && used_percent < RESET_RECOVERY_MAX_USED_PERCENT)
|
||||
.then_some(AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
|
||||
|
||||
@@ -13,6 +13,8 @@ use serde_json::{Map, Value};
|
||||
|
||||
use super::{CodexCredentialQuotaError, QUOTA_SCHEDULING_TTL};
|
||||
|
||||
const CORE_PRIMARY_WINDOW_KEY: &str = "core-primary";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CodexQuotaFact {
|
||||
remaining_percent: Option<u8>,
|
||||
@@ -103,6 +105,7 @@ pub struct CodexAccountQuotaSnapshot {
|
||||
credential_revision: CredentialRevision,
|
||||
observed_at: SystemTime,
|
||||
fact: CodexQuotaFact,
|
||||
authoritative_core_primary_exhausted: bool,
|
||||
windows: Vec<CodexQuotaWindow>,
|
||||
}
|
||||
|
||||
@@ -132,6 +135,19 @@ impl CodexAccountQuotaSnapshot {
|
||||
&self.windows
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn core_primary_window(&self) -> Option<&CodexQuotaWindow> {
|
||||
self.windows
|
||||
.iter()
|
||||
.find(|window| window.key() == CORE_PRIMARY_WINDOW_KEY)
|
||||
}
|
||||
|
||||
/// 上游成功 usage 明确拒绝请求,且 canonical core primary 已确认触顶。
|
||||
#[must_use]
|
||||
pub(crate) const fn authoritative_core_primary_exhausted(&self) -> bool {
|
||||
self.authoritative_core_primary_exhausted
|
||||
}
|
||||
|
||||
/// 投影滚动已过期窗口:`reset_at` 已过的窗口 `used_percent=0`、
|
||||
/// `limit_reached=false`、`reset_at` 滚到下一个周期(reset 到期自动归零)。
|
||||
/// 只作用于展示/调度投影,不写回持久化 raw JSON。
|
||||
@@ -307,6 +323,7 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
let object = usage
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let authoritative_core_primary_exhausted = authoritative_core_primary_exhausted(object)?;
|
||||
let mut windows = Vec::new();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
parse_rate_limit_windows(
|
||||
@@ -323,10 +340,55 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
credential_revision,
|
||||
observed_at,
|
||||
fact,
|
||||
authoritative_core_primary_exhausted,
|
||||
windows,
|
||||
})
|
||||
}
|
||||
|
||||
/// 账号状态只能由顶层 canonical Codex 额度桶的明确拒绝来改变。
|
||||
///
|
||||
/// `fact.exhausted()` 和窗口 `limit_reached` 都会聚合或继承 secondary、附加桶等事实,
|
||||
/// 因此不能单独用来将整个账号标为 `QuotaExhausted`。
|
||||
fn authoritative_core_primary_exhausted(
|
||||
usage: &Map<String, Value>,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
let Some(rate_limit) = usage.get("rate_limit") else {
|
||||
return Ok(false);
|
||||
};
|
||||
if rate_limit.is_null() {
|
||||
return Ok(false);
|
||||
}
|
||||
let rate_limit = rate_limit
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
if optional_bool(rate_limit, "allowed")? != Some(false) {
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(primary_window) = rate_limit.get("primary_window") else {
|
||||
return Ok(false);
|
||||
};
|
||||
if primary_window.is_null() {
|
||||
return Ok(false);
|
||||
}
|
||||
let primary_window = primary_window
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let primary_window_reached = optional_bool(rate_limit, "limit_reached")?.unwrap_or(false)
|
||||
|| optional_bool(primary_window, "limit_reached")?.unwrap_or(false)
|
||||
|| primary_window
|
||||
.get("used_percent")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.map(|value| value >= 100.0)
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(false);
|
||||
Ok(primary_window_reached)
|
||||
}
|
||||
|
||||
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
|
||||
struct CanonicalRateLimit<'a> {
|
||||
key: String,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! 成功取得 OpenAI OAuth token 后写入普通应用日志的恢复记录。
|
||||
//! 成功取得 OAuth token 后写入独立恢复日志的记录。
|
||||
//!
|
||||
//! 此日志会刻意保存原始 AT/RT,避免后续数据库写入或校验失败时,已成功
|
||||
//! 交换的账号无法恢复。记录直接进入现有结构化日志,沿用其滚动和清理策略。
|
||||
//! 交换的账号无法恢复。记录独立落盘,但沿用普通结构化日志的滚动和清理策略。
|
||||
|
||||
const RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
|
||||
/// 产生可恢复 token 组的 OAuth 操作。
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -32,10 +33,10 @@ impl CodexOAuthRecoveryOperation {
|
||||
}
|
||||
}
|
||||
|
||||
/// 将已取得的 AT/RT 写入普通结构化日志。
|
||||
/// 将已取得的 AT/RT 写入独立恢复结构化日志。
|
||||
///
|
||||
/// 调用方必须在任何可能拒绝响应的本地校验、画像补全或数据库写入之前调用。
|
||||
/// 该调用不执行额外 I/O,也不返回业务错误。Host 负责普通日志文件的分割、清理和
|
||||
/// 该调用不执行额外 I/O,也不返回业务错误。Host 负责恢复日志文件的分割、清理和
|
||||
/// 输出;日志系统不可用时按其既有策略处理,不能阻断 OAuth 业务流程。
|
||||
pub(crate) fn record_oauth_recovery(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
@@ -46,13 +47,14 @@ pub(crate) fn record_oauth_recovery(
|
||||
let has_refresh_token = refresh_token.is_some();
|
||||
let refresh_token = refresh_token.unwrap_or_default();
|
||||
tracing::info!(
|
||||
target: RECOVERY_LOG_TARGET,
|
||||
event = "openai_oauth_recovery",
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
event = "oauth_recovery",
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
operation = operation.as_str(),
|
||||
account_id = account_id.unwrap_or_default(),
|
||||
access_token,
|
||||
refresh_token,
|
||||
has_refresh_token,
|
||||
"OpenAI OAuth recovery token record"
|
||||
"OAuth recovery token record"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -139,8 +139,6 @@ pub async fn initialize(
|
||||
.map_err(OpenAiInitializeError::Provider)?
|
||||
.with_session_identity(session_identity),
|
||||
);
|
||||
|
||||
tracing::info!(target: "openai_oauth_recovery", "OpenAI OAuth recovery records use normal application logs");
|
||||
let token_client = Arc::new(
|
||||
credential::token_client::openai_token_client(config.token_client_config())
|
||||
.map_err(|_| OpenAiInitializeError::TokenClient)?,
|
||||
|
||||
@@ -270,6 +270,11 @@ impl CodexCanonicalDecoder {
|
||||
.event
|
||||
.as_deref()
|
||||
.or_else(|| value.get("type").and_then(Value::as_str));
|
||||
if event_type == Some("codex.rate_limits") {
|
||||
// HTTP transport has already projected this control frame into local quota facts.
|
||||
// It must not become client output or start first-output timing.
|
||||
return Ok(());
|
||||
}
|
||||
self.observe_response_service_tier(&value);
|
||||
let signals = response_event_signals(event_type, &value);
|
||||
self.merge_timing_signals(signals);
|
||||
@@ -411,7 +416,7 @@ impl CodexCanonicalDecoder {
|
||||
self.complete(event_type, value, output)
|
||||
}
|
||||
"response.failed" | "error" => Err(protocol_error_marker()),
|
||||
"response.rate_limits.updated" | "codex.rate_limits" | "response.metadata" => Ok(()),
|
||||
"response.rate_limits.updated" | "response.metadata" => Ok(()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use gateway_protocol::openai::events::is_rate_limit_header_name;
|
||||
use gateway_protocol::openai::events::{is_codex_quota_header_name, is_rate_limit_header_name};
|
||||
use reqwest::header::{HeaderMap, SET_COOKIE};
|
||||
|
||||
use super::diagnostics::CodexUpstreamDiagnostics;
|
||||
@@ -155,6 +155,9 @@ fn client_response_header_is_forwardable(name: &str, connection_options: &[Strin
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if is_codex_quota_header_name(&name) {
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name.as_str(),
|
||||
|
||||
@@ -487,7 +487,7 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_drops_empty_secondary_window() {
|
||||
async fn manual_quota_refresh_marks_ready_account_exhausted_and_drops_empty_secondary_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_secondary_placeholder";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -533,6 +533,13 @@ async fn manual_quota_refresh_drops_empty_secondary_window() {
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0));
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after refresh")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -824,7 +831,7 @@ async fn periodic_quota_synchronization_does_not_scan_stale_ready_accounts() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks() {
|
||||
async fn periodic_quota_synchronization_marks_ready_account_exhausted_when_primary_is_exhausted() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_stuck_cooldown").await;
|
||||
let account = store
|
||||
@@ -871,6 +878,50 @@ async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks(
|
||||
"primary_window": {"used_percent": 100, "reset_at": reset_at}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
service
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
.await;
|
||||
|
||||
let summary = service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
let reconciled = store
|
||||
.account("acct_periodic_stuck_cooldown")
|
||||
.expect("reconciled account");
|
||||
assert_eq!(
|
||||
reconciled.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_keeps_ready_when_only_secondary_window_is_exhausted() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_secondary_only").await;
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 5, "reset_at": reset_at},
|
||||
"secondary_window": {"used_percent": 100, "reset_at": reset_at}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
@@ -882,13 +933,99 @@ async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks(
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
service.synchronize().await.expect("quota synchronization");
|
||||
let summary = service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
// 429/触顶不再写入 availability(限流不改变账号可用性),worker 只更新 quota 数据。
|
||||
let reconciled = store
|
||||
.account("acct_periodic_stuck_cooldown")
|
||||
.expect("reconciled account");
|
||||
assert_eq!(reconciled.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_periodic_secondary_only")
|
||||
.expect("reconciled account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_keeps_ready_when_primary_is_full_but_requests_are_allowed()
|
||||
{
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_primary_full_but_allowed").await;
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": reset_at}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_periodic_primary_full_but_allowed")
|
||||
.expect("reconciled account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_keeps_ready_without_primary_exhaustion_evidence() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(
|
||||
&store,
|
||||
"acct_periodic_disallowed_without_primary_exhaustion",
|
||||
)
|
||||
.await;
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 5, "reset_at": reset_at}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_periodic_disallowed_without_primary_exhaustion")
|
||||
.expect("reconciled account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1254,7 +1391,7 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_before_reset() {
|
||||
async fn periodic_quota_synchronization_keeps_exhaustion_at_ten_percent_after_advanced_reset() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_recovery_evidence";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -1333,8 +1470,8 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
// 无 allowed/limit_reached 标记的模糊快照。
|
||||
"primary_window": {"used_percent": 99, "reset_at": reset_at}
|
||||
// reset 已后移,但 10% 仍不能证明这是刚重置的新窗口。
|
||||
"primary_window": {"used_percent": 10, "reset_at": reset_at + 3600}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
@@ -1350,19 +1487,19 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let lower_percent = next_cycle
|
||||
let ten_percent = next_cycle
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("lower percent quota recheck");
|
||||
.expect("ten-percent quota recheck");
|
||||
|
||||
assert_eq!(lower_percent.exhausted, 1);
|
||||
assert_eq!(ten_percent.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after lower percent snapshot")
|
||||
.expect("account after ten-percent snapshot")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"a provider percentage change is not sufficient to release a confirmed exhaustion"
|
||||
"an advanced reset at exactly 10% is not sufficient to release confirmed exhaustion"
|
||||
);
|
||||
let snapshot = next_cycle
|
||||
.read_account(account.id())
|
||||
@@ -1373,12 +1510,12 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
async fn periodic_quota_synchronization_recovers_with_advanced_reset_and_low_primary_usage() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_reset_elapsed";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let elapsed_reset = Utc::now().timestamp() - 60;
|
||||
let old_reset = Utc::now().timestamp() + 3_600;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
@@ -1388,7 +1525,7 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": elapsed_reset}
|
||||
"primary_window": {"used_percent": 100, "reset_at": old_reset}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
@@ -1399,7 +1536,7 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("record elapsed exhausted reset");
|
||||
.expect("record future exhausted reset");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
@@ -1409,9 +1546,9 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted with an elapsed reset");
|
||||
.expect("mark account exhausted with a future reset");
|
||||
let server = MockServer::start().await;
|
||||
let next_reset = Utc::now().timestamp() + 3600;
|
||||
let next_reset = old_reset + 3_600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
@@ -1433,7 +1570,7 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
// 定时复核可提前执行,但只有已记录的 reset 到期后才释放额度耗尽状态。
|
||||
// 定时复核可在旧 reset 前执行;新主窗口已推进且用量低于 10% 才释放额度耗尽状态。
|
||||
let summary = service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
assert_eq!(summary.updated, 1);
|
||||
@@ -1585,7 +1722,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {"used_percent": 25.0, "reset_at": next_reset}
|
||||
"primary_window": {"used_percent": 1.0, "reset_at": next_reset}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
@@ -1614,5 +1751,5 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
.await
|
||||
.expect("read good quota")
|
||||
.expect("good quota persisted");
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(75));
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(99));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 402 恢复回归。
|
||||
//!
|
||||
//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:新 reset 后移 + 新快照未耗尽。
|
||||
//! 覆盖 `QuotaExhausted -> Ready` 的证据规则:新 core primary reset 后移 + 新快照用量低于 10%。
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -127,8 +127,8 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn advanced_reset_recovers_ready_before_the_recorded_reset_time() {
|
||||
// 旧 reset 仍在未来;只要新窗口 reset 后移且额度正常,仍必须恢复 Ready。
|
||||
async fn advanced_reset_with_primary_usage_below_ten_percent_recovers_ready() {
|
||||
// 旧 reset 仍在未来;同一主窗口 reset 后移且用量低于 10% 时才恢复 Ready。
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_cross_window_recovery";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -181,7 +181,7 @@ async fn advanced_reset_recovers_ready_before_the_recorded_reset_time() {
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 5, "reset_at": future_reset}
|
||||
"primary_window": {"used_percent": 9.9, "reset_at": future_reset}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
@@ -207,7 +207,91 @@ async fn advanced_reset_recovers_ready_before_the_recorded_reset_time() {
|
||||
.expect("account after cross-window recovery")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
"an advanced reset must recover QuotaExhausted before the recorded reset time"
|
||||
"an advanced reset with a nearly unused primary window must recover QuotaExhausted"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn advanced_reset_with_primary_usage_at_ten_percent_keeps_confirmed_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_cross_window_ten_percent";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let old_reset = Utc::now().timestamp() + 3_600;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": old_reset,
|
||||
"limit_window_seconds": 3600
|
||||
}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed old exhausted snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
let future_reset = old_reset + 3_600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 10.0, "reset_at": future_reset}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let held = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh non-empty new window quota");
|
||||
|
||||
assert!(held.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after ten-percent snapshot")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"a reset advance at exactly 10% must not release confirmed exhaustion"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,9 @@ const METADATA_PREFIX_FIXTURE: &str = include_str!(
|
||||
);
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_existing_metadata_fixture_as_openai_wire() {
|
||||
fn decoder_should_not_forward_codex_rate_limit_metadata_fixture_as_openai_wire() {
|
||||
let events = CodexCanonicalDecoder::new("fallback")
|
||||
.with_raw_sse_passthrough()
|
||||
.push(METADATA_PREFIX_FIXTURE.as_bytes())
|
||||
.expect("metadata fixture should remain open-world");
|
||||
let wire_types = events
|
||||
@@ -20,16 +21,32 @@ fn decoder_should_preserve_existing_metadata_fixture_as_openai_wire() {
|
||||
.filter_map(|wire| wire.event_type())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
wire_types,
|
||||
vec!["response.created", "codex.rate_limits", "response.metadata"]
|
||||
);
|
||||
assert_eq!(wire_types, vec!["response.created", "response.metadata"]);
|
||||
assert!(matches!(
|
||||
canonical_facts(&events).as_slice(),
|
||||
[GatewayEvent::Started(_)]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_sse_passthrough_should_drop_rate_limit_control_without_timing_side_effects() {
|
||||
let frame = concat!(
|
||||
"event: codex.rate_limits\n",
|
||||
"data: {\"type\":\"codex.rate_limits\",\"rate_limits\":{\"primary\":{\"used_percent\":90}}}\n\n",
|
||||
);
|
||||
let mut decoder = CodexCanonicalDecoder::new("fallback").with_raw_sse_passthrough();
|
||||
|
||||
let events = decoder
|
||||
.push(frame.as_bytes())
|
||||
.expect("quota control frame should be accepted");
|
||||
let signals = decoder.take_timing_signals();
|
||||
|
||||
assert!(events.is_empty());
|
||||
assert!(!signals.protocol_progress);
|
||||
assert!(!signals.output_start);
|
||||
assert!(!signals.semantic_output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_empty_opaque_response_id_in_canonical_facts() {
|
||||
let body = concat!(
|
||||
@@ -407,7 +424,7 @@ fn decoder_should_preserve_image_tool_tokens_in_canonical_usage() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_accept_official_codex_metadata_lifecycle_events() {
|
||||
fn decoder_should_drop_rate_limit_control_and_preserve_metadata_lifecycle_events() {
|
||||
let body = concat!(
|
||||
"event: codex.rate_limits\n",
|
||||
"data: {\"type\":\"codex.rate_limits\",\"rate_limits\":{\"primary\":{\"used_percent\":42.0}}}\n\n",
|
||||
@@ -422,10 +439,25 @@ fn decoder_should_accept_official_codex_metadata_lifecycle_events() {
|
||||
);
|
||||
|
||||
let events = CodexCanonicalDecoder::new("fallback")
|
||||
.with_raw_sse_passthrough()
|
||||
.push(body.as_bytes())
|
||||
.expect("official Codex metadata lifecycle events");
|
||||
let canonical = canonical_facts(&events);
|
||||
let wire_types = events
|
||||
.iter()
|
||||
.filter_map(ProviderEvent::wire_event)
|
||||
.filter_map(|wire| wire.event_type())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
wire_types,
|
||||
vec![
|
||||
"response.metadata",
|
||||
"response.in_progress",
|
||||
"response.output_text.delta",
|
||||
"response.completed"
|
||||
]
|
||||
);
|
||||
assert!(matches!(
|
||||
canonical.as_slice(),
|
||||
[
|
||||
|
||||
@@ -385,6 +385,11 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() {
|
||||
.insert_header("connection", "x-hop-secret")
|
||||
.insert_header("x-hop-secret", "hop-secret")
|
||||
.insert_header("x-codex-primary-used-percent", "15")
|
||||
.insert_header("x-codex-code-review-primary-used-percent", "20")
|
||||
.insert_header("x-codex-promo-message", "internal quota notice")
|
||||
.insert_header("x-codex-safety-buffering-enabled", "true")
|
||||
.insert_header("retry-after", "17")
|
||||
.insert_header("x-ratelimit-remaining", "23")
|
||||
.set_body_string(body),
|
||||
)
|
||||
.mount(&server)
|
||||
@@ -445,11 +450,13 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() {
|
||||
"date",
|
||||
"openai-model",
|
||||
"openai-processing-ms",
|
||||
"x-codex-primary-used-percent",
|
||||
"retry-after",
|
||||
"x-codex-safety-buffering-enabled",
|
||||
"x-future-bytes",
|
||||
"x-future-multi",
|
||||
"x-future-multi",
|
||||
"x-models-etag",
|
||||
"x-ratelimit-remaining",
|
||||
"x-reasoning-included",
|
||||
"x-request-id"
|
||||
]
|
||||
@@ -486,10 +493,22 @@ async fn codex_backend_client_should_capture_forwardable_response_metadata() {
|
||||
);
|
||||
}
|
||||
assert_eq!(response.set_cookie_headers, vec!["secret=value"]);
|
||||
assert_eq!(
|
||||
response.rate_limit_headers,
|
||||
vec![("x-codex-primary-used-percent".to_string(), "15".to_string())]
|
||||
);
|
||||
for expected in [
|
||||
("x-codex-primary-used-percent", "15"),
|
||||
("x-codex-code-review-primary-used-percent", "20"),
|
||||
("x-codex-promo-message", "internal quota notice"),
|
||||
("retry-after", "17"),
|
||||
("x-ratelimit-remaining", "23"),
|
||||
] {
|
||||
assert!(
|
||||
response
|
||||
.rate_limit_headers
|
||||
.iter()
|
||||
.any(|(name, value)| name == expected.0 && value == expected.1),
|
||||
"missing locally observed rate-limit header {}",
|
||||
expected.0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+9
-7
@@ -111,9 +111,9 @@ PostgreSQL 是账号、Client Key、运行设置、请求记录与审计的权
|
||||
Provider schema 以明文 JSON 保存于 PostgreSQL。Redis 只保存可重建、可过期的协调状态,例如
|
||||
会话亲和、lease、cooldown、OAuth pending flow 与套餐模型目录 cache。
|
||||
|
||||
备份必须包含 `.runtime/postgres`。要保留 OpenAI OAuth 的 AT/RT 恢复能力,必须保持
|
||||
`host.logging.file.enabled: true` 并备份 `.runtime/logs`;该记录受普通日志的 `retention_days` 和
|
||||
`max_files` 约束。若希望保留短期 Redis 状态和会话锚点,也同时备份 `.runtime/redis` 与
|
||||
备份必须包含 `.runtime/postgres`。要保留 OAuth 的 AT/RT 恢复能力,必须保持
|
||||
`host.logging.file.enabled: true` 并备份 `.runtime/logs`;恢复记录位于独立文件集,仍按普通日志的
|
||||
`retention_days` 和 `max_files` 分别约束。若希望保留短期 Redis 状态和会话锚点,也同时备份 `.runtime/redis` 与
|
||||
`.runtime/data`。
|
||||
|
||||
完整运行时、Provider、revision 与恢复边界见 [架构文档](../docs/architecture.md)。
|
||||
@@ -203,10 +203,12 @@ Release 必须提供当前 OS/架构的 `codex-proxy-rs_<version>_<os>_<arch>.ta
|
||||
Debian 官方源补 `libpq5` 依赖,不引入第三方 APT 源,也不依赖运行时动态安装。
|
||||
- 备份暂存目录为 `/app/.runtime/data/backup-staging`,由 `.runtime/data` 卷持久化,权限
|
||||
`0700`(仅 `cpr` 用户可读写)。部署卷至少预留一个最大数据库归档的空间。
|
||||
- OpenAI OAuth 每次成功取得 AT/RT 后,都会在账号资料补全、过期时间计算和数据库写入前,以
|
||||
`openai_oauth_recovery` 结构化日志事件写入普通应用日志。事件含原始 AT/RT,完全沿用
|
||||
`.runtime/logs` 的按日、按大小分割和保留清理策略;文件日志开启时不会被普通日志级别筛掉。
|
||||
它遵循普通日志现有的非阻塞写入机制,不会阻断导入、授权或刷新。
|
||||
- OAuth 恢复记录通道使用 `oauth_recovery` 结构化日志事件;当前 OpenAI provider 每次成功取得
|
||||
AT/RT 后,都会在账号资料补全、过期时间计算和数据库写入前写入独立的
|
||||
`codex-proxy-rs-oath.YYYY-MM-DD[.N].log` 文件集。事件含原始 AT/RT,并以 `provider`
|
||||
字段标记来源,且与普通 `.runtime/logs` 完全相同地按日、按大小分割,并分别按相同 `retention_days` 与 `max_files`
|
||||
清理;文件日志开启时不会被普通日志级别筛掉。它遵循普通日志现有的非阻塞写入机制,不会阻断
|
||||
导入、授权或刷新。
|
||||
`.runtime/logs` 因此属于敏感数据,必须按现有普通日志的访问控制和加密备份策略处理。
|
||||
- S3/R2 存储、Cron 计划、保留策略与备份记录都保存在 PostgreSQL(`backup_settings` /
|
||||
`backup_records`),备份记录行在删除成功后硬删除,操作历史进入 `admin_audit_events`。
|
||||
|
||||
@@ -110,13 +110,10 @@ const labelClasses = computed(() => [
|
||||
>
|
||||
<span
|
||||
v-if="loading"
|
||||
class="inline-flex shrink-0 items-center justify-center leading-none"
|
||||
class="inline-flex shrink-0 animate-spin items-center justify-center origin-center leading-none will-change-transform"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<LoaderCircle
|
||||
:size="loadingIconSize[size]"
|
||||
class="block animate-spin origin-center [transform-box:view-box] will-change-transform"
|
||||
/>
|
||||
<LoaderCircle :size="loadingIconSize[size]" class="block" />
|
||||
</span>
|
||||
<span v-if="$slots.icon && !loading" class="inline-flex shrink-0">
|
||||
<slot name="icon" />
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.1.18"
|
||||
version: "3.1.20"
|
||||
|
||||
Reference in New Issue
Block a user