mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-08 16:05:33 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c22a2ad144 | ||
|
|
7bd362d544 | ||
|
|
e7036a616f | ||
|
|
98652d4c68 | ||
|
|
d0dc7ef92b | ||
|
|
194fb20d5f | ||
|
|
30f528f084 | ||
|
|
28d024ceda | ||
|
|
6b9c98c59f | ||
|
|
1294e50b76 | ||
|
|
a0e540642b | ||
|
|
14d6a7de31 | ||
|
|
cd7b60a3ba | ||
|
|
ebbed22f36 | ||
|
|
f6e5d8f673 | ||
|
|
bf1c6211b8 | ||
|
|
d388fa344a | ||
|
|
eabb61b08c | ||
|
|
a5c3438927 | ||
|
|
b6948542bb | ||
|
|
0242bd3193 | ||
|
|
229ef48470 | ||
|
|
19c35ba901 | ||
|
|
3c06ae97e8 | ||
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab |
Generated
+1
@@ -2628,6 +2628,7 @@ dependencies = [
|
||||
"tokio-tungstenite 0.28.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tungstenite 0.27.0",
|
||||
"url",
|
||||
"uuid",
|
||||
|
||||
@@ -134,7 +134,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_core_version() {
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
|
||||
}
|
||||
|
||||
|
||||
@@ -220,7 +220,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -229,9 +230,12 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -394,7 +398,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -403,10 +408,13 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -448,8 +456,12 @@ impl BackupTask {
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "上传到对象存储失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -475,19 +487,46 @@ impl BackupTask {
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(Some(remote)) => {
|
||||
let size_matches = remote.size_bytes == size_bytes;
|
||||
let sha256_matches = remote.sha256 == sha256;
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
expected_size_bytes = size_bytes,
|
||||
remote_size_bytes = remote.size_bytes,
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"远端对象大小或校验值不一致",
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"上传完成后未找到远端对象",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "远端对象校验失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -500,10 +539,17 @@ impl BackupTask {
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let message = message.into();
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
error_code,
|
||||
error_message = %message,
|
||||
"备份任务失败"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
error_code: Some(error_code.to_string()),
|
||||
error_message: Some(message.into()),
|
||||
error_message: Some(message),
|
||||
completed_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -13,15 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
RateLimited,
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Invalid,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -78,6 +209,8 @@ pub struct AccountRecord {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -170,13 +303,15 @@ pub struct AccountPage {
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
|
||||
@@ -11,7 +11,10 @@ use gateway_core::{
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -592,6 +595,9 @@ pub struct ProviderQuota {
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
@@ -688,6 +694,8 @@ impl fmt::Debug for ProviderExport {
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,7 +18,8 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -216,8 +217,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, quota.limit_reached, now),
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -301,11 +305,16 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, quota.limit_reached, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -512,43 +521,39 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 互斥页面状态:availability 终态优先,quota 快照级 `limit_reached` 派生
|
||||
/// `rate_limited`,`Unknown` 保守映射 `invalid`(不可调度,不显示为 active)。
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Invalid
|
||||
) {
|
||||
AccountStatus::Invalid
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else if quota_limit_reached {
|
||||
AccountStatus::RateLimited
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Unknown {
|
||||
AccountStatus::Invalid
|
||||
} else {
|
||||
AccountStatus::Active
|
||||
}
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
|
||||
@@ -13,9 +13,10 @@ use gateway_admin::{
|
||||
MutationActor, MutationContext, Revision,
|
||||
auth::AdminAuditEvent,
|
||||
backup::{
|
||||
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
|
||||
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
|
||||
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
|
||||
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
|
||||
UpdateBackupStorageCommand,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -497,15 +498,23 @@ impl DatabaseDumpPort for FakeDumpPort {
|
||||
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让上传返回指定的脱敏 S3 错误。
|
||||
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
|
||||
self.upload_error = Some(BackupError::new(code, message.into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
@@ -530,6 +539,9 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
_config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.upload_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let content = std::fs::read(&request.source).map_err(|_| {
|
||||
gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
|
||||
|
||||
@@ -6,9 +6,9 @@ use chrono::{Duration, Utc};
|
||||
use gateway_admin::{
|
||||
backup::task::BackupTask,
|
||||
model::backup::{
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
|
||||
},
|
||||
ports::backup::BackupRepository,
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
|
||||
@@ -89,6 +89,78 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
assert!(object_store.object(&seed.object_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("recovered-upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
dump.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("stage dump");
|
||||
repository.force_status(&seed.id, BackupStatus::Dumping);
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&cancellation).await.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_cleans_expired_scheduled_backups() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
|
||||
@@ -128,6 +128,7 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
|
||||
@@ -451,9 +451,11 @@ impl AccountStore for FakeAccountStore {
|
||||
total: accounts.len() as u64,
|
||||
summary: AccountSummary {
|
||||
total: accounts.len() as u64,
|
||||
active: 1,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -844,7 +846,7 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_rate_limited_probe_preserves_active_account_status() {
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
@@ -867,7 +869,7 @@ async fn connection_test_rate_limited_probe_preserves_active_account_status() {
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Active,
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
..
|
||||
})
|
||||
));
|
||||
@@ -948,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -984,12 +986,12 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1013,6 +1015,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
@@ -1088,8 +1091,8 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
|
||||
// Unknown 不可调度,Admin 不得显示为 active。
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
@@ -1111,13 +1114,14 @@ async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Invalid,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → rate_limited。
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
@@ -1135,6 +1139,7 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1159,6 +1164,51 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1179,6 +1229,7 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1287,6 +1338,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1332,6 +1384,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
|
||||
@@ -91,7 +91,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -127,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -172,9 +162,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -194,6 +186,10 @@ pub struct AccountView {
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
@@ -214,6 +210,8 @@ pub struct AccountView {
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -527,7 +525,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -545,7 +543,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -1333,9 +1331,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1366,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = status.as_str().to_owned();
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -1388,6 +1389,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
access_token_expires_at: expires_at,
|
||||
@@ -1417,10 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1713,18 +1718,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::RateLimited => "rate_limited",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::Invalid => "invalid",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
|
||||
@@ -310,16 +310,6 @@ pub(crate) fn usage_range(
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start =
|
||||
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_today_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
|
||||
@@ -38,7 +38,8 @@ where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
// 概览默认按中国时区当日统计,与单独趋势接口保持同一口径。
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -331,7 +344,7 @@ mod actions {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
@@ -365,7 +378,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
|
||||
@@ -83,6 +83,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -100,12 +101,14 @@ impl AdminTestFixture {
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let dashboard_summary_range = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
@@ -142,6 +145,7 @@ impl AdminTestFixture {
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
dashboard_summary_range,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,6 +417,7 @@ struct UnusedStore {
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -523,7 +528,11 @@ impl AccountStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
*self
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range") = Some(range);
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
|
||||
@@ -270,6 +270,61 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_default_to_current_china_day() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
china_day_start,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_today_range")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let range = fixture
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range")
|
||||
.take()
|
||||
.expect("recorded dashboard summary range");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(range.start, china_day_start(range.end));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
use axum::{
|
||||
|
||||
@@ -660,6 +660,9 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -1096,6 +1099,41 @@ where
|
||||
true
|
||||
}
|
||||
|
||||
/// 原生续写的原账号在本地调度阶段已不可用时,改用完整输入重放。
|
||||
///
|
||||
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
|
||||
/// 对应 Provider 的可携带会话状态时,才可安全移除 native handle 并换号。
|
||||
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt != ContinuationAttempt::Native
|
||||
|| !matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(pin) = self
|
||||
.continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if self
|
||||
.operation
|
||||
.provider_session_state(pin.provider().as_str())
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn reset_uncommitted_observations(&mut self) {
|
||||
self.usage = Usage::new();
|
||||
self.cost = CostEstimate::unavailable();
|
||||
|
||||
@@ -215,6 +215,20 @@ impl AccountAvailability {
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。
|
||||
///
|
||||
/// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用
|
||||
/// 上一次无关的错误信息。
|
||||
#[must_use]
|
||||
pub const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Expired => Some("credential_expired"),
|
||||
Self::Banned => Some("account_banned"),
|
||||
Self::Invalid => Some("credential_invalid"),
|
||||
Self::Unknown | Self::Ready | Self::QuotaExhausted => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析数据库稳定值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
@@ -607,6 +621,8 @@ pub struct AccountStateChange {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口。
|
||||
|
||||
@@ -513,17 +513,27 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
|
||||
}
|
||||
|
||||
fn plan(operation: &Operation) -> RoutingPlan {
|
||||
plan_with_policy(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
account_selection_policy,
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -1514,6 +1524,70 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_native_continuation_replays_with_the_configured_account_policy() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let policy = AccountSelectionPolicy::new(
|
||||
RotationStrategy::RoundRobin,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
);
|
||||
let route_plan = plan_with_policy(&operation, policy);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let original = ProviderAccountId::new("acct_one").expect("account");
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.clone(),
|
||||
);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].account_selection_policy().strategy(),
|
||||
RotationStrategy::RoundRobin
|
||||
);
|
||||
assert!(contexts[1].excluded_accounts().contains(&original));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
|
||||
@@ -13,6 +13,8 @@ use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::Subscriber
|
||||
use crate::config::LoggingConfig;
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
/// OpenAI OAuth 恢复记录含原始 AT/RT,文件日志开启时不能被常规级别过滤掉。
|
||||
const OPENAI_OAUTH_RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
@@ -34,7 +36,7 @@ pub enum LogError {
|
||||
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = EnvFilter::try_new(&directive).map_err(|_| LogError::InvalidFilter)?;
|
||||
let file_filter = file_filter(&directive)?;
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
@@ -97,6 +99,15 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
Ok(LogGuard { _writers: guards })
|
||||
}
|
||||
|
||||
fn file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OPENAI_OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OpenAI OAuth recovery log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
directive.to_owned()
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
use std::path::PathBuf;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use gateway_host::config::{FileLoggingConfig, HostConfig, ListenConfig, LoggingConfig};
|
||||
use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_sink() {
|
||||
let mut config = HostConfig {
|
||||
@@ -32,3 +40,87 @@ fn logging_requires_at_least_one_sink() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_file_logging_overrides_global_log_level() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
write_oauth_recovery_log(PathBuf::from(
|
||||
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
let directory = tempfile::tempdir().expect("create log directory");
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_file_logging_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off,openai_oauth_recovery=off")
|
||||
.output()
|
||||
.expect("run logging child process");
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
|
||||
let log = read_log_directory(directory.path());
|
||||
assert!(log.contains(OAUTH_RECOVERY_LOG_TARGET));
|
||||
assert!(log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
}
|
||||
|
||||
fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
let config = HostConfig {
|
||||
listen: ListenConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
},
|
||||
logging: LoggingConfig {
|
||||
level: "off".to_owned(),
|
||||
stdout: false,
|
||||
file: FileLoggingConfig {
|
||||
enabled: true,
|
||||
directory,
|
||||
retention_days: 1,
|
||||
max_file_size_mb: 1,
|
||||
max_files: 1,
|
||||
},
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create logging runtime");
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OpenAI OAuth recovery test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
fn read_log_directory(directory: &Path) -> String {
|
||||
let mut files = fs::read_dir(directory)
|
||||
.expect("read log directory")
|
||||
.map(|entry| entry.expect("read log entry").path())
|
||||
.collect::<Vec<_>>();
|
||||
files.sort();
|
||||
|
||||
files
|
||||
.into_iter()
|
||||
.map(|path| fs::read_to_string(path).expect("read log file"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.23"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "process", "time"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "net", "process", "time"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -385,7 +385,13 @@ async fn upload_parts(
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "上传并发闸门关闭".to_owned())
|
||||
})?;
|
||||
let offset = (part - 1) * (PART_SIZE as u64);
|
||||
let bytes = read_part(&source, offset, PART_SIZE).await?;
|
||||
let remaining = file_size.checked_sub(offset).ok_or_else(|| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片偏移超出归档大小".to_owned())
|
||||
})?;
|
||||
let part_len = usize::try_from(remaining.min(PART_SIZE as u64)).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片大小溢出".to_owned())
|
||||
})?;
|
||||
let bytes = read_part(&source, offset, part_len).await?;
|
||||
let part_number = i32::try_from(part).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片号溢出".to_owned())
|
||||
})?;
|
||||
@@ -450,20 +456,19 @@ async fn upload_parts(
|
||||
.map_err(map_s3_error)
|
||||
}
|
||||
|
||||
/// 从文件中读取一个分片(最多 `PART_SIZE` 字节)。
|
||||
async fn read_part(source: &Path, offset: u64, max_len: usize) -> Result<Vec<u8>, BackupError> {
|
||||
/// 从文件中读取指定长度的完整分片。
|
||||
async fn read_part(source: &Path, offset: u64, len: usize) -> Result<Vec<u8>, BackupError> {
|
||||
let mut file = tokio::fs::File::open(source)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "打开暂存归档失败".to_owned()))?;
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "定位暂存归档失败".to_owned()))?;
|
||||
let mut buffer = vec![0_u8; max_len];
|
||||
let n = file
|
||||
.read(&mut buffer)
|
||||
let mut buffer = vec![0_u8; len];
|
||||
// Tokio 默认单次文件读取最多 2 MiB;分片必须读取完整,不能把短读当作末尾分片上传。
|
||||
file.read_exact(&mut buffer)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?;
|
||||
buffer.truncate(n);
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountStore;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -85,7 +86,10 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
)?);
|
||||
|
||||
let admin_ports = AdminStorePorts::new(
|
||||
Arc::new(postgres::PgAdminAccountStore::new(pool.clone())),
|
||||
Arc::new(postgres::PgAdminAccountStore::new(
|
||||
pool.clone(),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminAuthStoreAdapter {
|
||||
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
|
||||
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
|
||||
@@ -98,6 +102,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
Arc::new(postgres::PgAdminObservabilityStore::new(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminSettingsStoreAdapter {
|
||||
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
@@ -7,12 +12,82 @@ use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepositor
|
||||
#[derive(Clone)]
|
||||
pub struct PgObservabilityRepository {
|
||||
pool: PgPool,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgObservabilityRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,9 +103,13 @@ pub struct PgAdminObservabilityStore {
|
||||
|
||||
impl PgAdminObservabilityStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool, runtime_signals: Option<RedisCredentialLeaseRepository>) -> Self {
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository: PgObservabilityRepository::new(pool),
|
||||
repository: PgObservabilityRepository::new(pool, cooldowns),
|
||||
runtime_signals,
|
||||
}
|
||||
}
|
||||
@@ -62,11 +141,13 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts, provider_accounts) = futures::try_join!(
|
||||
let (requests, attempts) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
provider_account_metrics(&self.pool, range.end),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
|
||||
@@ -357,8 +357,8 @@ pub(crate) async fn provider_account_metrics(
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when quota_limit_reached then 'rate_limited'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
@@ -425,6 +425,35 @@ pub(crate) async fn active_provider_account_ids(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
//! `gateway-admin` 账号端口的 PostgreSQL adapter。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Admin 账号用例所需的公共账号、留存观测与 revision 事务能力。
|
||||
@@ -11,19 +17,49 @@ pub struct PgAdminAccountStore {
|
||||
accounts: PgProviderAccountRepository,
|
||||
observability: PgObservabilityRepository,
|
||||
control_plane: PgControlPlaneRepository,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgAdminAccountStore {
|
||||
#[must_use]
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
pub fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self {
|
||||
pool: pool.clone(),
|
||||
accounts: PgProviderAccountRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone()),
|
||||
observability: PgObservabilityRepository::new(pool.clone(), cooldowns.clone()),
|
||||
control_plane: PgControlPlaneRepository::new(pool),
|
||||
cooldowns,
|
||||
}
|
||||
}
|
||||
|
||||
/// 批量读取 Redis 429 冷却,返回 `account_id → rate_limited_until` 映射。
|
||||
/// 冷却不可用(Redis 挂/无 cooldown 端口)时返回空映射,账号归入配额耗尽(近似)。
|
||||
async fn load_rate_limited_until(
|
||||
&self,
|
||||
accounts: &[ProviderAccountSummary],
|
||||
) -> AdminStoreResult<BTreeMap<String, SystemTime>> {
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
let mut map = BTreeMap::new();
|
||||
for account in accounts {
|
||||
let Ok(account_id) = CoreProviderAccountId::new(account.id.clone()) else {
|
||||
continue;
|
||||
};
|
||||
let Some(cooldown) = cooldowns.read(&account_id).await.ok().flatten() else {
|
||||
continue;
|
||||
};
|
||||
if cooldown.credential_revision().get() != account.credential_revision.get() {
|
||||
continue;
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until > SystemTime::now() {
|
||||
map.insert(account.id.clone(), until);
|
||||
}
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
async fn usage_observations(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
@@ -242,11 +278,17 @@ impl AccountStore for PgAdminAccountStore {
|
||||
)
|
||||
.map_err(|error| admin_store_error(ENTITY, error))?;
|
||||
let now = Utc::now();
|
||||
let summary = admin_account_summary(&accounts, now);
|
||||
// 批量读取 Redis 429 冷却,供列表状态派生与筛选使用。
|
||||
let rate_limited_until = self
|
||||
.load_rate_limited_until(&accounts)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let summary = admin_account_summary(&accounts, now, &rate_limited_until);
|
||||
let mut items = accounts
|
||||
.into_iter()
|
||||
.filter_map(|account| {
|
||||
let status = admin_account_status(&account, now);
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
let status = admin_account_status(&account, now, until);
|
||||
account_matches_admin_query(&account, status, &query).then_some(
|
||||
AdminAccountListItem {
|
||||
account,
|
||||
|
||||
@@ -247,6 +247,7 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
.map_err(core_store_error)?,
|
||||
availability: change.availability,
|
||||
availability_observed_at: DateTime::<Utc>::from(change.observed_at),
|
||||
message: change.message,
|
||||
})
|
||||
.await
|
||||
.map_err(core_store_error)?;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
use gateway_admin::model::accounts::AccountStatusSignals;
|
||||
|
||||
pub(crate) struct AdminAccountListItem {
|
||||
pub(crate) account: ProviderAccountSummary,
|
||||
pub(crate) status: AdminAccountStatus,
|
||||
@@ -58,52 +60,48 @@ pub(crate) fn account_matches_admin_query(
|
||||
pub(crate) fn admin_account_status(
|
||||
account: &ProviderAccountSummary,
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: Option<SystemTime>,
|
||||
) -> AdminAccountStatus {
|
||||
if !account.enabled {
|
||||
AdminAccountStatus::Disabled
|
||||
} else if account.availability == AccountAvailability::Banned {
|
||||
AdminAccountStatus::Banned
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted {
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == AccountAvailability::Expired
|
||||
|| account.availability == AccountAvailability::Invalid
|
||||
|| account
|
||||
// 派生规则唯一归属 gateway-admin 域层,此处只做类型适配。
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AdminAccountStatus::Expired
|
||||
} else if account.quota_limit_reached {
|
||||
AdminAccountStatus::RateLimited
|
||||
} else if account.availability == AccountAvailability::Unknown {
|
||||
AdminAccountStatus::Invalid
|
||||
} else {
|
||||
AdminAccountStatus::Active
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: account.quota_limit_reached,
|
||||
rate_limited_until,
|
||||
}
|
||||
.derive()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_summary(
|
||||
accounts: &[ProviderAccountSummary],
|
||||
now: DateTime<Utc>,
|
||||
rate_limited_until: &BTreeMap<String, SystemTime>,
|
||||
) -> AccountSummary {
|
||||
let mut summary = AccountSummary {
|
||||
total: u64::try_from(accounts.len()).unwrap_or(u64::MAX),
|
||||
active: 0,
|
||||
normal: 0,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
};
|
||||
for account in accounts {
|
||||
match admin_account_status(account, now) {
|
||||
AdminAccountStatus::Active => summary.active = summary.active.saturating_add(1),
|
||||
let until = rate_limited_until.get(&account.id).copied();
|
||||
match admin_account_status(account, now, until) {
|
||||
AdminAccountStatus::Normal => summary.normal = summary.normal.saturating_add(1),
|
||||
AdminAccountStatus::QuotaExhausted => {
|
||||
summary.quota_exhausted = summary.quota_exhausted.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::RateLimited
|
||||
| AdminAccountStatus::Expired
|
||||
| AdminAccountStatus::Invalid
|
||||
| AdminAccountStatus::Disabled
|
||||
| AdminAccountStatus::Banned => {
|
||||
summary.unavailable = summary.unavailable.saturating_add(1);
|
||||
AdminAccountStatus::RateLimited => {
|
||||
summary.rate_limited = summary.rate_limited.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Disabled => {
|
||||
summary.disabled = summary.disabled.saturating_add(1);
|
||||
}
|
||||
AdminAccountStatus::Error => summary.error = summary.error.saturating_add(1),
|
||||
}
|
||||
}
|
||||
summary
|
||||
@@ -120,9 +118,7 @@ pub(crate) fn sort_admin_account_items(
|
||||
items.sort_by(|left, right| {
|
||||
let ordering = match sort.field {
|
||||
AdminAccountSortField::Email => left.account.email.cmp(&right.account.email),
|
||||
AdminAccountSortField::Status => {
|
||||
admin_account_status_name(left.status).cmp(admin_account_status_name(right.status))
|
||||
}
|
||||
AdminAccountSortField::Status => left.status.as_str().cmp(right.status.as_str()),
|
||||
AdminAccountSortField::PlanType => left.account.plan_type.cmp(&right.account.plan_type),
|
||||
AdminAccountSortField::Usage => left
|
||||
.usage
|
||||
@@ -152,18 +148,6 @@ pub(crate) fn sort_admin_account_items(
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_account_status_name(status: AdminAccountStatus) -> &'static str {
|
||||
match status {
|
||||
AdminAccountStatus::Active => "active",
|
||||
AdminAccountStatus::RateLimited => "rate_limited",
|
||||
AdminAccountStatus::Expired => "expired",
|
||||
AdminAccountStatus::Invalid => "invalid",
|
||||
AdminAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
AdminAccountStatus::Disabled => "disabled",
|
||||
AdminAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_account_record(
|
||||
summary: ProviderAccountSummary,
|
||||
) -> AdminStoreResult<AccountRecord> {
|
||||
@@ -189,6 +173,7 @@ pub(crate) fn admin_account_record(
|
||||
enabled: summary.enabled,
|
||||
availability: summary.availability,
|
||||
availability_observed_at: summary.availability_observed_at,
|
||||
last_error_message: summary.last_error_message,
|
||||
quota_observed_at: summary.quota_observed_at,
|
||||
created_at: summary.created_at,
|
||||
updated_at: summary.updated_at,
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt,
|
||||
str::FromStr,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -95,7 +95,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
availability_observed_at,
|
||||
quota_observed_at, quota_limit_reached, created_at, updated_at
|
||||
quota_observed_at, quota_limit_reached, last_error_message, created_at, updated_at
|
||||
from provider_accounts
|
||||
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
|
||||
order by provider_kind, name, id",
|
||||
@@ -214,6 +214,11 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
when enabled then $4
|
||||
else availability_observed_at
|
||||
end,
|
||||
last_error_message = case
|
||||
when enabled then
|
||||
case when $3 = 'ready' then null else coalesce($5, last_error_message) end
|
||||
else last_error_message
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $4) else updated_at end
|
||||
where id = $1 and credential_revision = $2
|
||||
and (availability_observed_at is null or availability_observed_at <= $4)",
|
||||
@@ -222,6 +227,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
.bind(to_i64(update.expected_revision.get())?)
|
||||
.bind(update.availability.as_str())
|
||||
.bind(update.availability_observed_at)
|
||||
.bind(update.message.as_deref())
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("apply provider account state"))?;
|
||||
|
||||
@@ -90,6 +90,8 @@ pub struct ProviderAccountSummary {
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
/// 快照级限流事实(Provider 物化)。
|
||||
pub quota_limit_reached: bool,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -310,6 +312,8 @@ pub struct ProviderAccountStateUpdate {
|
||||
pub expected_revision: Revision,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 上游原因(错误原文 / 原因码);`None` 不更新错误信息列,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
impl ProviderAccountStateUpdate {
|
||||
@@ -335,14 +339,14 @@ impl ProviderAccountObservation {
|
||||
pub(crate) const ACCOUNT_SELECT: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
provider_quota_json, quota_limit_reached,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts where id = $1";
|
||||
|
||||
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select id, provider_kind, name, email, upstream_user_id,
|
||||
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
|
||||
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, availability,
|
||||
provider_quota_json, quota_limit_reached,
|
||||
provider_quota_json, quota_limit_reached, last_error_message,
|
||||
availability_observed_at, quota_observed_at, created_at, updated_at
|
||||
from provider_accounts
|
||||
where id = any($1::text[]) and provider_kind = $2
|
||||
@@ -445,6 +449,7 @@ pub(crate) fn account_summary_from_row(
|
||||
availability_observed_at: get(&row, "availability_observed_at")?,
|
||||
quota_observed_at: get(&row, "quota_observed_at")?,
|
||||
quota_limit_reached: get(&row, "quota_limit_reached")?,
|
||||
last_error_message: get(&row, "last_error_message")?,
|
||||
created_at: get(&row, "created_at")?,
|
||||
updated_at: get(&row, "updated_at")?,
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
mod postgres;
|
||||
mod redis;
|
||||
mod s3;
|
||||
mod support;
|
||||
|
||||
@@ -156,7 +156,7 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (10, 4));
|
||||
assert_eq!((first_table_count, migration_count), (10, 1));
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,23 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
num::NonZeroU32,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
model::{PageSize, observability as admin_observability},
|
||||
ports::store::ObservabilityStore as AdminObservabilityStore,
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::credential::{CredentialRevision, ProviderAccountId},
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError,
|
||||
},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
DiagnosticDimension, ObservabilityPageNumber, ObservabilityPageSize, ObservabilityRange,
|
||||
ObservabilityRepository, OpsErrorFilter, OpsErrorQuery, PgAdminObservabilityStore,
|
||||
@@ -83,7 +96,7 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
.expect("persist opaque response IDs");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let records = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
@@ -147,7 +160,7 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed account metric states");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
@@ -179,6 +192,184 @@ async fn dashboard_account_metrics_should_partition_account_statuses() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_with_cooldowns_should_only_reclassify_eligible_accounts() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics_cooldowns").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability, quota_limit_reached,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_metrics_active', 'openai', 'active', 'user-active', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_active_cooling', 'openai', 'active-cooling', 'user-active-cooling', 'oauth',
|
||||
'{}'::jsonb, 2, false, $1 + interval '1 day', true, 'ready', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_ready_quota', 'openai', 'ready-quota', 'user-ready-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_cooling', 'openai', 'cooling', 'user-cooling', 'oauth',
|
||||
'{}'::jsonb, 3, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_stale_cooldown', 'openai', 'stale-cooldown', 'user-stale-cooldown', 'oauth',
|
||||
'{}'::jsonb, 4, false, $1 + interval '1 day', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_persistent_quota', 'openai', 'persistent-quota', 'user-persistent-quota', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'quota_exhausted', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 5, false, $1 - interval '1 second', true, 'ready', true,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_unknown', 'openai', 'unknown', 'user-unknown', 'oauth',
|
||||
'{}'::jsonb, 6, false, $1 + interval '1 day', true, 'unknown', false,
|
||||
$1, $1, $1),
|
||||
('acct_metrics_disabled', 'openai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', false,
|
||||
$1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed cooldown account metric states");
|
||||
let cooldowns = StaticCooldowns::new([
|
||||
test_cooldown("acct_metrics_active_cooling", 2),
|
||||
test_cooldown("acct_metrics_cooling", 3),
|
||||
test_cooldown("acct_metrics_expired", 5),
|
||||
test_cooldown("acct_metrics_unknown", 6),
|
||||
test_cooldown("acct_metrics_stale_cooldown", 3),
|
||||
]);
|
||||
let repository =
|
||||
PgObservabilityRepository::new(database.pool.clone(), Some(Arc::new(cooldowns)));
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("dashboard range");
|
||||
|
||||
let metrics = repository
|
||||
.dashboard_summary(range)
|
||||
.await
|
||||
.expect("dashboard summary")
|
||||
.provider_accounts;
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
metrics.total,
|
||||
metrics.enabled,
|
||||
metrics.active,
|
||||
metrics.rate_limited,
|
||||
metrics.quota_exhausted,
|
||||
metrics.expired,
|
||||
metrics.invalid,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
metrics.unavailable,
|
||||
),
|
||||
(9, 8, 1, 2, 3, 1, 1, 1, 0, 8),
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.total,
|
||||
metrics.active
|
||||
+ metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
assert_eq!(
|
||||
metrics.unavailable,
|
||||
metrics.rate_limited
|
||||
+ metrics.quota_exhausted
|
||||
+ metrics.expired
|
||||
+ metrics.invalid
|
||||
+ metrics.disabled
|
||||
+ metrics.banned
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
struct StaticCooldowns {
|
||||
cooldowns: BTreeMap<ProviderAccountId, ProviderCooldown>,
|
||||
}
|
||||
|
||||
impl StaticCooldowns {
|
||||
fn new(cooldowns: impl IntoIterator<Item = ProviderCooldown>) -> Self {
|
||||
Self {
|
||||
cooldowns: cooldowns
|
||||
.into_iter()
|
||||
.map(|cooldown| (cooldown.account_id().clone(), cooldown))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for StaticCooldowns {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move { Ok(self.cooldowns.get(account_id).cloned()) })
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
fn test_cooldown(account_id: &str, revision: u64) -> ProviderCooldown {
|
||||
ProviderCooldown::new(
|
||||
ProviderAccountId::new(account_id).expect("test account ID"),
|
||||
CredentialRevision::new(revision).expect("test revision"),
|
||||
SystemTime::now() + Duration::from_secs(60),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
let Some(database) = TestDatabase::create("calculated_usage_billing_facts").await else {
|
||||
@@ -193,7 +384,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
.expect("seed calculated billing facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let facts = repository
|
||||
.usage_calculated_billing_facts(range, UsageRecordFilter::default())
|
||||
@@ -207,7 +398,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
let facts = store
|
||||
.usage_calculated_billing_facts(
|
||||
admin_observability::TimeRange::new(range.start, range.end)
|
||||
@@ -235,7 +426,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
let range =
|
||||
admin_observability::TimeRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("admin observability range");
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None, None);
|
||||
|
||||
let dashboard = store
|
||||
.dashboard_summary(range)
|
||||
@@ -524,7 +715,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("seed observability facts");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
|
||||
let dashboard = repository
|
||||
.dashboard_summary(range)
|
||||
|
||||
@@ -160,6 +160,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -170,6 +171,7 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
assert!(
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
@@ -200,6 +202,61 @@ async fn delayed_provider_observations_cannot_overwrite_newer_state_or_quota() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_state_message_is_persisted_and_cleared_on_recovery() {
|
||||
let Some(database) = TestDatabase::create("provider_account_state_message").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let account_id = ProviderAccountId::new("acct_state_message").expect("account ID");
|
||||
repository
|
||||
.insert_provider_account(account(account_id.as_str(), "user-state-message"))
|
||||
.await
|
||||
.expect("insert state message fixture");
|
||||
let revision = CredentialRevision::new(1).expect("revision");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: Some("upstream returned 402 payment required".to_owned()),
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("persist error message");
|
||||
let stored: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read error message");
|
||||
assert_eq!(
|
||||
stored.as_deref(),
|
||||
Some("upstream returned 402 payment required")
|
||||
);
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: revision,
|
||||
availability: AccountAvailability::Ready,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("recover account");
|
||||
let cleared: Option<String> =
|
||||
sqlx::query_scalar("select last_error_message from provider_accounts where id = $1")
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read cleared error message");
|
||||
assert_eq!(cleared, None);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_usage() {
|
||||
let Some(database) = TestDatabase::create("provider_account_terminal_list").await else {
|
||||
@@ -231,6 +288,21 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("insert account list fixture");
|
||||
}
|
||||
repository
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: ProviderAccountId::new("acct_charlie").expect("account id"),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({"rate_limit": {"limit_reached": true}})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed rate-limited account");
|
||||
|
||||
seed_model_request(
|
||||
&database.pool,
|
||||
@@ -289,7 +361,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
.await
|
||||
.expect("seed charlie usage");
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let usage_page = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
@@ -307,14 +379,18 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(usage_page.config_revision.get(), 1);
|
||||
assert_eq!(usage_page.total, 6);
|
||||
assert_eq!(usage_page.summary.total, 6);
|
||||
assert_eq!(usage_page.summary.active, 2);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 1);
|
||||
assert_eq!(usage_page.summary.unavailable, 3);
|
||||
assert_eq!(usage_page.summary.normal, 1);
|
||||
assert_eq!(usage_page.summary.quota_exhausted, 2);
|
||||
assert_eq!(usage_page.summary.rate_limited, 0);
|
||||
assert_eq!(usage_page.summary.disabled, 1);
|
||||
assert_eq!(usage_page.summary.error, 2);
|
||||
assert_eq!(
|
||||
usage_page.summary.total,
|
||||
usage_page.summary.active
|
||||
usage_page.summary.normal
|
||||
+ usage_page.summary.quota_exhausted
|
||||
+ usage_page.summary.unavailable
|
||||
+ usage_page.summary.rate_limited
|
||||
+ usage_page.summary.disabled
|
||||
+ usage_page.summary.error
|
||||
);
|
||||
assert_eq!(
|
||||
usage_page
|
||||
@@ -354,7 +430,7 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("Provider kind")),
|
||||
search: Some("ALPHA@EXAMPLE".to_owned()),
|
||||
status: Some(AccountStatus::Active),
|
||||
status: Some(AccountStatus::Normal),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
@@ -364,18 +440,20 @@ async fn terminal_admin_list_filters_and_sorts_before_pagination_with_retained_u
|
||||
assert_eq!(filtered.items[0].id, "acct_alpha");
|
||||
assert_eq!(filtered.items[0].provider_kind.as_str(), "openai");
|
||||
|
||||
let banned = store
|
||||
let error_accounts = store
|
||||
.list_accounts(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: Some(AccountStatus::Banned),
|
||||
status: Some(AccountStatus::Error),
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("filter banned accounts");
|
||||
assert_eq!(banned.items[0].id, "acct_beta");
|
||||
.expect("filter error accounts");
|
||||
assert_eq!(error_accounts.items.len(), 2);
|
||||
assert_eq!(error_accounts.items[0].id, "acct_beta");
|
||||
assert_eq!(error_accounts.items[1].id, "acct_invalid");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -403,7 +481,7 @@ async fn terminal_credential_list_preserves_grouped_filters_and_unpaged_collecti
|
||||
.expect("insert xAI credential fixture");
|
||||
}
|
||||
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let provider = ProviderKind::new("xai").expect("xAI Provider kind");
|
||||
let complete = store
|
||||
.list_credentials(
|
||||
@@ -484,7 +562,7 @@ async fn terminal_admin_usage_chunks_large_selections_and_preserves_exact_costs(
|
||||
let mut account_ids = vec!["acct_usage_exact".to_owned()];
|
||||
account_ids.extend((0..200).map(|index| format!("missing_account_{index}")));
|
||||
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage(
|
||||
TimeRange {
|
||||
start: now - TimeDelta::hours(1),
|
||||
@@ -576,7 +654,7 @@ async fn terminal_admin_quota_window_usage_prefers_provider_total_and_falls_back
|
||||
.await
|
||||
.expect("separate provider total from fallback components");
|
||||
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone())
|
||||
let mut usage = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.load_account_usage_by_windows(&[
|
||||
AccountUsageWindowQuery {
|
||||
account_id: "acct_quota_window".to_owned(),
|
||||
@@ -616,7 +694,7 @@ async fn terminal_admin_mutations_keep_revision_account_and_audit_atomic() {
|
||||
.insert_provider_account(account("acct_terminal_mutation", "user-terminal-mutation"))
|
||||
.await
|
||||
.expect("insert mutation account");
|
||||
let store = PgAdminAccountStore::new(database.pool.clone());
|
||||
let store = PgAdminAccountStore::new(database.pool.clone(), None);
|
||||
let context = MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "request_terminal_mutation".to_owned(),
|
||||
@@ -689,7 +767,7 @@ async fn terminal_admin_delete_removes_enabled_accounts_in_one_transaction() {
|
||||
.expect("insert enabled account");
|
||||
}
|
||||
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone())
|
||||
let revision = PgAdminAccountStore::new(database.pool.clone(), None)
|
||||
.delete_accounts(
|
||||
DeleteAccounts {
|
||||
account_ids: vec![
|
||||
@@ -1193,6 +1271,7 @@ async fn xai_free_quota_exhaustion_state_is_persisted() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -1231,6 +1310,7 @@ async fn xai_resettable_usage_limit_state_is_persisted() {
|
||||
let observed_at = SystemTime::now();
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
@@ -1280,6 +1360,7 @@ async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
|
||||
@@ -39,7 +39,7 @@ async fn request_snapshots_should_survive_account_deletion() {
|
||||
.await
|
||||
.expect("delete provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
@@ -154,7 +154,7 @@ async fn attempts_should_keep_their_own_account_snapshots() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_snap_ab")
|
||||
.await
|
||||
@@ -276,7 +276,7 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let errors = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
@@ -376,7 +376,7 @@ async fn diagnostics_should_group_same_email_accounts_by_stable_ref() {
|
||||
.await
|
||||
.expect("delete provider accounts");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -445,7 +445,7 @@ async fn diagnostics_should_fallback_to_name_then_ref_for_missing_snapshots() {
|
||||
.expect("insert legacy request without live account");
|
||||
finalize_request(&database.pool, "req_snap_legacy", started_at).await;
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -511,7 +511,7 @@ async fn failure_diagnostics_should_only_include_errored_requests() {
|
||||
.await
|
||||
.expect("mark request failed");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -566,7 +566,7 @@ async fn api_key_diagnostics_should_display_key_name_and_fallback_to_ref() {
|
||||
.await
|
||||
.expect("set request b latency");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let diagnostics = repository
|
||||
.usage_diagnostics(
|
||||
range_around(started_at),
|
||||
@@ -633,7 +633,7 @@ async fn renaming_account_should_not_rewrite_historical_snapshots() {
|
||||
.await
|
||||
.expect("rename provider account");
|
||||
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone(), None);
|
||||
let page = repository
|
||||
.list_usage_records(usage_query(started_at, UsageRecordFilter::default()))
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
use std::io;
|
||||
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::backup::{BackupObjectMetadata, BackupStorageConfig};
|
||||
use gateway_admin::ports::backup::{BackupObjectStorePort, UploadObjectRequest};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
use gateway_store::backup::s3::S3ObjectStoreAdapter;
|
||||
use secrecy::SecretString;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
const PART_SIZE_ABOVE_TOKIO_DEFAULT: usize = 2 * 1024 * 1024 + 1;
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_file_sends_full_part_beyond_tokio_default_buffer_limit() {
|
||||
let directory = tempfile::tempdir().expect("create temporary directory");
|
||||
let source = directory.path().join("archive.dump");
|
||||
let expected = vec![b'x'; PART_SIZE_ABOVE_TOKIO_DEFAULT];
|
||||
tokio::fs::write(&source, &expected)
|
||||
.await
|
||||
.expect("write temporary archive");
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind multipart probe server");
|
||||
let endpoint = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("read listener address")
|
||||
);
|
||||
let server = tokio::spawn(serve_multipart_probe(listener));
|
||||
let storage = BackupStorageConfig {
|
||||
storage_revision: 1,
|
||||
endpoint,
|
||||
region: "auto".to_owned(),
|
||||
bucket: "backup-bucket".to_owned(),
|
||||
access_key_id: "test-access-key".to_owned(),
|
||||
secret_access_key: SecretString::from("test-secret-key"),
|
||||
prefix: "backups".to_owned(),
|
||||
force_path_style: true,
|
||||
};
|
||||
let request = UploadObjectRequest {
|
||||
object_key: "backups/archive.dump".to_owned(),
|
||||
source,
|
||||
metadata: BackupObjectMetadata::new(
|
||||
"backup_test".to_owned(),
|
||||
"a".repeat(64),
|
||||
Utc::now(),
|
||||
PART_SIZE_ABOVE_TOKIO_DEFAULT as u64,
|
||||
),
|
||||
cancellation: CancellationToken::new(),
|
||||
};
|
||||
|
||||
S3ObjectStoreAdapter::new()
|
||||
.upload_file(&storage, request)
|
||||
.await
|
||||
.expect("upload multipart archive");
|
||||
let uploaded_len = server
|
||||
.await
|
||||
.expect("multipart probe task did not panic")
|
||||
.expect("serve multipart probe");
|
||||
|
||||
assert_eq!(uploaded_len, PART_SIZE_ABOVE_TOKIO_DEFAULT);
|
||||
}
|
||||
|
||||
async fn serve_multipart_probe(listener: TcpListener) -> io::Result<usize> {
|
||||
let mut uploaded_len = None;
|
||||
for _ in 0..3 {
|
||||
let (mut stream, _) = listener.accept().await?;
|
||||
let (request, body) = read_http_request(&mut stream).await?;
|
||||
let response = if request.starts_with("POST ") && request.contains("uploads") {
|
||||
xml_response(
|
||||
"<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><UploadId>probe-upload</UploadId></InitiateMultipartUploadResult>",
|
||||
)
|
||||
} else if request.starts_with("PUT ") && request.contains("partNumber=") {
|
||||
uploaded_len = Some(body.len());
|
||||
"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\netag: \"probe-part\"\r\n\r\n".to_owned()
|
||||
} else if request.starts_with("POST ") && request.contains("uploadId=") {
|
||||
xml_response(
|
||||
"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"probe-part\"</ETag></CompleteMultipartUploadResult>",
|
||||
)
|
||||
} else {
|
||||
return Err(io::Error::other("unexpected multipart probe request"));
|
||||
};
|
||||
stream.write_all(response.as_bytes()).await?;
|
||||
stream.shutdown().await?;
|
||||
}
|
||||
uploaded_len.ok_or_else(|| io::Error::other("multipart probe did not receive an upload part"))
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) -> io::Result<(String, Vec<u8>)> {
|
||||
let mut bytes = Vec::new();
|
||||
let header_end = loop {
|
||||
if let Some(position) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
|
||||
break position + 4;
|
||||
}
|
||||
let mut chunk = [0_u8; 8 * 1024];
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"multipart probe request ended before headers",
|
||||
));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
};
|
||||
let headers = String::from_utf8_lossy(&bytes[..header_end]).into_owned();
|
||||
let content_len = headers
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.and_then(|(_, value)| value.trim().parse::<usize>().ok())
|
||||
.unwrap_or_default();
|
||||
if headers
|
||||
.to_ascii_lowercase()
|
||||
.contains("expect: 100-continue")
|
||||
{
|
||||
stream.write_all(b"HTTP/1.1 100 Continue\r\n\r\n").await?;
|
||||
}
|
||||
while bytes.len() < header_end + content_len {
|
||||
let mut chunk = [0_u8; 8 * 1024];
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"multipart probe request ended before body",
|
||||
));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
Ok((
|
||||
headers,
|
||||
bytes[header_end..header_end + content_len].to_vec(),
|
||||
))
|
||||
}
|
||||
|
||||
fn xml_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
@@ -58,6 +58,7 @@ zstd = "0.13"
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-thread", "test-util"] }
|
||||
tracing-subscriber = { version = "0.3.23", features = ["json"] }
|
||||
wiremock = "0.6.5"
|
||||
|
||||
[[test]]
|
||||
|
||||
@@ -50,7 +50,7 @@ use crate::credential::{
|
||||
CodexCredentialAdminService, CodexCredentialCatalogError, CodexCredentialCatalogService,
|
||||
CodexCredentialQuotaError, CodexCredentialQuotaService, CodexOAuthAdmin, CodexOAuthAdminError,
|
||||
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
|
||||
CodexPendingAuthorization, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CodexPendingAuthorization, CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential, ExportManagedCodexCredential,
|
||||
RotateManagedCodexCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
refresh_time,
|
||||
@@ -194,12 +194,13 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
let profile = self.profile.snapshot();
|
||||
let release = self.desktop_release.snapshot();
|
||||
let user_agent = profile.user_agent();
|
||||
let client_identity = format!("{}; {}", profile.originator, profile.desktop_version);
|
||||
let release = dashboard_desktop_release(&profile, release);
|
||||
Some(DashboardWireProfile {
|
||||
provider: self.provider_kind.as_str().to_owned(),
|
||||
product: profile.originator,
|
||||
version: profile.desktop_version,
|
||||
build: Some(profile.desktop_build),
|
||||
version: profile.codex_version,
|
||||
build: None,
|
||||
target: DashboardWireTarget {
|
||||
os_type: profile.os_type,
|
||||
os_version: profile.os_version,
|
||||
@@ -208,8 +209,8 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
},
|
||||
user_agent,
|
||||
attributes: vec![DashboardWireAttribute {
|
||||
label: "Codex Core".to_owned(),
|
||||
value: profile.codex_version,
|
||||
label: "客户端标识".to_owned(),
|
||||
value: client_identity,
|
||||
}],
|
||||
verified_at: Some(profile.verified_at),
|
||||
release: Some(release),
|
||||
@@ -475,7 +476,14 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
};
|
||||
Ok(project_quota(snapshot, &account))
|
||||
let mut quota = project_quota(snapshot, &account);
|
||||
quota.rate_limited_until = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.map_err(map_quota_error)?
|
||||
.map(DateTime::<Utc>::from);
|
||||
Ok(quota)
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -740,6 +748,7 @@ fn empty_quota() -> ProviderQuota {
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -774,6 +783,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
let windows: Vec<ProviderQuotaWindow> = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| should_project_quota_window(window))
|
||||
.map(|window| {
|
||||
let mut data = Map::new();
|
||||
data.insert(
|
||||
@@ -790,6 +800,7 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
label: codex_quota_window_label(
|
||||
window.limit_name(),
|
||||
window.kind(),
|
||||
window.role(),
|
||||
window.source(),
|
||||
window.window_seconds(),
|
||||
),
|
||||
@@ -811,12 +822,24 @@ fn project_quota_snapshot(snapshot: CodexAccountQuotaSnapshot) -> ProviderQuota
|
||||
refresh_token_expires_at: None,
|
||||
windows,
|
||||
limit_reached,
|
||||
rate_limited_until: None,
|
||||
provider_data: Some(ProviderDocument::new(OpaqueProviderData::new(
|
||||
provider_data,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// `secondary_window` 有时只是上游的空占位:没有时长、重置时间,也没有用量。
|
||||
/// 它不能被可靠翻译成 5 小时或周额度,因此不应占用账号面板;带有实际事实的
|
||||
/// 次级窗口(时长、重置、非零用量或触顶)仍完整保留。
|
||||
fn should_project_quota_window(window: &CodexQuotaWindow) -> bool {
|
||||
window.role() != CodexQuotaWindowRole::Secondary
|
||||
|| window.window_seconds().is_some()
|
||||
|| window.reset_at().is_some()
|
||||
|| window.used_percent().is_some_and(|used| used > 0.0)
|
||||
|| window.limit_reached()
|
||||
}
|
||||
|
||||
fn quota_windows_limit_reached(windows: &[ProviderQuotaWindow]) -> bool {
|
||||
windows.iter().any(|window| window.limit_reached)
|
||||
}
|
||||
@@ -882,45 +905,66 @@ const fn quota_role(role: CodexQuotaWindowRole) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// 窗口展示名,优先使用上游 `limit_name`(原文,`_` 转空格);
|
||||
/// 无 limit_name 时按窗口时长猜(±5% 容差,汉化输出),猜不中兜底「额度」。
|
||||
/// 按窗口时长显示汉化额度名;非核心桶补上上游桶名称作为前缀。
|
||||
fn codex_quota_window_label(
|
||||
limit_name: Option<&str>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
role: CodexQuotaWindowRole,
|
||||
source: &str,
|
||||
window_seconds: Option<u64>,
|
||||
) -> String {
|
||||
if let Some(limit_name) = limit_name {
|
||||
return limit_name.replace('_', " ");
|
||||
}
|
||||
let base = match kind {
|
||||
CodexQuotaWindowKind::Monthly => "月限额".to_owned(),
|
||||
CodexQuotaWindowKind::Weekly => "周限额".to_owned(),
|
||||
CodexQuotaWindowKind::Monthly => "月额度".to_owned(),
|
||||
CodexQuotaWindowKind::Weekly => "周额度".to_owned(),
|
||||
CodexQuotaWindowKind::ShortTerm => {
|
||||
if window_seconds.is_some_and(|seconds| seconds > 86_400) {
|
||||
"周限额".to_owned()
|
||||
"周额度".to_owned()
|
||||
} else {
|
||||
"5小时限额".to_owned()
|
||||
"5小时额度".to_owned()
|
||||
}
|
||||
}
|
||||
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds),
|
||||
CodexQuotaWindowKind::Other => custom_quota_window_label(window_seconds, role),
|
||||
};
|
||||
if matches!(source, "core" | "codex" | "code_review" | "spend_control") {
|
||||
return base;
|
||||
if let Some(prefix) = codex_quota_window_prefix(limit_name, source) {
|
||||
return format!("{prefix} · {base}");
|
||||
}
|
||||
format!("{source} · {base}")
|
||||
base
|
||||
}
|
||||
|
||||
fn custom_quota_window_label(window_seconds: Option<u64>) -> String {
|
||||
fn codex_quota_window_prefix(limit_name: Option<&str>, source: &str) -> Option<String> {
|
||||
let source_identifier = quota_identifier(source);
|
||||
if matches!(source_identifier.as_str(), "core" | "codex") {
|
||||
return None;
|
||||
}
|
||||
let label = limit_name.unwrap_or(source);
|
||||
match quota_identifier(label).as_str() {
|
||||
"premium" => Some("Premium".to_owned()),
|
||||
"code_review" => Some("代码审查".to_owned()),
|
||||
"spend_control" => Some("消费控制".to_owned()),
|
||||
_ => Some(label.replace('_', " ")),
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_identifier(value: &str) -> String {
|
||||
value.trim().to_ascii_lowercase().replace(['-', ' '], "_")
|
||||
}
|
||||
|
||||
fn custom_quota_window_label(window_seconds: Option<u64>, role: CodexQuotaWindowRole) -> String {
|
||||
let Some(seconds) = window_seconds.filter(|seconds| *seconds > 0) else {
|
||||
return "额度".to_owned();
|
||||
// 与官方客户端一致:没有时长时不臆测为 5 小时或周额度;保留
|
||||
// primary/secondary 语义,让用户知道这是上游未标明时长的独立窗口。
|
||||
return match role {
|
||||
CodexQuotaWindowRole::Primary => "主额度".to_owned(),
|
||||
CodexQuotaWindowRole::Secondary => "次级额度".to_owned(),
|
||||
CodexQuotaWindowRole::Monthly => "月额度".to_owned(),
|
||||
};
|
||||
};
|
||||
if seconds % 86_400 == 0 {
|
||||
format!("{}日限额", seconds / 86_400)
|
||||
format!("{}日额度", seconds / 86_400)
|
||||
} else if seconds % 3_600 == 0 {
|
||||
format!("{}小时限额", seconds / 3_600)
|
||||
format!("{}小时额度", seconds / 3_600)
|
||||
} else {
|
||||
format!("{}分钟限额", seconds.div_ceil(60))
|
||||
format!("{}分钟额度", seconds.div_ceil(60))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1337,14 +1381,15 @@ fn map_credential_admin_error(error: CodexCredentialAdminError) -> ProviderAdmin
|
||||
| Error::MissingRefreshToken
|
||||
| Error::RefreshRejected
|
||||
| Error::AccountBanned
|
||||
| Error::AccountProfileRejected
|
||||
| Error::IdentityRejected => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound => ProviderAdminErrorKind::NotFound,
|
||||
Error::RefreshLeaseUnavailable | Error::RefreshAmbiguous => {
|
||||
ProviderAdminErrorKind::Conflict
|
||||
}
|
||||
Error::RefreshUnavailable | Error::IdentityUnavailable => {
|
||||
ProviderAdminErrorKind::Unavailable
|
||||
}
|
||||
Error::RefreshUnavailable
|
||||
| Error::AccountProfileUnavailable
|
||||
| Error::IdentityUnavailable => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1358,6 +1403,8 @@ const fn credential_admin_error_code(error: CodexCredentialAdminError) -> &'stat
|
||||
CodexCredentialAdminError::RefreshLeaseUnavailable => "refresh_lease_unavailable",
|
||||
CodexCredentialAdminError::RefreshRejected => "refresh_rejected",
|
||||
CodexCredentialAdminError::AccountBanned => "account_banned",
|
||||
CodexCredentialAdminError::AccountProfileRejected => "account_profile_rejected",
|
||||
CodexCredentialAdminError::AccountProfileUnavailable => "account_profile_unavailable",
|
||||
CodexCredentialAdminError::RefreshUnavailable => "refresh_unavailable",
|
||||
CodexCredentialAdminError::RefreshAmbiguous => "refresh_ambiguous",
|
||||
CodexCredentialAdminError::IdentityRejected => "identity_rejected",
|
||||
|
||||
@@ -43,10 +43,8 @@ impl OpenAiConfig {
|
||||
self.quota.validate()?;
|
||||
self.auth.validate()?;
|
||||
self.wire_profile.validate()?;
|
||||
self.identity_secret_path = source_dir
|
||||
.parent()
|
||||
.unwrap_or(source_dir)
|
||||
.join(".runtime/data/identity_hmac_secret");
|
||||
let project_root = source_dir.parent().unwrap_or(source_dir);
|
||||
self.identity_secret_path = project_root.join(".runtime/data/identity_hmac_secret");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -278,6 +276,7 @@ impl CodexAuthSettings {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CodexWireProfileConfig {
|
||||
pub originator: String,
|
||||
/// npm `@openai/codex` 启动版本;运行时发布任务会更新此画像字段。
|
||||
pub codex_version: String,
|
||||
pub desktop_version: String,
|
||||
pub desktop_build: String,
|
||||
@@ -292,10 +291,10 @@ impl Default for CodexWireProfileConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
originator: "Codex Desktop".to_owned(),
|
||||
codex_version: "0.145.0".to_owned(),
|
||||
desktop_version: "26.721.31836".to_owned(),
|
||||
desktop_build: "5828".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
codex_version: "0.146.0".to_owned(),
|
||||
desktop_version: "26.727.51351".to_owned(),
|
||||
desktop_build: "6119".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
|
||||
@@ -26,8 +26,9 @@ use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
CodexAuthenticatedAccountSource, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
};
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{RefreshFailure, TokenRefresher};
|
||||
use super::types::{
|
||||
@@ -40,6 +41,8 @@ const PROVIDER_NAME: &str = "openai";
|
||||
const CODEX_CREDENTIAL_SCHEMA_VERSION: u32 = 1;
|
||||
const MAX_BATCH: usize = 200;
|
||||
const MAX_IMPORT_DOCUMENT_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_IMPORT_EMAIL_BYTES: usize = 512;
|
||||
const MAX_IMPORT_PLAN_BYTES: usize = 128;
|
||||
|
||||
pub struct ImportCodexOAuthCredential {
|
||||
pub account_id: String,
|
||||
@@ -365,6 +368,10 @@ pub enum CodexCredentialAdminError {
|
||||
RefreshUnavailable,
|
||||
#[error("Codex refresh send state is ambiguous")]
|
||||
RefreshAmbiguous,
|
||||
#[error("Codex account profile request was rejected")]
|
||||
AccountProfileRejected,
|
||||
#[error("Codex account profile request is unavailable")]
|
||||
AccountProfileUnavailable,
|
||||
#[error("Codex refreshed identity was rejected")]
|
||||
IdentityRejected,
|
||||
#[error("Codex identity verification is unavailable")]
|
||||
@@ -387,8 +394,7 @@ impl CodexCredentialAdmin {
|
||||
if input.name.trim().is_empty() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let access_token_expires_at =
|
||||
required_time(input.verified_account.access_token_expires_at)?;
|
||||
let access_token_expires_at = optional_time(input.verified_account.access_token_expires_at);
|
||||
let revision =
|
||||
CredentialRevision::new(1).map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let upstream_user_id = input.verified_account.chatgpt_user_id.clone();
|
||||
@@ -402,7 +408,7 @@ impl CodexCredentialAdmin {
|
||||
upstream_user_id,
|
||||
CODEX_AUTHENTICATION_KIND_OAUTH.to_owned(),
|
||||
revision,
|
||||
Some(access_token_expires_at),
|
||||
access_token_expires_at,
|
||||
)
|
||||
.with_profile(
|
||||
input.verified_account.email,
|
||||
@@ -499,12 +505,60 @@ impl CodexCredentialAdmin {
|
||||
self.prepare_oauth_rotation(input, false)
|
||||
}
|
||||
|
||||
/// 完整 OAuth 授权按目标数据 ID 替换凭据与上游账号身份。
|
||||
pub(crate) fn prepare_reauthorization(
|
||||
/// 构建一次成功 RT exchange 的 CAS 写入,保留已存账号身份资料。
|
||||
///
|
||||
/// Refresh endpoint 已经完成本次 token 交换的授权;这里不对新 access
|
||||
/// token 重新执行身份验证。
|
||||
pub(crate) fn prepare_refreshed_oauth_rotation(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
current: LoadedCredential,
|
||||
secret: CodexOAuthSecret,
|
||||
access_token_expires_at: DateTime<Utc>,
|
||||
next_refresh_at: Option<DateTime<Utc>>,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
self.prepare_oauth_rotation(input, true)
|
||||
if current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| current.account.authentication_kind() != CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
{
|
||||
return Err(CodexCredentialAdminError::IdentityMismatch);
|
||||
}
|
||||
let mut data = CodexCredentialCodec::decode_complete(¤t.credential)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CodexCredentialAdminError::IdentityMismatch)?;
|
||||
oauth.access_token = secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(|value| value.expose_secret().to_owned());
|
||||
oauth.id_token = secret
|
||||
.id_token
|
||||
.as_ref()
|
||||
.map(|value| value.expose_secret().to_owned());
|
||||
let credential = CodexCredentialCodec::encode_complete(data)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let profile = ProviderAccountUpdate {
|
||||
account_id: current.account.id().clone(),
|
||||
name: current.account.name().to_owned(),
|
||||
email: current.account.email().map(str::to_owned),
|
||||
plan_type: current.account.plan_type().map(str::to_owned),
|
||||
};
|
||||
let credential = CredentialCasUpdate::new(
|
||||
current.account.id().clone(),
|
||||
current.account.revision(),
|
||||
profile.clone(),
|
||||
credential,
|
||||
secret.refresh_token.is_some(),
|
||||
Some(SystemTime::from(access_token_expires_at)),
|
||||
optional_time(next_refresh_at),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
Ok(PreparedCodexCredentialRotation {
|
||||
profile,
|
||||
credential,
|
||||
replacement_identity: None,
|
||||
refresh_guards: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn prepare_oauth_rotation(
|
||||
@@ -588,7 +642,7 @@ impl CodexCredentialAdmin {
|
||||
/// 有状态的 Codex 手工刷新边界;消费调用方刚读取的当前 credential 并准备 CAS。
|
||||
pub struct CodexCredentialAdminService {
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
}
|
||||
@@ -598,7 +652,7 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
formatter
|
||||
.debug_struct("CodexCredentialAdminService")
|
||||
.field("refresher", &"TokenRefresher")
|
||||
.field("verifier", &"CodexAccountIdentityVerifier")
|
||||
.field("account_source", &"CodexAuthenticatedAccountSource")
|
||||
.field("leases", &"ProviderLeasePort")
|
||||
.field("runtime_policy", &"ProviderRuntimePolicyPort")
|
||||
.finish()
|
||||
@@ -608,19 +662,19 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
impl CodexCredentialAdminService {
|
||||
pub fn new(
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
refresher,
|
||||
verifier,
|
||||
account_source,
|
||||
leases,
|
||||
runtime_policy,
|
||||
}
|
||||
}
|
||||
|
||||
/// 官方 RT exchange + AT 身份验证;结果由 App 在同一 revision/audit 事务中提交。
|
||||
/// 官方 RT exchange;结果由 App 在同一 revision/audit 事务中提交。
|
||||
pub async fn manual_refresh(
|
||||
&self,
|
||||
current: LoadedCredential,
|
||||
@@ -632,7 +686,6 @@ impl CodexCredentialAdminService {
|
||||
}
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)?;
|
||||
let expectation = current_identity_expectation(¤t.account, &runtime)?;
|
||||
let oauth = runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
@@ -677,9 +730,6 @@ impl CodexCredentialAdminService {
|
||||
.refresh(refresh_token.expose_secret())
|
||||
.await
|
||||
.map_err(map_refresh_failure)?;
|
||||
if tokens.access_token.trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
refresh_token: tokens
|
||||
@@ -688,24 +738,30 @@ impl CodexCredentialAdminService {
|
||||
.or_else(|| oauth.refresh_token.clone()),
|
||||
id_token: oauth.id_token.clone(),
|
||||
};
|
||||
let verified_account = self
|
||||
.verifier
|
||||
.verify(&secret, &expectation)
|
||||
.await
|
||||
.and_then(super::identity::CodexIdentityVerification::into_complete)
|
||||
.map_err(map_identity_error)?;
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ManualRefresh,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
if secret.access_token.expose_secret().trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let access_token_expires_at = SystemTime::now()
|
||||
.checked_add(tokens.expires_in)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
&account_id,
|
||||
verified_account.access_token_expires_at,
|
||||
Some(access_token_expires_at),
|
||||
secret.refresh_token.is_some(),
|
||||
)?;
|
||||
let mut prepared = CodexCredentialAdmin.prepare_rotation(RotateManagedCodexCredential {
|
||||
let mut prepared = CodexCredentialAdmin.prepare_refreshed_oauth_rotation(
|
||||
current,
|
||||
secret,
|
||||
verified_account,
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
})?;
|
||||
)?;
|
||||
prepared.refresh_guards = Some(ProviderRefreshGuards {
|
||||
_capacity: capacity_guard,
|
||||
_account: account_guard,
|
||||
@@ -713,7 +769,7 @@ impl CodexCredentialAdminService {
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
/// 从 OAuth token 对补全账号事实,并归一到唯一 `NewProviderAccount` 写入路径。
|
||||
/// 用 access token 补全账号事实,并归一到唯一 `NewProviderAccount` 写入路径。
|
||||
pub async fn prepare_import_document(
|
||||
&self,
|
||||
payload: Value,
|
||||
@@ -743,9 +799,23 @@ impl CodexCredentialAdminService {
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (secret, verified_account) = self
|
||||
.verify_import_tokens(access_token.clone(), refresh_token.clone())
|
||||
let typed_account_id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?;
|
||||
let (secret, access_token_expires_at) = self
|
||||
.resolve_import_tokens(
|
||||
&typed_account_id,
|
||||
access_token.clone(),
|
||||
refresh_token.clone(),
|
||||
)
|
||||
.await?;
|
||||
let mut verified_account = complete_oauth_account_profile(
|
||||
self.account_source.as_ref(),
|
||||
&secret,
|
||||
access_token_expires_at,
|
||||
)
|
||||
.await
|
||||
.map_err(map_account_profile_error)?;
|
||||
apply_import_metadata(&mut verified_account, &candidate);
|
||||
if !upstream_identities.insert((
|
||||
verified_account.chatgpt_user_id.clone(),
|
||||
verified_account.chatgpt_account_id.clone(),
|
||||
@@ -765,16 +835,17 @@ impl CodexCredentialAdminService {
|
||||
};
|
||||
let next_refresh_at = refresh_time(
|
||||
refresh_policy,
|
||||
&ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?,
|
||||
&typed_account_id,
|
||||
verified_account.access_token_expires_at,
|
||||
secret.refresh_token.is_some(),
|
||||
)?;
|
||||
CodexCredentialAdmin.prepare_import(ImportCodexOAuthCredential {
|
||||
account_id,
|
||||
name: verified_account
|
||||
.email
|
||||
name: candidate
|
||||
.name
|
||||
.clone()
|
||||
.or_else(|| candidate.email.clone())
|
||||
.or_else(|| verified_account.email.clone())
|
||||
.filter(|name| !name.trim().is_empty())
|
||||
.unwrap_or_else(|| "Codex OAuth".to_owned()),
|
||||
secret,
|
||||
@@ -815,11 +886,12 @@ impl CodexCredentialAdminService {
|
||||
Ok(PreparedCodexAccountImport { accounts })
|
||||
}
|
||||
|
||||
async fn verify_import_tokens(
|
||||
async fn resolve_import_tokens(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
) -> Result<(CodexOAuthSecret, CodexAccountProfile), CodexCredentialAdminError> {
|
||||
) -> Result<(CodexOAuthSecret, Option<DateTime<Utc>>), CodexCredentialAdminError> {
|
||||
let access_token = access_token
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
@@ -831,19 +903,15 @@ impl CodexCredentialAdminService {
|
||||
if let Some(access_token) = access_token {
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(access_token),
|
||||
refresh_token: refresh_token.clone().map(SecretString::from),
|
||||
refresh_token: refresh_token.map(SecretString::from),
|
||||
id_token: None,
|
||||
};
|
||||
match self.verifier.verify_import(&secret).await {
|
||||
Ok(profile) => return Ok((secret, profile)),
|
||||
Err(CodexIdentityVerificationError::Unavailable) => {
|
||||
return Err(CodexCredentialAdminError::IdentityUnavailable);
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Rejected) if refresh_token.is_none() => {
|
||||
return Err(CodexCredentialAdminError::IdentityRejected);
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Rejected) => {}
|
||||
}
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ImportDirect,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
return Ok((secret, None));
|
||||
}
|
||||
let refresh_token = refresh_token.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
let tokens = self
|
||||
@@ -851,9 +919,6 @@ impl CodexCredentialAdminService {
|
||||
.refresh(&refresh_token)
|
||||
.await
|
||||
.map_err(map_refresh_failure)?;
|
||||
if tokens.access_token.trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
// RT 未轮换时仍保留导入提供的 RT,保证本次补全不会丢失后续刷新能力。
|
||||
@@ -863,12 +928,35 @@ impl CodexCredentialAdminService {
|
||||
.or_else(|| Some(SecretString::from(refresh_token))),
|
||||
id_token: None,
|
||||
};
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_import(&secret)
|
||||
.await
|
||||
.map_err(map_identity_error)?;
|
||||
Ok((secret, profile))
|
||||
Self::record_recovery_log(
|
||||
CodexOAuthRecoveryOperation::ImportRefreshToken,
|
||||
Some(account_id.as_str()),
|
||||
&secret,
|
||||
);
|
||||
if secret.access_token.expose_secret().trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
let access_token_expires_at = SystemTime::now()
|
||||
.checked_add(tokens.expires_in)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
Ok((secret, Some(access_token_expires_at)))
|
||||
}
|
||||
|
||||
fn record_recovery_log(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
account_id: Option<&str>,
|
||||
secret: &CodexOAuthSecret,
|
||||
) {
|
||||
record_oauth_recovery(
|
||||
operation,
|
||||
account_id,
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
}
|
||||
|
||||
fn prepare_agent_identity_import(
|
||||
@@ -939,27 +1027,6 @@ impl CodexCredentialAdminService {
|
||||
}
|
||||
}
|
||||
|
||||
fn current_identity_expectation(
|
||||
account: &ProviderAccount,
|
||||
credential: &super::security::CodexRuntimeCredential,
|
||||
) -> Result<CodexIdentityExpectation, CodexCredentialAdminError> {
|
||||
let account_id = account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
let principal = credential
|
||||
.principal
|
||||
.as_ref()
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
CodexIdentityExpectation::current(
|
||||
principal.oauth_subject.clone(),
|
||||
principal.poid.clone(),
|
||||
account_id.to_owned(),
|
||||
account.upstream_user_id().to_owned(),
|
||||
credential.installation_id.clone(),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)
|
||||
}
|
||||
|
||||
fn required_time(
|
||||
value: Option<chrono::DateTime<chrono::Utc>>,
|
||||
) -> Result<SystemTime, CodexCredentialAdminError> {
|
||||
@@ -981,9 +1048,12 @@ pub(crate) fn refresh_time(
|
||||
if !has_refresh_token {
|
||||
return Ok(None);
|
||||
}
|
||||
let expires_at = required_time(access_token_expires_at)?;
|
||||
let observed_at = SystemTime::now();
|
||||
let expires_at = access_token_expires_at
|
||||
.map(SystemTime::from)
|
||||
.unwrap_or(observed_at);
|
||||
policy
|
||||
.next_attempt_at(account_id, expires_at, SystemTime::now())
|
||||
.next_attempt_at(account_id, expires_at, observed_at)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.map(Some)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidCredential)
|
||||
@@ -1016,15 +1086,6 @@ const fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError
|
||||
}
|
||||
}
|
||||
|
||||
const fn map_identity_error(error: CodexIdentityVerificationError) -> CodexCredentialAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => CodexCredentialAdminError::IdentityRejected,
|
||||
CodexIdentityVerificationError::Unavailable => {
|
||||
CodexCredentialAdminError::IdentityUnavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_import_document(
|
||||
payload: &Value,
|
||||
) -> Result<Vec<ParsedCodexImportAccount>, CodexCredentialAdminError> {
|
||||
@@ -1037,20 +1098,28 @@ fn parse_import_document(
|
||||
if !is_openai_oauth_candidate(value) {
|
||||
continue;
|
||||
}
|
||||
accounts.push(ParsedCodexImportAccount {
|
||||
id: None,
|
||||
name: None,
|
||||
email: None,
|
||||
plan_type: None,
|
||||
chatgpt_account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
authentication: parse_oauth_import_tokens(value)?,
|
||||
status: None,
|
||||
});
|
||||
accounts.push(parse_oauth_import_account(value)?);
|
||||
}
|
||||
Ok(accounts)
|
||||
}
|
||||
|
||||
fn parse_oauth_import_account(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
let credentials = value.get("credentials").unwrap_or(value);
|
||||
Ok(ParsedCodexImportAccount {
|
||||
id: first_string(value, &["id"]),
|
||||
name: first_string(value, &["name", "label"]),
|
||||
email: first_string(value, &["email"]).or_else(|| first_string(credentials, &["email"])),
|
||||
plan_type: first_string(value, &["plan_type", "planType"])
|
||||
.or_else(|| first_string(credentials, &["plan_type", "planType"])),
|
||||
chatgpt_account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
authentication: parse_oauth_import_tokens(value)?,
|
||||
status: first_string(value, &["status"]),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_agent_identity_account(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
@@ -1212,6 +1281,59 @@ fn normalize_bearer(value: &str) -> String {
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_oauth_account_profile(
|
||||
account_source: &dyn CodexAuthenticatedAccountSource,
|
||||
secret: &CodexOAuthSecret,
|
||||
access_token_expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<CodexAccountProfile, CodexIdentityVerificationError> {
|
||||
let account = account_source
|
||||
.fetch(secret, &CodexIdentityExpectation::default())
|
||||
.await?;
|
||||
Ok(CodexAccountProfile {
|
||||
email: account.email,
|
||||
oauth_subject: account.chatgpt_user_id.clone(),
|
||||
poid: None,
|
||||
chatgpt_account_id: account.chatgpt_account_id,
|
||||
chatgpt_user_id: account.chatgpt_user_id,
|
||||
plan_type: account.plan_type,
|
||||
access_token_expires_at,
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_import_metadata(profile: &mut CodexAccountProfile, candidate: &ParsedCodexImportAccount) {
|
||||
if profile.email.is_none() {
|
||||
profile.email = candidate
|
||||
.email
|
||||
.as_deref()
|
||||
.and_then(|value| normalize_import_text(value, MAX_IMPORT_EMAIL_BYTES));
|
||||
}
|
||||
if profile.plan_type.is_none() {
|
||||
profile.plan_type = candidate
|
||||
.plan_type
|
||||
.as_deref()
|
||||
.and_then(|value| normalize_import_text(value, MAX_IMPORT_PLAN_BYTES));
|
||||
}
|
||||
}
|
||||
|
||||
const fn map_account_profile_error(
|
||||
error: CodexIdentityVerificationError,
|
||||
) -> CodexCredentialAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => {
|
||||
CodexCredentialAdminError::AccountProfileRejected
|
||||
}
|
||||
CodexIdentityVerificationError::Unavailable => {
|
||||
CodexCredentialAdminError::AccountProfileUnavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_import_text(value: &str, max_bytes: usize) -> Option<String> {
|
||||
let value = value.trim();
|
||||
(!value.is_empty() && value.len() <= max_bytes && !value.chars().any(char::is_control))
|
||||
.then(|| value.to_owned())
|
||||
}
|
||||
|
||||
fn import_runtime_state(
|
||||
status: Option<&str>,
|
||||
) -> Result<(bool, AccountAvailability), CodexCredentialAdminError> {
|
||||
|
||||
@@ -10,19 +10,33 @@ use crate::transport::request::derive_conversation_anchor;
|
||||
pub(crate) fn derive_codex_session_affinity_key(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<ProviderSessionAffinityKey> {
|
||||
if let Some(conversation_id) = request
|
||||
let session_key = if let Some(conversation_id) = request
|
||||
.local_conversation_id
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
if let Ok(key) = ProviderSessionAffinityKey::try_new(conversation_id.to_owned()) {
|
||||
return Some(key);
|
||||
Some(key)
|
||||
} else {
|
||||
opaque_affinity_key("local-conversation", conversation_id)
|
||||
}
|
||||
return opaque_affinity_key("local-conversation", conversation_id);
|
||||
}
|
||||
let (domain, value) = derive_conversation_anchor(request)?;
|
||||
opaque_affinity_key(domain, &value)
|
||||
} else {
|
||||
let (domain, value) = derive_conversation_anchor(request)?;
|
||||
opaque_affinity_key(domain, &value)
|
||||
};
|
||||
|
||||
let session_discriminator = request.subagent_kind();
|
||||
session_key.and_then(|session_key| match session_discriminator.as_deref() {
|
||||
None => Some(session_key),
|
||||
Some(discriminator) => {
|
||||
// 仅扩展原会话锚点的键空间;后续查找、绑定和调度仍使用同一套亲和逻辑。
|
||||
opaque_affinity_key(
|
||||
"subagent-session",
|
||||
&format!("{discriminator}\0{}", session_key.expose_to_store()),
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn opaque_affinity_key(domain: &str, value: &str) -> Option<ProviderSessionAffinityKey> {
|
||||
|
||||
@@ -8,6 +8,7 @@ mod cookie;
|
||||
mod identity;
|
||||
mod oauth;
|
||||
mod quota;
|
||||
mod recovery_log;
|
||||
mod refresh;
|
||||
mod repository;
|
||||
mod security;
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
//! Codex Authorization Code + PKCE/OIDC 管理流。
|
||||
|
||||
use std::{fmt, sync::Arc, time::Duration};
|
||||
use std::{
|
||||
fmt,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
@@ -13,7 +17,7 @@ use gateway_admin::model::{
|
||||
},
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
LoadedCredential, NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderRuntimePolicyPort;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
@@ -24,11 +28,11 @@ use uuid::Uuid;
|
||||
|
||||
use super::admin::{
|
||||
CodexCredentialAdmin, CodexCredentialAdminError, ImportCodexOAuthCredential,
|
||||
PreparedCodexCredentialRotation, RotateManagedCodexCredential, refresh_time,
|
||||
};
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
PreparedCodexCredentialRotation, complete_oauth_account_profile, refresh_time,
|
||||
};
|
||||
use super::identity::{CodexAuthenticatedAccountSource, CodexIdentityVerificationError};
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{
|
||||
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
|
||||
OFFICIAL_CODEX_OAUTH_CLIENT_ID, OFFICIAL_CODEX_REDIRECT_URI,
|
||||
@@ -344,7 +348,7 @@ pub trait CodexOAuthAdmin: Send + Sync {
|
||||
pub struct CodexOAuthAdminService {
|
||||
pending: Arc<dyn CodexOAuthPendingStore>,
|
||||
exchanger: Arc<dyn AuthorizationCodeExchanger>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
store: Arc<dyn ProviderAccountStore>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
credentials: CodexCredentialAdmin,
|
||||
@@ -396,7 +400,7 @@ impl CodexOAuthAdminService {
|
||||
pub fn new(
|
||||
pending: Arc<dyn CodexOAuthPendingStore>,
|
||||
exchanger: Arc<dyn AuthorizationCodeExchanger>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
store: Arc<dyn ProviderAccountStore>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
credentials: CodexCredentialAdmin,
|
||||
@@ -404,7 +408,7 @@ impl CodexOAuthAdminService {
|
||||
Self {
|
||||
pending,
|
||||
exchanger,
|
||||
verifier,
|
||||
account_source,
|
||||
store,
|
||||
runtime_policy,
|
||||
credentials,
|
||||
@@ -487,8 +491,6 @@ impl CodexOAuthAdminService {
|
||||
callback_url: &str,
|
||||
) -> Result<(PendingAuthorizationMutation, CompletedCodexOAuthCredential), CodexOAuthAdminError>
|
||||
{
|
||||
let (mut secret, id_token) = self.exchange_pending(&pending, callback_url).await?;
|
||||
let mutation = pending.mutation.clone();
|
||||
let current = if let Some(target) = pending.reauthorization() {
|
||||
let current = self
|
||||
.store
|
||||
@@ -499,13 +501,19 @@ impl CodexOAuthAdminService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let expectation = CodexIdentityExpectation::default();
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_authorization(&secret, &id_token, &pending.nonce, &expectation)
|
||||
.await
|
||||
.and_then(super::identity::CodexIdentityVerification::into_complete)
|
||||
.map_err(map_identity_error)?;
|
||||
let fallback_refresh_token = current
|
||||
.as_ref()
|
||||
.map(current_oauth_refresh_token)
|
||||
.transpose()?
|
||||
.flatten();
|
||||
let (mut secret, id_token, expires_in) = self
|
||||
.exchange_pending(&pending, callback_url, fallback_refresh_token)
|
||||
.await?;
|
||||
let mutation = pending.mutation.clone();
|
||||
let access_token_expires_at = SystemTime::now()
|
||||
.checked_add(expires_in)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.ok_or(CodexOAuthAdminError::Credential)?;
|
||||
secret.id_token = Some(id_token);
|
||||
let policy = self
|
||||
.runtime_policy
|
||||
@@ -516,28 +524,35 @@ impl CodexOAuthAdminService {
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
current.account.id(),
|
||||
profile.access_token_expires_at,
|
||||
Some(access_token_expires_at),
|
||||
secret.refresh_token.is_some(),
|
||||
)
|
||||
.map_err(map_admin_error)?;
|
||||
CompletedCodexOAuthCredential::Reauthorize(
|
||||
self.credentials
|
||||
.prepare_reauthorization(RotateManagedCodexCredential {
|
||||
.prepare_refreshed_oauth_rotation(
|
||||
current,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
access_token_expires_at,
|
||||
next_refresh_at,
|
||||
})
|
||||
)
|
||||
.map_err(map_admin_error)?,
|
||||
)
|
||||
} else {
|
||||
let account_id = format!("acct_{}", Uuid::now_v7().simple());
|
||||
let typed_account_id = ProviderAccountId::new(account_id.clone())
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
let profile = complete_oauth_account_profile(
|
||||
self.account_source.as_ref(),
|
||||
&secret,
|
||||
Some(access_token_expires_at),
|
||||
)
|
||||
.await
|
||||
.map_err(map_account_profile_error)?;
|
||||
let next_refresh_at = refresh_time(
|
||||
policy,
|
||||
&typed_account_id,
|
||||
profile.access_token_expires_at,
|
||||
Some(access_token_expires_at),
|
||||
secret.refresh_token.is_some(),
|
||||
)
|
||||
.map_err(map_admin_error)?;
|
||||
@@ -654,7 +669,9 @@ impl CodexOAuthAdminService {
|
||||
&self,
|
||||
pending: &CodexPendingAuthorization,
|
||||
callback_url: &str,
|
||||
) -> Result<(super::types::CodexOAuthSecret, SecretString), CodexOAuthAdminError> {
|
||||
fallback_refresh_token: Option<SecretString>,
|
||||
) -> Result<(super::types::CodexOAuthSecret, SecretString, Duration), CodexOAuthAdminError>
|
||||
{
|
||||
let (code, callback_state) = callback_parts(callback_url)?;
|
||||
if !constant_time_equal(
|
||||
pending.state.expose_secret().as_bytes(),
|
||||
@@ -670,10 +687,36 @@ impl CodexOAuthAdminService {
|
||||
})
|
||||
.await
|
||||
.map_err(map_exchange_error)?;
|
||||
Ok((tokens.secret, tokens.id_token))
|
||||
let mut secret = tokens.secret;
|
||||
if secret.refresh_token.is_none() {
|
||||
secret.refresh_token = fallback_refresh_token;
|
||||
}
|
||||
record_oauth_recovery(
|
||||
CodexOAuthRecoveryOperation::AuthorizationCode,
|
||||
pending
|
||||
.reauthorization()
|
||||
.map(|account_id| account_id.as_str()),
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
Ok((secret, tokens.id_token, tokens.expires_in))
|
||||
}
|
||||
}
|
||||
|
||||
fn current_oauth_refresh_token(
|
||||
current: &LoadedCredential,
|
||||
) -> Result<Option<SecretString>, CodexOAuthAdminError> {
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
Ok(runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.and_then(|oauth| oauth.refresh_token.clone()))
|
||||
}
|
||||
|
||||
fn authorization_url(
|
||||
pending: &CodexPendingAuthorization,
|
||||
oauth_client_id: &str,
|
||||
@@ -699,27 +742,19 @@ fn authorization_url(
|
||||
|
||||
fn callback_parts(value: &str) -> Result<(SecretString, SecretString), CodexOAuthAdminError> {
|
||||
let url = Url::parse(value).map_err(|_| CodexOAuthAdminError::CallbackRejected)?;
|
||||
if url.scheme() != "http"
|
||||
|| url.host_str() != Some("localhost")
|
||||
|| url.port() != Some(1455)
|
||||
|| url.path() != "/auth/callback"
|
||||
|| url.fragment().is_some()
|
||||
{
|
||||
return Err(CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
let mut code = None;
|
||||
let mut state = None;
|
||||
for (key, value) in url.query_pairs() {
|
||||
match key.as_ref() {
|
||||
"code" => set_unique_callback_parameter(&mut code, value.into_owned())?,
|
||||
"state" => set_unique_callback_parameter(&mut state, value.into_owned())?,
|
||||
// OAuth 服务可能附加 `scope`、`iss` 等标准参数。
|
||||
// 固定回调地址与唯一的 `code`/`state` 才是安全边界,无关参数不参与校验。
|
||||
// 回调地址只承载 query 参数;安全绑定由唯一的 code/state、
|
||||
// 服务端保存的 state 和 PKCE token exchange 共同完成。
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let code = code.filter(|value| valid_secret(value));
|
||||
let state = state.filter(|value| valid_secret(value));
|
||||
let code = code.filter(|value| !value.is_empty());
|
||||
let state = state.filter(|value| !value.is_empty());
|
||||
match (code, state) {
|
||||
(Some(code), Some(state)) => Ok((SecretString::from(code), SecretString::from(state))),
|
||||
_ => Err(CodexOAuthAdminError::CallbackRejected),
|
||||
@@ -813,9 +848,9 @@ fn map_exchange_error(error: AuthorizationCodeExchangeError) -> CodexOAuthAdminE
|
||||
}
|
||||
}
|
||||
|
||||
fn map_identity_error(error: CodexIdentityVerificationError) -> CodexOAuthAdminError {
|
||||
const fn map_account_profile_error(error: CodexIdentityVerificationError) -> CodexOAuthAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => CodexOAuthAdminError::IdentityRejected,
|
||||
CodexIdentityVerificationError::Rejected => CodexOAuthAdminError::TokenRejected,
|
||||
CodexIdentityVerificationError::Unavailable => CodexOAuthAdminError::UpstreamUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,12 +20,13 @@ use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountQuotaSignals, CredentialRevision, OpaqueProviderData,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountStore, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::provider_ports::{ProviderCooldown, ProviderCooldownPort};
|
||||
use gateway_protocol::openai::events::{
|
||||
ParsedRateLimits, RateLimitDetails, RateLimitWindow, parse_rate_limit_headers,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::{Map, Number, Value};
|
||||
use serde_json::{Map, Value};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
@@ -35,7 +36,7 @@ use crate::transport::{CodexBackendClient, CodexClientError, CodexRequestContext
|
||||
|
||||
use super::agent_identity::{CodexAgentIdentityTaskService, PreparedCodexRuntimeCredential};
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use recovery::{QuotaRecoveryEvidence, quota_state_transition, quota_success_state};
|
||||
use recovery::{quota_state_reason, quota_state_transition, quota_success_state};
|
||||
use snapshot::{
|
||||
parse_account_quota_snapshot, quota_projection_ttl, quota_snapshot_from_observation,
|
||||
scheduling_signals_from_snapshot,
|
||||
@@ -120,6 +121,7 @@ pub struct CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
scheduling: CodexQuotaSchedulingProjection,
|
||||
}
|
||||
|
||||
@@ -379,6 +381,7 @@ impl CodexCredentialQuotaService {
|
||||
http: Client,
|
||||
base_url: String,
|
||||
agent_identity: Arc<CodexAgentIdentityTaskService>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store: Arc::clone(repository.store()),
|
||||
@@ -387,11 +390,12 @@ impl CodexCredentialQuotaService {
|
||||
http,
|
||||
base_url,
|
||||
agent_identity,
|
||||
cooldowns,
|
||||
scheduling: CodexQuotaSchedulingProjection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 写入新快照前读取旧确认 reset(402 恢复用「旧 reset 已到期」判定)。
|
||||
/// 写入新快照前读取已确认耗尽窗口的 reset,用于判断窗口是否真的推进。
|
||||
async fn previous_confirmed_reset(&self, account: &ProviderAccount) -> Option<SystemTime> {
|
||||
let observations = self
|
||||
.store
|
||||
@@ -406,6 +410,55 @@ impl CodexCredentialQuotaService {
|
||||
snapshot.fact().resets_at().map(SystemTime::from)
|
||||
}
|
||||
|
||||
/// 将已确认的额度拒绝投影为 100%,使展示、调度和恢复共用同一窗口事实。
|
||||
///
|
||||
/// `reset_at` 来自官方 `usage_limit_reached` 错误体;缺失时保留当前快照的 reset。
|
||||
pub(crate) async fn record_confirmed_exhaustion(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
reset_at: Option<i64>,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
let quota = confirmed_exhaustion_projection(
|
||||
normalize_quota_window_placeholders(existing),
|
||||
reset_at,
|
||||
);
|
||||
let observed_at = SystemTime::now();
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(quota.clone())),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
return Err(CodexCredentialQuotaError::RevisionConflict);
|
||||
}
|
||||
if let Ok(snapshot) = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(quota),
|
||||
) {
|
||||
self.scheduling.observe(&snapshot);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 批量预热请求级额度投影;持久层或 Provider JSON 异常只退化为未知额度。
|
||||
pub async fn prepare_scheduling(&self, accounts: &[ProviderAccount]) {
|
||||
if self.scheduling.hydration_targets(accounts).is_empty() {
|
||||
@@ -495,21 +548,23 @@ impl CodexCredentialQuotaService {
|
||||
// 402:真额度耗尽,写入 QuotaExhausted(worker 按 reset_at 自动恢复)。
|
||||
Some(AccountAvailability::QuotaExhausted) => {
|
||||
summary.exhausted += 1;
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(
|
||||
&account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
self.persist_upstream_state(
|
||||
&account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
&error,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Some(AccountAvailability::Banned) => {
|
||||
summary.banned += 1;
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(&account, AccountAvailability::Banned, observed_at)
|
||||
.await;
|
||||
self.persist_upstream_state(
|
||||
&account,
|
||||
AccountAvailability::Banned,
|
||||
&error,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
// 429/503 等临时拒绝:不写 availability(限流不改变账号可用性),仅记失败待重试。
|
||||
None | Some(_) => {
|
||||
@@ -535,18 +590,44 @@ impl CodexCredentialQuotaService {
|
||||
observed_at: SystemTime,
|
||||
summary: &mut CodexQuotaSyncSummary,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
value,
|
||||
)?;
|
||||
let fact = snapshot.fact();
|
||||
let refreshed_fact = refreshed_snapshot.fact();
|
||||
let previous_reset_at = self.previous_confirmed_reset(account).await;
|
||||
let object = value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let now = SystemTime::now();
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
refreshed_fact,
|
||||
previous_reset_at,
|
||||
now,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(object.clone()),
|
||||
)?;
|
||||
let fact = snapshot.fact();
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
@@ -577,10 +658,9 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
if let Some(availability) = quota_success_state(
|
||||
current.availability(),
|
||||
fact,
|
||||
refreshed_fact,
|
||||
previous_reset_at,
|
||||
SystemTime::now(),
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot,
|
||||
now,
|
||||
) {
|
||||
let _ = self
|
||||
.repository
|
||||
@@ -599,6 +679,15 @@ impl CodexCredentialQuotaService {
|
||||
let Some(rate_limits) = parse_rate_limit_headers(headers) else {
|
||||
return Ok(false);
|
||||
};
|
||||
let has_quota_facts = rate_limits
|
||||
.limits
|
||||
.values()
|
||||
.any(|details| passive_rate_limit_snapshot(details).is_some());
|
||||
let previous_reset_at = if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
self.previous_confirmed_reset(account).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let existing = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
@@ -607,12 +696,48 @@ impl CodexCredentialQuotaService {
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
});
|
||||
let existing_observed_at = existing
|
||||
.as_ref()
|
||||
.and_then(|observation| observation.observed_at);
|
||||
let existing = existing
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
|
||||
// 也不能借旧快照重新推导 quota state。
|
||||
if !has_quota_facts {
|
||||
let Some(observed_at) = existing_observed_at else {
|
||||
return Ok(false);
|
||||
};
|
||||
let outcome = self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(merge_passive_quota(
|
||||
existing,
|
||||
&rate_limits,
|
||||
))),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await?;
|
||||
return Ok(outcome != QuotaWriteOutcome::Conflict);
|
||||
}
|
||||
let quota = merge_passive_quota(existing, &rate_limits);
|
||||
let observed_at = SystemTime::now();
|
||||
let refreshed_fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
let quota = if account.availability() == AccountAvailability::QuotaExhausted {
|
||||
// 被动成功响应不能解除确认额度耗尽;继续展示 100%,直到权威 usage 快照
|
||||
// 证明旧 reset 已跨过且新窗口已经推进。
|
||||
confirmed_exhaustion_projection(
|
||||
quota,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
)
|
||||
} else {
|
||||
quota
|
||||
};
|
||||
let fact = parse_codex_quota_usage(&Value::Object(quota.clone()))?;
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
@@ -627,123 +752,68 @@ impl CodexCredentialQuotaService {
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(quota)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(true),
|
||||
limit_reached: Some(fact.exhausted()),
|
||||
})
|
||||
.await?;
|
||||
if outcome == QuotaWriteOutcome::Conflict {
|
||||
return Ok(false);
|
||||
}
|
||||
// 可用性写回必须基于最新账号状态:revision fence 只覆盖凭据轮换,
|
||||
// 不覆盖其他路径的仅状态变化(Cooldown/QuotaExhausted 转换)。
|
||||
let Some(current) = self.store.get_account(account.id()).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if current.revision() != account.revision() {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Ok(snapshot) = snapshot {
|
||||
self.scheduling.observe(&snapshot);
|
||||
}
|
||||
if let Some(availability) = quota_success_state(
|
||||
current.availability(),
|
||||
fact,
|
||||
None,
|
||||
observed_at,
|
||||
QuotaRecoveryEvidence::SuccessfulResponse,
|
||||
) {
|
||||
self.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
.await?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// 真实 429 的单一事实入口:合成/更新 quota 主窗口(`limit_reached=true`、
|
||||
/// `used_percent=100`、`reset_at=retry-after`)。availability 不改变,
|
||||
/// 窗口 reset 到期后由投影自动滚动恢复(429 只记录限流窗口)。
|
||||
/// 已有更晚的 `reset_at` 不会被缩短(never-shrink)。
|
||||
/// 真实 429 的单一事实入口:写入 Redis 临时限流冷却(`until = now + retry_after`)。
|
||||
/// availability 与 quota 主窗口(额度重置时间)都不改变——临时限流是独立维度,
|
||||
/// 到期由 Redis key 过期自动解除,不污染配额耗尽状态。
|
||||
/// 已有更晚的冷却不会被缩短(put_if_later)。
|
||||
pub async fn apply_rate_limit_429(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CodexCredentialQuotaError> {
|
||||
let mut object = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await?
|
||||
.into_iter()
|
||||
.find(|observation| {
|
||||
observation.account_id == *account.id()
|
||||
&& observation.expected_revision == account.revision()
|
||||
})
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.unwrap_or_default();
|
||||
let new_reset_at = observed_at
|
||||
let until = observed_at
|
||||
.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN))
|
||||
.and_then(|reset| reset.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|duration| duration.as_secs() as i64);
|
||||
let existing_reset_at = object
|
||||
.get("rate_limit")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|rate_limit| rate_limit.get("primary_window"))
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|window| window.get("reset_at"))
|
||||
.and_then(Value::as_i64);
|
||||
let reset_at = match (existing_reset_at, new_reset_at) {
|
||||
(Some(existing), Some(next)) if existing > next => Some(existing),
|
||||
(_, next) => next,
|
||||
};
|
||||
let mut rate_limit = object
|
||||
.get("rate_limit")
|
||||
.cloned()
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
|
||||
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
|
||||
let mut primary = rate_limit
|
||||
.get("primary_window")
|
||||
.cloned()
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
let used_percent = primary
|
||||
.get("used_percent")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(0.0)
|
||||
.max(100.0);
|
||||
primary.insert(
|
||||
"used_percent".to_owned(),
|
||||
Number::from_f64(used_percent)
|
||||
.map(Value::Number)
|
||||
.unwrap_or_else(|| Value::Number(Number::from(100))),
|
||||
);
|
||||
if let Some(reset_at) = reset_at {
|
||||
primary.insert("reset_at".to_owned(), Value::from(reset_at));
|
||||
}
|
||||
rate_limit.insert("primary_window".to_owned(), Value::Object(primary));
|
||||
object.insert("rate_limit".to_owned(), Value::Object(rate_limit));
|
||||
if self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(object)),
|
||||
observed_at: Some(observed_at),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await?
|
||||
== QuotaWriteOutcome::Conflict
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
// 429 合成的限流窗口要立即反映到调度信号(reset 到期由投影滚动解除)。
|
||||
if let Ok(Some(snapshot)) = self.read_snapshot_for(account).await {
|
||||
self.scheduling.observe(&snapshot);
|
||||
}
|
||||
.unwrap_or(observed_at);
|
||||
self.cooldowns
|
||||
.put_if_later(ProviderCooldown::new(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
until,
|
||||
))
|
||||
.await
|
||||
.map_err(|error| CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 读取账号当前是否处于临时限流(429)冷却,及到期时间。
|
||||
pub async fn rate_limited_until(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
) -> Result<Option<SystemTime>, CodexCredentialQuotaError> {
|
||||
let Some(cooldown) = self.cooldowns.read(account_id).await.map_err(|error| {
|
||||
CodexCredentialQuotaError::Store {
|
||||
detail: error.to_string(),
|
||||
}
|
||||
})?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if cooldown.credential_revision() != credential_revision {
|
||||
return Ok(None);
|
||||
}
|
||||
let until = cooldown.until();
|
||||
if until <= SystemTime::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(until))
|
||||
}
|
||||
|
||||
/// 读取单账号最后一次落库的 Provider quota,并由 Codex 域解析展示窗口。
|
||||
pub async fn read_account(
|
||||
&self,
|
||||
@@ -845,9 +915,7 @@ impl CodexCredentialQuotaService {
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
// 402/deactivated → 写终态;429/503 等临时拒绝不写 availability(限流不改变账号可用性)。
|
||||
if let Some(availability) = quota_state_transition(&error) {
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(&account, availability, observed_at)
|
||||
self.persist_upstream_state(&account, availability, &error, observed_at)
|
||||
.await;
|
||||
}
|
||||
return Err(CodexCredentialQuotaError::Upstream {
|
||||
@@ -855,17 +923,43 @@ impl CodexCredentialQuotaService {
|
||||
});
|
||||
}
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
let refreshed_snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&value,
|
||||
)?;
|
||||
let refreshed_fact = refreshed_snapshot.fact();
|
||||
let previous_reset_at = self.previous_confirmed_reset(&account).await;
|
||||
let object = value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let now = SystemTime::now();
|
||||
let object = normalize_quota_window_placeholders(
|
||||
value
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?,
|
||||
);
|
||||
let object = if account.availability() == AccountAvailability::QuotaExhausted
|
||||
&& quota_success_state(
|
||||
account.availability(),
|
||||
refreshed_fact,
|
||||
previous_reset_at,
|
||||
now,
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
confirmed_exhaustion_projection(
|
||||
object,
|
||||
confirmed_exhaustion_reset_at(previous_reset_at, refreshed_fact),
|
||||
)
|
||||
} else {
|
||||
object
|
||||
};
|
||||
let snapshot = parse_account_quota_snapshot(
|
||||
account.id().clone(),
|
||||
account.revision(),
|
||||
observed_at,
|
||||
&Value::Object(object.clone()),
|
||||
)?;
|
||||
if self
|
||||
.store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
@@ -891,10 +985,9 @@ impl CodexCredentialQuotaService {
|
||||
self.scheduling.observe(&snapshot);
|
||||
if let Some(availability) = quota_success_state(
|
||||
current.availability(),
|
||||
snapshot.fact(),
|
||||
refreshed_fact,
|
||||
previous_reset_at,
|
||||
SystemTime::now(),
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot,
|
||||
now,
|
||||
) {
|
||||
self.repository
|
||||
.apply_state(¤t, availability, observed_at)
|
||||
@@ -903,6 +996,29 @@ impl CodexCredentialQuotaService {
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn persist_upstream_state(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
upstream_error: &CodexClientError,
|
||||
observed_at: SystemTime,
|
||||
) {
|
||||
let reason = quota_state_reason(upstream_error).map(str::to_owned);
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state_with_message(account, availability, observed_at, reason.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
?availability,
|
||||
reason = reason.as_deref().unwrap_or_default(),
|
||||
error = %error,
|
||||
"OpenAI quota upstream state write failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_usage_with_recovery(
|
||||
&self,
|
||||
client: &CodexBackendClient,
|
||||
@@ -1095,6 +1211,103 @@ fn merge_passive_quota(
|
||||
quota
|
||||
}
|
||||
|
||||
/// 丢弃 core 限额中上游给出的无事实 `secondary_window` 占位。
|
||||
///
|
||||
/// 生产响应头可能携带 `used_percent=0`、零时长和空 reset 的占位,`/usage` 也会
|
||||
/// 返回 `secondary_window: null`。正常被动同步保留该响应事实,Admin 展示会将其
|
||||
/// 隐藏;但主动 `/usage` 刷新或 402 确认投影会把一个存在但无事实的字段写成
|
||||
/// 100%,从而显示并不存在的“次级额度”。因此这些非被动写入口在落库前移除
|
||||
/// 无事实值。带 reset、时长、正用量、触顶、未知或非法字段的次级窗口均完全按
|
||||
/// 原有额度逻辑保留。
|
||||
fn normalize_quota_window_placeholders(mut quota: Map<String, Value>) -> Map<String, Value> {
|
||||
if let Some(rate_limit) = quota.get_mut("rate_limit").and_then(Value::as_object_mut) {
|
||||
drop_secondary_window_placeholder(rate_limit);
|
||||
}
|
||||
quota
|
||||
}
|
||||
|
||||
fn drop_secondary_window_placeholder(rate_limit: &mut Map<String, Value>) {
|
||||
let placeholder = rate_limit.get("secondary_window").is_some_and(|window| {
|
||||
window.is_null()
|
||||
|| window
|
||||
.as_object()
|
||||
.is_some_and(secondary_window_is_placeholder)
|
||||
});
|
||||
if placeholder {
|
||||
rate_limit.remove("secondary_window");
|
||||
}
|
||||
}
|
||||
|
||||
fn secondary_window_is_placeholder(window: &Map<String, Value>) -> bool {
|
||||
window.iter().all(|(field, value)| match field.as_str() {
|
||||
"used_percent" => value
|
||||
.as_f64()
|
||||
.is_some_and(|used_percent| used_percent.is_finite() && used_percent == 0.0),
|
||||
"limit_reached" => value.as_bool() == Some(false),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
/// 用已确认的上游额度拒绝覆盖 core 限额投影。
|
||||
///
|
||||
/// 上游 usage 快照可能在结算期间短暂滞后为 98/99% 或 `allowed=true`;保留其他
|
||||
/// 原始 metadata,但把 core 窗口固定为 100%,直到权威快照证明进入下一窗口。
|
||||
fn confirmed_exhaustion_projection(
|
||||
mut quota: Map<String, Value>,
|
||||
reset_at: Option<i64>,
|
||||
) -> Map<String, Value> {
|
||||
let mut rate_limit = quota
|
||||
.remove("rate_limit")
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
rate_limit.insert("allowed".to_owned(), Value::Bool(false));
|
||||
rate_limit.insert("limit_reached".to_owned(), Value::Bool(true));
|
||||
for field in ["primary_window", "secondary_window"] {
|
||||
let existed = rate_limit.contains_key(field);
|
||||
let mut window = rate_limit
|
||||
.remove(field)
|
||||
.and_then(|value| value.as_object().cloned())
|
||||
.unwrap_or_default();
|
||||
if field == "primary_window" || existed {
|
||||
window.insert("used_percent".to_owned(), Value::from(100_u64));
|
||||
if field == "primary_window"
|
||||
&& let Some(reset_at) = reset_at
|
||||
{
|
||||
window.insert("reset_at".to_owned(), Value::from(reset_at));
|
||||
}
|
||||
rate_limit.insert(field.to_owned(), Value::Object(window));
|
||||
}
|
||||
}
|
||||
quota.insert("rate_limit".to_owned(), Value::Object(rate_limit));
|
||||
quota
|
||||
}
|
||||
|
||||
/// 选择应继续锁定的 reset:新鲜快照仍确认耗尽时可把锁推进到更晚窗口;
|
||||
/// 否则保留已确认失败的旧窗口,避免 98/99% 快照反向解锁。
|
||||
fn confirmed_exhaustion_reset_at(
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
refreshed_fact: CodexQuotaFact,
|
||||
) -> Option<i64> {
|
||||
let refreshed_reset_at = refreshed_fact.resets_at().map(SystemTime::from);
|
||||
let reset_at = match (previous_reset_at, refreshed_reset_at) {
|
||||
(Some(previous), Some(refreshed)) if refreshed_fact.exhausted() && refreshed > previous => {
|
||||
refreshed
|
||||
}
|
||||
(Some(previous), _) => previous,
|
||||
(None, refreshed) => refreshed?,
|
||||
};
|
||||
system_time_to_unix_seconds(reset_at)
|
||||
}
|
||||
|
||||
fn system_time_to_unix_seconds(value: SystemTime) -> Option<i64> {
|
||||
value
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs()
|
||||
.try_into()
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn passive_rate_limit_snapshot(details: &RateLimitDetails) -> Option<Map<String, Value>> {
|
||||
// 响应头的窗口属于同一次上游观测;跨响应合并会制造不存在的额度窗口。
|
||||
let mut snapshot = Map::new();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 402 恢复证据与 availability 转换。
|
||||
//! 确认额度耗尽后的恢复证据与 availability 转换。
|
||||
//!
|
||||
//! `QuotaExhausted -> Ready` 只接受同一 credential revision 的真实成功响应、
|
||||
//! 新鲜快照显式 `allowed=true` 且无 exhaustion、或已到期旧 reset 配合未耗尽新快照。
|
||||
//! `QuotaExhausted -> Ready` 只能由同一 credential revision 的权威 usage 快照证明:
|
||||
//! 已确认的旧窗口 reset 已到期,且新快照的 reset 已推进到下一窗口。
|
||||
|
||||
use std::time::SystemTime;
|
||||
|
||||
@@ -12,47 +12,24 @@ use serde_json::Value;
|
||||
use super::snapshot::CodexQuotaFact;
|
||||
use crate::transport::CodexClientError;
|
||||
|
||||
/// 402 恢复可接受的证据强度。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum QuotaRecoveryEvidence {
|
||||
/// 一次真实成功推理响应(最强证据)。
|
||||
SuccessfulResponse,
|
||||
/// 周期 worker 刷新出的新快照(需旧 reset 已到期佐证)。
|
||||
RefreshedSnapshot,
|
||||
}
|
||||
|
||||
/// 只处理 `QuotaExhausted` 的恢复——429/触顶不再进入该状态,由 quota 数据
|
||||
/// 驱动调度排除(限流不改变账号可用性)。`Invalid/Expired/Banned` 等终态由 selector 的
|
||||
/// 成功响应恢复路径处理。
|
||||
/// 只处理 `QuotaExhausted` 的恢复。`allowed=true`、成功响应和百分比回落都不能单独
|
||||
/// 覆盖一次已确认的额度拒绝;它们在 reset 未推进时可能只是上游滞后的观测。
|
||||
///
|
||||
/// `previous_reset_at` 是写入新快照前读取的旧确认 reset;`RefreshedSnapshot`
|
||||
/// 恢复要求它已到期(窗口真实滚动后新快照的 reset 指向下一个未来周期,
|
||||
/// 不能用它判定旧耗尽窗口是否已结束)。
|
||||
/// `previous_reset_at` 是写入新快照前读取的已确认耗尽窗口。恢复必须同时满足:
|
||||
/// 旧窗口已经到期,且权威快照的 reset 指向比旧窗口更晚的新周期。
|
||||
pub(crate) fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
fact: CodexQuotaFact,
|
||||
previous_reset_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
recovery_evidence: QuotaRecoveryEvidence,
|
||||
) -> Option<AccountAvailability> {
|
||||
if current != AccountAvailability::QuotaExhausted || fact.exhausted() {
|
||||
return None;
|
||||
}
|
||||
// 新鲜快照显式 `allowed=true` 且无 exhaustion,直接恢复。
|
||||
if fact.explicit_allowed() {
|
||||
return Some(AccountAvailability::Ready);
|
||||
}
|
||||
// 一次真实成功响应是最强证据;周期刷新需要旧确认 reset 已到期
|
||||
// (used_percent 是可滞后、会取整的观测值,其回落不能覆盖仍未到期的旧 reset)。
|
||||
match recovery_evidence {
|
||||
QuotaRecoveryEvidence::SuccessfulResponse => Some(AccountAvailability::Ready),
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot
|
||||
if previous_reset_at.is_some_and(|reset_at| reset_at <= now) =>
|
||||
{
|
||||
Some(AccountAvailability::Ready)
|
||||
}
|
||||
QuotaRecoveryEvidence::RefreshedSnapshot => None,
|
||||
}
|
||||
let previous_reset_at = previous_reset_at?;
|
||||
let refreshed_reset_at = fact.resets_at().map(SystemTime::from)?;
|
||||
(previous_reset_at <= now && refreshed_reset_at > previous_reset_at)
|
||||
.then_some(AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<AccountAvailability> {
|
||||
@@ -73,6 +50,18 @@ pub(crate) fn quota_state_transition(error: &CodexClientError) -> Option<Account
|
||||
}
|
||||
}
|
||||
|
||||
/// 配额接口将账号变为非 ready 状态时可安全持久化的上游原因码。
|
||||
pub(crate) fn quota_state_reason(error: &CodexClientError) -> Option<&'static str> {
|
||||
match quota_state_transition(error)? {
|
||||
AccountAvailability::Banned => Some("deactivated_workspace"),
|
||||
AccountAvailability::QuotaExhausted => Some("payment_required"),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_deactivated_workspace(body: &str) -> bool {
|
||||
serde_json::from_str::<Value>(body)
|
||||
.ok()
|
||||
|
||||
@@ -18,8 +18,6 @@ pub struct CodexQuotaFact {
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
exhausted: bool,
|
||||
/// 上游显式 `allowed=true`(恢复证据:新鲜快照明确放行)。
|
||||
explicit_allowed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -41,7 +39,7 @@ pub enum CodexQuotaWindowRole {
|
||||
pub struct CodexQuotaWindow {
|
||||
key: String,
|
||||
source: String,
|
||||
/// 上游提供的限流名称(`metered_feature`/`limit_name`/`limitId`);显示优先用它。
|
||||
/// 上游提供的限流桶名称(`metered_feature`/`limit_name`/`limitId`);用于非核心桶的展示前缀。
|
||||
limit_name: Option<String>,
|
||||
kind: CodexQuotaWindowKind,
|
||||
role: CodexQuotaWindowRole,
|
||||
@@ -189,12 +187,6 @@ impl CodexQuotaFact {
|
||||
pub const fn exhausted(&self) -> bool {
|
||||
self.exhausted
|
||||
}
|
||||
|
||||
/// 上游是否显式 `allowed=true`(新鲜快照的恢复证据)。
|
||||
#[must_use]
|
||||
pub const fn explicit_allowed(&self) -> bool {
|
||||
self.explicit_allowed
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCredentialQuotaError> {
|
||||
@@ -220,7 +212,6 @@ pub fn parse_codex_quota_usage(usage: &Value) -> Result<CodexQuotaFact, CodexCre
|
||||
remaining_percent: aggregate.remaining_percent,
|
||||
resets_at: aggregate.resets_at,
|
||||
exhausted: aggregate.exhausted,
|
||||
explicit_allowed: aggregate.explicit_allowed,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -319,6 +310,7 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
let mut windows = Vec::new();
|
||||
for limit in canonical_rate_limits(object)? {
|
||||
parse_rate_limit_windows(
|
||||
&limit.key,
|
||||
&limit.source,
|
||||
limit.limit_name.as_deref(),
|
||||
limit.rate_limit,
|
||||
@@ -335,8 +327,9 @@ pub(crate) fn parse_account_quota_snapshot(
|
||||
})
|
||||
}
|
||||
|
||||
/// 一个规范化后的限流桶:source 是调度/展示分组键,limit_name 是上游显示名。
|
||||
/// 一个规范化后的限流桶:`key` 用于稳定区分窗口,`source` 用于展示分组。
|
||||
struct CanonicalRateLimit<'a> {
|
||||
key: String,
|
||||
source: String,
|
||||
limit_name: Option<String>,
|
||||
rate_limit: &'a Value,
|
||||
@@ -346,9 +339,15 @@ fn canonical_rate_limits(
|
||||
object: &Map<String, Value>,
|
||||
) -> Result<Vec<CanonicalRateLimit<'_>>, CodexCredentialQuotaError> {
|
||||
let mut limits = Vec::new();
|
||||
let has_primary_codex_rate_limit = object
|
||||
.get("rate_limit")
|
||||
.is_some_and(|value| !value.is_null());
|
||||
if let Some(rate_limit) = object.get("rate_limit") {
|
||||
// 官方 Codex 将顶层 `rate_limit` 固定识别为 `codex`,而不把
|
||||
// `active_limit` 投影成另一个展示桶。后者只是当前活动套餐的元数据。
|
||||
limits.push(CanonicalRateLimit {
|
||||
source: "core".to_owned(),
|
||||
key: "core".to_owned(),
|
||||
source: "codex".to_owned(),
|
||||
limit_name: None,
|
||||
rate_limit,
|
||||
});
|
||||
@@ -358,6 +357,7 @@ fn canonical_rate_limits(
|
||||
.filter(|value| !value.is_null())
|
||||
{
|
||||
limits.push(CanonicalRateLimit {
|
||||
key: "code-review".to_owned(),
|
||||
source: "code_review".to_owned(),
|
||||
limit_name: Some("code_review".to_owned()),
|
||||
rate_limit,
|
||||
@@ -379,23 +379,29 @@ fn canonical_rate_limits(
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialQuotaError::InvalidCredentialData)?;
|
||||
let (source, limit_name) = canonical_rate_limit_source(item, index);
|
||||
// 部分上游响应会在 `additional_rate_limits` 中重复默认的 `codex`
|
||||
// 桶。它没有独立的 limit id,不能在账号面板再次展示。
|
||||
if has_primary_codex_rate_limit && is_codex_limit_id(&source) {
|
||||
continue;
|
||||
}
|
||||
if let Some(rate_limit) = item.get("rate_limit") {
|
||||
// additional 里的 codex 桶覆盖顶层 rate_limit(同源替换)。
|
||||
if let Some(existing) = limits.iter_mut().find(|existing| existing.source == source) {
|
||||
existing.rate_limit = rate_limit;
|
||||
existing.limit_name = limit_name;
|
||||
} else {
|
||||
limits.push(CanonicalRateLimit {
|
||||
source,
|
||||
limit_name,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
// 非 Codex additional 桶保留为独立快照;额外用索引生成窗口键,
|
||||
// 避免同名 additional 产生重复前端 key。
|
||||
limits.push(CanonicalRateLimit {
|
||||
key: format!("additional-{index}-{source}"),
|
||||
source,
|
||||
limit_name,
|
||||
rate_limit,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(limits)
|
||||
}
|
||||
|
||||
fn is_codex_limit_id(value: &str) -> bool {
|
||||
value.trim().eq_ignore_ascii_case("codex")
|
||||
}
|
||||
|
||||
fn canonical_rate_limit_source(
|
||||
item: &Map<String, Value>,
|
||||
index: usize,
|
||||
@@ -417,14 +423,11 @@ fn canonical_rate_limit_source(
|
||||
!value.is_empty() && value.len() <= 128 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_owned);
|
||||
if source.trim().to_ascii_lowercase().replace(['-', ' '], "_") == "codex" {
|
||||
("core".to_owned(), limit_name)
|
||||
} else {
|
||||
(source, limit_name)
|
||||
}
|
||||
(source, limit_name)
|
||||
}
|
||||
|
||||
fn parse_rate_limit_windows(
|
||||
key_source: &str,
|
||||
source: &str,
|
||||
limit_name: Option<&str>,
|
||||
value: &Value,
|
||||
@@ -482,7 +485,7 @@ fn parse_rate_limit_windows(
|
||||
|| used_percent.is_some_and(|used| used >= 100.0);
|
||||
let kind = quota_window_kind(window_seconds);
|
||||
output.push(CodexQuotaWindow {
|
||||
key: format!("{}-{}", quota_key(source), quota_role_name(role, kind)),
|
||||
key: format!("{}-{}", quota_key(key_source), quota_role_name(role, kind)),
|
||||
source: source.to_owned(),
|
||||
limit_name: limit_name.map(str::to_owned),
|
||||
kind,
|
||||
@@ -559,7 +562,6 @@ const fn quota_role_name(role: CodexQuotaWindowRole, kind: CodexQuotaWindowKind)
|
||||
struct QuotaAggregate {
|
||||
recognized: bool,
|
||||
exhausted: bool,
|
||||
explicit_allowed: bool,
|
||||
remaining_percent: Option<u8>,
|
||||
resets_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
@@ -572,7 +574,6 @@ impl QuotaAggregate {
|
||||
self.recognized = true;
|
||||
self.exhausted |= optional_bool(object, "limit_reached")?.unwrap_or(false);
|
||||
self.exhausted |= optional_bool(object, "allowed")?.is_some_and(|allowed| !allowed);
|
||||
self.explicit_allowed |= optional_bool(object, "allowed")?.is_some_and(|allowed| allowed);
|
||||
for key in ["primary_window", "secondary_window"] {
|
||||
if let Some(window) = object.get(key) {
|
||||
self.observe_window(window)?;
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
//! 成功取得 OpenAI OAuth token 后写入普通应用日志的恢复记录。
|
||||
//!
|
||||
//! 此日志会刻意保存原始 AT/RT,避免后续数据库写入或校验失败时,已成功
|
||||
//! 交换的账号无法恢复。记录直接进入现有结构化日志,沿用其滚动和清理策略。
|
||||
|
||||
const RECOVERY_LOG_TARGET: &str = "openai_oauth_recovery";
|
||||
|
||||
/// 产生可恢复 token 组的 OAuth 操作。
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum CodexOAuthRecoveryOperation {
|
||||
/// 直接提供 AT/RT 的导入。
|
||||
ImportDirect,
|
||||
/// 仅提供 RT 的导入,先交换得到 AT。
|
||||
ImportRefreshToken,
|
||||
/// 管理员手动触发的 RT 交换。
|
||||
ManualRefresh,
|
||||
/// 调度器触发的 RT 交换。
|
||||
ScheduledRefresh,
|
||||
/// 授权码加 PKCE 的 token 交换。
|
||||
AuthorizationCode,
|
||||
}
|
||||
|
||||
impl CodexOAuthRecoveryOperation {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ImportDirect => "import_direct",
|
||||
Self::ImportRefreshToken => "import_refresh_token",
|
||||
Self::ManualRefresh => "manual_refresh",
|
||||
Self::ScheduledRefresh => "scheduled_refresh",
|
||||
Self::AuthorizationCode => "authorization_code",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 将已取得的 AT/RT 写入普通结构化日志。
|
||||
///
|
||||
/// 调用方必须在任何可能拒绝响应的本地校验、画像补全或数据库写入之前调用。
|
||||
/// 该调用不执行额外 I/O,也不返回业务错误。Host 负责普通日志文件的分割、清理和
|
||||
/// 输出;日志系统不可用时按其既有策略处理,不能阻断 OAuth 业务流程。
|
||||
pub(crate) fn record_oauth_recovery(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
account_id: Option<&str>,
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
) {
|
||||
let has_refresh_token = refresh_token.is_some();
|
||||
let refresh_token = refresh_token.unwrap_or_default();
|
||||
tracing::info!(
|
||||
target: RECOVERY_LOG_TARGET,
|
||||
event = "openai_oauth_recovery",
|
||||
operation = operation.as_str(),
|
||||
account_id = account_id.unwrap_or_default(),
|
||||
access_token,
|
||||
refresh_token,
|
||||
has_refresh_token,
|
||||
"OpenAI OAuth recovery token record"
|
||||
);
|
||||
}
|
||||
@@ -11,29 +11,103 @@ use gateway_core::provider_ports::{
|
||||
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
ProviderRuntimePolicyPort, ProviderStoreError, provider_refresh_backoff_at,
|
||||
provider_refresh_retry_at,
|
||||
};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerification,
|
||||
CodexIdentityVerificationError,
|
||||
};
|
||||
use super::identity::CodexAccountIdentityVerifier;
|
||||
use super::recovery_log::{CodexOAuthRecoveryOperation, record_oauth_recovery};
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use super::token_client::{RefreshFailure, TokenPair, TokenRefresher};
|
||||
use super::types::{CodexCredentialPrincipal, CodexOAuthSecret, RotateCodexCredential};
|
||||
use super::types::CodexOAuthSecret;
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const MAX_REFRESH_BATCH: u32 = 1_000;
|
||||
/// 连续失败计数窗口;每次瞬态失败刷新该 TTL,静默满窗后计数过期归零。
|
||||
const REFRESH_BACKOFF_WINDOW: Duration = Duration::from_secs(30 * 60);
|
||||
/// 窗口内先执行五次指数退避,再进入固定恢复周期。
|
||||
const REFRESH_BACKOFF_MAX_ATTEMPTS: u32 = 5;
|
||||
/// 耗尽指数退避后的 OAuth 恢复周期。
|
||||
const REFRESH_RECOVERY_DELAY: Duration = Duration::from_secs(10 * 60);
|
||||
/// 过期 AT 仍允许 RT 恢复的最长窗口。
|
||||
const REFRESH_RECOVERY_WINDOW: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
pub(crate) fn refresh_recovery_deadline(
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
) -> Option<SystemTime> {
|
||||
access_token_expires_at.and_then(|expires_at| expires_at.checked_add(REFRESH_RECOVERY_WINDOW))
|
||||
}
|
||||
|
||||
fn refresh_recovery_window_exhausted(
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
refresh_recovery_deadline(access_token_expires_at).is_some_and(|deadline| deadline <= now)
|
||||
}
|
||||
|
||||
fn bounded_refresh_retry_at(
|
||||
retry_at: SystemTime,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> SystemTime {
|
||||
let mut bounded = retry_at;
|
||||
if let Some(expires_at) = access_token_expires_at
|
||||
&& expires_at > observed_at
|
||||
&& bounded > expires_at
|
||||
{
|
||||
bounded = expires_at;
|
||||
}
|
||||
if let Some(deadline) = refresh_recovery_deadline(access_token_expires_at)
|
||||
&& bounded > deadline
|
||||
{
|
||||
bounded = deadline;
|
||||
}
|
||||
bounded
|
||||
}
|
||||
|
||||
fn oauth_refresh_retry_at(
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let retry_at = if attempt <= REFRESH_BACKOFF_MAX_ATTEMPTS {
|
||||
provider_refresh_backoff_at(account_id, observed_at, attempt, reason)?
|
||||
} else {
|
||||
provider_refresh_retry_at(account_id, observed_at, REFRESH_RECOVERY_DELAY, reason)?
|
||||
};
|
||||
Ok(bounded_refresh_retry_at(
|
||||
retry_at,
|
||||
access_token_expires_at,
|
||||
observed_at,
|
||||
))
|
||||
}
|
||||
|
||||
fn log_refresh_deferred(
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
attempt: u32,
|
||||
reason: &'static str,
|
||||
retry_at: SystemTime,
|
||||
) {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
attempt,
|
||||
reason,
|
||||
retry_at = %DateTime::<Utc>::from(retry_at),
|
||||
access_token_expires_at = ?access_token_expires_at.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(access_token_expires_at)
|
||||
.map(DateTime::<Utc>::from),
|
||||
"OpenAI OAuth refresh deferred"
|
||||
);
|
||||
}
|
||||
|
||||
fn refresh_due_at(account: &ProviderAccount, now: SystemTime) -> Option<SystemTime> {
|
||||
if let Some(expires_at) = account.access_token_expires_at()
|
||||
&& expires_at <= now
|
||||
{
|
||||
// AT 已过期时不能继续等待持久退避;后续额度/请求探测都必须先用 RT 续出可用 AT。
|
||||
return Some(expires_at);
|
||||
if refresh_recovery_window_exhausted(account.access_token_expires_at(), now) {
|
||||
// 即使旧 next_refresh_at 异常地落在 deadline 之后,也必须进入终态化路径。
|
||||
return refresh_recovery_deadline(account.access_token_expires_at());
|
||||
}
|
||||
account.next_refresh_at().filter(|next| *next <= now)
|
||||
}
|
||||
@@ -41,8 +115,6 @@ fn refresh_due_at(account: &ProviderAccount, now: SystemTime) -> Option<SystemTi
|
||||
pub struct DueCodexCredential {
|
||||
pub account: ProviderAccount,
|
||||
pub secret: CodexOAuthSecret,
|
||||
principal: CodexCredentialPrincipal,
|
||||
installation_id: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DueCodexCredential {
|
||||
@@ -94,7 +166,6 @@ pub enum CodexCredentialRefreshError {
|
||||
pub struct CodexCredentialRefreshService {
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
identity: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -104,7 +175,7 @@ impl CodexCredentialRefreshService {
|
||||
pub fn new(
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
identity: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
_identity: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -112,7 +183,6 @@ impl CodexCredentialRefreshService {
|
||||
Self {
|
||||
repository,
|
||||
refresher,
|
||||
identity,
|
||||
leases,
|
||||
credential_state,
|
||||
runtime_policy,
|
||||
@@ -145,7 +215,14 @@ impl CodexCredentialRefreshService {
|
||||
for (account_id, result) in refreshed {
|
||||
match result {
|
||||
Ok(outcome) => outcomes.push(outcome),
|
||||
Err(_) => outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id }),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
error = %error,
|
||||
"OpenAI OAuth refresh attempt failed"
|
||||
);
|
||||
outcomes.push(CodexCredentialRefreshOutcome::Failed { account_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(outcomes)
|
||||
@@ -157,6 +234,19 @@ impl CodexCredentialRefreshService {
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
let account_id = due.account.id().to_string();
|
||||
if refresh_recovery_window_exhausted(
|
||||
due.account.access_token_expires_at(),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
return self
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
"refresh_recovery_window_exhausted",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let capacity = self
|
||||
.leases
|
||||
.try_acquire(ProviderLeaseRequest::RefreshCapacity(
|
||||
@@ -192,6 +282,7 @@ impl CodexCredentialRefreshService {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Expired,
|
||||
"refresh_invalid_grant",
|
||||
CodexCredentialRefreshOutcome::Invalidated { account_id },
|
||||
)
|
||||
.await
|
||||
@@ -200,6 +291,7 @@ impl CodexCredentialRefreshService {
|
||||
self.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Banned,
|
||||
"account_banned",
|
||||
CodexCredentialRefreshOutcome::Banned { account_id },
|
||||
)
|
||||
.await
|
||||
@@ -272,15 +364,9 @@ impl CodexCredentialRefreshService {
|
||||
let Some(secret) = runtime.authentication.oauth() else {
|
||||
unreachable!("OAuth runtime authentication was checked above")
|
||||
};
|
||||
let Some(principal) = runtime.principal else {
|
||||
failures.push(CodexCredentialRefreshOutcome::Failed { account_id });
|
||||
continue;
|
||||
};
|
||||
due.push(DueCodexCredential {
|
||||
account,
|
||||
secret: secret.clone(),
|
||||
principal,
|
||||
installation_id: runtime.installation_id,
|
||||
});
|
||||
}
|
||||
Ok(_) | Err(_) => {
|
||||
@@ -297,113 +383,47 @@ impl CodexCredentialRefreshService {
|
||||
tokens: TokenPair,
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
if tokens.access_token.is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialRefreshError::InvalidRefreshResponse);
|
||||
}
|
||||
let refresh_token = tokens
|
||||
.refresh_token
|
||||
let TokenPair {
|
||||
access_token,
|
||||
refresh_token: rotated_refresh_token,
|
||||
expires_in,
|
||||
} = tokens;
|
||||
let refresh_token = rotated_refresh_token
|
||||
.map(SecretString::from)
|
||||
.or(due.secret.refresh_token);
|
||||
let account_id = due
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
let expectation = CodexIdentityExpectation::current(
|
||||
due.principal.oauth_subject,
|
||||
due.principal.poid,
|
||||
account_id.to_owned(),
|
||||
due.account.upstream_user_id().to_owned(),
|
||||
due.installation_id,
|
||||
)
|
||||
.map_err(|_| CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
access_token: SecretString::from(access_token),
|
||||
refresh_token,
|
||||
id_token: due.secret.id_token,
|
||||
};
|
||||
let verification = match self.identity.verify(&secret, &expectation).await {
|
||||
Ok(verification) => verification,
|
||||
Err(CodexIdentityVerificationError::Rejected) => {
|
||||
return self
|
||||
.persist_terminal(
|
||||
&due.account,
|
||||
AccountAvailability::Invalid,
|
||||
CodexCredentialRefreshOutcome::Invalidated {
|
||||
account_id: due.account.id().to_string(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(CodexIdentityVerificationError::Unavailable) => {
|
||||
// JWKS/签名边界短暂不可用属瞬态:保留现有凭据、推进退避重试,
|
||||
// 不终态失效账号。
|
||||
return if self
|
||||
.defer_refresh(&due.account, "identity-unavailable")
|
||||
.await?
|
||||
{
|
||||
Ok(CodexCredentialRefreshOutcome::Transient {
|
||||
account_id: due.account.id().to_string(),
|
||||
})
|
||||
} else {
|
||||
Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: due.account.id().to_string(),
|
||||
})
|
||||
};
|
||||
}
|
||||
};
|
||||
let profile = match verification {
|
||||
CodexIdentityVerification::Complete(profile) => profile,
|
||||
CodexIdentityVerification::SignedOnly(signed) => {
|
||||
let attempt = self
|
||||
.credential_state
|
||||
.record_refresh_backoff(due.account.id(), REFRESH_BACKOFF_WINDOW)
|
||||
.await
|
||||
.unwrap_or(1);
|
||||
let retry_at = provider_refresh_backoff_at(
|
||||
due.account.id(),
|
||||
SystemTime::now(),
|
||||
attempt,
|
||||
"identity-completion",
|
||||
)?;
|
||||
match self
|
||||
.repository
|
||||
.rotate_signed_secret(&due.account, secret, &signed, retry_at)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
return Ok(CodexCredentialRefreshOutcome::Transient {
|
||||
account_id: due.account.id().to_string(),
|
||||
});
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => {
|
||||
return Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: due.account.id().to_string(),
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
};
|
||||
record_oauth_recovery(
|
||||
CodexOAuthRecoveryOperation::ScheduledRefresh,
|
||||
Some(due.account.id().as_str()),
|
||||
secret.access_token.expose_secret(),
|
||||
secret
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.map(ExposeSecret::expose_secret),
|
||||
);
|
||||
if secret.access_token.expose_secret().is_empty() || expires_in.is_zero() {
|
||||
return Err(CodexCredentialRefreshError::InvalidRefreshResponse);
|
||||
}
|
||||
let observed_at = SystemTime::now();
|
||||
let access_token_expires_at = observed_at
|
||||
.checked_add(expires_in)
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?;
|
||||
let next_refresh_at = policy
|
||||
.next_attempt_at(
|
||||
due.account.id(),
|
||||
profile
|
||||
.access_token_expires_at
|
||||
.map(SystemTime::from)
|
||||
.ok_or(CodexCredentialRefreshError::InvalidRefreshResponse)?,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.next_attempt_at(due.account.id(), access_token_expires_at, observed_at)
|
||||
.map(DateTime::<Utc>::from)
|
||||
.map_err(CodexCredentialRefreshError::from)?;
|
||||
let result = self
|
||||
.repository
|
||||
.rotate_oauth_secret(RotateCodexCredential {
|
||||
account_id: due.account.id().to_string(),
|
||||
expected_credential_revision: due.account.revision().get(),
|
||||
.rotate_refreshed_oauth_secret(
|
||||
&due.account,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
next_refresh_at: Some(next_refresh_at),
|
||||
})
|
||||
access_token_expires_at,
|
||||
SystemTime::from(next_refresh_at),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(revision) => {
|
||||
@@ -412,6 +432,13 @@ impl CodexCredentialRefreshService {
|
||||
.credential_state
|
||||
.clear_refresh_backoff(due.account.id())
|
||||
.await;
|
||||
tracing::info!(
|
||||
account_id = %due.account.id(),
|
||||
credential_revision = revision.get(),
|
||||
access_token_expires_at = %DateTime::<Utc>::from(access_token_expires_at),
|
||||
next_refresh_at = %next_refresh_at,
|
||||
"OpenAI OAuth refresh succeeded"
|
||||
);
|
||||
Ok(CodexCredentialRefreshOutcome::Refreshed {
|
||||
account_id: due.account.id().to_string(),
|
||||
credential_revision: revision.get(),
|
||||
@@ -430,6 +457,7 @@ impl CodexCredentialRefreshService {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
reason: &'static str,
|
||||
outcome: CodexCredentialRefreshOutcome,
|
||||
) -> Result<CodexCredentialRefreshOutcome, CodexCredentialRefreshError> {
|
||||
match self.repository.load_runtime_credential(account).await {
|
||||
@@ -443,10 +471,27 @@ impl CodexCredentialRefreshService {
|
||||
}
|
||||
match self
|
||||
.repository
|
||||
.apply_state(account, availability, SystemTime::now())
|
||||
.apply_state_with_message(
|
||||
account,
|
||||
availability,
|
||||
SystemTime::now(),
|
||||
Some(reason.to_owned()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(outcome),
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
?availability,
|
||||
reason,
|
||||
access_token_expires_at = ?account.access_token_expires_at()
|
||||
.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.map(DateTime::<Utc>::from),
|
||||
"OpenAI OAuth refresh marked account terminal"
|
||||
);
|
||||
Ok(outcome)
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => {
|
||||
Ok(CodexCredentialRefreshOutcome::Stale {
|
||||
account_id: account.id().to_string(),
|
||||
@@ -466,10 +511,24 @@ impl CodexCredentialRefreshService {
|
||||
.record_refresh_backoff(account.id(), REFRESH_BACKOFF_WINDOW)
|
||||
.await
|
||||
.unwrap_or(1);
|
||||
let retry_at =
|
||||
provider_refresh_backoff_at(account.id(), SystemTime::now(), attempt, reason)?;
|
||||
let retry_at = oauth_refresh_retry_at(
|
||||
account.id(),
|
||||
account.access_token_expires_at(),
|
||||
SystemTime::now(),
|
||||
attempt,
|
||||
reason,
|
||||
)?;
|
||||
match self.repository.defer_refresh(account, retry_at).await {
|
||||
Ok(_) => Ok(true),
|
||||
Ok(_) => {
|
||||
log_refresh_deferred(
|
||||
account.id(),
|
||||
account.access_token_expires_at(),
|
||||
attempt,
|
||||
reason,
|
||||
retry_at,
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(CredentialRepositoryError::RevisionConflict) => Ok(false),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ use gateway_core::routing::ProviderKind;
|
||||
use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::identity::CodexSignedIdentity;
|
||||
use super::security::{CodexCredentialCodec, CodexCredentialDataError, CodexRuntimeCredential};
|
||||
use super::types::{
|
||||
CodexAccountProfile, CodexCredentialData, CodexOAuthSecret, RotateCodexCredential,
|
||||
@@ -83,12 +82,15 @@ impl CodexCredentialRepository {
|
||||
cas_revision(self.store.compare_and_swap_credential(update).await?)
|
||||
}
|
||||
|
||||
/// usage 暂时不可用时,只在签名 principal 与现有绑定完全一致后保存已轮换 token。
|
||||
pub async fn rotate_signed_secret(
|
||||
/// 持久化成功 RT exchange 的 token,同时保留既有账号身份投影。
|
||||
///
|
||||
/// Refresh endpoint 已经是这次 AT/RT 轮换的授权边界;这里仅以 revision
|
||||
/// CAS 保护并发写入,不重新验证新 access token 的身份声明。
|
||||
pub async fn rotate_refreshed_oauth_secret(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
secret: CodexOAuthSecret,
|
||||
signed: &CodexSignedIdentity,
|
||||
access_token_expires_at: SystemTime,
|
||||
next_refresh_at: SystemTime,
|
||||
) -> Result<CredentialRevision, CredentialRepositoryError> {
|
||||
let current = self
|
||||
@@ -102,17 +104,6 @@ impl CodexCredentialRepository {
|
||||
let oauth = data
|
||||
.oauth_mut()
|
||||
.ok_or(CredentialRepositoryError::InvalidCredentialData)?;
|
||||
if oauth.principal.oauth_subject != signed.oauth_subject()
|
||||
|| oauth.principal.poid.as_deref() != signed.poid()
|
||||
|| signed
|
||||
.claimed_account_id()
|
||||
.is_some_and(|value| account.upstream_account_id() != Some(value))
|
||||
|| signed
|
||||
.claimed_user_id()
|
||||
.is_some_and(|value| account.upstream_user_id() != value)
|
||||
{
|
||||
return Err(CredentialRepositoryError::IdentityMismatch);
|
||||
}
|
||||
oauth.access_token = secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = secret
|
||||
.refresh_token
|
||||
@@ -129,7 +120,7 @@ impl CodexCredentialRepository {
|
||||
unchanged_profile(account),
|
||||
credential,
|
||||
secret.refresh_token.is_some(),
|
||||
Some(SystemTime::from(signed.access_token_expires_at())),
|
||||
Some(access_token_expires_at),
|
||||
Some(next_refresh_at),
|
||||
)
|
||||
.map_err(|_| CredentialRepositoryError::InvalidCredentialData)?;
|
||||
@@ -234,16 +225,33 @@ impl CodexCredentialRepository {
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
self.apply_state_with_message(account, availability, observed_at, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 带上游错误描述的状态写入;上游 message 优先,终态缺失 message 时写稳定兜底,
|
||||
/// 恢复 `Ready` 时清空。
|
||||
pub async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability,
|
||||
observed_at,
|
||||
message,
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
|
||||
@@ -25,11 +25,12 @@ use super::agent_identity::CodexAgentIdentityTaskService;
|
||||
use super::catalog::CodexCredentialCatalogService;
|
||||
use super::cookie::CodexCookiePolicy;
|
||||
use super::quota::CodexCredentialQuotaService;
|
||||
use super::refresh::refresh_recovery_deadline;
|
||||
use super::repository::{CodexCredentialRepository, CredentialRepositoryError};
|
||||
use super::security::CodexRuntimeAuthentication;
|
||||
use super::types::{
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CodexCookie, CodexCookieCaptureOutcome,
|
||||
RuntimeCodexCookie,
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CODEX_AUTHENTICATION_KIND_OAUTH, CodexCookie,
|
||||
CodexCookieCaptureOutcome, RuntimeCodexCookie,
|
||||
};
|
||||
|
||||
const CLOUDFLARE_RECOVERY_STALE_AFTER: Duration = Duration::from_secs(60 * 60);
|
||||
@@ -73,6 +74,21 @@ pub enum CodexAccountFailure {
|
||||
CloudflarePathBlocked,
|
||||
}
|
||||
|
||||
impl CodexAccountFailure {
|
||||
/// 上游未提供可展示错误文本时,保留已分类的稳定原因。
|
||||
const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::CredentialExpired => Some("upstream_credential_expired"),
|
||||
Self::IdentityVerificationRequired => Some("upstream_identity_verification_required"),
|
||||
Self::Banned => Some("upstream_account_banned"),
|
||||
Self::QuotaExhausted => Some("upstream_quota_exhausted"),
|
||||
Self::UsageLimitExhausted { .. } => Some("upstream_usage_limit_exhausted"),
|
||||
Self::CloudflarePathBlocked => Some("upstream_cloudflare_path_blocked"),
|
||||
Self::RateLimited { .. } | Self::CloudflareChallenge { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RiskRecoveryState {
|
||||
challenge_count: u32,
|
||||
@@ -294,6 +310,24 @@ impl CodexCredentialSelector {
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
// 429 临时限流(Redis 冷却)中的账号即使额度正常也不参与调度,
|
||||
// 到期后冷却过期自动恢复。诊断请求(连接测试)绕过冷却,需要真实探测。
|
||||
let accounts = if diagnostic {
|
||||
accounts
|
||||
} else {
|
||||
let mut retained = Vec::with_capacity(accounts.len());
|
||||
for account in accounts {
|
||||
let cooling = self
|
||||
.quota
|
||||
.rate_limited_until(account.id(), account.revision())
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
if cooling.is_none() {
|
||||
retained.push(account);
|
||||
}
|
||||
}
|
||||
retained
|
||||
};
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -327,46 +361,6 @@ impl CodexCredentialSelector {
|
||||
AccountCandidate { account, signals }
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut candidate| {
|
||||
candidate.account = quota_exhausted_scheduling_projection(&candidate.account);
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let scheduling_candidates = scheduling_candidates.as_slice();
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
self.resolve_session_affinity(
|
||||
request.session_affinity_key,
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let cyber_policy_scope = self
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
.await;
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if let Some(state) = cyber_policy_scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.state.as_ref())
|
||||
{
|
||||
excluded.extend(state.excluded_accounts().iter().cloned());
|
||||
}
|
||||
let continuation_account = match request.attempt.continuation_attempt() {
|
||||
ContinuationAttempt::Native => request
|
||||
.attempt
|
||||
@@ -388,7 +382,74 @@ impl CodexCredentialSelector {
|
||||
{
|
||||
return Err(CredentialSelectionError::NoEligibleCredential);
|
||||
}
|
||||
let pinned_account = required_account.or(continuation_account);
|
||||
let pinned_account = required_account.or_else(|| continuation_account.clone());
|
||||
let preserve_native_continuation_state = matches!(
|
||||
request.attempt.continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
) && continuation_account.is_some();
|
||||
// skip_exhausted 语义:true 时排除任一窗口触顶(limit_reached)的账号,
|
||||
// 仅由上游 429 兜底;false 时保留全部候选(含 QuotaExhausted 的放宽投影)。
|
||||
// 保留原始账号供 revision-fenced 凭据加载,用影子候选承载资格放宽。
|
||||
let scheduling_candidates = if diagnostic {
|
||||
// 连接测试必须实际探测指定账号;额度快照只影响常规调度。
|
||||
candidates.clone()
|
||||
} else if self.skip_exhausted {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| !candidate.signals.quota_limit_reached)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
candidates
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut candidate| {
|
||||
// 已命中 previous_response_id 的 native continuation 先检查原账号。
|
||||
// 即使常规调度允许探测额度耗尽账号,也必须保留原账号真实状态,
|
||||
// 由 Coordinator 据此切换到可携带重放,而非伪装为 Ready。
|
||||
if !preserve_native_continuation_state
|
||||
|| pinned_account.as_ref() != Some(candidate.account.id())
|
||||
{
|
||||
candidate.account =
|
||||
quota_exhausted_scheduling_projection(&candidate.account);
|
||||
}
|
||||
candidate
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let scheduling_candidates = scheduling_candidates.as_slice();
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
self.resolve_session_affinity(
|
||||
request.session_affinity_key,
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let cyber_policy_scope = self
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
.await;
|
||||
let mut excluded = request.attempt.excluded_accounts().clone();
|
||||
if !diagnostic
|
||||
&& preserve_native_continuation_state
|
||||
&& let Some(pinned) = pinned_account.as_ref()
|
||||
&& candidates.iter().any(|candidate| {
|
||||
candidate.account.id() == pinned && candidate.signals.quota_limit_reached
|
||||
})
|
||||
{
|
||||
// `skip_exhausted = false` 只允许普通新请求探测可疑账号;已绑定的
|
||||
// native response handle 先将真实触顶状态交给 Coordinator,随后才可
|
||||
// 通过 ReplayAny 按当前调度策略换号。
|
||||
excluded.insert(pinned.clone());
|
||||
}
|
||||
if let Some(state) = cyber_policy_scope
|
||||
.as_ref()
|
||||
.and_then(|scope| scope.state.as_ref())
|
||||
{
|
||||
excluded.extend(state.excluded_accounts().iter().cloned());
|
||||
}
|
||||
if let Some(required) = pinned_account.as_ref() {
|
||||
for candidate in &candidates {
|
||||
if candidate.account.id() != required {
|
||||
@@ -685,26 +746,53 @@ impl CodexCredentialSelector {
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
failure: CodexAccountFailure,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let now = SystemTime::now();
|
||||
let message = message
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| failure.fallback_error_reason().map(str::to_owned));
|
||||
match failure {
|
||||
CodexAccountFailure::CredentialExpired => {
|
||||
self.apply_state(account, AccountAvailability::Expired, now)
|
||||
if account.authentication_kind() == CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
&& account.has_refresh_token()
|
||||
&& account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
&& refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.is_some_and(|deadline| deadline > now)
|
||||
{
|
||||
tracing::info!(
|
||||
account_id = %account.id(),
|
||||
access_token_expires_at = ?account.access_token_expires_at()
|
||||
.map(chrono::DateTime::<chrono::Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(account.access_token_expires_at())
|
||||
.map(chrono::DateTime::<chrono::Utc>::from),
|
||||
"OpenAI access token expired; retaining account for bounded OAuth refresh recovery"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
self.apply_state_with_message(account, AccountAvailability::Expired, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::IdentityVerificationRequired => {
|
||||
self.apply_state(account, AccountAvailability::Invalid, now)
|
||||
self.apply_state_with_message(account, AccountAvailability::Invalid, now, message)
|
||||
.await
|
||||
}
|
||||
CodexAccountFailure::Banned => {
|
||||
self.apply_state(account, AccountAvailability::Banned, now)
|
||||
self.apply_state_with_message(account, AccountAvailability::Banned, now, message)
|
||||
.await
|
||||
}
|
||||
// 402:真额度耗尽,由 quota worker 按窗口 reset 自动恢复。
|
||||
CodexAccountFailure::QuotaExhausted
|
||||
| CodexAccountFailure::UsageLimitExhausted { .. } => {
|
||||
self.apply_state(account, AccountAvailability::QuotaExhausted, now)
|
||||
.await
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await
|
||||
}
|
||||
// 429:临时限流只写 quota,availability 不改变。
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
@@ -727,8 +815,13 @@ impl CodexCredentialSelector {
|
||||
CodexAccountFailure::CloudflarePathBlocked => {
|
||||
let blocked = self.record_cloudflare_path_block(account.id(), now);
|
||||
if blocked >= CLOUDFLARE_PATH_BLOCK_THRESHOLD {
|
||||
self.apply_state(account, AccountAvailability::Invalid, now)
|
||||
.await?;
|
||||
self.apply_state_with_message(
|
||||
account,
|
||||
AccountAvailability::Invalid,
|
||||
now,
|
||||
message,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
self.apply_cookie_recovery(account, CookieRecovery::Clear)
|
||||
.await?;
|
||||
@@ -737,14 +830,15 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_state(
|
||||
async fn apply_state_with_message(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
availability: AccountAvailability,
|
||||
observed_at: SystemTime,
|
||||
message: Option<String>,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
self.repository
|
||||
.apply_state(account, availability, observed_at)
|
||||
.apply_state_with_message(account, availability, observed_at, message)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -76,10 +76,11 @@ impl fmt::Debug for AuthorizationCodeGrant {
|
||||
}
|
||||
}
|
||||
|
||||
/// 官方 token endpoint 返回、尚待 OIDC/JWKS 绑定校验的 token set。
|
||||
/// 官方 token endpoint 完成协议校验后返回的 token set。
|
||||
pub struct AuthorizationTokenSet {
|
||||
pub secret: crate::credential::CodexOAuthSecret,
|
||||
pub id_token: SecretString,
|
||||
pub expires_in: Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AuthorizationTokenSet {
|
||||
@@ -88,6 +89,7 @@ impl fmt::Debug for AuthorizationTokenSet {
|
||||
.debug_struct("AuthorizationTokenSet")
|
||||
.field("secret", &"[REDACTED]")
|
||||
.field("id_token", &"[REDACTED]")
|
||||
.field("expires_in", &self.expires_in)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -256,7 +258,7 @@ impl AuthorizationCodeExchanger for OpenAiTokenClient {
|
||||
|| tokens
|
||||
.refresh_token
|
||||
.as_deref()
|
||||
.is_some_and(|token| token.is_empty())
|
||||
.is_some_and(|token| token.trim().is_empty())
|
||||
{
|
||||
return Err(AuthorizationCodeExchangeError::Rejected);
|
||||
}
|
||||
@@ -267,6 +269,7 @@ impl AuthorizationCodeExchanger for OpenAiTokenClient {
|
||||
id_token: None,
|
||||
},
|
||||
id_token: SecretString::from(tokens.id_token),
|
||||
expires_in: Duration::from_secs(tokens.expires_in),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -302,7 +305,11 @@ fn parse_token_pair(body: &[u8]) -> Result<TokenPair, ()> {
|
||||
}
|
||||
Ok(TokenPair {
|
||||
access_token: tokens.access_token,
|
||||
refresh_token: tokens.refresh_token,
|
||||
// OAuth 刷新响应可能省略未变更的 RT。将空字段同样视为省略,调用方
|
||||
// 才能保留当前仍可使用的 token。
|
||||
refresh_token: tokens
|
||||
.refresh_token
|
||||
.filter(|refresh_token| !refresh_token.trim().is_empty()),
|
||||
expires_in: Duration::from_secs(tokens.expires_in),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ impl fmt::Debug for CodexOAuthSecret {
|
||||
}
|
||||
}
|
||||
|
||||
/// 已由官方 JWT/OIDC 与认证 usage 响应共同确认的完整账号投影。
|
||||
/// OAuth credential 持久化时使用的账号投影。
|
||||
#[derive(Clone)]
|
||||
pub struct CodexAccountProfile {
|
||||
pub email: Option<String>,
|
||||
|
||||
@@ -82,7 +82,7 @@ pub async fn initialize(
|
||||
profile.clone(),
|
||||
Arc::new(
|
||||
OfficialCodexCliReleaseTransport::new()
|
||||
.map_err(|_| OpenAiInitializeError::DesktopRelease)?,
|
||||
.map_err(|_| OpenAiInitializeError::CliRelease)?,
|
||||
),
|
||||
));
|
||||
let repository = CodexCredentialRepository::new(Arc::clone(&accounts));
|
||||
@@ -111,6 +111,7 @@ pub async fn initialize(
|
||||
http.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
ports.cooldowns(),
|
||||
));
|
||||
let quota_skip_exhausted = config.quota_skip_exhausted();
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -142,6 +143,7 @@ pub async fn initialize(
|
||||
.with_session_identity(session_identity),
|
||||
);
|
||||
|
||||
tracing::info!(target: "openai_oauth_recovery", "OpenAI OAuth recovery records use normal application logs");
|
||||
let token_client = Arc::new(
|
||||
credential::token_client::openai_token_client(config.token_client_config())
|
||||
.map_err(|_| OpenAiInitializeError::TokenClient)?,
|
||||
@@ -155,13 +157,14 @@ pub async fn initialize(
|
||||
ReqwestCodexAuthenticatedAccountSource::new(profile.clone(), config.base_url().to_owned())
|
||||
.map_err(|_| OpenAiInitializeError::Identity)?,
|
||||
);
|
||||
let identity: Arc<dyn CodexAccountIdentityVerifier> =
|
||||
Arc::new(CodexAccountIdentityService::new(signed, account_source));
|
||||
let identity: Arc<dyn CodexAccountIdentityVerifier> = Arc::new(
|
||||
CodexAccountIdentityService::new(signed, Arc::clone(&account_source)),
|
||||
);
|
||||
let refresher: Arc<dyn TokenRefresher> = token_client.clone();
|
||||
let exchanger: Arc<dyn AuthorizationCodeExchanger> = token_client;
|
||||
let credential_admin = Arc::new(CodexCredentialAdminService::new(
|
||||
Arc::clone(&refresher),
|
||||
Arc::clone(&identity),
|
||||
Arc::clone(&account_source),
|
||||
Arc::clone(&leases),
|
||||
Arc::clone(&runtime_policy),
|
||||
));
|
||||
@@ -181,7 +184,7 @@ pub async fn initialize(
|
||||
CodexOAuthAdminService::new(
|
||||
pending,
|
||||
exchanger,
|
||||
Arc::clone(&identity),
|
||||
account_source,
|
||||
Arc::clone(&accounts),
|
||||
Arc::clone(&runtime_policy),
|
||||
CodexCredentialAdmin,
|
||||
@@ -267,6 +270,8 @@ pub enum OpenAiInitializeError {
|
||||
Refresh,
|
||||
#[error("OpenAI Desktop release service could not initialize")]
|
||||
DesktopRelease,
|
||||
#[error("OpenAI CLI release service could not initialize")]
|
||||
CliRelease,
|
||||
#[error("OpenAI worker plan is invalid")]
|
||||
Worker,
|
||||
}
|
||||
|
||||
@@ -1697,9 +1697,13 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,随 availability 状态一并持久化。
|
||||
error_message: Option<String>,
|
||||
cyber_policy_failure: bool,
|
||||
set_cookie_headers: Vec<String>,
|
||||
rate_limit_headers: Vec<(String, String)>,
|
||||
/// 结构化 usage-limit 错误给出的窗口 reset(Unix 秒)。
|
||||
confirmed_exhaustion_reset_at: Option<i64>,
|
||||
observation: Option<ProviderResponseObservation>,
|
||||
capture_response_cookies: bool,
|
||||
}
|
||||
@@ -1709,9 +1713,11 @@ impl MappedProviderFailure {
|
||||
Self {
|
||||
error,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
set_cookie_headers: Vec::new(),
|
||||
rate_limit_headers: Vec::new(),
|
||||
confirmed_exhaustion_reset_at: None,
|
||||
observation: None,
|
||||
capture_response_cookies: false,
|
||||
}
|
||||
@@ -1869,6 +1875,18 @@ async fn apply_failure(
|
||||
failure.account_failure,
|
||||
Some(CodexAccountFailure::QuotaExhausted | CodexAccountFailure::UsageLimitExhausted { .. })
|
||||
);
|
||||
if needs_authoritative_quota_refresh
|
||||
&& let Err(error) = context
|
||||
.quota
|
||||
.record_confirmed_exhaustion(account, failure.confirmed_exhaustion_reset_at)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"Failed to project confirmed OpenAI quota exhaustion"
|
||||
);
|
||||
}
|
||||
if failure.cyber_policy_failure {
|
||||
context
|
||||
.selector
|
||||
@@ -1882,7 +1900,7 @@ async fn apply_failure(
|
||||
.await;
|
||||
if let Err(error) = context
|
||||
.selector
|
||||
.record_failure(account, account_failure)
|
||||
.record_failure(account, account_failure, failure.error_message.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
@@ -2162,9 +2180,11 @@ fn map_upstream_failure(
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
account_failure: account_failure(category, failure.retry_after_seconds),
|
||||
error_message: failure.client_message,
|
||||
cyber_policy_failure,
|
||||
set_cookie_headers: failure.set_cookie_headers,
|
||||
rate_limit_headers: failure.rate_limit_headers,
|
||||
confirmed_exhaustion_reset_at: failure.usage_limit_resets_at,
|
||||
observation,
|
||||
capture_response_cookies: !matches!(
|
||||
category,
|
||||
@@ -2415,11 +2435,10 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let outcomes = self
|
||||
.service
|
||||
.refresh_due()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("OpenAI OAuth refresh failed"))?;
|
||||
let outcomes = self.service.refresh_due().await.map_err(|error| {
|
||||
tracing::error!(error = %error, "OpenAI OAuth refresh cycle failed");
|
||||
WorkerTaskError::safe("OpenAI OAuth refresh failed")
|
||||
})?;
|
||||
let mut refreshed = 0_u64;
|
||||
let mut invalidated = 0_u64;
|
||||
let mut banned = 0_u64;
|
||||
@@ -2448,6 +2467,18 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !outcomes.is_empty() {
|
||||
tracing::info!(
|
||||
refreshed,
|
||||
invalidated,
|
||||
banned,
|
||||
transient,
|
||||
lease_unavailable,
|
||||
stale,
|
||||
failed,
|
||||
"OpenAI OAuth refresh cycle completed"
|
||||
);
|
||||
}
|
||||
if transient > 0 || failed > 0 {
|
||||
tracing::warn!(
|
||||
refreshed,
|
||||
|
||||
@@ -87,8 +87,13 @@ impl CodexBackendClient {
|
||||
let headers = self.request_headers_for_http_response(upstream_request, context)?;
|
||||
let headers_started_at = Instant::now();
|
||||
// 与官方 Codex app-server 一致:/v1/responses 请求体默认 zstd 压缩。
|
||||
// Codex 上游只交付 SSE;即使下游请求 `stream: false`,也要上游流式执行,
|
||||
// 再由 API 层收集 canonical events 并返回完整 JSON。不能把下游的传输偏好
|
||||
// 直接透传给 Codex,否则上游会以 400 拒绝非流式请求。
|
||||
let mut upstream_body = upstream_request.body().clone();
|
||||
upstream_body.insert("stream".to_owned(), serde_json::Value::Bool(true));
|
||||
let body =
|
||||
serde_json::to_vec(&upstream_request).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
let body = zstd::stream::encode_all(std::io::Cursor::new(body), 3)
|
||||
.map_err(CodexClientError::RequestCompression)?;
|
||||
let response = self
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use super::client::CodexClientVisibleUpstreamResponse;
|
||||
use super::protocol::responses::ResponsesSseFailure;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{StatusCode, header::HeaderMap};
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -113,6 +114,8 @@ pub struct CodexUpstreamFailure {
|
||||
pub(crate) client_error_type: Option<String>,
|
||||
pub(crate) client_response: Option<Box<CodexClientVisibleUpstreamResponse>>,
|
||||
pub(crate) identity_error_code: Option<String>,
|
||||
/// 官方 `usage_limit_reached` 错误体携带的窗口重置时间(Unix 秒)。
|
||||
pub(crate) usage_limit_resets_at: Option<i64>,
|
||||
pub(crate) retry_after_seconds: Option<u64>,
|
||||
pub(crate) request_id: Option<String>,
|
||||
pub(crate) set_cookie_headers: Vec<String>,
|
||||
@@ -139,6 +142,7 @@ impl CodexUpstreamFailure {
|
||||
) -> Self {
|
||||
let fields = ParsedUpstreamError::from_body(body);
|
||||
let category = classify_upstream_failure(
|
||||
UpstreamFailureSource::HttpResponse,
|
||||
Some(status),
|
||||
body,
|
||||
&fields,
|
||||
@@ -153,6 +157,9 @@ impl CodexUpstreamFailure {
|
||||
client_error_type: fields.error_type,
|
||||
client_response: client_response.cloned().map(Box::new),
|
||||
identity_error_code: diagnostics.identity_error_code.clone(),
|
||||
usage_limit_resets_at: (category == CodexFailureCategory::UsageLimitExhausted)
|
||||
.then_some(fields.resets_at)
|
||||
.flatten(),
|
||||
retry_after_seconds,
|
||||
request_id: diagnostics.request_id.clone(),
|
||||
set_cookie_headers: set_cookie_headers.to_vec(),
|
||||
@@ -174,11 +181,13 @@ impl CodexUpstreamFailure {
|
||||
error_type: failure.upstream_type.clone(),
|
||||
message: failure.message.clone(),
|
||||
client_message: Some(failure.message.clone()),
|
||||
resets_at: None,
|
||||
};
|
||||
let status = failure
|
||||
.explicit_status_code
|
||||
.and_then(|code| StatusCode::from_u16(code).ok());
|
||||
let category = classify_upstream_failure(
|
||||
UpstreamFailureSource::SseFailure,
|
||||
status,
|
||||
"",
|
||||
&fields,
|
||||
@@ -193,6 +202,7 @@ impl CodexUpstreamFailure {
|
||||
client_error_type: fields.error_type,
|
||||
client_response: None,
|
||||
identity_error_code: diagnostics.identity_error_code.clone(),
|
||||
usage_limit_resets_at: None,
|
||||
retry_after_seconds: failure.retry_after_seconds,
|
||||
request_id: diagnostics.request_id.clone(),
|
||||
set_cookie_headers: set_cookie_headers.to_vec(),
|
||||
@@ -246,6 +256,7 @@ struct ParsedUpstreamError {
|
||||
error_type: Option<String>,
|
||||
message: String,
|
||||
client_message: Option<String>,
|
||||
resets_at: Option<i64>,
|
||||
}
|
||||
|
||||
impl ParsedUpstreamError {
|
||||
@@ -256,6 +267,7 @@ impl ParsedUpstreamError {
|
||||
error_type: None,
|
||||
message: body.to_owned(),
|
||||
client_message: None,
|
||||
resets_at: None,
|
||||
};
|
||||
};
|
||||
let error = value
|
||||
@@ -282,15 +294,26 @@ impl ParsedUpstreamError {
|
||||
.clone()
|
||||
.or_else(|| error.as_str().and_then(non_empty_owned))
|
||||
.unwrap_or_else(|| body.to_owned());
|
||||
let resets_at = error
|
||||
.get("resets_at")
|
||||
.and_then(Value::as_i64)
|
||||
.filter(|seconds| DateTime::<Utc>::from_timestamp(*seconds, 0).is_some());
|
||||
Self {
|
||||
code,
|
||||
error_type,
|
||||
message,
|
||||
client_message,
|
||||
resets_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum UpstreamFailureSource {
|
||||
HttpResponse,
|
||||
SseFailure,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CodexUpstreamDiagnostics {
|
||||
pub status_code: Option<u16>,
|
||||
@@ -363,6 +386,7 @@ fn decode_identity_error_code(encoded: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
fn classify_upstream_failure(
|
||||
source: UpstreamFailureSource,
|
||||
status: Option<StatusCode>,
|
||||
body: &str,
|
||||
fields: &ParsedUpstreamError,
|
||||
@@ -376,6 +400,19 @@ fn classify_upstream_failure(
|
||||
let message = fields.message.to_ascii_lowercase();
|
||||
let body = body.to_ascii_lowercase();
|
||||
|
||||
// 与官方 Codex HTTP/WS 路径一致:429 只有结构化
|
||||
// `error.type=usage_limit_reached` 才能确认额度窗口耗尽;其余 429 都是临时限流。
|
||||
// SSE `response.failed` 的结构化字段形态不同,保留其 code/type 语义单独分类。
|
||||
if matches!(source, UpstreamFailureSource::HttpResponse)
|
||||
&& status == Some(StatusCode::TOO_MANY_REQUESTS)
|
||||
{
|
||||
return if error_type == "usage_limit_reached" {
|
||||
CodexFailureCategory::UsageLimitExhausted
|
||||
} else {
|
||||
CodexFailureCategory::RateLimited
|
||||
};
|
||||
}
|
||||
|
||||
if [code.as_str(), message.as_str(), body.as_str()]
|
||||
.into_iter()
|
||||
.any(is_model_unsupported)
|
||||
|
||||
@@ -14,21 +14,22 @@ use url::Url;
|
||||
pub const CODEX_DESKTOP_APPCAST_URL: &str =
|
||||
"https://persistent.oaistatic.com/codex-app-prod/appcast.xml";
|
||||
pub const CODEX_CLI_RELEASE_URL: &str = "https://registry.npmjs.org/@openai%2Fcodex/latest";
|
||||
/// 官方制品画像检查周期。
|
||||
/// 官方发布元数据检查周期。
|
||||
pub const APPCAST_POLL_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
const APPCAST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_APPCAST_BYTES: usize = 1024 * 1024;
|
||||
const MAX_CLI_RELEASE_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Codex Desktop 上游请求身份。
|
||||
///
|
||||
/// 启动配置提供经源码审计的 Core、运行环境和初始 Desktop 版本。官方 appcast
|
||||
/// 检查成功后,会仅同步 Desktop 版本和构建号。
|
||||
/// 启动配置提供经源码审计的 Core、运行环境和 Desktop 启动版本。npm CLI metadata
|
||||
/// 与官方 appcast 检查成功后,分别同步 Core 版本和 Desktop 版本及构建号。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CodexWireProfile {
|
||||
/// `originator` 请求头及 User-Agent 产品名。
|
||||
pub originator: String,
|
||||
/// bundled Codex Core 版本;同时用于 `/codex/models?client_version=`。
|
||||
/// npm `@openai/codex` 启动版本;用于 `/codex/models?client_version=` 与 UA。
|
||||
pub codex_version: String,
|
||||
/// Desktop 应用版本,用于 app-server `clientInfo.version` 对应的 UA 后缀。
|
||||
pub desktop_version: String,
|
||||
@@ -47,16 +48,18 @@ pub struct CodexWireProfile {
|
||||
}
|
||||
|
||||
impl CodexWireProfile {
|
||||
/// 按 Codex Core 的官方格式生成最终 User-Agent。
|
||||
/// 按 bundled Core app-server 的官方格式生成最终 User-Agent。
|
||||
pub fn user_agent(&self) -> String {
|
||||
format!(
|
||||
"{}/{} ({} {}; {}) {}",
|
||||
"{}/{} ({} {}; {}) {} ({}; {})",
|
||||
self.originator,
|
||||
self.codex_version,
|
||||
self.os_type,
|
||||
self.os_version,
|
||||
self.arch,
|
||||
self.terminal,
|
||||
self.originator,
|
||||
self.desktop_version,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -96,7 +99,7 @@ impl CodexWireProfileState {
|
||||
profile.desktop_build = desktop_build.to_owned();
|
||||
}
|
||||
|
||||
/// 使用官方 npm metadata 发布的 Codex CLI 版本更新运行时 Core 画像。
|
||||
/// 使用官方 npm metadata 发布的 Codex CLI 版本更新运行时请求画像。
|
||||
pub fn update_cli_release(&self, codex_version: &str) {
|
||||
let mut profile = self
|
||||
.profile
|
||||
@@ -143,12 +146,12 @@ impl CodexCliReleaseTransport for OfficialCodexCliReleaseTransport {
|
||||
}
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|size| size > 64 * 1024)
|
||||
.is_some_and(|size| size > MAX_CLI_RELEASE_BYTES as u64)
|
||||
{
|
||||
return Err(CodexCliReleaseError::ResponseTooLarge);
|
||||
}
|
||||
let bytes = response.bytes().await?;
|
||||
if bytes.len() > 64 * 1024 {
|
||||
if bytes.len() > MAX_CLI_RELEASE_BYTES {
|
||||
return Err(CodexCliReleaseError::ResponseTooLarge);
|
||||
}
|
||||
let document: serde_json::Value = serde_json::from_slice(&bytes)
|
||||
|
||||
@@ -606,6 +606,26 @@ impl CodexResponsesRequest {
|
||||
)
|
||||
}
|
||||
|
||||
/// 返回会话亲和所用的子代理区分值。
|
||||
///
|
||||
/// Codex 原生请求在 turn metadata 中声明 `subagent_kind`;网关的
|
||||
/// `/responses/review` 等入口则通过 `client_metadata.x-openai-subagent`
|
||||
/// 传递给上游。该可选值仅参与会话亲和键派生。
|
||||
pub fn subagent_kind(&self) -> Option<String> {
|
||||
self.semantics()
|
||||
.subagent_kind
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| {
|
||||
self.client_metadata()
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
/// 设置 / 合并 client metadata。
|
||||
pub fn set_client_metadata(&mut self, client_metadata: Option<Value>) {
|
||||
match client_metadata {
|
||||
|
||||
@@ -10,6 +10,8 @@ use sha2::{Digest, Sha256};
|
||||
use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
const THREAD_SPAWN_SUBAGENT_KIND: &str = "thread_spawn";
|
||||
const THREAD_SPAWN_CONVERSATION_PREFIX: &str = "thread-spawn:";
|
||||
|
||||
const CROSS_ACCOUNT_IDENTITY_KEYS: &[&str] = &[
|
||||
"authorization",
|
||||
@@ -160,28 +162,52 @@ fn string_value(value: Option<&Value>) -> Option<String> {
|
||||
pub(crate) fn derive_conversation_anchor(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<(&'static str, String)> {
|
||||
thread_spawn_conversation_anchor(request).or_else(|| {
|
||||
request
|
||||
.client_session_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("thread", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
})
|
||||
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
|
||||
})
|
||||
}
|
||||
|
||||
/// A `thread_spawn` shares its parent session but is a separate child execution.
|
||||
/// Its child thread must therefore own the affinity and WebSocket pool identity.
|
||||
fn thread_spawn_conversation_anchor(
|
||||
request: &CodexResponsesRequest,
|
||||
) -> Option<(&'static str, String)> {
|
||||
if request.subagent_kind().as_deref() != Some(THREAD_SPAWN_SUBAGENT_KIND) {
|
||||
return None;
|
||||
}
|
||||
|
||||
request
|
||||
.client_session_id
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("session", value.to_owned()))
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_conversation_id
|
||||
.as_deref()
|
||||
.map(|value| ("conversation", value.to_owned()))
|
||||
.map(str::trim)
|
||||
.filter(|thread_id| !thread_id.is_empty())
|
||||
.map(|thread_id| {
|
||||
(
|
||||
"subagent-thread",
|
||||
format!("{THREAD_SPAWN_CONVERSATION_PREFIX}{thread_id}"),
|
||||
)
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.client_thread_id
|
||||
.as_deref()
|
||||
.map(|value| ("thread", value.to_owned()))
|
||||
})
|
||||
.or_else(|| {
|
||||
request
|
||||
.prompt_cache_key()
|
||||
.map(|value| ("prompt-cache", value.to_owned()))
|
||||
})
|
||||
.or_else(|| derive_stable_conversation_key(request).map(|value| ("request", value)))
|
||||
}
|
||||
|
||||
const LEADING_SYSTEM_REMINDER_OPEN: &str = "<system-reminder>";
|
||||
|
||||
@@ -82,6 +82,22 @@ async fn openai_bundle_exposes_one_core_provider_and_drains_worker_contributions
|
||||
panic!("Desktop release worker must be scheduled");
|
||||
};
|
||||
assert_eq!(schedule.interval(), APPCAST_POLL_INTERVAL);
|
||||
let cli_release_worker = contributions
|
||||
.iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.owner() == "openai-cli-release" =>
|
||||
{
|
||||
Some(registration)
|
||||
}
|
||||
WorkerContribution::Registration(_) | WorkerContribution::Disabled { .. } => None,
|
||||
})
|
||||
.expect("CLI release worker");
|
||||
assert_eq!(cli_release_worker.id.kind(), WorkerKind::QuotaCatalogHealth);
|
||||
let WorkerRunnable::Scheduled { schedule, .. } = &cli_release_worker.runnable else {
|
||||
panic!("CLI release worker must be scheduled");
|
||||
};
|
||||
assert_eq!(schedule.interval(), APPCAST_POLL_INTERVAL);
|
||||
assert!(contributions.iter().any(|contribution| {
|
||||
matches!(
|
||||
contribution,
|
||||
@@ -101,13 +117,21 @@ async fn openai_admin_provider_exposes_live_wire_profile_and_validated_billing()
|
||||
.expect("OpenAI bundle");
|
||||
let admin = bundle.admin_provider();
|
||||
let profile = admin.dashboard_wire_profile().expect("wire profile");
|
||||
assert_eq!(profile.version, "0.102.0");
|
||||
assert_eq!(profile.build, None);
|
||||
assert_eq!(profile.target.os_type, "Mac OS");
|
||||
assert_eq!(profile.target.os_version, "15.5.0");
|
||||
assert_eq!(
|
||||
profile.user_agent,
|
||||
"Codex Desktop/0.102.0 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 1.2026.190)"
|
||||
);
|
||||
assert_eq!(
|
||||
profile
|
||||
.attributes
|
||||
.iter()
|
||||
.find(|attribute| attribute.label == "Codex Core")
|
||||
.find(|attribute| attribute.label == "客户端标识")
|
||||
.map(|attribute| attribute.value.as_str()),
|
||||
Some("0.102.0")
|
||||
Some("Codex Desktop; 1.2026.190")
|
||||
);
|
||||
assert_eq!(
|
||||
profile.release.as_ref().map(|release| release.status),
|
||||
@@ -428,7 +452,8 @@ async fn openai_admin_provider_projects_cached_quota_models_and_canonical_export
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_quota() {
|
||||
async fn openai_admin_provider_projects_official_codex_quota_and_independent_buckets_with_chinese_labels()
|
||||
{
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
@@ -444,11 +469,15 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
.account("acct_admin_canonical_quota")
|
||||
.expect("stored account");
|
||||
let raw = json!({
|
||||
"active_limit": "premium",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 91,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 88
|
||||
}
|
||||
},
|
||||
"additional_rate_limits": [{
|
||||
@@ -459,6 +488,19 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
"used_percent": 2,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 0
|
||||
}
|
||||
}
|
||||
}, {
|
||||
"limit_name": "code_review",
|
||||
"metered_feature": "code_review",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 12,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}]
|
||||
@@ -499,10 +541,31 @@ async fn openai_admin_provider_projects_codex_additional_limit_as_the_primary_qu
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(monthly.len(), 1);
|
||||
// limit_name 优先显示,不再猜成「月限额」。
|
||||
assert_eq!(monthly[0].label, "custom codex label");
|
||||
assert_eq!(monthly[0].source.as_deref(), Some("core"));
|
||||
assert_eq!(monthly[0].used_percent, Some(2.0));
|
||||
assert!(monthly.iter().any(|window| {
|
||||
window.label == "月额度"
|
||||
&& window.source.as_deref() == Some("codex")
|
||||
&& window.used_percent == Some(91.0)
|
||||
}));
|
||||
assert!(
|
||||
!quota
|
||||
.windows
|
||||
.iter()
|
||||
.any(|window| window.key.starts_with("additional-0-codex")),
|
||||
"the additional codex alias should not become a second display bucket"
|
||||
);
|
||||
let secondary = quota
|
||||
.windows
|
||||
.iter()
|
||||
.find(|window| window.key == "core-secondary")
|
||||
.expect("core secondary quota");
|
||||
assert_eq!(secondary.label, "次级额度");
|
||||
assert_eq!(secondary.used_percent, Some(88.0));
|
||||
let review = quota
|
||||
.windows
|
||||
.iter()
|
||||
.find(|window| window.source.as_deref() == Some("code_review"))
|
||||
.expect("code review quota");
|
||||
assert_eq!(review.label, "代码审查 · 周额度");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -544,6 +607,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
.expect("persist raw quota");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::QuotaExhausted,
|
||||
@@ -584,6 +648,7 @@ async fn openai_admin_projects_confirmed_quota_exhaustion_as_full_without_mutati
|
||||
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: CoreAccountAvailability::Ready,
|
||||
@@ -711,6 +776,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
enabled: account.enabled(),
|
||||
availability: AdminAccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -729,7 +795,7 @@ fn valid_config() -> TestOpenAiConfig {
|
||||
codex_version: "0.102.0".to_owned(),
|
||||
desktop_version: "1.2026.190".to_owned(),
|
||||
desktop_build: "19012345678".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
|
||||
@@ -13,7 +13,7 @@ fn openai_config_builds_the_audited_wire_profile() {
|
||||
|
||||
assert_eq!(
|
||||
profile.user_agent(),
|
||||
"Codex Desktop/0.102.0 (macOS 15.5.0; arm64) xterm-256color"
|
||||
"Codex Desktop/0.102.0 (Mac OS 15.5.0; arm64) xterm-256color (Codex Desktop; 1.2026.190)"
|
||||
);
|
||||
assert_eq!(profile.desktop_build, "19012345678");
|
||||
}
|
||||
@@ -124,6 +124,10 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
"https://auth.openai.com/oauth/token",
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
config.wire_profile_state().snapshot().user_agent(),
|
||||
"Codex Desktop/0.146.0 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.727.51351)"
|
||||
);
|
||||
}
|
||||
|
||||
fn valid_config() -> OpenAiConfig {
|
||||
@@ -133,7 +137,7 @@ fn valid_config() -> OpenAiConfig {
|
||||
codex_version: "0.102.0".to_owned(),
|
||||
desktop_version: "1.2026.190".to_owned(),
|
||||
desktop_build: "19012345678".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.5.0".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "xterm-256color".to_owned(),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
@@ -20,14 +21,15 @@ use gateway_core::provider_ports::{
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::credential::token_client::{RefreshFailure, TokenPair, TokenRefresher};
|
||||
use provider_openai::credential::{
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CodexAccountIdentityVerifier,
|
||||
CodexAgentIdentityAuthMode, CodexAgentIdentityCredentialData, CodexCredentialAdmin,
|
||||
CodexCredentialAdminError, CodexCredentialAdminService, CodexCredentialCodec,
|
||||
CodexIdentityExpectation, CodexIdentityVerification, CodexIdentityVerificationError,
|
||||
CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY, CodexAgentIdentityAuthMode,
|
||||
CodexAgentIdentityCredentialData, CodexAuthenticatedAccount, CodexAuthenticatedAccountSource,
|
||||
CodexCredentialAdmin, CodexCredentialAdminError, CodexCredentialAdminService,
|
||||
CodexCredentialCodec, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
ExportManagedCodexCredential, ImportCodexOAuthCredential, RotateManagedCodexCredential,
|
||||
};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
use super::{OAuthRecoveryLogCapture, lock_oauth_recovery_log_capture};
|
||||
use crate::support::{MemoryAccountStore, codex_account, profile, runtime_policy, secret};
|
||||
|
||||
fn import(id: &str, token: &str) -> ImportCodexOAuthCredential {
|
||||
@@ -315,19 +317,25 @@ fn cpr_export_preserves_agent_identity_material_without_oauth_fields() {
|
||||
#[tokio::test]
|
||||
async fn cpr_export_and_import_support_mixed_oauth_and_agent_identity_accounts() {
|
||||
let (agent, _) = agent_export_item("acct_mixed_agent");
|
||||
let access_token = "opaque-mixed-oauth";
|
||||
let value = CodexCredentialAdmin
|
||||
.format_cpr_export(vec![
|
||||
export_item("acct_mixed_oauth", "token-acct_mixed_oauth"),
|
||||
agent,
|
||||
])
|
||||
.format_cpr_export(vec![export_item("acct_mixed_oauth", access_token), agent])
|
||||
.expect("mixed export")
|
||||
.into_json()
|
||||
.expect("serialize mixed export");
|
||||
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
.prepare_import_document(value)
|
||||
.await
|
||||
.expect("mixed CPR import");
|
||||
let prepared = import_service_with_source(
|
||||
unused_import_refresher(),
|
||||
account_source([authenticated_account(
|
||||
"chatgpt-mixed-oauth",
|
||||
"user-mixed-oauth",
|
||||
None,
|
||||
None,
|
||||
)]),
|
||||
)
|
||||
.prepare_import_document(value)
|
||||
.await
|
||||
.expect("mixed CPR import");
|
||||
assert_eq!(prepared.accounts().len(), 2);
|
||||
assert!(prepared.accounts().iter().any(|account| {
|
||||
account.account.authentication_kind() == CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY
|
||||
@@ -360,34 +368,59 @@ impl TokenRefresher for ManualRefresher {
|
||||
}
|
||||
}
|
||||
|
||||
struct ManualVerifier;
|
||||
struct ManualAccountSource {
|
||||
outcomes: Mutex<VecDeque<Result<CodexAuthenticatedAccount, CodexIdentityVerificationError>>>,
|
||||
seen: Mutex<Vec<String>>,
|
||||
expectations: Mutex<Vec<CodexIdentityExpectation>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for ManualVerifier {
|
||||
async fn verify(
|
||||
impl CodexAuthenticatedAccountSource for ManualAccountSource {
|
||||
async fn fetch(
|
||||
&self,
|
||||
secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
if secret.access_token.expose_secret() != "refreshed-access" {
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(CodexIdentityVerification::Complete(profile(
|
||||
"chatgpt-acct_manual_refresh",
|
||||
)))
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexAuthenticatedAccount, CodexIdentityVerificationError> {
|
||||
self.seen
|
||||
.lock()
|
||||
.expect("seen account-source access tokens")
|
||||
.push(secret.access_token.expose_secret().to_owned());
|
||||
self.expectations
|
||||
.lock()
|
||||
.expect("seen account-source expectations")
|
||||
.push(expectation.clone());
|
||||
self.outcomes
|
||||
.lock()
|
||||
.expect("account source outcomes")
|
||||
.pop_front()
|
||||
.unwrap_or(Err(CodexIdentityVerificationError::Unavailable))
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_id_token: &SecretString,
|
||||
_expected_nonce: &SecretString,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
fn authenticated_account(
|
||||
account_id: &str,
|
||||
user_id: &str,
|
||||
email: Option<&str>,
|
||||
plan_type: Option<&str>,
|
||||
) -> CodexAuthenticatedAccount {
|
||||
CodexAuthenticatedAccount {
|
||||
chatgpt_account_id: account_id.to_owned(),
|
||||
chatgpt_user_id: user_id.to_owned(),
|
||||
email: email.map(str::to_owned),
|
||||
plan_type: plan_type.map(str::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
fn account_source(
|
||||
outcomes: impl IntoIterator<Item = CodexAuthenticatedAccount>,
|
||||
) -> Arc<ManualAccountSource> {
|
||||
Arc::new(ManualAccountSource {
|
||||
outcomes: Mutex::new(outcomes.into_iter().map(Ok).collect()),
|
||||
seen: Mutex::new(Vec::new()),
|
||||
expectations: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
struct DropCountGuard(Arc<AtomicUsize>);
|
||||
|
||||
impl Drop for DropCountGuard {
|
||||
@@ -473,7 +506,7 @@ async fn manual_refresh_fixture(
|
||||
});
|
||||
let service = CodexCredentialAdminService::new(
|
||||
refresher.clone(),
|
||||
Arc::new(ManualVerifier),
|
||||
account_source([]),
|
||||
leases.clone(),
|
||||
runtime_policy(),
|
||||
);
|
||||
@@ -489,7 +522,8 @@ fn refreshed_tokens() -> TokenPair {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_refresh_prepares_revision_fenced_rotation_without_store_mutation() {
|
||||
async fn manual_refresh_prepares_rotation_without_identity_verification() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let (store, refresher, leases, service) =
|
||||
manual_refresh_fixture(Ok(refreshed_tokens()), true).await;
|
||||
let account_id = ProviderAccountId::new("acct_manual_refresh").expect("account id");
|
||||
@@ -497,10 +531,14 @@ async fn manual_refresh_prepares_revision_fenced_rotation_without_store_mutation
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.expect("current credential");
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let prepared = service
|
||||
.manual_refresh(current)
|
||||
.await
|
||||
.expect("manual refresh");
|
||||
drop(guard);
|
||||
|
||||
assert_eq!(prepared.credential.account_id(), &account_id);
|
||||
assert_eq!(prepared.credential.expected_revision().get(), 1);
|
||||
@@ -539,6 +577,12 @@ async fn manual_refresh_prepares_revision_fenced_rotation_without_store_mutation
|
||||
["rt-old-access"]
|
||||
);
|
||||
assert_eq!(leases.drops.load(Ordering::SeqCst), 0);
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "manual_refresh");
|
||||
assert_eq!(record["fields"]["account_id"], "acct_manual_refresh");
|
||||
assert_eq!(record["fields"]["access_token"], "refreshed-access");
|
||||
assert_eq!(record["fields"]["refresh_token"], "rotated-refresh");
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
drop(prepared);
|
||||
assert_eq!(leases.drops.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
@@ -569,54 +613,19 @@ async fn manual_refresh_missing_lease_fails_before_exchange() {
|
||||
);
|
||||
}
|
||||
|
||||
struct ImportVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for ImportVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
let suffix = secret
|
||||
.access_token
|
||||
.expose_secret()
|
||||
.strip_prefix("token-")
|
||||
.ok_or(CodexIdentityVerificationError::Rejected)?;
|
||||
let mut profile = profile(&format!("chatgpt-{suffix}"));
|
||||
if let Some(user) = suffix.strip_prefix("shared-") {
|
||||
profile.chatgpt_account_id = "chatgpt-shared".to_owned();
|
||||
profile.chatgpt_user_id = format!("user-{user}");
|
||||
}
|
||||
if expectation
|
||||
.chatgpt_account_id()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_account_id)
|
||||
|| expectation
|
||||
.chatgpt_user_id()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_user_id)
|
||||
{
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(CodexIdentityVerification::Complete(profile))
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_secret: &provider_openai::credential::CodexOAuthSecret,
|
||||
_id_token: &SecretString,
|
||||
_expected_nonce: &SecretString,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn import_service(
|
||||
refresher: Arc<ManualRefresher>,
|
||||
) -> provider_openai::credential::CodexCredentialAdminService {
|
||||
import_service_with_source(refresher, account_source([]))
|
||||
}
|
||||
|
||||
fn import_service_with_source(
|
||||
refresher: Arc<ManualRefresher>,
|
||||
source: Arc<ManualAccountSource>,
|
||||
) -> provider_openai::credential::CodexCredentialAdminService {
|
||||
CodexCredentialAdminService::new(
|
||||
refresher,
|
||||
Arc::new(ImportVerifier),
|
||||
source,
|
||||
Arc::new(ManualLeases {
|
||||
requests: Mutex::new(Vec::new()),
|
||||
drops: Arc::new(AtomicUsize::new(0)),
|
||||
@@ -634,21 +643,25 @@ pub(super) fn unused_import_refresher() -> Arc<ManualRefresher> {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_import_uses_only_tokens_and_verified_identity() {
|
||||
async fn oauth_import_accepts_opaque_access_token_and_completes_profile_from_account_source() {
|
||||
let refresher = unused_import_refresher();
|
||||
let prepared = import_service(refresher.clone())
|
||||
let access_token = "opaque-cpr-access";
|
||||
let source = account_source([authenticated_account(
|
||||
"chatgpt-cpr",
|
||||
"user-cpr",
|
||||
Some("remote-cpr@example.com"),
|
||||
Some("team"),
|
||||
)]);
|
||||
let prepared = import_service_with_source(refresher.clone(), Arc::clone(&source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_cpr_import",
|
||||
"email": "cpr@example.com",
|
||||
"accountId": "chatgpt-cpr",
|
||||
"userId": "user-chatgpt-cpr",
|
||||
"email": "untrusted-cpr@example.com",
|
||||
"label": "CPR account",
|
||||
"planType": "pro",
|
||||
"token": "Bearer token-cpr",
|
||||
"planType": "untrusted-plan",
|
||||
"token": format!("Bearer {access_token}"),
|
||||
"refreshToken": "refresh-cpr",
|
||||
"accessTokenExpiresAt": "2100-01-01T00:00:00+00:00",
|
||||
"status": "disabled",
|
||||
"addedAt": "2026-07-18T10:47:01+08:00",
|
||||
"updatedAt": "2026-07-19T11:00:00+08:00",
|
||||
@@ -662,6 +675,11 @@ async fn oauth_import_uses_only_tokens_and_verified_identity() {
|
||||
assert!(account.account.id().as_str().starts_with("acct_"));
|
||||
assert!(account.account.enabled());
|
||||
assert_eq!(account.account.upstream_account_id(), Some("chatgpt-cpr"));
|
||||
assert_eq!(account.account.upstream_user_id(), "user-cpr");
|
||||
assert_eq!(account.account.name(), "CPR account");
|
||||
assert_eq!(account.account.email(), Some("remote-cpr@example.com"));
|
||||
assert_eq!(account.account.plan_type(), Some("team"));
|
||||
assert_eq!(account.account.access_token_expires_at(), None);
|
||||
let runtime = CodexCredentialCodec::decode(&account.credential).expect("credential");
|
||||
assert_eq!(
|
||||
runtime
|
||||
@@ -670,26 +688,181 @@ async fn oauth_import_uses_only_tokens_and_verified_identity() {
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"token-cpr"
|
||||
access_token
|
||||
);
|
||||
assert!(refresher.seen.lock().expect("seen tokens").is_empty());
|
||||
assert_eq!(
|
||||
source
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen account-source access tokens")
|
||||
.as_slice(),
|
||||
[access_token]
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.expectations
|
||||
.lock()
|
||||
.expect("seen account-source expectations")
|
||||
.as_slice(),
|
||||
[CodexIdentityExpectation::default()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_import_logs_raw_tokens_before_account_profile_failure() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let error = import_service_with_source(unused_import_refresher(), account_source([]))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"token": "logged-direct-access",
|
||||
"refreshToken": "logged-direct-refresh"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect_err("profile source fails after direct import logging");
|
||||
drop(guard);
|
||||
|
||||
assert_eq!(error, CodexCredentialAdminError::AccountProfileUnavailable);
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "import_direct");
|
||||
assert!(
|
||||
record["fields"]["account_id"]
|
||||
.as_str()
|
||||
.is_some_and(|account_id| account_id.starts_with("acct_"))
|
||||
);
|
||||
assert_eq!(record["fields"]["access_token"], "logged-direct-access");
|
||||
assert_eq!(record["fields"]["refresh_token"], "logged-direct-refresh");
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_token_only_import_persists_exchanged_tokens_and_completes_profile() {
|
||||
let refreshed_access = "opaque-refreshed-import";
|
||||
let refresher = Arc::new(ManualRefresher {
|
||||
outcome: Mutex::new(Some(Ok(TokenPair {
|
||||
access_token: refreshed_access.to_owned(),
|
||||
refresh_token: Some("rotated-import-refresh".to_owned()),
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
}))),
|
||||
seen: Mutex::new(Vec::new()),
|
||||
});
|
||||
let source = account_source([authenticated_account(
|
||||
"chatgpt-refreshed-import",
|
||||
"user-refreshed-import",
|
||||
Some("refreshed@example.com"),
|
||||
Some("plus"),
|
||||
)]);
|
||||
|
||||
let prepared = import_service_with_source(Arc::clone(&refresher), Arc::clone(&source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"refreshToken": "refresh-only"
|
||||
}))
|
||||
.await
|
||||
.expect("refresh-token-only import");
|
||||
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
let account = &prepared.accounts()[0];
|
||||
assert_eq!(
|
||||
account.account.upstream_account_id(),
|
||||
Some("chatgpt-refreshed-import")
|
||||
);
|
||||
assert_eq!(account.account.upstream_user_id(), "user-refreshed-import");
|
||||
assert_eq!(account.account.email(), Some("refreshed@example.com"));
|
||||
assert_eq!(account.account.plan_type(), Some("plus"));
|
||||
let runtime = CodexCredentialCodec::decode(&account.credential).expect("credential");
|
||||
let oauth = runtime.authentication.oauth().expect("OAuth credential");
|
||||
assert_eq!(oauth.access_token.expose_secret(), refreshed_access);
|
||||
assert_eq!(
|
||||
oauth
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.expect("rotated RT")
|
||||
.expose_secret(),
|
||||
"rotated-import-refresh"
|
||||
);
|
||||
assert_eq!(
|
||||
refresher
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen refresh tokens")
|
||||
.as_slice(),
|
||||
["refresh-only"]
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen account-source access tokens")
|
||||
.as_slice(),
|
||||
[refreshed_access]
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.expectations
|
||||
.lock()
|
||||
.expect("seen account-source expectations")
|
||||
.as_slice(),
|
||||
[CodexIdentityExpectation::default()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_token_only_import_logs_fallback_refresh_token_before_account_profile_failure() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let refresher = Arc::new(ManualRefresher {
|
||||
outcome: Mutex::new(Some(Ok(TokenPair {
|
||||
access_token: "logged-import-access".to_owned(),
|
||||
refresh_token: None,
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
}))),
|
||||
seen: Mutex::new(Vec::new()),
|
||||
});
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let error = import_service_with_source(refresher, account_source([]))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"refreshToken": "logged-import-original-refresh"
|
||||
}))
|
||||
.await
|
||||
.expect_err("profile source fails after refresh-token import logging");
|
||||
drop(guard);
|
||||
|
||||
assert_eq!(error, CodexCredentialAdminError::AccountProfileUnavailable);
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "import_refresh_token");
|
||||
assert_eq!(record["fields"]["access_token"], "logged-import-access");
|
||||
assert_eq!(
|
||||
record["fields"]["refresh_token"],
|
||||
"logged-import-original-refresh"
|
||||
);
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cpr_batch_allows_distinct_users_in_the_same_workspace() {
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
let source = account_source([
|
||||
authenticated_account("chatgpt-shared", "user-a", None, None),
|
||||
authenticated_account("chatgpt-shared", "user-b", None, None),
|
||||
]);
|
||||
let prepared = import_service_with_source(unused_import_refresher(), Arc::clone(&source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_shared_a",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
"token": "opaque-shared-a"
|
||||
}, {
|
||||
"id": "acct_shared_b",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-b",
|
||||
"token": "token-shared-b"
|
||||
"token": "opaque-shared-b"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
@@ -703,19 +876,23 @@ async fn cpr_batch_allows_distinct_users_in_the_same_workspace() {
|
||||
.all(|account| { account.account.upstream_account_id() == Some("chatgpt-shared") })
|
||||
);
|
||||
|
||||
let duplicate = import_service(unused_import_refresher())
|
||||
let duplicate_source = account_source([
|
||||
authenticated_account("chatgpt-shared", "user-a", None, None),
|
||||
authenticated_account("chatgpt-shared", "user-a", None, None),
|
||||
]);
|
||||
let duplicate = import_service_with_source(unused_import_refresher(), duplicate_source)
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"sourceFormat": "cpr",
|
||||
"accounts": [{
|
||||
"id": "acct_duplicate_a",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
"token": "opaque-duplicate-a"
|
||||
}, {
|
||||
"id": "acct_duplicate_b",
|
||||
"accountId": "chatgpt-shared",
|
||||
"userId": "user-a",
|
||||
"token": "token-shared-a"
|
||||
"token": "opaque-duplicate-b"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
@@ -724,8 +901,14 @@ async fn cpr_batch_allows_distinct_users_in_the_same_workspace() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_import_extracts_tokens_without_parsing_source_projections() {
|
||||
let bundle = import_service(unused_import_refresher())
|
||||
async fn oauth_import_completes_profiles_from_account_source() {
|
||||
let bundle_source = account_source([authenticated_account(
|
||||
"chatgpt-bundle",
|
||||
"user-bundle",
|
||||
Some("remote-bundle@example.com"),
|
||||
Some("team"),
|
||||
)]);
|
||||
let bundle = import_service_with_source(unused_import_refresher(), Arc::clone(&bundle_source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"exported_at": "2026-07-03T15:46:38.717Z",
|
||||
"proxies": [],
|
||||
@@ -734,10 +917,12 @@ async fn oauth_import_extracts_tokens_without_parsing_source_projections() {
|
||||
"platform": "openai",
|
||||
"type": "oauth",
|
||||
"credentials": {
|
||||
"access_token": "token-bundle",
|
||||
"access_token": "opaque-bundle",
|
||||
"refresh_token": "refresh-bundle",
|
||||
"chatgpt_account_id": "chatgpt-bundle",
|
||||
"chatgpt_user_id": "user-chatgpt-bundle"
|
||||
"chatgpt_account_id": "source-bundle",
|
||||
"chatgpt_user_id": "source-user-bundle",
|
||||
"email": "bundle-source@example.com",
|
||||
"plan_type": "team"
|
||||
},
|
||||
"concurrency": 3,
|
||||
"priority": 50
|
||||
@@ -745,48 +930,85 @@ async fn oauth_import_extracts_tokens_without_parsing_source_projections() {
|
||||
}))
|
||||
.await
|
||||
.expect("credential bundle import");
|
||||
let auth_document = import_service(unused_import_refresher())
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"accounts": [{
|
||||
"type": "openai",
|
||||
"access_token": "token-auth-document",
|
||||
"refresh_token": "refresh-auth-document",
|
||||
"chatgpt_account_id": "chatgpt-auth-document",
|
||||
"chatgpt_user_id": "user-chatgpt-auth-document",
|
||||
"email": "auth-document@example.com",
|
||||
"label": "Auth document"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect("auth document import");
|
||||
let auth_document_source = account_source([authenticated_account(
|
||||
"chatgpt-auth-document",
|
||||
"user-auth-document",
|
||||
Some("remote-auth-document@example.com"),
|
||||
Some("plus"),
|
||||
)]);
|
||||
let auth_document =
|
||||
import_service_with_source(unused_import_refresher(), Arc::clone(&auth_document_source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"accounts": [{
|
||||
"type": "openai",
|
||||
"access_token": "opaque-auth-document",
|
||||
"refresh_token": "refresh-auth-document",
|
||||
"chatgpt_account_id": "source-auth-document",
|
||||
"chatgpt_user_id": "source-user-auth-document",
|
||||
"email": "auth-document@example.com",
|
||||
"label": "Auth document",
|
||||
"planType": "plus"
|
||||
}]
|
||||
}))
|
||||
.await
|
||||
.expect("auth document import");
|
||||
|
||||
for (prepared, expected) in [
|
||||
(&bundle, "chatgpt-bundle"),
|
||||
(&auth_document, "chatgpt-auth-document"),
|
||||
] {
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
assert_eq!(
|
||||
prepared.accounts()[0].account.upstream_account_id(),
|
||||
Some(expected)
|
||||
);
|
||||
assert!(
|
||||
prepared.accounts()[0]
|
||||
.account
|
||||
.id()
|
||||
.as_str()
|
||||
.starts_with("acct_")
|
||||
);
|
||||
}
|
||||
let bundle_account = &bundle.accounts()[0].account;
|
||||
assert_eq!(bundle.accounts().len(), 1);
|
||||
assert_eq!(bundle_account.upstream_account_id(), Some("chatgpt-bundle"));
|
||||
assert_eq!(bundle_account.upstream_user_id(), "user-bundle");
|
||||
assert_eq!(bundle_account.name(), "bundle@example.com");
|
||||
assert_eq!(bundle_account.email(), Some("remote-bundle@example.com"));
|
||||
assert_eq!(bundle_account.plan_type(), Some("team"));
|
||||
|
||||
let auth_document_account = &auth_document.accounts()[0].account;
|
||||
assert_eq!(auth_document.accounts().len(), 1);
|
||||
assert_eq!(
|
||||
auth_document_account.upstream_account_id(),
|
||||
Some("chatgpt-auth-document")
|
||||
);
|
||||
assert_eq!(
|
||||
auth_document_account.upstream_user_id(),
|
||||
"user-auth-document"
|
||||
);
|
||||
assert_eq!(auth_document_account.name(), "Auth document");
|
||||
assert_eq!(
|
||||
auth_document_account.email(),
|
||||
Some("remote-auth-document@example.com")
|
||||
);
|
||||
assert_eq!(auth_document_account.plan_type(), Some("plus"));
|
||||
assert_eq!(
|
||||
bundle_source
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen account-source access tokens")
|
||||
.as_slice(),
|
||||
["opaque-bundle"]
|
||||
);
|
||||
assert_eq!(
|
||||
auth_document_source
|
||||
.seen
|
||||
.lock()
|
||||
.expect("seen account-source access tokens")
|
||||
.as_slice(),
|
||||
["opaque-auth-document"]
|
||||
);
|
||||
assert!(!format!("{bundle:?} {auth_document:?}").contains("refresh-bundle"));
|
||||
assert!(!format!("{bundle:?} {auth_document:?}").contains("refresh-auth-document"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cliproxyapi_codex_auth_file_is_recognized_as_an_openai_auth_document() {
|
||||
let prepared = import_service(unused_import_refresher())
|
||||
let source = account_source([authenticated_account(
|
||||
"chatgpt-cpa",
|
||||
"user-cpa",
|
||||
Some("remote-cpa@example.com"),
|
||||
Some("pro"),
|
||||
)]);
|
||||
let prepared = import_service_with_source(unused_import_refresher(), Arc::clone(&source))
|
||||
.prepare_import_document(serde_json::json!({
|
||||
"type": "codex",
|
||||
"access_token": "token-cpa",
|
||||
"access_token": "opaque-cpa",
|
||||
"refresh_token": "refresh-cpa",
|
||||
"id_token": "id-cpa",
|
||||
"account_id": "chatgpt-cpa",
|
||||
@@ -799,6 +1021,8 @@ async fn cliproxyapi_codex_auth_file_is_recognized_as_an_openai_auth_document()
|
||||
assert_eq!(prepared.accounts().len(), 1);
|
||||
let account = &prepared.accounts()[0].account;
|
||||
assert_eq!(account.upstream_account_id(), Some("chatgpt-cpa"));
|
||||
assert_eq!(account.email(), Some("chatgpt-cpa@example.com"));
|
||||
assert_eq!(account.upstream_user_id(), "user-cpa");
|
||||
assert_eq!(account.email(), Some("remote-cpa@example.com"));
|
||||
assert_eq!(account.plan_type(), Some("pro"));
|
||||
assert_eq!(account.authentication_kind(), "oauth");
|
||||
}
|
||||
|
||||
@@ -4,15 +4,21 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use futures::executor::block_on;
|
||||
use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
|
||||
AccountStateChange, ProviderAccountId, ProviderAccountStore as _, RotationStrategy,
|
||||
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
|
||||
QuotaObservation, QuotaWriteOutcome, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::{ProviderSessionAffinityKey, ProviderSessionAffinityPort};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldownPort, ProviderSessionAffinityKey, ProviderSessionAffinityPort,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use provider_openai::OFFICIAL_CODEX_BASE_URL;
|
||||
use provider_openai::credential::{
|
||||
@@ -22,11 +28,12 @@ use provider_openai::credential::{
|
||||
};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::json;
|
||||
use url::Url;
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service, catalog_cache, profile, secret,
|
||||
};
|
||||
|
||||
fn create_account(store: &Arc<MemoryAccountStore>, id: &str, token: &str) {
|
||||
@@ -99,6 +106,7 @@ fn selector_with_affinity(
|
||||
session_affinity,
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,6 +116,7 @@ fn selector_with_runtime(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
skip_exhausted: bool,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
) -> CodexCredentialSelector {
|
||||
let profile = CodexWireProfileState::new(CodexWireProfile {
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -136,6 +145,7 @@ fn selector_with_runtime(
|
||||
http,
|
||||
OFFICIAL_CODEX_BASE_URL.to_owned(),
|
||||
Arc::clone(&agent_identity),
|
||||
cooldowns,
|
||||
));
|
||||
CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
@@ -464,6 +474,7 @@ async fn selector_should_keep_a_schedulable_affinity_account_despite_soft_health
|
||||
affinity,
|
||||
Arc::clone(&account_feedback),
|
||||
true,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
for _ in 0..4 {
|
||||
account_feedback.report(
|
||||
@@ -507,6 +518,7 @@ async fn selector_should_escape_a_quota_exhausted_affinity_account() {
|
||||
let first = store.account("acct_first").expect("first account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: first.id().clone(),
|
||||
expected_revision: first.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -663,8 +675,12 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
}),
|
||||
)
|
||||
.expect("select account");
|
||||
block_on(selector.record_failure(lease.account(), CodexAccountFailure::CredentialExpired))
|
||||
.expect("record credential expiry");
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CredentialExpired,
|
||||
None,
|
||||
))
|
||||
.expect("record credential expiry");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
@@ -672,13 +688,58 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
.availability(),
|
||||
AccountAvailability::Expired
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message("acct_primary").as_deref(),
|
||||
Some("upstream_credential_expired")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_expired_failure_keeps_expired_oauth_for_bounded_refresh_recovery() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let mut expired_profile = profile("chatgpt-acct_primary");
|
||||
expired_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() - chrono::Duration::minutes(1));
|
||||
block_on(store.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_primary".to_owned(),
|
||||
name: "acct_primary".to_owned(),
|
||||
secret: secret("at-primary"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(10)),
|
||||
enabled: true,
|
||||
}));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::CredentialExpired,
|
||||
Some("token_expired".to_owned()),
|
||||
))
|
||||
.expect("record credential expiry");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("account retained for refresh")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
let attempt = attempt(BTreeSet::new());
|
||||
let lease =
|
||||
block_on(
|
||||
@@ -697,36 +758,22 @@ fn rate_limited_failure_records_quota_window_without_changing_availability() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
// 429 不改变 availability(限流不改变账号可用性),只把限流窗口合成进 quota JSON。
|
||||
// 429 不改变 availability(限流不改变账号可用性),也不污染 quota JSON——
|
||||
// 临时限流写入 Redis 冷却(到期自动解除),额度重置时间保持不变。
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
let quota = store.quota_json("acct_primary").expect("quota");
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/limit_reached")
|
||||
.and_then(serde_json::Value::as_bool),
|
||||
Some(true)
|
||||
);
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(serde_json::Value::as_f64),
|
||||
Some(100.0)
|
||||
);
|
||||
assert!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/reset_at")
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
.is_some_and(|reset_at| {
|
||||
reset_at
|
||||
> SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock")
|
||||
.as_secs() as i64
|
||||
})
|
||||
store.quota_json("acct_primary").is_none(),
|
||||
"429 must not synthesize quota window"
|
||||
);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown expiring in the future"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -743,6 +790,7 @@ async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("record usage-limit exhaustion");
|
||||
@@ -758,6 +806,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -771,6 +820,7 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
@@ -789,13 +839,22 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
true,
|
||||
Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>,
|
||||
);
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
@@ -806,8 +865,13 @@ fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
);
|
||||
// 429 合成限流窗口需要读写 quota JSON。
|
||||
assert!(store.quota_reads() >= 1);
|
||||
// 429 临时限流写入 Redis 冷却,不读写 quota JSON。
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
let cooldown = block_on(cooldowns.read(account.id())).expect("read cooldown");
|
||||
assert!(
|
||||
cooldown.is_some_and(|cooling| cooling.until() > SystemTime::now()),
|
||||
"429 must record a Redis cooldown"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -816,6 +880,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Invalid,
|
||||
@@ -830,6 +895,7 @@ fn rate_limited_failure_does_not_overwrite_stale_authentication_state() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
))
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
@@ -854,6 +920,7 @@ fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: stale,
|
||||
@@ -892,12 +959,14 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
selector.record_failure(
|
||||
&second,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
None,
|
||||
),
|
||||
);
|
||||
first_result.expect("record first account failure");
|
||||
@@ -925,7 +994,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(strict_selector.record_failure(&account, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
@@ -934,6 +1003,7 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let request_url =
|
||||
Url::parse("https://chatgpt.com/backend-api/codex/responses").expect("request URL");
|
||||
@@ -957,6 +1027,131 @@ fn selector_keeps_a_quota_exhausted_account_eligible_when_configured_not_to_skip
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_surfaces_the_original_accounts_quota_status_to_the_coordinator() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_original", "at-original");
|
||||
create_account(&store, "acct_fallback", "at-fallback");
|
||||
let strict_selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let original = store.account("acct_original").expect("original account");
|
||||
block_on(strict_selector.record_failure(&original, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("mark original account exhausted");
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.id().clone(),
|
||||
);
|
||||
let attempt = AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_native_continuation").expect("request id"),
|
||||
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
|
||||
),
|
||||
NonZeroU32::new(1).expect("attempt"),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, None),
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
let error =
|
||||
block_on(
|
||||
selector.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
|
||||
.expect("request URL"),
|
||||
attempt: &attempt,
|
||||
session_affinity_key: None,
|
||||
}),
|
||||
)
|
||||
.expect_err("the selector must surface the unavailable native account");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CredentialSelectionError::NoEligibleCredential
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_surfaces_the_original_accounts_quota_signal_to_the_coordinator() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_original_signal", "at-original-signal");
|
||||
create_account(&store, "acct_fallback_signal", "at-fallback-signal");
|
||||
let original = store
|
||||
.account("acct_original_signal")
|
||||
.expect("original account");
|
||||
let quota = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 98}
|
||||
}
|
||||
});
|
||||
let outcome = block_on(store.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: original.id().clone(),
|
||||
expected_revision: original.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
quota.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
}))
|
||||
.expect("persist quota signal");
|
||||
assert!(matches!(outcome, QuotaWriteOutcome::Updated));
|
||||
|
||||
let selector = selector_with_runtime(
|
||||
&store,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(AccountFeedbackStats::default()),
|
||||
false,
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.id().clone(),
|
||||
);
|
||||
let attempt = AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new("req_native_continuation_signal").expect("request id"),
|
||||
ClientApiKeyId::new("key_codex_contract").expect("client key id"),
|
||||
),
|
||||
NonZeroU32::new(1).expect("attempt"),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, None),
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
);
|
||||
let error =
|
||||
block_on(
|
||||
selector.select(&SelectCodexCredential {
|
||||
upstream_model: "gpt-5.4",
|
||||
request_url: &Url::parse("https://chatgpt.com/backend-api/codex/responses")
|
||||
.expect("request URL"),
|
||||
attempt: &attempt,
|
||||
session_affinity_key: None,
|
||||
}),
|
||||
)
|
||||
.expect_err("the selector must surface the quota-limited native account");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CredentialSelectionError::NoEligibleCredential
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_verification_failure_isolates_only_selected_account() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -979,6 +1174,7 @@ fn identity_verification_failure_isolates_only_selected_account() {
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::IdentityVerificationRequired,
|
||||
None,
|
||||
))
|
||||
.expect("record identity verification failure");
|
||||
|
||||
@@ -1022,6 +1218,7 @@ fn cloudflare_challenge_does_not_change_availability() {
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record challenge");
|
||||
|
||||
@@ -1058,9 +1255,11 @@ fn repeated_cloudflare_path_block_marks_only_the_affected_account_invalid() {
|
||||
.expect("select account");
|
||||
|
||||
for _ in 0..3 {
|
||||
block_on(
|
||||
selector.record_failure(lease.account(), CodexAccountFailure::CloudflarePathBlocked),
|
||||
)
|
||||
block_on(selector.record_failure(
|
||||
lease.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
}
|
||||
|
||||
@@ -1114,6 +1313,7 @@ fn cloudflare_challenge_expires_provider_owned_cookies_at_cooldown_boundary() {
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after: None },
|
||||
None,
|
||||
))
|
||||
.expect("record challenge");
|
||||
|
||||
@@ -1157,8 +1357,12 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
|
||||
session_affinity_key: None,
|
||||
}))
|
||||
.expect("select revised account");
|
||||
block_on(selector.record_failure(second.account(), CodexAccountFailure::CloudflarePathBlocked))
|
||||
.expect("record path block");
|
||||
block_on(selector.record_failure(
|
||||
second.account(),
|
||||
CodexAccountFailure::CloudflarePathBlocked,
|
||||
None,
|
||||
))
|
||||
.expect("record path block");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let data = block_on(store.repository().load_complete_data(&account)).expect("credential data");
|
||||
@@ -1194,7 +1398,7 @@ fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes()
|
||||
|
||||
assert_eq!(outcome.credential_revision, Some(current.revision().get()));
|
||||
assert_ne!(current.revision(), lease.account().revision());
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted))
|
||||
block_on(selector.record_failure(¤t, CodexAccountFailure::QuotaExhausted, None))
|
||||
.expect("record failure with current revision");
|
||||
assert_eq!(
|
||||
store
|
||||
|
||||
@@ -1,3 +1,72 @@
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde_json::Value;
|
||||
use tokio::sync::{Mutex as AsyncMutex, MutexGuard};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct OAuthRecoveryLogCapture(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
static OAUTH_RECOVERY_LOG_CAPTURE_LOCK: AsyncMutex<()> = AsyncMutex::const_new(());
|
||||
|
||||
pub(super) struct OAuthRecoveryLogWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for OAuthRecoveryLogWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("OAuth recovery log buffer lock")
|
||||
.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for OAuthRecoveryLogCapture {
|
||||
type Writer = OAuthRecoveryLogWriter;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
OAuthRecoveryLogWriter(Arc::clone(&self.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl OAuthRecoveryLogCapture {
|
||||
pub(super) fn dispatch(&self) -> tracing::Dispatch {
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.with_target(true)
|
||||
.without_time()
|
||||
.with_writer(self.clone())
|
||||
.finish();
|
||||
tracing::Dispatch::new(subscriber)
|
||||
}
|
||||
|
||||
pub(super) fn recovery_record(&self) -> Value {
|
||||
let bytes = self
|
||||
.0
|
||||
.lock()
|
||||
.expect("OAuth recovery log buffer lock")
|
||||
.clone();
|
||||
std::str::from_utf8(&bytes)
|
||||
.expect("JSON log output is UTF-8")
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).expect("JSON log record"))
|
||||
.find(|record: &Value| {
|
||||
record["target"] == "openai_oauth_recovery"
|
||||
&& record["fields"]["event"] == "openai_oauth_recovery"
|
||||
})
|
||||
.expect("OAuth recovery log record")
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn lock_oauth_recovery_log_capture() -> MutexGuard<'static, ()> {
|
||||
OAUTH_RECOVERY_LOG_CAPTURE_LOCK.lock().await
|
||||
}
|
||||
|
||||
mod admin;
|
||||
mod agent_identity;
|
||||
mod catalog;
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
@@ -13,8 +19,8 @@ use provider_openai::credential::token_client::{
|
||||
AuthorizationTokenSet,
|
||||
};
|
||||
use provider_openai::credential::{
|
||||
CodexAccountIdentityVerifier, CodexCredentialAdmin, CodexIdentityExpectation,
|
||||
CodexIdentityVerification, CodexIdentityVerificationError, CodexOAuthAdmin,
|
||||
CodexAuthenticatedAccount, CodexAuthenticatedAccountSource, CodexCredentialAdmin,
|
||||
CodexIdentityExpectation, CodexIdentityVerificationError, CodexOAuthAdmin,
|
||||
CodexOAuthAdminError, CodexOAuthAdminService, CodexOAuthPendingClaimOutcome,
|
||||
CodexOAuthPendingStore, CodexOAuthPendingStoreError, CodexOAuthSecret,
|
||||
CodexPendingAuthorization, CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential,
|
||||
@@ -24,6 +30,7 @@ use secrecy::{ExposeSecret, SecretString};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use url::Url;
|
||||
|
||||
use super::{OAuthRecoveryLogCapture, lock_oauth_recovery_log_capture};
|
||||
use crate::support::{MemoryAccountStore, profile, runtime_policy, secret};
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -158,51 +165,100 @@ impl AuthorizationCodeExchanger for Exchanger {
|
||||
id_token: None,
|
||||
},
|
||||
id_token: SecretString::from("oauth-id-token"),
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Verifier;
|
||||
struct NoRefreshTokenExchanger;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for Verifier {
|
||||
async fn verify(
|
||||
impl AuthorizationCodeExchanger for NoRefreshTokenExchanger {
|
||||
async fn exchange_authorization_code(
|
||||
&self,
|
||||
_secret: &CodexOAuthSecret,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_secret: &CodexOAuthSecret,
|
||||
id_token: &SecretString,
|
||||
expected_nonce: &SecretString,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
if id_token.expose_secret() != "oauth-id-token" || expected_nonce.expose_secret().len() < 16
|
||||
{
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
grant: AuthorizationCodeGrant,
|
||||
) -> Result<AuthorizationTokenSet, AuthorizationCodeExchangeError> {
|
||||
if grant.code.expose_secret() != "authorization-code" {
|
||||
return Err(AuthorizationCodeExchangeError::Rejected);
|
||||
}
|
||||
Ok(CodexIdentityVerification::Complete(profile(
|
||||
"chatgpt-oauth",
|
||||
)))
|
||||
Ok(AuthorizationTokenSet {
|
||||
secret: CodexOAuthSecret {
|
||||
access_token: SecretString::from("oauth-access-without-refresh-token"),
|
||||
refresh_token: None,
|
||||
id_token: None,
|
||||
},
|
||||
id_token: SecretString::from("oauth-id-token"),
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StaticAccountSource {
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl StaticAccountSource {
|
||||
fn call_count(&self) -> usize {
|
||||
self.calls.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAuthenticatedAccountSource for StaticAccountSource {
|
||||
async fn fetch(
|
||||
&self,
|
||||
_secret: &CodexOAuthSecret,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexAuthenticatedAccount, CodexIdentityVerificationError> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
assert_eq!(expectation, &CodexIdentityExpectation::default());
|
||||
Ok(CodexAuthenticatedAccount {
|
||||
chatgpt_account_id: "chatgpt-oauth".to_owned(),
|
||||
chatgpt_user_id: "user-chatgpt-oauth".to_owned(),
|
||||
email: Some("oauth@example.com".to_owned()),
|
||||
plan_type: Some("plus".to_owned()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableAccountSource;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAuthenticatedAccountSource for UnavailableAccountSource {
|
||||
async fn fetch(
|
||||
&self,
|
||||
_secret: &CodexOAuthSecret,
|
||||
_expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexAuthenticatedAccount, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Unavailable)
|
||||
}
|
||||
}
|
||||
|
||||
fn service() -> CodexOAuthAdminService {
|
||||
service_with_store(Arc::new(MemoryAccountStore::default()), Arc::new(Verifier))
|
||||
service_with_store(Arc::new(MemoryAccountStore::default()))
|
||||
}
|
||||
|
||||
fn service_with_store(
|
||||
fn service_with_store(store: Arc<MemoryAccountStore>) -> CodexOAuthAdminService {
|
||||
service_with_account_source(store, Arc::new(StaticAccountSource::default()))
|
||||
}
|
||||
|
||||
fn service_with_account_source(
|
||||
store: Arc<MemoryAccountStore>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
) -> CodexOAuthAdminService {
|
||||
service_with_exchanger(store, account_source, Arc::new(Exchanger))
|
||||
}
|
||||
|
||||
fn service_with_exchanger(
|
||||
store: Arc<MemoryAccountStore>,
|
||||
account_source: Arc<dyn CodexAuthenticatedAccountSource>,
|
||||
exchanger: Arc<dyn AuthorizationCodeExchanger>,
|
||||
) -> CodexOAuthAdminService {
|
||||
CodexOAuthAdminService::new(
|
||||
Arc::new(PendingStore::default()),
|
||||
Arc::new(Exchanger),
|
||||
verifier,
|
||||
exchanger,
|
||||
account_source,
|
||||
store,
|
||||
runtime_policy(),
|
||||
CodexCredentialAdmin,
|
||||
@@ -283,8 +339,10 @@ async fn start_authorization_uses_pkce_nonce_and_fixed_official_redirect() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_returns_verified_core_account_without_writing_store() {
|
||||
let service = service();
|
||||
async fn completion_populates_create_profile_from_authenticated_account_source() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_source = Arc::new(StaticAccountSource::default());
|
||||
let service = service_with_account_source(store, account_source.clone());
|
||||
let started = started(&service).await;
|
||||
let url = Url::parse(&started.authorization_url).expect("authorization URL");
|
||||
let state = url
|
||||
@@ -307,25 +365,61 @@ async fn completion_returns_verified_core_account_without_writing_store() {
|
||||
assert_eq!(prepared.account.provider().as_str(), "openai");
|
||||
assert!(prepared.account.id().as_str().starts_with("acct_"));
|
||||
assert_eq!(
|
||||
prepared
|
||||
.credential
|
||||
.expose_to_provider()
|
||||
.get("access_token")
|
||||
.and_then(serde_json::Value::as_str),
|
||||
Some("oauth-access-token")
|
||||
prepared.account.upstream_account_id(),
|
||||
Some("chatgpt-oauth")
|
||||
);
|
||||
assert_eq!(prepared.account.upstream_user_id(), "user-chatgpt-oauth");
|
||||
assert_eq!(prepared.account.email(), Some("oauth@example.com"));
|
||||
assert_eq!(prepared.account.plan_type(), Some("plus"));
|
||||
assert_eq!(
|
||||
prepared
|
||||
.credential
|
||||
.expose_to_provider()
|
||||
.get("id_token")
|
||||
.and_then(serde_json::Value::as_str),
|
||||
Some("oauth-id-token")
|
||||
account_source.call_count(),
|
||||
1,
|
||||
"first OAuth authorization completes the account profile through the access-token source"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_accepts_unrelated_oauth_callback_parameters() {
|
||||
async fn authorization_code_profile_failure_is_upstream_unavailable() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let service = service_with_account_source(
|
||||
Arc::new(MemoryAccountStore::default()),
|
||||
Arc::new(UnavailableAccountSource),
|
||||
);
|
||||
let started = started(&service).await;
|
||||
let state = Url::parse(&started.authorization_url)
|
||||
.expect("authorization URL")
|
||||
.query_pairs()
|
||||
.find_map(|(key, value)| (key == "state").then(|| value.into_owned()))
|
||||
.expect("state");
|
||||
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let error = service
|
||||
.complete_authorization(CompleteCodexOAuthAuthorization {
|
||||
owner_ref: owner_ref(),
|
||||
flow_id: started.flow_id,
|
||||
callback_url: SecretString::from(format!(
|
||||
"http://localhost:1455/auth/callback?code=authorization-code&state={state}"
|
||||
)),
|
||||
})
|
||||
.await
|
||||
.expect_err("profile source fails after successful authorization-code exchange");
|
||||
drop(guard);
|
||||
|
||||
assert_eq!(error, CodexOAuthAdminError::UpstreamUnavailable);
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "authorization_code");
|
||||
assert_eq!(record["fields"]["account_id"], "");
|
||||
assert_eq!(record["fields"]["access_token"], "oauth-access-token");
|
||||
assert_eq!(record["fields"]["refresh_token"], "oauth-refresh-token");
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
assert!(record["fields"].get("flow_id").is_none());
|
||||
assert!(record["fields"].get("id_token").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_accepts_parseable_non_localhost_callback_url() {
|
||||
let service = service();
|
||||
let started = started(&service).await;
|
||||
let state = Url::parse(&started.authorization_url)
|
||||
@@ -339,11 +433,11 @@ async fn completion_accepts_unrelated_oauth_callback_parameters() {
|
||||
owner_ref: owner_ref(),
|
||||
flow_id: started.flow_id,
|
||||
callback_url: SecretString::from(format!(
|
||||
"http://localhost:1455/auth/callback?code=authorization-code&state={state}&scope=openid%20profile&iss=https%3A%2F%2Fauth.openai.com"
|
||||
"https://relay-next.aivify.cc/relay/openai/callback?code=authorization-code&state={state}&scope=openid%20profile&iss=https%3A%2F%2Fauth.openai.com"
|
||||
)),
|
||||
})
|
||||
.await
|
||||
.expect("complete OAuth with standard callback parameters");
|
||||
.expect("complete OAuth from a relayed callback URL");
|
||||
|
||||
assert!(matches!(
|
||||
prepared.credential,
|
||||
@@ -399,54 +493,6 @@ async fn completion_rejects_repeated_state_callback_parameter() {
|
||||
assert_eq!(error, CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
|
||||
struct UnavailableVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for UnavailableVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Unavailable)
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &SecretString,
|
||||
_: &SecretString,
|
||||
_: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Unavailable)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_classifies_unavailable_identity_verification_as_upstream_unavailable() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let service = service_with_store(store, Arc::new(UnavailableVerifier));
|
||||
let started = started(&service).await;
|
||||
let state = Url::parse(&started.authorization_url)
|
||||
.expect("authorization URL")
|
||||
.query_pairs()
|
||||
.find_map(|(key, value)| (key == "state").then(|| value.into_owned()))
|
||||
.expect("state");
|
||||
|
||||
let error = service
|
||||
.complete_authorization(CompleteCodexOAuthAuthorization {
|
||||
owner_ref: owner_ref(),
|
||||
flow_id: started.flow_id,
|
||||
callback_url: SecretString::from(format!(
|
||||
"http://localhost:1455/auth/callback?code=authorization-code&state={state}"
|
||||
)),
|
||||
})
|
||||
.await
|
||||
.expect_err("identity verification unavailable");
|
||||
|
||||
assert_eq!(error, CodexOAuthAdminError::UpstreamUnavailable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_state_mismatch_releases_flow_for_retry() {
|
||||
let service = service();
|
||||
@@ -523,7 +569,7 @@ fn oauth_command_debug_redacts_owner_flow_and_callback() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reauthorization_uses_revision_advanced_after_start_for_prepared_rotation() {
|
||||
async fn reauthorization_preserves_existing_principal_without_identity_verification() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
@@ -538,13 +584,15 @@ async fn reauthorization_uses_revision_advanced_after_start_for_prepared_rotatio
|
||||
let original_account = store
|
||||
.account("acct_oauth_reauth")
|
||||
.expect("original account");
|
||||
let original_installation_id = store
|
||||
let original_runtime = store
|
||||
.repository()
|
||||
.load_runtime_credential(&original_account)
|
||||
.await
|
||||
.expect("original credential")
|
||||
.installation_id;
|
||||
let service = service_with_store(store.clone(), Arc::new(Verifier));
|
||||
.expect("original credential");
|
||||
let original_installation_id = original_runtime.installation_id;
|
||||
let original_principal = original_runtime.principal;
|
||||
let account_source = Arc::new(StaticAccountSource::default());
|
||||
let service = service_with_account_source(store.clone(), account_source.clone());
|
||||
let started = service
|
||||
.start_authorization(StartCodexOAuthAuthorization {
|
||||
mutation: reauthorization_mutation("request-reauth", "acct_oauth_reauth"),
|
||||
@@ -580,16 +628,10 @@ async fn reauthorization_uses_revision_advanced_after_start_for_prepared_rotatio
|
||||
let runtime =
|
||||
provider_openai::credential::CodexCredentialCodec::decode(prepared.credential.credential())
|
||||
.expect("prepared credential");
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"oauth-access-token"
|
||||
);
|
||||
assert_eq!(runtime.installation_id, original_installation_id);
|
||||
assert_eq!(runtime.principal, original_principal);
|
||||
assert!(prepared.replacement_identity.is_none());
|
||||
assert_eq!(account_source.call_count(), 0);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_oauth_reauth")
|
||||
@@ -600,75 +642,31 @@ async fn reauthorization_uses_revision_advanced_after_start_for_prepared_rotatio
|
||||
);
|
||||
}
|
||||
|
||||
struct ReplacementIdentityVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for ReplacementIdentityVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &SecretString,
|
||||
_: &SecretString,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
assert_eq!(expectation, &CodexIdentityExpectation::default());
|
||||
Ok(CodexIdentityVerification::Complete(profile(
|
||||
"chatgpt-different-owner",
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
struct RotatedPrincipalVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for RotatedPrincipalVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
Err(CodexIdentityVerificationError::Rejected)
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
_: &SecretString,
|
||||
_: &SecretString,
|
||||
_: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
let mut verified = profile("chatgpt-oauth");
|
||||
verified.oauth_subject = "rotated-oauth-subject".to_owned();
|
||||
verified.poid = None;
|
||||
Ok(CodexIdentityVerification::Complete(verified))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reauthorization_updates_changed_free_principal_for_same_account() {
|
||||
async fn reauthorization_preserves_existing_refresh_token_when_exchange_omits_one() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_oauth_principal".to_owned(),
|
||||
name: "reauthorize".to_owned(),
|
||||
account_id: "acct_oauth_reauth_fallback".to_owned(),
|
||||
name: "reauthorize fallback".to_owned(),
|
||||
secret: secret("old-oauth-access"),
|
||||
verified_account: profile("chatgpt-oauth"),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let service = service_with_store(store, Arc::new(RotatedPrincipalVerifier));
|
||||
let service = service_with_exchanger(
|
||||
store,
|
||||
Arc::new(StaticAccountSource::default()),
|
||||
Arc::new(NoRefreshTokenExchanger),
|
||||
);
|
||||
let started = service
|
||||
.start_authorization(StartCodexOAuthAuthorization {
|
||||
mutation: reauthorization_mutation("request-rotated-principal", "acct_oauth_principal"),
|
||||
mutation: reauthorization_mutation(
|
||||
"request-reauth-fallback",
|
||||
"acct_oauth_reauth_fallback",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("start reauthorization");
|
||||
@@ -677,6 +675,10 @@ async fn reauthorization_updates_changed_free_principal_for_same_account() {
|
||||
.query_pairs()
|
||||
.find_map(|(key, value)| (key == "state").then(|| value.into_owned()))
|
||||
.expect("state");
|
||||
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let prepared = service
|
||||
.complete_authorization(CompleteCodexOAuthAuthorization {
|
||||
owner_ref: owner_ref(),
|
||||
@@ -687,70 +689,31 @@ async fn reauthorization_updates_changed_free_principal_for_same_account() {
|
||||
})
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
drop(guard);
|
||||
let CompletedCodexOAuthCredential::Reauthorize(prepared) = prepared.credential else {
|
||||
panic!("expected prepared reauthorization");
|
||||
};
|
||||
let runtime =
|
||||
provider_openai::credential::CodexCredentialCodec::decode(prepared.credential.credential())
|
||||
.expect("prepared credential");
|
||||
let principal = runtime.principal.expect("OAuth principal");
|
||||
|
||||
assert_eq!(principal.oauth_subject, "rotated-oauth-subject");
|
||||
assert_eq!(principal.poid, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reauthorization_rebinds_target_record_to_authorized_identity() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_oauth_rebind".to_owned(),
|
||||
name: "reauthorize".to_owned(),
|
||||
secret: secret("old-oauth-access"),
|
||||
verified_account: profile("chatgpt-oauth"),
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let service = service_with_store(store, Arc::new(ReplacementIdentityVerifier));
|
||||
let started = service
|
||||
.start_authorization(StartCodexOAuthAuthorization {
|
||||
mutation: reauthorization_mutation("request-rebind", "acct_oauth_rebind"),
|
||||
})
|
||||
.await
|
||||
.expect("start reauthorization");
|
||||
let state = Url::parse(&started.authorization_url)
|
||||
.expect("authorization URL")
|
||||
.query_pairs()
|
||||
.find_map(|(key, value)| (key == "state").then(|| value.into_owned()))
|
||||
.expect("state");
|
||||
let completed = service
|
||||
.complete_authorization(CompleteCodexOAuthAuthorization {
|
||||
owner_ref: owner_ref(),
|
||||
flow_id: started.flow_id,
|
||||
callback_url: SecretString::from(format!(
|
||||
"http://localhost:1455/auth/callback?code=authorization-code&state={state}"
|
||||
)),
|
||||
})
|
||||
.await
|
||||
.expect("complete identity replacement");
|
||||
let CompletedCodexOAuthCredential::Reauthorize(prepared) = completed.credential else {
|
||||
panic!("expected prepared reauthorization");
|
||||
};
|
||||
assert_eq!(
|
||||
prepared.credential.account_id().as_str(),
|
||||
"acct_oauth_rebind"
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.expect("preserved refresh token")
|
||||
.expose_secret(),
|
||||
"rt-old-oauth-access"
|
||||
);
|
||||
let replacement = prepared
|
||||
.replacement_identity
|
||||
.as_ref()
|
||||
.expect("replacement identity");
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "authorization_code");
|
||||
assert_eq!(record["fields"]["account_id"], "acct_oauth_reauth_fallback");
|
||||
assert_eq!(
|
||||
replacement.upstream_account_id(),
|
||||
Some("chatgpt-different-owner")
|
||||
record["fields"]["access_token"],
|
||||
"oauth-access-without-refresh-token"
|
||||
);
|
||||
assert_eq!(
|
||||
replacement.upstream_user_id(),
|
||||
"user-chatgpt-different-owner"
|
||||
);
|
||||
assert_eq!(record["fields"]["refresh_token"], "rt-old-oauth-access");
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ use provider_openai::credential::{
|
||||
CodexQuotaWindowKind, ImportCodexOAuthCredential,
|
||||
};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use serde_json::json;
|
||||
use serde_json::{Value, json};
|
||||
use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
@@ -71,6 +71,7 @@ fn quota_service_with_base_url(
|
||||
http,
|
||||
base_url,
|
||||
crate::support::agent_identity_service(store),
|
||||
Arc::new(crate::support::MemoryCooldownPort::new()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -193,6 +194,40 @@ async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_headers_should_materialize_false_when_quota_is_not_exhausted() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_quota_available").await;
|
||||
let account = store
|
||||
.account("acct_passive_quota_available")
|
||||
.expect("account");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "codex".to_owned()),
|
||||
("x-codex-primary-used-percent".to_owned(), "31".to_owned()),
|
||||
("x-codex-allowed".to_owned(), "true".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
|
||||
let observations = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read quota observation");
|
||||
assert_eq!(
|
||||
observations
|
||||
.first()
|
||||
.and_then(|observation| observation.limit_reached),
|
||||
Some(false)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_preserves_disabled_account_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -201,6 +236,7 @@ async fn manual_quota_refresh_preserves_disabled_account_state() {
|
||||
let account = store.account(account_id).expect("created disabled account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
@@ -254,6 +290,7 @@ async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential(
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -387,14 +424,306 @@ async fn passive_rate_limit_headers_replace_stale_secondary_window() {
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
let raw = store
|
||||
.quota_json("acct_passive_quota_snapshot")
|
||||
.expect("stored passive quota");
|
||||
let monthly = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| window.kind() == CodexQuotaWindowKind::Monthly)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].used_percent(), Some(92.0));
|
||||
assert!(
|
||||
raw.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"a normal response without a secondary window must delete stale raw state"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_drops_empty_secondary_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_secondary_placeholder";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 0,
|
||||
"limit_reached": false
|
||||
}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("manual quota refresh");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_drops_null_secondary_window_during_confirmed_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_null_secondary_placeholder";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let reset_at = 1_900_000_000;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {
|
||||
"used_percent": 100,
|
||||
"reset_at": reset_at,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed confirmed exhaustion");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"used_percent": 86,
|
||||
"reset_at": reset_at,
|
||||
"reset_after_seconds": 2_133_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": null
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let snapshot = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("manual quota refresh");
|
||||
let persisted = store
|
||||
.get_quotas(&[account.id().clone()])
|
||||
.await
|
||||
.expect("read persisted quota")
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.map(Value::Object)
|
||||
.expect("persisted quota");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0));
|
||||
assert_eq!(
|
||||
snapshot.windows()[0]
|
||||
.reset_at()
|
||||
.map(|value| value.timestamp()),
|
||||
Some(reset_at)
|
||||
);
|
||||
assert!(snapshot.fact().exhausted());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after refresh")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(
|
||||
persisted.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"a null secondary window must not be projected as 100%"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_active_limit_keeps_the_top_level_codex_bucket_canonical() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_active_limit").await;
|
||||
let account = store.account("acct_passive_active_limit").expect("account");
|
||||
let baseline = json!({
|
||||
"additional_rate_limits": [{
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "premium".to_owned()),
|
||||
("x-premium-primary-used-percent".to_owned(), "55".to_owned()),
|
||||
(
|
||||
"x-premium-primary-window-minutes".to_owned(),
|
||||
"10080".to_owned(),
|
||||
),
|
||||
(
|
||||
"x-premium-primary-reset-at".to_owned(),
|
||||
"1900000000".to_owned(),
|
||||
),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert!(
|
||||
snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.any(|window| { window.source() == "codex" && window.used_percent() == Some(55.0) })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_metadata_headers_preserve_quota_observation_time() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_metadata").await;
|
||||
let account = store.account("acct_passive_metadata").expect("account");
|
||||
let baseline = json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {
|
||||
"used_percent": 20,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
baseline.as_object().expect("quota object").clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: None,
|
||||
})
|
||||
.await
|
||||
.expect("persist baseline quota");
|
||||
let before = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read baseline quota");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-plan-type".to_owned(), "free".to_owned()),
|
||||
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("metadata-only passive response")
|
||||
);
|
||||
let after = store
|
||||
.get_quotas(std::slice::from_ref(account.id()))
|
||||
.await
|
||||
.expect("read updated quota metadata");
|
||||
|
||||
assert_eq!(after[0].observed_at, before[0].observed_at);
|
||||
assert_eq!(after[0].limit_reached, before[0].limit_reached);
|
||||
assert_eq!(
|
||||
store
|
||||
.quota_json("acct_passive_metadata")
|
||||
.and_then(|quota| quota.get("plan_type").cloned()),
|
||||
Some(json!("free"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -460,6 +789,7 @@ async fn periodic_quota_synchronization_preserves_availability_when_usage_peaks(
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Ready,
|
||||
@@ -526,6 +856,7 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -556,6 +887,183 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_refresh_drops_empty_secondary_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_secondary_placeholder";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"used_percent": 99,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.synchronize().await.expect("periodic quota refresh");
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_refresh_drops_null_secondary_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_null_secondary_placeholder";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"used_percent": 86,
|
||||
"reset_at": 1_900_000_000,
|
||||
"reset_after_seconds": 2_133_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": null
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.synchronize().await.expect("periodic quota refresh");
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(snapshot.windows().len(), 1);
|
||||
assert_eq!(snapshot.windows()[0].used_percent(), Some(100.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_refresh_keeps_nonempty_secondary_window_during_confirmed_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_real_secondary";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
let secondary_reset_at = 1_900_604_800;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {
|
||||
"used_percent": 99,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": 0,
|
||||
"reset_at": secondary_reset_at,
|
||||
"limit_window_seconds": 604_800
|
||||
}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.synchronize().await.expect("periodic quota refresh");
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(summary.exhausted, 1);
|
||||
assert_eq!(snapshot.windows().len(), 2);
|
||||
assert!(snapshot.windows().iter().any(|window| {
|
||||
window.window_seconds() == Some(604_800)
|
||||
&& window.used_percent() == Some(100.0)
|
||||
&& window
|
||||
.reset_at()
|
||||
.is_some_and(|reset_at| reset_at.timestamp() == secondary_reset_at)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_respects_retry_after_on_service_unavailable() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -565,6 +1073,7 @@ async fn periodic_quota_synchronization_respects_retry_after_on_service_unavaila
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -605,6 +1114,7 @@ async fn periodic_quota_synchronization_skips_ready_accounts_without_quota_reach
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Ready,
|
||||
@@ -637,6 +1147,7 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() {
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -662,6 +1173,7 @@ async fn periodic_quota_synchronization_rechecks_exhausted_accounts_without_wait
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -727,6 +1239,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.expect("seed post-failure quota snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -798,7 +1311,7 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
.await
|
||||
.expect("lower percent quota recheck");
|
||||
|
||||
assert_eq!(lower_percent.updated, 1);
|
||||
assert_eq!(lower_percent.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
@@ -807,6 +1320,12 @@ async fn periodic_quota_synchronization_does_not_recover_from_percent_drop_befor
|
||||
AccountAvailability::QuotaExhausted,
|
||||
"a provider percentage change is not sufficient to release a confirmed exhaustion"
|
||||
);
|
||||
let snapshot = next_cycle
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read held exhaustion projection")
|
||||
.expect("held exhaustion projection");
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -815,8 +1334,31 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
let account_id = "acct_periodic_reset_elapsed";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let elapsed_reset = Utc::now().timestamp() - 60;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": elapsed_reset}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("recorded exhausted quota")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("record elapsed exhausted reset");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -825,14 +1367,14 @@ async fn periodic_quota_synchronization_recovers_after_recorded_reset() {
|
||||
.await
|
||||
.expect("mark account exhausted with an elapsed reset");
|
||||
let server = MockServer::start().await;
|
||||
let elapsed_reset = Utc::now().timestamp() - 1;
|
||||
let next_reset = Utc::now().timestamp() + 3600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 1, "reset_at": elapsed_reset}
|
||||
"primary_window": {"used_percent": 1, "reset_at": next_reset}
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
@@ -866,6 +1408,7 @@ async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() {
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -889,6 +1432,7 @@ async fn periodic_quota_synchronization_does_not_bypass_throttle_for_a_new_reset
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -943,6 +1487,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -951,7 +1496,33 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
.await
|
||||
.expect("mark account exhausted");
|
||||
}
|
||||
let good = store
|
||||
.account("acct_quota_batch_good")
|
||||
.expect("good account");
|
||||
let elapsed_reset = Utc::now().timestamp() - 60;
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: good.id().clone(),
|
||||
expected_revision: good.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": elapsed_reset}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("recorded exhausted quota")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed good account old reset");
|
||||
let server = MockServer::start().await;
|
||||
let next_reset = Utc::now().timestamp() + 3600;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header("authorization", "Bearer token-acct_quota_batch_bad"))
|
||||
@@ -970,7 +1541,7 @@ async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"primary_window": {"used_percent": 25.0, "reset_at": 1_900_000_000}
|
||||
"primary_window": {"used_percent": 25.0, "reset_at": next_reset}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
|
||||
@@ -12,6 +12,7 @@ async fn manual_quota_refresh_keeps_confirmed_exhaustion_until_the_recorded_rese
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
@@ -161,6 +162,7 @@ async fn expired_old_reset_with_fresh_allowed_snapshot_recovers_ready() {
|
||||
.expect("seed old exhausted snapshot");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -209,3 +211,47 @@ async fn expired_old_reset_with_fresh_allowed_snapshot_recovers_ready() {
|
||||
"old reset elapsed + fresh allowed snapshot must recover QuotaExhausted"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deactivated_workspace_quota_response_persists_the_upstream_reason() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_deactivated_workspace";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(402).set_body_json(json!({
|
||||
"detail": {
|
||||
"code": "deactivated_workspace",
|
||||
"message": "workspace disabled"
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
assert!(service.refresh_account(account.id()).await.is_err());
|
||||
assert_eq!(
|
||||
store
|
||||
.account(account_id)
|
||||
.expect("account after deactivation")
|
||||
.availability(),
|
||||
AccountAvailability::Banned
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(account_id).as_deref(),
|
||||
Some("deactivated_workspace")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -305,7 +305,7 @@ async fn scheduling_signal_survives_limit_reached_without_percent_or_reset() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
|
||||
async fn persisted_codex_alias_keeps_the_top_level_rate_limit_canonical() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_canonical_codex_limit").await;
|
||||
let account = store
|
||||
@@ -323,7 +323,7 @@ async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
|
||||
"metered_feature": "codex",
|
||||
"rate_limit": {
|
||||
"primary_window": {
|
||||
"used_percent": 2,
|
||||
"used_percent": 98,
|
||||
"reset_at": 1_900_000_000,
|
||||
"limit_window_seconds": 2_592_000
|
||||
}
|
||||
@@ -355,10 +355,13 @@ async fn persisted_codex_additional_limit_replaces_the_top_level_rate_limit() {
|
||||
.filter(|window| window.kind() == CodexQuotaWindowKind::Monthly)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(98));
|
||||
assert_eq!(snapshot.fact().remaining_percent(), Some(9));
|
||||
assert_eq!(monthly.len(), 1);
|
||||
assert_eq!(monthly[0].source(), "core");
|
||||
assert_eq!(monthly[0].used_percent(), Some(2.0));
|
||||
assert!(
|
||||
monthly
|
||||
.iter()
|
||||
.any(|window| { window.source() == "codex" && window.used_percent() == Some(91.0) })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -419,16 +422,21 @@ async fn code_review_limit_projects_as_independent_window_with_limit_name() {
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
// spend_control 不再生成窗口(只作 exhaustion 信号),
|
||||
// 顶层 code_review_rate_limit 与 additional 的 code_review 桶同源替换后保留一个。
|
||||
// spend_control 不再生成窗口(只作 exhaustion 信号);顶层和 additional
|
||||
// 的 code_review 桶分别保留,避免一个快照覆盖另一个。
|
||||
let review = snapshot
|
||||
.windows()
|
||||
.iter()
|
||||
.filter(|window| window.source() == "code_review")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(review.len(), 1);
|
||||
assert_eq!(review[0].limit_name(), Some("code review"));
|
||||
assert_eq!(review[0].used_percent(), Some(55.0));
|
||||
assert_eq!(snapshot.windows().len(), 2);
|
||||
assert_eq!(review.len(), 2);
|
||||
assert!(review.iter().any(|window| {
|
||||
window.limit_name() == Some("code_review") && window.used_percent() == Some(80.0)
|
||||
}));
|
||||
assert!(review.iter().any(|window| {
|
||||
window.limit_name() == Some("code review") && window.used_percent() == Some(55.0)
|
||||
}));
|
||||
assert_ne!(review[0].key(), review[1].key());
|
||||
assert_eq!(snapshot.windows().len(), 3);
|
||||
assert!(!snapshot.fact().exhausted());
|
||||
}
|
||||
|
||||
@@ -17,8 +17,11 @@ use provider_openai::credential::{
|
||||
};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
|
||||
use super::{OAuthRecoveryLogCapture, lock_oauth_recovery_log_capture};
|
||||
use crate::support::{MemoryAccountStore, profile, runtime_policy, secret};
|
||||
|
||||
const OAUTH_BACKOFF_ATTEMPTS: u32 = 5;
|
||||
|
||||
struct Refresher {
|
||||
outcomes: Mutex<VecDeque<Result<TokenPair, RefreshFailure>>>,
|
||||
seen: Mutex<Vec<String>>,
|
||||
@@ -377,7 +380,88 @@ async fn successful_refresh_uses_redis_lease_and_cas_rotates_plaintext_tokens()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshed_identity_rejection_revision_fences_account_as_invalid() {
|
||||
async fn scheduled_refresh_preserves_existing_refresh_token_when_exchange_omits_one() {
|
||||
let (store, _, service) = setup(
|
||||
Ok(TokenPair {
|
||||
access_token: "scheduled-access-without-rotated-refresh".to_owned(),
|
||||
refresh_token: None,
|
||||
expires_in: Duration::from_secs(3_600),
|
||||
}),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("scheduled refresh");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Refreshed { account_id, .. }] if account_id == "acct_refresh"
|
||||
));
|
||||
let account = store.account("acct_refresh").expect("rotated account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
let runtime = store
|
||||
.repository()
|
||||
.load_runtime_credential(&account)
|
||||
.await
|
||||
.expect("rotated credential");
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"scheduled-access-without-rotated-refresh"
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.refresh_token
|
||||
.as_ref()
|
||||
.expect("preserved refresh token")
|
||||
.expose_secret(),
|
||||
"rt-old-access"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_refresh_response_is_logged_before_local_rejection() {
|
||||
let _capture_lock = lock_oauth_recovery_log_capture().await;
|
||||
let (_, _, service) = setup(
|
||||
Ok(TokenPair {
|
||||
access_token: "rejected-access".to_owned(),
|
||||
refresh_token: None,
|
||||
expires_in: Duration::ZERO,
|
||||
}),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
let recovery_log = OAuthRecoveryLogCapture::default();
|
||||
let dispatch = recovery_log.dispatch();
|
||||
let guard = tracing::dispatcher::set_default(&dispatch);
|
||||
|
||||
let outcomes = service
|
||||
.refresh_due()
|
||||
.await
|
||||
.expect("refresh cycle completes");
|
||||
drop(guard);
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Failed { account_id }] if account_id == "acct_refresh"
|
||||
));
|
||||
let record = recovery_log.recovery_record();
|
||||
assert_eq!(record["fields"]["operation"], "scheduled_refresh");
|
||||
assert_eq!(record["fields"]["account_id"], "acct_refresh");
|
||||
assert_eq!(record["fields"]["access_token"], "rejected-access");
|
||||
assert_eq!(record["fields"]["refresh_token"], "rt-old-access");
|
||||
assert_eq!(record["fields"]["has_refresh_token"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshed_tokens_persist_when_identity_verification_rejects() {
|
||||
let (store, _, _) = setup(Ok(success_tokens()), true).await;
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
@@ -395,19 +479,31 @@ async fn refreshed_identity_rejection_revision_fences_account_as_invalid() {
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Invalidated { .. }]
|
||||
[CodexCredentialRefreshOutcome::Refreshed {
|
||||
account_id,
|
||||
credential_revision: 2
|
||||
}] if account_id == "acct_refresh"
|
||||
));
|
||||
let account = store.account("acct_refresh").expect("refreshed account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
let runtime = store
|
||||
.repository()
|
||||
.load_runtime_credential(&account)
|
||||
.await
|
||||
.expect("refreshed credential");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_refresh")
|
||||
.expect("invalid account")
|
||||
.availability(),
|
||||
AccountAvailability::Invalid
|
||||
runtime
|
||||
.authentication
|
||||
.oauth()
|
||||
.expect("OAuth credential")
|
||||
.access_token
|
||||
.expose_secret(),
|
||||
"new-access"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unavailable_signature_verification_persists_refresh_backoff() {
|
||||
async fn refreshed_tokens_persist_when_identity_verification_is_unavailable() {
|
||||
let (store, _, _) = setup(Ok(success_tokens()), true).await;
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
@@ -426,11 +522,10 @@ async fn unavailable_signature_verification_persists_refresh_backoff() {
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Transient { .. }]
|
||||
[CodexCredentialRefreshOutcome::Refreshed { .. }]
|
||||
));
|
||||
let account = store.account("acct_refresh").expect("deferred account");
|
||||
let account = store.account("acct_refresh").expect("refreshed account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
// JWKS/签名边界短暂不可用是瞬态:账号保持可用、不被永久失效,仅推进退避重试。
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(
|
||||
account
|
||||
@@ -440,7 +535,7 @@ async fn unavailable_signature_verification_persists_refresh_backoff() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unavailable_usage_preserves_rotated_tokens_and_persists_backoff() {
|
||||
async fn refreshed_tokens_do_not_require_identity_completion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let signed = super::identity::signed_identity_fixture().await;
|
||||
let mut account_profile = profile("account-signed");
|
||||
@@ -474,7 +569,7 @@ async fn unavailable_usage_preserves_rotated_tokens_and_persists_backoff() {
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Transient { .. }]
|
||||
[CodexCredentialRefreshOutcome::Refreshed { .. }]
|
||||
));
|
||||
let account = store.account("acct_signed_only").expect("rotated account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
@@ -553,6 +648,10 @@ async fn upstream_ban_marks_unified_account_banned() {
|
||||
.availability(),
|
||||
AccountAvailability::Banned
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message("acct_refresh").as_deref(),
|
||||
Some("account_banned")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -597,7 +696,7 @@ async fn transient_refresh_defers_without_invalidating_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() {
|
||||
async fn expired_access_token_respects_persisted_refresh_backoff() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
seed_due_account(&store).await;
|
||||
let original = store.account("acct_refresh").expect("seeded account");
|
||||
@@ -615,16 +714,6 @@ async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() {
|
||||
})
|
||||
.await
|
||||
.expect("seed expired access token with deferred refresh");
|
||||
let deferred = store.account("acct_refresh").expect("deferred account");
|
||||
store
|
||||
.repository()
|
||||
.apply_state(
|
||||
&deferred,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
.expect("mark account quota exhausted");
|
||||
let refresher = Arc::new(Refresher::new(Ok(success_tokens())));
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
@@ -640,31 +729,19 @@ async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() {
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Refreshed {
|
||||
account_id,
|
||||
credential_revision: 3,
|
||||
}] if account_id == "acct_refresh"
|
||||
));
|
||||
assert_eq!(
|
||||
refresher.seen.lock().expect("seen tokens lock").as_slice(),
|
||||
["rt-old-access"]
|
||||
);
|
||||
let refreshed = store.account("acct_refresh").expect("refreshed account");
|
||||
assert_eq!(
|
||||
refreshed.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(outcomes.is_empty());
|
||||
assert!(refresher.seen.lock().expect("seen tokens lock").is_empty());
|
||||
assert!(
|
||||
refreshed
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at > SystemTime::now())
|
||||
store
|
||||
.account("acct_refresh")
|
||||
.expect("deferred account")
|
||||
.next_refresh_at()
|
||||
.is_some_and(|next_refresh_at| next_refresh_at > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_access_token_refreshes_during_active_cooldown() {
|
||||
async fn expired_access_token_refreshes_when_persisted_refresh_is_due() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
seed_due_account(&store).await;
|
||||
let original = store.account("acct_refresh").expect("seeded account");
|
||||
@@ -678,7 +755,7 @@ async fn expired_access_token_refreshes_during_active_cooldown() {
|
||||
expected_credential_revision: original.revision().get(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::hours(12)),
|
||||
next_refresh_at: Some(chrono::Utc::now() - chrono::Duration::seconds(1)),
|
||||
})
|
||||
.await
|
||||
.expect("seed expired access token with deferred refresh");
|
||||
@@ -825,6 +902,15 @@ async fn seed_due_account(store: &MemoryAccountStore) {
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn force_due_account(store: &Arc<MemoryAccountStore>) {
|
||||
let account = store.account("acct_refresh").expect("account to reset");
|
||||
store
|
||||
.repository()
|
||||
.defer_refresh(&account, SystemTime::now() - Duration::from_secs(1))
|
||||
.await
|
||||
.expect("reset next_refresh_at to the past");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_backoff_grows_exponentially_across_attempts() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -891,6 +977,175 @@ async fn refresh_backoff_grows_exponentially_across_attempts() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausted_oauth_backoff_continues_after_access_token_expiry() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let mut expired_profile = profile("chatgpt-acct_refresh");
|
||||
expired_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() - chrono::Duration::minutes(1));
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_refresh".to_owned(),
|
||||
name: "refresh".to_owned(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() - chrono::Duration::seconds(1)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let credential_state = Arc::new(CountingCredentialState::default());
|
||||
let state_port: Arc<dyn ProviderCredentialStatePort> = credential_state.clone();
|
||||
let refresher = Arc::new(Refresher::scripted([
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
]));
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
refresher.clone(),
|
||||
Arc::new(VerifiedIdentity),
|
||||
Arc::new(RefreshLeases {
|
||||
available: true,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
}),
|
||||
state_port,
|
||||
runtime_policy(),
|
||||
);
|
||||
|
||||
for _ in 0..OAUTH_BACKOFF_ATTEMPTS {
|
||||
service.refresh_due().await.expect("backoff attempt");
|
||||
force_due_account(&store).await;
|
||||
}
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
service.refresh_due().await.expect("recovery attempt");
|
||||
let recovery_delay = store
|
||||
.account("acct_refresh")
|
||||
.expect("account after recovery defer")
|
||||
.next_refresh_at()
|
||||
.expect("recovery scheduled")
|
||||
.duration_since(observed_at)
|
||||
.expect("recovery is in the future");
|
||||
assert!(
|
||||
(Duration::from_secs(8 * 60)..=Duration::from_secs(12 * 60 + 1)).contains(&recovery_delay),
|
||||
"recovery delay should be about ten minutes, got {recovery_delay:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
credential_state.count("acct_refresh"),
|
||||
OAUTH_BACKOFF_ATTEMPTS + 1
|
||||
);
|
||||
assert_eq!(refresher.seen.lock().expect("seen tokens lock").len(), 6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_backoff_never_passes_an_unexpired_access_token_expiry() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let mut account_profile = profile("chatgpt-acct_refresh");
|
||||
account_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() + chrono::Duration::seconds(30));
|
||||
let access_token_expires_at: SystemTime = account_profile
|
||||
.access_token_expires_at
|
||||
.expect("access expiry")
|
||||
.into();
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_refresh".to_owned(),
|
||||
name: "refresh".to_owned(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: account_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() - chrono::Duration::seconds(1)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let credential_state = Arc::new(CountingCredentialState::default());
|
||||
let state_port: Arc<dyn ProviderCredentialStatePort> = credential_state.clone();
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
Arc::new(Refresher::scripted([
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
Err(RefreshFailure::Transport),
|
||||
])),
|
||||
Arc::new(VerifiedIdentity),
|
||||
Arc::new(RefreshLeases {
|
||||
available: true,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
}),
|
||||
state_port,
|
||||
runtime_policy(),
|
||||
);
|
||||
|
||||
for attempt in 0..=OAUTH_BACKOFF_ATTEMPTS {
|
||||
service.refresh_due().await.expect("backoff attempt");
|
||||
if attempt < OAUTH_BACKOFF_ATTEMPTS {
|
||||
force_due_account(&store).await;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_refresh")
|
||||
.expect("account after capped retry")
|
||||
.next_refresh_at(),
|
||||
Some(access_token_expires_at)
|
||||
);
|
||||
assert_eq!(
|
||||
credential_state.count("acct_refresh"),
|
||||
OAUTH_BACKOFF_ATTEMPTS + 1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_window_exhaustion_marks_account_expired_without_token_exchange() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let mut expired_profile = profile("chatgpt-acct_refresh");
|
||||
expired_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() - chrono::Duration::hours(25));
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: "acct_refresh".to_owned(),
|
||||
name: "refresh".to_owned(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::hours(12)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let refresher = Arc::new(Refresher::new(Ok(success_tokens())));
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
refresher.clone(),
|
||||
Arc::new(VerifiedIdentity),
|
||||
Arc::new(RefreshLeases {
|
||||
available: true,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
}),
|
||||
Arc::new(CountingCredentialState::default()),
|
||||
runtime_policy(),
|
||||
);
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Invalidated { account_id }] if account_id == "acct_refresh"
|
||||
));
|
||||
assert!(refresher.seen.lock().expect("seen tokens lock").is_empty());
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_refresh")
|
||||
.expect("expired account")
|
||||
.availability(),
|
||||
AccountAvailability::Expired
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_refresh_clears_backoff_counter() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -75,6 +75,28 @@ async fn bounded_oauth_response_should_parse_lifetime_and_rotated_token() {
|
||||
assert_eq!(tokens.expires_in, Duration::from_secs(3600));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_response_with_blank_rotated_token_keeps_the_existing_token_eligible() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/oauth/token"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"access_token": "access-rotated",
|
||||
"refresh_token": " ",
|
||||
"expires_in": 3600
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let tokens = client(&server)
|
||||
.refresh("refresh-initial")
|
||||
.await
|
||||
.expect("blank rotation field is treated as absent");
|
||||
|
||||
assert_eq!(tokens.refresh_token, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_should_exchange_the_official_form_fields() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -43,8 +43,9 @@ use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::support::{
|
||||
MemoryAccountStore, MemorySessionAffinity, MemorySessionExclusions, TestLeaseCoordinator,
|
||||
account_policy, agent_identity_service_with_pool, catalog_cache, profile, secret,
|
||||
MemoryAccountStore, MemoryCooldownPort, MemorySessionAffinity, MemorySessionExclusions,
|
||||
TestLeaseCoordinator, account_policy, agent_identity_service_with_pool, catalog_cache, profile,
|
||||
secret,
|
||||
};
|
||||
use crate::transport::accept_codex_test_websocket;
|
||||
|
||||
@@ -115,6 +116,21 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> CodexProvider {
|
||||
provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store,
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
)
|
||||
.0
|
||||
}
|
||||
|
||||
fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -136,6 +152,7 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
Arc::clone(&agent_identity),
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
));
|
||||
let account_feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
@@ -152,10 +169,10 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
true,
|
||||
));
|
||||
|
||||
CodexProvider::new(
|
||||
let provider = CodexProvider::new(
|
||||
selector,
|
||||
catalog,
|
||||
quota,
|
||||
Arc::clone("a),
|
||||
agent_identity,
|
||||
account_feedback,
|
||||
http,
|
||||
@@ -163,7 +180,8 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
base_url,
|
||||
websocket_pool,
|
||||
)
|
||||
.expect("official OpenAI provider")
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, id: &str) {
|
||||
@@ -196,6 +214,27 @@ fn generate_operation() -> Operation {
|
||||
))
|
||||
}
|
||||
|
||||
fn generate_with_session_context(
|
||||
session_id: &str,
|
||||
thread_id: Option<&str>,
|
||||
turn_metadata: Option<&str>,
|
||||
) -> GenerateRequest {
|
||||
let mut body = Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
("session_id".to_owned(), json!(session_id)),
|
||||
]);
|
||||
if let Some(thread_id) = thread_id {
|
||||
body.insert("thread_id".to_owned(), json!(thread_id));
|
||||
}
|
||||
if let Some(turn_metadata) = turn_metadata {
|
||||
body.insert("turnMetadata".to_owned(), json!(turn_metadata));
|
||||
}
|
||||
GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body).expect("OpenAI payload"),
|
||||
)
|
||||
}
|
||||
|
||||
fn http_generate_operation() -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
@@ -1053,6 +1092,66 @@ async fn prompt_cache_key_should_become_an_opaque_session_affinity_lookup_key()
|
||||
assert!(keys[0].bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_requests_should_use_an_affinity_key_isolated_from_the_root_session() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_subagent").await;
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(CAPTURE_COMPLETED_SSE),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let provider = provider_with_affinity_and_base_url(&store, Arc::clone(&affinity), server.uri());
|
||||
|
||||
for (request_id, subagent_kind) in [
|
||||
("req_root_affinity", None),
|
||||
("req_subagent_affinity_first", Some("review")),
|
||||
("req_subagent_affinity_second", Some("review")),
|
||||
] {
|
||||
let mut body = Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("new task")),
|
||||
("prompt_cache_key".to_owned(), json!("root-session-key")),
|
||||
]);
|
||||
if let Some(subagent_kind) = subagent_kind {
|
||||
body.insert(
|
||||
"client_metadata".to_owned(),
|
||||
json!({"x-openai-subagent": subagent_kind}),
|
||||
);
|
||||
}
|
||||
let generation = GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body)
|
||||
.expect("OpenAI payload")
|
||||
.with_context(Map::from_iter([("use_websocket".to_owned(), json!(false))])),
|
||||
);
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", Operation::Generate(generation)),
|
||||
context(request_id, CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare subagent provider stream");
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("subagent response");
|
||||
}
|
||||
drop(stream);
|
||||
}
|
||||
|
||||
let keys = affinity.lookup_keys();
|
||||
assert_eq!(keys.len(), 3);
|
||||
assert_ne!(keys[0], keys[1], "subagent must not inherit the root key");
|
||||
assert_eq!(
|
||||
keys[1], keys[2],
|
||||
"the same subagent should retain its own key"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_session_id_should_override_turn_specific_prompt_cache_keys_for_affinity() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1095,6 +1194,58 @@ async fn explicit_session_id_should_override_turn_specific_prompt_cache_keys_for
|
||||
assert_eq!(keys[0], keys[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_spawn_children_should_have_independent_session_affinity_keys() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_thread_spawn_affinity").await;
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let provider = provider_with_affinity(&store, Arc::clone(&affinity));
|
||||
let thread_spawn = r#"{"subagent_kind":"thread_spawn"}"#;
|
||||
|
||||
for (request_id, thread_id, turn_metadata) in [
|
||||
(
|
||||
"req_thread_spawn_first",
|
||||
Some("child-one"),
|
||||
Some(thread_spawn),
|
||||
),
|
||||
(
|
||||
"req_thread_spawn_second",
|
||||
Some("child-two"),
|
||||
Some(thread_spawn),
|
||||
),
|
||||
(
|
||||
"req_thread_spawn_repeat",
|
||||
Some("child-one"),
|
||||
Some(thread_spawn),
|
||||
),
|
||||
("req_thread_spawn_fallback", None, Some(thread_spawn)),
|
||||
("req_thread_spawn_parent", None, None),
|
||||
] {
|
||||
let stream = provider
|
||||
.execute(
|
||||
planned_request(
|
||||
"openai",
|
||||
Operation::Generate(generate_with_session_context(
|
||||
"parent-session",
|
||||
thread_id,
|
||||
turn_metadata,
|
||||
)),
|
||||
),
|
||||
context(request_id, CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
drop(stream);
|
||||
}
|
||||
|
||||
let keys = affinity.lookup_keys();
|
||||
assert_eq!(keys.len(), 5);
|
||||
assert_ne!(keys[0], keys[1]);
|
||||
assert_eq!(keys[0], keys[2]);
|
||||
assert_ne!(keys[3], keys[4]);
|
||||
assert_ne!(keys[0], keys[3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_conversation_id_should_fall_back_to_an_opaque_affinity_key() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1271,9 +1422,10 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_loc
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_quota_snapshot() {
|
||||
async fn official_usage_limit_failure_projects_100_percent_and_holds_until_reset_advances() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_usage_limit_request_path";
|
||||
let reset_at = 1_900_000_000;
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
@@ -1285,7 +1437,8 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 99, "reset_at": 1_900_000_000}
|
||||
"primary_window": {"used_percent": 99, "reset_at": reset_at},
|
||||
"secondary_window": {"used_percent": 0}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
@@ -1301,8 +1454,8 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header("authorization", format!("Bearer at-{account_id}")))
|
||||
// 生产环境会短暂返回互相矛盾的快照:数字已到 100%,但布尔位仍声称可用。
|
||||
// 后台同步应覆盖展示数据,不能据此撤销真实请求已经确认的额度耗尽。
|
||||
// 生产环境会短暂返回互相矛盾的快照:98/99% 且布尔位仍声称可用。
|
||||
// 后台同步不能覆盖真实请求已确认的额度耗尽。
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
// 验证后台 usage 同步不能把原始的额度错误响应拖到查询完成之后。
|
||||
@@ -1311,7 +1464,8 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
"primary_window": {"used_percent": 99, "reset_at": reset_at},
|
||||
"secondary_window": {"used_percent": 0}
|
||||
}
|
||||
})),
|
||||
)
|
||||
@@ -1320,16 +1474,13 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_usage_limit_confirm\",\"model\":\"gpt-5.4\",\"status\":\"in_progress\"}}\n\n",
|
||||
"event: response.failed\n",
|
||||
"data: {\"type\":\"response.failed\",\"status_code\":429,\"retry_after_seconds\":86400,\"response\":{\"id\":\"resp_usage_limit_confirm\",\"status\":\"failed\",\"error\":{\"code\":\"usage_limit_reached\",\"message\":\"usage limit reached\"}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.respond_with(ResponseTemplate::new(429).set_body_json(json!({
|
||||
"error": {
|
||||
"type": "usage_limit_reached",
|
||||
"message": "usage limit reached",
|
||||
"resets_at": reset_at
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
@@ -1355,6 +1506,30 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
let account = store.account(account_id).expect("usage-limit account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
let projected = store
|
||||
.get_quotas(&[account.id().clone()])
|
||||
.await
|
||||
.expect("read immediate quota projection")
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("confirmed quota observation");
|
||||
let projected_observed_at = projected.observed_at.expect("projection observed at");
|
||||
let projected = Value::Object(
|
||||
projected
|
||||
.quota
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.expect("confirmed quota projection"),
|
||||
);
|
||||
assert_eq!(
|
||||
projected
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(Value::as_u64),
|
||||
Some(100)
|
||||
);
|
||||
assert!(
|
||||
projected.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"request failure must not manufacture a secondary quota"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
assert_eq!(
|
||||
@@ -1366,35 +1541,49 @@ async fn usage_limit_failure_returns_promptly_and_refreshes_the_authoritative_qu
|
||||
"usage refresh must wait for the upstream quota settlement delay"
|
||||
);
|
||||
|
||||
let quota = timeout(Duration::from_secs(5), async {
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let quota = store
|
||||
let observed_at = store
|
||||
.get_quotas(&[account.id().clone()])
|
||||
.await
|
||||
.expect("read refreshed quota")
|
||||
.expect("read refreshed quota observation")
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|observation| observation.quota)
|
||||
.expect("authoritative quota snapshot");
|
||||
let quota = Value::Object(quota.into_inner());
|
||||
if quota
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(Value::as_u64)
|
||||
== Some(100)
|
||||
{
|
||||
break quota;
|
||||
.and_then(|observation| observation.observed_at);
|
||||
if observed_at.is_some_and(|observed_at| observed_at > projected_observed_at) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("background usage refresh must replace the stale quota snapshot");
|
||||
.expect("background usage refresh must run");
|
||||
let quota = store
|
||||
.get_quotas(&[account.id().clone()])
|
||||
.await
|
||||
.expect("read refreshed quota")
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|observation| observation.quota)
|
||||
.map(OpaqueProviderData::into_inner)
|
||||
.map(Value::Object)
|
||||
.expect("authoritative quota projection");
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/used_percent")
|
||||
.and_then(Value::as_u64),
|
||||
Some(100)
|
||||
);
|
||||
assert_eq!(
|
||||
quota
|
||||
.pointer("/rate_limit/primary_window/reset_at")
|
||||
.and_then(Value::as_i64),
|
||||
Some(reset_at)
|
||||
);
|
||||
assert!(
|
||||
quota.pointer("/rate_limit/secondary_window").is_none(),
|
||||
"background quota refresh must retain the same window projection"
|
||||
);
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
assert_eq!(
|
||||
requests
|
||||
@@ -1849,6 +2038,85 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
assert_eq!(affinity.binding_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_quota_limited_diagnostic";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("test account");
|
||||
let raw_quota = json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
});
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
raw_quota
|
||||
.as_object()
|
||||
.expect("quota snapshot object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
limit_reached: Some(true),
|
||||
})
|
||||
.await
|
||||
.expect("seed quota-limited snapshot");
|
||||
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_quota_limited_diagnostic\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (provider, quota) = provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
&store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
);
|
||||
quota
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
.await;
|
||||
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
diagnostic_context("req_quota_limited_diagnostic", account_id),
|
||||
)
|
||||
.await
|
||||
.expect("quota-limited diagnostic should prepare a fixed-account stream");
|
||||
let mut completed = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("quota-limited diagnostic upstream response");
|
||||
completed |= event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
}
|
||||
|
||||
assert!(completed);
|
||||
let requests = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("captured quota-limited diagnostic request");
|
||||
assert_eq!(requests.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -16,9 +16,10 @@ use gateway_core::engine::credential::{
|
||||
};
|
||||
use gateway_core::error::{StoreError, StoreErrorKind};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderLeaseAcquisition, ProviderLeasePort,
|
||||
ProviderLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
|
||||
ProviderSchedulingLeaseRequest, ProviderSchedulingState, ProviderSessionAffinityKey,
|
||||
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCooldown, ProviderCooldownPort,
|
||||
ProviderCooldownScope, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshPolicy, ProviderRuntimePolicyPort, ProviderSchedulingLeaseRequest,
|
||||
ProviderSchedulingState, ProviderScopedCooldown, ProviderSessionAffinityKey,
|
||||
ProviderSessionAffinityPort, ProviderSessionExclusionPort, ProviderSessionExclusions,
|
||||
ProviderStoreError,
|
||||
};
|
||||
@@ -37,6 +38,7 @@ struct StoredAccount {
|
||||
credential: gateway_core::engine::credential::PlaintextCredential,
|
||||
quota: Option<QuotaObservation>,
|
||||
state_observed_at: Option<SystemTime>,
|
||||
last_error_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -69,6 +71,15 @@ impl MemoryAccountStore {
|
||||
.map(|stored| stored.account.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn last_error_message(&self, id: &str) -> Option<String> {
|
||||
let id = ProviderAccountId::new(id).ok()?;
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("account store lock")
|
||||
.get(&id)
|
||||
.and_then(|stored| stored.last_error_message.clone())
|
||||
}
|
||||
|
||||
pub(crate) async fn advance_credential_revision(&self, id: &str) -> CredentialRevision {
|
||||
let account_id = ProviderAccountId::new(id).expect("test account id");
|
||||
let current = self
|
||||
@@ -147,6 +158,7 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
credential: input.credential,
|
||||
quota: None,
|
||||
state_observed_at: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
@@ -320,6 +332,11 @@ impl ProviderAccountStore for MemoryAccountStore {
|
||||
},
|
||||
);
|
||||
stored.state_observed_at = Some(change.observed_at);
|
||||
if change.availability == AccountAvailability::Ready {
|
||||
stored.last_error_message = None;
|
||||
} else if let Some(message) = change.message {
|
||||
stored.last_error_message = Some(message);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -771,3 +788,90 @@ pub(crate) fn codex_account(id: &str) -> ProviderAccount {
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_refresh_schedule(true, Some(SystemTime::now() + Duration::from_secs(2_700)))
|
||||
}
|
||||
|
||||
/// 内存 `ProviderCooldownPort`:只实现 `read`/`put_if_later`(openai selector
|
||||
/// 429 冷却路径用),其余 scope 变体测试不涉及,返回占位。
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct MemoryCooldownPort {
|
||||
pub(crate) cooldowns: Arc<Mutex<BTreeMap<ProviderAccountId, ProviderCooldown>>>,
|
||||
}
|
||||
|
||||
impl MemoryCooldownPort {
|
||||
#[must_use]
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCooldownPort for MemoryCooldownPort {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
cooldown: ProviderCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut cooldowns = self.cooldowns.lock().expect("cooldown lock");
|
||||
let should_write = cooldowns.get(cooldown.account_id()).is_none_or(|current| {
|
||||
current.credential_revision() < cooldown.credential_revision()
|
||||
|| (current.credential_revision() == cooldown.credential_revision()
|
||||
&& current.until() < cooldown.until())
|
||||
});
|
||||
if should_write {
|
||||
cooldowns.insert(cooldown.account_id().clone(), cooldown);
|
||||
}
|
||||
Ok(should_write)
|
||||
})
|
||||
}
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
Ok(self
|
||||
.cooldowns
|
||||
.lock()
|
||||
.expect("cooldown lock")
|
||||
.get(account_id)
|
||||
.cloned())
|
||||
})
|
||||
}
|
||||
|
||||
fn clear<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
_cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
_scope: &'a ProviderCooldownScope,
|
||||
_through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
_account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async { Ok(false) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,6 +219,10 @@ async fn fetch_should_send_official_catalog_headers_and_query() {
|
||||
.and(header("authorization", "Bearer oauth-access"))
|
||||
.and(header("chatgpt-account-id", "acct_123"))
|
||||
.and(header("originator", "codex_cli_rs"))
|
||||
.and(header(
|
||||
"user-agent",
|
||||
"codex_cli_rs/0.144.0 (linux 6.8; x86_64) xterm (codex_cli_rs; 1.0.0)",
|
||||
))
|
||||
.and(header("accept", "application/json"))
|
||||
.and(header("x-codex-installation-id", "installation-123"))
|
||||
.respond_with(
|
||||
|
||||
@@ -146,13 +146,13 @@ fn structured_expiry_code_classifies_without_an_auth_status() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_quota_signal_takes_precedence_over_http_429() {
|
||||
fn http_429_without_usage_limit_type_remains_a_temporary_rate_limit() {
|
||||
assert_eq!(
|
||||
classify(
|
||||
429,
|
||||
r#"{"error":{"code":"insufficient_quota","message":"rate limit reached"}}"#
|
||||
),
|
||||
CodexFailureCategory::QuotaExhausted
|
||||
CodexFailureCategory::RateLimited
|
||||
);
|
||||
}
|
||||
|
||||
@@ -168,13 +168,24 @@ fn structured_usage_limit_type_is_resettable_quota_exhaustion() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_limit_message_without_a_type_is_resettable_quota_exhaustion() {
|
||||
fn http_429_usage_limit_message_without_a_type_remains_a_temporary_rate_limit() {
|
||||
assert_eq!(
|
||||
classify(
|
||||
429,
|
||||
r#"{"error":{"message":"The usage limit has been reached"}}"#
|
||||
),
|
||||
CodexFailureCategory::UsageLimitExhausted
|
||||
CodexFailureCategory::RateLimited
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_429_usage_limit_code_without_a_type_remains_a_temporary_rate_limit() {
|
||||
assert_eq!(
|
||||
classify(
|
||||
429,
|
||||
r#"{"error":{"code":"usage_limit_reached","message":"The usage limit has been reached"}}"#
|
||||
),
|
||||
CodexFailureCategory::RateLimited
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -789,6 +789,70 @@ async fn backend_http_should_zstd_compress_codex_responses_request_body() {
|
||||
assert_eq!(json["instructions"], "compress me");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_http_should_force_upstream_sse_for_non_streaming_client_request() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind non-streaming HTTP server");
|
||||
let address = listener
|
||||
.local_addr()
|
||||
.expect("non-streaming HTTP server address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept HTTP client");
|
||||
let request = read_http_request_with_body(&mut stream).await;
|
||||
write_completed_sse_response(&mut stream).await;
|
||||
request
|
||||
});
|
||||
let mut request = CodexResponsesRequest::from_body(
|
||||
json!({
|
||||
"model": "gpt-test",
|
||||
"instructions": "collect this response",
|
||||
"input": [],
|
||||
"stream": false
|
||||
})
|
||||
.as_object()
|
||||
.expect("request body")
|
||||
.clone(),
|
||||
);
|
||||
request.force_http_sse = true;
|
||||
assert!(
|
||||
!request.stream(),
|
||||
"client-facing preference remains non-streaming"
|
||||
);
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("HTTP client"),
|
||||
format!("http://{address}"),
|
||||
test_wire_profile(),
|
||||
);
|
||||
|
||||
client
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_non_streaming_client", Some("acct-non-streaming")),
|
||||
)
|
||||
.await
|
||||
.expect("upstream SSE response");
|
||||
|
||||
let raw = server.await.expect("non-streaming HTTP server task");
|
||||
let separator = raw
|
||||
.windows(4)
|
||||
.position(|window| window == b"\r\n\r\n")
|
||||
.expect("HTTP head/body separator");
|
||||
let compressed_body = &raw[separator + 4..];
|
||||
let decompressed = zstd::stream::decode_all(std::io::Cursor::new(compressed_body))
|
||||
.expect("zstd body should decode");
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&decompressed).expect("decompressed body should be valid JSON");
|
||||
assert_eq!(json["stream"], true);
|
||||
assert!(
|
||||
!request.stream(),
|
||||
"upstream normalization must not mutate local metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_should_keep_an_exact_chain_while_new_connections_adopt_the_latest_codex_core_wire_profile()
|
||||
{
|
||||
|
||||
@@ -60,13 +60,13 @@ impl CodexCliReleaseTransport for CliReleaseTransport {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_profile_should_generate_codex_core_user_agent() {
|
||||
fn wire_profile_should_generate_bundled_core_app_server_user_agent() {
|
||||
let profile = CodexWireProfile {
|
||||
originator: "Codex Desktop".to_owned(),
|
||||
codex_version: "0.144.2".to_owned(),
|
||||
desktop_version: "26.707.72221".to_owned(),
|
||||
desktop_build: "72221".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
@@ -78,7 +78,7 @@ fn wire_profile_should_generate_codex_core_user_agent() {
|
||||
|
||||
assert_eq!(
|
||||
profile.user_agent(),
|
||||
"Codex Desktop/0.144.2 (macOS 15.7.1; arm64) unknown"
|
||||
"Codex Desktop/0.144.2 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.707.72221)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@ fn desktop_appcast_should_reject_invalid_version_and_build() {
|
||||
#[tokio::test]
|
||||
async fn wire_profile_release_update_should_change_only_desktop_identity() {
|
||||
let original = wire_profile();
|
||||
let original_user_agent = original.user_agent();
|
||||
let state = CodexWireProfileState::new(original.clone());
|
||||
let service = CodexDesktopReleaseService::new(
|
||||
state.clone(),
|
||||
@@ -155,6 +156,11 @@ async fn wire_profile_release_update_should_change_only_desktop_identity() {
|
||||
assert_eq!(updated.arch, original.arch);
|
||||
assert_eq!(updated.terminal, original.terminal);
|
||||
assert_eq!(updated.verified_at, original.verified_at);
|
||||
assert_ne!(updated.user_agent(), original_user_agent);
|
||||
assert_eq!(
|
||||
updated.user_agent(),
|
||||
"Codex Desktop/0.144.2 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.720.1)"
|
||||
);
|
||||
let status = service.status().snapshot();
|
||||
assert_eq!(status.latest, Some(release("26.720.1", "72001")));
|
||||
assert!(status.checked_at.is_some());
|
||||
@@ -164,6 +170,7 @@ async fn wire_profile_release_update_should_change_only_desktop_identity() {
|
||||
#[tokio::test]
|
||||
async fn cli_release_update_should_change_only_codex_core_identity() {
|
||||
let original = wire_profile();
|
||||
let original_user_agent = original.user_agent();
|
||||
let state = CodexWireProfileState::new(original.clone());
|
||||
let service = CodexCliReleaseService::new(
|
||||
state.clone(),
|
||||
@@ -182,6 +189,11 @@ async fn cli_release_update_should_change_only_codex_core_identity() {
|
||||
assert_eq!(updated.arch, original.arch);
|
||||
assert_eq!(updated.terminal, original.terminal);
|
||||
assert_eq!(updated.verified_at, original.verified_at);
|
||||
assert_ne!(updated.user_agent(), original_user_agent);
|
||||
assert_eq!(
|
||||
updated.user_agent(),
|
||||
"Codex Desktop/0.145.0 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.707.72221)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -219,7 +231,7 @@ fn wire_profile() -> CodexWireProfile {
|
||||
codex_version: "0.144.2".to_owned(),
|
||||
desktop_version: "26.707.72221".to_owned(),
|
||||
desktop_build: "72221".to_owned(),
|
||||
os_type: "macOS".to_owned(),
|
||||
os_type: "Mac OS".to_owned(),
|
||||
os_version: "15.7.1".to_owned(),
|
||||
arch: "arm64".to_owned(),
|
||||
terminal: "unknown".to_owned(),
|
||||
|
||||
@@ -401,6 +401,41 @@ fn observability_semantics_should_reuse_codex_turn_metadata() {
|
||||
assert!(semantics.compact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_extract_subagent_kind_from_wire_or_turn_metadata() {
|
||||
let metadata_request = request(Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
(
|
||||
"client_metadata".to_owned(),
|
||||
json!({"x-openai-subagent": "review"}),
|
||||
),
|
||||
]));
|
||||
let metadata_encoded =
|
||||
encode_generate_request(&metadata_request, "gpt-test").expect("encode metadata request");
|
||||
assert_eq!(metadata_encoded.subagent_kind().as_deref(), Some("review"));
|
||||
|
||||
let turn_metadata_payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"turn_metadata".to_owned(),
|
||||
Value::String(r#"{"subagent_kind":"worker"}"#.to_owned()),
|
||||
)]));
|
||||
let turn_metadata_request = GenerateRequest::from_protocol_payload(turn_metadata_payload);
|
||||
let turn_metadata_encoded = encode_generate_request(&turn_metadata_request, "gpt-test")
|
||||
.expect("encode turn metadata request");
|
||||
assert_eq!(
|
||||
turn_metadata_encoded.subagent_kind().as_deref(),
|
||||
Some("worker")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observability_semantics_should_use_the_transparent_openai_payload() {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
|
||||
@@ -163,7 +163,12 @@ async fn fetch_should_use_wham_usage_headers_only() {
|
||||
"unexpected {name} header"
|
||||
);
|
||||
}
|
||||
assert!(headers.get("user-agent").is_some());
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("user-agent")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("codex_cli_rs/0.144.0 (linux 6.8; x86_64) xterm (codex_cli_rs; 1.0.0)")
|
||||
);
|
||||
let quota_header_names = headers
|
||||
.keys()
|
||||
.map(|name| name.as_str())
|
||||
|
||||
@@ -1048,6 +1048,7 @@ fn project_quota(
|
||||
refresh_token_expires_at,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
};
|
||||
@@ -1115,6 +1116,7 @@ fn project_quota(
|
||||
limit_reached: billing
|
||||
.used_percent()
|
||||
.is_some_and(|used| used.is_finite() && used >= 100.0),
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ impl GrokOAuthClient {
|
||||
/// # Errors
|
||||
///
|
||||
/// 返回已分类的 OAuth 错误。refresh token 可能已轮换,Ambiguous 类
|
||||
/// transport 失败不得自动重试。
|
||||
/// transport 失败不在本次 exchange 内重试,后续交由 refresh scheduler 退避协调。
|
||||
pub async fn refresh(
|
||||
&self,
|
||||
discovery: &DiscoveryDocument,
|
||||
|
||||
@@ -37,7 +37,7 @@ pub enum OAuthErrorCode {
|
||||
pub enum FailureClass {
|
||||
/// 瞬时失败,允许后续独立协调的重试。
|
||||
Transient,
|
||||
/// 发送状态不确定,一次性凭据不得重放。
|
||||
/// 发送状态不确定,不应在同一 exchange 内重放一次性凭据。
|
||||
Ambiguous,
|
||||
/// 当前 revision 的凭据被永久拒绝。
|
||||
CredentialPermanent,
|
||||
|
||||
@@ -12,10 +12,10 @@ use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, LoadedCredential, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCredentialStatePort, ProviderLeaseAcquisition,
|
||||
ProviderLeasePort, ProviderLeaseRequest, ProviderRefreshCapacityRequest,
|
||||
ProviderRefreshLeaseRequest, ProviderRefreshPolicy, ProviderRuntimePolicyPort,
|
||||
ProviderStoreError, provider_refresh_backoff_at,
|
||||
ProviderCredentialStatePort, ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
ProviderRuntimePolicyPort, ProviderStoreError, provider_refresh_backoff_at,
|
||||
provider_refresh_retry_at,
|
||||
};
|
||||
|
||||
use super::catalog::GrokCredentialCatalogService;
|
||||
@@ -38,12 +38,49 @@ const MAX_SECRET_BYTES: usize = 64 * 1_024;
|
||||
const DISCOVERY_NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(5);
|
||||
/// 连续失败计数窗口;静默满窗后计数过期归零。
|
||||
const REFRESH_BACKOFF_WINDOW: Duration = Duration::from_secs(30 * 60);
|
||||
const REFRESH_AMBIGUITY_COOLDOWN: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
/// 窗口内先执行五次指数退避,再进入固定恢复周期。
|
||||
const REFRESH_BACKOFF_MAX_ATTEMPTS: u32 = 5;
|
||||
/// 耗尽指数退避后的 OAuth 恢复周期。
|
||||
const REFRESH_RECOVERY_DELAY: Duration = Duration::from_secs(10 * 60);
|
||||
/// 过期 AT 仍允许 RT 恢复的最长窗口。
|
||||
const REFRESH_RECOVERY_WINDOW: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
fn refresh_recovery_deadline(access_token_expires_at: Option<SystemTime>) -> Option<SystemTime> {
|
||||
access_token_expires_at.and_then(|expires_at| expires_at.checked_add(REFRESH_RECOVERY_WINDOW))
|
||||
}
|
||||
|
||||
fn refresh_recovery_window_exhausted(
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
refresh_recovery_deadline(access_token_expires_at).is_some_and(|deadline| deadline <= now)
|
||||
}
|
||||
|
||||
fn bounded_refresh_retry_at(
|
||||
retry_at: SystemTime,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> SystemTime {
|
||||
let mut bounded = retry_at;
|
||||
if let Some(expires_at) = access_token_expires_at
|
||||
&& expires_at > observed_at
|
||||
&& bounded > expires_at
|
||||
{
|
||||
bounded = expires_at;
|
||||
}
|
||||
if let Some(deadline) = refresh_recovery_deadline(access_token_expires_at)
|
||||
&& bounded > deadline
|
||||
{
|
||||
bounded = deadline;
|
||||
}
|
||||
bounded
|
||||
}
|
||||
|
||||
/// 一个到期且已按 revision 读取明文 RT 的 xAI account。
|
||||
pub struct DueGrokCredential {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
refresh_token: SecretValue,
|
||||
id_token: Option<SecretValue>,
|
||||
scope: String,
|
||||
@@ -77,6 +114,7 @@ impl fmt::Debug for DueGrokCredential {
|
||||
.debug_struct("DueGrokCredential")
|
||||
.field("account_id", &self.account_id)
|
||||
.field("credential_revision", &self.credential_revision)
|
||||
.field("access_token_expires_at", &self.access_token_expires_at)
|
||||
.field("refresh_token", &"[REDACTED]")
|
||||
.field("id_token", &self.id_token.as_ref().map(|_| "[REDACTED]"))
|
||||
.field("scope", &"[REDACTED]")
|
||||
@@ -277,7 +315,6 @@ pub struct GrokCredentialRefreshService {
|
||||
refresher: Arc<dyn GrokCredentialRefresher>,
|
||||
catalog: Arc<GrokCredentialCatalogService>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
}
|
||||
@@ -288,7 +325,6 @@ impl GrokCredentialRefreshService {
|
||||
refresher: Arc<dyn GrokCredentialRefresher>,
|
||||
catalog: Arc<GrokCredentialCatalogService>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
) -> Self {
|
||||
@@ -297,7 +333,6 @@ impl GrokCredentialRefreshService {
|
||||
refresher,
|
||||
catalog,
|
||||
leases,
|
||||
cooldowns,
|
||||
credential_state,
|
||||
runtime_policy,
|
||||
}
|
||||
@@ -331,6 +366,7 @@ impl GrokCredentialRefreshService {
|
||||
let credential = DueGrokCredential {
|
||||
account_id: account_id.clone(),
|
||||
credential_revision,
|
||||
access_token_expires_at: loaded.account.access_token_expires_at(),
|
||||
refresh_token: loaded.refresh_token,
|
||||
id_token: loaded.id_token,
|
||||
scope: loaded.scope,
|
||||
@@ -447,12 +483,7 @@ impl GrokCredentialRefreshService {
|
||||
.into_iter()
|
||||
.map(|account_id| GrokCredentialRefreshOutcome::Failed { account_id })
|
||||
.collect::<Vec<_>>();
|
||||
let mut credentials = Vec::with_capacity(batch.credentials.len());
|
||||
for credential in batch.credentials {
|
||||
if !self.runtime_cooldown_active(&credential).await {
|
||||
credentials.push(credential);
|
||||
}
|
||||
}
|
||||
let credentials = batch.credentials;
|
||||
if credentials.is_empty() {
|
||||
return Ok(outcomes);
|
||||
}
|
||||
@@ -474,7 +505,14 @@ impl GrokCredentialRefreshService {
|
||||
for (account_id, result) in refreshed {
|
||||
match result {
|
||||
Ok(outcome) => outcomes.push(outcome),
|
||||
Err(_) => outcomes.push(GrokCredentialRefreshOutcome::Failed { account_id }),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
error = %error,
|
||||
"xAI OAuth refresh attempt failed"
|
||||
);
|
||||
outcomes.push(GrokCredentialRefreshOutcome::Failed { account_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(outcomes)
|
||||
@@ -486,9 +524,10 @@ impl GrokCredentialRefreshService {
|
||||
policy: ProviderRefreshPolicy,
|
||||
) -> Result<GrokCredentialRefreshOutcome, GrokCredentialRefreshError> {
|
||||
let account_id = credential.account_id.clone();
|
||||
let observed_at = SystemTime::now();
|
||||
if credential
|
||||
.refresh_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= Utc::now())
|
||||
.is_some_and(|expires_at| expires_at <= DateTime::<Utc>::from(observed_at))
|
||||
{
|
||||
return self
|
||||
.persist_terminal_failure(
|
||||
@@ -498,6 +537,15 @@ impl GrokCredentialRefreshService {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if refresh_recovery_window_exhausted(credential.access_token_expires_at, observed_at) {
|
||||
return self
|
||||
.persist_terminal_failure(
|
||||
credential,
|
||||
GrokCredentialAvailability::Expired,
|
||||
"refresh_recovery_window_exhausted",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let capacity = self
|
||||
.leases
|
||||
.try_acquire(ProviderLeaseRequest::RefreshCapacity(
|
||||
@@ -545,9 +593,14 @@ impl GrokCredentialRefreshService {
|
||||
.await
|
||||
}
|
||||
Err(GrokRefreshFailure::Ambiguous) => {
|
||||
self.record_runtime_cooldown(&credential, REFRESH_AMBIGUITY_COOLDOWN)
|
||||
.await;
|
||||
Ok(GrokCredentialRefreshOutcome::Ambiguous { account_id })
|
||||
if self
|
||||
.persist_backoff(&credential, "refresh-ambiguous")
|
||||
.await?
|
||||
{
|
||||
Ok(GrokCredentialRefreshOutcome::Ambiguous { account_id })
|
||||
} else {
|
||||
Ok(GrokCredentialRefreshOutcome::Stale { account_id })
|
||||
}
|
||||
}
|
||||
Err(GrokRefreshFailure::Transient) => {
|
||||
if self
|
||||
@@ -644,35 +697,19 @@ impl GrokCredentialRefreshService {
|
||||
.credential_state
|
||||
.clear_refresh_backoff(&account_id)
|
||||
.await;
|
||||
tracing::info!(
|
||||
account_id = %account_id,
|
||||
credential_revision = record.credential_revision.get(),
|
||||
access_token_expires_at = %access_expires_at,
|
||||
next_refresh_at = %next_refresh_at,
|
||||
"xAI OAuth refresh succeeded"
|
||||
);
|
||||
Ok(GrokCredentialRefreshOutcome::Refreshed {
|
||||
account_id,
|
||||
credential_revision: record.credential_revision,
|
||||
})
|
||||
}
|
||||
|
||||
async fn runtime_cooldown_active(&self, credential: &DueGrokCredential) -> bool {
|
||||
// runtime cooldown 是账号/model 级生命周期,与 credential revision 无关:
|
||||
// 轮换不清除、不因 revision 变化判失效(与 selector 的读取语义一致)。
|
||||
let Ok(Some(cooldown)) = self.cooldowns.read(credential.account_id()).await else {
|
||||
return false;
|
||||
};
|
||||
cooldown.until() > SystemTime::now()
|
||||
}
|
||||
|
||||
async fn record_runtime_cooldown(&self, credential: &DueGrokCredential, duration: Duration) {
|
||||
let cooldown_until = SystemTime::now()
|
||||
.checked_add(duration)
|
||||
.unwrap_or_else(SystemTime::now);
|
||||
let _ = self
|
||||
.cooldowns
|
||||
.put_if_later(ProviderCooldown::new(
|
||||
credential.account_id.clone(),
|
||||
credential.credential_revision,
|
||||
cooldown_until,
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn persist_terminal_failure(
|
||||
&self,
|
||||
credential: DueGrokCredential,
|
||||
@@ -702,7 +739,19 @@ impl GrokCredentialRefreshService {
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(GrokCredentialRefreshOutcome::Invalidated { account_id }),
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %account_id,
|
||||
?availability,
|
||||
reason,
|
||||
access_token_expires_at = ?credential.access_token_expires_at
|
||||
.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(credential.access_token_expires_at)
|
||||
.map(DateTime::<Utc>::from),
|
||||
"xAI OAuth refresh marked account terminal"
|
||||
);
|
||||
Ok(GrokCredentialRefreshOutcome::Invalidated { account_id })
|
||||
}
|
||||
Err(error) if stale_repository_error(&error) => {
|
||||
Ok(GrokCredentialRefreshOutcome::Stale { account_id })
|
||||
}
|
||||
@@ -720,12 +769,19 @@ impl GrokCredentialRefreshService {
|
||||
.record_refresh_backoff(&credential.account_id, REFRESH_BACKOFF_WINDOW)
|
||||
.await
|
||||
.unwrap_or(1);
|
||||
let retry_at = provider_refresh_backoff_at(
|
||||
&credential.account_id,
|
||||
SystemTime::now(),
|
||||
attempt,
|
||||
reason,
|
||||
)?;
|
||||
let observed_at = SystemTime::now();
|
||||
let retry_at = if attempt <= REFRESH_BACKOFF_MAX_ATTEMPTS {
|
||||
provider_refresh_backoff_at(&credential.account_id, observed_at, attempt, reason)?
|
||||
} else {
|
||||
provider_refresh_retry_at(
|
||||
&credential.account_id,
|
||||
observed_at,
|
||||
REFRESH_RECOVERY_DELAY,
|
||||
reason,
|
||||
)?
|
||||
};
|
||||
let retry_at =
|
||||
bounded_refresh_retry_at(retry_at, credential.access_token_expires_at, observed_at);
|
||||
match self
|
||||
.repository
|
||||
.defer_refresh(
|
||||
@@ -735,7 +791,20 @@ impl GrokCredentialRefreshService {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(true),
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
account_id = %credential.account_id,
|
||||
attempt,
|
||||
reason,
|
||||
retry_at = %DateTime::<Utc>::from(retry_at),
|
||||
access_token_expires_at = ?credential.access_token_expires_at
|
||||
.map(DateTime::<Utc>::from),
|
||||
recovery_deadline = ?refresh_recovery_deadline(credential.access_token_expires_at)
|
||||
.map(DateTime::<Utc>::from),
|
||||
"xAI OAuth refresh deferred"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(error) if stale_repository_error(&error) => Ok(false),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
@@ -814,6 +883,7 @@ impl GrokCredentialRepository {
|
||||
due.push(DueGrokCredential {
|
||||
account_id: account.id().clone(),
|
||||
credential_revision: account.revision(),
|
||||
access_token_expires_at: account.access_token_expires_at(),
|
||||
refresh_token: loaded.refresh_token,
|
||||
id_token: loaded.id_token,
|
||||
scope: loaded.scope,
|
||||
@@ -844,7 +914,8 @@ fn account_due(
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
)
|
||||
&& account.next_refresh_at().is_some_and(|next| next <= now)
|
||||
&& (refresh_recovery_window_exhausted(account.access_token_expires_at(), now)
|
||||
|| account.next_refresh_at().is_some_and(|next| next <= now))
|
||||
}
|
||||
|
||||
fn refreshed_tokens(tokens: RefreshedTokenSet) -> Result<GrokRefreshTokens, GrokRefreshFailure> {
|
||||
|
||||
@@ -6,10 +6,10 @@ use std::time::SystemTime;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountStateChange, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
|
||||
LoadedCredential, NewProviderAccount, OpaqueProviderData, PlaintextCredential, ProviderAccount,
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, LoadedCredential, NewProviderAccount, OpaqueProviderData,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::provider_ports::ProviderRefreshPolicy;
|
||||
@@ -428,14 +428,21 @@ impl GrokCredentialRepository {
|
||||
&self,
|
||||
input: &UpdateGrokCredentialState,
|
||||
) -> Result<(), GrokCredentialRepositoryError> {
|
||||
validate_reason(input.availability_reason.as_deref())?;
|
||||
let availability = AccountAvailability::from(input.availability);
|
||||
let availability_reason = input
|
||||
.availability_reason
|
||||
.clone()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.or_else(|| availability.fallback_error_reason().map(str::to_owned));
|
||||
validate_reason(availability_reason.as_deref())?;
|
||||
self.ensure_xai_account(&input.account_id).await?;
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: input.expected_revision,
|
||||
availability: input.availability.into(),
|
||||
availability,
|
||||
observed_at: to_system_time(input.observed_at),
|
||||
message: availability_reason,
|
||||
})
|
||||
.await
|
||||
.map_err(map_store_error)
|
||||
|
||||
@@ -171,7 +171,6 @@ pub async fn initialize(
|
||||
refresher,
|
||||
Arc::clone(&catalog),
|
||||
Arc::clone(&leases),
|
||||
cooldowns.clone(),
|
||||
ports.credential_state(),
|
||||
Arc::clone(&runtime_policy),
|
||||
));
|
||||
|
||||
@@ -2040,11 +2040,10 @@ impl ScheduledTask for XaiOAuthRefreshTask {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let outcomes = self
|
||||
.service
|
||||
.refresh_due()
|
||||
.await
|
||||
.map_err(|_| WorkerTaskError::safe("xAI OAuth refresh failed"))?;
|
||||
let outcomes = self.service.refresh_due().await.map_err(|error| {
|
||||
tracing::error!(error = %error, "xAI OAuth refresh cycle failed");
|
||||
WorkerTaskError::safe("xAI OAuth refresh failed")
|
||||
})?;
|
||||
let failures = outcomes
|
||||
.iter()
|
||||
.filter(|outcome| {
|
||||
@@ -2056,6 +2055,13 @@ impl ScheduledTask for XaiOAuthRefreshTask {
|
||||
)
|
||||
})
|
||||
.count();
|
||||
if !outcomes.is_empty() {
|
||||
tracing::info!(
|
||||
accounts = outcomes.len(),
|
||||
failures,
|
||||
"xAI OAuth refresh cycle completed"
|
||||
);
|
||||
}
|
||||
if failures > 0 {
|
||||
tracing::warn!(failures, "xAI OAuth refresh cycle contained failures");
|
||||
}
|
||||
|
||||
@@ -473,6 +473,7 @@ fn account_record(account: &ProviderAccount) -> AccountRecord {
|
||||
enabled: account.enabled(),
|
||||
availability: account.availability(),
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
|
||||
@@ -204,6 +204,7 @@ async fn set_account_state(
|
||||
) {
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability,
|
||||
@@ -485,6 +486,7 @@ async fn quota_exhausted_account_remains_eligible_for_catalog_discovery() {
|
||||
let id = account_id("quota-exhausted");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id,
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
@@ -755,6 +757,7 @@ async fn quota_refresh_preserves_ready_availability_across_concurrent_state_writ
|
||||
let transport = Arc::new(MutatingBillingTransport {
|
||||
store: Arc::clone(&store),
|
||||
mutation: Mutex::new(Some(BillingMutation::State(AccountStateChange {
|
||||
message: None,
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
|
||||
@@ -233,6 +233,29 @@ async fn state_update_uses_credential_revision_fence() {
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_state_without_upstream_message_uses_stable_error_reason() {
|
||||
let (store, repository) = repository();
|
||||
let input = create_input("state-fallback", "subject-state-fallback");
|
||||
seed_input(&store, &input).await.expect("create");
|
||||
|
||||
repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: input.account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
availability: GrokCredentialAvailability::Banned,
|
||||
availability_reason: None,
|
||||
observed_at: Utc::now(),
|
||||
})
|
||||
.await
|
||||
.expect("state update");
|
||||
|
||||
assert_eq!(
|
||||
store.last_error_message(&input.account_id).as_deref(),
|
||||
Some("account_banned")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_prepare_does_not_mutate_provider_account_store() {
|
||||
let (store, _) = repository();
|
||||
|
||||
@@ -10,8 +10,8 @@ use gateway_core::engine::credential::{
|
||||
ProviderAccountId, ProviderAccountStore, ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCredentialState, ProviderCredentialStatePort,
|
||||
ProviderLeaseAcquisition, ProviderLeasePort, ProviderLeaseRequest, ProviderStoreError,
|
||||
ProviderCredentialState, ProviderCredentialStatePort, ProviderLeaseAcquisition,
|
||||
ProviderLeasePort, ProviderLeaseRequest, ProviderStoreError,
|
||||
};
|
||||
use provider_xai::{
|
||||
GrokCredentialCatalogCache, GrokCredentialRecovery, GrokCredentialRecoveryOutcome,
|
||||
@@ -22,12 +22,13 @@ use provider_xai::{
|
||||
};
|
||||
|
||||
use crate::support::{
|
||||
MemoryCooldownPort, MemoryGrokCatalogCache, MemoryProviderAccountStore, create_input,
|
||||
credential_object, runtime_policy, seed_input,
|
||||
MemoryGrokCatalogCache, MemoryProviderAccountStore, create_input, credential_object,
|
||||
runtime_policy, seed_input,
|
||||
};
|
||||
|
||||
const OFFICIAL_FIXTURE: &[u8] =
|
||||
include_bytes!("../transport/catalog/fixtures/official_grok_models_snapshot.json");
|
||||
const OAUTH_BACKOFF_ATTEMPTS: u32 = 5;
|
||||
|
||||
struct StaticCatalogTransport;
|
||||
|
||||
@@ -143,48 +144,6 @@ async fn fixture(
|
||||
fixture_many([input], responses, lease_available).await
|
||||
}
|
||||
|
||||
/// 带外部注入 cooldown 端口的 fixture(AUD-05 组合测试用)。
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn fixture_with_cooldowns(
|
||||
input: provider_xai::CreateGrokCredential,
|
||||
responses: impl IntoIterator<Item = Result<GrokRefreshTokens, GrokRefreshFailure>>,
|
||||
lease_available: bool,
|
||||
cooldowns: Arc<MemoryCooldownPort>,
|
||||
) -> (
|
||||
Arc<MemoryProviderAccountStore>,
|
||||
GrokCredentialRepository,
|
||||
Arc<QueueRefresher>,
|
||||
GrokCredentialRefreshService,
|
||||
Arc<MemoryCooldownPort>,
|
||||
) {
|
||||
let store = MemoryProviderAccountStore::shared();
|
||||
let account_store: Arc<dyn ProviderAccountStore> = store.clone();
|
||||
let repository = GrokCredentialRepository::new(account_store);
|
||||
seed_input(&store, &input).await.expect("create account");
|
||||
let refresher = QueueRefresher::new(responses);
|
||||
let refresher_port: Arc<dyn GrokCredentialRefresher> = refresher.clone();
|
||||
let cache: Arc<dyn GrokCredentialCatalogCache> = MemoryGrokCatalogCache::shared();
|
||||
let catalog = Arc::new(crate::support::grok_catalog_service(
|
||||
repository.clone(),
|
||||
Arc::new(StaticCatalogTransport),
|
||||
cache,
|
||||
));
|
||||
let leases = Arc::new(TestRefreshLeases {
|
||||
available: lease_available,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let service = GrokCredentialRefreshService::new(
|
||||
repository.clone(),
|
||||
refresher_port,
|
||||
catalog,
|
||||
leases,
|
||||
cooldowns.clone(),
|
||||
Arc::new(CountingCredentialState::default()),
|
||||
runtime_policy(),
|
||||
);
|
||||
(store, repository, refresher, service, cooldowns)
|
||||
}
|
||||
|
||||
/// 真实累加连续失败计数的测试 double,用于断言退避的指数增长与成功清零。
|
||||
#[derive(Default)]
|
||||
struct CountingCredentialState {
|
||||
@@ -313,13 +272,11 @@ async fn fixture_many_with_state(
|
||||
available: lease_available,
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::default());
|
||||
let service = GrokCredentialRefreshService::new(
|
||||
repository.clone(),
|
||||
refresher_port,
|
||||
catalog,
|
||||
leases,
|
||||
cooldowns,
|
||||
credential_state,
|
||||
runtime_policy(),
|
||||
);
|
||||
@@ -354,6 +311,38 @@ async fn force_due(store: &MemoryProviderAccountStore, id: &ProviderAccountId) {
|
||||
));
|
||||
}
|
||||
|
||||
async fn set_token_deadlines(
|
||||
store: &MemoryProviderAccountStore,
|
||||
id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
next_refresh_at: SystemTime,
|
||||
) {
|
||||
let account = store.account(id).expect("account to update");
|
||||
let credential = store.credential(id).expect("credential to update");
|
||||
let update = CredentialCasUpdate::new(
|
||||
id.clone(),
|
||||
account.revision(),
|
||||
ProviderAccountUpdate {
|
||||
account_id: id.clone(),
|
||||
name: account.name().to_owned(),
|
||||
email: account.email().map(str::to_owned),
|
||||
plan_type: account.plan_type().map(str::to_owned),
|
||||
},
|
||||
credential,
|
||||
true,
|
||||
Some(access_token_expires_at),
|
||||
Some(next_refresh_at),
|
||||
)
|
||||
.expect("valid token deadline update");
|
||||
assert!(matches!(
|
||||
store
|
||||
.compare_and_swap_credential(update)
|
||||
.await
|
||||
.expect("update token deadlines"),
|
||||
CredentialCasOutcome::Updated(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_refresh_rotates_plaintext_tokens_once() {
|
||||
let input = due_input("success");
|
||||
@@ -415,6 +404,10 @@ async fn unauthorized_recovery_marks_a_permanently_rejected_refresh_expired() {
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Expired
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(&id).as_deref(),
|
||||
Some("refresh_invalid_grant")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -573,14 +566,25 @@ async fn banned_failure_marks_account_banned() {
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Banned
|
||||
);
|
||||
assert_eq!(
|
||||
store.last_error_message(&id).as_deref(),
|
||||
Some("account_banned")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ambiguous_refresh_uses_runtime_cooldown_without_persisting_account_state() {
|
||||
async fn ambiguous_refresh_applies_bounded_oauth_backoff() {
|
||||
let input = due_input("ambiguous");
|
||||
let id = input.account_id.clone();
|
||||
let (store, _, refresher, service) =
|
||||
fixture(input, [Err(GrokRefreshFailure::Ambiguous)], true).await;
|
||||
let counting = Arc::new(CountingCredentialState::default());
|
||||
let (store, _, refresher, service) = fixture_many_with_state(
|
||||
[input],
|
||||
[Err(GrokRefreshFailure::Ambiguous)],
|
||||
true,
|
||||
counting.clone(),
|
||||
)
|
||||
.await;
|
||||
let started_at = SystemTime::now();
|
||||
let outcomes = service.refresh_due().await.expect("refresh");
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
@@ -590,7 +594,18 @@ async fn ambiguous_refresh_uses_runtime_cooldown_without_persisting_account_stat
|
||||
store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::Unknown
|
||||
);
|
||||
assert_eq!(store.account(&id).expect("account").revision().get(), 1);
|
||||
let account = store.account(&id).expect("account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
let retry_delay = account
|
||||
.next_refresh_at()
|
||||
.expect("OAuth retry scheduled")
|
||||
.duration_since(started_at)
|
||||
.expect("OAuth retry is in the future");
|
||||
assert!(
|
||||
retry_delay <= Duration::from_secs(7),
|
||||
"first retry must use bounded OAuth backoff, got {retry_delay:?}"
|
||||
);
|
||||
assert_eq!(counting.count(&id), 1);
|
||||
assert!(
|
||||
service
|
||||
.refresh_due()
|
||||
@@ -602,7 +617,7 @@ async fn ambiguous_refresh_uses_runtime_cooldown_without_persisting_account_stat
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pre_send_transient_failure_applies_bounded_cooldown() {
|
||||
async fn pre_send_transient_failure_applies_bounded_oauth_backoff() {
|
||||
let input = due_input("transient");
|
||||
let id = input.account_id.clone();
|
||||
let (store, _, _, service) = fixture(input, [Err(GrokRefreshFailure::Transient)], true).await;
|
||||
@@ -672,6 +687,121 @@ async fn refresh_backoff_grows_exponentially_across_attempts() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausted_oauth_backoff_continues_after_access_token_expiry() {
|
||||
let input = due_input("backoff-recovery");
|
||||
let id = input.account_id.clone();
|
||||
let counting = Arc::new(CountingCredentialState::default());
|
||||
let (store, _, refresher, service) = fixture_many_with_state(
|
||||
[input],
|
||||
[
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
],
|
||||
true,
|
||||
counting.clone(),
|
||||
)
|
||||
.await;
|
||||
set_token_deadlines(
|
||||
&store,
|
||||
&id,
|
||||
(Utc::now() - chrono::Duration::minutes(1)).into(),
|
||||
SystemTime::now() - Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
|
||||
for _ in 0..OAUTH_BACKOFF_ATTEMPTS {
|
||||
service.refresh_due().await.expect("backoff attempt");
|
||||
force_due(&store, &id).await;
|
||||
}
|
||||
|
||||
let observed_at = SystemTime::now();
|
||||
service.refresh_due().await.expect("recovery attempt");
|
||||
let recovery_delay = store
|
||||
.account(&id)
|
||||
.expect("account after recovery defer")
|
||||
.next_refresh_at()
|
||||
.expect("recovery scheduled")
|
||||
.duration_since(observed_at)
|
||||
.expect("recovery is in the future");
|
||||
assert!(
|
||||
(Duration::from_secs(8 * 60)..=Duration::from_secs(12 * 60 + 1)).contains(&recovery_delay),
|
||||
"recovery delay should be about ten minutes, got {recovery_delay:?}"
|
||||
);
|
||||
assert_eq!(counting.count(&id), OAUTH_BACKOFF_ATTEMPTS + 1);
|
||||
assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oauth_backoff_never_passes_an_unexpired_access_token_expiry() {
|
||||
let mut input = due_input("backoff-expiry-cap");
|
||||
input.account.access_token_expires_at = Utc::now() + chrono::Duration::seconds(30);
|
||||
let access_expires_at: SystemTime = input.account.access_token_expires_at.into();
|
||||
let id = input.account_id.clone();
|
||||
let counting = Arc::new(CountingCredentialState::default());
|
||||
let (store, _, _, service) = fixture_many_with_state(
|
||||
[input],
|
||||
[
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
Err(GrokRefreshFailure::Transient),
|
||||
],
|
||||
true,
|
||||
counting.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
for attempt in 0..=OAUTH_BACKOFF_ATTEMPTS {
|
||||
service.refresh_due().await.expect("backoff attempt");
|
||||
if attempt < OAUTH_BACKOFF_ATTEMPTS {
|
||||
force_due(&store, &id).await;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account(&id)
|
||||
.expect("account after capped retry")
|
||||
.next_refresh_at(),
|
||||
Some(access_expires_at)
|
||||
);
|
||||
assert_eq!(counting.count(&id), OAUTH_BACKOFF_ATTEMPTS + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_window_exhaustion_marks_account_expired_without_token_exchange() {
|
||||
let input = due_input("recovery-window-exhausted");
|
||||
let id = input.account_id.clone();
|
||||
let (store, _, refresher, service) = fixture(input, [Ok(success_tokens(None))], true).await;
|
||||
set_token_deadlines(
|
||||
&store,
|
||||
&id,
|
||||
(Utc::now() - chrono::Duration::hours(25)).into(),
|
||||
(Utc::now() + chrono::Duration::hours(12)).into(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[GrokCredentialRefreshOutcome::Invalidated { account_id }] if account_id == &id
|
||||
));
|
||||
assert_eq!(
|
||||
store.account(&id).expect("expired account").availability(),
|
||||
AccountAvailability::Expired
|
||||
);
|
||||
assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(refresher.responses.lock().expect("queue").len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_refresh_clears_backoff_counter() {
|
||||
let input = due_input("backoff-clear");
|
||||
@@ -776,52 +906,33 @@ async fn malformed_account_refresh_does_not_stop_later_accounts() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_cooldown_survives_credential_rotation_and_blocks_refresh() {
|
||||
// cooldown 写于 revision N,轮换到 N+1 后 refresh worker 不得清除
|
||||
// 或失效它;cooldown 活跃期间账号被跳过刷新。
|
||||
let input = create_input(
|
||||
"cooldown-survives-rotation",
|
||||
"subject-cooldown-survives-rotation",
|
||||
);
|
||||
async fn scheduled_oauth_refresh_runs_after_credential_rotation() {
|
||||
let input = due_input("refresh-after-rotation");
|
||||
let id = input.account_id.clone();
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::default());
|
||||
let (store, _, _, service, cooldowns) = fixture_with_cooldowns(
|
||||
let (store, _, refresher, service) = fixture(
|
||||
input,
|
||||
[Ok(success_tokens(Some("rotated-refresh")))],
|
||||
[
|
||||
Ok(success_tokens(Some("recovered-refresh"))),
|
||||
Ok(success_tokens(Some("scheduled-refresh"))),
|
||||
],
|
||||
true,
|
||||
cooldowns,
|
||||
)
|
||||
.await;
|
||||
// 先写 account cooldown(revision 1,未来到期)。
|
||||
let until = SystemTime::now() + Duration::from_secs(3600);
|
||||
cooldowns
|
||||
.put_if_later(ProviderCooldown::new(
|
||||
id.clone(),
|
||||
CredentialRevision::new(1).expect("revision"),
|
||||
until,
|
||||
))
|
||||
.await
|
||||
.expect("write cooldown");
|
||||
|
||||
// 轮换到 revision 2。
|
||||
let outcome = service
|
||||
.recover_unauthorized(&id, CredentialRevision::new(1).expect("revision"))
|
||||
.await;
|
||||
assert_eq!(outcome, GrokCredentialRecoveryOutcome::Recovered);
|
||||
assert_eq!(store.account(&id).expect("account").revision().get(), 2);
|
||||
|
||||
// cooldown 仍有效(轮换不清除)。
|
||||
assert!(
|
||||
cooldowns
|
||||
.cooldown(&id)
|
||||
.is_some_and(|cooldown| cooldown.until() > SystemTime::now()),
|
||||
"rotation must not clear account runtime cooldown"
|
||||
);
|
||||
// cooldown 活跃期间 refresh worker 跳过该账号(不清除、不刷新)。
|
||||
force_due(&store, &id).await;
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
assert!(
|
||||
outcomes.is_empty(),
|
||||
"cooldown-active account must be skipped by refresh worker"
|
||||
);
|
||||
assert_eq!(store.account(&id).expect("account").revision().get(), 2);
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[GrokCredentialRefreshOutcome::Refreshed {
|
||||
account_id,
|
||||
credential_revision,
|
||||
}] if account_id == &id && credential_revision.get() == 4
|
||||
));
|
||||
assert_eq!(refresher.prepare_calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
@@ -194,6 +194,7 @@ struct StoredAccount {
|
||||
account: ProviderAccount,
|
||||
credential: PlaintextCredential,
|
||||
quota: Option<QuotaObservation>,
|
||||
last_error_message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -214,6 +215,12 @@ impl MemoryProviderAccountStore {
|
||||
.map(|stored| stored.account.clone())
|
||||
}
|
||||
|
||||
pub fn last_error_message(&self, id: &ProviderAccountId) -> Option<String> {
|
||||
lock(&self.accounts)
|
||||
.get(id)
|
||||
.and_then(|stored| stored.last_error_message.clone())
|
||||
}
|
||||
|
||||
pub fn credential(&self, id: &ProviderAccountId) -> Option<PlaintextCredential> {
|
||||
lock(&self.accounts)
|
||||
.get(id)
|
||||
@@ -246,6 +253,7 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
account: account.account,
|
||||
credential: account.credential,
|
||||
quota: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
@@ -388,6 +396,11 @@ impl ProviderAccountStore for MemoryProviderAccountStore {
|
||||
plan_type: stored.account.plan_type().map(str::to_owned),
|
||||
},
|
||||
);
|
||||
if change.availability == AccountAvailability::Ready {
|
||||
stored.last_error_message = None;
|
||||
} else if let Some(message) = change.message {
|
||||
stored.last_error_message = Some(message);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1 @@
|
||||
54be07512951a561391b6e611355a1bfc119d82363253fb9bbdb225c9e3e39bb 0001_initial.sql
|
||||
f480ac4cb70f6963eee71c0b8a67777ebb492f85501c90ebe7544bee0b2c0525 0002_remove_account_cooldown.sql
|
||||
5cf5b5387c66b4d437d79c002cec46cb85196789060fa87c39e97a382487cac9 0004_fix_quota_limit_reached_null.sql
|
||||
a3f236d558e7598f69277520458c801ee43130f7704ca25455df468cf8899ff2 0003_materialize_quota_limit_reached.sql
|
||||
5f79e74c9a7a7366c39cd1afe43175eff6c44afe9217c0f26f33f173bd40c781 0001_initial.sql
|
||||
|
||||
@@ -131,9 +131,9 @@ create table provider_accounts (
|
||||
next_refresh_at timestamptz,
|
||||
enabled boolean not null default true,
|
||||
availability text not null default 'unknown',
|
||||
availability_reason text,
|
||||
cooldown_until timestamptz,
|
||||
provider_quota_json jsonb,
|
||||
quota_limit_reached boolean not null default false,
|
||||
last_error_message text,
|
||||
availability_observed_at timestamptz not null,
|
||||
quota_observed_at timestamptz,
|
||||
created_at timestamptz not null,
|
||||
@@ -163,23 +163,12 @@ create table provider_accounts (
|
||||
availability in (
|
||||
'unknown',
|
||||
'ready',
|
||||
'cooldown',
|
||||
'quota_exhausted',
|
||||
'expired',
|
||||
'banned',
|
||||
'invalid'
|
||||
)
|
||||
),
|
||||
constraint provider_accounts_cooldown_ck check (
|
||||
(
|
||||
availability != 'cooldown'
|
||||
or cooldown_until is not null
|
||||
)
|
||||
and (
|
||||
cooldown_until is null
|
||||
or availability in ('cooldown', 'quota_exhausted')
|
||||
)
|
||||
),
|
||||
constraint provider_accounts_time_ck check (
|
||||
created_at <= updated_at
|
||||
and availability_observed_at <= updated_at
|
||||
@@ -196,7 +185,7 @@ create unique index provider_accounts_upstream_identity_uq
|
||||
create index provider_accounts_runtime_idx
|
||||
on provider_accounts (provider_kind, enabled, id);
|
||||
create index provider_accounts_availability_idx
|
||||
on provider_accounts (availability, cooldown_until, id);
|
||||
on provider_accounts (availability, id);
|
||||
create index provider_accounts_access_expiry_idx
|
||||
on provider_accounts (access_token_expires_at, id);
|
||||
create index provider_accounts_refresh_due_idx
|
||||
@@ -235,8 +224,6 @@ create table model_requests (
|
||||
outcome text not null default 'running',
|
||||
client_status_code integer,
|
||||
upstream_status_code integer,
|
||||
-- Provider response IDs are opaque JSON strings. Bytes preserve embedded NUL
|
||||
-- and avoid imposing PostgreSQL text-index limits on protocol handles.
|
||||
client_response_id bytea,
|
||||
upstream_request_id text,
|
||||
upstream_response_id bytea,
|
||||
@@ -438,6 +425,7 @@ create index model_requests_running_deadline_idx
|
||||
create index model_requests_retention_idx
|
||||
on model_requests (completed_at)
|
||||
where outcome <> 'running';
|
||||
|
||||
-- 运行期警告与错误事件。
|
||||
create table ops_events (
|
||||
id text primary key,
|
||||
@@ -691,11 +679,9 @@ create unique index backup_records_active_uq
|
||||
create unique index backup_records_object_key_uq
|
||||
on backup_records (object_key)
|
||||
where object_key is not null;
|
||||
-- 同时服务 retentionDays 与 retentionCount。
|
||||
create index backup_records_scheduled_completed_idx
|
||||
on backup_records (completed_at desc, id desc)
|
||||
where trigger_kind = 'scheduled' and status = 'completed';
|
||||
-- 可清理记录(expires_at 已到期且非活跃)的扫描索引。
|
||||
create index backup_records_retention_idx
|
||||
on backup_records (expires_at)
|
||||
where expires_at is not null and status in ('completed', 'failed');
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
-- 移除账号级 cooldown 状态模型(照 v2 语义:429 限流不进 availability,由 quota 数据驱动)。
|
||||
--
|
||||
-- 存量迁移:
|
||||
-- - availability='cooldown' 的账号:429/Cloudflare 等临时冷却 → 'ready'
|
||||
-- (限流事实留在 provider_quota_json,由投影滚动兜底);
|
||||
-- 402 语义的 usage_limit_exhausted 且未到期 → 'quota_exhausted'。
|
||||
|
||||
-- 1. 迁移存量 cooldown 账号。
|
||||
update provider_accounts
|
||||
set availability = case
|
||||
when availability_reason = 'usage_limit_exhausted'
|
||||
and cooldown_until is not null
|
||||
and cooldown_until > now()
|
||||
then 'quota_exhausted'
|
||||
else 'ready'
|
||||
end
|
||||
where availability = 'cooldown';
|
||||
|
||||
-- 2. 删除依赖 cooldown_until 的约束与索引。
|
||||
alter table provider_accounts drop constraint provider_accounts_cooldown_ck;
|
||||
drop index provider_accounts_availability_idx;
|
||||
|
||||
-- 3. availability CHECK 去掉 'cooldown'。
|
||||
alter table provider_accounts drop constraint provider_accounts_availability_ck;
|
||||
alter table provider_accounts
|
||||
add constraint provider_accounts_availability_ck check (
|
||||
availability in (
|
||||
'unknown',
|
||||
'ready',
|
||||
'quota_exhausted',
|
||||
'expired',
|
||||
'banned',
|
||||
'invalid'
|
||||
)
|
||||
);
|
||||
|
||||
-- 4. 删除列。
|
||||
alter table provider_accounts drop column availability_reason;
|
||||
alter table provider_accounts drop column cooldown_until;
|
||||
|
||||
-- 5. 重建可用性索引(不含 cooldown_until)。
|
||||
create index provider_accounts_availability_idx
|
||||
on provider_accounts (availability, id);
|
||||
@@ -1,20 +0,0 @@
|
||||
-- 物化快照级限流事实:Dashboard 复用 Provider 解析结果,不再 SQL 解析 raw quota JSON。
|
||||
--
|
||||
-- `quota_limit_reached` 由 Provider 在写 quota 时计算(快照级:顶层或任一窗口触顶,
|
||||
-- 已过 reset 的窗口不计数),Admin/Dashboard 直接读该列。
|
||||
|
||||
alter table provider_accounts
|
||||
add column quota_limit_reached boolean not null default false;
|
||||
|
||||
-- 存量回填:从现有 raw JSON 推导(仅 primary_window 的 limit_reached + reset 未过期)。
|
||||
update provider_accounts
|
||||
set quota_limit_reached = coalesce(
|
||||
provider_quota_json is not null
|
||||
and provider_quota_json->'rate_limit'->>'limit_reached' = 'true'
|
||||
and (
|
||||
provider_quota_json->'rate_limit'->'primary_window'->>'reset_at' is null
|
||||
or (provider_quota_json->'rate_limit'->'primary_window'->>'reset_at')::bigint > extract(epoch from now())::bigint
|
||||
),
|
||||
false
|
||||
)
|
||||
where provider_quota_json is not null;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- 修正 0003 存量回填:raw JSON 缺 rate_limit 键时表达式为 NULL,
|
||||
-- 赋 NULL 违反 not-null 约束。0004 把仍为 NULL 的行兜底为 false
|
||||
-- (0003 已应用的环境,add column 用 default false 填充,通常无 NULL;
|
||||
-- 本迁移保证任何环境下该列都非 NULL)。
|
||||
|
||||
update provider_accounts
|
||||
set quota_limit_reached = coalesce(quota_limit_reached, false)
|
||||
where quota_limit_reached is null;
|
||||
+9
-3
@@ -111,9 +111,10 @@ PostgreSQL 是账号、Client Key、运行设置、请求记录与审计的权
|
||||
Provider schema 以明文 JSON 保存于 PostgreSQL。Redis 只保存可重建、可过期的协调状态,例如
|
||||
会话亲和、lease、cooldown、OAuth pending flow 与套餐模型目录 cache。
|
||||
|
||||
备份必须包含 `.runtime/postgres`。若希望保留短期 Redis 状态、OpenAI 会话亲和和更新状态,
|
||||
同时备份 `.runtime/redis` 与 `.runtime/data`;后两者丢失不会造成账号 credential 丢失,但会使
|
||||
缓存和会话锚点重新建立。
|
||||
备份必须包含 `.runtime/postgres`。要保留 OpenAI OAuth 的 AT/RT 恢复能力,必须保持
|
||||
`host.logging.file.enabled: true` 并备份 `.runtime/logs`;该记录受普通日志的 `retention_days` 和
|
||||
`max_files` 约束。若希望保留短期 Redis 状态和会话锚点,也同时备份 `.runtime/redis` 与
|
||||
`.runtime/data`。
|
||||
|
||||
完整运行时、Provider、revision 与恢复边界见 [架构文档](../docs/architecture.md)。
|
||||
|
||||
@@ -202,6 +203,11 @@ Release 必须提供当前 OS/架构的 `codex-proxy-rs_<version>_<os>_<arch>.ta
|
||||
Debian 官方源补 `libpq5` 依赖,不引入第三方 APT 源,也不依赖运行时动态安装。
|
||||
- 备份暂存目录为 `/app/.runtime/data/backup-staging`,由 `.runtime/data` 卷持久化,权限
|
||||
`0700`(仅 `cpr` 用户可读写)。部署卷至少预留一个最大数据库归档的空间。
|
||||
- OpenAI OAuth 每次成功取得 AT/RT 后,都会在账号资料补全、过期时间计算和数据库写入前,以
|
||||
`openai_oauth_recovery` 结构化日志事件写入普通应用日志。事件含原始 AT/RT,完全沿用
|
||||
`.runtime/logs` 的按日、按大小分割和保留清理策略;文件日志开启时不会被普通日志级别筛掉。
|
||||
它遵循普通日志现有的非阻塞写入机制,不会阻断导入、授权或刷新。
|
||||
`.runtime/logs` 因此属于敏感数据,必须按现有普通日志的访问控制和加密备份策略处理。
|
||||
- S3/R2 存储、Cron 计划、保留策略与备份记录都保存在 PostgreSQL(`backup_settings` /
|
||||
`backup_records`),备份记录行在删除成功后硬删除,操作历史进入 `admin_audit_events`。
|
||||
|
||||
|
||||
@@ -57,16 +57,17 @@ api:
|
||||
openai:
|
||||
# 上游地址、WebSocket 池、额度刷新和 OAuth 使用 Provider 内置默认值;模板只保留画像启动基线。
|
||||
wire_profile:
|
||||
# Provider 每 24 小时读取官方 npm metadata 与 Desktop appcast,仅在进程内更新版本字段。
|
||||
# codex_version 是 npm CLI 的启动基线,并用于模型目录与请求 UA;Provider 每 24 小时更新它。
|
||||
# Desktop appcast 同期只更新 Desktop 版本与构建号。
|
||||
originator: 'Codex Desktop'
|
||||
codex_version: '0.146.0'
|
||||
desktop_version: '26.727.51351'
|
||||
desktop_build: '6119'
|
||||
os_type: 'macOS'
|
||||
os_type: 'Mac OS'
|
||||
os_version: '15.7.1'
|
||||
arch: 'arm64'
|
||||
terminal: 'unknown'
|
||||
verified_at: '2026-07-25T00:00:00+08:00'
|
||||
verified_at: '2026-08-04T00:00:00+08:00'
|
||||
|
||||
xai:
|
||||
# OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理;这里只声明请求画像启动基线。
|
||||
|
||||
+10
-8
@@ -122,7 +122,7 @@ OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段
|
||||
- xAI 从单账号 object 或 `accounts` 数组中提取 OAuth token;包装中的代理、并发、优先级等字段不参与认证;
|
||||
- xAI 批量导入逐条独立校验:失败条目跳过并记录日志,不中断其余条目,仅当没有任何条目成功时整个导入才报错;
|
||||
- xAI API Key 不是受支持的账号 credential;
|
||||
- 导入仍会通过 Provider 的身份或刷新链路校验,不能只凭文件外形写入账号。
|
||||
- 导入不会只凭文件外形写入账号;目标 Provider 使用认证材料完成必要的 token exchange 或已认证账号资料补全。
|
||||
|
||||
OpenAI 的 CPR 导出保持 OAuth 账号的既有字段;Agent Identity 账号则输出
|
||||
`authMode: "agentIdentity"`、`agentRuntimeId`、`agentPrivateKey` 和可选的
|
||||
@@ -171,20 +171,22 @@ OAuth start 使用:
|
||||
|
||||
### OpenAI 身份、额度与状态
|
||||
|
||||
- 导入会校验签名 access token 和稳定账号身份;`/wham/usage` 用于补全账号/套餐事实和额度,
|
||||
不是判断 token 是否有效的唯一依据。
|
||||
- OAuth 文件导入使用可用的 access token 调用 OpenAI 已认证账号接口,补全实际使用的账号 ID、用户 ID、
|
||||
邮箱和套餐;仅有 refresh token 时,先换取 access token 后执行同一补全。导入不要求 JWT payload 含有
|
||||
特定字段,不执行 JWKS 验签,也不将文件中的身份字段与上游资料比对。
|
||||
- 首次 OAuth 保留回调 `state`、PKCE 与官方 token exchange;换得 access token 后调用同一已认证账号接口
|
||||
补全账号资料。重新授权也保留这些回调保护,但只轮换目标账号的 token。回调地址只承载 `code`/`state`,
|
||||
不以 host/path 形式作为拒绝条件。
|
||||
- 账号文件导入和首次 OAuth 创建在 credential 提交后立即尝试一次额度观测。观测失败只记录告警,
|
||||
不回滚已提交的账号;重新授权只轮换目标账号 credential,不隐式等同于手工额度刷新。
|
||||
- 重新授权在 start 时绑定目标账号,complete 时重新读取目标的当前 credential revision,并验证上游
|
||||
account/user 身份没有换绑。Free token 缺少 `chatgpt_account_id` 时,后端使用目标账号 header 调用
|
||||
`/wham/usage` 交叉确认,而不是把该账号推断成 K12。
|
||||
不回滚已提交的账号;重新授权和手工或后台 RT 刷新只更新 token,不隐式等同于手工额度刷新,也不更新
|
||||
既有账号资料或 OAuth principal。
|
||||
- OAuth pending flow 先取得带过期时间的独占 claim,只有账号事务提交成功后才消费。失败会释放 claim,
|
||||
但上游 authorization code 本身通常只能交换一次;已完成过 token exchange 时应重新创建 OAuth flow。
|
||||
- `GET /accounts/quota` 只读取最后一次落库快照;`POST /accounts/quota/refresh` 才访问上游。access token
|
||||
已过期时,额度刷新要求先走 credential 刷新或重新授权,不会拿过期 token 探测额度。
|
||||
- 成功额度观测会 revision-fenced 写入 quota,并在 `active` 与 `quota_exhausted` 等额度所属状态间同步;
|
||||
不会清除 `expired`、`banned` 等 credential/封禁终态。额度接口的 401/403 也不足以判定 refresh token
|
||||
永久失效,credential 终态只由 OAuth refresh 或身份校验链路写入。
|
||||
永久失效,credential 终态只由 OAuth refresh 的明确永久错误写入。
|
||||
- 正常 Responses 请求会解析上游响应的 rate-limit headers,合并进同一 quota 快照并同步状态。Free、
|
||||
K12 等套餐共用该状态机;套餐只参与账号展示和按套餐隔离的模型目录 cache,不存在 K12 专属额度路径。
|
||||
- 账号页没有定时静默轮询。手工额度刷新完成后页面会重新读取该账号;请求驱动或后台任务产生的状态
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user