mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae5eaa46b4 | ||
|
|
65d5b32eba | ||
|
|
6c158447be | ||
|
|
81d939bd03 | ||
|
|
4a22479dff | ||
|
|
3ee454b750 | ||
|
|
b5bd63b585 | ||
|
|
286031a441 | ||
|
|
5358de53dd | ||
|
|
d814582c09 | ||
|
|
a9ce01f91f |
Generated
+1
@@ -1681,6 +1681,7 @@ dependencies = [
|
||||
"gateway-core",
|
||||
"heck",
|
||||
"hex",
|
||||
"hkdf",
|
||||
"prettyplease",
|
||||
"quote",
|
||||
"rand_core 0.6.4",
|
||||
|
||||
@@ -21,6 +21,7 @@ futures = "0.3.32"
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
heck = "0.5.0"
|
||||
hex = "0.4.3"
|
||||
hkdf = "0.13.0"
|
||||
rand_core = { version = "0.6.4", features = ["getrandom"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
|
||||
@@ -405,9 +405,7 @@ async fn initialize_inner(
|
||||
let registry = providers;
|
||||
|
||||
let auth = Arc::new(DefaultAuthService::new(
|
||||
config.default_username,
|
||||
config.session_ttl_minutes,
|
||||
config.session_absolute_ttl_minutes,
|
||||
&config,
|
||||
client_config.session_ttl_minutes,
|
||||
store.auth(),
|
||||
client_key_verifier.clone(),
|
||||
|
||||
@@ -76,18 +76,23 @@ impl std::fmt::Debug for LoginCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/// 登录成功后返回的会话事实
|
||||
/// 登录或刷新后仅向当前浏览器交付的会话凭据
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct LoginResult {
|
||||
pub struct SessionGrant {
|
||||
pub session_id: String,
|
||||
pub refresh_token: Option<String>,
|
||||
pub session: AuthSession,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LoginResult {
|
||||
impl std::fmt::Debug for SessionGrant {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("LoginResult")
|
||||
.debug_struct("SessionGrant")
|
||||
.field("session_id", &"[REDACTED]")
|
||||
.field(
|
||||
"refresh_token",
|
||||
&self.refresh_token.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("session", &self.session)
|
||||
.finish()
|
||||
}
|
||||
@@ -138,12 +143,41 @@ impl std::fmt::Debug for ChangePassword {
|
||||
}
|
||||
}
|
||||
|
||||
/// 会话有效期与管理员续期上限;没有上限字段的旧会话保持固定有效期
|
||||
/// 访问有效期与逻辑会话;没有续期凭据的会话保持固定期限
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AuthSession {
|
||||
pub subject: SessionSubject,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub absolute_expires_at: Option<DateTime<Utc>>,
|
||||
pub renewal: Option<SessionRenewal>,
|
||||
}
|
||||
|
||||
/// 长效登录的凭据摘要与续期窗口;访问到期不删除整段登录
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRenewal {
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub access_digest: [u8; 32],
|
||||
pub refresh_digest: [u8; 32],
|
||||
pub previous: Option<SessionRefreshReplay>,
|
||||
}
|
||||
|
||||
/// 仅保存上一代摘要及派生盐;响应丢失时须持有上一代凭据才能重建同一响应
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRefreshReplay {
|
||||
pub digest: [u8; 32],
|
||||
pub salt: [u8; 32],
|
||||
pub expires_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl AuthSession {
|
||||
#[must_use]
|
||||
pub fn retained_until(&self) -> DateTime<Utc> {
|
||||
self.renewal
|
||||
.as_ref()
|
||||
.map_or(self.expires_at, |renewal| renewal.expires_at)
|
||||
}
|
||||
}
|
||||
|
||||
/// 安全审计事件类型
|
||||
|
||||
@@ -294,12 +294,12 @@ pub trait AuthStore: Send + Sync {
|
||||
|
||||
async fn store_session(&self, session_id: &str, session: &AuthSession) -> AdminStoreResult<()>;
|
||||
|
||||
/// 只延长仍存在且匹配的会话;并发续期返回当前值,已退出或过期时不重建
|
||||
async fn renew_session(
|
||||
/// 原子替换或撤销仍匹配的会话并返回当前值;过期或退出后不重新创建
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>>;
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AuthSession>>;
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//! 控制面认证端口、会话策略与访问和续期共用的身份校验
|
||||
|
||||
use std::{net::IpAddr, sync::Arc};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_core::engine::execution::ClientKeyVerifier;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use super::map_store_error;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError,
|
||||
auth::{
|
||||
AuthSession, ChangePassword, LoginCommand, LoginError, SessionGrant, SessionSubject,
|
||||
},
|
||||
},
|
||||
ports::store::AuthStore,
|
||||
};
|
||||
|
||||
mod service;
|
||||
mod session;
|
||||
|
||||
/// 所有控制面接口消费同一个会话服务,权限由服务端身份决定
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
async fn change_password(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
command: ChangePassword,
|
||||
source_ip: IpAddr,
|
||||
) -> Result<(), AdminError>;
|
||||
async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminError>;
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn renew_session(
|
||||
&self,
|
||||
refresh_token: Option<&str>,
|
||||
access_token: Option<&str>,
|
||||
) -> Result<Option<SessionGrant>, AdminError>;
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<String>, AdminError>;
|
||||
async fn verify_admin_api_key(&self, key: &str) -> Result<bool, AdminError>;
|
||||
async fn login(
|
||||
&self,
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<SessionGrant, LoginError>;
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
admin_access_ttl: Duration,
|
||||
admin_session_ttl: Duration,
|
||||
admin_session_absolute_ttl: Duration,
|
||||
key_session_ttl: Duration,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
config: &crate::AdminConfig,
|
||||
key_session_ttl_minutes: u64,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
) -> Self {
|
||||
let admin_session_ttl = session_ttl(config.session_ttl_minutes);
|
||||
Self {
|
||||
default_admin_user_id: config.default_username.clone(),
|
||||
// 访问凭据最多有效 24 小时,较短的不活动期限也要留出续期窗口
|
||||
admin_access_ttl: Duration::hours(24).min(admin_session_ttl / 4),
|
||||
admin_session_ttl,
|
||||
admin_session_absolute_ttl: session_ttl(config.session_absolute_ttl_minutes),
|
||||
key_session_ttl: session_ttl(key_session_ttl_minutes),
|
||||
store,
|
||||
verifier,
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_live_session(&self, id: &str) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session) = self
|
||||
.store
|
||||
.load_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if session.retained_until() <= Utc::now()
|
||||
|| session
|
||||
.absolute_expires_at
|
||||
.is_some_and(|expiry| expiry <= Utc::now())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let valid = match &session.subject {
|
||||
SessionSubject::Admin {
|
||||
admin_user_id,
|
||||
credential_fingerprint,
|
||||
} => self
|
||||
.store
|
||||
.load_password_hash(admin_user_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator session"))?
|
||||
.is_some_and(|hash| password_fingerprint(&hash) == *credential_fingerprint),
|
||||
SessionSubject::Key { client_key_id } => self
|
||||
.store
|
||||
.client_key_enabled(client_key_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "session key"))?,
|
||||
};
|
||||
if !valid {
|
||||
let _ = self.store.delete_session(id).await;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(session))
|
||||
}
|
||||
}
|
||||
|
||||
fn session_ttl(minutes: u64) -> Duration {
|
||||
Duration::minutes(
|
||||
i64::try_from(minutes)
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES),
|
||||
)
|
||||
}
|
||||
|
||||
// 指纹只绑定已加盐的密码哈希,不把密码或原始哈希复制到 Redis 会话
|
||||
fn password_fingerprint(password_hash: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(password_hash.as_bytes()))
|
||||
}
|
||||
+80
-156
@@ -1,93 +1,35 @@
|
||||
//! 控制面登录、会话恢复与身份校验的唯一 owner
|
||||
//! 管理员和 Key 的登录、退出、密码策略与会话访问
|
||||
|
||||
use std::{net::IpAddr, sync::Arc, time::Duration as StdDuration};
|
||||
use std::{net::IpAddr, time::Duration as StdDuration};
|
||||
|
||||
use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_core::engine::execution::ClientKeyVerifier;
|
||||
use chrono::Utc;
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
AuthService, DefaultAuthService, password_fingerprint,
|
||||
session::{self, Credential, storage_id},
|
||||
};
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind,
|
||||
auth::{
|
||||
AdminAuditEvent, AuditActorKind, AuthSession, ChangePassword, LoginCommand, LoginError,
|
||||
LoginResult, SessionSubject,
|
||||
SessionGrant, SessionSubject,
|
||||
},
|
||||
},
|
||||
ports::store::AuthStore,
|
||||
use_case::map_store_error,
|
||||
};
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
/// 所有控制面接口消费同一个会话服务,权限由服务端身份决定
|
||||
#[async_trait]
|
||||
pub trait AuthService: Send + Sync {
|
||||
async fn change_password(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
command: ChangePassword,
|
||||
source_ip: IpAddr,
|
||||
) -> Result<(), AdminError>;
|
||||
async fn ensure_default_admin(&self, password: &str) -> Result<bool, AdminError>;
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError>;
|
||||
async fn resolve_admin_user_id(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<String>, AdminError>;
|
||||
async fn verify_admin_api_key(&self, key: &str) -> Result<bool, AdminError>;
|
||||
async fn login(
|
||||
&self,
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<LoginResult, LoginError>;
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
const MAX_SESSION_TTL_MINUTES: i64 = 366 * 24 * 60;
|
||||
const LOGIN_WINDOW: StdDuration = StdDuration::from_secs(60);
|
||||
const LOGIN_ATTEMPTS_PER_SOURCE: u32 = 10;
|
||||
const LOGIN_ATTEMPTS_GLOBAL: u32 = 200;
|
||||
|
||||
pub(crate) struct DefaultAuthService {
|
||||
default_admin_user_id: String,
|
||||
admin_session_ttl: Duration,
|
||||
admin_session_absolute_ttl: Duration,
|
||||
key_session_ttl: Duration,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
default_admin_user_id: impl Into<String>,
|
||||
admin_session_ttl_minutes: u64,
|
||||
admin_session_absolute_ttl_minutes: u64,
|
||||
key_session_ttl_minutes: u64,
|
||||
store: Arc<dyn AuthStore>,
|
||||
verifier: Arc<dyn ClientKeyVerifier>,
|
||||
) -> Self {
|
||||
Self {
|
||||
default_admin_user_id: default_admin_user_id.into(),
|
||||
admin_session_ttl: session_ttl(admin_session_ttl_minutes),
|
||||
admin_session_absolute_ttl: session_ttl(admin_session_absolute_ttl_minutes),
|
||||
key_session_ttl: session_ttl(key_session_ttl_minutes),
|
||||
store,
|
||||
verifier,
|
||||
}
|
||||
}
|
||||
|
||||
fn auth_audit(&self, action: &str, admin_user_id: &str) -> AdminAuditEvent {
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
@@ -228,50 +170,23 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
|
||||
async fn session(&self, session_id: Option<&str>) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session_id) = session_id.filter(|value| !value.is_empty()) else {
|
||||
let Some(raw) = session_id.filter(|value| !value.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(session) = self
|
||||
.store
|
||||
.load_session(session_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
let Some(session) = self.load_live_session(storage_id(raw)).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if session.expires_at <= Utc::now()
|
||||
|| session
|
||||
.absolute_expires_at
|
||||
.is_some_and(|expiry| expiry <= Utc::now())
|
||||
{
|
||||
// Redis 的 TTL 负责过期清理,避免旧读取删除另一个请求刚续期的会话
|
||||
if session.expires_at <= Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if let SessionSubject::Admin {
|
||||
admin_user_id,
|
||||
credential_fingerprint,
|
||||
} = &session.subject
|
||||
{
|
||||
let password_hash = self
|
||||
.store
|
||||
.load_password_hash(admin_user_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "administrator session"))?;
|
||||
if password_hash
|
||||
.is_none_or(|hash| password_fingerprint(&hash) != *credential_fingerprint)
|
||||
{
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
if let Some(renewal) = &session.renewal {
|
||||
let Some(credential) = Credential::parse(raw).filter(|value| !value.refresh) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !credential.owns(renewal) {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
if let SessionSubject::Key { client_key_id } = &session.subject
|
||||
&& !self
|
||||
.store
|
||||
.client_key_enabled(client_key_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "session key"))?
|
||||
{
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
} else if Credential::parse(raw).is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(session))
|
||||
@@ -279,33 +194,25 @@ impl AuthService for DefaultAuthService {
|
||||
|
||||
async fn renew_session(
|
||||
&self,
|
||||
session_id: Option<&str>,
|
||||
) -> Result<Option<AuthSession>, AdminError> {
|
||||
let Some(session_id) = session_id else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(session) = self.session(Some(session_id)).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Key 与升级前的会话保留原有期限,不能凭续期获得新的最长登录窗口
|
||||
let Some(absolute) = session
|
||||
.absolute_expires_at
|
||||
.filter(|_| matches!(session.subject, SessionSubject::Admin { .. }))
|
||||
else {
|
||||
return Ok(Some(session));
|
||||
};
|
||||
let now = Utc::now();
|
||||
let expires_at = (now + self.admin_session_ttl).min(absolute);
|
||||
let interval = (self.admin_session_ttl / 4).min(Duration::minutes(1));
|
||||
if expires_at <= session.expires_at
|
||||
|| session.expires_at - now > self.admin_session_ttl - interval
|
||||
{
|
||||
return Ok(Some(session));
|
||||
refresh_token: Option<&str>,
|
||||
access_token: Option<&str>,
|
||||
) -> Result<Option<SessionGrant>, AdminError> {
|
||||
if let Some(refresh) = refresh_token {
|
||||
return self.refresh(refresh).await;
|
||||
}
|
||||
self.store
|
||||
.renew_session(session_id, &session, expires_at)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session renewal"))
|
||||
// Key 与升级前的单凭据仅返回仍有效的原会话,不授予新的续期能力
|
||||
let Some(session) = self
|
||||
.session(access_token)
|
||||
.await?
|
||||
.filter(|session| session.renewal.is_none())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(SessionGrant {
|
||||
session_id: access_token.unwrap_or_default().to_owned(),
|
||||
refresh_token: None,
|
||||
session,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn resolve_admin_user_id(
|
||||
@@ -346,7 +253,7 @@ impl AuthService for DefaultAuthService {
|
||||
command: LoginCommand,
|
||||
source_ip: IpAddr,
|
||||
previous_session_id: Option<&str>,
|
||||
) -> Result<LoginResult, LoginError> {
|
||||
) -> Result<SessionGrant, LoginError> {
|
||||
if let Some(retry_after) = self
|
||||
.store
|
||||
.consume_login_attempt(
|
||||
@@ -370,14 +277,31 @@ impl AuthService for DefaultAuthService {
|
||||
let now = Utc::now();
|
||||
let absolute_expires_at = matches!(subject, SessionSubject::Admin { .. })
|
||||
.then_some(now + self.admin_session_absolute_ttl);
|
||||
let session = AuthSession {
|
||||
subject,
|
||||
expires_at: (now + ttl).min(absolute_expires_at.unwrap_or(now + ttl)),
|
||||
absolute_expires_at,
|
||||
let refresh_expiry = (now + ttl).min(absolute_expires_at.unwrap_or(now + ttl));
|
||||
let grant = if let Some(absolute) = absolute_expires_at {
|
||||
session::issue(
|
||||
subject,
|
||||
(now + self.admin_access_ttl).min(refresh_expiry),
|
||||
refresh_expiry,
|
||||
absolute,
|
||||
)
|
||||
.map_err(|_| LoginError::Unavailable)?
|
||||
} else {
|
||||
SessionGrant {
|
||||
session_id: random_session_token(),
|
||||
refresh_token: None,
|
||||
session: AuthSession {
|
||||
subject,
|
||||
expires_at: refresh_expiry,
|
||||
absolute_expires_at: None,
|
||||
renewal: None,
|
||||
},
|
||||
}
|
||||
};
|
||||
let session_id = random_session_token();
|
||||
let session_id = storage_id(&grant.session_id);
|
||||
let session = &grant.session;
|
||||
self.store
|
||||
.store_session(&session_id, &session)
|
||||
.store_session(session_id, session)
|
||||
.await
|
||||
.map_err(|_| LoginError::Unavailable)?;
|
||||
if let SessionSubject::Admin { admin_user_id, .. } = &session.subject
|
||||
@@ -387,26 +311,39 @@ impl AuthService for DefaultAuthService {
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
let _ = self.store.delete_session(&session_id).await;
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
return Err(LoginError::Unavailable);
|
||||
}
|
||||
// 新身份验证成功后才撤销旧会话;撤销失败时不向浏览器提交新会话
|
||||
if let Some(previous) = previous_session_id.filter(|value| !value.is_empty())
|
||||
&& self.logout(previous).await.is_err()
|
||||
{
|
||||
let _ = self.store.delete_session(&session_id).await;
|
||||
let _ = self.store.delete_session(session_id).await;
|
||||
return Err(LoginError::Unavailable);
|
||||
}
|
||||
Ok(LoginResult {
|
||||
session_id,
|
||||
session,
|
||||
})
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
async fn logout(&self, session_id: &str) -> Result<(), AdminError> {
|
||||
let id = storage_id(session_id);
|
||||
let Some(stored) = self
|
||||
.store
|
||||
.load_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(renewal) = &stored.renewal {
|
||||
if !Credential::parse(session_id).is_some_and(|credential| credential.owns(renewal)) {
|
||||
return Ok(());
|
||||
}
|
||||
} else if Credential::parse(session_id).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let session = self
|
||||
.store
|
||||
.delete_session(session_id)
|
||||
.delete_session(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session"))?;
|
||||
if let Some(AuthSession {
|
||||
@@ -423,14 +360,6 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
}
|
||||
|
||||
fn session_ttl(minutes: u64) -> Duration {
|
||||
Duration::minutes(
|
||||
i64::try_from(minutes)
|
||||
.unwrap_or(MAX_SESSION_TTL_MINUTES)
|
||||
.clamp(1, MAX_SESSION_TTL_MINUTES),
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_new_password(password: &str) -> Result<(), AdminError> {
|
||||
if password.trim().chars().count() < 12
|
||||
|| password.len() > 1024
|
||||
@@ -470,8 +399,3 @@ fn valid_admin_api_key_shape(value: &str) -> bool {
|
||||
&& value.starts_with("admin-")
|
||||
&& value[6..].bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
// 指纹只绑定已加盐的密码哈希,不把密码或原始哈希复制到 Redis 会话
|
||||
fn password_fingerprint(password_hash: &str) -> String {
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(password_hash.as_bytes()))
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
//! 双凭据会话的签发、摘要校验与短窗口重试,不持久化可直接使用的凭据
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use rand_core::{OsRng, RngCore as _};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
|
||||
use super::DefaultAuthService;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind,
|
||||
auth::{AuthSession, SessionGrant, SessionRefreshReplay, SessionRenewal, SessionSubject},
|
||||
},
|
||||
use_case::map_store_error,
|
||||
};
|
||||
|
||||
const REPLAY_WINDOW: Duration = Duration::seconds(30);
|
||||
|
||||
pub(super) struct Credential<'a> {
|
||||
pub id: &'a str,
|
||||
pub secret: &'a str,
|
||||
pub refresh: bool,
|
||||
}
|
||||
|
||||
impl<'a> Credential<'a> {
|
||||
pub fn parse(value: &'a str) -> Option<Self> {
|
||||
let mut parts = value.split('.');
|
||||
let refresh = match parts.next()? {
|
||||
"a2" => false,
|
||||
"r2" => true,
|
||||
_ => return None,
|
||||
};
|
||||
let id = parts.next()?;
|
||||
let secret = parts.next()?;
|
||||
if parts.next().is_some() || !valid_secret(id) || !valid_secret(secret) {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
id,
|
||||
secret,
|
||||
refresh,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn digest(&self) -> [u8; 32] {
|
||||
digest(self.secret)
|
||||
}
|
||||
|
||||
pub fn owns(&self, renewal: &SessionRenewal) -> bool {
|
||||
if self.refresh {
|
||||
matches_digest(&renewal.refresh_digest, &self.digest())
|
||||
|| renewal
|
||||
.previous
|
||||
.as_ref()
|
||||
.is_some_and(|previous| matches_digest(&previous.digest, &self.digest()))
|
||||
} else {
|
||||
matches_digest(&renewal.access_digest, &self.digest())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_secret(value: &str) -> bool {
|
||||
value.len() == 43
|
||||
&& URL_SAFE_NO_PAD
|
||||
.decode(value)
|
||||
.is_ok_and(|bytes| bytes.len() == 32)
|
||||
}
|
||||
|
||||
pub(super) fn storage_id(value: &str) -> &str {
|
||||
Credential::parse(value).map_or(value, |credential| credential.id)
|
||||
}
|
||||
|
||||
fn matches_digest(left: &[u8; 32], right: &[u8; 32]) -> bool {
|
||||
bool::from(left.ct_eq(right))
|
||||
}
|
||||
|
||||
fn digest(secret: &str) -> [u8; 32] {
|
||||
Sha256::digest(secret.as_bytes()).into()
|
||||
}
|
||||
|
||||
fn random_bytes() -> [u8; 32] {
|
||||
let mut bytes = [0; 32];
|
||||
OsRng.fill_bytes(&mut bytes);
|
||||
bytes
|
||||
}
|
||||
|
||||
// 使用标准 HKDF 的独立用途标签;只有盐和摘要留在 Store,重试仍须持有原始凭据
|
||||
fn derive(secret: &str, salt: Option<&[u8]>, purpose: &[u8]) -> Result<String, AdminError> {
|
||||
let mut output = [0; 32];
|
||||
hkdf::Hkdf::<Sha256>::new(salt, secret.as_bytes())
|
||||
.expand(purpose, &mut output)
|
||||
.map_err(|_| AdminError::internal("会话凭据派生失败"))?;
|
||||
Ok(URL_SAFE_NO_PAD.encode(output))
|
||||
}
|
||||
|
||||
fn access_secret(refresh: &str) -> Result<String, AdminError> {
|
||||
derive(refresh, None, b"cpr-session-v2/access")
|
||||
}
|
||||
|
||||
pub(super) fn issue(
|
||||
subject: SessionSubject,
|
||||
access_expiry: DateTime<Utc>,
|
||||
refresh_expiry: DateTime<Utc>,
|
||||
absolute: DateTime<Utc>,
|
||||
) -> Result<SessionGrant, AdminError> {
|
||||
let id = URL_SAFE_NO_PAD.encode(random_bytes());
|
||||
let secret = URL_SAFE_NO_PAD.encode(random_bytes());
|
||||
let session = AuthSession {
|
||||
subject,
|
||||
expires_at: access_expiry,
|
||||
absolute_expires_at: Some(absolute),
|
||||
renewal: Some(SessionRenewal {
|
||||
expires_at: refresh_expiry,
|
||||
access_digest: digest(&access_secret(&secret)?),
|
||||
refresh_digest: digest(&secret),
|
||||
previous: None,
|
||||
}),
|
||||
};
|
||||
grant(&id, &secret, session)
|
||||
}
|
||||
|
||||
fn grant(id: &str, refresh: &str, session: AuthSession) -> Result<SessionGrant, AdminError> {
|
||||
Ok(SessionGrant {
|
||||
session_id: format!("a2.{id}.{}", access_secret(refresh)?),
|
||||
refresh_token: Some(format!("r2.{id}.{refresh}")),
|
||||
session,
|
||||
})
|
||||
}
|
||||
|
||||
impl DefaultAuthService {
|
||||
pub(super) async fn refresh(&self, raw: &str) -> Result<Option<SessionGrant>, AdminError> {
|
||||
let Some(credential) = Credential::parse(raw).filter(|value| value.refresh) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut current = self.load_live_session(credential.id).await?;
|
||||
// CAS 冲突只重新解释同一请求的凭据,不对并发退出进行补建
|
||||
for _ in 0..3 {
|
||||
let Some(session) = current else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(renewal) = &session.renewal else {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = Utc::now();
|
||||
if session.retained_until() <= now
|
||||
|| session.absolute_expires_at.is_none_or(|limit| limit <= now)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if matches_digest(&renewal.refresh_digest, &credential.digest()) {
|
||||
// 其他标签页已经刷新或客户端重试时复用当前代,避免 Cookie 响应相互覆盖
|
||||
if session.expires_at > now {
|
||||
return grant(credential.id, credential.secret, session).map(Some);
|
||||
}
|
||||
let absolute = session
|
||||
.absolute_expires_at
|
||||
.ok_or_else(|| AdminError::internal("缺少会话最长有效期"))?;
|
||||
let salt = random_bytes();
|
||||
let next_secret =
|
||||
derive(credential.secret, Some(&salt), b"cpr-session-v2/refresh")?;
|
||||
let refresh_expiry = (now + self.admin_session_ttl).min(absolute);
|
||||
let access_expiry = (now + self.admin_access_ttl).min(refresh_expiry);
|
||||
let replacement = AuthSession {
|
||||
expires_at: access_expiry,
|
||||
renewal: Some(SessionRenewal {
|
||||
expires_at: refresh_expiry,
|
||||
access_digest: digest(&access_secret(&next_secret)?),
|
||||
refresh_digest: digest(&next_secret),
|
||||
previous: Some(SessionRefreshReplay {
|
||||
digest: credential.digest(),
|
||||
salt,
|
||||
expires_at: (now + REPLAY_WINDOW).min(access_expiry),
|
||||
}),
|
||||
}),
|
||||
..session.clone()
|
||||
};
|
||||
current = self
|
||||
.store
|
||||
.compare_exchange_session(credential.id, &session, Some(&replacement))
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session rotation"))?;
|
||||
if current.as_ref() == Some(&replacement) {
|
||||
return grant(credential.id, &next_secret, replacement).map(Some);
|
||||
}
|
||||
} else if let Some(previous) = renewal
|
||||
.previous
|
||||
.as_ref()
|
||||
.filter(|previous| matches_digest(&previous.digest, &credential.digest()))
|
||||
{
|
||||
if previous.expires_at > now {
|
||||
let secret = derive(
|
||||
credential.secret,
|
||||
Some(&previous.salt),
|
||||
b"cpr-session-v2/refresh",
|
||||
)?;
|
||||
return grant(credential.id, &secret, session).map(Some);
|
||||
}
|
||||
// 已知上一代在重试窗口外再次使用,原子撤销整个会话
|
||||
current = self
|
||||
.store
|
||||
.compare_exchange_session(credential.id, &session, None)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "authentication session revocation"))?;
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Err(AdminError::new(
|
||||
AdminErrorKind::Unavailable,
|
||||
"会话正在变更,请重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -725,11 +725,11 @@ impl AuthStore for FakeAuthStore {
|
||||
) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
_: Option<&gateway_admin::model::auth::AuthSession>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
+47
-110
@@ -1,4 +1,4 @@
|
||||
//! 验证管理员认证、密码变更、会话绑定与限流行为
|
||||
//! 验证管理员认证、密码变更与限流,提供会话轮换的内存存储辅助
|
||||
|
||||
use std::{collections::BTreeMap, sync::Mutex};
|
||||
|
||||
@@ -69,6 +69,7 @@ async fn password_change_obeys_rate_limit_and_rejects_unbound_legacy_sessions()
|
||||
admin_user_id: "admin".into(),
|
||||
credential_fingerprint: String::new(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + TimeDelta::hours(1),
|
||||
},
|
||||
@@ -138,19 +139,24 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
if session.retained_until() <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
if let Some(replacement) = replacement {
|
||||
*session = replacement.clone();
|
||||
} else {
|
||||
sessions.remove(session_id);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
@@ -190,8 +196,13 @@ impl AuthStore for MemoryAuthStore {
|
||||
#[tokio::test]
|
||||
async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
let services = super::AdminHarness::new()
|
||||
.session_ttl_minutes(7 * 24 * 60)
|
||||
.auth(store.clone())
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let before = Utc::now();
|
||||
let result = services
|
||||
.auth()
|
||||
.login(
|
||||
@@ -205,6 +216,17 @@ async fn successful_login_should_create_expiring_session_and_audit() {
|
||||
.await
|
||||
.expect("login");
|
||||
|
||||
let after = Utc::now();
|
||||
for (expiry, ttl) in [
|
||||
(result.session.expires_at, TimeDelta::hours(24)),
|
||||
(result.session.retained_until(), TimeDelta::days(7)),
|
||||
(
|
||||
result.session.absolute_expires_at.unwrap(),
|
||||
TimeDelta::days(30),
|
||||
),
|
||||
] {
|
||||
assert!((before + ttl..=after + ttl).contains(&expiry));
|
||||
}
|
||||
assert!(
|
||||
services
|
||||
.auth()
|
||||
@@ -293,6 +315,7 @@ async fn expired_sessions_are_rejected_and_store_outages_are_not_treated_as_logo
|
||||
credential_fingerprint: String::new(),
|
||||
admin_user_id: "admin".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() - TimeDelta::seconds(1),
|
||||
},
|
||||
@@ -362,7 +385,7 @@ async fn failed_rotation_discards_the_new_session_and_leaves_the_old_session_ret
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!format!("{original:?}").contains(&original.session_id));
|
||||
*store.reject_delete.lock().unwrap() = Some(original.session_id.clone());
|
||||
*store.reject_delete.lock().unwrap() = Some(stored_id(&original.session_id).to_owned());
|
||||
assert_eq!(
|
||||
services
|
||||
.auth()
|
||||
@@ -391,109 +414,23 @@ async fn failed_rotation_discards_the_new_session_and_leaves_the_old_session_ret
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_is_throttled_and_capped_by_the_original_absolute_expiry() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new()
|
||||
.auth(store.clone())
|
||||
.session_ttl_minutes(60)
|
||||
.build()
|
||||
.await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(login.session.clone())
|
||||
);
|
||||
let limit = Utc::now() + TimeDelta::minutes(30);
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(&login.session_id).unwrap();
|
||||
session.expires_at = Utc::now() + TimeDelta::minutes(1);
|
||||
session.absolute_expires_at = Some(limit);
|
||||
}
|
||||
let renewed = auth
|
||||
.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(renewed.expires_at, limit);
|
||||
assert_eq!(renewed.absolute_expires_at, Some(limit));
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(renewed)
|
||||
);
|
||||
fn stored_id(token: &str) -> &str {
|
||||
token.split('.').nth(1).unwrap_or(token)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_never_revives_expired_revoked_or_legacy_sessions() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = super::AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = auth
|
||||
.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut session = login.session.clone();
|
||||
session.absolute_expires_at = None;
|
||||
session.expires_at = Utc::now() + TimeDelta::seconds(30);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session.clone());
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(&login.session_id)).await.unwrap(),
|
||||
Some(session.clone())
|
||||
);
|
||||
session.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let mut session = login.session;
|
||||
session.absolute_expires_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(login.session_id.clone(), session);
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
auth.logout(&login.session_id).await.unwrap();
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.sessions.lock().unwrap().is_empty());
|
||||
async fn admin_login(
|
||||
auth: &dyn gateway_admin::AuthService,
|
||||
) -> gateway_admin::model::auth::SessionGrant {
|
||||
auth.login(
|
||||
LoginCommand::Admin {
|
||||
username: None,
|
||||
password: "strong-test-password".into(),
|
||||
},
|
||||
std::net::Ipv4Addr::LOCALHOST.into(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
mod session;
|
||||
@@ -0,0 +1,268 @@
|
||||
//! 验证双凭据续期、响应重试、并发撤销与固定期限边界
|
||||
|
||||
use super::super::AdminHarness;
|
||||
use super::{MemoryAuthStore, admin_login, stored_id};
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_admin::model::auth::AuthSession;
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_access_is_restored_by_refresh_and_duplicate_refresh_replays_the_same_grant() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let refresh = login.refresh_token.as_deref().unwrap();
|
||||
assert!(auth.session(Some(refresh)).await.unwrap().is_none());
|
||||
assert!(
|
||||
auth.renew_session(Some(&login.session_id), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.renew_session(None, Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(refresh), None).await.unwrap(),
|
||||
Some(login.clone())
|
||||
);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let (first, concurrent) = tokio::join!(
|
||||
auth.renew_session(Some(refresh), None),
|
||||
auth.renew_session(Some(refresh), None)
|
||||
);
|
||||
let first = first.unwrap().unwrap();
|
||||
assert_eq!(concurrent.unwrap().as_ref(), Some(&first));
|
||||
assert_ne!(first.session_id, login.session_id);
|
||||
assert_ne!(first.refresh_token, login.refresh_token);
|
||||
assert_eq!(
|
||||
first.session.absolute_expires_at,
|
||||
login.session.absolute_expires_at
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&first.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(first.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
auth.renew_session(Some(refresh), None).await.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
let persisted = format!("{:?}", store.sessions.lock().unwrap());
|
||||
for token in [
|
||||
&first.session_id,
|
||||
first.refresh_token.as_ref().unwrap(),
|
||||
&login.session_id,
|
||||
login.refresh_token.as_ref().unwrap(),
|
||||
] {
|
||||
assert!(!persisted.contains(token));
|
||||
assert!(!persisted.contains(token.rsplit('.').next().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_refresh_reuse_revokes_the_family_but_unknown_secrets_do_not() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
let next = auth
|
||||
.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let unknown = format!("r2.{}.{}", stored_id(&login.session_id), "A".repeat(43));
|
||||
assert!(
|
||||
auth.renew_session(Some(&unknown), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
auth.logout(&unknown).await.unwrap();
|
||||
auth.logout(stored_id(&login.session_id)).await.unwrap();
|
||||
assert!(
|
||||
auth.session(Some(&next.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.renewal
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.previous
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&next.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.renew_session(next.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_respects_idle_and_absolute_expiry_and_logout_after_access_expiry() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
for absolute in [false, true] {
|
||||
let login = admin_login(auth).await;
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(stored_id(&login.session_id)).unwrap();
|
||||
if absolute {
|
||||
session.absolute_expires_at = Some(Utc::now() - TimeDelta::seconds(1));
|
||||
} else {
|
||||
session.renewal.as_mut().unwrap().expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
auth.session(Some(&login.session_id))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
let login = admin_login(auth).await;
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(stored_id(&login.session_id))
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
auth.logout(login.refresh_token.as_deref().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refreshed_expiry_is_capped_and_store_outages_preserve_the_session() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let limit = Utc::now() + TimeDelta::minutes(1);
|
||||
{
|
||||
let mut sessions = store.sessions.lock().unwrap();
|
||||
let session = sessions.get_mut(stored_id(&login.session_id)).unwrap();
|
||||
session.absolute_expires_at = Some(limit);
|
||||
session.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
}
|
||||
store
|
||||
.unavailable
|
||||
.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
assert!(
|
||||
auth.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.unavailable
|
||||
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let refreshed = auth
|
||||
.renew_session(login.refresh_token.as_deref(), None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(refreshed.session.expires_at, limit);
|
||||
assert_eq!(refreshed.session.retained_until(), limit);
|
||||
assert_eq!(refreshed.session.absolute_expires_at, Some(limit));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fixed_sessions_cannot_acquire_refresh_credentials() {
|
||||
let store = std::sync::Arc::new(MemoryAuthStore::default());
|
||||
let services = AdminHarness::new().auth(store.clone()).build().await;
|
||||
let auth = services.auth();
|
||||
let login = admin_login(auth).await;
|
||||
let fixed = AuthSession {
|
||||
renewal: None,
|
||||
..login.session
|
||||
};
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("fixed".into(), fixed.clone());
|
||||
let grant = auth
|
||||
.renew_session(None, Some("fixed"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(grant.session, fixed);
|
||||
assert!(grant.refresh_token.is_none());
|
||||
store
|
||||
.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut("fixed")
|
||||
.unwrap()
|
||||
.expires_at = Utc::now() - TimeDelta::seconds(1);
|
||||
assert!(
|
||||
auth.renew_session(None, Some("fixed"))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -285,19 +285,24 @@ impl AuthStore for MemoryKeyAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
if session.retained_until() <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
if let Some(replacement) = replacement {
|
||||
*session = replacement.clone();
|
||||
} else {
|
||||
sessions.remove(session_id);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
|
||||
@@ -369,11 +369,11 @@ impl AuthStore for BootstrapAuthStore {
|
||||
Err(unavailable("admin session"))
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &gateway_admin::model::auth::AuthSession,
|
||||
_: chrono::DateTime<chrono::Utc>,
|
||||
_: Option<&gateway_admin::model::auth::AuthSession>,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AuthSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -138,7 +138,9 @@ where
|
||||
.login(
|
||||
payload.into(),
|
||||
peer.ip(),
|
||||
session_cookie::value(&headers).as_deref(),
|
||||
session_cookie::refresh_value(&headers)
|
||||
.or_else(|| session_cookie::value(&headers))
|
||||
.as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(map_login_error)?;
|
||||
@@ -149,8 +151,7 @@ where
|
||||
)
|
||||
.into_response(),
|
||||
&headers,
|
||||
&result.session_id,
|
||||
&result.session,
|
||||
&result,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -177,16 +178,18 @@ async fn refresh_session<S>(
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let session_id = session_cookie::value(&headers);
|
||||
let session = state
|
||||
let grant = state
|
||||
.admin_services()
|
||||
.auth()
|
||||
.renew_session(session_id.as_deref())
|
||||
.renew_session(
|
||||
session_cookie::refresh_value(&headers).as_deref(),
|
||||
session_cookie::value(&headers).as_deref(),
|
||||
)
|
||||
.await
|
||||
.map_err(map_admin_service_error)?;
|
||||
let response = session_response(session.as_ref());
|
||||
if let (Some(session_id), Some(session)) = (session_id, session) {
|
||||
return set_session_cookie(response, &headers, &session_id, &session);
|
||||
let response = session_response(grant.as_ref().map(|grant| &grant.session));
|
||||
if let Some(grant) = grant {
|
||||
return set_session_cookie(response, &headers, &grant);
|
||||
}
|
||||
// 失效响应不清 Cookie,避免晚到的请求覆盖另一个标签页刚建立的会话
|
||||
Ok(response)
|
||||
@@ -206,32 +209,72 @@ fn session_response(session: Option<&AuthSession>) -> Response {
|
||||
fn set_session_cookie(
|
||||
mut response: Response,
|
||||
headers: &HeaderMap,
|
||||
session_id: &str,
|
||||
session: &AuthSession,
|
||||
grant: &gateway_admin::model::auth::SessionGrant,
|
||||
) -> Result<Response, AdminError> {
|
||||
let max_age = session
|
||||
.expires_at
|
||||
.signed_duration_since(chrono::Utc::now())
|
||||
.num_seconds()
|
||||
.max(1);
|
||||
let expires = session.expires_at.format("%a, %d %b %Y %H:%M:%S GMT");
|
||||
let cookie = format!(
|
||||
"{}={session_id}; {}; Max-Age={max_age}; Expires={expires}",
|
||||
append_cookie(
|
||||
&mut response,
|
||||
headers,
|
||||
session_cookie::NAME,
|
||||
session_cookie::attributes(headers)
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::SET_COOKIE,
|
||||
HeaderValue::from_str(&cookie).map_err(|_| AdminError::internal())?,
|
||||
);
|
||||
&grant.session_id,
|
||||
Some(grant.session.expires_at),
|
||||
)?;
|
||||
append_cookie(
|
||||
&mut response,
|
||||
headers,
|
||||
session_cookie::REFRESH_NAME,
|
||||
grant.refresh_token.as_deref().unwrap_or_default(),
|
||||
grant
|
||||
.session
|
||||
.renewal
|
||||
.as_ref()
|
||||
.map(|renewal| renewal.expires_at),
|
||||
)?;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn append_cookie(
|
||||
response: &mut Response,
|
||||
headers: &HeaderMap,
|
||||
name: &str,
|
||||
value: &str,
|
||||
expires_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
) -> Result<(), AdminError> {
|
||||
let attributes = session_cookie::attributes(headers);
|
||||
let path = if name == session_cookie::REFRESH_NAME {
|
||||
"/api/auth"
|
||||
} else {
|
||||
"/"
|
||||
};
|
||||
let lifetime = expires_at.map_or_else(
|
||||
|| "Max-Age=0".to_owned(),
|
||||
|expiry| {
|
||||
format!(
|
||||
"Max-Age={}; Expires={}",
|
||||
expiry
|
||||
.signed_duration_since(chrono::Utc::now())
|
||||
.num_seconds()
|
||||
.max(1),
|
||||
expiry.format("%a, %d %b %Y %H:%M:%S GMT")
|
||||
)
|
||||
},
|
||||
);
|
||||
response.headers_mut().append(
|
||||
header::SET_COOKIE,
|
||||
HeaderValue::from_str(&format!(
|
||||
"{name}={value}; Path={path}; {attributes}; {lifetime}"
|
||||
))
|
||||
.map_err(|_| AdminError::internal())?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn logout<S>(State(state): State<S>, headers: HeaderMap) -> Result<Response, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
if let Some(session_id) = session_cookie::value(&headers) {
|
||||
if let Some(session_id) =
|
||||
session_cookie::refresh_value(&headers).or_else(|| session_cookie::value(&headers))
|
||||
{
|
||||
state
|
||||
.admin_services()
|
||||
.auth()
|
||||
@@ -285,15 +328,9 @@ fn clear_session_cookie(
|
||||
mut response: Response,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<Response, AdminError> {
|
||||
let cookie = format!(
|
||||
"{}=; {}; Max-Age=0",
|
||||
session_cookie::NAME,
|
||||
session_cookie::attributes(headers)
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::SET_COOKIE,
|
||||
HeaderValue::from_str(&cookie).map_err(|_| AdminError::internal())?,
|
||||
);
|
||||
for name in [session_cookie::NAME, session_cookie::REFRESH_NAME] {
|
||||
append_cookie(&mut response, headers, name, "", None)?;
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ use axum::http::{HeaderMap, header::ORIGIN};
|
||||
use url::Url;
|
||||
|
||||
pub(crate) const NAME: &str = "cpr_session";
|
||||
pub(crate) const REFRESH_NAME: &str = "cpr_refresh";
|
||||
|
||||
pub(crate) fn attributes(headers: &HeaderMap) -> &'static str {
|
||||
// HTTPS 反代回源 HTTP 不改变浏览器 Origin
|
||||
@@ -19,16 +20,24 @@ pub(crate) fn attributes(headers: &HeaderMap) -> &'static str {
|
||||
})
|
||||
});
|
||||
if http_origin {
|
||||
"Path=/; HttpOnly; SameSite=Lax"
|
||||
"HttpOnly; SameSite=Lax"
|
||||
} else {
|
||||
"Path=/; Secure; HttpOnly; SameSite=Lax"
|
||||
"Secure; HttpOnly; SameSite=Lax"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn value(headers: &HeaderMap) -> Option<String> {
|
||||
named_value(headers, NAME)
|
||||
}
|
||||
|
||||
pub(crate) fn refresh_value(headers: &HeaderMap) -> Option<String> {
|
||||
named_value(headers, REFRESH_NAME)
|
||||
}
|
||||
|
||||
fn named_value(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
let cookie = headers.get("cookie")?.to_str().ok()?;
|
||||
cookie.split(';').find_map(|part| {
|
||||
let (candidate, value) = part.trim().split_once('=')?;
|
||||
(candidate == NAME).then(|| value.to_owned())
|
||||
(candidate == name && !value.is_empty()).then(|| value.to_owned())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -321,6 +321,7 @@ impl MemoryAuthStore {
|
||||
},
|
||||
admin_user_id: "admin_1".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + Duration::hours(1),
|
||||
},
|
||||
@@ -339,7 +340,7 @@ impl MemoryAuthStore {
|
||||
self.sessions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(session_id)
|
||||
.get_mut(session_id.split('.').nth(1).unwrap_or(session_id))
|
||||
.unwrap()
|
||||
.expires_at = expires_at;
|
||||
}
|
||||
@@ -415,19 +416,24 @@ impl AuthStore for MemoryAuthStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
let mut sessions = self.sessions.lock().unwrap();
|
||||
if let Some(session) = sessions.get_mut(session_id) {
|
||||
if session.expires_at <= chrono::Utc::now() {
|
||||
if session.retained_until() <= chrono::Utc::now() {
|
||||
return Ok(None);
|
||||
}
|
||||
if session == expected {
|
||||
session.expires_at = expires_at;
|
||||
if let Some(replacement) = replacement {
|
||||
*session = replacement.clone();
|
||||
} else {
|
||||
sessions.remove(session_id);
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
return Ok(Some(session.clone()));
|
||||
}
|
||||
|
||||
@@ -23,6 +23,27 @@ fn session_cookie(response: &axum::response::Response) -> String {
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
fn refresh_cookie(response: &axum::response::Response) -> String {
|
||||
response
|
||||
.headers()
|
||||
.get_all(header::SET_COOKIE)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.find(|value| value.starts_with("cpr_refresh="))
|
||||
.expect("refresh cookie")
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
async fn refresh(app: &axum::Router, cookie: &str) -> axum::response::Response {
|
||||
app.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", cookie))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn login(
|
||||
app: &axum::Router,
|
||||
body: Value,
|
||||
@@ -537,64 +558,129 @@ async fn key_auth_should_issue_restore_and_clear_a_unified_cookie() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_renews_admin_cookie_and_preserves_fixed_key_expiry() {
|
||||
async fn refresh_restores_expired_access_with_only_the_refresh_cookie() {
|
||||
let (app, fixture) = auth_app().await;
|
||||
for body in [
|
||||
let response = login(
|
||||
&app,
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "apiKey": RAW_KEY}),
|
||||
] {
|
||||
let response = login(&app, body.clone(), None).await;
|
||||
let cookie = session_cookie(&response);
|
||||
let original = response_json(response).await["data"]["expiresAt"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
if body["mode"] == "admin" {
|
||||
fixture.set_session_expiry(
|
||||
cookie.split_once('=').unwrap().1,
|
||||
chrono::Utc::now() + chrono::Duration::minutes(1),
|
||||
);
|
||||
}
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", &cookie))
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let access = session_cookie(&response);
|
||||
let renewal = refresh_cookie(&response);
|
||||
let cookies: Vec<_> = response
|
||||
.headers()
|
||||
.get_all(header::SET_COOKIE)
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(cookies.len(), 2);
|
||||
assert!(cookies[0].contains("Path=/;"));
|
||||
assert!(cookies[1].contains("Path=/api/auth;"));
|
||||
assert!(
|
||||
cookies
|
||||
.iter()
|
||||
.all(|value| value.contains("HttpOnly") && value.contains("SameSite=Lax"))
|
||||
);
|
||||
assert_eq!(
|
||||
response_json(refresh(&app, &access).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
fixture.set_session_expiry(
|
||||
access.split_once('=').unwrap().1,
|
||||
chrono::Utc::now() - chrono::Duration::seconds(1),
|
||||
);
|
||||
assert_eq!(
|
||||
get(&app, "/api/admin/system/version", &access)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
assert_eq!(session_cookie(&response), cookie);
|
||||
assert!(
|
||||
response.headers()[header::SET_COOKIE]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains("HttpOnly")
|
||||
);
|
||||
let data = response_json(response).await;
|
||||
assert_eq!(data["data"]["authenticated"], true);
|
||||
let expiry = data["data"]["session"]["expiresAt"].as_str().unwrap();
|
||||
if body["mode"] == "key" {
|
||||
assert_eq!(expiry, original);
|
||||
} else {
|
||||
assert!(
|
||||
chrono::DateTime::parse_from_rfc3339(expiry).unwrap()
|
||||
> chrono::Utc::now() + chrono::Duration::minutes(1)
|
||||
);
|
||||
}
|
||||
}
|
||||
.status(),
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
assert_eq!(
|
||||
response_json(get(&app, "/api/auth/status", &renewal).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
|
||||
let response = refresh(&app, &renewal).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
let next_access = session_cookie(&response);
|
||||
let next_renewal = refresh_cookie(&response);
|
||||
assert_ne!(next_access, access);
|
||||
assert_ne!(next_renewal, renewal);
|
||||
assert_eq!(response_json(response).await["data"]["authenticated"], true);
|
||||
assert_eq!(
|
||||
get(&app, "/api/admin/system/version", &next_access)
|
||||
.await
|
||||
.status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(
|
||||
get(&app, "/api/admin/system/version", &access)
|
||||
.await
|
||||
.status(),
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
// 模拟响应丢失后的重试,返回同一代 Cookie
|
||||
let retry = refresh(&app, &renewal).await;
|
||||
assert_eq!(session_cookie(&retry), next_access);
|
||||
assert_eq!(refresh_cookie(&retry), next_renewal);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn key_refresh_preserves_fixed_expiry_and_clears_admin_refresh_cookie() {
|
||||
let (app, _) = auth_app().await;
|
||||
let admin = login(
|
||||
&app,
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let renewal = refresh_cookie(&admin);
|
||||
let response = login(
|
||||
&app,
|
||||
json!({"mode": "key", "apiKey": RAW_KEY}),
|
||||
Some(&renewal),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(refresh_cookie(&response), "cpr_refresh=");
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get_all(header::SET_COOKIE)
|
||||
.iter()
|
||||
.any(|value| {
|
||||
let value = value.to_str().unwrap();
|
||||
value.starts_with("cpr_refresh=")
|
||||
&& value.contains("Max-Age=0")
|
||||
&& value.contains("Path=/api/auth;")
|
||||
})
|
||||
);
|
||||
let cookie = session_cookie(&response);
|
||||
let original = response_json(response).await["data"]["expiresAt"].clone();
|
||||
let response = refresh(&app, &cookie).await;
|
||||
assert_eq!(session_cookie(&response), cookie);
|
||||
let data = response_json(response).await;
|
||||
assert_eq!(data["data"]["authenticated"], true);
|
||||
assert_eq!(data["data"]["session"]["expiresAt"], original);
|
||||
assert_eq!(
|
||||
response_json(refresh(&app, &renewal).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_after_logout_or_password_change_cannot_recreate_a_session() {
|
||||
let (app, _) = auth_app().await;
|
||||
let (app, fixture) = auth_app().await;
|
||||
for password_change in [false, true] {
|
||||
let cookie = session_cookie(
|
||||
&login(
|
||||
&app,
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
None,
|
||||
)
|
||||
.await,
|
||||
);
|
||||
let login_response = login(
|
||||
&app,
|
||||
json!({"mode": "admin", "password": "strong-admin-password"}),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let cookie = session_cookie(&login_response);
|
||||
let renewal = refresh_cookie(&login_response);
|
||||
let response = if password_change {
|
||||
change_password(
|
||||
&app,
|
||||
@@ -604,15 +690,34 @@ async fn refresh_after_logout_or_password_change_cannot_recreate_a_session() {
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
fixture.set_session_expiry(
|
||||
cookie.split_once('=').unwrap().1,
|
||||
chrono::Utc::now() - chrono::Duration::seconds(1),
|
||||
);
|
||||
app.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/logout", &cookie))
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/logout", &renewal))
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all(header::SET_COOKIE)
|
||||
.iter()
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.get_all(header::SET_COOKIE)
|
||||
.iter()
|
||||
.all(|value| value.to_str().unwrap().contains("Max-Age=0"))
|
||||
);
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", &cookie))
|
||||
.oneshot(cookie_request(Method::POST, "/api/auth/refresh", &renewal))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
@@ -560,8 +560,6 @@ where
|
||||
match self.pull().await? {
|
||||
PullOutcome::Events(events) => {
|
||||
self.replay_barrier_reached |= self.middleware.is_some()
|
||||
|| self.client_transport != super::execution::ClientTransport::WebSocket
|
||||
|| self.response_control.is_none()
|
||||
|| events.iter().any(|event| !event.is_replayable_prelude());
|
||||
let requirement = if self.delivery_committed {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
|
||||
@@ -559,12 +559,10 @@ impl AttemptContext {
|
||||
self.request.client_transport
|
||||
}
|
||||
|
||||
/// 原生双向通道可提前交付可替换前导事件;中间件可能改变其输出语义
|
||||
/// 原生响应可交付可替换前导事件;中间件可能改变其输出语义
|
||||
#[must_use]
|
||||
pub fn allows_replayable_prelude(&self) -> bool {
|
||||
self.client_transport() == execution::ClientTransport::WebSocket
|
||||
&& self.request.middleware.is_none()
|
||||
&& self.response_control().is_some()
|
||||
self.request.middleware.is_none()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -4286,6 +4286,85 @@ fn structural_event_before_replay_safe_failure_should_switch_account_before_comm
|
||||
assert_eq!(state.finalizations[0].client_status_code, Some(200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delivered_prelude_allows_retry_without_control_until_non_replayable_delivery() {
|
||||
for close_window in [false, true] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let prelude = || {
|
||||
ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({"type":"response.created","response":{"id":"response-1","output":[]}}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.with_replayable_prelude()
|
||||
};
|
||||
let mut items = vec![Ok(prelude())];
|
||||
if close_window {
|
||||
items.push(Ok(ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.future".to_owned()),
|
||||
json!({"type":"response.future"}),
|
||||
)
|
||||
.unwrap(),
|
||||
)));
|
||||
// 后续前导不能撤销已经交付未知事件的事实
|
||||
items.push(Ok(prelude()));
|
||||
}
|
||||
items.push(Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_replay_safe()));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items,
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
let first = block_on(session.next_event()).unwrap().unwrap();
|
||||
assert!(first.into_provider_events()[0].is_replayable_prelude());
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let result = loop {
|
||||
match block_on(session.next_event()) {
|
||||
Ok(Some(_)) => {}
|
||||
terminal => break terminal,
|
||||
}
|
||||
};
|
||||
assert_eq!(result.is_err(), close_window);
|
||||
let attempts = if close_window { 1 } else { 2 };
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), attempts);
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.commits, 1);
|
||||
assert_eq!(state.intermediate_failures, attempts - 1);
|
||||
assert_eq!(
|
||||
state.finalizations[0].outcome,
|
||||
if close_window {
|
||||
ExecutionOutcome::Incomplete
|
||||
} else {
|
||||
ExecutionOutcome::Succeeded
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_before_pending_delivery_commit_reaches_terminal_state() {
|
||||
let operation = generate_operation();
|
||||
|
||||
@@ -24,6 +24,7 @@ const MAXIMUM_RESPONSE_BYTES: usize = 512 * 1024;
|
||||
const DYNAMIC_CACHE_SECONDS: i64 = 300;
|
||||
const FALLBACK_CACHE_SECONDS: u64 = 60;
|
||||
const MINIMUM_REMAINING_LINK_SECONDS: i64 = 600;
|
||||
const OFFICIAL_METADATA_TIMEOUT: Duration = Duration::from_secs(3);
|
||||
|
||||
const OFFICIAL_X64_URL: &str = "https://persistent.oaistatic.com/codex-app-prod/ChatGPT-x64.msix";
|
||||
const OFFICIAL_ARM64_URL: &str =
|
||||
@@ -53,7 +54,7 @@ impl RgAdguardClientDistribution {
|
||||
pub(crate) fn new() -> Self {
|
||||
let client = Client::builder()
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.timeout(Duration::from_secs(12))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(Policy::none())
|
||||
.build()
|
||||
.inspect_err(|_| {
|
||||
@@ -69,10 +70,11 @@ impl RgAdguardClientDistribution {
|
||||
}
|
||||
|
||||
async fn resolve_uncached(&self) -> (CodexDesktopWindowsDownloads, Duration) {
|
||||
let now = Utc::now();
|
||||
match self.fetch_store_packages(now).await {
|
||||
match self.fetch_store_packages().await {
|
||||
Ok(packages) => {
|
||||
let (packages, warning) = complete_with_official_fallback(packages);
|
||||
let (mut packages, warning) = complete_with_official_fallback(packages);
|
||||
self.fill_official_package_sizes(&mut packages).await;
|
||||
let now = Utc::now();
|
||||
let valid_seconds = packages
|
||||
.iter()
|
||||
.filter_map(|package| package.expires_at)
|
||||
@@ -97,17 +99,33 @@ impl RgAdguardClientDistribution {
|
||||
error_kind = ?error_kind,
|
||||
"RG-Adguard package resolution failed; using official downloads"
|
||||
);
|
||||
(
|
||||
official_fallback(now),
|
||||
Duration::from_secs(FALLBACK_CACHE_SECONDS),
|
||||
)
|
||||
let mut downloads = official_fallback(Utc::now());
|
||||
self.fill_official_package_sizes(&mut downloads.packages)
|
||||
.await;
|
||||
downloads.resolved_at = Utc::now();
|
||||
(downloads, Duration::from_secs(FALLBACK_CACHE_SECONDS))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fill_official_package_sizes(&self, packages: &mut [ClientDownloadPackage]) {
|
||||
let Some(client) = self.client.as_ref() else {
|
||||
return;
|
||||
};
|
||||
// 只补充回退包的展示大小,超时或缺失长度不能撤销官方稳定下载地址
|
||||
futures::future::join_all(
|
||||
packages
|
||||
.iter_mut()
|
||||
.filter(|package| package.source == ClientDownloadSource::OfficialOpenAi)
|
||||
.map(|package| async {
|
||||
package.size_bytes = official_package_size(client, &package.download_url).await;
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn fetch_store_packages(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> Result<Vec<ClientDownloadPackage>, ClientDistributionResolutionErrorKind> {
|
||||
let client = self
|
||||
.client
|
||||
@@ -145,7 +163,8 @@ impl RgAdguardClientDistribution {
|
||||
}
|
||||
let html = std::str::from_utf8(&bytes)
|
||||
.map_err(|_| ClientDistributionResolutionErrorKind::InvalidResponse)?;
|
||||
parse_store_packages(html, now)
|
||||
// 上游解析可能耗时几十秒,按收到结果时的时间检查剩余有效期
|
||||
parse_store_packages(html, Utc::now())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,11 +337,12 @@ fn parse_size_bytes(value: &str) -> Option<u64> {
|
||||
if parts.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
// RG 页面以 KB/MB/GB 标记二进制大小,换算后才能与 CDN Content-Length 对齐
|
||||
let multiplier = match unit {
|
||||
"B" => 1_u128,
|
||||
"KB" => 1_000,
|
||||
"MB" => 1_000_000,
|
||||
"GB" => 1_000_000_000,
|
||||
"KB" => 1_024,
|
||||
"MB" => 1_048_576,
|
||||
"GB" => 1_073_741_824,
|
||||
_ => return None,
|
||||
};
|
||||
let (whole, fraction) = amount.split_once('.').unwrap_or((amount.as_str(), ""));
|
||||
@@ -351,7 +371,7 @@ fn official_fallback(now: DateTime<Utc>) -> CodexDesktopWindowsDownloads {
|
||||
resolved_at: now,
|
||||
cached: false,
|
||||
warning: Some(
|
||||
"Microsoft Store 离线直链暂不可用,已切换为 OpenAI 官方稳定安装包。".to_owned(),
|
||||
"Microsoft Store 离线直链暂不可用,已切换为 OpenAI 官方稳定安装包".to_owned(),
|
||||
),
|
||||
packages,
|
||||
}
|
||||
@@ -375,7 +395,7 @@ pub fn complete_with_official_fallback(
|
||||
}
|
||||
let warning = (!missing.is_empty()).then(|| {
|
||||
format!(
|
||||
"{} 的 Microsoft Store 临时直链不可用,已切换为 OpenAI 官方稳定安装包。",
|
||||
"{} 的 Microsoft Store 临时直链不可用,已切换为 OpenAI 官方稳定安装包",
|
||||
missing.join("、")
|
||||
)
|
||||
});
|
||||
@@ -397,3 +417,25 @@ fn official_package(architecture: ClientArchitecture) -> ClientDownloadPackage {
|
||||
expires_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 读取官方安装包的响应头大小,不下载包字节
|
||||
pub async fn official_package_size(client: &Client, download_url: &str) -> Option<u64> {
|
||||
let response = client
|
||||
.head(download_url)
|
||||
.timeout(OFFICIAL_METADATA_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !response.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
// HEAD 没有正文,不能用正文长度推断实际安装包大小
|
||||
response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_LENGTH)?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.filter(|size| *size > 0)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ fn missing_config_field(error: &config::ConfigError) -> Option<String> {
|
||||
}
|
||||
|
||||
/// Host 唯一拥有的进程配置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct HostConfig {
|
||||
#[serde(default)]
|
||||
pub timezone: gateway_core::time::DeploymentTimeZone,
|
||||
@@ -145,7 +145,7 @@ pub struct ListenConfig {
|
||||
}
|
||||
|
||||
/// Host 结构化日志配置
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq)]
|
||||
pub struct LoggingConfig {
|
||||
pub level: String,
|
||||
pub stdout: bool,
|
||||
@@ -157,17 +157,17 @@ pub struct LoggingConfig {
|
||||
/// 请求报文的留存窗口由 request_dump_retention_days 独立控制
|
||||
#[serde(default)]
|
||||
pub request_dump: bool,
|
||||
/// 完整报文的最少保留天数;按 UTC 日期整组清理,默认 1 天
|
||||
/// 完整报文的最少保留天数;支持小数,每天按 24 小时换算,默认 1 天
|
||||
#[serde(default = "default_request_dump_retention_days")]
|
||||
pub request_dump_retention_days: usize,
|
||||
pub request_dump_retention_days: f64,
|
||||
}
|
||||
|
||||
fn default_file_retention_days() -> usize {
|
||||
7
|
||||
}
|
||||
|
||||
fn default_request_dump_retention_days() -> usize {
|
||||
1
|
||||
fn default_request_dump_retention_days() -> f64 {
|
||||
1.0
|
||||
}
|
||||
|
||||
impl LoggingConfig {
|
||||
@@ -190,14 +190,19 @@ impl LoggingConfig {
|
||||
"host.logging.file.max_file_size_mb",
|
||||
));
|
||||
}
|
||||
if self.request_dump_retention_days == 0 {
|
||||
return Err(ConfigError::InvalidField(
|
||||
"host.logging.request_dump_retention_days",
|
||||
));
|
||||
}
|
||||
self.request_dump_retention()?;
|
||||
resolve_relative_path(source_dir, &mut self.file.directory);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn request_dump_retention(&self) -> Result<Duration, ConfigError> {
|
||||
Duration::try_from_secs_f64(self.request_dump_retention_days * 86_400.0)
|
||||
.ok()
|
||||
.filter(|duration| !duration.is_zero())
|
||||
.ok_or(ConfigError::InvalidField(
|
||||
"host.logging.request_dump_retention_days",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Host 文件日志配置
|
||||
|
||||
@@ -15,7 +15,7 @@ mod sink;
|
||||
mod writer;
|
||||
|
||||
use sink::{FileLogGuard, FileLogSink, LogHealth};
|
||||
use writer::RotatingLogWriter;
|
||||
use writer::{LogRetention, RotatingLogWriter};
|
||||
|
||||
// 文件前缀统一为 codex-proxy-rs-<kebab-case 类别>;专用 tracing target 使用 snake_case
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs-application";
|
||||
@@ -45,9 +45,11 @@ pub enum LogError {
|
||||
AlreadyInitialized,
|
||||
#[error("logging size limit is too large")]
|
||||
SizeOverflow,
|
||||
#[error(transparent)]
|
||||
InvalidConfig(#[from] crate::config::ConfigError),
|
||||
}
|
||||
|
||||
/// 初始化日志;文件按完整日期留存,不以分片数量淘汰窗口内记录
|
||||
/// 初始化日志;按各通道的保留窗口清理,不以分片数量淘汰记录
|
||||
pub fn initialize_logging(
|
||||
config: &LoggingConfig,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
@@ -74,7 +76,7 @@ pub fn initialize_logging(
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
timezone,
|
||||
config.file.retention_days,
|
||||
LogRetention::CalendarDays(config.file.retention_days),
|
||||
APPLICATION_LOG_FILE_PREFIX,
|
||||
"gateway-log-application-file",
|
||||
Arc::clone(&health),
|
||||
@@ -88,7 +90,7 @@ pub fn initialize_logging(
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
timezone,
|
||||
config.file.retention_days,
|
||||
LogRetention::CalendarDays(config.file.retention_days),
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
"gateway-log-oauth-recovery-file",
|
||||
Arc::clone(&health),
|
||||
@@ -102,7 +104,7 @@ pub fn initialize_logging(
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
timezone,
|
||||
config.request_dump_retention_days,
|
||||
LogRetention::Elapsed(config.request_dump_retention()?),
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
"gateway-log-request-dump-file",
|
||||
Arc::clone(&health),
|
||||
@@ -220,7 +222,7 @@ impl LogGuard {
|
||||
fn create_file_writer(
|
||||
config: &LoggingConfig,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
retention_days: usize,
|
||||
retention: LogRetention,
|
||||
prefix: &'static str,
|
||||
thread_name: &'static str,
|
||||
health: Arc<LogHealth>,
|
||||
@@ -234,7 +236,7 @@ fn create_file_writer(
|
||||
config.file.directory.clone(),
|
||||
prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
retention,
|
||||
Arc::clone(&health),
|
||||
timezone,
|
||||
)?;
|
||||
|
||||
@@ -1,22 +1,30 @@
|
||||
//! 按大小轮转、压缩已关闭分片、按部署时区自然日期组清理
|
||||
//! 按大小与时间轮转、压缩已关闭分片,按自然日期或实际时长清理
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use chrono::{NaiveDate, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
|
||||
use super::sink::LogHealth;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum LogRetention {
|
||||
CalendarDays(usize),
|
||||
Elapsed(Duration),
|
||||
}
|
||||
|
||||
pub(super) struct RotatingLogWriter {
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
directory: PathBuf,
|
||||
prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
retention: LogRetention,
|
||||
opened_at: Instant,
|
||||
date: NaiveDate,
|
||||
segment: usize,
|
||||
bytes_written: u64,
|
||||
@@ -29,19 +37,23 @@ impl RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
retention: LogRetention,
|
||||
health: Arc<LogHealth>,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = timezone.local(Utc::now()).date_naive();
|
||||
cleanup_log_files(&directory, prefix, date, retention_days, timezone)?;
|
||||
cleanup_log_files(&directory, prefix, retention, timezone, None)?;
|
||||
let latest = managed_log_files(&directory, prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| (entry.segment, entry.compressed));
|
||||
let (segment, bytes_written) = match latest {
|
||||
Some(entry) if !entry.compressed && entry.path.metadata()?.len() < maximum_bytes => {
|
||||
Some(entry)
|
||||
if matches!(retention, LogRetention::CalendarDays(_))
|
||||
&& !entry.compressed
|
||||
&& entry.path.metadata()?.len() < maximum_bytes =>
|
||||
{
|
||||
(entry.segment, entry.path.metadata()?.len())
|
||||
}
|
||||
Some(entry) => (next_segment(entry.segment)?, 0),
|
||||
@@ -54,7 +66,8 @@ impl RotatingLogWriter {
|
||||
directory,
|
||||
prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
retention,
|
||||
opened_at: Instant::now(),
|
||||
date,
|
||||
segment,
|
||||
bytes_written,
|
||||
@@ -78,7 +91,14 @@ impl RotatingLogWriter {
|
||||
let day_changed = date != self.date;
|
||||
let size_exceeded = self.bytes_written > 0
|
||||
&& self.bytes_written.saturating_add(incoming_bytes as u64) > self.maximum_bytes;
|
||||
if !day_changed && !size_exceeded {
|
||||
// 小流量转储也需及时关闭分片,避免小数保留期仍要等自然日切换才能生效
|
||||
let age_exceeded = match self.retention {
|
||||
LogRetention::CalendarDays(_) => false,
|
||||
LogRetention::Elapsed(duration) => {
|
||||
self.opened_at.elapsed() >= duration.min(Duration::from_secs(3600))
|
||||
}
|
||||
};
|
||||
if !day_changed && !size_exceeded && !age_exceeded {
|
||||
return Ok(());
|
||||
}
|
||||
let previous = self
|
||||
@@ -104,6 +124,7 @@ impl RotatingLogWriter {
|
||||
self.date = date;
|
||||
self.segment = segment;
|
||||
self.bytes_written = self.file.get_ref().metadata()?.len();
|
||||
self.opened_at = Instant::now();
|
||||
if let Err(error) = compress_log_file(&previous) {
|
||||
self.health.maintenance_failed(error.kind());
|
||||
}
|
||||
@@ -111,9 +132,9 @@ impl RotatingLogWriter {
|
||||
if let Err(error) = cleanup_log_files(
|
||||
&self.directory,
|
||||
self.prefix,
|
||||
date,
|
||||
self.retention_days,
|
||||
self.retention,
|
||||
self.timezone,
|
||||
Some(&path),
|
||||
) {
|
||||
self.health.maintenance_failed(error.kind());
|
||||
}
|
||||
@@ -203,17 +224,38 @@ fn managed_log_files(directory: &Path, prefix: &str) -> io::Result<Vec<ManagedLo
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
prefix: &str,
|
||||
today: NaiveDate,
|
||||
retention_days: usize,
|
||||
retention: LogRetention,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
active_path: Option<&Path>,
|
||||
) -> io::Result<()> {
|
||||
let files = managed_log_files(directory, prefix)?;
|
||||
let retention_days = match retention {
|
||||
LogRetention::CalendarDays(days) => days,
|
||||
LogRetention::Elapsed(duration) => {
|
||||
let now = SystemTime::now();
|
||||
for entry in files {
|
||||
if active_path == Some(entry.path.as_path()) {
|
||||
continue;
|
||||
}
|
||||
let modified = entry.path.metadata()?.modified()?;
|
||||
if now
|
||||
.duration_since(modified)
|
||||
.is_ok_and(|age| age >= duration)
|
||||
{
|
||||
fs::remove_file(entry.path)?;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let today = timezone.local(Utc::now()).date_naive();
|
||||
let days =
|
||||
u64::try_from(retention_days).map_err(|_| io::Error::other("log retention overflow"))?;
|
||||
let cutoff = today
|
||||
.checked_sub_days(chrono::Days::new(days))
|
||||
.unwrap_or(NaiveDate::MIN);
|
||||
let mut dates = BTreeMap::<NaiveDate, Vec<PathBuf>>::new();
|
||||
for entry in managed_log_files(directory, prefix)? {
|
||||
for entry in files {
|
||||
dates.entry(entry.date).or_default().push(entry.path);
|
||||
}
|
||||
for (date, paths) in dates {
|
||||
|
||||
@@ -30,7 +30,7 @@ fn parser_selects_latest_valid_package_for_each_architecture() {
|
||||
assert_eq!(packages[0].architecture, ClientArchitecture::X64);
|
||||
assert_eq!(packages[0].version.as_deref(), Some("26.825.6671.0"));
|
||||
assert!(packages[0].download_url.starts_with("http://"));
|
||||
assert_eq!(packages[0].size_bytes, Some(744_250_000));
|
||||
assert_eq!(packages[0].size_bytes, Some(780_402_688));
|
||||
assert_eq!(packages[1].architecture, ClientArchitecture::Arm64);
|
||||
assert!(packages[1].download_url.starts_with("https://"));
|
||||
}
|
||||
@@ -72,3 +72,74 @@ fn missing_architecture_uses_only_its_official_fallback() {
|
||||
assert_eq!(packages[1].source, ClientDownloadSource::OfficialOpenAi);
|
||||
assert!(warning.is_some_and(|value| value.contains("arm64")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn official_package_size_reads_head_length_without_downloading_the_body() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}/ChatGPT-x64.msix", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
|
||||
let read = stream.read(&mut buffer).await.unwrap();
|
||||
assert!(read > 0);
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
}
|
||||
assert!(request.starts_with(b"HEAD /ChatGPT-x64.msix HTTP/1.1\r\n"));
|
||||
// 只发送响应头,不发送大包正文,证明解析使用文件长度而不是 HEAD 正文长度
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 944929441\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
||||
assert_eq!(
|
||||
gateway_host::client_distribution::official_package_size(&client, &url).await,
|
||||
Some(944_929_441)
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn official_package_size_failure_keeps_metadata_optional() {
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{method, path},
|
||||
};
|
||||
|
||||
let server = MockServer::start().await;
|
||||
for (route, response) in [
|
||||
("/missing.msix", ResponseTemplate::new(404)),
|
||||
("/empty.msix", ResponseTemplate::new(200)),
|
||||
(
|
||||
"/slow.msix",
|
||||
ResponseTemplate::new(200).set_delay(std::time::Duration::from_secs(4)),
|
||||
),
|
||||
] {
|
||||
Mock::given(method("HEAD"))
|
||||
.and(path(route))
|
||||
.respond_with(response)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
}
|
||||
let client = reqwest::Client::builder().no_proxy().build().unwrap();
|
||||
for route in ["/missing.msix", "/empty.msix", "/slow.msix"] {
|
||||
assert_eq!(
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(3500),
|
||||
gateway_host::client_distribution::official_package_size(
|
||||
&client,
|
||||
&format!("{}{route}", server.uri()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("metadata timeout must not wait for the slow response"),
|
||||
None
|
||||
);
|
||||
}
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ fn logging_rejects_zero_retention_windows() {
|
||||
for request_dump in [false, true] {
|
||||
let mut config = valid_config();
|
||||
if request_dump {
|
||||
config.logging.request_dump_retention_days = 0;
|
||||
config.logging.request_dump_retention_days = 0.0;
|
||||
} else {
|
||||
config.logging.file.retention_days = 0;
|
||||
}
|
||||
@@ -168,7 +168,7 @@ fn logging_defaults_to_time_retention_and_ignores_removed_count_retention() {
|
||||
"file": { "enabled": true, "directory": "logs", "max_file_size_mb": 20 }
|
||||
});
|
||||
let config: LoggingConfig = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(config.request_dump_retention_days, 1);
|
||||
assert_eq!(config.request_dump_retention_days, 1.0);
|
||||
assert_eq!(config.file.retention_days, 7);
|
||||
assert!(!config.oauth_recovery);
|
||||
value["file"]["max_files"] = serde_json::json!(20);
|
||||
@@ -178,6 +178,55 @@ fn logging_defaults_to_time_retention_and_ignores_removed_count_retention() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_dump_retention_accepts_integer_and_fractional_yaml_days() {
|
||||
for (input, expected) in [("1", 1.0), ("0.5", 0.5), ("0.25", 0.25), ("1.5", 1.5)] {
|
||||
let yaml = format!(
|
||||
"level: info\nstdout: true\nrequest_dump_retention_days: {input}\nfile:\n enabled: true\n directory: logs\n max_file_size_mb: 20\n"
|
||||
);
|
||||
let mut config = valid_config();
|
||||
config.logging = config::Config::builder()
|
||||
.add_source(config::File::from_str(&yaml, config::FileFormat::Yaml))
|
||||
.build()
|
||||
.unwrap()
|
||||
.try_deserialize()
|
||||
.unwrap();
|
||||
config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(config.logging.request_dump_retention_days, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_dump_retention_rejects_invalid_durations() {
|
||||
for days in [
|
||||
0.0,
|
||||
-0.5,
|
||||
f64::NAN,
|
||||
f64::INFINITY,
|
||||
f64::NEG_INFINITY,
|
||||
f64::MAX,
|
||||
1e-20,
|
||||
] {
|
||||
let mut config = valid_config();
|
||||
config.logging.request_dump_retention_days = days;
|
||||
let error = config
|
||||
.resolve_and_validate(
|
||||
std::path::Path::new("/srv/gateway"),
|
||||
std::path::Path::new("/srv/gateway/web/dist"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_host::ConfigError::InvalidField("host.logging.request_dump_retention_days")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedicated_file_channels_can_be_enabled_without_application_sinks() {
|
||||
for oauth_recovery in [false, true] {
|
||||
@@ -220,7 +269,7 @@ fn valid_config() -> HostConfig {
|
||||
},
|
||||
oauth_recovery: false,
|
||||
request_dump: false,
|
||||
request_dump_retention_days: 1,
|
||||
request_dump_retention_days: 1.0,
|
||||
},
|
||||
system_update,
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
|
||||
@@ -48,7 +48,7 @@ fn logging_requires_at_least_one_sink() {
|
||||
},
|
||||
oauth_recovery: false,
|
||||
request_dump: false,
|
||||
request_dump_retention_days: 1,
|
||||
request_dump_retention_days: 1.0,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
@@ -325,7 +325,7 @@ fn logging_config(directory: PathBuf, request_dump: bool) -> HostConfig {
|
||||
},
|
||||
oauth_recovery: true,
|
||||
request_dump,
|
||||
request_dump_retention_days: 1,
|
||||
request_dump_retention_days: 1.0,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证日志压缩、轮转与重启后按日期保留完整记录
|
||||
//! 验证日志压缩、轮转,以及重启后的自然日与小数天数保留窗口
|
||||
|
||||
use super::*;
|
||||
use std::time::SystemTime;
|
||||
@@ -10,12 +10,11 @@ fn retention_preserves_complete_dates_across_compression_rotation_and_restart()
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
with_file_logging(
|
||||
PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()),
|
||||
true,
|
||||
false,
|
||||
|| {
|
||||
let payload = "x".repeat(1024 * 1024);
|
||||
// 三条记录已跨过 1 MiB 轮转边界;保留日期的 25 个分段由下方单独构造
|
||||
for sequence in 0..ROTATION_RECORDS {
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, sequence, payload, "retention record");
|
||||
tracing::info!(target: APPLICATION_LOG_TARGET, sequence, payload, "retention record");
|
||||
tracing::info!(target: OAUTH_RECOVERY_LOG_TARGET, sequence, payload, "retention record");
|
||||
}
|
||||
@@ -32,7 +31,6 @@ fn retention_preserves_complete_dates_across_compression_rotation_and_restart()
|
||||
for (prefix, days) in [
|
||||
(APPLICATION_LOG_FILE_PREFIX, 7),
|
||||
(OAUTH_RECOVERY_LOG_FILE_PREFIX, 7),
|
||||
(REQUEST_DUMP_LOG_FILE_PREFIX, 1),
|
||||
] {
|
||||
let boundary = today - chrono::Days::new(days);
|
||||
for segment in 1..=25 {
|
||||
@@ -90,11 +88,7 @@ fn retention_preserves_complete_dates_across_compression_rotation_and_restart()
|
||||
);
|
||||
assert!(unrelated.exists());
|
||||
|
||||
for prefix in [
|
||||
APPLICATION_LOG_FILE_PREFIX,
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
] {
|
||||
for prefix in [APPLICATION_LOG_FILE_PREFIX, OAUTH_RECOVERY_LOG_FILE_PREFIX] {
|
||||
let body = read_log_file_set(directory.path(), prefix);
|
||||
let mut sequences = [0; ROTATION_RECORDS];
|
||||
let mut boundaries = std::collections::BTreeSet::new();
|
||||
@@ -141,6 +135,151 @@ fn retention_preserves_complete_dates_across_compression_rotation_and_restart()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_dump_retention_uses_elapsed_time_across_compression_and_restart() {
|
||||
const RETENTION_ENV: &str = "CPR_LOGGING_TEST_RETENTION_DAYS";
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
let mut config =
|
||||
logging_config(PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()), true);
|
||||
config.logging.request_dump_retention_days =
|
||||
env::var(RETENTION_ENV).unwrap().parse().unwrap();
|
||||
with_logging(config, || {
|
||||
let payload = "x".repeat(1024 * 1024);
|
||||
for sequence in 0..ROTATION_RECORDS {
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, sequence, payload, "elapsed retention record");
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for days in [0.25, 0.5, 1.0, 1.5] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let now = chrono::Utc::now();
|
||||
let cutoff = now - chrono::Duration::seconds((days * 86_400.0) as i64);
|
||||
let mut retained = Vec::new();
|
||||
let mut expired = Vec::new();
|
||||
// 同一日期内既有过期文件也有有效文件,文件名日期不覆盖实际修改时间
|
||||
for (segment, modified, keep) in [
|
||||
(0, cutoff - chrono::Duration::minutes(1), false),
|
||||
(1, cutoff + chrono::Duration::minutes(1), true),
|
||||
(2, now + chrono::Duration::hours(1), true),
|
||||
] {
|
||||
let path = directory.path().join(format!(
|
||||
"{REQUEST_DUMP_LOG_FILE_PREFIX}{}.{segment}.log",
|
||||
now.date_naive()
|
||||
));
|
||||
fs::write(&path, format!("seed-{segment}\n")).unwrap();
|
||||
fs::File::open(&path)
|
||||
.unwrap()
|
||||
.set_modified(modified.into())
|
||||
.unwrap();
|
||||
if keep {
|
||||
let modified = path.metadata().unwrap().modified().unwrap();
|
||||
retained.push((path, modified));
|
||||
} else {
|
||||
expired.push(path);
|
||||
}
|
||||
}
|
||||
// 归档的修改时间也参与清理,压缩不能重置保留窗口
|
||||
let archive = directory.path().join(format!(
|
||||
"{REQUEST_DUMP_LOG_FILE_PREFIX}{}.3.log.gz",
|
||||
now.date_naive() - chrono::Days::new(3)
|
||||
));
|
||||
let mut encoder = flate2::write::GzEncoder::new(
|
||||
fs::File::create(&archive).unwrap(),
|
||||
flate2::Compression::fast(),
|
||||
);
|
||||
std::io::Write::write_all(&mut encoder, b"expired archive\n").unwrap();
|
||||
encoder
|
||||
.finish()
|
||||
.unwrap()
|
||||
.set_modified((cutoff - chrono::Duration::hours(1)).into())
|
||||
.unwrap();
|
||||
expired.push(archive);
|
||||
let unrelated = directory.path().join("unmanaged.log");
|
||||
fs::write(&unrelated, "unmanaged\n").unwrap();
|
||||
|
||||
for run in 1..=2 {
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args(["--exact", "logging::writer::request_dump_retention_uses_elapsed_time_across_compression_and_restart"])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env(LOG_TIMEZONE_ENV, "America/New_York")
|
||||
.env(RETENTION_ENV, days.to_string())
|
||||
.env("RUST_LOG", "off")
|
||||
.output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
for (path, modified) in &retained {
|
||||
let archive = path.with_extension("log.gz");
|
||||
assert!(archive.exists());
|
||||
assert_eq!(archive.metadata().unwrap().modified().unwrap(), *modified);
|
||||
}
|
||||
assert!(
|
||||
expired
|
||||
.iter()
|
||||
.all(|path| !path.exists() && !path.with_extension("log.gz").exists())
|
||||
);
|
||||
assert!(unrelated.exists());
|
||||
let body = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
assert!(body.contains("seed-1\n") && body.contains("seed-2\n"));
|
||||
assert_eq!(
|
||||
body.matches("elapsed retention record").count(),
|
||||
run * ROTATION_RECORDS
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_dump_rotates_small_files_when_the_time_window_elapses() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
let directory = PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap());
|
||||
let mut config = logging_config(directory.clone(), true);
|
||||
config.logging.request_dump_retention_days = 1.0 / 86_400.0;
|
||||
with_logging(config, || {
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, "before retention window");
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
loop {
|
||||
let body = read_log_file_set(&directory, REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
if body.contains("before retention window") {
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"first record was not flushed"
|
||||
);
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(1100));
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, "after retention window");
|
||||
});
|
||||
return;
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::writer::request_dump_rotates_small_files_when_the_time_window_elapses",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let body = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
assert!(!body.contains("before retention window"));
|
||||
assert!(body.contains("after retention window"));
|
||||
}
|
||||
|
||||
fn seed_log(path: &Path, date: chrono::NaiveDate, body: &str) {
|
||||
fs::write(path, body).unwrap();
|
||||
let modified = gateway_core::time::DeploymentTimeZone::default()
|
||||
|
||||
@@ -56,7 +56,7 @@ pub fn configuration(directory: &Path) -> HostConfig {
|
||||
},
|
||||
oauth_recovery: false,
|
||||
request_dump: false,
|
||||
request_dump_retention_days: 1,
|
||||
request_dump_retention_days: 1.0,
|
||||
},
|
||||
system_update: Default::default(),
|
||||
worker_shutdown_timeout_seconds: 1,
|
||||
|
||||
@@ -286,17 +286,17 @@ impl AuthStore for AuthStoreAdapter {
|
||||
.map_err(|error| admin_store_error("authentication session", error))
|
||||
}
|
||||
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSession,
|
||||
expires_at: chrono::DateTime<chrono::Utc>,
|
||||
replacement: Option<&AuthSession>,
|
||||
) -> AdminStoreResult<Option<AuthSession>> {
|
||||
redis::AuthStateRepository::renew_session(
|
||||
redis::AuthStateRepository::compare_exchange_session(
|
||||
&self.state,
|
||||
session_id,
|
||||
&auth_session_record(expected),
|
||||
expires_at,
|
||||
replacement.map(auth_session_record).as_ref(),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("authentication session renewal", error))?
|
||||
@@ -399,6 +399,7 @@ fn auth_session(record: redis::AuthSessionRecord) -> AdminStoreResult<AuthSessio
|
||||
subject,
|
||||
expires_at: record.expires_at,
|
||||
absolute_expires_at: record.absolute_expires_at,
|
||||
renewal: record.renewal,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -419,5 +420,6 @@ fn auth_session_record(session: &AuthSession) -> redis::AuthSessionRecord {
|
||||
subject,
|
||||
expires_at: session.expires_at,
|
||||
absolute_expires_at: session.absolute_expires_at,
|
||||
renewal: session.renewal.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,13 +55,28 @@ pub struct AuthSessionRecord {
|
||||
pub subject: SessionSubjectRecord,
|
||||
pub expires_at: DateTime<Utc>,
|
||||
pub absolute_expires_at: Option<DateTime<Utc>>,
|
||||
pub renewal: Option<gateway_admin::model::auth::SessionRenewal>,
|
||||
}
|
||||
|
||||
impl AuthSessionRecord {
|
||||
fn validate(&self) -> StoreResult<u64> {
|
||||
self.subject.validate()?;
|
||||
let expires_at_millis = u64::try_from(self.expires_at.timestamp_millis())
|
||||
.map_err(|_| auth_invalid("session expiry must be after the Unix epoch"))?;
|
||||
if let Some(renewal) = &self.renewal
|
||||
&& (!matches!(self.subject, SessionSubjectRecord::Admin { .. })
|
||||
|| self.expires_at > renewal.expires_at
|
||||
|| self
|
||||
.absolute_expires_at
|
||||
.is_none_or(|limit| renewal.expires_at > limit))
|
||||
{
|
||||
return Err(auth_invalid("invalid renewable session lifetime"));
|
||||
}
|
||||
let expires_at_millis = u64::try_from(
|
||||
self.renewal
|
||||
.as_ref()
|
||||
.map_or(self.expires_at, |renewal| renewal.expires_at)
|
||||
.timestamp_millis(),
|
||||
)
|
||||
.map_err(|_| auth_invalid("session expiry must be after the Unix epoch"))?;
|
||||
if expires_at_millis > MAX_REDIS_EXACT_INTEGER {
|
||||
return Err(auth_invalid(
|
||||
"session expiry is outside the supported range",
|
||||
@@ -82,11 +97,11 @@ pub trait AuthStateRepository: Send + Sync {
|
||||
async fn store_session(&self, session_id: &str, session: &AuthSessionRecord)
|
||||
-> StoreResult<()>;
|
||||
async fn delete_session(&self, session_id: &str) -> StoreResult<Option<AuthSessionRecord>>;
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSessionRecord,
|
||||
expires_at: DateTime<Utc>,
|
||||
replacement: Option<&AuthSessionRecord>,
|
||||
) -> StoreResult<Option<AuthSessionRecord>>;
|
||||
async fn consume_login_attempt(
|
||||
&self,
|
||||
@@ -127,29 +142,26 @@ impl RedisAuthStateRepository {
|
||||
|
||||
#[async_trait]
|
||||
impl AuthStateRepository for RedisAuthStateRepository {
|
||||
async fn renew_session(
|
||||
async fn compare_exchange_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
expected: &AuthSessionRecord,
|
||||
expires_at: DateTime<Utc>,
|
||||
replacement: Option<&AuthSessionRecord>,
|
||||
) -> StoreResult<Option<AuthSessionRecord>> {
|
||||
if expires_at < expected.expires_at
|
||||
|| expected
|
||||
.absolute_expires_at
|
||||
.is_none_or(|limit| expires_at > limit)
|
||||
{
|
||||
return Err(auth_invalid("session renewal is outside its lifetime"));
|
||||
}
|
||||
let renewed = AuthSessionRecord {
|
||||
expires_at,
|
||||
..expected.clone()
|
||||
};
|
||||
let expiry = renewed.validate()?;
|
||||
let payload = replacement.map(encode_session).transpose()?;
|
||||
let expiry = replacement
|
||||
.map(AuthSessionRecord::validate)
|
||||
.transpose()?
|
||||
.unwrap_or(0);
|
||||
// 比较和写入在同一脚本完成;GET 会检查 Redis 到期,退出后的键不能复活
|
||||
let payload: Option<String> = Script::new(
|
||||
r#"
|
||||
local current = redis.call('GET', KEYS[1])
|
||||
if current == ARGV[1] then
|
||||
if ARGV[2] == '' then
|
||||
redis.call('DEL', KEYS[1])
|
||||
return nil
|
||||
end
|
||||
redis.call('SET', KEYS[1], ARGV[2], 'PXAT', ARGV[3], 'XX')
|
||||
return ARGV[2]
|
||||
end
|
||||
@@ -158,7 +170,7 @@ return current
|
||||
)
|
||||
.key(self.session_key(session_id)?)
|
||||
.arg(encode_session(expected)?)
|
||||
.arg(encode_session(&renewed)?)
|
||||
.arg(payload.unwrap_or_default())
|
||||
.arg(expiry)
|
||||
.invoke_async(&mut self.connection.clone())
|
||||
.await
|
||||
@@ -250,11 +262,14 @@ struct AuthSessionWire {
|
||||
expires_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
absolute_expires_at: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
renewal: Option<gateway_admin::model::auth::SessionRenewal>,
|
||||
}
|
||||
|
||||
fn encode_session(session: &AuthSessionRecord) -> StoreResult<String> {
|
||||
serde_json::to_string(&AuthSessionWire {
|
||||
subject: session.subject.clone(),
|
||||
renewal: session.renewal.clone(),
|
||||
expires_at: session
|
||||
.expires_at
|
||||
.to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
@@ -277,6 +292,7 @@ fn decode_session(value: &str) -> StoreResult<AuthSessionRecord> {
|
||||
let expires_at = parse(&wire.expires_at)?;
|
||||
Ok(AuthSessionRecord {
|
||||
subject: wire.subject,
|
||||
renewal: wire.renewal,
|
||||
expires_at,
|
||||
absolute_expires_at: wire.absolute_expires_at.as_deref().map(parse).transpose()?,
|
||||
})
|
||||
|
||||
@@ -20,6 +20,7 @@ async fn client_session_should_use_fixed_ttl_without_storing_session_or_api_key_
|
||||
subject: SessionSubjectRecord::Key {
|
||||
client_key_id: "key-42".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + chrono::Duration::seconds(60),
|
||||
};
|
||||
@@ -116,6 +117,7 @@ async fn client_session_should_reject_invalid_intervals_and_rate_limit_policies(
|
||||
subject: SessionSubjectRecord::Key {
|
||||
client_key_id: "key-42".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: now,
|
||||
};
|
||||
@@ -156,6 +158,7 @@ fn admin_auth_state_rejects_invalid_ttl_boundaries() {
|
||||
credential_fingerprint: String::new(),
|
||||
admin_user_id: "admin".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() - chrono::Duration::seconds(1),
|
||||
};
|
||||
@@ -183,6 +186,7 @@ async fn admin_auth_state_keeps_fixed_ttl_and_opaque_keys() {
|
||||
credential_fingerprint: "test-password-fingerprint".to_owned(),
|
||||
admin_user_id: "default-admin".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + chrono::Duration::seconds(60),
|
||||
};
|
||||
@@ -252,6 +256,7 @@ async fn unified_session_rejects_invalid_identities_and_loads_legacy_admin_paylo
|
||||
subject: SessionSubjectRecord::Key {
|
||||
client_key_id: "key-42".to_owned(),
|
||||
},
|
||||
renewal: None,
|
||||
absolute_expires_at: None,
|
||||
expires_at: Utc::now() + chrono::Duration::seconds(60),
|
||||
},
|
||||
@@ -309,83 +314,83 @@ async fn unified_session_rejects_invalid_identities_and_loads_legacy_admin_paylo
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_is_atomic_monotonic_and_cannot_resurrect_a_deleted_session() {
|
||||
let Some((repository, _, _)) = auth_repository().await else {
|
||||
return;
|
||||
};
|
||||
let original = AuthSessionRecord {
|
||||
fn renewable_record() -> AuthSessionRecord {
|
||||
AuthSessionRecord {
|
||||
subject: SessionSubjectRecord::Admin {
|
||||
admin_user_id: "admin".into(),
|
||||
credential_fingerprint: "fingerprint".into(),
|
||||
},
|
||||
expires_at: Utc::now() + chrono::Duration::minutes(1),
|
||||
expires_at: Utc::now() - chrono::Duration::minutes(1),
|
||||
absolute_expires_at: Some(Utc::now() + chrono::Duration::minutes(10)),
|
||||
renewal: Some(gateway_admin::model::auth::SessionRenewal {
|
||||
expires_at: Utc::now() + chrono::Duration::minutes(5),
|
||||
access_digest: [1; 32],
|
||||
refresh_digest: [2; 32],
|
||||
previous: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rotation_is_atomic_and_logout_cannot_be_undone_by_inflight_refresh() {
|
||||
let Some((repository, _, _)) = auth_repository().await else {
|
||||
return;
|
||||
};
|
||||
let original = renewable_record();
|
||||
repository
|
||||
.store_session("renewal", &original)
|
||||
.store_session("rotation", &original)
|
||||
.await
|
||||
.unwrap();
|
||||
let expiry = Utc::now() + chrono::Duration::minutes(5);
|
||||
let mut replacement = original.clone();
|
||||
replacement.expires_at = Utc::now() + chrono::Duration::minutes(1);
|
||||
replacement.renewal.as_mut().unwrap().refresh_digest = [3; 32];
|
||||
let mut competing = replacement.clone();
|
||||
competing.renewal.as_mut().unwrap().refresh_digest = [4; 32];
|
||||
let (first, second) = tokio::join!(
|
||||
repository.renew_session("renewal", &original, expiry),
|
||||
repository.renew_session("renewal", &original, expiry + chrono::Duration::minutes(1)),
|
||||
repository.compare_exchange_session("rotation", &original, Some(&replacement)),
|
||||
repository.compare_exchange_session("rotation", &original, Some(&competing)),
|
||||
);
|
||||
let first = first.unwrap().unwrap();
|
||||
assert_eq!(Some(first.clone()), second.unwrap());
|
||||
assert_eq!(first.absolute_expires_at, original.absolute_expires_at);
|
||||
assert!(first.expires_at >= expiry);
|
||||
assert_eq!(second.unwrap(), Some(first.clone()));
|
||||
assert_eq!(
|
||||
repository.load_session("rotation").await.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
// 过时快照不能撤销已轮换的状态;领域层重新核对后才决定是否撤销
|
||||
assert_eq!(
|
||||
repository
|
||||
.renew_session("renewal", &original, original.expires_at)
|
||||
.compare_exchange_session("rotation", &original, None)
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(first.clone())
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.renew_session(
|
||||
"renewal",
|
||||
&first,
|
||||
original.absolute_expires_at.unwrap() + chrono::Duration::seconds(1)
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
repository.delete_session("renewal").await.unwrap();
|
||||
assert!(
|
||||
repository
|
||||
.renew_session("renewal", &first, expiry + chrono::Duration::minutes(2))
|
||||
.compare_exchange_session("rotation", &first, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(repository.load_session("renewal").await.unwrap().is_none());
|
||||
assert!(
|
||||
repository
|
||||
.compare_exchange_session("rotation", &first, Some(&replacement))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(repository.load_session("rotation").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_updates_redis_ttl_and_rejects_a_key_that_expired_after_loading() {
|
||||
async fn redis_retains_refresh_after_access_expiry_but_never_resurrects_expired_sessions() {
|
||||
let Some((repository, mut connection, namespace)) = auth_repository().await else {
|
||||
return;
|
||||
};
|
||||
let original = AuthSessionRecord {
|
||||
subject: SessionSubjectRecord::Admin {
|
||||
admin_user_id: "admin".into(),
|
||||
credential_fingerprint: "fingerprint".into(),
|
||||
},
|
||||
expires_at: Utc::now() + chrono::Duration::minutes(1),
|
||||
absolute_expires_at: Some(Utc::now() + chrono::Duration::minutes(10)),
|
||||
};
|
||||
let original = renewable_record();
|
||||
repository
|
||||
.store_session("expiry-race", &original)
|
||||
.await
|
||||
.unwrap();
|
||||
let expiry = Utc::now() + chrono::Duration::minutes(5);
|
||||
let renewed = repository
|
||||
.renew_session("expiry-race", &original, expiry)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let keys: Vec<String> = redis::cmd("KEYS")
|
||||
.arg(format!("{namespace}:session:*"))
|
||||
.query_async(&mut connection)
|
||||
@@ -397,6 +402,25 @@ async fn renewal_updates_redis_ttl_and_rejects_a_key_that_expired_after_loading(
|
||||
.await
|
||||
.unwrap();
|
||||
assert!((240_000..=300_000).contains(&ttl));
|
||||
assert_eq!(
|
||||
repository.load_session("expiry-race").await.unwrap(),
|
||||
Some(original.clone())
|
||||
);
|
||||
let mut replacement = original.clone();
|
||||
replacement.renewal.as_mut().unwrap().expires_at = Utc::now() + chrono::Duration::minutes(6);
|
||||
assert_eq!(
|
||||
repository
|
||||
.compare_exchange_session("expiry-race", &original, Some(&replacement))
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(replacement.clone())
|
||||
);
|
||||
let ttl: i64 = redis::cmd("PTTL")
|
||||
.arg(&keys[0])
|
||||
.query_async(&mut connection)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!((300_000..=360_000).contains(&ttl));
|
||||
redis::cmd("PEXPIRE")
|
||||
.arg(&keys[0])
|
||||
.arg(1)
|
||||
@@ -406,11 +430,7 @@ async fn renewal_updates_redis_ttl_and_rejects_a_key_that_expired_after_loading(
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
assert!(
|
||||
repository
|
||||
.renew_session(
|
||||
"expiry-race",
|
||||
&renewed,
|
||||
expiry + chrono::Duration::minutes(1)
|
||||
)
|
||||
.compare_exchange_session("expiry-race", &replacement, Some(&replacement))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
|
||||
@@ -164,10 +164,8 @@ impl CodexCredentialQuotaService {
|
||||
"service_tier".to_owned(),
|
||||
Value::String("default".to_owned()),
|
||||
);
|
||||
body.insert(
|
||||
"reasoning".to_owned(),
|
||||
serde_json::json!({"effort": "none"}),
|
||||
);
|
||||
// 目录里各模型的最低档都是 low,none 会收到上游 error 终态并让整次预热记为失败
|
||||
body.insert("reasoning".to_owned(), serde_json::json!({"effort": "low"}));
|
||||
body.insert("text".to_owned(), serde_json::json!({"verbosity": "low"}));
|
||||
|
||||
let upstream_request = CodexResponsesRequest::from_body(body);
|
||||
|
||||
@@ -1758,6 +1758,8 @@ pub enum CredentialSelectionError {
|
||||
InvalidCredential(#[source] Option<ErrorSource>),
|
||||
#[error("Codex account changed repeatedly during selection")]
|
||||
AccountSnapshotChanged,
|
||||
#[error("Codex credential revision conflict")]
|
||||
RevisionConflict(#[source] ErrorSource),
|
||||
#[error("Codex account store is unavailable")]
|
||||
Store(#[source] ErrorSource),
|
||||
#[error("Codex account lease runtime is unavailable")]
|
||||
@@ -1776,8 +1778,10 @@ impl From<CredentialRepositoryError> for CredentialSelectionError {
|
||||
CredentialRepositoryError::InvalidCredentialData(_) => {
|
||||
Self::InvalidCredential(Some(ErrorSource::new(error)))
|
||||
}
|
||||
CredentialRepositoryError::RevisionConflict(_)
|
||||
| CredentialRepositoryError::Store(_) => Self::Store(ErrorSource::new(error)),
|
||||
CredentialRepositoryError::RevisionConflict(_) => {
|
||||
Self::RevisionConflict(ErrorSource::new(error))
|
||||
}
|
||||
CredentialRepositoryError::Store(_) => Self::Store(ErrorSource::new(error)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -919,18 +919,33 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
if allows_account_state_mutation
|
||||
&& !response.set_cookie_headers.is_empty()
|
||||
&& let Ok(outcome) = selector
|
||||
{
|
||||
match selector
|
||||
.capture_response_cookies(
|
||||
&active_account,
|
||||
&response_origin,
|
||||
&response.set_cookie_headers,
|
||||
)
|
||||
.await
|
||||
&& let Some(revision) = outcome.credential_revision
|
||||
&& let Ok(current) = selector.current_account(active_account.id()).await
|
||||
&& current.revision().get() == revision
|
||||
{
|
||||
active_account = current;
|
||||
{
|
||||
Ok(outcome) => {
|
||||
if let Some(revision) = outcome.credential_revision
|
||||
&& let Ok(current) = selector.current_account(active_account.id()).await
|
||||
&& current.revision().get() == revision
|
||||
{
|
||||
active_account = current;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
request_id = %request_id,
|
||||
account_id = %active_account.id(),
|
||||
credential_revision = active_account.revision().get(),
|
||||
error = %error,
|
||||
"Failed to persist OpenAI response cookies"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let response_transport = response.transport;
|
||||
let websocket_connection_id = response.websocket_connection_id;
|
||||
@@ -940,7 +955,6 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
// opening ID 标识连接,不可作为缺失请求级错误头时的当前请求 ID
|
||||
failure_diagnostics.request_id = None;
|
||||
}
|
||||
let failure_set_cookie_headers = response.set_cookie_headers.clone();
|
||||
let failure_rate_limit_headers = response.rate_limit_headers.clone();
|
||||
let mut passive_quota_observation =
|
||||
OpenAiPassiveQuotaObservation::new(response.rate_limit_headers);
|
||||
@@ -954,11 +968,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
.with_requested_service_tier(request.service_tier())
|
||||
.with_request_tool_pricing(upstream_model.as_str(), request.tools())
|
||||
.with_raw_sse_passthrough();
|
||||
let mut pre_commit_events = PreCommitClientEvents::new(trace).with_replayable_prelude(
|
||||
websocket_transport
|
||||
&& request.responses_lite.as_deref() == Some("true")
|
||||
&& context.allows_replayable_prelude(),
|
||||
);
|
||||
let mut client_events = ClientEventDelivery::new(context.allows_replayable_prelude());
|
||||
loop {
|
||||
if context.deadline().is_elapsed() {
|
||||
if allows_account_state_mutation {
|
||||
@@ -1016,8 +1026,10 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
&mut decoder,
|
||||
)
|
||||
.await;
|
||||
let observation_event = if rate_limits_changed || metadata_merge.is_some() {
|
||||
observation_state.observation(None).map(ProviderEvent::observation)
|
||||
let observation_event = if rate_limits_changed || metadata_merge.is_some()
|
||||
|| response_transport == CodexBackendTransport::WebSocket
|
||||
{
|
||||
observation_state.observation(Some(&failure.error)).map(ProviderEvent::observation)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -1028,7 +1040,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy: WebSocketFailurePolicy::Budgeted,
|
||||
replay_boundary: ReplayBoundary::from_semantic_output(pre_commit_events.is_committed()),
|
||||
replay_boundary: ReplayBoundary::from_semantic_output(client_events.replay_closed()),
|
||||
requirement: request_transport_requirement,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
@@ -1057,7 +1069,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
Err(quota_continuation_replay_error(
|
||||
failure.error,
|
||||
&request,
|
||||
ReplayBoundary::from_semantic_output(pre_commit_events.is_committed()),
|
||||
ReplayBoundary::from_semantic_output(client_events.replay_closed()),
|
||||
))?;
|
||||
return;
|
||||
}
|
||||
@@ -1072,7 +1084,6 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
};
|
||||
let first_event_changed =
|
||||
observation_state.observe_stream_chunk(&chunk, context.timing_started_at());
|
||||
let chunk_len = chunk.len();
|
||||
let decoded = if websocket_transport {
|
||||
let text = std::str::from_utf8(&chunk).map_err(|_| provider_error(ProviderErrorKind::Protocol, UpstreamSendState::Sent))?;
|
||||
decoder.push_websocket(text)
|
||||
@@ -1094,7 +1105,6 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
)
|
||||
.await;
|
||||
let metadata_changed = metadata_merge.unwrap_or(false);
|
||||
pre_commit_events.observe_chunk(chunk_len);
|
||||
let response_model_changed = observation_state
|
||||
.observe_upstream_response_model(decoder.response_model());
|
||||
let service_tier_changed = observation_state
|
||||
@@ -1114,10 +1124,9 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
map_canonical_error(
|
||||
error,
|
||||
&failure_diagnostics,
|
||||
&failure_set_cookie_headers,
|
||||
&failure_rate_limit_headers,
|
||||
ReplayBoundary::from_semantic_output(
|
||||
semantic_output_seen || pre_commit_events.is_committed(),
|
||||
semantic_output_seen || client_events.replay_closed(),
|
||||
),
|
||||
),
|
||||
atomic_upstream_failure,
|
||||
@@ -1183,15 +1192,15 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
if let Some((mut failure, atomic_upstream_failure)) = terminal_failure {
|
||||
let failure_after_commit =
|
||||
timing_signals.semantic_output || pre_commit_events.is_committed();
|
||||
timing_signals.semantic_output || client_events.replay_closed();
|
||||
if failure_after_commit {
|
||||
for event in pre_commit_events.commit(events) {
|
||||
for event in client_events.commit(events) {
|
||||
yield event;
|
||||
}
|
||||
} else if atomic_upstream_failure {
|
||||
failure.error = failure
|
||||
.error
|
||||
.with_atomic_client_events(pre_commit_events.take_for_failure(events));
|
||||
.with_atomic_client_events(events);
|
||||
}
|
||||
Err(quota_continuation_replay_error(
|
||||
failure.error,
|
||||
@@ -1200,7 +1209,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
))?;
|
||||
return;
|
||||
}
|
||||
let events = pre_commit_events.stage(events, timing_signals, completed);
|
||||
let events = client_events.forward(events, timing_signals, completed);
|
||||
for event in events {
|
||||
yield event;
|
||||
}
|
||||
@@ -1230,10 +1239,9 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
map_canonical_error(
|
||||
error,
|
||||
&failure_diagnostics,
|
||||
&failure_set_cookie_headers,
|
||||
&failure_rate_limit_headers,
|
||||
ReplayBoundary::from_semantic_output(
|
||||
semantic_output_seen || pre_commit_events.is_committed(),
|
||||
semantic_output_seen || client_events.replay_closed(),
|
||||
),
|
||||
),
|
||||
atomic_upstream_failure,
|
||||
@@ -1317,15 +1325,15 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
if let Some((mut failure, atomic_upstream_failure)) = terminal_failure {
|
||||
let failure_after_commit =
|
||||
timing_signals.semantic_output || pre_commit_events.is_committed();
|
||||
timing_signals.semantic_output || client_events.replay_closed();
|
||||
if failure_after_commit {
|
||||
for event in pre_commit_events.commit(events) {
|
||||
for event in client_events.commit(events) {
|
||||
yield event;
|
||||
}
|
||||
} else if atomic_upstream_failure {
|
||||
failure.error = failure
|
||||
.error
|
||||
.with_atomic_client_events(pre_commit_events.take_for_failure(events));
|
||||
.with_atomic_client_events(events);
|
||||
}
|
||||
Err(quota_continuation_replay_error(
|
||||
failure.error,
|
||||
@@ -1334,7 +1342,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
))?;
|
||||
return;
|
||||
}
|
||||
let events = pre_commit_events.finish(events, timing_signals, completed);
|
||||
let events = client_events.forward(events, timing_signals, completed);
|
||||
for event in events {
|
||||
yield event;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! OpenAI 上游失败分类、恢复决策与稳定错误投影
|
||||
|
||||
use super::*;
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use crate::transport::protocol::responses::is_replaceable_response_prelude;
|
||||
|
||||
/// OpenAI 失败对 Smart 账号分数的结构化 reason 闭集
|
||||
///
|
||||
@@ -69,128 +69,50 @@ impl MappedProviderFailure {
|
||||
}
|
||||
}
|
||||
|
||||
/// 提交边界前的上游事件预取
|
||||
///
|
||||
/// 原生 Lite 双向通道立即交付正常事件,其他通道保留到语义输出、终态或 EOF
|
||||
/// committed 只表示重放窗口已结束,不等同于首次 wire 已交付
|
||||
pub(super) struct PreCommitClientEvents {
|
||||
pending: Vec<ProviderEvent>,
|
||||
prefetched_bytes: usize,
|
||||
started_at: tokio::time::Instant,
|
||||
committed: bool,
|
||||
/// 正常事件立即交付,只记录后续失败能否请求重放
|
||||
/// 可替换前导不关闭窗口;实际输出、终态和未知事件使边界单向推进
|
||||
pub(super) struct ClientEventDelivery {
|
||||
replay_closed: bool,
|
||||
replayable_prelude: bool,
|
||||
trace: TraceContext,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum PreCommitReleaseReason {
|
||||
SemanticOutput,
|
||||
NonReplayableEvent,
|
||||
Terminal,
|
||||
Eof,
|
||||
}
|
||||
|
||||
impl PreCommitClientEvents {
|
||||
pub(super) fn new(trace: TraceContext) -> Self {
|
||||
impl ClientEventDelivery {
|
||||
pub(super) fn new(replayable_prelude: bool) -> Self {
|
||||
Self {
|
||||
pending: Vec::new(),
|
||||
prefetched_bytes: 0,
|
||||
started_at: tokio::time::Instant::now(),
|
||||
committed: false,
|
||||
replayable_prelude: false,
|
||||
trace,
|
||||
replay_closed: false,
|
||||
replayable_prelude,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn with_replayable_prelude(mut self, enabled: bool) -> Self {
|
||||
self.replayable_prelude = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn observe_chunk(&mut self, bytes: usize) {
|
||||
if !self.committed {
|
||||
self.prefetched_bytes = self.prefetched_bytes.saturating_add(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn stage(
|
||||
pub(super) fn forward(
|
||||
&mut self,
|
||||
incoming: Vec<ProviderEvent>,
|
||||
timing_signals: ResponseEventSignals,
|
||||
completed: bool,
|
||||
) -> Vec<ProviderEvent> {
|
||||
if self.committed {
|
||||
if self.replay_closed || incoming.is_empty() {
|
||||
return incoming;
|
||||
}
|
||||
self.pending.extend(incoming);
|
||||
if timing_signals.semantic_output {
|
||||
return self.commit_pending(PreCommitReleaseReason::SemanticOutput);
|
||||
if timing_signals.semantic_output
|
||||
|| completed
|
||||
|| !self.replayable_prelude
|
||||
|| !incoming.iter().all(is_replaceable_prelude)
|
||||
{
|
||||
return self.commit(incoming);
|
||||
}
|
||||
if completed {
|
||||
return self.commit_pending(PreCommitReleaseReason::Terminal);
|
||||
}
|
||||
if self.replayable_prelude {
|
||||
// 未确认可重放的事件也立即交付,但后续前导不能重新开启重试窗口
|
||||
if !self.pending.iter().all(is_replaceable_prelude) {
|
||||
return self.commit_pending(PreCommitReleaseReason::NonReplayableEvent);
|
||||
}
|
||||
self.prefetched_bytes = 0;
|
||||
return std::mem::take(&mut self.pending)
|
||||
.into_iter()
|
||||
.map(ProviderEvent::with_replayable_prelude)
|
||||
.collect();
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
pub(super) fn finish(
|
||||
&mut self,
|
||||
incoming: Vec<ProviderEvent>,
|
||||
timing_signals: ResponseEventSignals,
|
||||
completed: bool,
|
||||
) -> Vec<ProviderEvent> {
|
||||
let mut events = self.stage(incoming, timing_signals, completed);
|
||||
if !self.committed {
|
||||
events.extend(self.commit_pending(PreCommitReleaseReason::Eof));
|
||||
}
|
||||
events
|
||||
incoming
|
||||
.into_iter()
|
||||
.map(ProviderEvent::with_replayable_prelude)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn commit(&mut self, incoming: Vec<ProviderEvent>) -> Vec<ProviderEvent> {
|
||||
if self.committed {
|
||||
return incoming;
|
||||
}
|
||||
self.pending.extend(incoming);
|
||||
// 此入口只在同批次已出现语义输出、随后失败时释放原始事件
|
||||
self.commit_pending(PreCommitReleaseReason::SemanticOutput)
|
||||
self.replay_closed = true;
|
||||
incoming
|
||||
}
|
||||
|
||||
pub(super) fn take_for_failure(&mut self, incoming: Vec<ProviderEvent>) -> Vec<ProviderEvent> {
|
||||
self.pending.extend(incoming);
|
||||
self.prefetched_bytes = 0;
|
||||
std::mem::take(&mut self.pending)
|
||||
}
|
||||
|
||||
pub(super) const fn is_committed(&self) -> bool {
|
||||
self.committed
|
||||
}
|
||||
|
||||
pub(super) fn commit_pending(&mut self, reason: PreCommitReleaseReason) -> Vec<ProviderEvent> {
|
||||
// Provider 只记录释放缓存的原因;实际下游提交仍由 Core 记录和裁决
|
||||
if self.trace.is_enabled() {
|
||||
self.trace.record(
|
||||
"provider.precommit.released",
|
||||
json!({
|
||||
"reason": reason,
|
||||
"prefetchedBytes": self.prefetched_bytes,
|
||||
"waitMs": self.started_at.elapsed().as_millis(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
self.committed = true;
|
||||
self.prefetched_bytes = 0;
|
||||
std::mem::take(&mut self.pending)
|
||||
pub(super) const fn replay_closed(&self) -> bool {
|
||||
self.replay_closed
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,12 +120,14 @@ fn is_replaceable_prelude(event: &ProviderEvent) -> bool {
|
||||
event
|
||||
.wire_event()
|
||||
.is_some_and(|wire| match wire.event_type() {
|
||||
Some("codex.response.metadata" | "codex.rate_limits") => true,
|
||||
Some("response.created" | "response.in_progress") => wire
|
||||
.data()
|
||||
.pointer("/response/output")
|
||||
.is_none_or(|output| output.as_array().is_some_and(Vec::is_empty)),
|
||||
_ => false,
|
||||
Some(event_type) => is_replaceable_response_prelude(event_type, wire.data()),
|
||||
// 上游 SSE 注释也即时透传,但不能把传输心跳当作业务交付
|
||||
None => wire.raw_sse_frame().is_some_and(|frame| {
|
||||
std::str::from_utf8(frame).is_ok_and(|text| {
|
||||
text.lines()
|
||||
.all(|line| line.is_empty() || line.starts_with(':'))
|
||||
})
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -427,6 +351,7 @@ pub(super) async fn apply_failure(
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
credential_revision = account.revision().get(),
|
||||
error = %error,
|
||||
"Failed to persist OpenAI response cookies"
|
||||
);
|
||||
@@ -538,7 +463,7 @@ pub(super) fn apply_websocket_recovery_policy(
|
||||
if failure.error.retry_is_prohibited() {
|
||||
return;
|
||||
}
|
||||
// 明确账号拒绝走已有换号路径,容量拒绝走请求内退避;两者都不消耗 WS 传输预算
|
||||
// 明确账号拒绝走已有换号路径,业务瞬态失败走请求内退避,不消耗 WS 传输预算
|
||||
if failure.error.replay_is_safe()
|
||||
&& (failure.account_failure.is_some()
|
||||
|| matches!(
|
||||
@@ -787,7 +712,6 @@ pub(super) fn stream_transport_allows_pre_delivery_retry(error: &CodexClientErro
|
||||
pub(super) fn map_canonical_error(
|
||||
error: CodexCanonicalError,
|
||||
diagnostics: &CodexUpstreamDiagnostics,
|
||||
set_cookie_headers: &[String],
|
||||
rate_limit_headers: &[(String, String)],
|
||||
replay_boundary: ReplayBoundary,
|
||||
) -> MappedProviderFailure {
|
||||
@@ -797,7 +721,8 @@ pub(super) fn map_canonical_error(
|
||||
CodexUpstreamFailure::from_sse_failure(
|
||||
&failure,
|
||||
diagnostics,
|
||||
set_cookie_headers,
|
||||
// 响应头在开始消费流时已写回,流内失败不重放同一批 Cookie
|
||||
&[],
|
||||
rate_limit_headers,
|
||||
CodexUpstreamSendPhase::AfterPayload,
|
||||
),
|
||||
@@ -1402,6 +1327,7 @@ pub(super) fn map_upstream_failure(
|
||||
) -> MappedProviderFailure {
|
||||
let category = failure.category();
|
||||
let capacity_unavailable = category == CodexFailureCategory::CapacityUnavailable;
|
||||
let transient_retry = capacity_unavailable || failure.is_retryable_server_error();
|
||||
let cyber_policy_failure = failure
|
||||
.status
|
||||
.is_some_and(|status| status.is_client_error())
|
||||
@@ -1468,7 +1394,7 @@ pub(super) fn map_upstream_failure(
|
||||
if let Some(retry_after) = failure.retry_after_seconds.map(Duration::from_secs) {
|
||||
error = error.with_retry_after(retry_after);
|
||||
}
|
||||
if capacity_unavailable && error.replay_is_safe() {
|
||||
if transient_retry && error.replay_is_safe() {
|
||||
let max_delay = Duration::from_secs(8);
|
||||
error = error.with_transient_retry(
|
||||
NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
|
||||
@@ -1538,7 +1464,9 @@ pub(super) const fn provider_error_kind(category: CodexFailureCategory) -> Provi
|
||||
ProviderErrorKind::PermissionDenied
|
||||
}
|
||||
CodexFailureCategory::UsageLimitExhausted => ProviderErrorKind::QuotaExhausted,
|
||||
CodexFailureCategory::RateLimited => ProviderErrorKind::RateLimited,
|
||||
CodexFailureCategory::RateLimited | CodexFailureCategory::TurnRateLimited => {
|
||||
ProviderErrorKind::RateLimited
|
||||
}
|
||||
CodexFailureCategory::QuotaExhausted => ProviderErrorKind::QuotaExhausted,
|
||||
CodexFailureCategory::CloudflareChallenge
|
||||
| CodexFailureCategory::CloudflarePathBlocked
|
||||
@@ -1593,6 +1521,7 @@ pub(super) fn account_failure(
|
||||
Some(CodexAccountFailure::CloudflarePathBlocked)
|
||||
}
|
||||
CodexFailureCategory::ModelUnsupported
|
||||
| CodexFailureCategory::TurnRateLimited
|
||||
| CodexFailureCategory::InvalidRequest
|
||||
| CodexFailureCategory::PermissionDenied
|
||||
| CodexFailureCategory::Timeout
|
||||
|
||||
@@ -812,6 +812,9 @@ pub(super) fn map_selection_error(error: CredentialSelectionError) -> ProviderEr
|
||||
fn selection_failure(error: &CredentialSelectionError) -> ProviderError {
|
||||
let diagnostic_code = match error {
|
||||
CredentialSelectionError::InvalidCredential(_) => Some("account_credential_invalid"),
|
||||
CredentialSelectionError::RevisionConflict(_) => {
|
||||
Some("account_credential_revision_conflict")
|
||||
}
|
||||
CredentialSelectionError::Store(_) => Some("account_store_unavailable"),
|
||||
CredentialSelectionError::Coordinator(_) => Some("account_coordinator_unavailable"),
|
||||
CredentialSelectionError::CookiePolicy(_) => Some("account_cookie_policy_rejected"),
|
||||
@@ -861,6 +864,7 @@ fn selection_failure(error: &CredentialSelectionError) -> ProviderError {
|
||||
.with_classification("account_selection", "account_snapshot_conflict"),
|
||||
),
|
||||
CredentialSelectionError::InvalidCredential(_)
|
||||
| CredentialSelectionError::RevisionConflict(_)
|
||||
| CredentialSelectionError::Store(_)
|
||||
| CredentialSelectionError::Coordinator(_)
|
||||
| CredentialSelectionError::CookiePolicy(_) => provider_error(
|
||||
|
||||
@@ -27,7 +27,8 @@ use crate::transport::{
|
||||
profile::CodexWireProfileState,
|
||||
protocol::{
|
||||
responses::{
|
||||
CodexResponsesRequest, ResponsesSseFailure, TransportRequirement, transport_requirement,
|
||||
CodexResponsesRequest, TransportRequirement, is_replaceable_response_prelude,
|
||||
transport_requirement,
|
||||
},
|
||||
websocket::{
|
||||
websocket_audit_artifact_from_attempt, websocket_connection_limit_failure,
|
||||
@@ -37,10 +38,9 @@ use crate::transport::{
|
||||
response_meta,
|
||||
websocket::{
|
||||
CodexWebSocketConnection, CodexWebSocketExchangeError, CodexWebSocketPool,
|
||||
CodexWebSocketPoolKey, CodexWebSocketStreamingExchange, DEFAULT_STREAM_IDLE_TIMEOUT,
|
||||
WEBSOCKET_FAST_PATH_BUDGET, WebSocketFastPath, WebSocketOriginBreaker,
|
||||
execute_prepared_response_create_request_stream, post_send_ambiguous,
|
||||
prepare_response_create_request_with_pool, websocket_audit_dir,
|
||||
CodexWebSocketPoolKey, DEFAULT_STREAM_IDLE_TIMEOUT, WEBSOCKET_FAST_PATH_BUDGET,
|
||||
WebSocketFastPath, WebSocketOriginBreaker, execute_prepared_response_create_request_stream,
|
||||
post_send_ambiguous, prepare_response_create_request_with_pool, websocket_audit_dir,
|
||||
write_websocket_audit_artifact_from_env,
|
||||
},
|
||||
};
|
||||
@@ -488,15 +488,12 @@ impl CodexBackendClient {
|
||||
)
|
||||
.await
|
||||
.map_err(websocket_exchange_error_to_client_error)?;
|
||||
if requirement.allows_connection_restart()
|
||||
&& request.responses_lite.as_deref() == Some("true")
|
||||
&& request.downstream_websocket_connection_id.is_some()
|
||||
{
|
||||
// Lite 客户端需先拿到响应 ID 才能中断;连接级拒绝仍沿用原有分类
|
||||
if requirement.allows_connection_restart() {
|
||||
// 前导帧即时交付;连接寿命拒绝仍沿用原有分类与恢复预算
|
||||
let mut boundary_reached = false;
|
||||
exchange.body = Box::pin(exchange.body.map(move |frame| {
|
||||
let frame = frame?;
|
||||
if !boundary_reached && !is_websocket_lifecycle_prelude(&frame) {
|
||||
if !boundary_reached && !is_websocket_replayable_prelude(&frame) {
|
||||
boundary_reached = true;
|
||||
if let Some(failure) = websocket_connection_limit_failure(&frame) {
|
||||
return Err(CodexWebSocketExchangeError::ConnectionLimitReached(
|
||||
@@ -506,20 +503,6 @@ impl CodexBackendClient {
|
||||
}
|
||||
Ok(frame)
|
||||
}));
|
||||
} else if requirement.allows_connection_restart() {
|
||||
match await_websocket_delivery_boundary(&mut exchange).await {
|
||||
Ok(DeliveryBoundary::Ready) => {}
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached(failure)) => {
|
||||
return Err(CodexClientError::WebSocket(
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached(failure),
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(websocket_exchange_error_to_client_error(
|
||||
post_send_ambiguous(error),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
body: Box::pin(
|
||||
@@ -630,54 +613,14 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// 首个可投递帧前的交付边界结果
|
||||
enum DeliveryBoundary {
|
||||
/// 已越过边界,可开始向下游投递
|
||||
Ready,
|
||||
/// 首个可投递帧是上游连接寿命限制错误
|
||||
ConnectionLimitReached(Box<ResponsesSseFailure>),
|
||||
}
|
||||
|
||||
async fn await_websocket_delivery_boundary(
|
||||
exchange: &mut CodexWebSocketStreamingExchange,
|
||||
) -> Result<DeliveryBoundary, CodexWebSocketExchangeError> {
|
||||
let mut prelude = Vec::new();
|
||||
loop {
|
||||
match exchange.body.next().await {
|
||||
Some(Ok(frame)) if is_websocket_lifecycle_prelude(&frame) => prelude.push(frame),
|
||||
Some(Ok(frame)) => {
|
||||
let connection_limit_failure = websocket_connection_limit_failure(&frame);
|
||||
prelude.push(frame);
|
||||
let remaining =
|
||||
std::mem::replace(&mut exchange.body, Box::pin(futures::stream::empty()));
|
||||
exchange.body =
|
||||
Box::pin(futures::stream::iter(prelude.into_iter().map(Ok)).chain(remaining));
|
||||
return Ok(if let Some(failure) = connection_limit_failure {
|
||||
DeliveryBoundary::ConnectionLimitReached(Box::new(failure))
|
||||
} else {
|
||||
DeliveryBoundary::Ready
|
||||
});
|
||||
}
|
||||
Some(Err(error)) => return Err(error),
|
||||
None => {
|
||||
return Err(CodexWebSocketExchangeError::StreamEndedBeforeTerminal {
|
||||
reason: "stream_eof",
|
||||
timeout: None,
|
||||
last_event_type: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_websocket_lifecycle_prelude(frame: &[u8]) -> bool {
|
||||
fn is_websocket_replayable_prelude(frame: &[u8]) -> bool {
|
||||
serde_json::from_slice::<serde_json::Value>(frame)
|
||||
.ok()
|
||||
.is_some_and(|value| {
|
||||
matches!(
|
||||
value.get("type").and_then(serde_json::Value::as_str),
|
||||
Some("response.created" | "response.in_progress")
|
||||
)
|
||||
value
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|event_type| is_replaceable_response_prelude(event_type, &value))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ pub enum CodexUpstreamSendPhase {
|
||||
Ambiguous,
|
||||
}
|
||||
|
||||
/// Transport 边界对上游失败的账号级分类
|
||||
/// Transport 边界对上游失败的分类
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CodexFailureCategory {
|
||||
ModelUnsupported,
|
||||
@@ -46,6 +46,8 @@ pub enum CodexFailureCategory {
|
||||
/// 当前用量窗口已经耗尽,但存在自动恢复的重置时间
|
||||
UsageLimitExhausted,
|
||||
RateLimited,
|
||||
/// 当前 turn 的限流交给客户端等待,不据此推导账号级冷却
|
||||
TurnRateLimited,
|
||||
QuotaExhausted,
|
||||
CloudflareChallenge,
|
||||
CloudflarePathBlocked,
|
||||
@@ -174,29 +176,40 @@ impl CodexUpstreamFailure {
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回账号级失败分类
|
||||
/// 返回上游失败分类
|
||||
#[must_use]
|
||||
pub const fn category(&self) -> CodexFailureCategory {
|
||||
self.category
|
||||
}
|
||||
|
||||
/// 只有明确识别的账号或容量拒绝才允许网关重放,普通上游错误交给客户端
|
||||
/// 明确的 503 server_error 允许有界恢复,不把普通 5xx 当作容量或账号故障
|
||||
pub(crate) fn is_retryable_server_error(&self) -> bool {
|
||||
self.category == CodexFailureCategory::Unavailable
|
||||
&& self.status == Some(StatusCode::SERVICE_UNAVAILABLE)
|
||||
&& self
|
||||
.code
|
||||
.as_deref()
|
||||
.is_some_and(|code| code.trim().eq_ignore_ascii_case("server_error"))
|
||||
}
|
||||
|
||||
/// 只对明确识别的账号、容量拒绝或 503 server_error 提供重放资格
|
||||
#[must_use]
|
||||
pub const fn replay_is_safe(&self) -> bool {
|
||||
pub fn replay_is_safe(&self) -> bool {
|
||||
!matches!(self.send_phase, CodexUpstreamSendPhase::Ambiguous)
|
||||
&& matches!(
|
||||
self.category,
|
||||
CodexFailureCategory::ModelUnsupported
|
||||
| CodexFailureCategory::CredentialExpired
|
||||
| CodexFailureCategory::IdentityVerificationRequired
|
||||
| CodexFailureCategory::Banned
|
||||
| CodexFailureCategory::UsageLimitExhausted
|
||||
| CodexFailureCategory::RateLimited
|
||||
| CodexFailureCategory::QuotaExhausted
|
||||
| CodexFailureCategory::CloudflareChallenge
|
||||
| CodexFailureCategory::CloudflarePathBlocked
|
||||
| CodexFailureCategory::CapacityUnavailable
|
||||
)
|
||||
&& (self.is_retryable_server_error()
|
||||
|| matches!(
|
||||
self.category,
|
||||
CodexFailureCategory::ModelUnsupported
|
||||
| CodexFailureCategory::CredentialExpired
|
||||
| CodexFailureCategory::IdentityVerificationRequired
|
||||
| CodexFailureCategory::Banned
|
||||
| CodexFailureCategory::UsageLimitExhausted
|
||||
| CodexFailureCategory::RateLimited
|
||||
| CodexFailureCategory::QuotaExhausted
|
||||
| CodexFailureCategory::CloudflareChallenge
|
||||
| CodexFailureCategory::CloudflarePathBlocked
|
||||
| CodexFailureCategory::CapacityUnavailable
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_code(&self) -> Option<&str> {
|
||||
@@ -378,6 +391,9 @@ fn classify_upstream_failure(
|
||||
if code == "flex_unavailable" {
|
||||
return CodexFailureCategory::FlexUnavailable;
|
||||
}
|
||||
if code == "turn_rate_limit_exceeded" {
|
||||
return CodexFailureCategory::TurnRateLimited;
|
||||
}
|
||||
|
||||
// 容量拒绝可能带 400/429/503;仅用结构化错误字段识别,不能扫描任意正文
|
||||
if is_capacity_error(
|
||||
|
||||
@@ -210,6 +210,17 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅空响应前导和元数据可在已交付后被下一次尝试替换
|
||||
pub(crate) fn is_replaceable_response_prelude(event_type: &str, value: &Value) -> bool {
|
||||
match event_type {
|
||||
"codex.response.metadata" | "codex.rate_limits" => true,
|
||||
"response.created" | "response.in_progress" => value
|
||||
.pointer("/response/output")
|
||||
.is_none_or(|output| output.as_array().is_some_and(Vec::is_empty)),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 单个 Responses 事件对计时系统提供的稳定语义信号
|
||||
///
|
||||
/// `output_start` 只表示官方输出项开始事件;`semantic_output` 仍要求
|
||||
|
||||
@@ -68,6 +68,8 @@ pub type CodexWebSocketResponseMetadataUpdates = Arc<Mutex<CodexWebSocketRespons
|
||||
pub(super) fn reusable_websocket_metadata(
|
||||
mut metadata: CodexWebSocketConnectionMetadata,
|
||||
) -> CodexWebSocketConnectionMetadata {
|
||||
// 握手 Cookie 只属于首次响应,复用旧头会重复写入并覆盖后续轮换结果
|
||||
metadata.set_cookie_headers.clear();
|
||||
metadata.rate_limit_headers.clear();
|
||||
metadata.turn_state = None;
|
||||
// 模型报告属于上一轮请求,池中连接不能把它带到下一轮缺失报告的响应
|
||||
|
||||
@@ -1840,6 +1840,41 @@ fn cloudflare_path_block_deletes_provider_owned_cookies() {
|
||||
assert!(data.cookies().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_response_cookie_capture_reports_revision_conflict_and_preserves_newer_cookie() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
let origin = Url::parse("https://chatgpt.com/backend-api/codex/responses").unwrap();
|
||||
let stale = store.account("acct_primary").unwrap();
|
||||
block_on(selector.capture_response_cookies(
|
||||
&stale,
|
||||
&origin,
|
||||
&["cf_clearance=newer; Path=/; Secure; Max-Age=3600".to_owned()],
|
||||
))
|
||||
.unwrap();
|
||||
let current = store.account("acct_primary").unwrap();
|
||||
|
||||
let error = block_on(selector.capture_response_cookies(
|
||||
&stale,
|
||||
&origin,
|
||||
&["cf_clearance=older; Path=/; Secure; Max-Age=3600".to_owned()],
|
||||
))
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
CredentialSelectionError::RevisionConflict(_)
|
||||
));
|
||||
assert_eq!(error.to_string(), "Codex credential revision conflict");
|
||||
assert_eq!(
|
||||
store.account("acct_primary").unwrap().revision(),
|
||||
current.revision()
|
||||
);
|
||||
let data = block_on(store.repository().load_complete_data(¤t)).unwrap();
|
||||
assert_eq!(data.cookies()[0].value, "newer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_cookie_rotation_returns_a_current_account_for_later_fenced_writes() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证额度预热按 SSE 终态判定结果并跳过过期凭据
|
||||
//! 验证额度预热按 SSE 终态判定结果、请求档位并跳过过期凭据
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -40,6 +40,43 @@ async fn warmup_consumes_completed_sse_and_persists_rate_limit_event() {
|
||||
assert!(store.quota_json("acct_warmup_completed").is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_sends_the_lowest_reasoning_effort_supported_by_the_catalog() {
|
||||
let server = MockServer::start().await;
|
||||
let sse = concat!(
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_warmup_effort\",\"status\":\"completed\",\"output\":[]}}\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
);
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_warmup_effort").await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let summary = service.execute_warmup("test-model").await.expect("warmup");
|
||||
|
||||
assert_eq!(summary.warmed_up, 1);
|
||||
let requests = server.received_requests().await.expect("recorded requests");
|
||||
let decoded =
|
||||
zstd::stream::decode_all(requests[0].body.as_slice()).expect("zstd body should decode");
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(&decoded).expect("warmup body should be valid JSON");
|
||||
// 上游按目录拒绝目录外的档位,none 曾让全部定时预热只留下 error 终态
|
||||
assert_eq!(body["reasoning"]["effort"], "low");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_rejects_failed_sse_despite_successful_http_headers() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! 验证真实响应头只写回一次,流内失败不重复处理 Cookie
|
||||
|
||||
use super::*;
|
||||
|
||||
fn cookie_provider(store: &Arc<MemoryAccountStore>, base_url: String) -> Arc<CodexProvider> {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder().no_proxy().build().unwrap();
|
||||
let leases = Arc::new(TestLeaseCoordinator::default());
|
||||
let catalog = Arc::new(CodexCredentialCatalogService::new(
|
||||
store.repository(),
|
||||
profile.clone(),
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
catalog_cache(),
|
||||
));
|
||||
let quota = Arc::new(CodexCredentialQuotaService::new(
|
||||
store.repository(),
|
||||
profile.clone(),
|
||||
http.clone(),
|
||||
base_url.clone(),
|
||||
Arc::new(MemoryCooldownPort::new()),
|
||||
Arc::clone(&leases) as Arc<dyn ProviderLeasePort>,
|
||||
crate::support::runtime_policy(),
|
||||
));
|
||||
let feedback = Arc::new(AccountFeedbackStats::default());
|
||||
let selector = Arc::new(CodexCredentialSelector::new(
|
||||
ProviderKind::new("openai").unwrap(),
|
||||
store.repository(),
|
||||
leases,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
Arc::new(MemorySessionExclusions::default()),
|
||||
Arc::clone("a),
|
||||
Arc::clone(&feedback),
|
||||
// 测试仅允许本机模拟上游,生产 Cookie 白名单不放宽
|
||||
CodexCookiePolicy::new(["cf_clearance"], ["127.0.0.1"]).unwrap(),
|
||||
));
|
||||
Arc::new(
|
||||
CodexProvider::new(
|
||||
selector,
|
||||
catalog,
|
||||
quota,
|
||||
feedback,
|
||||
http,
|
||||
profile,
|
||||
base_url,
|
||||
Arc::new(CodexWebSocketPool::default()),
|
||||
0,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_cookies_are_saved_once_for_stream_and_opening_failures() {
|
||||
for streaming in [true, false] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let original = store.account("acct_provider_contract").unwrap();
|
||||
let server = MockServer::start().await;
|
||||
let failure = json!({"type":"error", "error":{"type":"server_error", "code":"server_error", "message":"backend unavailable"}});
|
||||
let response = if streaming {
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(format!("event: error\ndata: {failure}\n\n"))
|
||||
} else {
|
||||
ResponseTemplate::new(503).set_body_json(&failure)
|
||||
}
|
||||
.insert_header("set-cookie", "cf_clearance=current; Path=/; Max-Age=3600");
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(response)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let mut stream = cookie_provider(&store, server.uri())
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
context("req_cookie_failure", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut failure = None;
|
||||
while let Some(event) = stream.next().await {
|
||||
if let Err(error) = event {
|
||||
failure = Some(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(failure.is_some(), "simulated upstream must fail");
|
||||
let current = store.account("acct_provider_contract").unwrap();
|
||||
assert_eq!(
|
||||
current.revision().get(),
|
||||
original.revision().get() + 1,
|
||||
"streaming={streaming}"
|
||||
);
|
||||
let data = store
|
||||
.repository()
|
||||
.load_complete_data(¤t)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(data.cookies()[0].value, "current");
|
||||
server.verify().await;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod account_isolation;
|
||||
mod capacity;
|
||||
mod cookies;
|
||||
mod error_details;
|
||||
mod image_account_eligibility;
|
||||
mod image_model_access;
|
||||
@@ -4374,7 +4375,10 @@ async fn websocket_idle_timeout_diagnosis_survives_ambiguous_send_wrapping() {
|
||||
.await
|
||||
.expect("partial response")
|
||||
.expect("valid partial response");
|
||||
if event.has_client_event() {
|
||||
if event
|
||||
.wire_event()
|
||||
.is_some_and(|wire| wire.event_type() == Some("response.output_text.delta"))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -4593,7 +4597,7 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation()
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_item.added",
|
||||
"type": "response.in_progress",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": "msg_structural",
|
||||
@@ -4710,8 +4714,8 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation()
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!event.has_client_event(),
|
||||
"structural events must remain behind the replay boundary"
|
||||
!event.has_client_event() || event.is_replayable_prelude(),
|
||||
"delivered preludes preserve the replay boundary"
|
||||
);
|
||||
}
|
||||
Some(Err(error)) => break error,
|
||||
@@ -4736,7 +4740,7 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation()
|
||||
assert_eq!(
|
||||
error.diagnostic().map(|diagnostic| diagnostic.as_str()),
|
||||
Some(
|
||||
"OpenAI WebSocket closed before a terminal response (close code 1000); last event type: response.output_item.added"
|
||||
"OpenAI WebSocket closed before a terminal response (close code 1000); last event type: response.in_progress"
|
||||
)
|
||||
);
|
||||
let raw_close: Value = serde_json::from_str(
|
||||
@@ -4751,7 +4755,7 @@ async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation()
|
||||
assert_eq!(raw_close.get("reason"), Some(&json!("")));
|
||||
assert_eq!(
|
||||
raw_close.get("last_event_type"),
|
||||
Some(&json!("response.output_item.added"))
|
||||
Some(&json!("response.in_progress"))
|
||||
);
|
||||
let connection = error
|
||||
.connection_observation()
|
||||
@@ -8494,15 +8498,27 @@ async fn capacity_http_and_websocket_opening_rejections_use_bounded_business_ret
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn flex_http_and_websocket_opening_rejections_should_not_retry_or_cool_down_accounts() {
|
||||
async fn request_rejections_should_not_retry_or_cool_down_accounts() {
|
||||
use gateway_core::provider_ports::ProviderCooldownPort as _;
|
||||
for use_websocket in [false, true] {
|
||||
for (use_websocket, (code, expected_kind)) in [false, true].into_iter().flat_map(|websocket| {
|
||||
[
|
||||
("flex_unavailable", ProviderErrorKind::Unavailable),
|
||||
("turn_rate_limit_exceeded", ProviderErrorKind::RateLimited),
|
||||
]
|
||||
.map(|case| (websocket, case))
|
||||
}) {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_provider_contract";
|
||||
create_account(&store, account_id).await;
|
||||
let cooldowns = Arc::new(MemoryCooldownPort::new());
|
||||
let server = MockServer::start().await;
|
||||
let body = json!({"error":{"type":"resource_unavailable","code":"flex_unavailable","message":"Flex capacity unavailable."}});
|
||||
let error_type = if code == "turn_rate_limit_exceeded" {
|
||||
"rate_limit_error"
|
||||
} else {
|
||||
"resource_unavailable"
|
||||
};
|
||||
let body =
|
||||
json!({"error":{"type":error_type,"code":code,"message":"Retry this request later."}});
|
||||
Mock::given(method(if use_websocket { "GET" } else { "POST" }))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
@@ -8523,7 +8539,7 @@ async fn flex_http_and_websocket_opening_rejections_should_not_retry_or_cool_dow
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_flex", CancellationToken::new()),
|
||||
context("req_terminal_rejection", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -8531,14 +8547,12 @@ async fn flex_http_and_websocket_opening_rejections_should_not_retry_or_cool_dow
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("expected Flex failure"),
|
||||
None => panic!("expected request rejection"),
|
||||
}
|
||||
};
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("flex_unavailable")
|
||||
);
|
||||
assert_eq!(error.kind(), expected_kind);
|
||||
assert_eq!(error.upstream_code().map(|code| code.as_str()), Some(code));
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(300)));
|
||||
assert!(!error.replay_is_safe());
|
||||
assert!(error.pre_delivery_retry().is_none());
|
||||
assert!(error.retry_is_prohibited());
|
||||
@@ -8548,6 +8562,11 @@ async fn flex_http_and_websocket_opening_rejections_should_not_retry_or_cool_dow
|
||||
let response = error.client_visible_upstream_response().unwrap();
|
||||
assert_eq!(response.status(), 429);
|
||||
assert_eq!(response.body().as_ref(), body.to_string().as_bytes());
|
||||
assert!(
|
||||
response.headers().iter().any(|header| {
|
||||
header.name() == "retry-after" && header.value().as_ref() == b"300"
|
||||
})
|
||||
);
|
||||
let account = store.account(account_id).unwrap();
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
|
||||
@@ -8811,6 +8830,12 @@ async fn capacity_feedback_in_stream_only_counts_explicit_overload() {
|
||||
ProviderErrorKind::Unavailable,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"turn_rate_limit_exceeded",
|
||||
"Retry this turn later.",
|
||||
ProviderErrorKind::RateLimited,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"invalid_prompt",
|
||||
"Invalid prompt: we've limited access to this content for safety reasons.",
|
||||
@@ -8852,7 +8877,10 @@ async fn capacity_feedback_in_stream_only_counts_explicit_overload() {
|
||||
if semantic_output {
|
||||
events.push(json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"}));
|
||||
}
|
||||
let original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": message}}});
|
||||
let mut original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": message}}});
|
||||
if code == "turn_rate_limit_exceeded" {
|
||||
original["response"]["error"]["headers"] = json!({"retry-after": "7"});
|
||||
}
|
||||
events.push(original.clone());
|
||||
let (base_url, _http_server, websocket_server) = if use_websocket {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
|
||||
@@ -8924,6 +8952,10 @@ async fn capacity_feedback_in_stream_only_counts_explicit_overload() {
|
||||
);
|
||||
assert_eq!(error.replay_is_safe(), capacity && !semantic_output);
|
||||
assert_eq!(error.upstream_status(), None);
|
||||
if code == "turn_rate_limit_exceeded" {
|
||||
assert!(error.retry_is_prohibited());
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(7)));
|
||||
}
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry().is_some(),
|
||||
capacity && !semantic_output
|
||||
@@ -8963,7 +8995,11 @@ async fn capacity_feedback_in_stream_only_counts_explicit_overload() {
|
||||
original
|
||||
);
|
||||
if !semantic_output {
|
||||
assert!(client_events.is_empty());
|
||||
assert!(
|
||||
client_events
|
||||
.iter()
|
||||
.all(|event| event.is_replayable_prelude())
|
||||
);
|
||||
}
|
||||
client_events.extend(error.take_atomic_client_events());
|
||||
let wire = client_events
|
||||
@@ -8980,7 +9016,7 @@ async fn capacity_feedback_in_stream_only_counts_explicit_overload() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_request_should_hold_created_until_later_failure_can_rotate() {
|
||||
async fn ordinary_request_forwards_created_and_preserves_later_rotation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_first_event_latency").await;
|
||||
let (base_url, release, first_chunk_sent, server) = paused_chunked_sse_server(
|
||||
@@ -9004,39 +9040,18 @@ async fn ordinary_request_should_hold_created_until_later_failure_can_rotate() {
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
|
||||
let mut first_chunk_sent = Box::pin(first_chunk_sent);
|
||||
loop {
|
||||
tokio::select! {
|
||||
sent = &mut first_chunk_sent => {
|
||||
sent.expect("first upstream chunk");
|
||||
break;
|
||||
}
|
||||
next = stream.next() => {
|
||||
let event = next
|
||||
.expect("provider stream must stay open")
|
||||
.expect("provider event");
|
||||
assert!(!event.has_client_event(), "response.created must remain replayable");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let exposed = timeout(Duration::from_millis(100), async {
|
||||
loop {
|
||||
let next = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("provider stream must stay open")
|
||||
.expect("provider event");
|
||||
if next.has_client_event() {
|
||||
return next;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
exposed.is_err(),
|
||||
"a structural event must not commit the downstream before a later 429"
|
||||
let created = timeout(
|
||||
Duration::from_secs(1),
|
||||
precommit::next_client_event(&mut stream),
|
||||
)
|
||||
.await
|
||||
.expect("created must arrive before the next upstream chunk");
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
assert!(created.is_replayable_prelude());
|
||||
first_chunk_sent.await.expect("first chunk sent");
|
||||
|
||||
release.send(()).expect("release second upstream chunk");
|
||||
let mut failure = loop {
|
||||
@@ -9060,13 +9075,13 @@ async fn ordinary_request_should_hold_created_until_later_failure_can_rotate() {
|
||||
.iter()
|
||||
.filter_map(|event| event.wire_event()?.event_type())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["response.created", "response.failed"]
|
||||
vec!["response.failed"]
|
||||
);
|
||||
server.await.expect("chunked SSE server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_request_should_hold_structural_events_until_upstream_finishes() {
|
||||
async fn ordinary_request_forwards_structural_events_before_upstream_finishes() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_bounded_replay_grace").await;
|
||||
let (base_url, release, _first_chunk_sent, server) = paused_chunked_sse_server(
|
||||
@@ -9086,29 +9101,29 @@ async fn ordinary_request_should_hold_structural_events_until_upstream_finishes(
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
|
||||
assert!(
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while let Some(event) = stream.next().await {
|
||||
assert!(!event.expect("provider event").has_client_event());
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
let created = timeout(
|
||||
Duration::from_secs(1),
|
||||
precommit::next_client_event(&mut stream),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
assert!(created.is_replayable_prelude());
|
||||
release.send(()).expect("finish upstream response");
|
||||
let events = stream.collect::<Vec<_>>().await;
|
||||
assert!(events.iter().any(|event| {
|
||||
event
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.wire_event()
|
||||
.is_some_and(|wire| wire.event_type() == Some("response.created"))
|
||||
}));
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.all(|event| !event.as_ref().unwrap().has_client_event())
|
||||
);
|
||||
server.await.expect("chunked SSE server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuation_should_hold_created_until_semantic_output() {
|
||||
async fn continuation_forwards_created_before_semantic_output() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_continuation_prefetch").await;
|
||||
let (base_url, release, first_chunk_sent, server) = paused_chunked_sse_server(
|
||||
@@ -9137,42 +9152,24 @@ async fn continuation_should_hold_created_until_semantic_output() {
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
|
||||
let mut first_chunk_sent = Box::pin(first_chunk_sent);
|
||||
loop {
|
||||
tokio::select! {
|
||||
sent = &mut first_chunk_sent => {
|
||||
sent.expect("first upstream chunk");
|
||||
break;
|
||||
}
|
||||
next = stream.next() => {
|
||||
let event = next
|
||||
.expect("provider stream must stay open")
|
||||
.expect("provider event");
|
||||
assert!(!event.has_client_event(), "created was exposed before the first chunk barrier");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let blocked = timeout(Duration::from_millis(100), async {
|
||||
loop {
|
||||
let next = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("provider stream must stay open")
|
||||
.expect("provider event");
|
||||
if next.has_client_event() {
|
||||
return next;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(blocked.is_err());
|
||||
let created = timeout(
|
||||
Duration::from_secs(1),
|
||||
precommit::next_client_event(&mut stream),
|
||||
)
|
||||
.await
|
||||
.expect("created must arrive before the next upstream chunk");
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
assert!(created.is_replayable_prelude());
|
||||
first_chunk_sent.await.expect("first chunk sent");
|
||||
|
||||
release.send(()).expect("release semantic output chunk");
|
||||
let first_event = loop {
|
||||
let next = timeout(Duration::from_secs(1), stream.next())
|
||||
.await
|
||||
.expect("semantic output must release the prefetched batch")
|
||||
.expect("semantic output arrives after release")
|
||||
.expect("provider stream must stay open")
|
||||
.expect("provider event");
|
||||
if next.has_client_event() {
|
||||
@@ -9181,7 +9178,7 @@ async fn continuation_should_hold_created_until_semantic_output() {
|
||||
};
|
||||
assert_eq!(
|
||||
first_event.wire_event().and_then(|wire| wire.event_type()),
|
||||
Some("response.created")
|
||||
Some("response.content_part.added")
|
||||
);
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("continuation response must complete");
|
||||
@@ -9595,7 +9592,7 @@ async fn exact_websocket_busy_then_replay_scope_relaxation_is_rejected_before_se
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuation_prefetch_over_128_kib_stays_buffered_until_eof_without_protocol_failure() {
|
||||
async fn large_continuation_prelude_is_delivered_before_eof_without_protocol_failure() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_prefetch_limit").await;
|
||||
let padding = "x".repeat(128 * 1024);
|
||||
@@ -9622,36 +9619,13 @@ async fn continuation_prefetch_over_128_kib_stays_buffered_until_eof_without_pro
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
let visible = {
|
||||
let next_visible = async {
|
||||
loop {
|
||||
let event = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("provider stream remains open")
|
||||
.expect("large prefetch cannot create a protocol failure");
|
||||
if event.has_client_event() {
|
||||
break event;
|
||||
}
|
||||
}
|
||||
};
|
||||
tokio::pin!(next_visible);
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), next_visible.as_mut())
|
||||
.await
|
||||
.is_err(),
|
||||
"large structural events must remain buffered before semantic output"
|
||||
);
|
||||
assert!(
|
||||
timeout(Duration::from_secs(3), next_visible.as_mut())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
release.send(()).expect("finish upstream response");
|
||||
timeout(Duration::from_secs(1), next_visible)
|
||||
.await
|
||||
.expect("EOF releases original wire")
|
||||
};
|
||||
let visible = timeout(
|
||||
Duration::from_secs(1),
|
||||
precommit::next_client_event(&mut stream),
|
||||
)
|
||||
.await
|
||||
.expect("large prelude is delivered before EOF");
|
||||
release.send(()).expect("finish upstream response");
|
||||
|
||||
assert_eq!(
|
||||
visible.wire_event().and_then(|wire| wire.event_type()),
|
||||
@@ -10415,6 +10389,18 @@ async fn connection_limit_rejection_requests_one_provider_retry_and_reconnects_i
|
||||
let (stream, _) = listener.accept().await.expect("accept WS");
|
||||
let mut ws = accept_codex_test_websocket(stream).await;
|
||||
ws.next().await.expect("request").expect("valid frame");
|
||||
// 真实上游先发送元数据,随后仍可能在空前导后拒绝连接寿命
|
||||
if first {
|
||||
for event in [
|
||||
json!({"type":"codex.response.metadata","model":"gpt-5.4"}),
|
||||
json!({"type":"response.created","response":{"id":"resp_expired","output":[]}}),
|
||||
json!({"type":"response.in_progress","response":{"id":"resp_expired","output":[]}}),
|
||||
] {
|
||||
ws.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.expect("prelude");
|
||||
}
|
||||
}
|
||||
let event = if first {
|
||||
json!({"type":"error","status":400,"headers":{"x-request-id":"req-limit-retry"},"error":{"code":"websocket_connection_limit_reached","type":"invalid_request_error","message":"connection expired"}})
|
||||
} else {
|
||||
@@ -10725,6 +10711,7 @@ async fn assert_websocket_failure_headers(headers: Value, request_id: Option<&st
|
||||
)
|
||||
.await
|
||||
.expect("prepare");
|
||||
let mut delivered = Vec::new();
|
||||
let mut observation = None;
|
||||
let mut error = timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
@@ -10733,6 +10720,7 @@ async fn assert_websocket_failure_headers(headers: Value, request_id: Option<&st
|
||||
if let Some(current) = event.response_observation() {
|
||||
observation = Some(current.clone());
|
||||
}
|
||||
delivered.push(event);
|
||||
}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("expected error"),
|
||||
@@ -10765,7 +10753,8 @@ async fn assert_websocket_failure_headers(headers: Value, request_id: Option<&st
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert!(!error.replay_is_safe());
|
||||
assert_eq!(error.raw_upstream_error().expect("raw error").as_str(), raw);
|
||||
let events = error.take_atomic_client_events();
|
||||
delivered.extend(error.take_atomic_client_events());
|
||||
let events = delivered;
|
||||
let event_type = if reuse { "response.failed" } else { "error" };
|
||||
assert_eq!(
|
||||
events
|
||||
@@ -10773,7 +10762,7 @@ async fn assert_websocket_failure_headers(headers: Value, request_id: Option<&st
|
||||
.filter_map(|event| event.wire_event()?.event_type())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["response.metadata", event_type],
|
||||
"original metadata and failure stay atomically deliverable",
|
||||
"original metadata and failure are delivered in order",
|
||||
);
|
||||
let wire = events
|
||||
.iter()
|
||||
@@ -11633,7 +11622,9 @@ async fn quota_continuation_stream_rejection_only_projects_before_delivery() {
|
||||
client_events.extend(error.take_atomic_client_events());
|
||||
if native_continuation && !semantic_output {
|
||||
assert!(
|
||||
client_events.is_empty(),
|
||||
client_events
|
||||
.iter()
|
||||
.all(|event| event.is_replayable_prelude()),
|
||||
"raw quota error must not override replay projection"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -11694,15 +11685,17 @@ async fn late_quota_continuation_rejection_requires_client_replay_before_output(
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while let Some(event) = stream.next().await {
|
||||
assert!(!event.unwrap().has_client_event());
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
let created = timeout(
|
||||
Duration::from_secs(1),
|
||||
precommit::next_client_event(&mut stream),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
assert!(created.is_replayable_prelude());
|
||||
release.send(()).unwrap();
|
||||
let mut delivered_failure = false;
|
||||
let error = loop {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证结构事件缓冲、长时间等待与取消对重试边界的影响
|
||||
//! 验证事件即时交付、长时间进度与取消对重试边界的影响
|
||||
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
use gateway_core::engine::provider::ProviderStream;
|
||||
@@ -49,16 +49,6 @@ fn sse(event: &Value) -> String {
|
||||
encode_sse_event(event["type"].as_str().unwrap(), &event.to_string())
|
||||
}
|
||||
|
||||
fn releases(trace: &TraceContext) -> Vec<Value> {
|
||||
trace.snapshot().unwrap()["events"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|event| event["stage"] == "provider.precommit.released")
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) async fn next_client_event(stream: &mut ProviderStream) -> ProviderEvent {
|
||||
loop {
|
||||
let event = stream.next().await.unwrap().unwrap();
|
||||
@@ -81,7 +71,7 @@ pub(super) async fn stream_failure(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_structural_events_preserve_overload_replay_on_http_and_websocket() {
|
||||
async fn large_preludes_are_forwarded_before_failure_on_http_and_websocket() {
|
||||
for websocket in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
@@ -89,7 +79,7 @@ async fn large_structural_events_preserve_overload_replay_on_http_and_websocket(
|
||||
let created = structural_event("response.created", 300 * 1024);
|
||||
let progress = structural_event("response.in_progress", 300 * 1024);
|
||||
let failure = overload();
|
||||
let expected = vec![created.clone(), progress.clone(), failure.clone()];
|
||||
let expected = vec![failure.clone()];
|
||||
let (base_url, release, server) = if websocket {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
@@ -128,11 +118,13 @@ async fn large_structural_events_preserve_overload_replay_on_http_and_websocket(
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
timeout(Duration::from_millis(3_500), next_client_event(&mut stream))
|
||||
for kind in ["response.created", "response.in_progress"] {
|
||||
let event = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
.unwrap();
|
||||
assert_eq!(event.wire_event().unwrap().event_type(), Some(kind));
|
||||
assert!(event.is_replayable_prelude());
|
||||
}
|
||||
release.send(()).unwrap();
|
||||
let (events, mut error) = timeout(Duration::from_secs(1), stream_failure(&mut stream))
|
||||
.await
|
||||
@@ -149,10 +141,6 @@ async fn large_structural_events_preserve_overload_replay_on_http_and_websocket(
|
||||
.filter_map(|event| event.wire_event().map(|wire| wire.data().clone()))
|
||||
.collect();
|
||||
assert_eq!(actual, expected);
|
||||
assert!(
|
||||
releases(&trace).is_empty(),
|
||||
"discardable failure must not be reported as a release"
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -185,11 +173,13 @@ async fn late_overload_after_multiple_structural_events_remains_replayable() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_millis(3_500), next_client_event(&mut stream))
|
||||
for kind in ["response.created", "response.in_progress"] {
|
||||
let event = timeout(Duration::from_secs(2), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
.unwrap();
|
||||
assert_eq!(event.wire_event().unwrap().event_type(), Some(kind));
|
||||
assert!(event.is_replayable_prelude());
|
||||
}
|
||||
release.send(()).unwrap();
|
||||
let (events, mut error) = timeout(Duration::from_secs(1), stream_failure(&mut stream))
|
||||
.await
|
||||
@@ -206,14 +196,13 @@ async fn late_overload_after_multiple_structural_events_remains_replayable() {
|
||||
.iter()
|
||||
.filter_map(|event| event.wire_event()?.event_type())
|
||||
.collect::<Vec<_>>(),
|
||||
["response.created", "response.in_progress", "error"]
|
||||
["error"]
|
||||
);
|
||||
assert!(releases(&trace).is_empty());
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_release_preserves_wire_and_records_the_boundary_once() {
|
||||
async fn immediate_delivery_preserves_wire_through_output_terminal_and_eof() {
|
||||
let semantic = json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"});
|
||||
let terminal = json!({"type": "response.completed", "response": {"id": "resp_precommit", "model": "gpt-5.4", "status": "completed", "output": []}});
|
||||
let tool = json!({"type": "response.web_search_call.in_progress", "item_id": "search_precommit", "output_index": 0});
|
||||
@@ -270,16 +259,12 @@ async fn immediate_release_preserves_wire_and_records_the_boundary_once() {
|
||||
}
|
||||
}
|
||||
assert_eq!(wire, body.as_bytes());
|
||||
let releases = releases(&trace);
|
||||
assert_eq!(releases.len(), 1);
|
||||
assert_eq!(releases[0]["data"]["reason"], reason);
|
||||
assert_eq!(releases[0]["data"]["prefetchedBytes"], body.len());
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_buffered_structural_events_does_not_release_or_offer_replay() {
|
||||
async fn cancelling_after_prelude_delivery_does_not_offer_replay() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let (base_url, release, _, server) = paused_chunked_sse_server(
|
||||
@@ -296,10 +281,12 @@ async fn cancelling_buffered_structural_events_does_not_release_or_offer_replay(
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_millis(100), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
let created = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
cancellation.cancel();
|
||||
let (events, error) = timeout(Duration::from_secs(1), stream_failure(&mut stream))
|
||||
@@ -308,7 +295,6 @@ async fn cancelling_buffered_structural_events_does_not_release_or_offer_replay(
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Cancelled);
|
||||
assert!(!error.replay_is_safe());
|
||||
assert!(events.iter().all(|event| !event.has_client_event()));
|
||||
assert!(releases(&trace).is_empty());
|
||||
release.send(()).unwrap();
|
||||
server.await.unwrap();
|
||||
}
|
||||
@@ -347,10 +333,12 @@ async fn twenty_minute_structural_progress_preserves_late_overload_recovery() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_millis(100), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
let created = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
created.wire_event().unwrap().event_type(),
|
||||
Some("response.created")
|
||||
);
|
||||
// 结构进度保持上游传输活跃,长思考不能额外触发首内容等待期限
|
||||
for _ in 0..20 {
|
||||
@@ -358,11 +346,14 @@ async fn twenty_minute_structural_progress_preserves_late_overload_recovery() {
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
tokio::time::resume();
|
||||
progress.send(()).await.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_millis(50), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
let event = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
event.wire_event().unwrap().event_type(),
|
||||
Some("response.in_progress")
|
||||
);
|
||||
assert!(event.is_replayable_prelude());
|
||||
}
|
||||
drop(progress);
|
||||
let (events, error) = timeout(Duration::from_secs(1), stream_failure(&mut stream))
|
||||
@@ -374,12 +365,11 @@ async fn twenty_minute_structural_progress_preserves_late_overload_recovery() {
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransientRetry { .. })
|
||||
));
|
||||
assert!(releases(&trace).is_empty());
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structural_preamble_over_16_mib_preserves_late_overload_recovery() {
|
||||
async fn large_prelude_stream_is_forwarded_without_retaining_it_for_failure() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
// 大段配置回显仍属于前导事件,累计体积不应改变上游失败的恢复方式
|
||||
@@ -394,11 +384,16 @@ async fn structural_preamble_over_16_mib_preserves_late_overload_recovery() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
for _ in 0..17 {
|
||||
let event = timeout(Duration::from_secs(2), next_client_event(&mut stream))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
event.wire_event().unwrap().event_type(),
|
||||
Some("response.in_progress")
|
||||
);
|
||||
assert!(event.is_replayable_prelude());
|
||||
}
|
||||
release.send(()).unwrap();
|
||||
let (events, mut error) = timeout(Duration::from_secs(10), stream_failure(&mut stream))
|
||||
.await
|
||||
@@ -415,8 +410,7 @@ async fn structural_preamble_over_16_mib_preserves_late_overload_recovery() {
|
||||
.filter_map(|event| event.wire_event()?.raw_sse_frame())
|
||||
.map(bytes::Bytes::len)
|
||||
.sum();
|
||||
assert!(buffered_bytes > 16 * 1024 * 1024);
|
||||
assert!(releases(&trace).is_empty());
|
||||
assert_eq!(buffered_bytes, sse(&overload()).len());
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
@@ -455,7 +449,12 @@ async fn websocket_disconnect_recovery_honors_request_retry_budget_without_sessi
|
||||
.await
|
||||
.unwrap();
|
||||
let (events, error) = stream_failure(&mut stream).await;
|
||||
assert!(events.iter().all(|event| !event.has_client_event()));
|
||||
let delivered: Vec<_> = events
|
||||
.iter()
|
||||
.filter(|event| event.has_client_event())
|
||||
.collect();
|
||||
assert_eq!(delivered.len(), 1);
|
||||
assert!(delivered[0].is_replayable_prelude());
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(!error.replay_is_safe());
|
||||
if retry_count < max_retries {
|
||||
@@ -468,8 +467,107 @@ async fn websocket_disconnect_recovery_honors_request_retry_budget_without_sessi
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback)
|
||||
);
|
||||
}
|
||||
assert!(releases(&trace).is_empty());
|
||||
}
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sse_comments_are_forwarded_without_closing_the_retry_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let heartbeat = ": keep-alive\r\n\r\n";
|
||||
let (base_url, release, _, server) =
|
||||
paused_chunked_sse_server(heartbeat.to_owned(), sse(&overload())).await;
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
context("req_heartbeat_retry", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.expect("heartbeat cannot wait for a later event");
|
||||
assert_eq!(
|
||||
event.wire_event().unwrap().raw_sse_frame().unwrap(),
|
||||
heartbeat.as_bytes()
|
||||
);
|
||||
assert!(event.is_replayable_prelude());
|
||||
release.send(()).unwrap();
|
||||
let (_, error) = stream_failure(&mut stream).await;
|
||||
assert!(error.replay_is_safe());
|
||||
assert!(matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransientRetry { .. })
|
||||
));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_streams_forward_compaction_output_and_unknown_events_without_reopening_retry() {
|
||||
for websocket in [false, true] {
|
||||
for first in [
|
||||
json!({"type":"response.compaction.compacting","item_id":"compact_test","output_index":0}),
|
||||
json!({"type":"response.output_text.delta","output_index":0,"content_index":0,"delta":"partial"}),
|
||||
json!({"type":"response.future","value":"opaque"}),
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let prelude = structural_event("response.created", 1024);
|
||||
let (base_url, release, server) = if websocket {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (release, released) = oneshot::channel();
|
||||
let first = first.clone();
|
||||
let server = tokio::spawn(async move {
|
||||
let (socket, _) = listener.accept().await.unwrap();
|
||||
let mut socket = accept_codex_test_websocket(socket).await;
|
||||
socket.next().await.unwrap().unwrap();
|
||||
for event in [first, prelude] {
|
||||
socket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
released.await.unwrap();
|
||||
socket
|
||||
.send(Message::Text(overload().to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
(base_url, release, server)
|
||||
} else {
|
||||
let (base_url, release, _, server) =
|
||||
paused_chunked_sse_server(sse(&first) + &sse(&prelude), sse(&overload())).await;
|
||||
(base_url, release, server)
|
||||
};
|
||||
let operation = if websocket {
|
||||
generate_operation()
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_closed_retry", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let delivered = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(delivered.wire_event().unwrap().data(), &first);
|
||||
assert!(!delivered.is_replayable_prelude());
|
||||
let created = timeout(Duration::from_secs(1), next_client_event(&mut stream))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!created.is_replayable_prelude());
|
||||
release.send(()).unwrap();
|
||||
let (_, error) = stream_failure(&mut stream).await;
|
||||
assert!(!error.replay_is_safe());
|
||||
assert!(error.pre_delivery_retry().is_none());
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,149 @@
|
||||
use super::precommit::stream_failure;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_error_503_stream_recovery_requires_no_semantic_output() {
|
||||
for websocket in [false, true] {
|
||||
for semantic_output in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let failure = json!({"type":"error","status":503,"error":{"code":"server_error","type":"server_error","message":"You can retry your request."}});
|
||||
let mut upstream_events = vec![
|
||||
json!({"type":"codex.response.metadata","metadata":{}}),
|
||||
json!({"type":"response.created","response":{"id":"resp_503","status":"in_progress","output":[]}}),
|
||||
json!({"type":"response.in_progress","response":{"id":"resp_503","status":"in_progress","output":[]}}),
|
||||
];
|
||||
if semantic_output {
|
||||
upstream_events.push(json!({"type":"response.output_text.delta","response_id":"resp_503","item_id":"msg_503","output_index":0,"content_index":0,"delta":"hello"}));
|
||||
}
|
||||
upstream_events.push(failure.clone());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
if websocket {
|
||||
let mut socket = accept_codex_test_websocket(socket).await;
|
||||
socket.next().await.unwrap().unwrap();
|
||||
for event in upstream_events {
|
||||
socket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
} else {
|
||||
read_http_request(&mut socket).await;
|
||||
let body = upstream_events
|
||||
.iter()
|
||||
.map(|event| {
|
||||
gateway_protocol::openai::sse::encode_sse_event(
|
||||
event["type"].as_str().unwrap(),
|
||||
&event.to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<String>();
|
||||
socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap();
|
||||
}
|
||||
});
|
||||
let operation = if websocket {
|
||||
capacity_websocket_operation()
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, base)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_server_error_503", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (mut events, mut error) = stream_failure(&mut stream).await;
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
|
||||
assert_eq!(error.upstream_status(), Some(503));
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert_eq!(error.replay_is_safe(), !semantic_output);
|
||||
if semantic_output {
|
||||
assert!(error.pre_delivery_retry().is_none());
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|event| event.has_client_event() && !event.is_replayable_prelude())
|
||||
);
|
||||
} else {
|
||||
assert!(!error.retry_is_prohibited());
|
||||
assert!(
|
||||
matches!(error.pre_delivery_retry(), Some(PreDeliveryRetry::SameAccountTransientRetry { max_retries, initial_delay, max_delay })
|
||||
if max_retries.get() == 3 && initial_delay == Duration::from_millis(500) && max_delay == Duration::from_secs(8))
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|event| event.has_client_event())
|
||||
.all(|event| event.is_replayable_prelude())
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Value>(error.raw_upstream_error().unwrap().as_str())
|
||||
.unwrap(),
|
||||
failure
|
||||
);
|
||||
events.extend(error.take_atomic_client_events());
|
||||
assert_eq!(
|
||||
events.last().unwrap().wire_event().unwrap().data(),
|
||||
&failure
|
||||
);
|
||||
let account = store.account("acct_provider_contract").unwrap();
|
||||
assert_eq!(account.credential_state(), CredentialState::Ready);
|
||||
assert_eq!(account.quota().access(), QuotaAccessState::Unknown);
|
||||
server.await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_error_503_opening_rejections_request_bounded_retry_and_keep_response() {
|
||||
for websocket in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let server = MockServer::start().await;
|
||||
let body =
|
||||
r#"{ "error":{"code":"server_error","type":"server_error","message":"retry later"} }"#;
|
||||
Mock::given(method(if websocket { "GET" } else { "POST" }))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(503)
|
||||
.insert_header("retry-after", "17")
|
||||
.set_body_string(body),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let operation = if websocket {
|
||||
capacity_websocket_operation()
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_server_error_opening", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_, error) = stream_failure(&mut stream).await;
|
||||
assert!(error.replay_is_safe());
|
||||
assert!(!error.retry_is_prohibited());
|
||||
assert!(matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransientRetry { .. })
|
||||
));
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(17)));
|
||||
let response = error.client_visible_upstream_response().unwrap();
|
||||
assert_eq!(response.status(), 503);
|
||||
assert_eq!(response.body().as_ref(), body.as_bytes());
|
||||
server.verify().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_http_and_websocket_opening_errors_remain_client_decisions() {
|
||||
for websocket in [false, true] {
|
||||
@@ -105,12 +248,18 @@ async fn ordinary_stream_failures_preserve_original_events_without_proxy_retry()
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (events, mut error) = stream_failure(&mut stream).await;
|
||||
assert!(events.iter().all(|event| !event.has_client_event()));
|
||||
let (mut events, mut error) = stream_failure(&mut stream).await;
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|event| event.has_client_event())
|
||||
.all(|event| event.is_replayable_prelude())
|
||||
);
|
||||
assert!(error.retry_is_prohibited());
|
||||
assert!(error.pre_delivery_retry().is_none());
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(17)));
|
||||
let atomic = error.take_atomic_client_events();
|
||||
events.extend(error.take_atomic_client_events());
|
||||
let atomic = events;
|
||||
assert_eq!(
|
||||
atomic
|
||||
.iter()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证上游诊断的容量错误分类、请求标识与响应事实提取
|
||||
//! 验证上游失败分类、重放资格、请求标识与响应事实提取
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use provider_openai::transport::CodexUpstreamDiagnostics;
|
||||
@@ -43,6 +43,87 @@ fn flex_capacity_rejection_should_be_terminal_even_before_payload_send() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turn_rate_limit_does_not_grant_proxy_replay_or_account_cooldown() {
|
||||
for phase in [
|
||||
CodexUpstreamSendPhase::BeforePayload,
|
||||
CodexUpstreamSendPhase::AfterPayload,
|
||||
CodexUpstreamSendPhase::Ambiguous,
|
||||
] {
|
||||
let failure = CodexUpstreamFailure::from_response(
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||
r#"{"error":{"code":"turn_rate_limit_exceeded","type":"rate_limit_error","message":"Retry this turn later."}}"#,
|
||||
Some(53),
|
||||
&CodexUpstreamDiagnostics::default(),
|
||||
None,
|
||||
&[],
|
||||
&[],
|
||||
phase,
|
||||
);
|
||||
assert_eq!(failure.category(), CodexFailureCategory::TurnRateLimited);
|
||||
assert!(!failure.replay_is_safe());
|
||||
}
|
||||
assert_eq!(
|
||||
classify(429, r#"{"error":{"message":"turn_rate_limit_exceeded"}}"#),
|
||||
CodexFailureCategory::RateLimited
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_error_replay_requires_explicit_503_and_unambiguous_send() {
|
||||
for (status, body, retryable) in [
|
||||
(
|
||||
503,
|
||||
r#"{"error":{"code":"server_error","type":"server_error"}}"#,
|
||||
true,
|
||||
),
|
||||
(
|
||||
500,
|
||||
r#"{"error":{"code":"server_error","type":"server_error"}}"#,
|
||||
false,
|
||||
),
|
||||
(
|
||||
504,
|
||||
r#"{"error":{"code":"server_error","type":"server_error"}}"#,
|
||||
false,
|
||||
),
|
||||
(503, r#"{"error":{"type":"server_error"}}"#, false),
|
||||
(
|
||||
503,
|
||||
r#"{"error":{"code":"Vendor.Future.v2","type":"server_error"}}"#,
|
||||
false,
|
||||
),
|
||||
(
|
||||
503,
|
||||
r#"{"error":{"code":"flex_unavailable","type":"server_error"}}"#,
|
||||
false,
|
||||
),
|
||||
(503, "server_error", false),
|
||||
] {
|
||||
for phase in [
|
||||
CodexUpstreamSendPhase::BeforePayload,
|
||||
CodexUpstreamSendPhase::AfterPayload,
|
||||
CodexUpstreamSendPhase::Ambiguous,
|
||||
] {
|
||||
let failure = CodexUpstreamFailure::from_response(
|
||||
reqwest::StatusCode::from_u16(status).unwrap(),
|
||||
body,
|
||||
None,
|
||||
&CodexUpstreamDiagnostics::default(),
|
||||
None,
|
||||
&[],
|
||||
&[],
|
||||
phase,
|
||||
);
|
||||
assert_eq!(
|
||||
failure.replay_is_safe(),
|
||||
retryable && phase != CodexUpstreamSendPhase::Ambiguous,
|
||||
"status={status}, body={body}, phase={phase:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capacity_rejections_are_request_scoped_across_http_statuses() {
|
||||
for status in [400, 429, 503] {
|
||||
|
||||
@@ -3,6 +3,62 @@
|
||||
use super::*;
|
||||
use provider_openai::transport::websocket::PreviousResponseUnavailableReason;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_websocket_does_not_replay_opening_response_cookies() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (socket, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket_with(socket, |_, response| {
|
||||
response.headers_mut().append(
|
||||
"set-cookie",
|
||||
"__cf_bm=opening; Domain=chatgpt.com; Path=/; Secure; Max-Age=3600"
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
for response_id in ["resp_cookie_first", "resp_cookie_reuse"] {
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response(response_id, 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::new(CodexWebSocketPool::new(Duration::from_mins(1))));
|
||||
let request = pooled_websocket_request("conversation-cookie-reuse");
|
||||
let first = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_cookie_first", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_cookie_reuse", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.set_cookie_headers.len(), 1);
|
||||
assert!(
|
||||
second
|
||||
.websocket_pool_decision
|
||||
.is_some_and(WebSocketPoolDecision::is_reuse)
|
||||
);
|
||||
assert!(second.set_cookie_headers.is_empty());
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_should_reuse_pooled_websocket_for_same_account_and_conversation() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
+15
-9
@@ -151,11 +151,14 @@ openssl rand -hex 24
|
||||
|
||||
| 会话配置 | 含义 | 配置模板 / 默认值 |
|
||||
| --- | --- | --- |
|
||||
| `admin.session_ttl_minutes` | 管理员不活动期限 | 模板为 10080 分钟(7 天) |
|
||||
| `admin.session_ttl_minutes` | 管理员续期凭据有效期,成功续期后重新计时 | 模板为 10080 分钟(7 天) |
|
||||
| `admin.session_absolute_ttl_minutes` | 从登录起计算的最长有效期 | 模板及省略时均为 43200 分钟(30 天) |
|
||||
| `client.session_ttl_minutes` | 密钥身份固定有效期,不随活动续期 | 默认 1440 分钟(1 天) |
|
||||
|
||||
修改配置需重启,不能延长已建立会话的最长有效期。浏览器会话与续期规则见[认证 API](../docs/api.md#统一登录与会话),独立于 `/v1/*` 鉴权和限额
|
||||
管理员访问凭据最多有效 24 小时,由内部策略控制;不活动期限较短时,访问期限缩短到其四分之一,为自动续期留出窗口。
|
||||
推荐使用模板中的 7 天不活动期限和 30 天最长登录期限。修改配置需重启,不能延长已建立会话的最长有效期。
|
||||
升级后仅有单凭据的已有会话保留到原期限,重新登录后获得自动续期能力。
|
||||
浏览器会话与续期规则见[认证 API](../docs/api.md#统一登录与会话),独立于 `/v1/*` 鉴权和限额
|
||||
|
||||
PostgreSQL 与 Redis 密码必须是 48 位十六进制字符。Compose 通过 `config.yaml` 的凭据桥接区
|
||||
引用同一密码;三个值都不需要额外导出为环境变量,数据库和 Redis 密码也不能嵌入连接 URL
|
||||
@@ -454,19 +457,26 @@ Provider schema 以明文 JSON 保存于 PostgreSQL。Redis 只保存可重建
|
||||
| 文件集 | 保留配置 | 默认完整窗口 |
|
||||
| --- | --- | --- |
|
||||
| 普通日志、OAuth 恢复日志 | `host.logging.file.retention_days` | 前 7 个完整自然日及当天 |
|
||||
| 全量请求/响应报文 | `host.logging.request_dump_retention_days` | 前 1 个完整自然日及当天 |
|
||||
| 全量请求/响应报文 | `host.logging.request_dump_retention_days` | 最近 24 小时 |
|
||||
|
||||
文件名统一为 `codex-proxy-rs-<类别>.YYYY-MM-DD[.N].log[.gz]`,类别分别为
|
||||
`application`、`oauth-recovery`、`request-dump`。专用 tracing target 为 `oauth_recovery` 和
|
||||
`request_dump`;普通日志保留各 Rust 模块的 target,便于按模块过滤。
|
||||
程序只管理上述规范名称的日志,其他命名的文件由运维手动清理。
|
||||
|
||||
日志时间戳采用部署时区并携带数字偏移,文件按该时区的日期轮转和整组保留:例如 9 月 8 日配置 1 天,会保留 9 月 7 日全天及 9 月 8 日的所有分片,
|
||||
日志时间戳采用部署时区并携带数字偏移。普通日志与 OAuth 恢复日志按该时区的日期轮转和整组保留:例如 9 月 8 日配置 1 天,会保留 9 月 7 日全天及 9 月 8 日的所有分片,
|
||||
到 9 月 9 日才允许清理 9 月 7 日,避免留下半天日志。夏令时日期允许为 23 或 25 小时。切换时区不重命名历史文件;
|
||||
按文件名日期与部署时区中的实际修改日期取较近值,旧日期分片近期被写入时整组延后删除。
|
||||
|
||||
请求转储保留期支持正小数,按每天 24 小时换算,例如 `0.5` 为 12 小时、`0.25` 为 6 小时。
|
||||
已关闭分片以最后修改时间判断过期,整片删除,不受时区或夏令时影响。
|
||||
时间轮转间隔取一小时与保留期中的较小值,在下一条记录写入时触发。
|
||||
实际保存时间可能长于配置值,取决于分片的最后写入时间和下次清理时机
|
||||
|
||||
`max_file_size_mb` 仅决定分片大小(默认 20 MiB),不决定保存时长,单条大记录不会被截断。
|
||||
关闭的分片压缩为 `.log.gz`;成功压缩、同步并发布归档后才删除原文件,保留原修改时间。
|
||||
清理发生在启动和轮转时;空闲期间过期文件可能暂时多保留。检索时须同时读取 `.log` 与 `.log.gz`
|
||||
清理发生在启动和轮转时;空闲期间过期文件可能暂时多保留。请求转储重启时新建分片,避免反复追加延后旧记录的过期点。
|
||||
检索时须同时读取 `.log` 与 `.log.gz`
|
||||
|
||||
报文开关为 `host.logging.request_dump`,默认关闭;开启时原始报文按块写入独立文件,
|
||||
缓冲拒收或写入失败可能造成缺块,需结合 `file_logging` 健康状态判断完整性。
|
||||
@@ -564,10 +574,6 @@ docker compose -f deploy/compose.yaml up -d --no-build --wait
|
||||
|
||||
升级后通过管理端版本接口或容器 image digest 确认运行实例的版本和 revision
|
||||
|
||||
管理端请求默认等待 120 秒。直接构建前端时可通过进程环境 `VITE_API_TIMEOUT_MS` 指定毫秒数,
|
||||
例如 `VITE_API_TIMEOUT_MS=180000 pnpm --dir frontend build`,取值须为正整数。
|
||||
该值在构建时写入静态资源,修改后需要重新构建和部署前端。单个请求显式指定的时限仍优先
|
||||
|
||||
构建元数据通过一次性进程环境传入:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -24,6 +24,7 @@ host:
|
||||
oauth_recovery: false
|
||||
# 将完整请求与响应头、正文写入独立文件;包含凭据,仅排障时开启。
|
||||
request_dump: false
|
||||
# 按实际时长保留,支持正小数天数,例如 0.5 为 12 小时
|
||||
request_dump_retention_days: 1
|
||||
file:
|
||||
enabled: true
|
||||
|
||||
+33
-22
@@ -443,20 +443,25 @@ xAI 的 `reasoning.effort` 接受 `none / minimal / low / medium / high / xhigh
|
||||
|
||||
#### 容量拒绝与重试
|
||||
|
||||
原生 Responses Lite WebSocket 在未启用中间件链时,按顺序及时交付正常事件,
|
||||
让客户端取得中断所需的响应 ID。无输出的 `response.created`、`response.in_progress`
|
||||
`turn_rate_limit_exceeded` 保留原始错误和 `Retry-After`,由客户端按自身重试预算等待。
|
||||
代理不因此写入账号冷却、换号或透明重试,其他会话仍可使用该账号
|
||||
|
||||
Responses 的 HTTP/SSE 与 WebSocket 按顺序即时交付正常事件,不暂存前导等待后续输出。
|
||||
未启用中间件链时,无输出的 `response.created`、`response.in_progress`
|
||||
及 `codex.response.metadata`、`codex.rate_limits` 元数据交付后仍可按已有安全条件重试,
|
||||
客户端可能收到下一次尝试的新 `response.created`。
|
||||
其他事件也立即交付,但会关闭透明重试窗口,后续前导事件不再恢复重试资格;
|
||||
错误事件先按既有重试规则处理,无法重试时交付错误
|
||||
其他事件也立即交付,但会关闭透明重试窗口,后续前导事件不再恢复重试资格。
|
||||
中间件链不启用可替换前导例外;错误事件先按既有重试规则处理,无法重试时交付错误
|
||||
|
||||
其他 WebSocket 与 HTTP/SSE 的前导事件保留到首次有效文本、推理、工具活动或终态再交付。
|
||||
前导缓冲不另设等待时长或累计体积上限,等待沿用[请求期限](#请求期限)与传输超时配置。
|
||||
等待沿用[请求期限](#请求期限)与传输超时配置。
|
||||
WebSocket 独立 Ping 保活;Responses HTTP 流式请求完成准入后,执行仍在等待且连续 15 秒无输出时提交 `200`
|
||||
和 SSE 注释心跳,此后的等待继续每 15 秒保活,包括请求中间件等待和前导事件缓冲。
|
||||
和 SSE 注释心跳,此后的等待继续每 15 秒保活,包括请求中间件等待和上游尚未返回事件。
|
||||
首次心跳前仍保留最终 HTTP 状态与响应头;心跳发出后发生的失败使用 `response.failed` 和 `[DONE]`,
|
||||
后续取得的上游或插件响应头无法再补发,`Retry-After` 转入流内错误的 `response.error.headers`。
|
||||
心跳不代表业务输出,不关闭已有的安全重试窗口
|
||||
心跳不代表业务输出,不关闭已有的安全重试窗口。
|
||||
SSE 注释和 WebSocket Ping/Pong 不重置官方 Codex 客户端的事件空闲超时,
|
||||
其 `stream_idle_timeout_ms` 默认是 300 秒;长请求需持续收到真实事件,
|
||||
或同时调整客户端与上游传输的空闲等待预算,单独缩短心跳间隔不能延长业务等待
|
||||
|
||||
WebSocket 在交付有效输出前断线或超时,可按 `stream_max_retries` 做同账号、有退避的尽力重试
|
||||
(默认最多 5 次),耗尽后仅在请求允许时改用 HTTP。已发送或发送结果不明不单独阻止这项恢复;
|
||||
@@ -568,15 +573,16 @@ call 与账号的绑定保存在网关进程内,未连接时一小时过期;
|
||||
### 统一登录与会话
|
||||
|
||||
管理员和密钥登录共用 `/api/auth/*`。登录模式 `mode` 只用于选择凭据验证方式,不直接授予权限;
|
||||
验证成功后,由后端写入身份和绑定 ID。一个浏览器只持有一份 `cpr_session` HttpOnly Cookie,
|
||||
原始 Key 不进入 URL、Pinia 或浏览器存储。登录页的切换只改变本地表单,不改变 URL。
|
||||
验证成功后,由后端写入身份和绑定 ID。浏览器通过 `cpr_session` HttpOnly Cookie 访问业务接口,
|
||||
管理员另持有仅发送到 `/api/auth` 的 `cpr_refresh` HttpOnly Cookie。原始 Key 不进入 URL、Pinia 或浏览器存储。
|
||||
登录页的切换只改变本地表单,不改变 URL。
|
||||
管理员进入管理端;Key 登录后进入 `/key-usage`,只读取当前会话绑定 Key 的数据
|
||||
|
||||
| 方法 | 路由 | 请求 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `POST` | `/api/auth/login` | `{ mode: "admin", username?, password }` 或 `{ mode: "key", apiKey }` | 验证凭据、创建会话;成功后撤销请求携带的旧会话 |
|
||||
| `GET` | `/api/auth/status` | 无 | 只读校验 Cookie 对应身份,返回 `{ authenticated, session }` |
|
||||
| `POST` | `/api/auth/refresh` | 无 | 校验身份并按管理员策略续期,返回同 status 的结构;有效会话同步设置 Cookie |
|
||||
| `POST` | `/api/auth/refresh` | 无 | 使用管理员续期 Cookie 恢复访问凭据;返回同 status 的结构并同步 Cookie,Key 会话保持原期限 |
|
||||
| `POST` | `/api/auth/logout` | 无 | 删除当前会话并清除 Cookie;存储失败返回 503,不假装退出成功 |
|
||||
| `POST` | `/api/auth/password` | `{ currentPassword, newPassword }` | 仅管理员会话可用;验证当前密码后修改密码,撤销全部管理员会话并清除当前 Cookie |
|
||||
|
||||
@@ -589,11 +595,14 @@ call 与账号的绑定保存在网关进程内,未连接时一小时过期;
|
||||
并发修改中只有旧密码哈希仍匹配的请求可以提交,冲突返回 409;密码更新与安全审计在同一事务提交。
|
||||
该入口共用登录尝试限流,超限返回 429。普通设置变更和管理员 API Key 变更不撤销密码登录会话,密钥身份会话也不受改密影响
|
||||
|
||||
会话由服务端保存,Cookie 属性为 `Path=/; HttpOnly; SameSite=Lax`,`Max-Age` /
|
||||
`Expires` 对齐服务端有效期,`Secure` 沿用上述 Origin 规则。普通业务请求和状态轮询不会续期。
|
||||
管理员通过 refresh 在不活动期限内续期,不能超过登录时确定的最长有效期;已过期或撤销的会话不会重新创建。
|
||||
refresh 未认证时返回 `{ authenticated: false, session: null }`,不写 Cookie;依赖故障返回 503。
|
||||
密钥身份和缺少最长有效期记录的会话保持固定期限。期限配置见[部署配置](../deploy/README.md#手动安装)
|
||||
会话由服务端保存,两种 Cookie 均为 `HttpOnly; SameSite=Lax`,`Max-Age` / `Expires` 对齐各自有效期,
|
||||
`Secure` 沿用上述 Origin 规则。访问 Cookie 使用 `Path=/`,续期 Cookie 使用 `Path=/api/auth`,不能直接访问业务接口。
|
||||
普通业务请求和状态轮询不会续期。管理员访问凭据到期后,仍可通过续期 Cookie 恢复登录;续期成功会轮换两种凭据,
|
||||
并重新计算不活动期限,但不能超过登录时确定的最长有效期。访问凭据仍有效时,refresh 返回同一组凭据和期限。
|
||||
轮换后 30 秒内重试上一代续期凭据返回同一组新凭据;超出窗口再次使用已识别的上一代凭据会撤销该会话。
|
||||
续期已过期或会话已撤销时不会重新创建;未认证返回 `{ authenticated: false, session: null }`,不写 Cookie;依赖故障返回 503。
|
||||
退出可以只携带续期 Cookie,并清除两种 Cookie。Key 登录清除续期 Cookie;密钥身份和仅有单凭据的会话保持固定期限。
|
||||
期限配置及升级注意事项见[部署配置](../deploy/README.md#手动安装)
|
||||
|
||||
每次恢复密钥会话时重新确认 Key 存在且启用;停用或删除后会话失效,重新启用不会恢复已撤销会话。
|
||||
依赖不可用时返回 503,不返回已认证或假装未登录。预算耗尽不妨碍登录
|
||||
@@ -606,7 +615,8 @@ refresh 未认证时返回 `{ authenticated: false, session: null }`,不写 Co
|
||||
拒绝时使用 `42901` 和 `Retry-After`
|
||||
|
||||
认证错误共用 `40101`(会话失效)、`40102`(凭据错误)和 `40301`(权限不足)。
|
||||
前端在 `40101` 后统一尝试一次会话恢复,并发请求等待同一恢复结果;恢复成功后最多重放一次,确认失效才退出。
|
||||
前端首次读取状态未认证时尝试续期;业务请求在 `40101` 后统一尝试一次会话恢复,并发请求等待同一恢复结果;
|
||||
恢复成功后最多重放一次,确认失效才退出。
|
||||
`40102` 和 `40301` 不触发恢复,网络或依赖故障不清除身份
|
||||
|
||||
### Key 用量与客户端配置
|
||||
@@ -1693,7 +1703,8 @@ Windows 离线包接口固定解析 Microsoft Store Product ID `9PLM9XGG6VKS`
|
||||
产品 ID、上游地址、ring 或文件名。后端只返回通过包名、架构、Microsoft CDN host/path、scheme 和失效
|
||||
时间校验的 `x64` / `arm64` MSIX 直链,不代理安装包字节。Store 内容通道返回 HTTP/80 临时地址时保留
|
||||
原始 scheme,不强制改写为该 host 不保证支持的 HTTPS。动态链接不足 10 分钟即失效时不会下发;某个架构
|
||||
解析失败时只将该架构降级到 OpenAI 官方 HTTPS 稳定 MSIX,并通过 `warning` 说明。响应形状为:
|
||||
解析失败时只将该架构降级到 OpenAI 官方 HTTPS 稳定 MSIX,并通过 `warning` 说明。Store 解析最多等待
|
||||
30 秒,官方回退包通过最多 3 秒的 HEAD 请求补充大小,大小查询失败不影响下载地址。响应形状为:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -1706,7 +1717,7 @@ Windows 离线包接口固定解析 Microsoft Store Product ID `9PLM9XGG6VKS`
|
||||
"source": "microsoft_store",
|
||||
"version": "26.825.6671.0",
|
||||
"fileName": "OpenAI.Codex_26.825.6671.0_x64__2p2nqsd0c76g0.msix",
|
||||
"sizeBytes": 744250000,
|
||||
"sizeBytes": 780402688,
|
||||
"downloadUrl": "http://dl.delivery.mp.microsoft.com/filestreamingservice/files/...",
|
||||
"expiresAt": "2026-09-01T07:30:00Z"
|
||||
}
|
||||
@@ -1952,9 +1963,9 @@ WebSocket 握手默认请求专项计时,客户端显式提供的开关保留
|
||||
|
||||
管理端下载的诊断包 `schemaVersion: 4` 用于人工反馈,不是备份或导入格式。它包含关联 ID、错误分类摘要、
|
||||
请求与错误事件各自的状态、attempt、时间线阶段、计时和上述失败分类。
|
||||
`provider.precommit.released` 事件的 `precommitRelease` 保留 `reason / prefetchedBytes / waitMs`,
|
||||
分别表示 Provider 释放缓存的原因、累计预取字节数和等待毫秒数,不等同于下游已提交。
|
||||
当前释放原因为 `semantic_output / terminal / eof`,导出也接受历史记录中的 `grace_timeout / byte_limit`。
|
||||
诊断包兼容已有 `provider.precommit.released` 记录,其 `precommitRelease` 保留 `reason / prefetchedBytes / waitMs`,
|
||||
分别表示 Provider 释放暂存的原因、累计预取字节数和等待毫秒数,不等同于下游已提交。
|
||||
导出接受记录中的 `semantic_output / terminal / eof / grace_timeout / byte_limit`。
|
||||
数值只允许非负安全整数,缺失或未通过校验的字段为 `null`;其他事件的 `precommitRelease` 为 `null`
|
||||
|
||||
诊断包不自动导出 message/raw error、任意 metadata、其他 trace event data、请求响应正文和头部。
|
||||
|
||||
+13
-11
@@ -376,23 +376,23 @@ Client Key 鉴权完成后,API adapter 从有界请求头识别客户端,Cor
|
||||
|
||||
- 一个客户端请求对应一条 `model_requests`;attempt 是请求内事实,不建立第二张权威表
|
||||
- Provider 的一次 `execute` 只选择一个 credential 并返回一个冷流;换号、重试和 fallback 由 Core 决定
|
||||
- OpenAI 的自动恢复限于明确识别的账号、容量、续接和传输场景;普通上游拒绝保留响应与重试提示,
|
||||
交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围,明确的 `426` 仍允许 HTTP 回退
|
||||
- OpenAI 的自动恢复限于明确识别的账号、容量、`503 + error.code=server_error`、续接和传输场景;
|
||||
其他上游拒绝保留响应与重试提示,交由客户端决策。WS opening 尚未发送业务 payload 也不扩大该恢复范围,明确的 `426` 仍允许 HTTP 回退
|
||||
- `not_sent`、`sent`、`ambiguous` 是单调的上游发送边界;结果不明确时不能假定上游未收到请求
|
||||
- HTTP 明确未发送的瞬时建连失败(含 TLS 握手 EOF)共享最多 4 次 opening、30 秒的请求内预算;
|
||||
证书、TLS 协议及配置错误不进入同账号建连恢复,取消和执行 deadline 优先终止等待
|
||||
- downstream commit 记录不可撤回的首次交付;不可重放事件交付后禁止换号、重试和 fallback
|
||||
HTTP 保活可先提交响应头,Core 记录该首字节时间与实际状态,但首个业务批次仍独立提交;
|
||||
SSE 注释不构成业务输出或上游已发送的证明,也不改变重放资格
|
||||
原生 Responses Lite WebSocket 及时交付正常事件,仅 Provider 明确标记的可替换前导保留重试资格;Core 单独维护重放屏障,
|
||||
原生 Responses HTTP/SSE 与 WebSocket 即时交付正常事件,仅 Provider 明确标记的可替换前导保留重试资格;Core 单独维护重放屏障,
|
||||
仍检查错误分类、续接归属、发送状态、副作用及预算。中间件链不启用此前导例外
|
||||
- API 在最终错误编码出口投影客户端恢复信号;该投影不修改 Provider 上游事实,也不改变 Core 的
|
||||
重试与提交边界。具体错误合同见 [数据面接口](api.md#3-openai-数据面与模型目录)
|
||||
- Provider 可将明确容量拒绝标记为有界同账号退避,Core 在既有安全重放边界内执行,按账号维护请求内
|
||||
预算,耗尽后复用普通换号路径。该退避消耗总路由预算,与 WS 传输恢复、OAuth 刷新及账号额度冷却分开
|
||||
- OpenAI Provider 在原生 Responses Lite WebSocket 上不等待后续事件再交付正常帧;
|
||||
未知事件立即交付并结束重放窗口。其他 WebSocket 及 HTTP/SSE 保留到有效输出或终态,
|
||||
不按等待时长或累计体积提前交付,中间件链沿用缓冲策略。
|
||||
- Provider 可将明确容量拒绝或 `503 + error.code=server_error` 标记为有界同账号退避,Core 在既有安全重放边界内执行,
|
||||
按账号维护请求内预算,耗尽后复用普通换号路径。该退避消耗总路由预算,与 WS 传输恢复、OAuth 刷新及账号额度冷却分开
|
||||
- OpenAI Provider 与 WS 传输层不暂存前导等待后续事件,正常帧按顺序即时交付;
|
||||
Provider 只保留单向推进的重放边界,实际输出、终态或未知事件关闭重放窗口。
|
||||
中间件链中的正常事件同样即时交付,但不保留可替换前导的重试资格。
|
||||
有效输出交付前的 WS 传输错误可请求有界同账号尽力恢复,即使发送结果不明也保留此恢复意图;
|
||||
发送事实与安全重放证明不因此改写。Provider 限定传输次数与续接范围,Core 保持原账号并检查
|
||||
取消、deadline、已观测外部副作用及下游交付边界
|
||||
@@ -587,9 +587,11 @@ Vue 普通管理请求的错误提示由 `api/request/` 请求封装统一负责
|
||||
AuthService 每次恢复 Key 会话时重新检查 Key 是否存在且启用;Key 会话不能访问管理员页面和管理接口。
|
||||
前端只维护一份 Auth Store,不在每个 API 请求上标记身份;401 会话失效、403 权限不足和 503 依赖故障分别处理。
|
||||
成功登录替换当前浏览器的旧会话,登出必须确认服务端撤销。
|
||||
管理员的不活动期限和最长有效期由 AuthService 持有;显式续期验证当前身份并限制写入频率,Store 原子比较会话后延长期限,
|
||||
不重新创建已退出或过期的键,API 将返回的有效期同步到 Cookie。Key 与缺少最长有效期记录的会话保持固定期限。
|
||||
前端首次进入受保护页面只读取登录状态,点击、滚动、聚焦等页面活动不触发续期;
|
||||
管理员访问凭据和续期凭据绑定同一逻辑会话,AuthService 负责签发、身份校验、轮换、不活动期限和最长有效期。
|
||||
Store 保存凭据摘要与上一代重试所需的派生盐,原子比较后替换记录或撤销;Redis TTL 对齐续期期限,
|
||||
访问到期不删除逻辑会话,刷新不能重建已退出或过期的键。短窗口重试返回同一代凭据,避免响应丢失与并发刷新互相撤销。
|
||||
API 只把签发结果投影为两种 HttpOnly Cookie;Key 与单凭据会话保持固定期限。
|
||||
前端首次进入受保护页面先读取状态,未认证时尝试续期;点击、滚动、聚焦等页面活动不单独触发续期;
|
||||
认证 Cookie 写入在支持 Web Locks 的环境中跨标签页串行,其他环境在当前页面串行。
|
||||
请求层仅在业务接口返回 `40101` 时共用一次会话续期,失败请求及恢复期间的新请求等待同一结果;
|
||||
成功后原请求最多重放一次,取消或身份已切换的请求不重放,确认失效才退出。恢复接口跳过该协调以避免循环等待。
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# PostgreSQL 3100 万请求优化与验收
|
||||
|
||||
本报告对应 [Issue #413](https://github.com/zyycn/codex-proxy-rs/issues/413),范围为大数据量下的统计、请求分页、账号用量、分组费用与额度预测
|
||||
|
||||
3100 万条源请求、窄事实和完整汇总已构建并独立对账。当前 HTTP 矩阵执行 167 次请求,包含 145 次合法业务请求、22 次预期参数或权限拒绝,未出现超时或非预期失败。按实际注册路由计,84 条已有通过记录、70 条未覆盖、6 条不适用;不宣称全接口或长期负载验收通过
|
||||
|
||||
## 问题与实现边界
|
||||
|
||||
原账号用量读取在命中事实超过 10 万条时拒绝查询;其他统计的事实回退与汇总读取也有固定行数门槛。仅放宽门槛会继续放大应用端逐行合并、深页 OFFSET 和预测窗口排序的成本
|
||||
|
||||
实现继续使用原生 PostgreSQL、现有窄事实、十五分钟/小时汇总和投影 Worker:
|
||||
|
||||
- 账号与 Key 的必要模型联合维度进入共享汇总,完整桶和精确边界在数据库内聚合,不以命中数据量拒绝查询
|
||||
- 账号窗口、用量排序、分组费用和分页计数在同一读快照补齐待投影修改与删除,保留精确总数、金额及未知值语义
|
||||
- 请求深页通过时间分段定位,错误列表合并两类来源后保持稳定排序
|
||||
- 预测使用开始时间与完成时间二维汇总,累计量保留完整窗口,Provider 文档独立有界采样
|
||||
- 投影在同一事务撤销旧贡献并加入新贡献;普通更新维护构建前缀,不反复重启整个桶
|
||||
- 观测操作与前端默认保护时限为 120 秒,可分别配置;30 秒为优化方向,不能将提高超时时限当作查询提速
|
||||
|
||||
使用记录搜索支持密钥、账号与模型,不匹配请求 ID;请求 ID 的显式精确筛选仍保留。汇总、洞察和诊断不接受搜索参数,管理观测接口不接受路由分组筛选
|
||||
|
||||
当前合同与职责见 [API 参考](../api.md)、[系统架构](../architecture.md#10-观测与后台任务)与[部署配置](../../deploy/README.md),本报告不重复字段清单
|
||||
|
||||
## 环境与样本
|
||||
|
||||
| 项目 | 实际条件 |
|
||||
| --- | --- |
|
||||
| 数据 | 31 天、3100 万条合成源请求;50 个账号、100 个 Key、20 个模型、10 个分组 |
|
||||
| 时间范围 | `2026-09-12T04:45:00Z` 至 `2026-10-13T04:45:00Z` |
|
||||
| 数据库 | PostgreSQL 18.6,容器限制 4 CPU、4 GiB 内存,持久磁盘 |
|
||||
| 主要参数 | shared_buffers 512 MiB、work_mem 16 MiB、max_parallel_workers 8、max_parallel_workers_per_gather 2、JIT 开启 |
|
||||
| 持久性 | fsync、full_page_writes、synchronous_commit 均开启 |
|
||||
| 应用 | release 构建;核心轮次重启应用清空应用缓存,16 个查询客户端并发 |
|
||||
| 其他轮次 | 剩余只读接口 4 并发;合成配置写入、鉴权及本机 Provider 场景串行执行 |
|
||||
| 缓存边界 | 未清空数据库与宿主文件缓存,应用冷缓存不等于数据库冷缓存 |
|
||||
|
||||
造数与离线汇总构建使用的临时资源配置已恢复,搜索索引试验已撤销。3100 万条容量基线不是生产 Worker 全量回填吞吐的测量结果;生产投影的正确性通过真实源表回归及样本对账验证
|
||||
|
||||
## HTTP 结果
|
||||
|
||||
以下为当前矩阵中的单次短样本,不能推断稳定的 P95/P99。并发样本包含资源竞争,不能与隔离单请求耗时直接当作前后优化对照
|
||||
|
||||
| 场景 | 耗时 |
|
||||
| --- | ---: |
|
||||
| 用量明细宽范围模型前缀搜索 | 32.63 秒 |
|
||||
| 账号用量降序 | 20.50 秒 |
|
||||
| 账号最近使用降序 | 20.39 秒 |
|
||||
| 账号额度预测 | 19.94 秒 |
|
||||
| Key 维度诊断 | 18.46 秒 |
|
||||
| Dashboard 概览 | 12.30 秒 |
|
||||
| 账号维度诊断 | 10.82 秒 |
|
||||
| 用量洞察 | 6.20 秒 |
|
||||
| 用量汇总 | 6.17 秒 |
|
||||
| 用量中间页 | 3.60 秒 |
|
||||
| 用量首/尾/超尾页 | 3.46/1.63/1.39 秒 |
|
||||
| 错误列表第 6200 页 | 1.70 秒 |
|
||||
|
||||
宽范围搜索命中大量明细,32.63 秒超过 30 秒优化期望,但未触发 120 秒保护时限。本次保留该性能局限,不再为此扩展索引或引入其他存储系统
|
||||
|
||||
正确性核对覆盖源请求总量、Token、费用、主要筛选组合、诊断分母及分页结果。移除的搜索与分组参数按预期返回 400,权限负例按预期返回拒绝;这些负例不计作合法容量查询通过
|
||||
|
||||
## 回归与自审范围
|
||||
|
||||
- 超过 10 万条真实源请求经过生产 Worker,验证账号/Key 汇总、热点边界和未知 Token;锁住窄事实表时,干净完整桶仍可读取
|
||||
- 待投影跨账号/模型/时间桶修改、删除、重复投递、写入失败回滚及构建前缀持续更新
|
||||
- 精确分页首中尾页、超尾页、跨时间段与同时间戳排序,两类错误来源合并
|
||||
- 预测微秒边界、同完成时间、截止时间、缺失费用、恢复请求、重建与待投影补差
|
||||
- 连接排队计入总预算,超时与取消后的事务回滚及连接归还仍持有查询名额
|
||||
- 分组费用沿用当前成员关系;账号窗口的 Token 兜底与普通统计的未知值口径分别保留
|
||||
- 投影与极值重算共用维度定义,已删除请求 ID 搜索捷径及淘汰的逐行读取映射
|
||||
- Rustfmt、相关 Store/Admin/API 回归、严格 Clippy、架构边界、迁移冻结清单、前端类型构建与 ESLint 对应实际最终工作区执行
|
||||
|
||||
页面仅调整搜索提示;按本次任务要求不提供截图
|
||||
|
||||
## 升级与验证局限
|
||||
|
||||
迁移 0030/0031 增加统计字段和索引,将既有时间桶排入可恢复重建,不清空窄事实或重置已完成的源投影游标。重建中的桶走事实回退,索引创建与桶重建仍需要额外 I/O、磁盘空间和时间;未完成生产环境在线升级的时延与负载验收
|
||||
|
||||
本次收尾聚焦 3100 万请求瓶颈,不再扩展外围插件、OAuth、备份和升级场景。数据库冷缓存、长期持续负载、生产 Worker 全量回填性能及未覆盖外部集成保留为验证局限;小规模生命周期回归不等于相同场景的 3100 万级持续写入验收
|
||||
|
||||
临时造数、HTTP 驱动、SQL 计划及逐请求记录保存在忽略跟踪的 `.runtime/capacity`,不进入工程提交。完整请求与路由结果为 `http-31m-latest.json`/`http-31m-latest.md`,原失败轮次分别保留,最终检查日志使用 `final-*` 前缀
|
||||
Vendored
-4
@@ -1,9 +1,5 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
readonly VITE_API_TIMEOUT_MS?: string
|
||||
}
|
||||
|
||||
declare module '@fontsource-variable/inter'
|
||||
declare module '@fontsource-variable/jetbrains-mono'
|
||||
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
export const API_BASE_URL = import.meta.env.DEV ? '/dev' : ''
|
||||
const timeout = Number(import.meta.env.VITE_API_TIMEOUT_MS ?? 120_000)
|
||||
if (!Number.isSafeInteger(timeout) || timeout <= 0 || timeout > 2_147_483_647) {
|
||||
throw new Error('VITE_API_TIMEOUT_MS must be a positive 32-bit integer')
|
||||
}
|
||||
export const API_TIMEOUT_MS = timeout
|
||||
export const API_TIMEOUT_MS = 120_000
|
||||
|
||||
@@ -422,6 +422,7 @@ onBeforeUnmount(() => {
|
||||
|
||||
<ZScrollbar class="my-6 w-full flex-1">
|
||||
<div class="px-4">
|
||||
<!-- 焦点环绘制在按钮内侧,避免被滚动区和折叠菜单裁切 -->
|
||||
<nav class="relative grid gap-3" :class="isCollapsed ? 'mx-auto w-11.5' : 'w-full'" aria-label="主导航">
|
||||
<span
|
||||
class="pointer-events-none absolute inset-x-0 top-0 h-11.5 overflow-hidden rounded-cp bg-cp-menu-item-selected-bg transition-[transform,opacity] duration-260 ease-[cubic-bezier(0.22,1,0.36,1)] motion-reduce:transition-none"
|
||||
@@ -436,7 +437,7 @@ onBeforeUnmount(() => {
|
||||
<div v-if="item.path === '/plugins'" class="relative z-10 grid min-w-0">
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex h-11.5 cursor-pointer items-center rounded-cp border-0 bg-transparent text-sm leading-[1.15] outline-none focus-visible:ring-2 focus-visible:ring-cp-control-outline focus-visible:ring-offset-2 focus-visible:ring-offset-cp-bg-container"
|
||||
class="inline-flex h-11.5 cursor-pointer items-center rounded-cp border-0 bg-transparent text-sm leading-[1.15] outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-cp-control-outline"
|
||||
:class="[
|
||||
isCollapsed ? 'w-11.5 justify-center' : 'w-full gap-3 px-4',
|
||||
isActive(item.path)
|
||||
@@ -490,7 +491,7 @@ onBeforeUnmount(() => {
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
class="relative inline-flex h-10 min-w-0 cursor-pointer items-center gap-2 rounded-cp border-0 bg-transparent px-3 text-left text-cp-sm outline-none transition-colors focus-visible:ring-2 focus-visible:ring-cp-control-outline motion-reduce:transition-none"
|
||||
class="relative inline-flex h-10 min-w-0 cursor-pointer items-center gap-2 rounded-cp border-0 bg-transparent px-3 text-left text-cp-sm outline-none transition-colors focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-cp-control-outline motion-reduce:transition-none"
|
||||
:class="route.name === 'plugins'
|
||||
? 'font-emphasis text-cp-text'
|
||||
: navFeedbackMuted ? 'font-semibold text-cp-text-secondary' : 'font-semibold text-cp-text-secondary hover:bg-cp-fill-quaternary hover:text-cp-text'"
|
||||
@@ -504,7 +505,7 @@ onBeforeUnmount(() => {
|
||||
v-for="view in pluginPageViews"
|
||||
:key="view.target.instanceId"
|
||||
type="button"
|
||||
class="relative inline-flex h-10 min-w-0 cursor-pointer items-center gap-2 rounded-cp border-0 bg-transparent px-3 text-left text-cp-sm outline-none transition-colors focus-visible:ring-2 focus-visible:ring-cp-control-outline motion-reduce:transition-none"
|
||||
class="relative inline-flex h-10 min-w-0 cursor-pointer items-center gap-2 rounded-cp border-0 bg-transparent px-3 text-left text-cp-sm outline-none transition-colors focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-cp-control-outline motion-reduce:transition-none"
|
||||
:class="isPluginViewActive(view)
|
||||
? 'font-emphasis text-cp-text'
|
||||
: navFeedbackMuted ? 'font-semibold text-cp-text-secondary' : 'font-semibold text-cp-text-secondary hover:bg-cp-fill-quaternary hover:text-cp-text'"
|
||||
@@ -522,7 +523,7 @@ onBeforeUnmount(() => {
|
||||
<button
|
||||
v-else
|
||||
type="button"
|
||||
class="relative z-10 inline-flex h-11.5 cursor-pointer items-center rounded-cp border-0 text-sm leading-[1.15] outline-none focus-visible:ring-2 focus-visible:ring-cp-control-outline focus-visible:ring-offset-2 focus-visible:ring-offset-cp-bg-container"
|
||||
class="relative z-10 inline-flex h-11.5 cursor-pointer items-center rounded-cp border-0 text-sm leading-[1.15] outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-cp-control-outline"
|
||||
:class="[
|
||||
isCollapsed ? 'w-11.5 justify-center' : 'w-full gap-3 px-4',
|
||||
isActive(item.path)
|
||||
|
||||
@@ -19,7 +19,10 @@ export const useAuthStore = defineStore('auth', () => {
|
||||
function checkAuth(): Promise<boolean> {
|
||||
if (pendingCheck.value)
|
||||
return pendingCheck.value
|
||||
const check = loadSession(getAuthStatus).finally(() => {
|
||||
const check = loadSession(async () => {
|
||||
const status = await getAuthStatus()
|
||||
return status.authenticated ? status : refreshAuthSession()
|
||||
}).finally(() => {
|
||||
if (pendingCheck.value === check)
|
||||
pendingCheck.value = undefined
|
||||
})
|
||||
|
||||
@@ -10,7 +10,6 @@ defineProps<{
|
||||
groups: AccountGroupRef[]
|
||||
groupsLoading: boolean
|
||||
saving: boolean
|
||||
hasChanges: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -59,7 +58,7 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
<ZButton
|
||||
type="primary"
|
||||
:loading="saving"
|
||||
:disabled="selectedCount === 0 || groupsLoading || !hasChanges"
|
||||
:disabled="selectedCount === 0 || groupsLoading"
|
||||
@click="emit('save')"
|
||||
>
|
||||
保存更改
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { Ref } from 'vue'
|
||||
import type { Account, AccountModelAccess } from '@/api'
|
||||
|
||||
import { ZNotification } from '@codex-proxy/ui'
|
||||
import { computed, ref, shallowRef, watch } from 'vue'
|
||||
import { ref, shallowRef, watch } from 'vue'
|
||||
import { batchUpdateAccounts } from '@/api'
|
||||
import { useAsyncAction } from '@/composables/useAsyncAction'
|
||||
import { accountModelAccessError } from '../utils/modelAccess'
|
||||
@@ -17,24 +17,23 @@ export function useAccountBatchEditor(options: {
|
||||
const selectedAccountsById = new Map<string, Account>()
|
||||
const showBatchEditModal = shallowRef(false)
|
||||
const editingCount = shallowRef(0)
|
||||
const schedulingEnabled = shallowRef(true)
|
||||
const schedulingEnabled = shallowRef(false)
|
||||
const concurrencyLimit = shallowRef('')
|
||||
const weight = shallowRef('1')
|
||||
const modelAccess = ref<AccountModelAccess | undefined>()
|
||||
const editedFields = ref(new Set<'enabled' | 'concurrencyLimit' | 'weight' | 'groupIds'>())
|
||||
const editedFields = ref(new Set<'concurrencyLimit' | 'weight' | 'groupIds'>())
|
||||
const catalogAccountId = shallowRef<string>()
|
||||
const proxyMode = shallowRef('preserve')
|
||||
const proxyId = shallowRef('')
|
||||
const selectedGroupIds = ref<string[]>([])
|
||||
const saveAction = useAsyncAction()
|
||||
const saving = saveAction.loading
|
||||
const hasChanges = computed(() => Boolean(modelAccess.value) || editedFields.value.size > 0 || proxyMode.value !== 'preserve')
|
||||
|
||||
// 未操作的字段保持每个账号原值,避免展开表单就覆盖混合设置。
|
||||
watch([schedulingEnabled, concurrencyLimit, weight, selectedGroupIds], (values, previous) => {
|
||||
// 调度按开关值统一更新,其他未操作字段保留每个账号原值
|
||||
watch([concurrencyLimit, weight, selectedGroupIds], (values, previous) => {
|
||||
if (!showBatchEditModal.value || saving.value)
|
||||
return
|
||||
const fields = ['enabled', 'concurrencyLimit', 'weight', 'groupIds'] as const
|
||||
const fields = ['concurrencyLimit', 'weight', 'groupIds'] as const
|
||||
fields.forEach((field, index) => {
|
||||
if (values[index] !== previous[index])
|
||||
editedFields.value.add(field)
|
||||
@@ -50,7 +49,7 @@ export function useAccountBatchEditor(options: {
|
||||
modelAccess.value = undefined
|
||||
editedFields.value.clear()
|
||||
catalogAccountId.value = accounts[0]?.id
|
||||
schedulingEnabled.value = accounts.every(account => account.enabled)
|
||||
schedulingEnabled.value = false
|
||||
proxyMode.value = 'preserve'
|
||||
proxyId.value = ''
|
||||
concurrencyLimit.value = sharedConcurrencyLimit(accounts)
|
||||
@@ -67,10 +66,6 @@ export function useAccountBatchEditor(options: {
|
||||
ZNotification.warning({ message: modelError })
|
||||
return
|
||||
}
|
||||
if (!hasChanges.value) {
|
||||
ZNotification.warning({ message: '请选择需要更新的设置' })
|
||||
return
|
||||
}
|
||||
const scheduling = parseAccountSchedulingForm(editedFields.value.has('concurrencyLimit') ? concurrencyLimit.value : '', editedFields.value.has('weight') ? weight.value : '1')
|
||||
if (proxyMode.value === 'proxy' && !proxyId.value.trim()) {
|
||||
ZNotification.warning({ message: '请选择已通过测试的代理' })
|
||||
@@ -87,7 +82,7 @@ export function useAccountBatchEditor(options: {
|
||||
accountIds,
|
||||
modelAccess: modelAccess.value,
|
||||
outboundProxyId: proxyMode.value === 'preserve' ? undefined : proxyMode.value === 'direct' ? '' : proxyId.value.trim(),
|
||||
enabled: editedFields.value.has('enabled') ? schedulingEnabled.value : undefined,
|
||||
enabled: schedulingEnabled.value,
|
||||
concurrencyLimit: editedFields.value.has('concurrencyLimit') ? scheduling.values.concurrencyLimit : undefined,
|
||||
weight: editedFields.value.has('weight') ? scheduling.values.weight : undefined,
|
||||
groupIds: editedFields.value.has('groupIds') ? [...new Set(selectedGroupIds.value)] : undefined,
|
||||
@@ -130,7 +125,6 @@ export function useAccountBatchEditor(options: {
|
||||
concurrencyLimit,
|
||||
weight,
|
||||
modelAccess,
|
||||
hasChanges,
|
||||
catalogAccountId,
|
||||
proxyMode,
|
||||
proxyId,
|
||||
|
||||
@@ -152,7 +152,6 @@ const {
|
||||
concurrencyLimit: batchConcurrencyLimit,
|
||||
weight: batchWeight,
|
||||
modelAccess: batchModelAccess,
|
||||
hasChanges: batchHasChanges,
|
||||
catalogAccountId: batchCatalogAccountId,
|
||||
editingCount: batchEditingCount,
|
||||
proxyMode: batchProxyMode,
|
||||
@@ -454,7 +453,6 @@ const {
|
||||
:groups="groups"
|
||||
:groups-loading="groupsLoading"
|
||||
:saving="savingBatchEdit"
|
||||
:has-changes="batchHasChanges"
|
||||
@save="saveBatchEdit"
|
||||
/>
|
||||
|
||||
|
||||
+13
-4
@@ -4,11 +4,13 @@ import { ZButton, ZEmpty, ZIconButton, ZSkeleton } from '@codex-proxy/ui'
|
||||
|
||||
import { ArrowDownToLine, PackageOpen, RefreshCw } from '@lucide/vue'
|
||||
|
||||
import { computed } from 'vue'
|
||||
|
||||
import ClientInstallCommandList from './ClientInstallCommandList.vue'
|
||||
|
||||
defineOptions({ name: 'ClientDesktopOfflineDownloads' })
|
||||
|
||||
withDefaults(
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
downloads?: CodexDesktopWindowsDownloads | null
|
||||
loading?: boolean
|
||||
@@ -26,6 +28,13 @@ const emit = defineEmits<{
|
||||
retry: []
|
||||
}>()
|
||||
|
||||
const downloadDescription = computed(() => {
|
||||
const packages = props.downloads?.packages
|
||||
if (packages?.length && packages.every(packageItem => packageItem.source === 'official_openai'))
|
||||
return 'OpenAI 官方安装包,请选择与设备一致的架构'
|
||||
return 'Microsoft Store 离线安装包,请选择与设备一致的架构'
|
||||
})
|
||||
|
||||
const localInstallCommand = [
|
||||
{
|
||||
label: '下载后使用 PowerShell 安装 / 升级',
|
||||
@@ -71,12 +80,12 @@ function formatFileSize(value: number | null): string {
|
||||
</span>
|
||||
</div>
|
||||
<p class="mt-1.5 mb-0 text-cp-sm leading-[1.45] font-semibold text-cp-text-secondary">
|
||||
后端实时校验 Microsoft CDN 直链,请选择与设备一致的架构
|
||||
{{ downloadDescription }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<ZIconButton
|
||||
:aria-label="loading ? '正在重新提取离线下载链接' : '重新提取离线下载链接'" variant="solid"
|
||||
:aria-label="loading ? '正在刷新离线下载链接' : '刷新离线下载链接'" variant="solid"
|
||||
:disabled="loading"
|
||||
:aria-busy="loading || undefined"
|
||||
@click="emit('refresh')"
|
||||
@@ -100,7 +109,7 @@ function formatFileSize(value: number | null): string {
|
||||
{{ error }},当前仍显示上次成功结果
|
||||
</p>
|
||||
|
||||
<div v-if="loading && !downloads" class="grid gap-3 sm:grid-cols-2" aria-label="正在提取离线安装包">
|
||||
<div v-if="loading && !downloads" class="grid gap-3 sm:grid-cols-2" aria-label="正在加载离线安装包">
|
||||
<div v-for="index in 2" :key="index" class="grid gap-3 rounded-cp bg-cp-bg-container p-4 shadow-cp-tertiary">
|
||||
<ZSkeleton class="h-4 w-28" />
|
||||
<ZSkeleton class="h-3 w-44" shape="text" />
|
||||
|
||||
@@ -21,6 +21,7 @@ export default defineConfig(({ mode }) => {
|
||||
'/dev': {
|
||||
target: 'http://127.0.0.1:8080',
|
||||
changeOrigin: true,
|
||||
cookiePathRewrite: { '/api/auth': '/dev/api/auth' },
|
||||
rewrite: path => path.replace(/^\/dev/, ''),
|
||||
},
|
||||
},
|
||||
|
||||
+12
-20
@@ -1,25 +1,17 @@
|
||||
# v3.26.0
|
||||
|
||||
## 新增与调整
|
||||
|
||||
- 移除「最小请求间隔」及其前后端、调度、数据库和插件设置字段;同一账号不再因上次请求开始时间而等待,并发、排队、额度与冷却限制继续生效
|
||||
- 使用记录的首字耗时和总耗时按固定阈值分档着色,便于识别慢请求,不改变请求成功或失败状态
|
||||
- 管理端请求和后端观测操作的默认保护时限调整为 120 秒,分别支持前端构建变量 `VITE_API_TIMEOUT_MS` 与部署配置 `store.pool.observability_timeout_seconds`;模型请求期限仍独立控制
|
||||
# v3.26.2
|
||||
|
||||
## 修复与改进
|
||||
|
||||
- 优化大规模请求历史下的统计、账号与 Key 用量、精确分页和额度预测,复用数据库聚合与时间分段查询,移除固定十万行拒绝条件;保留精确总数、金额及未知值语义,并及时修正尚未投影的修改和删除
|
||||
- 修复同一额度窗口内上游额度使用比例回落时预测累计量被错误重置的问题,保留已确认的历史用量
|
||||
- 管理端导入 OpenAI 多账号 JSON 文件时逐账号处理、分别记录结果,自动按条目数和请求体大小分批;单个账号失败不再影响整份文件,已知过期且没有刷新令牌的账号会明确拒绝导入
|
||||
- 优化 Codex 隐私规则对长请求的处理,减少重复复制、序列化和上下文遍历;取消固定 100 毫秒墙钟限制,保留大小、执行步数与请求取消检查,以及规则失败时的回滚语义
|
||||
- 保留租约丢失、续期超时、升级会话结束和备份清理的真实失败原因;S3 连接测试在探针删除失败时报告失败,文件交换回滚失败时保留主错误及恢复路径
|
||||
- 调整隐私策略文案与规则弹窗按钮,优化模型价格表的行高及单位提示
|
||||
- 更新 `quinn-proto`,修复相关依赖安全问题
|
||||
- 修复 OpenAI `turn_rate_limit_exceeded` 被错误扩大为账号冷却的问题,保留原始错误和 `Retry-After`,由客户端按自身重试预算处理,避免后续请求在本地被挡成 `no_available_provider`
|
||||
- OpenAI 明确返回 `503 + error.code=server_error` 时,在既有安全重放边界内进行有界重试,已交付正文或工具活动后不重放
|
||||
- 修复 OpenAI 流式失败和 WebSocket 复用时重复写回旧 Cookie 的问题,并区分凭据版本冲突与存储故障
|
||||
- OpenAI 账号预热改用模型目录支持的推理强度,避免因不支持的 `none` 导致预热失败
|
||||
- 管理员登录分离访问凭据与续期凭据,访问凭据到期后可自动恢复;续期轮换支持短窗口内的并发与重试,并识别超出重试窗口的旧续期凭据重用
|
||||
- 请求转储保留期支持正小数天数,例如 `0.5` 为 12 小时、`0.25` 为 6 小时,按已关闭分片的最后写入时间清理
|
||||
- Windows 离线安装包优先解析 Microsoft Store,最多等待 30 秒后按架构回退到 OpenAI 官方包;修正包大小并按实际来源展示下载说明
|
||||
- 账号批量编辑默认关闭调度,保存时始终应用界面显示的调度开关值;修复侧边栏键盘焦点环被裁切的问题
|
||||
|
||||
## 升级事项
|
||||
## 升级说明
|
||||
|
||||
- 新增 `0030_analytics_and_scheduling.sql`,创建统计索引、安排时间桶重建并删除请求间隔字段;保留已有统计事实和已完成的历史投影进度。升级前备份数据库,大数据量下索引创建会增加启动时间、磁盘占用与 I/O,迁移完成后服务才开始监听
|
||||
- 已发布的 `0001–0029` 迁移保持不变,可从 `v3.25.0` 正常升级。若源码部署已执行本轮 `main` 中合并前的 `0030–0032`,不能直接沿用该迁移历史升级,需单独处理开发库数据;不要修改 `_sqlx_migrations` 的校验值绕过检查
|
||||
- 管理 API 和插件设置不再提供 `requestIntervalMs`/`request_interval_ms`,依赖该字段的自定义调用方需同步调整
|
||||
- 使用记录搜索仅按密钥、账号和模型做前缀匹配,密钥与账号名称使用历史快照;请求 ID 和上游 ID 改用对应精确筛选参数。搜索不再改变顶部统计范围,汇总、洞察和诊断不接受搜索参数,管理观测接口不再接受路由分组筛选
|
||||
- 分组费用按当前组成员的请求费用汇总,调整成员关系后,留存窗口内的费用随当前成员重新计算
|
||||
- 已有单凭据登录会话保留到原期限,重新登录后获得新的自动续期能力
|
||||
- `host.logging.request_dump_retention_days` 的默认值仍为 `1`,含义改为最近 24 小时;清理按分片进行,实际保存时间受最后写入时间和下次清理时机影响
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.26.0"
|
||||
version: "3.26.2"
|
||||
|
||||
Reference in New Issue
Block a user