mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-08 16:05:33 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5671a57dc8 | ||
|
|
166e8754a2 | ||
|
|
fbaf6c1bdd | ||
|
|
7cdc2dba6f | ||
|
|
a321ccf737 | ||
|
|
4533ae8850 | ||
|
|
7542ed365e | ||
|
|
bf4da7a3e7 | ||
|
|
0fe354b083 | ||
|
|
6d0c583599 | ||
|
|
2e861c03c4 | ||
|
|
d5fc778ea7 | ||
|
|
404cd41ff4 | ||
|
|
851f473b88 | ||
|
|
e97da4f551 | ||
|
|
5544b0fb29 | ||
|
|
428e744710 | ||
|
|
7b406765d6 | ||
|
|
92ae7df115 | ||
|
|
cc42e78334 | ||
|
|
0c90ed30fb | ||
|
|
5a81856510 | ||
|
|
3110b71512 | ||
|
|
eac1b2c39e | ||
|
|
337bb80230 | ||
|
|
82f2a0da06 | ||
|
|
f0d8dad71e | ||
|
|
f5c1d773d7 | ||
|
|
9148e6b11e | ||
|
|
a7d8c3d3df | ||
|
|
997eb86aa0 | ||
|
|
3866acc269 | ||
|
|
38477f4328 |
@@ -21,15 +21,16 @@
|
||||
|
||||
## 能力
|
||||
|
||||
| 领域 | 实现 |
|
||||
| -------- | --------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI SSE 与 WebSocket 均原样转发上游字节;xAI 在 `providers/xai` 内做 Grok 与 Responses 协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因 |
|
||||
| 领域 | 实现 |
|
||||
| -------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -93,6 +94,11 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
|
||||
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
|
||||
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
|
||||
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
|
||||
[账号接口](docs/api.md#5-账号)。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
|
||||
@@ -145,7 +151,7 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
## 运维
|
||||
|
||||
```bash
|
||||
# 升级
|
||||
# 实例升级
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build
|
||||
|
||||
@@ -179,7 +185,8 @@ release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。
|
||||
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
|
||||
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
Generated
+3
@@ -1194,6 +1194,7 @@ dependencies = [
|
||||
"subtle",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -1214,6 +1215,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
"tower",
|
||||
"tower-http 0.7.0",
|
||||
"tracing",
|
||||
@@ -1232,6 +1234,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
|
||||
@@ -24,10 +24,11 @@ serde = { version = "1.0.228", features = ["derive"] }
|
||||
secrecy = { version = "0.10.3", features = ["serde"] }
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -212,6 +212,9 @@ pub enum AccountConnectionTestEvent {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
provider_error_type: Option<String>,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -330,6 +330,7 @@ pub struct LatencyPercentiles {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderBillingInput {
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -345,6 +346,7 @@ pub struct UsageCalculatedBillingFact {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub provider_kind: String,
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -574,6 +576,7 @@ pub struct UsageRecord {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;管理领域保持不透明。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
|
||||
@@ -2,14 +2,15 @@
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId},
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
};
|
||||
|
||||
use super::{
|
||||
MutationActor, MutationContext, PageSize, Revision,
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
};
|
||||
|
||||
@@ -170,13 +171,6 @@ pub struct CredentialImportCommit {
|
||||
pub prepared: PreparedCredentialImport,
|
||||
}
|
||||
|
||||
/// 可选的重新授权目标。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReauthorizationTarget {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub credential_revision: Revision,
|
||||
}
|
||||
|
||||
/// OAuth pending owner 的中立身份;不编码具体 Provider 的 Redis key 或 JSON。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationOwner {
|
||||
@@ -237,15 +231,13 @@ impl fmt::Debug for AuthorizationOwnerBinding {
|
||||
}
|
||||
|
||||
/// OAuth 完成时应创建新账号还是 CAS 更新既有 credential。
|
||||
///
|
||||
/// 重新授权只绑定稳定的账号身份:credential revision 由服务端在临近写入时读取,
|
||||
/// 长流程期间的后台刷新不得让恢复操作失效。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationMutationTarget {
|
||||
Create {
|
||||
name: String,
|
||||
},
|
||||
Reauthorize {
|
||||
account_id: ProviderAccountId,
|
||||
expected_credential_revision: Revision,
|
||||
},
|
||||
Create { name: String },
|
||||
Reauthorize { account_id: ProviderAccountId },
|
||||
}
|
||||
|
||||
/// 必须完整进入 Provider opaque pending payload、并在 complete 后原样恢复的事务信封。
|
||||
@@ -291,7 +283,7 @@ impl PendingAuthorizationMutation {
|
||||
pub struct StartAuthorization {
|
||||
pub context: MutationContext,
|
||||
pub name: String,
|
||||
pub reauthorization: Option<ReauthorizationTarget>,
|
||||
pub reauthorization: Option<ProviderAccountId>,
|
||||
}
|
||||
|
||||
/// OAuth 流程启动结果。
|
||||
@@ -310,6 +302,20 @@ pub trait CredentialCommitGuard: Send + 'static {
|
||||
fn finish(self: Box<Self>);
|
||||
}
|
||||
|
||||
/// OAuth pending claim 在 Store 事务结果确定后执行的显式结算动作。
|
||||
///
|
||||
/// OAuth provider 必须在 credential 准备阶段持有 claim:Store 提交成功后消费 flow,提交或
|
||||
/// 校验失败后释放 flow。异步结算不能依赖 `Drop`,否则请求提前返回会让同一授权流无谓地
|
||||
/// 进入短暂的不可重试状态。
|
||||
#[async_trait]
|
||||
pub trait AuthorizationCommitGuard: Send + 'static {
|
||||
/// Store 已提交 OAuth credential 后消费对应的一次性 flow。
|
||||
async fn commit(self: Box<Self>) -> Result<(), AdminError>;
|
||||
|
||||
/// Store 未提交 OAuth credential 时释放 claim,允许同一 flow 重试。
|
||||
async fn abort(self: Box<Self>) -> Result<(), AdminError>;
|
||||
}
|
||||
|
||||
/// OAuth complete 后由 Provider 返回的准备结果;Store 仍是唯一提交者。
|
||||
pub enum PreparedAuthorizationCredential {
|
||||
Create(PreparedCredentialCreate),
|
||||
@@ -320,6 +326,7 @@ pub enum PreparedAuthorizationCredential {
|
||||
pub struct PreparedAuthorizationCommit {
|
||||
pub pending: PendingAuthorizationMutation,
|
||||
pub credential: PreparedAuthorizationCredential,
|
||||
authorization_guard: Option<Box<dyn AuthorizationCommitGuard>>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreparedAuthorizationCommit {
|
||||
@@ -346,9 +353,37 @@ pub struct AuthorizationCommit {
|
||||
pub credential: AuthorizationCredentialCommit,
|
||||
}
|
||||
|
||||
/// OAuth 准备结果拆解后的 Store 命令与两个结算 guard。
|
||||
///
|
||||
/// Store 成功时先结束 credential guard,再消费 OAuth claim;失败时丢弃 credential guard 并
|
||||
/// 释放 OAuth claim,使同一授权流可以重试。
|
||||
pub(crate) struct AuthorizationCommitSettlement {
|
||||
pub(crate) command: AuthorizationCommit,
|
||||
pub(crate) credential_guard: Option<Box<dyn CredentialCommitGuard>>,
|
||||
pub(crate) authorization_guard: Option<Box<dyn AuthorizationCommitGuard>>,
|
||||
}
|
||||
|
||||
impl PreparedAuthorizationCommit {
|
||||
#[must_use]
|
||||
pub fn into_commit(self) -> (AuthorizationCommit, Option<Box<dyn CredentialCommitGuard>>) {
|
||||
pub fn new(
|
||||
pending: PendingAuthorizationMutation,
|
||||
credential: PreparedAuthorizationCredential,
|
||||
) -> Self {
|
||||
Self {
|
||||
pending,
|
||||
credential,
|
||||
authorization_guard: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让 OAuth claim 覆盖准备完成到 Store 事务结算之间的全部窗口。
|
||||
#[must_use]
|
||||
pub fn with_authorization_guard(mut self, guard: Box<dyn AuthorizationCommitGuard>) -> Self {
|
||||
self.authorization_guard = Some(guard);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_commit(self) -> AuthorizationCommitSettlement {
|
||||
let (credential, guard) = match self.credential {
|
||||
PreparedAuthorizationCredential::Create(credential) => {
|
||||
(AuthorizationCredentialCommit::Create(credential), None)
|
||||
@@ -361,13 +396,27 @@ impl PreparedAuthorizationCommit {
|
||||
)
|
||||
}
|
||||
};
|
||||
(
|
||||
AuthorizationCommit {
|
||||
AuthorizationCommitSettlement {
|
||||
command: AuthorizationCommit {
|
||||
pending: self.pending,
|
||||
credential,
|
||||
},
|
||||
guard,
|
||||
)
|
||||
credential_guard: guard,
|
||||
authorization_guard: self.authorization_guard,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn abort(self) -> Result<(), AdminError> {
|
||||
let AuthorizationCommitSettlement {
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
..
|
||||
} = self.into_commit();
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard {
|
||||
guard.abort().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,14 +471,12 @@ pub struct CredentialDeletionResult {
|
||||
/// Provider-owned token 轮换命令。
|
||||
pub struct RotateCredential {
|
||||
pub mutation: CredentialMutation,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
/// Provider 校验手工轮换材料时所需的非事务输入。
|
||||
pub struct PrepareCredentialRotation {
|
||||
pub account: AccountRecord,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
@@ -438,10 +485,6 @@ impl fmt::Debug for PrepareCredentialRotation {
|
||||
formatter
|
||||
.debug_struct("PrepareCredentialRotation")
|
||||
.field("account", &self.account)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
@@ -453,6 +496,7 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
@@ -514,10 +558,6 @@ impl fmt::Debug for RotateCredential {
|
||||
formatter
|
||||
.debug_struct("RotateCredential")
|
||||
.field("mutation", &self.mutation)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
|
||||
@@ -64,6 +64,12 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
/// 成功之后,不参与事务成败,也不得恢复或改写已经提交的账号状态。
|
||||
async fn account_unavailable(&self, account_id: &ProviderAccountId);
|
||||
|
||||
/// 账号资格事实已经由控制面提交,失效 Provider 持有的可重建派生状态。
|
||||
///
|
||||
/// 该通知发生在 Store 事务成功之后、下一份 RuntimeSnapshot 编译之前;通知
|
||||
/// 不参与已提交事务成败。没有账号派生状态的 Provider 可使用默认空实现。
|
||||
async fn account_facts_changed(&self, _account_ids: &[ProviderAccountId]) {}
|
||||
|
||||
/// 生成一次连接测试所需的 Provider-owned operation;Core 负责实际执行与落账。
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
|
||||
@@ -370,6 +370,9 @@ impl AccountsService for DefaultAccountsService {
|
||||
"provider credential refresh",
|
||||
)
|
||||
.await?;
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
let account = self.load_directory_item(&result.account_id, false).await?;
|
||||
Ok(AccountRefreshResult {
|
||||
@@ -449,12 +452,33 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_status: status,
|
||||
}))
|
||||
.collect(),
|
||||
Err(error) => vec![AccountConnectionTestEvent::Failed {
|
||||
message: error.client_message().to_owned(),
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
account_status: status,
|
||||
}],
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
.upstream_response()
|
||||
.map(gateway_core::engine::probe::AccountProbeUpstreamResponse::status);
|
||||
let upstream_content_type = error
|
||||
.upstream_response()
|
||||
.and_then(|response| response.content_type())
|
||||
.and_then(|value| std::str::from_utf8(value).ok())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_body = error
|
||||
.upstream_response()
|
||||
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
|
||||
let message = upstream_body
|
||||
.as_deref()
|
||||
.filter(|body| !body.is_empty())
|
||||
.unwrap_or_else(|| error.client_message())
|
||||
.to_owned();
|
||||
vec![AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
.flat_map(futures::stream::iter);
|
||||
@@ -465,22 +489,25 @@ impl AccountsService for DefaultAccountsService {
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
match account.availability {
|
||||
crate::model::accounts::AccountAvailability::Banned => AccountStatus::Banned,
|
||||
crate::model::accounts::AccountAvailability::QuotaExhausted => {
|
||||
AccountStatus::QuotaExhausted
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid => AccountStatus::Expired,
|
||||
_ if account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now) =>
|
||||
{
|
||||
AccountStatus::Expired
|
||||
}
|
||||
_ => AccountStatus::Active,
|
||||
}
|
||||
AccountStatus::Active
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -90,17 +90,10 @@ async fn pending_authorization(
|
||||
resource: &'static str,
|
||||
) -> Result<PendingAuthorizationMutation, AdminError> {
|
||||
let target = match &command.reauthorization {
|
||||
Some(target) => {
|
||||
let details =
|
||||
required_credential(accounts, provider_kind, &target.account_id, resource).await?;
|
||||
if details.credential.credential_revision != target.credential_revision {
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} reauthorization target is stale"
|
||||
)));
|
||||
}
|
||||
Some(account_id) => {
|
||||
required_credential(accounts, provider_kind, account_id, resource).await?;
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: target.account_id.clone(),
|
||||
expected_credential_revision: target.credential_revision,
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
}
|
||||
None => AuthorizationMutationTarget::Create {
|
||||
@@ -142,7 +135,6 @@ fn validate_prepared_rotation(
|
||||
let facts = prepared.facts();
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
|| facts.expected_credential_revision != account.credential_revision
|
||||
{
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} prepared facts do not match the current credential"
|
||||
@@ -151,18 +143,27 @@ fn validate_prepared_rotation(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_authorization_commit(
|
||||
async fn validate_authorization_commit(
|
||||
provider_kind: &ProviderKind,
|
||||
context: &MutationContext,
|
||||
prepared: &PreparedAuthorizationCommit,
|
||||
prepared: PreparedAuthorizationCommit,
|
||||
resource: &'static str,
|
||||
) -> Result<(), AdminError> {
|
||||
) -> Result<PreparedAuthorizationCommit, AdminError> {
|
||||
if prepared.pending.provider_kind() != provider_kind
|
||||
|| !prepared.pending.owner_binding().matches_context(context)
|
||||
{
|
||||
return Err(AdminError::conflict(format!(
|
||||
let validation_error = AdminError::conflict(format!(
|
||||
"{resource} pending authorization binding is invalid"
|
||||
)));
|
||||
));
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
let matches_target = match (prepared.pending.target(), &prepared.credential) {
|
||||
(
|
||||
@@ -170,25 +171,29 @@ fn validate_authorization_commit(
|
||||
PreparedAuthorizationCredential::Create(credential),
|
||||
) => credential.provider_kind == *provider_kind,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
},
|
||||
AuthorizationMutationTarget::Reauthorize { account_id },
|
||||
PreparedAuthorizationCredential::Reauthorize(credential),
|
||||
) => {
|
||||
let facts = credential.facts();
|
||||
facts.provider_kind == *provider_kind
|
||||
&& facts.account_id == *account_id
|
||||
&& facts.expected_credential_revision == *expected_credential_revision
|
||||
facts.provider_kind == *provider_kind && facts.account_id == *account_id
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if !matches_target {
|
||||
return Err(AdminError::conflict(format!(
|
||||
let validation_error = AdminError::conflict(format!(
|
||||
"{resource} prepared credential does not match its pending target"
|
||||
)));
|
||||
));
|
||||
if let Err(error) = prepared.abort().await {
|
||||
tracing::warn!(
|
||||
resource,
|
||||
settlement_error = %error,
|
||||
"OAuth authorization claim release failed after preparation validation"
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
return Err(validation_error);
|
||||
}
|
||||
Ok(())
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
async fn commit_authorization(
|
||||
@@ -197,16 +202,34 @@ async fn commit_authorization(
|
||||
context: &MutationContext,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let (command, guard) = prepared.into_commit();
|
||||
let crate::model::provider_credentials::AuthorizationCommitSettlement {
|
||||
command,
|
||||
credential_guard,
|
||||
authorization_guard,
|
||||
} = prepared.into_commit();
|
||||
match accounts.commit_authorization(command, context).await {
|
||||
Ok(result) => {
|
||||
if let Some(guard) = guard {
|
||||
if let Some(guard) = credential_guard {
|
||||
guard.finish();
|
||||
}
|
||||
if let Some(guard) = authorization_guard {
|
||||
guard.commit().await?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
drop(guard);
|
||||
drop(credential_guard);
|
||||
if let Some(guard) = authorization_guard
|
||||
&& let Err(settlement_error) = guard.abort().await
|
||||
{
|
||||
tracing::warn!(
|
||||
resource,
|
||||
store_error = %error,
|
||||
settlement_error = %settlement_error,
|
||||
"OAuth authorization claim release failed after Store commit failure"
|
||||
);
|
||||
return Err(settlement_error);
|
||||
}
|
||||
Err(map_store_error(error, resource))
|
||||
}
|
||||
}
|
||||
@@ -284,6 +307,9 @@ async fn set_credential_enabled(
|
||||
if !enabled {
|
||||
provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
Ok(CredentialMutationResult {
|
||||
config_revision: revision,
|
||||
account_id,
|
||||
@@ -316,6 +342,7 @@ async fn delete_credentials(
|
||||
for account_id in &account_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
provider.account_facts_changed(&account_ids).await;
|
||||
Ok(CredentialDeletionResult {
|
||||
config_revision: revision,
|
||||
account_ids,
|
||||
|
||||
@@ -428,6 +428,7 @@ fn recover_standard_costs(
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: fact.upstream_model_id,
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
@@ -491,14 +492,14 @@ impl DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 逐条尽力把 calculated 总额升级为完整分解;单条脏数据(非法 Provider kind
|
||||
/// 或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
/// 逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
|
||||
for record in records {
|
||||
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if source != "calculated" {
|
||||
if !matches!(source.as_str(), "calculated" | "provider_reported") {
|
||||
continue;
|
||||
}
|
||||
let (Some(provider), Some(upstream_model_id)) = (
|
||||
@@ -512,6 +513,7 @@ impl DefaultObservabilityService {
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: upstream_model_id.clone(),
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
|
||||
@@ -83,6 +83,27 @@ impl DefaultOpenAiService {
|
||||
snapshot,
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -129,18 +150,13 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "OpenAI credential import"))?;
|
||||
self.provider
|
||||
.account_facts_changed(&result.credential_ids)
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
// 导入已经提交,额度属于可重建观察事实;单个账号查询失败由周期任务重试。
|
||||
for account_id in &result.credential_ids {
|
||||
let _ = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
|
||||
.await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -171,12 +187,13 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.complete_authorization(command)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "OpenAI authorization"))?;
|
||||
validate_authorization_commit(
|
||||
let prepared = validate_authorization_commit(
|
||||
self.provider.provider_kind(),
|
||||
&context,
|
||||
&prepared,
|
||||
prepared,
|
||||
"OpenAI authorization",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
@@ -184,7 +201,15 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI authorization",
|
||||
)
|
||||
.await?;
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -201,17 +226,11 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
if details.credential.credential_revision != command.expected_credential_revision {
|
||||
return Err(AdminError::conflict(
|
||||
"OpenAI credential rotation revision is stale",
|
||||
));
|
||||
}
|
||||
let account = details.credential;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: account.clone(),
|
||||
expected_credential_revision: command.expected_credential_revision,
|
||||
provider_material: command.provider_material,
|
||||
})
|
||||
.await
|
||||
@@ -224,6 +243,9 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -148,12 +148,13 @@ impl XaiService for DefaultXaiService {
|
||||
.complete_authorization(command)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "xAI authorization"))?;
|
||||
validate_authorization_commit(
|
||||
let prepared = validate_authorization_commit(
|
||||
self.provider.provider_kind(),
|
||||
&context,
|
||||
&prepared,
|
||||
prepared,
|
||||
"xAI authorization",
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
|
||||
@@ -6,7 +6,7 @@ use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
operation::{GenerateRequest, Operation, ProtocolPayload},
|
||||
@@ -16,7 +16,7 @@ use gateway_core::{
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
AdminError, MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
|
||||
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
|
||||
@@ -24,12 +24,12 @@ use gateway_admin::{
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationCredentialCommit, AuthorizationMutationTarget,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialCommitGuard, CredentialDetails,
|
||||
CredentialImportCommit, CredentialImportResult, CredentialListQuery,
|
||||
CredentialMutationResult, CredentialPage, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
AuthorizationCommit, AuthorizationCommitGuard, AuthorizationCredentialCommit,
|
||||
AuthorizationMutationTarget, AuthorizationStarted, CompleteAuthorization,
|
||||
CredentialCommitGuard, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
|
||||
CredentialRotationCommit, PendingAuthorizationMutation, PrepareCredentialImport,
|
||||
PrepareCredentialRefresh, PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
@@ -58,11 +58,13 @@ pub(super) struct FakeProviderAdmin {
|
||||
events: EventLog,
|
||||
failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
pending: Mutex<Option<PendingAuthorizationMutation>>,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
retry_authorization_after_abort: Mutex<bool>,
|
||||
export_inputs: Mutex<Vec<ProviderExportCredentialInput>>,
|
||||
import_account_ids: Mutex<Vec<String>>,
|
||||
quota_requests: Mutex<Vec<ProviderQuotaRequest>>,
|
||||
quota: Mutex<ProviderQuota>,
|
||||
current_credential_revision: Mutex<Revision>,
|
||||
}
|
||||
|
||||
impl FakeProviderAdmin {
|
||||
@@ -72,11 +74,13 @@ impl FakeProviderAdmin {
|
||||
events,
|
||||
failure: Mutex::new(None),
|
||||
quota_failure: Mutex::new(None),
|
||||
pending: Mutex::new(None),
|
||||
pending: Arc::new(Mutex::new(None)),
|
||||
retry_authorization_after_abort: Mutex::new(false),
|
||||
export_inputs: Mutex::new(Vec::new()),
|
||||
import_account_ids: Mutex::new(vec!["acct_prepared".to_owned()]),
|
||||
quota_requests: Mutex::new(Vec::new()),
|
||||
quota: Mutex::new(empty_quota()),
|
||||
current_credential_revision: Mutex::new(revision(1)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -113,6 +117,20 @@ impl FakeProviderAdmin {
|
||||
self.pending.lock().expect("pending authorization").clone()
|
||||
}
|
||||
|
||||
pub(super) fn retry_authorization_after_abort(&self) {
|
||||
*self
|
||||
.retry_authorization_after_abort
|
||||
.lock()
|
||||
.expect("authorization retry") = true;
|
||||
}
|
||||
|
||||
pub(super) fn set_current_credential_revision(&self, revision: Revision) {
|
||||
*self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision") = revision;
|
||||
}
|
||||
|
||||
fn export_inputs(&self) -> Vec<ProviderExportCredentialInput> {
|
||||
self.export_inputs.lock().expect("export inputs").clone()
|
||||
}
|
||||
@@ -133,7 +151,11 @@ impl FakeProviderAdmin {
|
||||
PreparedCredentialRotationFacts {
|
||||
account_id: ProviderAccountId::new(account.id.clone()).expect("account ID"),
|
||||
provider_kind: account.provider_kind.clone(),
|
||||
expected_credential_revision: account.credential_revision,
|
||||
expected_credential_revision: *self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision"),
|
||||
replacement_identity: None,
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
@@ -159,6 +181,10 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
self.record("provider.account_unavailable");
|
||||
}
|
||||
|
||||
async fn account_facts_changed(&self, _: &[ProviderAccountId]) {
|
||||
self.record("provider.account_facts_changed");
|
||||
}
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
model: &gateway_core::routing::UpstreamModelId,
|
||||
@@ -246,23 +272,31 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
if !pending.owner_binding().matches_context(&command.context) {
|
||||
return Err(ProviderAdminError::new(ProviderAdminErrorKind::NotFound));
|
||||
}
|
||||
let retry_pending = (*self
|
||||
.retry_authorization_after_abort
|
||||
.lock()
|
||||
.expect("authorization retry"))
|
||||
.then(|| pending.clone());
|
||||
let credential = match pending.target() {
|
||||
AuthorizationMutationTarget::Create { name } => {
|
||||
PreparedAuthorizationCredential::Create(prepared_create(self.kind.clone(), name))
|
||||
}
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => {
|
||||
AuthorizationMutationTarget::Reauthorize { account_id } => {
|
||||
let mut account = account_record(self.kind.as_str());
|
||||
account.id = account_id.as_str().to_owned();
|
||||
account.credential_revision = *expected_credential_revision;
|
||||
PreparedAuthorizationCredential::Reauthorize(self.prepared_rotation(&account))
|
||||
}
|
||||
};
|
||||
Ok(PreparedAuthorizationCommit {
|
||||
pending,
|
||||
credential,
|
||||
let prepared = PreparedAuthorizationCommit::new(pending, credential);
|
||||
Ok(match retry_pending {
|
||||
Some(pending) => {
|
||||
prepared.with_authorization_guard(Box::new(RetryableAuthorizationGuard::new(
|
||||
self.events.clone(),
|
||||
Arc::clone(&self.pending),
|
||||
pending,
|
||||
)))
|
||||
}
|
||||
None => prepared,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -664,6 +698,51 @@ impl Drop for RecordingGuard {
|
||||
}
|
||||
}
|
||||
|
||||
struct RetryableAuthorizationGuard {
|
||||
events: EventLog,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
authorization: PendingAuthorizationMutation,
|
||||
}
|
||||
|
||||
impl RetryableAuthorizationGuard {
|
||||
fn new(
|
||||
events: EventLog,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
authorization: PendingAuthorizationMutation,
|
||||
) -> Self {
|
||||
Self {
|
||||
events,
|
||||
pending,
|
||||
authorization,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AuthorizationCommitGuard for RetryableAuthorizationGuard {
|
||||
async fn commit(self: Box<Self>) -> Result<(), AdminError> {
|
||||
self.events
|
||||
.lock()
|
||||
.expect("authorization guard events")
|
||||
.push("authorization_guard.commit");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn abort(self: Box<Self>) -> Result<(), AdminError> {
|
||||
let Self {
|
||||
events,
|
||||
pending,
|
||||
authorization,
|
||||
} = *self;
|
||||
*pending.lock().expect("pending authorization") = Some(authorization);
|
||||
events
|
||||
.lock()
|
||||
.expect("authorization guard events")
|
||||
.push("authorization_guard.abort");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_registry_should_resolve_custom_kind_without_central_match() {
|
||||
let provider = FakeProviderAdmin::new("custom-provider", events());
|
||||
@@ -716,12 +795,42 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
&& error_type == "invalid_request_error"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.enabled = false;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -778,6 +887,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
"provider.prepare_refresh",
|
||||
"store.commit_refresh",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"store.load_account",
|
||||
]
|
||||
);
|
||||
@@ -812,6 +922,36 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1062,7 +1202,7 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
@@ -1077,19 +1217,19 @@ impl AccountProbe for FailingAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"included usage exhausted",
|
||||
Some("usage_exhausted".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider error");
|
||||
);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
)
|
||||
.with_client_visible_upstream_error(detail))
|
||||
.with_client_visible_upstream_error(detail)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ use gateway_admin::{
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
policy::ClientApiKeyId,
|
||||
@@ -647,12 +647,13 @@ impl AccountProbe for UnavailableProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"test account probe is unavailable",
|
||||
))
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,6 +249,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
@@ -326,9 +327,19 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let invalid_kind = "x".repeat(65);
|
||||
store.replace_usage_records(vec![
|
||||
calculated_total_record("request_invalid_kind", Some(&invalid_kind), now),
|
||||
calculated_total_record("request_unregistered_kind", Some("anthropic"), now),
|
||||
calculated_total_record("request_enriched", Some("openai"), now),
|
||||
total_record(
|
||||
"request_invalid_kind",
|
||||
Some(&invalid_kind),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record(
|
||||
"request_unregistered_kind",
|
||||
Some("anthropic"),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record("request_enriched", Some("openai"), "calculated", now),
|
||||
]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
@@ -353,6 +364,30 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_usage_records(vec![total_record(
|
||||
"request_provider_reported",
|
||||
Some("openai"),
|
||||
"provider_reported",
|
||||
now,
|
||||
)]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("usage records");
|
||||
|
||||
assert!(matches!(
|
||||
page.items[0].billing,
|
||||
Some(UsageBilling::Calculated(_))
|
||||
));
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
@@ -566,9 +601,10 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
}
|
||||
}
|
||||
|
||||
fn calculated_total_record(
|
||||
fn total_record(
|
||||
id: &str,
|
||||
provider_kind: Option<&str>,
|
||||
source: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageRecord {
|
||||
UsageRecord {
|
||||
@@ -589,6 +625,7 @@ fn calculated_total_record(
|
||||
upstream_transport: None,
|
||||
http_version: None,
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
@@ -611,11 +648,11 @@ fn calculated_total_record(
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
cost_source: "calculated".to_owned(),
|
||||
cost_source: source.to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Total {
|
||||
source: "calculated".to_owned(),
|
||||
source: source.to_owned(),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
|
||||
@@ -6,7 +6,7 @@ use gateway_admin::{
|
||||
AdminServices,
|
||||
model::provider_credentials::{
|
||||
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, CredentialMutation,
|
||||
ImportCredentials, ProviderQuotaRequest, ReauthorizationTarget, StartAuthorization,
|
||||
ImportCredentials, ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
@@ -30,7 +30,11 @@ async fn openai_enable_should_use_scoped_store_without_provider_mutation() {
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.credential_details", "store.set_enabled"]
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
@@ -54,6 +58,7 @@ async fn openai_disable_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -78,6 +83,7 @@ async fn openai_delete_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.delete",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -124,6 +130,7 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
[
|
||||
"provider.prepare_import",
|
||||
"store.commit_import",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
@@ -203,6 +210,135 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_create_should_observe_initial_quota() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = service(provider.clone(), store.clone()).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai-create"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-start-openai-create"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("complete authorization");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.start_authorization",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
provider.quota_requests(),
|
||||
[ProviderQuotaRequest {
|
||||
account_id: ProviderAccountId::new("acct_prepared").expect("account ID"),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
}]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["oauth-start-openai-create"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_create_should_remain_successful_when_initial_quota_fails() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
provider.fail_next_quota(ProviderAdminErrorKind::Unavailable);
|
||||
let store = FakeAccountStore::new("openai", events);
|
||||
let services = service(provider.clone(), store).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai-quota-failure"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
let result = services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-start-openai-quota-failure"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
})
|
||||
.await
|
||||
.expect("quota observation must not fail authorization");
|
||||
|
||||
assert_eq!(result.account_id.as_str(), "acct_prepared");
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
provider.retry_authorization_after_abort();
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
store.fail_next_commit();
|
||||
let services = service(provider.clone(), store).await;
|
||||
let command = || CompleteAuthorization {
|
||||
context: context("oauth-store-retry"),
|
||||
flow_id: "flow-test".to_owned(),
|
||||
callback_url: "http://localhost/callback?code=test&state=test".to_owned(),
|
||||
};
|
||||
|
||||
services
|
||||
.openai()
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-store-retry"),
|
||||
name: "new OpenAI credential".to_owned(),
|
||||
reauthorization: None,
|
||||
})
|
||||
.await
|
||||
.expect("start authorization");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(command())
|
||||
.await
|
||||
.expect_err("Store commit must fail");
|
||||
services
|
||||
.openai()
|
||||
.complete_authorization(command())
|
||||
.await
|
||||
.expect("released claim must permit retry");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.start_authorization",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"authorization_guard.abort",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"authorization_guard.commit",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert!(provider.pending().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
let events = events();
|
||||
@@ -224,7 +360,7 @@ async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_through_commit() {
|
||||
async fn openai_reauthorization_should_commit_after_credential_revision_advances() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
@@ -234,10 +370,7 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai"),
|
||||
name: "reauthorize".to_owned(),
|
||||
reauthorization: Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
credential_revision: revision(1),
|
||||
}),
|
||||
reauthorization: Some(ProviderAccountId::new("acct_test").expect("account ID")),
|
||||
})
|
||||
.await
|
||||
.expect("start reauthorization");
|
||||
@@ -250,15 +383,14 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
);
|
||||
assert!(matches!(
|
||||
pending.target(),
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} if account_id.as_str() == "acct_test"
|
||||
&& *expected_credential_revision == revision(1)
|
||||
AuthorizationMutationTarget::Reauthorize { account_id }
|
||||
if account_id.as_str() == "acct_test"
|
||||
));
|
||||
assert!(!format!("{pending:?}").contains("admin-test"));
|
||||
|
||||
services
|
||||
provider.set_current_credential_revision(revision(2));
|
||||
|
||||
let result = services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-complete-openai"),
|
||||
@@ -268,6 +400,8 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
|
||||
assert_eq!(result.credential_revision, Some(revision(3)));
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -276,8 +410,11 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
assert_eq!(store.audit_requests(), ["oauth-complete-openai"]);
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-xai"]);
|
||||
|
||||
@@ -21,7 +21,7 @@ gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "time"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
@@ -31,7 +31,9 @@ uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1.89"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "net", "rt-multi-thread"] }
|
||||
tokio-tungstenite = "0.29.0"
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -16,7 +16,7 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use gateway_admin::model::{
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize, Revision,
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
|
||||
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
|
||||
@@ -26,8 +26,8 @@ use gateway_admin::model::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialImportResult, CredentialMutation, CredentialMutationResult, ImportCredentials,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow,
|
||||
ReauthorizationTarget, RotateCredential, StartAuthorization,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow, RotateCredential,
|
||||
StartAuthorization,
|
||||
},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -196,8 +196,6 @@ pub struct AccountView {
|
||||
pub token_refreshing: bool,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub credential_revision: u64,
|
||||
pub state_revision: Option<u64>,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
pub access_token_expires_at_display: Option<String>,
|
||||
pub refresh_token_expires_at: Option<String>,
|
||||
@@ -533,12 +531,18 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
message,
|
||||
provider_error_code,
|
||||
provider_error_type,
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
"providerErrorCode": provider_error_code,
|
||||
"providerErrorType": provider_error_type,
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
}),
|
||||
};
|
||||
@@ -604,24 +608,16 @@ pub struct StartAccountAuthorizationRequest {
|
||||
pub provider: String,
|
||||
pub name: String,
|
||||
pub account_id: Option<String>,
|
||||
pub expected_credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl StartAccountAuthorizationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_text(&self.name, MAX_NAME_BYTES, "name")?;
|
||||
match (
|
||||
self.account_id.as_deref(),
|
||||
self.expected_credential_revision,
|
||||
) {
|
||||
(None, None) => Ok(()),
|
||||
(Some(account_id), Some(credential_revision)) => {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
require_positive(credential_revision, "expectedCredentialRevision")
|
||||
}
|
||||
_ => Err(WireValidationError::new("reauthorization")),
|
||||
if let Some(account_id) = self.account_id.as_deref() {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -630,15 +626,11 @@ impl StartAccountAuthorizationRequest {
|
||||
) -> Result<(AccountProvider, StartAuthorization), WireValidationError> {
|
||||
self.validate()?;
|
||||
let provider = AccountProvider::parse(&self.provider)?;
|
||||
let reauthorization = match (self.account_id, self.expected_credential_revision) {
|
||||
(None, None) => None,
|
||||
(Some(account_id), Some(credential_revision)) => Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
credential_revision: revision(credential_revision, "expectedCredentialRevision")?,
|
||||
}),
|
||||
_ => return Err(WireValidationError::new("reauthorization")),
|
||||
};
|
||||
let reauthorization = self
|
||||
.account_id
|
||||
.map(ProviderAccountId::new)
|
||||
.transpose()
|
||||
.map_err(|_| WireValidationError::new("accountId"))?;
|
||||
Ok((
|
||||
provider,
|
||||
StartAuthorization {
|
||||
@@ -771,7 +763,6 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_credential_revision: u64,
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
}
|
||||
@@ -782,10 +773,6 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?;
|
||||
validate_oauth_material(&self.access_token, self.refresh_token.as_deref())
|
||||
}
|
||||
|
||||
@@ -806,10 +793,6 @@ impl RotateAccountRequest {
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
expected_credential_revision: revision(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?,
|
||||
provider_material: ProviderDocument::new(OpaqueProviderData::new(material)),
|
||||
})
|
||||
}
|
||||
@@ -858,15 +841,12 @@ impl From<AuthorizationStarted> for AccountAuthorizationData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountMutationData {
|
||||
pub account_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl From<CredentialMutationResult> for AccountMutationData {
|
||||
fn from(result: CredentialMutationResult) -> Self {
|
||||
Self {
|
||||
account_id: result.account_id.to_string(),
|
||||
credential_revision: result.credential_revision.map(Revision::get),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1409,8 +1389,6 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
token_refreshing: false,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
credential_revision: account.credential_revision.get(),
|
||||
state_revision: None,
|
||||
access_token_expires_at: expires_at,
|
||||
access_token_expires_at_display: account
|
||||
.access_token_expires_at
|
||||
@@ -1661,10 +1639,6 @@ fn empty_account_usage() -> AccountUsageView {
|
||||
}
|
||||
}
|
||||
|
||||
fn revision(value: u64, field: &'static str) -> Result<Revision, WireValidationError> {
|
||||
Revision::new(value).map_err(|_| WireValidationError::new(field))
|
||||
}
|
||||
|
||||
fn provider_document(
|
||||
value: Value,
|
||||
field: &'static str,
|
||||
@@ -1698,13 +1672,6 @@ fn require_text(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_positive(value: u64, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value == 0 || i64::try_from(value).is_err() {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_oauth_material(
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
|
||||
@@ -1800,7 +1800,7 @@ fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView {
|
||||
model,
|
||||
requested_model: Some(record.requested_model_id),
|
||||
upstream_model: record.upstream_model_id,
|
||||
service_tier: None,
|
||||
service_tier: record.service_tier,
|
||||
status_code,
|
||||
transport,
|
||||
attempt_index: None,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::StatusCode,
|
||||
body::Body,
|
||||
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -121,6 +122,33 @@ pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// 在下游尚未提交时优先交付 Provider 的原始 HTTP 失败响应。
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
if let EngineError::Provider(error) = error
|
||||
&& let Some(upstream) = error.client_visible_upstream_response()
|
||||
&& let Ok(status) = StatusCode::from_u16(upstream.status())
|
||||
{
|
||||
let mut response = Response::new(Body::from(upstream.body().clone()));
|
||||
*response.status_mut() = status;
|
||||
if let Some(content_type) = upstream.content_type()
|
||||
&& let Ok(content_type) = HeaderValue::from_bytes(content_type)
|
||||
{
|
||||
response.headers_mut().insert(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for header in upstream.headers() {
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
gateway_error_response(&gateway_error_from_engine(error))
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用。
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
|
||||
@@ -152,9 +152,6 @@ pub enum ResponseEncodeError {
|
||||
/// 协议原生流缺少可返回的终态 response object。
|
||||
#[error("responses wire stream has no terminal response")]
|
||||
MissingWireTerminal,
|
||||
/// 协议原生事件携带了不一致的响应身份。
|
||||
#[error("responses wire stream changed its response identity")]
|
||||
WireIdentityChanged,
|
||||
/// 终态 response object 无法序列化为完整 HTTP JSON 响应。
|
||||
#[error("responses wire terminal serialization failed")]
|
||||
Serialization,
|
||||
@@ -167,7 +164,7 @@ impl ResponseEncodeError {
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_canonical_response",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
|
||||
@@ -8,7 +8,7 @@ use axum::{
|
||||
body::{Body, Bytes},
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, StatusCode,
|
||||
HeaderMap, HeaderName, HeaderValue, StatusCode,
|
||||
header::{CACHE_CONTROL, CONTENT_TYPE, USER_AGENT},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
@@ -25,8 +25,8 @@ use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{
|
||||
gateway_error_contract, gateway_error_from_engine, gateway_error_response,
|
||||
protocol_error_response, runtime_unavailable_response,
|
||||
engine_error_response, gateway_error_contract, gateway_error_from_engine,
|
||||
gateway_error_response, protocol_error_response, runtime_unavailable_response,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -180,7 +180,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -206,7 +206,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
@@ -237,9 +237,7 @@ fn encode_collected_events(
|
||||
) -> Result<Vec<u8>, BufferedResponseEncodeError> {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
for event in events {
|
||||
encoder
|
||||
.push_sse(event)
|
||||
.map_err(BufferedResponseEncodeError::Canonical)?;
|
||||
encoder.push_sse(event);
|
||||
}
|
||||
let response = encoder
|
||||
.finish()
|
||||
@@ -253,23 +251,25 @@ fn json_body_response(encoded: Vec<u8>, response_headers: &[ProviderResponseHead
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_safe_response_headers(
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
let Some(name) = super::safe_response_header_name(header.name()) else {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_str(header.value().as_str()) else {
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::HeaderName::from_static(name), value);
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -295,7 +295,7 @@ pub async fn stream_execution_response(
|
||||
return response;
|
||||
}
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -310,18 +310,7 @@ pub async fn stream_execution_response(
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let mut frames = Vec::new();
|
||||
for event in &first_events {
|
||||
match encoder.push_sse(event) {
|
||||
Ok(encoded) => frames.extend(encoded),
|
||||
Err(error) => {
|
||||
let response = protocol_error_response(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
);
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
frames.extend(encoder.push_sse(event));
|
||||
}
|
||||
if frames.is_empty() {
|
||||
let response = internal_gateway_response("gateway commit batch encoded no output");
|
||||
@@ -338,7 +327,7 @@ pub async fn stream_execution_response(
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
let Some(session) = execution.into_session() else {
|
||||
@@ -371,7 +360,7 @@ fn event_stream_response(body: Body, response_headers: &[ProviderResponseHeader]
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CACHE_CONTROL, HeaderValue::from_static("no-cache"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn internal_gateway_response(message: &'static str) -> Response {
|
||||
@@ -403,15 +392,10 @@ impl PendingExecution {
|
||||
}
|
||||
|
||||
async fn record_response_status(&mut self, response: Response) -> Response {
|
||||
let Some(session) = self.session.as_mut() else {
|
||||
return internal_gateway_response("gateway response session is unavailable");
|
||||
};
|
||||
if session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return internal_gateway_response("gateway client status could not be persisted");
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
let _ = session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await;
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -499,10 +483,16 @@ impl ResponsesStreamState {
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.is_finalized());
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
));
|
||||
if self.execution_terminal {
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
} else {
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without finalizing the execution",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.execution_terminal = self
|
||||
@@ -521,18 +511,11 @@ impl ResponsesStreamState {
|
||||
}
|
||||
|
||||
async fn push_event(&mut self, event: ProviderEvent) {
|
||||
match self.encoder.push_sse(&event) {
|
||||
Ok(frames) => {
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.cancel_execution().await;
|
||||
self.finish_with_encode_error(error);
|
||||
}
|
||||
let frames = self.encoder.push_sse(&event);
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,20 +590,6 @@ impl ResponsesStreamState {
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
|
||||
fn finish_with_encode_error(&mut self, error: ResponseEncodeError) {
|
||||
let protocol = error.protocol_body().error;
|
||||
self.pending
|
||||
.push_back(Bytes::from(response_failed_sse_event_with_id(
|
||||
self.encoder.response_id(),
|
||||
protocol.kind,
|
||||
protocol.code,
|
||||
&protocol.message,
|
||||
)));
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponsesStreamState {
|
||||
|
||||
@@ -17,13 +17,63 @@ pub use response::OpenAiResponsesEncoder;
|
||||
pub(crate) use websocket::responses_websocket;
|
||||
pub use websocket::{ResponseCreateFrameError, decode_response_create_with_context};
|
||||
|
||||
pub(super) fn safe_response_header_name(name: &str) -> Option<&'static str> {
|
||||
match name {
|
||||
"x-request-id" => Some("x-request-id"),
|
||||
"openai-model" => Some("openai-model"),
|
||||
"x-models-etag" => Some("x-models-etag"),
|
||||
"x-reasoning-included" => Some("x-reasoning-included"),
|
||||
"openai-processing-ms" => Some("openai-processing-ms"),
|
||||
_ => None,
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
|
||||
pub(super) fn response_connection_options(headers: &[ProviderResponseHeader]) -> Vec<String> {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|header| header.name().trim().eq_ignore_ascii_case("connection"))
|
||||
.filter_map(|header| std::str::from_utf8(header.value()).ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn response_header_is_forwardable(name: &str, connection_options: &[String]) -> bool {
|
||||
let name = name.trim().to_ascii_lowercase();
|
||||
if connection_options
|
||||
.iter()
|
||||
.any(|option| option.eq_ignore_ascii_case(&name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name.as_str(),
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "content-length"
|
||||
| "content-type"
|
||||
| "content-encoding"
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "www-authenticate"
|
||||
| "authentication-info"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "proxy-authentication-info"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "set-cookie"
|
||||
| "set-cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
| "x-codex-installation-id"
|
||||
| "x-codex-turn-state"
|
||||
| "x-codex-turn-metadata"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::{fmt, net::IpAddr};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload, ProviderSessionState};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
@@ -14,6 +15,7 @@ use super::error::RequestDecodeError;
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OpenAiRequestHeaders {
|
||||
@@ -34,6 +36,7 @@ pub struct OpenAiRequestHeaders {
|
||||
responses_lite: Option<String>,
|
||||
memgen_request: Option<String>,
|
||||
subagent: Option<String>,
|
||||
passthrough_headers: Vec<Value>,
|
||||
}
|
||||
|
||||
impl OpenAiRequestHeaders {
|
||||
@@ -57,12 +60,8 @@ impl OpenAiRequestHeaders {
|
||||
turn_id: header_string(headers, "x-codex-turn-id"),
|
||||
responses_lite: header_string(headers, X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER),
|
||||
memgen_request: header_string(headers, X_OPENAI_MEMGEN_REQUEST_HEADER),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"review" | "compact" | "memory_consolidation" | "collab_spawn"
|
||||
)
|
||||
}),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY),
|
||||
passthrough_headers: passthrough_headers(headers),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +107,12 @@ impl OpenAiRequestHeaders {
|
||||
insert_protocol_context(&mut context, "turn_id", self.turn_id.as_ref());
|
||||
insert_protocol_context(&mut context, "responses_lite", self.responses_lite.as_ref());
|
||||
insert_protocol_context(&mut context, "memgen_request", self.memgen_request.as_ref());
|
||||
if !self.passthrough_headers.is_empty() {
|
||||
context.insert(
|
||||
PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
Value::Array(self.passthrough_headers.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(use_websocket) = use_websocket {
|
||||
context.insert("use_websocket".to_owned(), Value::Bool(use_websocket));
|
||||
}
|
||||
@@ -322,13 +327,14 @@ pub(super) fn decode_request_object(
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// `use_websocket` 只影响本地 transport。无论值是否可识别,都不得进入上游 body。
|
||||
let use_websocket = object
|
||||
.remove("use_websocket")
|
||||
.and_then(|value| value.as_bool());
|
||||
// 仅消费已识别的本地 transport 开关;未知同名值保留给未来上游协议。
|
||||
let use_websocket = object.get("use_websocket").and_then(Value::as_bool);
|
||||
if use_websocket.is_some() {
|
||||
object.remove("use_websocket");
|
||||
}
|
||||
|
||||
let model = required_non_empty_string(&object, "model", "model")?.trim();
|
||||
if model.is_empty() {
|
||||
let model = required_non_empty_string(&object, "model", "model")?;
|
||||
if model.trim().is_empty() {
|
||||
return Err(RequestDecodeError::EmptyField {
|
||||
field: "model".to_owned(),
|
||||
});
|
||||
@@ -345,7 +351,6 @@ pub(super) fn decode_request_object(
|
||||
let continuation = object
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|response_id| !response_id.is_empty())
|
||||
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
@@ -376,26 +381,98 @@ fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
.entry("client_metadata".to_owned())
|
||||
.or_insert_with(|| Value::Object(Map::new()));
|
||||
if !metadata.is_object() {
|
||||
*metadata = Value::Object(Map::new());
|
||||
}
|
||||
if let Some(metadata) = metadata.as_object_mut() {
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
let Some(metadata) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn passthrough_headers(headers: &HeaderMap) -> Vec<Value> {
|
||||
let connection_headers = headers
|
||||
.get_all("connection")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| passthrough_header_name(name.as_str(), &connection_headers))
|
||||
.map(|(name, value)| {
|
||||
Value::Array(vec![
|
||||
Value::String(name.as_str().to_owned()),
|
||||
Value::String(STANDARD.encode(value.as_bytes())),
|
||||
])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
if connection_headers
|
||||
.iter()
|
||||
.any(|connection_header| connection_header.eq_ignore_ascii_case(name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
|| name.starts_with("x-grok-")
|
||||
|| name.starts_with("x-xai-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name,
|
||||
// 逐跳头、上游 authority 和重序列化后的实体长度由 transport 重建。
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "host"
|
||||
| "content-length"
|
||||
// 下游连接诊断事实只用于本地观测,不能伪装成 OpenAI 请求事实。
|
||||
| "forwarded"
|
||||
| "x-forwarded-for"
|
||||
| "x-forwarded-host"
|
||||
| "x-forwarded-proto"
|
||||
| "x-forwarded-port"
|
||||
| "x-real-ip"
|
||||
| "true-client-ip"
|
||||
| "cf-connecting-ip"
|
||||
| "x-request-id"
|
||||
// 下游鉴权和账号 cookie 绝不能成为上游账号身份。
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
)
|
||||
}
|
||||
|
||||
fn required_non_empty_string<'a>(
|
||||
object: &'a Map<String, Value>,
|
||||
key: &str,
|
||||
|
||||
@@ -11,15 +11,14 @@ const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
/// OpenAI Responses wire 的转发器。
|
||||
///
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。这里仅保留同一
|
||||
/// 响应的身份和终态校验,不再从 canonical facts 重建客户端 JSON;因此
|
||||
/// OpenAI 上游的未知字段、字段顺序和事件语义不会被 API 层二次解释。
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。wire 是客户端交付
|
||||
/// 与终态判断的事实来源;canonical facts 只在 wire 暂未携带 ID 时提供旁路观测,
|
||||
/// 不得反过来拒绝或改写上游事件。
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OpenAiResponsesEncoder {
|
||||
response_id: Option<String>,
|
||||
wire_terminal: Option<Value>,
|
||||
wire_failure: bool,
|
||||
canonical_completed: bool,
|
||||
}
|
||||
|
||||
impl OpenAiResponsesEncoder {
|
||||
@@ -30,53 +29,41 @@ impl OpenAiResponsesEncoder {
|
||||
response_id: None,
|
||||
wire_terminal: None,
|
||||
wire_failure: false,
|
||||
canonical_completed: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 SSE frames。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Result<Vec<Bytes>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Vec<Bytes> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
self.observe_wire(wire);
|
||||
if let Some(raw_sse_frame) = wire.raw_sse_frame() {
|
||||
return Ok(vec![raw_sse_frame.clone()]);
|
||||
return vec![raw_sse_frame.clone()];
|
||||
}
|
||||
Ok(vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
wire.event_type().unwrap_or_default(),
|
||||
&wire.data().to_string(),
|
||||
wire.sse_id(),
|
||||
wire.sse_retry(),
|
||||
))])
|
||||
))]
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 WebSocket JSON messages。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_websocket(
|
||||
&mut self,
|
||||
event: &ProviderEvent,
|
||||
) -> Result<Vec<String>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_websocket(&mut self, event: &ProviderEvent) -> Vec<String> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event).filter(|wire| wire.has_json_data()) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
Ok(vec![wire.data().to_string()])
|
||||
self.observe_wire(wire);
|
||||
vec![wire.data().to_string()]
|
||||
}
|
||||
|
||||
/// 返回是否已经看到同一响应的 canonical 与 wire 终态。
|
||||
/// 返回是否已经看到客户端可见的 wire 终态。
|
||||
#[must_use]
|
||||
pub fn is_completed(&self) -> bool {
|
||||
self.wire_terminal.is_some() && self.canonical_completed
|
||||
self.wire_terminal.is_some()
|
||||
}
|
||||
|
||||
/// 返回是否已把 Provider 原生失败 event 交付给客户端。
|
||||
@@ -95,39 +82,31 @@ impl OpenAiResponsesEncoder {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 缺少 wire 或 canonical 终态时返回错误。
|
||||
/// 缺少可转换为非流式响应的 wire 终态时返回错误。
|
||||
pub fn finish(self) -> Result<Value, ResponseEncodeError> {
|
||||
if self.canonical_completed {
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
} else {
|
||||
Err(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
|
||||
fn observe_identity(&mut self, event: &ProviderEvent) -> Result<(), ResponseEncodeError> {
|
||||
fn observe_canonical_identity(&mut self, event: &ProviderEvent) {
|
||||
for fact in event.canonical_facts() {
|
||||
let metadata = match fact {
|
||||
GatewayEvent::Started(metadata) => metadata,
|
||||
GatewayEvent::Completed(metadata) => {
|
||||
self.canonical_completed = true;
|
||||
metadata
|
||||
}
|
||||
GatewayEvent::Started(metadata) | GatewayEvent::Completed(metadata) => metadata,
|
||||
_ => continue,
|
||||
};
|
||||
if self
|
||||
.response_id
|
||||
.as_deref()
|
||||
.is_some_and(|current| current != metadata.response_id())
|
||||
{
|
||||
return Err(ResponseEncodeError::WireIdentityChanged);
|
||||
}
|
||||
self.response_id = Some(metadata.response_id().to_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_wire_terminal(&mut self, wire: &ProtocolWireEvent) {
|
||||
fn observe_wire(&mut self, wire: &ProtocolWireEvent) {
|
||||
if let Some(response_id) = wire
|
||||
.data()
|
||||
.pointer("/response/id")
|
||||
.or_else(|| wire.data().get("response_id"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
self.response_id = Some(response_id.to_owned());
|
||||
}
|
||||
let effective_type = wire
|
||||
.event_type()
|
||||
.or_else(|| wire.data().get("type").and_then(Value::as_str));
|
||||
|
||||
@@ -9,7 +9,7 @@ use axum::{
|
||||
connect_info::ConnectInfo,
|
||||
ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade, close_code},
|
||||
},
|
||||
http::{HeaderMap, StatusCode},
|
||||
http::{HeaderMap, HeaderName, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::StreamExt;
|
||||
@@ -27,7 +27,6 @@ use crate::{
|
||||
openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{gateway_error_contract, gateway_error_from_engine, runtime_unavailable_response},
|
||||
router::MAX_CLIENT_REQUEST_BODY_BYTES,
|
||||
service::OpenAiService,
|
||||
},
|
||||
};
|
||||
@@ -102,8 +101,10 @@ impl ResponsesWebSocketAdapter {
|
||||
_connection_guard: connection_guard,
|
||||
};
|
||||
websocket
|
||||
.max_message_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
.max_frame_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
// 覆盖 axum/tungstenite 的私有 64 MiB message 与 16 MiB frame 默认值。
|
||||
// Responses JSON 的协议可接受性由上游决定,代理不另设 wire 长度上限。
|
||||
.max_message_size(usize::MAX)
|
||||
.max_frame_size(usize::MAX)
|
||||
.on_upgrade(move |socket| async move {
|
||||
serve_responses_websocket(socket, session).await;
|
||||
})
|
||||
@@ -303,18 +304,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(messages) => first_messages.extend(messages),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
first_messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if first_messages.is_empty() {
|
||||
let error = GatewayError::new(
|
||||
@@ -336,9 +326,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) = commit_connection_replay(replay, &encoder, provider_state.take()) {
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_text(
|
||||
socket,
|
||||
response_metadata_event(&request_id, &response_headers),
|
||||
@@ -378,18 +366,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(encoded) => messages.extend(encoded),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if encoder.is_completed() {
|
||||
if let Err(outcome) =
|
||||
@@ -397,11 +374,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) =
|
||||
commit_connection_replay(replay, &encoder, provider_state.take())
|
||||
{
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_messages(socket, messages).await {
|
||||
return ForwardOutcome::Disconnect;
|
||||
}
|
||||
@@ -413,9 +386,17 @@ async fn forward_execution(
|
||||
}
|
||||
}
|
||||
ActiveInput::Event(Ok(None)) => {
|
||||
if execution
|
||||
.session_mut()
|
||||
.is_some_and(|session| session.is_finalized())
|
||||
{
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
execution.disarm();
|
||||
return ForwardOutcome::Continue;
|
||||
}
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
"gateway response ended without finalizing the execution",
|
||||
);
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
@@ -441,15 +422,10 @@ fn commit_connection_replay(
|
||||
replay: &mut ConnectionReplaySnapshot,
|
||||
encoder: &OpenAiResponsesEncoder,
|
||||
provider_state: Option<ProviderSessionState>,
|
||||
) -> Result<(), GatewayError> {
|
||||
let response_id = encoder.response_id().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway completed a response without an identity",
|
||||
)
|
||||
})?;
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
Ok(())
|
||||
) {
|
||||
if let Some(response_id) = encoder.response_id() {
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
}
|
||||
}
|
||||
|
||||
async fn confirm_completed_execution(
|
||||
@@ -595,21 +571,24 @@ fn response_metadata_event(
|
||||
request_id: &str,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> String {
|
||||
let mut headers = response_headers
|
||||
.iter()
|
||||
.filter_map(|header| {
|
||||
super::safe_response_header_name(header.name()).map(|name| {
|
||||
(
|
||||
name.to_owned(),
|
||||
Value::String(header.value().as_str().to_owned()),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Map<String, Value>>();
|
||||
headers.insert(
|
||||
"x-request-id".to_owned(),
|
||||
Value::String(request_id.to_owned()),
|
||||
);
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
let mut headers = Map::new();
|
||||
for header in response_headers {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = std::str::from_utf8(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
// 官方 response.metadata 使用 string map,无法表达同名多值;后值保持既有覆盖语义。
|
||||
headers.insert(name.as_str().to_owned(), Value::String(value.to_owned()));
|
||||
}
|
||||
headers
|
||||
.entry("x-request-id".to_owned())
|
||||
.or_insert_with(|| Value::String(request_id.to_owned()));
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": headers,
|
||||
|
||||
@@ -13,9 +13,6 @@ use super::{
|
||||
|
||||
use crate::ApiState;
|
||||
|
||||
/// OpenAI 请求正文上限。
|
||||
pub const MAX_CLIENT_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// 构造 OpenAI 客户端协议路由。
|
||||
pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
@@ -26,5 +23,7 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
// Responses 正文属于 OpenAI/Codex 协议;代理不能用私有大小上限提前拒绝
|
||||
// 上游本可接受的未来 payload。
|
||||
.layer(DefaultBodyLimit::disable())
|
||||
}
|
||||
|
||||
@@ -81,16 +81,10 @@ impl OpenAiService {
|
||||
"requested model was not found",
|
||||
)
|
||||
})?;
|
||||
if !self.execution.contains_public_model(&client, &public_model) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let previous_response_id = match metadata.continuation() {
|
||||
ContinuationIntent::None => None,
|
||||
ContinuationIntent::PreviousResponseId(value) => {
|
||||
PreviousResponseId::new(value.clone()).ok()
|
||||
Some(PreviousResponseId::new(value.clone()))
|
||||
}
|
||||
};
|
||||
self.execution
|
||||
|
||||
@@ -105,12 +105,15 @@ mod actions {
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
|
||||
provider_credentials::{CredentialDeletionResult, CredentialImportResult},
|
||||
provider_credentials::{
|
||||
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
|
||||
},
|
||||
};
|
||||
use gateway_api::admin::accounts::{
|
||||
AccountActionRequest, AccountConnectionTestEvent, AccountDeletionData,
|
||||
AccountDeletionRequest, AccountExportData, AccountExportQuery, AccountIdQuery,
|
||||
AccountImportData, AccountImportRequest, AccountRefreshRequest, AccountTestQuery,
|
||||
AccountImportData, AccountImportRequest, AccountMutationData, AccountRefreshRequest,
|
||||
AccountTestQuery, RotateAccountRequest, StartAccountAuthorizationRequest,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use serde_json::json;
|
||||
@@ -168,6 +171,58 @@ mod actions {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1"
|
||||
}))
|
||||
.expect("decode reauthorization");
|
||||
assert!(authorization.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<StartAccountAuthorizationRequest>(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let rotation: RotateAccountRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"accessToken": "header.payload.signature",
|
||||
"refreshToken": "refresh-token"
|
||||
}))
|
||||
.expect("decode rotation");
|
||||
assert!(rotation.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<RotateAccountRequest>(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1,
|
||||
"accessToken": "header.payload.signature"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_mutation_response_should_not_expose_internal_revision() {
|
||||
let response = AccountMutationData::from(CredentialMutationResult {
|
||||
config_revision: Revision::new(8).expect("config revision"),
|
||||
account_id: ProviderAccountId::new("acct_1").expect("account ID"),
|
||||
credential_revision: Some(Revision::new(9).expect("credential revision")),
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize credential mutation"),
|
||||
json!({ "accountId": "acct_1" })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_import_should_use_provider_and_opaque_data_fields() {
|
||||
let valid: AccountImportRequest = serde_json::from_value(json!({
|
||||
@@ -282,6 +337,9 @@ mod actions {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
provider_error_type: Some("invalid_request_error".to_owned()),
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
},
|
||||
]
|
||||
@@ -313,6 +371,9 @@ mod actions {
|
||||
"error": "upstream unavailable",
|
||||
"providerErrorCode": "usage_exhausted",
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "active"
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -58,9 +58,8 @@ use gateway_api::admin::AdminSessionState;
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::{
|
||||
ConfigRevision, ProviderKind, PublicModelId, UpstreamModelId, snapshot::SnapshotControl,
|
||||
@@ -675,7 +674,7 @@ impl AccountProbe for NoopProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async { panic!("unused account probe") })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,6 +326,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
@@ -402,6 +403,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
@@ -428,6 +430,7 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
|
||||
@@ -141,8 +141,7 @@ fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
"no account is eligible for the requested model",
|
||||
Some("no_eligible_account".to_owned()),
|
||||
Some("account_unavailable_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider detail");
|
||||
);
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
@@ -327,14 +326,11 @@ fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
async fn gateway_error_response_should_expose_only_structured_client_visible_upstream_fields() {
|
||||
let error = GatewayError::from_provider(
|
||||
&ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
),
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)),
|
||||
);
|
||||
|
||||
let response = gateway_error_response(&error);
|
||||
|
||||
@@ -8,7 +8,10 @@ use std::sync::{
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
extract::connect_info::ConnectInfo,
|
||||
http::{HeaderMap, Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, Request, StatusCode,
|
||||
header::{AUTHORIZATION, CONTENT_TYPE},
|
||||
},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
@@ -18,8 +21,8 @@ use gateway_core::engine::execution::{
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
SafeUpstreamValue,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind,
|
||||
ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -101,6 +104,8 @@ struct ContextCaptureExecution {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedClientContext {
|
||||
public_model: String,
|
||||
wire_model: Option<String>,
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
endpoint: String,
|
||||
@@ -109,6 +114,7 @@ struct CapturedClientContext {
|
||||
client_metadata: Option<Value>,
|
||||
protocol_context: Option<Value>,
|
||||
prompt_cache_key: Option<String>,
|
||||
previous_response_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ExecutionService for ContextCaptureExecution {
|
||||
@@ -124,11 +130,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
["model-a", "model-b"]
|
||||
.into_iter()
|
||||
.map(|model| PublicModelId::new(model).expect("catalog model"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
true
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
matches!(model.as_str(), "model-a" | "model-b")
|
||||
}
|
||||
|
||||
fn start(
|
||||
@@ -136,11 +145,16 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let public_model = request.public_model.as_str().to_owned();
|
||||
let operation_kind = request.operation.kind();
|
||||
let generation = match &request.operation {
|
||||
Operation::Generate(generation) => Some(generation),
|
||||
_ => None,
|
||||
};
|
||||
let wire_model = generation
|
||||
.and_then(|generation| generation.protocol_payload().body().get("model"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned);
|
||||
let (client_metadata, protocol_context, prompt_cache_key, input) =
|
||||
generation.map_or((None, None, None, None), |generation| {
|
||||
let payload = generation.protocol_payload();
|
||||
@@ -152,7 +166,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
payload.body().get("input").cloned(),
|
||||
)
|
||||
});
|
||||
let previous_response_id = request
|
||||
.metadata
|
||||
.previous_response_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str().to_owned());
|
||||
*self.observed.lock().expect("context capture lock") = Some(CapturedClientContext {
|
||||
public_model,
|
||||
wire_model,
|
||||
client_ip: request.metadata.client_ip,
|
||||
user_agent: request.metadata.user_agent,
|
||||
endpoint: request.metadata.endpoint,
|
||||
@@ -161,6 +182,7 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
client_metadata,
|
||||
protocol_context,
|
||||
prompt_cache_key,
|
||||
previous_response_id,
|
||||
});
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -503,14 +525,19 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
assert_eq!(
|
||||
captured_client_context(headers, peer).await,
|
||||
CapturedClientContext {
|
||||
public_model: "smart-code".to_owned(),
|
||||
wire_model: Some("smart-code".to_owned()),
|
||||
client_ip: Some("198.51.100.1".parse().expect("expected IP")),
|
||||
user_agent: Some("Codex-CLI/1.0".to_owned()),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
operation_kind: OperationKind::Generate,
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
protocol_context: None,
|
||||
protocol_context: Some(json!({
|
||||
"opaque_request_headers": [["user-agent", "IENvZGV4LUNMSS8xLjAg"]]
|
||||
})),
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -533,14 +560,58 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_headers() {
|
||||
async fn stale_model_catalog_should_not_block_a_new_model_request() {
|
||||
let model = " gpt-future-codex ";
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({"model": model, "input": "hello"}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.public_model, model);
|
||||
assert_eq!(captured.wire_model.as_deref(), Some(model));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_pass_opaque_previous_response_id_to_core() {
|
||||
for response_id in [
|
||||
format!("resp_{}", "x".repeat(257)),
|
||||
"resp_control\0opaque".to_owned(),
|
||||
String::new(),
|
||||
] {
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
captured.previous_response_id.as_deref(),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_codex_headers_without_projecting_xai_headers() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-codex-turn-state", "turn-state".parse().expect("header"));
|
||||
headers.insert("conversation-id", "conversation-1".parse().expect("header"));
|
||||
headers.insert("session-id", "session-1".parse().expect("header"));
|
||||
headers.insert("x-grok-turn-idx", "7".parse().expect("header"));
|
||||
headers.insert("x-openai-subagent", "compact".parse().expect("header"));
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-responses-lite",
|
||||
"true".parse().expect("header"),
|
||||
@@ -566,10 +637,33 @@ async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_h
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent")),
|
||||
Some(&json!("compact"))
|
||||
Some(&json!("future_codex_mode"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_header_should_not_replace_non_object_client_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "preserve metadata",
|
||||
"client_metadata": "opaque-client-value"
|
||||
}),
|
||||
headers,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.client_metadata, Some(json!("opaque-client-value")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
@@ -632,6 +726,43 @@ async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_not_replace_non_object_client_metadata() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": ["opaque", "client", "value"]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(
|
||||
captured.client_metadata,
|
||||
Some(json!(["opaque", "client", "value"]))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -653,19 +784,22 @@ async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresentable_or_sensitive_ones()
|
||||
{
|
||||
let trace = Arc::new(Trace::default());
|
||||
let headers = vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-models-etag",
|
||||
SafeUpstreamValue::new("models-v2").expect("etag"),
|
||||
)
|
||||
.expect("safe header"),
|
||||
ProviderResponseHeader::new("x-models-etag", Bytes::from_static(b"models-v2")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
SafeUpstreamValue::new("should-not-cross-boundary").expect("header value"),
|
||||
)
|
||||
.expect("syntactically safe header"),
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-hop-secret")),
|
||||
ProviderResponseHeader::new("x-hop-secret", Bytes::from_static(b"hop-secret")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
];
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -685,7 +819,29 @@ async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("models-v2")
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-multi")
|
||||
.iter()
|
||||
.map(HeaderValue::as_bytes)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(HeaderValue::as_bytes),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-hop-secret").is_none());
|
||||
assert_eq!(
|
||||
response.headers().get(CONTENT_TYPE),
|
||||
Some(&HeaderValue::from_static("text/event-stream"))
|
||||
);
|
||||
std::mem::forget(response);
|
||||
}
|
||||
|
||||
@@ -801,7 +957,107 @@ async fn streaming_rate_limit_before_first_frame_should_persist_the_returned_429
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
async fn streaming_upstream_http_failure_before_first_frame_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(
|
||||
b"{\"error\":{\"message\":\"rate limited\",\"future_field\":{\"kept\":true}},\"top_level\":17}\x00",
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
429,
|
||||
Some(b"application/problem+json; charset=utf-8".to_vec()),
|
||||
raw_body.clone(),
|
||||
)
|
||||
.with_headers(vec![
|
||||
ProviderResponseHeader::new("retry-after", Bytes::from_static(b"17")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req-failure")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
]),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![NextStep::Error(EngineError::Provider(error))],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"application/problem+json; charset=utf-8".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("retry-after")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"17".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-request-id")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"req-failure".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-error")
|
||||
.iter()
|
||||
.map(|value| value.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![429]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_upstream_http_failure_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(b"not-json\xffstill-upstream");
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/octet-stream".to_vec()),
|
||||
raw_body.clone(),
|
||||
));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_collect_error(EngineError::Provider(error));
|
||||
|
||||
let response = collect_execution_response(Box::new(session)).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![503]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_canonical_identity_change_does_not_interrupt_wire_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -811,7 +1067,7 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
mismatched_terminal_event(),
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)),
|
||||
NextStep::FinalizeCancelled,
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
@@ -820,12 +1076,16 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
|
||||
assert!(String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
assert!(body.contains("response.completed"));
|
||||
assert!(body.contains("resp_other"));
|
||||
assert!(!body.contains("response.failed"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.client_statuses(), vec![200]);
|
||||
assert!(trace.is_cancelled());
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(
|
||||
trace.snapshot(),
|
||||
vec!["next_event", "commit", "next_event", "cancel_finalize"]
|
||||
vec!["next_event", "commit", "next_event", "next_end"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -874,6 +1134,38 @@ async fn streaming_success_should_emit_terminal_event_and_done_marker() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_finalized_unknown_wire_should_end_cleanly_without_synthetic_failure() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw = Bytes::from_static(
|
||||
b"event: response.future_terminal\ndata: {\"type\":\"response.future_terminal\",\"opaque\":true}\n\n",
|
||||
);
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_sse("openai", raw.clone()).expect("raw future event"),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(delivery_provider(
|
||||
event,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read transparent SSE body");
|
||||
|
||||
assert!(body.as_ref().starts_with(raw.as_ref()));
|
||||
assert!(body.as_ref().ends_with(b"data: [DONE]\n\n"));
|
||||
assert!(!String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_end"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_completed_event_without_finalized_execution_should_fail_closed() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -942,14 +1234,11 @@ async fn streaming_empty_terminal_should_emit_failure_done_and_cancel_execution(
|
||||
async fn streaming_error_should_emit_client_visible_upstream_details() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
);
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
));
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
@@ -1055,6 +1344,86 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atomic_uncommitted_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_created = Bytes::from_static(
|
||||
b"event: response.created\r\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_atomic_api\",\"model\":\"public-model\",\"status\":\"in_progress\"}}\r\n\r\n",
|
||||
);
|
||||
let raw_failure = Bytes::from_static(
|
||||
b"event: response.failed\r\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_atomic_api\",\"status\":\"failed\",\"error\":{\"code\":\"rate_limit_exceeded\",\"message\":\"atomic upstream marker\"}}}\r\n\r\n",
|
||||
);
|
||||
let started_wire = ProviderEvent::canonical_with_wire(
|
||||
vec![started()],
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"model": "public-model",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
raw_created.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream started wire"),
|
||||
);
|
||||
let failed_wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
raw_failure.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream failure wire"),
|
||||
);
|
||||
let batch = CoordinatedEvent::try_batch(
|
||||
vec![started_wire, failed_wire],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic failure delivery");
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(batch),
|
||||
NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
))),
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("SSE is UTF-8");
|
||||
|
||||
assert_eq!(body.matches("atomic upstream marker").count(), 1);
|
||||
assert_eq!(body.matches("event: response.failed").count(), 1);
|
||||
assert!(body.contains(std::str::from_utf8(&raw_created).expect("created frame")));
|
||||
assert!(body.contains(std::str::from_utf8(&raw_failure).expect("failure frame")));
|
||||
assert!(!body.contains("upstream capacity is temporarily unavailable"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_error"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_second_commit_request_should_fail_and_finalize_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -1132,13 +1501,10 @@ async fn buffered_response_commits_only_after_complete_json_is_encoded() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
async fn buffered_response_forwards_long_ordinary_provider_header_values() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let header = ProviderResponseHeader::new(
|
||||
"openai-model",
|
||||
SafeUpstreamValue::new("gpt-5.5-effective").expect("model header"),
|
||||
)
|
||||
.expect("safe header");
|
||||
let model = format!("gpt-{}", "x".repeat(512));
|
||||
let header = ProviderResponseHeader::new("openai-model", Bytes::from(model.clone()));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_response_headers(vec![header]);
|
||||
|
||||
@@ -1149,7 +1515,7 @@ async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
.headers()
|
||||
.get("openai-model")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("gpt-5.5-effective")
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ mod http;
|
||||
mod websocket;
|
||||
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_api::openai::responses::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, OpenAiResponsesEncoder,
|
||||
@@ -117,6 +118,28 @@ fn decoder_should_preserve_the_openai_body_and_only_derive_stable_routing_facts(
|
||||
assert!(decoded.metadata().store());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_client_model_values() {
|
||||
for model in [
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
"future\0model".to_owned(),
|
||||
" future-model ".to_owned(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": model.clone(),
|
||||
"input": "hello"
|
||||
}));
|
||||
|
||||
assert_eq!(decoded.metadata().public_model(), model);
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("model")
|
||||
.and_then(Value::as_str),
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_compaction_trigger_for_the_openai_provider() {
|
||||
let decoded = generate_request(json!({
|
||||
@@ -149,6 +172,21 @@ fn decoder_should_keep_transport_override_out_of_the_openai_wire_body() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unrecognized_use_websocket_values() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"use_websocket": {"future": "transport-mode"}
|
||||
}));
|
||||
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded).get("use_websocket"),
|
||||
Some(&json!({"future": "transport-mode"}))
|
||||
);
|
||||
assert!(!openai_protocol_context(&decoded).contains_key("use_websocket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -170,10 +208,142 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
("turn_state".to_owned(), json!("turn-state")),
|
||||
("turn_metadata".to_owned(), json!("{\"kind\":\"review\"}")),
|
||||
("conversation_id".to_owned(), json!("conversation")),
|
||||
(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([
|
||||
["x-codex-turn-state", STANDARD.encode(b"turn-state")],
|
||||
[
|
||||
"x-codex-turn-metadata",
|
||||
STANDARD.encode(br#"{"kind":"review"}"#)
|
||||
],
|
||||
["conversation-id", STANDARD.encode(b"conversation")]
|
||||
]),
|
||||
),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_static("future-ascii"),
|
||||
);
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_bytes(b"\x80\xff").expect("opaque header bytes"),
|
||||
);
|
||||
headers.append("version", HeaderValue::from_static("future-v1"));
|
||||
headers.append(
|
||||
"version",
|
||||
HeaderValue::from_bytes(b"future-\x80").expect("opaque known header bytes"),
|
||||
);
|
||||
headers.insert(
|
||||
"accept",
|
||||
HeaderValue::from_static("application/vnd.openai.responses+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"content-type",
|
||||
HeaderValue::from_static("application/vnd.openai.responses.request+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_static("Codex future-client"),
|
||||
);
|
||||
headers.insert("originator", HeaderValue::from_static("future-codex"));
|
||||
headers.append(
|
||||
"openai-beta",
|
||||
HeaderValue::from_static("future_responses=v2"),
|
||||
);
|
||||
headers.append("openai-beta", HeaderValue::from_static("future_tools=v3"));
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-residency",
|
||||
HeaderValue::from_static("future-region"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-turn-state",
|
||||
HeaderValue::from_static("same-account-turn"),
|
||||
);
|
||||
|
||||
headers.insert(
|
||||
"connection",
|
||||
HeaderValue::from_static("keep-alive, x-private-hop"),
|
||||
);
|
||||
headers.insert("x-private-hop", HeaderValue::from_static("drop-me"));
|
||||
headers.insert("host", HeaderValue::from_static("downstream.invalid"));
|
||||
headers.insert("content-length", HeaderValue::from_static("999"));
|
||||
headers.insert("authorization", HeaderValue::from_static("Bearer client"));
|
||||
headers.insert("x-api-key", HeaderValue::from_static("client-key"));
|
||||
headers.insert("cookie", HeaderValue::from_static("client=cookie"));
|
||||
headers.insert(
|
||||
"chatgpt-account-id",
|
||||
HeaderValue::from_static("client-account"),
|
||||
);
|
||||
headers.insert(
|
||||
"chatgpt-project-id",
|
||||
HeaderValue::from_static("client-project"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-installation-id",
|
||||
HeaderValue::from_static("client-installation"),
|
||||
);
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
.expect("opaque request headers should not affect body decoding");
|
||||
let entries = openai_protocol_context(&decoded)
|
||||
.get("opaque_request_headers")
|
||||
.and_then(Value::as_array)
|
||||
.expect("opaque request header context");
|
||||
let values = |target: &str| {
|
||||
entries
|
||||
.iter()
|
||||
.filter_map(Value::as_array)
|
||||
.filter(|entry| entry.first().and_then(Value::as_str) == Some(target))
|
||||
.filter_map(|entry| entry.get(1).and_then(Value::as_str))
|
||||
.map(|encoded| STANDARD.decode(encoded).expect("valid base64 header value"))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
values("x-openai-future-mode"),
|
||||
vec![b"future-ascii".to_vec(), b"\x80\xff".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("version"),
|
||||
vec![b"future-v1".to_vec(), b"future-\x80".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("openai-beta"),
|
||||
vec![b"future_responses=v2".to_vec(), b"future_tools=v3".to_vec()]
|
||||
);
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
"user-agent",
|
||||
"originator",
|
||||
"x-openai-internal-codex-residency",
|
||||
"x-codex-turn-state",
|
||||
] {
|
||||
assert_eq!(values(preserved).len(), 1, "missing {preserved}");
|
||||
}
|
||||
for excluded in [
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"host",
|
||||
"content-length",
|
||||
"authorization",
|
||||
"x-api-key",
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
] {
|
||||
assert!(values(excluded).is_empty(), "unexpected {excluded}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
let secret = "nested-private-value";
|
||||
@@ -191,25 +361,51 @@ fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_continuation_and_mark_the_routing_requirement() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": "resp_private_continuation"
|
||||
}));
|
||||
fn decoder_should_preserve_unknown_arbitrary_precision_numbers() {
|
||||
let number = "12345678901234567890123456789012345678901234567890";
|
||||
let body = format!(r#"{{"model":"smart-code","input":"hello","future_number":{number}}}"#);
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == "resp_private_continuation"
|
||||
));
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
assert!(!format!("{decoded:?}").contains("resp_private_continuation"));
|
||||
let decoded = decode_request_with_headers(body.as_bytes(), &HeaderMap::new())
|
||||
.expect("opaque numeric field should decode");
|
||||
let encoded = serde_json::to_string(openai_wire_body(&decoded)).expect("encode request body");
|
||||
|
||||
assert!(encoded.contains(&format!(r#""future_number":{number}"#)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_continuation_and_mark_the_routing_requirement() {
|
||||
for response_id in [
|
||||
"resp_private_continuation".to_owned(),
|
||||
format!("resp_{}\0opaque", "x".repeat(257)),
|
||||
String::new(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}));
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == &response_id
|
||||
));
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
if !response_id.is_empty() {
|
||||
assert!(!format!("{decoded:?}").contains(&response_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -316,7 +512,7 @@ fn transparent_encoder_should_forward_raw_sse_frames_unknown_events_and_terminal
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let frames = events
|
||||
.iter()
|
||||
.flat_map(|event| encoder.push_sse(event).expect("wire event should encode"))
|
||||
.flat_map(|event| encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let response = encoder.finish().expect("wire response should finish");
|
||||
let body = frames
|
||||
@@ -370,15 +566,11 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
|
||||
let sse = events
|
||||
.iter()
|
||||
.flat_map(|event| sse_encoder.push_sse(event).expect("SSE wire event"))
|
||||
.flat_map(|event| sse_encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let websocket = events
|
||||
.iter()
|
||||
.flat_map(|event| {
|
||||
websocket_encoder
|
||||
.push_websocket(event)
|
||||
.expect("WebSocket wire event")
|
||||
})
|
||||
.flat_map(|event| websocket_encoder.push_websocket(event))
|
||||
.collect::<Vec<_>>();
|
||||
let sse_text = String::from_utf8(
|
||||
sse.iter()
|
||||
@@ -400,8 +592,40 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
fn transparent_encoder_should_follow_wire_when_canonical_identity_changes() {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
))],
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "wire_resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started);
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "wire_resp_2"}}),
|
||||
);
|
||||
|
||||
let completed = encoder.push_sse(&changed_identity);
|
||||
|
||||
assert_eq!(completed.len(), 1);
|
||||
assert!(encoder.is_completed());
|
||||
assert_eq!(encoder.response_id(), Some("wire_resp_2"));
|
||||
assert_eq!(
|
||||
encoder
|
||||
.finish()
|
||||
.expect("wire terminal remains authoritative"),
|
||||
json!({"id": "wire_resp_2"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_only_require_wire_terminal_for_buffered_conversion() {
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
@@ -409,24 +633,8 @@ fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started).expect("started event");
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "resp_2"}}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
encoder
|
||||
.push_sse(&changed_identity)
|
||||
.expect_err("identity changes must fail"),
|
||||
ResponseEncodeError::WireIdentityChanged
|
||||
);
|
||||
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
missing_terminal.push_sse(&started).expect("started event");
|
||||
missing_terminal.push_sse(&started);
|
||||
assert_eq!(
|
||||
missing_terminal
|
||||
.finish()
|
||||
|
||||
@@ -1,14 +1,35 @@
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{GatewayError, ProviderError, ProviderErrorKind};
|
||||
use gateway_core::event::{
|
||||
GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader, ResponseMeta,
|
||||
};
|
||||
use gateway_core::operation::Operation;
|
||||
use serde_json::json;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use serde_json::{Value, json};
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as ClientMessage;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, models::ModelsExecution};
|
||||
use crate::openai::{api_router, authenticated_client, models::ModelsExecution};
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
@@ -148,11 +169,403 @@ fn response_create_should_reject_non_boolean_stream_without_disclosing_body_valu
|
||||
assert!(!rendered.contains(opaque_stream_value));
|
||||
}
|
||||
|
||||
// `serve_responses_websocket`/`forward_execution` 只有拿到 `hyper::upgrade::OnUpgrade`
|
||||
// 的真实 HTTP/1.1 升级连接才会运行;本 crate 的依赖图按 axum `ws` feature 编译
|
||||
// hyper(无 `http1`),测试进程内不存在能完成升级的服务端,`hyper::upgrade` 也不
|
||||
// 提供公开构造器。集成测试因此止步于 upgrade 边界:会话循环内的事件转发、错误帧
|
||||
// 与断连取消仍未被覆盖,需要 dev 依赖开启 `http1` 或 socket 泛化后才能落地。
|
||||
#[derive(Default)]
|
||||
struct AtomicFailureTrace {
|
||||
next_calls: AtomicUsize,
|
||||
committed: AtomicBool,
|
||||
finalized: AtomicBool,
|
||||
}
|
||||
|
||||
struct AtomicFailureSession {
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionSession for AtomicFailureSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
match self.trace.next_calls.fetch_add(1, Ordering::AcqRel) {
|
||||
0 => Ok(Some(atomic_failure_batch())),
|
||||
1 => {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
Err(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async { Err(EngineError::InvalidDeliveryState) })
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn commit_downstream(&mut self, _: Option<u16>) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.trace.committed.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.trace.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AtomicFailureExecution {
|
||||
client: AuthenticatedClient,
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionService for AtomicFailureExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_ws_atomic")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
vec![PublicModelId::new("model-a").expect("public model")]
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
model.as_str() == "model-a"
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
Ok(StartedExecution {
|
||||
request_id: gateway_core::engine::ModelRequestId::new("req_ws_atomic")
|
||||
.expect("request id"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: request.metadata.stream,
|
||||
session: Box::new(AtomicFailureSession {
|
||||
trace: Arc::clone(&self.trace),
|
||||
response_headers: self.response_headers.clone(),
|
||||
}),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn atomic_failure_batch() -> CoordinatedEvent {
|
||||
let started = ProviderEvent::canonical_with_wire(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_ws_atomic",
|
||||
"model-a",
|
||||
))],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"model": "model-a",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("created wire"),
|
||||
);
|
||||
let failed = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "websocket atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("failed wire"),
|
||||
);
|
||||
CoordinatedEvent::try_batch(
|
||||
vec![started, failed],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic WebSocket batch")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut messages = Vec::<Value>::new();
|
||||
loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
let failed = value.get("type").and_then(Value::as_str) == Some("response.failed");
|
||||
messages.push(value);
|
||||
if failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let failed = messages
|
||||
.iter()
|
||||
.find(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.expect("upstream failed event");
|
||||
assert_eq!(
|
||||
failed
|
||||
.pointer("/response/error/message")
|
||||
.and_then(Value::as_str),
|
||||
Some("websocket atomic upstream marker")
|
||||
);
|
||||
assert!(trace.committed.load(Ordering::Acquire));
|
||||
assert_eq!(trace.next_calls.load(Ordering::Acquire), 2);
|
||||
assert!(trace.finalized.load(Ordering::Acquire));
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_metadata_should_preserve_ordinary_headers_without_blocking_events() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: vec![
|
||||
ProviderResponseHeader::new("x-future-header", Bytes::from_static(b"future-value")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-private-hop")),
|
||||
ProviderResponseHeader::new("x-private-hop", Bytes::from_static(b"private-hop")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req_upstream")),
|
||||
],
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut metadata = None;
|
||||
let mut saw_failed = false;
|
||||
while metadata.is_none() || !saw_failed {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
match value.get("type").and_then(Value::as_str) {
|
||||
Some("response.metadata") => metadata = Some(value),
|
||||
Some("response.failed") => saw_failed = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let headers = metadata
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("headers"))
|
||||
.and_then(Value::as_object)
|
||||
.expect("response metadata headers");
|
||||
assert_eq!(headers.get("x-future-header"), Some(&json!("future-value")));
|
||||
// 官方 metadata 是 string map,同名多值只能保持既有的后值覆盖语义。
|
||||
assert_eq!(headers.get("x-future-multi"), Some(&json!("second")));
|
||||
assert_eq!(headers.get("x-request-id"), Some(&json!("req_upstream")));
|
||||
for omitted in [
|
||||
"x-future-bytes",
|
||||
"bad\0name",
|
||||
"authorization",
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"content-type",
|
||||
] {
|
||||
assert!(
|
||||
!headers.contains_key(omitted),
|
||||
"unexpected metadata header: {omitted:?}"
|
||||
);
|
||||
}
|
||||
assert!(saw_failed);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
let payload = json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "x".repeat(16 * 1024 * 1024 + 1),
|
||||
})
|
||||
.to_string();
|
||||
assert!(payload.len() > 16 * 1024 * 1024);
|
||||
|
||||
socket
|
||||
.send(ClientMessage::Text(payload.into()))
|
||||
.await
|
||||
.expect("send response.create above the former frame limit");
|
||||
|
||||
let mut saw_upstream_event = false;
|
||||
for _ in 0..3 {
|
||||
let message = tokio::time::timeout(Duration::from_secs(10), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
if matches!(
|
||||
value.get("type").and_then(Value::as_str),
|
||||
Some("response.created" | "response.failed")
|
||||
) {
|
||||
saw_upstream_event = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(saw_upstream_event);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
const TEST_WEBSOCKET_KEY: &str = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
|
||||
|
||||
@@ -1,57 +1,29 @@
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
extract::DefaultBodyLimit,
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode},
|
||||
routing::post,
|
||||
};
|
||||
use gateway_api::openai::router::MAX_CLIENT_REQUEST_BODY_BYTES;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
fn body_limit_app() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
"/v1/responses",
|
||||
post(|body: Bytes| async move {
|
||||
if body.len() == MAX_CLIENT_REQUEST_BODY_BYTES {
|
||||
StatusCode::NO_CONTENT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
}),
|
||||
)
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
}
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_accept_exactly_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES]))
|
||||
.expect("build exact-limit request"),
|
||||
.body(Body::from(vec![b'a'; REMOVED_BODY_LIMIT_BYTES + 1]))
|
||||
.expect("build request above the removed limit"),
|
||||
)
|
||||
.await
|
||||
.expect("route exact-limit request");
|
||||
.expect("route request above the removed limit");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_reject_one_byte_over_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES + 1]))
|
||||
.expect("build over-limit request"),
|
||||
)
|
||||
.await
|
||||
.expect("route over-limit request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
// The handler sees the body and rejects the missing credentials. A restored body limit
|
||||
// would return 413 before authentication runs.
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -17,8 +17,9 @@ bytes = "1.11.1"
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -8,24 +8,21 @@ use std::fmt;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::ProviderAccountId;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
use crate::routing::ProviderKind;
|
||||
|
||||
/// 客户端传入的 previous response ID。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// 客户端或 Provider 传递的 opaque response handle。
|
||||
///
|
||||
/// Codex 定义这个值的语义和格式。网关只将其作为亲和查找键或同账号上游
|
||||
/// continuation 的载体,不得按私有长度、字符集或前缀规则拒绝、归一化或改写。
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct PreviousResponseId(String);
|
||||
|
||||
impl PreviousResponseId {
|
||||
/// 校验并创建 previous response ID。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// ID 为空、过长或包含控制字符时返回错误。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, 256, false, None)?;
|
||||
Ok(Self(value))
|
||||
/// 按原样创建 opaque response handle。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -34,6 +31,12 @@ impl PreviousResponseId {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreviousResponseId {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PreviousResponseId(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 原生 response handle 的续接范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NativeContinuationScope {
|
||||
@@ -52,7 +55,7 @@ pub struct NativeContinuationPin {
|
||||
/// 客户端提交、用于查找可丢失会话亲和记录的 response ID。
|
||||
previous_response_id: PreviousResponseId,
|
||||
/// Provider 原生 response handle。
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
scope: NativeContinuationScope,
|
||||
@@ -63,7 +66,7 @@ impl NativeContinuationPin {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
previous_response_id: PreviousResponseId,
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
) -> Self {
|
||||
@@ -98,7 +101,7 @@ impl NativeContinuationPin {
|
||||
|
||||
/// 返回只允许发送给已冻结 Provider 目标的原生上游 handle。
|
||||
#[must_use]
|
||||
pub const fn upstream_response_id(&self) -> &SafeUpstreamValue {
|
||||
pub const fn upstream_response_id(&self) -> &PreviousResponseId {
|
||||
&self.upstream_response_id
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 唯一的账号重试、发送与下游 commit barrier owner。
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -17,7 +18,7 @@ use crate::engine::{
|
||||
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
|
||||
RequestAttemptContext, UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use crate::event::{
|
||||
GatewayEvent, ProviderEvent, ProviderResponseHeader, ProviderResponseObservation,
|
||||
};
|
||||
@@ -32,6 +33,21 @@ pub struct AttemptCoordinator<S: ?Sized> {
|
||||
engine: Arc<GatewayEngine<S>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum AccountSelection {
|
||||
Scheduled(Option<crate::engine::credential::ProviderAccountId>),
|
||||
Diagnostic(crate::engine::credential::ProviderAccountId),
|
||||
}
|
||||
|
||||
impl AccountSelection {
|
||||
fn required_account(&self) -> Option<&crate::engine::credential::ProviderAccountId> {
|
||||
match self {
|
||||
Self::Scheduled(account) => account.as_ref(),
|
||||
Self::Diagnostic(account) => Some(account),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: ?Sized> AttemptCoordinator<S>
|
||||
where
|
||||
S: ExecutionStore,
|
||||
@@ -56,6 +72,49 @@ where
|
||||
required_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<ResponseExecutionSession<S>, EngineError> {
|
||||
self.start_with_account_selection(
|
||||
request,
|
||||
operation,
|
||||
plan,
|
||||
AccountSelection::Scheduled(required_account),
|
||||
continuation,
|
||||
cancellation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 对固定账号执行诊断请求。
|
||||
///
|
||||
/// 仅跳过该账号的本地可用性投影;账号租约、并发和请求间隔仍必须满足。
|
||||
pub async fn start_diagnostic(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
operation: Operation,
|
||||
plan: RoutingPlan,
|
||||
required_account: crate::engine::credential::ProviderAccountId,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<ResponseExecutionSession<S>, EngineError> {
|
||||
self.start_with_account_selection(
|
||||
request,
|
||||
operation,
|
||||
plan,
|
||||
AccountSelection::Diagnostic(required_account),
|
||||
continuation,
|
||||
cancellation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn start_with_account_selection(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
operation: Operation,
|
||||
plan: RoutingPlan,
|
||||
account_selection: AccountSelection,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
cancellation: CancellationToken,
|
||||
) -> Result<ResponseExecutionSession<S>, EngineError> {
|
||||
let request_id = request.id.clone();
|
||||
let client_api_key_ref = request.client_api_key_ref.clone();
|
||||
@@ -84,7 +143,7 @@ where
|
||||
request_persisted: false,
|
||||
operation,
|
||||
plan,
|
||||
required_account,
|
||||
account_selection,
|
||||
continuation,
|
||||
continuation_attempt,
|
||||
account_state_owner,
|
||||
@@ -107,7 +166,9 @@ where
|
||||
client_response_id: None,
|
||||
upstream_response_id: None,
|
||||
last_account_exhaustion: None,
|
||||
last_account_exhaustion_events: Vec::new(),
|
||||
provider_attempt_outcomes: Vec::new(),
|
||||
pending_terminal_failure: None,
|
||||
};
|
||||
|
||||
if session.cancellation.is_cancelled() {
|
||||
@@ -158,7 +219,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
request_persisted: bool,
|
||||
operation: Operation,
|
||||
plan: RoutingPlan,
|
||||
required_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
account_selection: AccountSelection,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
account_state_owner: Option<ProviderAccountStateOwner>,
|
||||
@@ -185,7 +246,16 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
client_response_id: Option<String>,
|
||||
upstream_response_id: Option<String>,
|
||||
last_account_exhaustion: Option<ProviderError>,
|
||||
/// 最近一次账号耗尽在重试时被丢弃的原始失败批次;只用于后续空选路终态。
|
||||
last_account_exhaustion_events: Vec<ProviderEvent>,
|
||||
provider_attempt_outcomes: Vec<ProviderAttemptOutcome>,
|
||||
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误。
|
||||
pending_terminal_failure: Option<PendingTerminalFailure>,
|
||||
}
|
||||
|
||||
struct PendingTerminalFailure {
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
}
|
||||
|
||||
impl<S: ?Sized> ResponseExecutionSession<S>
|
||||
@@ -201,6 +271,11 @@ where
|
||||
if self.delivery_pending {
|
||||
return Err(EngineError::DownstreamCommitRequired);
|
||||
}
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
self.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(pending.error));
|
||||
}
|
||||
if self.finalized {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -220,6 +295,21 @@ where
|
||||
)));
|
||||
}
|
||||
PullOutcome::AttemptDiscarded => {}
|
||||
PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
} => {
|
||||
let requirement = if self.downstream_committed_at.is_some() {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
self.delivery_pending = true;
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
};
|
||||
self.pending_terminal_failure =
|
||||
Some(PendingTerminalFailure { error, send_state });
|
||||
return CoordinatedEvent::try_batch(events, requirement).map(Some);
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if self.downstream_committed_at.is_some() {
|
||||
self.finish_success().await?;
|
||||
@@ -248,6 +338,13 @@ where
|
||||
match self.pull().await? {
|
||||
PullOutcome::Event(event) => events.push(event),
|
||||
PullOutcome::AttemptDiscarded => events.clear(),
|
||||
PullOutcome::TerminalFailure {
|
||||
error, send_state, ..
|
||||
} => {
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(error));
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if events.is_empty() {
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
@@ -259,11 +356,11 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// 在协议 adapter 真正写出首字节前持久化下游不可撤回边界。
|
||||
/// 在协议 adapter 真正写出首字节前记录下游不可撤回边界。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 没有待提交结果、重复提交或 Store 失败时返回错误。
|
||||
/// 没有待提交结果或重复提交时返回错误;观测 Store 失败只记服务端告警。
|
||||
pub async fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
@@ -272,10 +369,18 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
let committed_at = SystemTime::now();
|
||||
self.engine
|
||||
.store()
|
||||
.mark_downstream_committed(&self.request_id, committed_at, client_status_code)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_downstream_committed",
|
||||
&self.request_id,
|
||||
self.engine.store().mark_downstream_committed(
|
||||
&self.request_id,
|
||||
committed_at,
|
||||
client_status_code,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.downstream_committed_at = Some(committed_at);
|
||||
self.client_status_code = client_status_code;
|
||||
self.delivery_pending = false;
|
||||
@@ -289,7 +394,7 @@ where
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 状态已经写入或 Store 无法写回时返回错误。
|
||||
/// 状态已经写入时返回错误;观测 Store 失败只记服务端告警。
|
||||
pub async fn record_client_status(
|
||||
&mut self,
|
||||
client_status_code: u16,
|
||||
@@ -298,10 +403,14 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
if self.request_persisted {
|
||||
self.engine
|
||||
.store()
|
||||
.record_client_status(&self.request_id, client_status_code)
|
||||
.await?;
|
||||
best_effort_store_write(
|
||||
"record_client_status",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.record_client_status(&self.request_id, client_status_code),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.client_status_code = Some(client_status_code);
|
||||
Ok(())
|
||||
@@ -345,9 +454,8 @@ where
|
||||
}
|
||||
let account = current.metadata.provider_account_id()?.clone();
|
||||
let response_id = self.upstream_response_id.as_deref()?;
|
||||
let previous_response_id = PreviousResponseId::new(response_id.to_owned()).ok()?;
|
||||
let upstream_response_id =
|
||||
crate::error::SafeUpstreamValue::new(response_id.to_owned()).ok()?;
|
||||
let previous_response_id = PreviousResponseId::new(response_id.to_owned());
|
||||
let upstream_response_id = PreviousResponseId::new(response_id.to_owned());
|
||||
Some(
|
||||
NativeContinuationPin::new(
|
||||
previous_response_id,
|
||||
@@ -376,13 +484,20 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
self.delivery_pending = false;
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
return self
|
||||
.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await;
|
||||
}
|
||||
self.finish_interruption(EngineError::Cancelled).await
|
||||
}
|
||||
|
||||
async fn pull(&mut self) -> Result<PullOutcome, EngineError> {
|
||||
loop {
|
||||
if self.current.is_none() {
|
||||
self.prepare_attempt().await?;
|
||||
if self.current.is_none()
|
||||
&& let Some(outcome) = self.prepare_attempt().await?
|
||||
{
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let boundary = {
|
||||
@@ -409,32 +524,17 @@ where
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
PollBoundary::Item(Some(Ok(mut event))) => {
|
||||
if let Some(observation) = event.take_observation()
|
||||
&& let Err(error) = self.observe_response(observation)
|
||||
{
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let mut identity_error = None;
|
||||
for fact in event.canonical_facts() {
|
||||
if let Err(error) = self.observe_response_identity(fact) {
|
||||
identity_error = Some(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(error) = identity_error {
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
continue;
|
||||
if let Some(observation) = event.take_observation() {
|
||||
self.observe_response(observation);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_event(fact).await?;
|
||||
self.observe_response_identity(fact);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_event(fact).await;
|
||||
}
|
||||
if event.wire_event().is_some() && !event.has_canonical_facts() {
|
||||
self.observe_wire_event().await?;
|
||||
self.observe_wire_event().await;
|
||||
}
|
||||
if !event.has_client_event() {
|
||||
continue;
|
||||
@@ -442,8 +542,13 @@ where
|
||||
return Ok(PullOutcome::Event(event));
|
||||
}
|
||||
PollBoundary::Item(Some(Err(error))) => {
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
match self.handle_stream_error(error).await? {
|
||||
StreamErrorOutcome::AttemptDiscarded => {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
terminal @ StreamErrorOutcome::TerminalFailure { .. } => {
|
||||
return Ok(terminal.into_pull_outcome());
|
||||
}
|
||||
}
|
||||
}
|
||||
PollBoundary::Item(None) => {
|
||||
@@ -455,7 +560,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<(), EngineError> {
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
if self.attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
@@ -483,27 +588,36 @@ where
|
||||
.ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
// recovery 钉选在此被一次性消费,只绑定本次 replay attempt;
|
||||
// 外部 required_account 每次 attempt 都重新生效。
|
||||
let pinned_account = self
|
||||
.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.required_account.clone());
|
||||
let pinned_account = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => Some(account.clone()),
|
||||
AccountSelection::Scheduled(_) => self
|
||||
.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => AccountAttemptContext::diagnostic(
|
||||
self.excluded_accounts.clone(),
|
||||
account.clone(),
|
||||
self.account_state_owner.clone(),
|
||||
),
|
||||
AccountSelection::Scheduled(_) => AccountAttemptContext::new(
|
||||
self.excluded_accounts.clone(),
|
||||
pinned_account.clone(),
|
||||
self.account_state_owner.clone(),
|
||||
),
|
||||
}
|
||||
.with_credential_recovery_attempted(pinned_account.as_ref().is_some_and(|account| {
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
}));
|
||||
let context = AttemptContext::new(
|
||||
RequestAttemptContext::new(self.request_id.clone(), self.client_api_key_ref.clone())
|
||||
.with_timing_started_at(self.timing_started_at),
|
||||
next_attempt,
|
||||
self.deadline,
|
||||
self.plan.account_selection_policy(),
|
||||
AccountAttemptContext::new(
|
||||
self.excluded_accounts.clone(),
|
||||
pinned_account.clone(),
|
||||
self.account_state_owner.clone(),
|
||||
)
|
||||
.with_credential_recovery_attempted(pinned_account.as_ref().is_some_and(
|
||||
|account| {
|
||||
self.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
},
|
||||
)),
|
||||
account_context,
|
||||
self.continuation.clone(),
|
||||
self.cancellation.clone(),
|
||||
)
|
||||
@@ -546,11 +660,20 @@ where
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if error.kind() == ProviderErrorKind::NoEligibleAccount
|
||||
&& let Some(exhaustion) = self.last_account_exhaustion.clone()
|
||||
&& let Some(exhaustion) = self.last_account_exhaustion.take()
|
||||
{
|
||||
let engine_error = provider_engine_error(&exhaustion);
|
||||
let send_state = self.current_send_state();
|
||||
let mut events = std::mem::take(&mut self.last_account_exhaustion_events);
|
||||
if !events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut events);
|
||||
return Ok(Some(PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error: exhaustion,
|
||||
send_state,
|
||||
}));
|
||||
}
|
||||
self.finish_provider_error(&exhaustion).await?;
|
||||
return Err(engine_error);
|
||||
return Err(provider_engine_error(exhaustion));
|
||||
}
|
||||
if !(matches!(
|
||||
error.kind(),
|
||||
@@ -559,9 +682,8 @@ where
|
||||
{
|
||||
self.record_provider_failure(candidate.provider().clone(), error.kind());
|
||||
}
|
||||
let engine_error = provider_engine_error(&error);
|
||||
self.finish_provider_error(&error).await?;
|
||||
return Err(engine_error);
|
||||
return Err(provider_engine_error(error));
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -650,17 +772,30 @@ where
|
||||
http_version: None,
|
||||
};
|
||||
if self.request_persisted {
|
||||
self.engine.store().record_attempt(attempt_record).await?;
|
||||
best_effort_store_write(
|
||||
"record_attempt",
|
||||
&self.request_id,
|
||||
self.engine.store().record_attempt(attempt_record),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
let request = self
|
||||
.pending_request
|
||||
.take()
|
||||
.as_ref()
|
||||
.ok_or(EngineError::InvalidDeliveryState)?;
|
||||
self.engine
|
||||
.store()
|
||||
.create_model_request_with_attempt(request, attempt_record)
|
||||
.await?;
|
||||
self.request_persisted = true;
|
||||
if best_effort_store_write(
|
||||
"create_model_request_with_attempt",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.create_model_request_with_attempt(request.clone(), attempt_record),
|
||||
)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
self.pending_request = None;
|
||||
self.request_persisted = true;
|
||||
}
|
||||
}
|
||||
self.attempts = next_attempt.get();
|
||||
self.current = Some(CurrentAttempt {
|
||||
@@ -672,10 +807,15 @@ where
|
||||
send_observed: false,
|
||||
response_observation: None,
|
||||
});
|
||||
Ok(())
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) -> Result<(), EngineError> {
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) {
|
||||
self.observe_event_facts(event);
|
||||
self.mark_send_observed().await;
|
||||
}
|
||||
|
||||
fn observe_event_facts(&mut self, event: &GatewayEvent) {
|
||||
let elapsed = elapsed_ms(self.timing_started_at);
|
||||
observe_event_timing(&mut self.timings, event, elapsed);
|
||||
if let GatewayEvent::Usage(observed) = event {
|
||||
@@ -689,46 +829,42 @@ where
|
||||
if let GatewayEvent::ProviderCost(observed) = event {
|
||||
self.cost = observed.into_estimate();
|
||||
}
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
}
|
||||
|
||||
async fn observe_wire_event(&mut self) {
|
||||
self.mark_send_observed().await;
|
||||
}
|
||||
|
||||
async fn mark_send_observed(&mut self) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn observe_wire_event(&mut self) -> Result<(), EngineError> {
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_response(
|
||||
&mut self,
|
||||
observation: ProviderResponseObservation,
|
||||
) -> Result<(), ProviderError> {
|
||||
let send_state = self.current_send_state();
|
||||
let current = self
|
||||
.current
|
||||
.as_mut()
|
||||
.ok_or_else(|| ProviderError::new(ProviderErrorKind::Protocol, send_state))?;
|
||||
fn observe_response(&mut self, observation: ProviderResponseObservation) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.is_some_and(|existing| existing.transport() != observation.transport())
|
||||
{
|
||||
return Err(ProviderError::new(ProviderErrorKind::Protocol, send_state));
|
||||
return;
|
||||
}
|
||||
let observed = observation.timings();
|
||||
if let Some(value) = observed.transport_decision_wait_ms {
|
||||
@@ -756,47 +892,31 @@ where
|
||||
self.timings.provider_processing_ms = Some(value);
|
||||
}
|
||||
current.response_observation = Some(observation);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_response_identity(&mut self, event: &GatewayEvent) -> Result<(), ProviderError> {
|
||||
fn observe_response_identity(&mut self, event: &GatewayEvent) {
|
||||
let metadata = match event {
|
||||
GatewayEvent::Started(metadata) | GatewayEvent::Completed(metadata) => metadata,
|
||||
_ => return Ok(()),
|
||||
_ => return,
|
||||
};
|
||||
let response_id = crate::error::SafeUpstreamValue::new(metadata.response_id().to_owned())
|
||||
.map_err(|_| {
|
||||
ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::Sent)
|
||||
})?;
|
||||
let response_id = response_id.as_str();
|
||||
if response_id.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if self
|
||||
.upstream_response_id
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != response_id)
|
||||
{
|
||||
return Err(ProviderError::new(
|
||||
ProviderErrorKind::Protocol,
|
||||
UpstreamSendState::Sent,
|
||||
));
|
||||
}
|
||||
let response_id = response_id.to_owned();
|
||||
let response_id = metadata.response_id().to_owned();
|
||||
self.client_response_id = Some(response_id.clone());
|
||||
self.upstream_response_id = Some(response_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 返回 `true` 表示调用方必须丢弃本 attempt 已收集的未提交事件。
|
||||
async fn handle_stream_error(&mut self, error: ProviderError) -> Result<bool, EngineError> {
|
||||
/// 失败可重试时要求丢弃本 attempt;预算耗尽时可返回最后一批原始失败事件。
|
||||
async fn handle_stream_error(
|
||||
&mut self,
|
||||
mut error: ProviderError,
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
// 原始 wire 只能活在本次决策栈内;后续 clone、attempt 记录与终态对象均只
|
||||
// 接触已剥离的稳定错误字段。
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if matches!(
|
||||
let account_exhausted = matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted
|
||||
) {
|
||||
self.last_account_exhaustion = Some(error.clone());
|
||||
}
|
||||
);
|
||||
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
|
||||
// attempt_send_state 是本 attempt 自身的发送事实,驱动重试门;
|
||||
// 持久化与终态用请求级水位,二者不可混用(水位会把早先 attempt 的
|
||||
@@ -807,10 +927,16 @@ where
|
||||
error.send_state()
|
||||
};
|
||||
let send_state = self.raise_send_watermark(attempt_send_state);
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let provider_proved_replay_safe = provider_proved_replay_safe(&error);
|
||||
let continuation_retry = self.prepare_continuation_retry(
|
||||
¤t,
|
||||
@@ -818,7 +944,13 @@ where
|
||||
attempt_send_state,
|
||||
provider_proved_replay_safe,
|
||||
);
|
||||
let ordinary_retry = self.required_account.is_none()
|
||||
let pre_delivery_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& error.allows_pre_delivery_retry()
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
let ordinary_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
@@ -840,9 +972,13 @@ where
|
||||
.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
});
|
||||
let retryable = continuation_retry || same_account_retry || ordinary_retry;
|
||||
let retryable =
|
||||
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
|
||||
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始失败响应只能随最后一次客户端可见
|
||||
// 的账号耗尽错误继续,不能进入中间失败记录。
|
||||
let persistence_error = self.request_persisted.then(|| error.clone());
|
||||
if same_account_retry {
|
||||
if let Some(account) = current.metadata.provider_account_id() {
|
||||
self.credential_recovery_attempted_accounts
|
||||
@@ -856,37 +992,66 @@ where
|
||||
{
|
||||
self.excluded_accounts.insert(account.clone());
|
||||
}
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: current.metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: current.metadata.upstream_model().clone(),
|
||||
upstream_status_code: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code),
|
||||
upstream_request_id: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::request_id)
|
||||
.map(|value| value.as_str().to_owned()),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
})
|
||||
.await?;
|
||||
if account_exhausted {
|
||||
self.last_account_exhaustion_events = atomic_client_events;
|
||||
self.last_account_exhaustion = Some(error);
|
||||
}
|
||||
if let Some(error) = persistence_error {
|
||||
best_effort_store_write(
|
||||
"record_intermediate_failure",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: current.metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: current.metadata.upstream_model().clone(),
|
||||
upstream_status_code: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code),
|
||||
upstream_request_id: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::request_id)
|
||||
.map(|value| value.as_str().to_owned()),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.reset_uncommitted_observations();
|
||||
return Ok(true);
|
||||
return Ok(StreamErrorOutcome::AttemptDiscarded);
|
||||
}
|
||||
|
||||
let engine_error = provider_engine_error(&error);
|
||||
self.current = Some(current);
|
||||
if !atomic_client_events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut atomic_client_events);
|
||||
return Ok(StreamErrorOutcome::TerminalFailure {
|
||||
events: atomic_client_events,
|
||||
error,
|
||||
send_state,
|
||||
});
|
||||
}
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
Err(engine_error)
|
||||
Err(provider_engine_error(error))
|
||||
}
|
||||
|
||||
fn observe_atomic_terminal_events(&mut self, events: &mut [ProviderEvent]) {
|
||||
for event in events {
|
||||
if let Some(observation) = event.take_observation() {
|
||||
self.observe_response(observation);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_response_identity(fact);
|
||||
self.observe_event_facts(fact);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_continuation_retry(
|
||||
@@ -896,7 +1061,7 @@ where
|
||||
send_state: UpstreamSendState,
|
||||
provider_proved_replay_safe: bool,
|
||||
) -> bool {
|
||||
if self.required_account.is_some()
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt == ContinuationAttempt::None
|
||||
|| self.downstream_committed_at.is_some()
|
||||
|| self.delivery_pending
|
||||
@@ -976,34 +1141,42 @@ where
|
||||
.and_then(ProviderResponseObservation::status_code);
|
||||
let (upstream_transport, http_version, websocket_pool) =
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: ExecutionOutcome::Succeeded,
|
||||
send_state: UpstreamSendState::Sent,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code,
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: self.upstream_response_id.clone(),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
provider_metadata_json,
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
retry_after_ms: None,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
})
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"finalize_model_request",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: ExecutionOutcome::Succeeded,
|
||||
send_state: UpstreamSendState::Sent,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code,
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: self.upstream_response_id.clone(),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
retry_after_ms: None,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finalized = true;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1072,11 +1245,15 @@ where
|
||||
UpstreamSendState::Ambiguous
|
||||
};
|
||||
let send_state = self.raise_send_watermark(send_state);
|
||||
if self.attempts > 0 {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state)
|
||||
.await?;
|
||||
if self.attempts > 0 && self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome,
|
||||
@@ -1126,36 +1303,44 @@ where
|
||||
.and_then(ProviderResponseObservation::status_code);
|
||||
let (upstream_transport, http_version, websocket_pool) =
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: finalization.outcome,
|
||||
send_state: finalization.send_state,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code: finalization.upstream_status_code.or(observed_status_code),
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: finalization
|
||||
.provider_response_id
|
||||
.or_else(|| self.upstream_response_id.clone()),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
provider_metadata_json,
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
})
|
||||
.await?;
|
||||
best_effort_store_write(
|
||||
"finalize_model_request",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: finalization.outcome,
|
||||
send_state: finalization.send_state,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code: finalization
|
||||
.upstream_status_code
|
||||
.or(observed_status_code),
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: finalization
|
||||
.provider_response_id
|
||||
.or_else(|| self.upstream_response_id.clone()),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
self.finalized = true;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1186,6 +1371,14 @@ where
|
||||
.map(|metadata| metadata.as_json().to_owned())
|
||||
}
|
||||
|
||||
fn current_service_tier(&self) -> Option<String> {
|
||||
self.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::service_tier)
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn image_generation_succeeded(&self) -> Option<bool> {
|
||||
self.image_generation_requested
|
||||
.then(|| self.usage.image_output_tokens.unwrap_or_default() > 0)
|
||||
@@ -1235,12 +1428,62 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn best_effort_store_write<T>(
|
||||
operation: &'static str,
|
||||
request_id: &ModelRequestId,
|
||||
write: impl Future<Output = Result<T, StoreError>>,
|
||||
) -> Option<T> {
|
||||
match write.await {
|
||||
Ok(value) => Some(value),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
operation,
|
||||
request_id = request_id.as_str(),
|
||||
error_kind = ?error.kind(),
|
||||
"执行观测写入失败,数据面不受影响"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PullOutcome {
|
||||
Event(ProviderEvent),
|
||||
AttemptDiscarded,
|
||||
TerminalFailure {
|
||||
events: Vec<ProviderEvent>,
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
},
|
||||
End,
|
||||
}
|
||||
|
||||
enum StreamErrorOutcome {
|
||||
AttemptDiscarded,
|
||||
TerminalFailure {
|
||||
events: Vec<ProviderEvent>,
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
},
|
||||
}
|
||||
|
||||
impl StreamErrorOutcome {
|
||||
fn into_pull_outcome(self) -> PullOutcome {
|
||||
match self {
|
||||
Self::AttemptDiscarded => PullOutcome::AttemptDiscarded,
|
||||
Self::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
} => PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PollBoundary {
|
||||
Item(Option<Result<ProviderEvent, ProviderError>>),
|
||||
Cancelled,
|
||||
@@ -1313,11 +1556,11 @@ fn initial_continuation_attempt(
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_engine_error(error: &ProviderError) -> EngineError {
|
||||
fn provider_engine_error(error: ProviderError) -> EngineError {
|
||||
if error.kind() == ProviderErrorKind::Cancelled {
|
||||
EngineError::Cancelled
|
||||
} else {
|
||||
EngineError::Provider(error.clone())
|
||||
EngineError::Provider(error)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,46 @@ impl fmt::Display for ProviderAccountId {
|
||||
}
|
||||
}
|
||||
|
||||
/// credential 轮换时可选替换的上游账号身份。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountIdentity {
|
||||
upstream_user_id: String,
|
||||
upstream_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderAccountIdentity {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderAccountIdentity")
|
||||
.field("upstream_user_id", &"<redacted>")
|
||||
.field(
|
||||
"upstream_account_id",
|
||||
&self.upstream_account_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAccountIdentity {
|
||||
#[must_use]
|
||||
pub const fn new(upstream_user_id: String, upstream_account_id: Option<String>) -> Self {
|
||||
Self {
|
||||
upstream_user_id,
|
||||
upstream_account_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_account_id(&self) -> Option<&str> {
|
||||
self.upstream_account_id.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// `provider_accounts.credential_revision` 的正数 CAS revision。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct CredentialRevision(NonZeroU64);
|
||||
@@ -602,6 +642,15 @@ pub trait ProviderAccountStore: Send + Sync {
|
||||
expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
/// 读取账号当前 credential 及其 revision,不做任何版本比对。
|
||||
///
|
||||
/// 管理写入必须在临近 CAS 时用它取 fence,而不是携带调用方持有的旧 revision:
|
||||
/// 后台刷新随时会推进 revision,用陈旧快照会把正常的恢复操作误判为冲突。
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
@@ -901,6 +950,63 @@ pub struct AccountSelectionContext {
|
||||
pub excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
pub preferred_account: Option<ProviderAccountId>,
|
||||
pub round_robin_cursor: u64,
|
||||
pub availability: AccountAvailabilityPolicy,
|
||||
}
|
||||
|
||||
/// 选择账号时是否信任本地可用性投影。
|
||||
///
|
||||
/// 管理端对指定账号执行诊断时,会直接向上游确认实际状态;该模式跳过
|
||||
/// `enabled`、可用性、冷却和 token 到期等本地投影,仍保留租约、并发和
|
||||
/// 请求间隔约束,且只能与固定账号约束组合使用。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AccountAvailabilityPolicy {
|
||||
#[default]
|
||||
Enforce,
|
||||
BypassForDiagnostic,
|
||||
}
|
||||
|
||||
impl AccountAvailabilityPolicy {
|
||||
#[must_use]
|
||||
pub const fn bypasses_local_availability(self) -> bool {
|
||||
matches!(self, Self::BypassForDiagnostic)
|
||||
}
|
||||
}
|
||||
|
||||
/// 候选账号未进入调度池的约束。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountSchedulingBlocker {
|
||||
LocalAvailability,
|
||||
Excluded,
|
||||
ConcurrencyLimit,
|
||||
RequestInterval,
|
||||
}
|
||||
|
||||
/// 优先账号在本次选择中的处理结果。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PreferredAccountSelection {
|
||||
NotRequested,
|
||||
Hit,
|
||||
Missing,
|
||||
Blocked(AccountSchedulingBlocker),
|
||||
}
|
||||
|
||||
/// 一次完整账号选择;同时保留优先账号决策,供 Provider 记录调度遥测。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AccountSelection<'a> {
|
||||
candidate: &'a AccountCandidate,
|
||||
preferred: PreferredAccountSelection,
|
||||
}
|
||||
|
||||
impl<'a> AccountSelection<'a> {
|
||||
#[must_use]
|
||||
pub const fn candidate(self) -> &'a AccountCandidate {
|
||||
self.candidate
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn preferred(self) -> PreferredAccountSelection {
|
||||
self.preferred
|
||||
}
|
||||
}
|
||||
|
||||
/// 同一 target 内唯一的账号排序器。
|
||||
@@ -914,43 +1020,55 @@ impl AccountSelector {
|
||||
&self,
|
||||
candidates: &'a [AccountCandidate],
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<&'a AccountCandidate> {
|
||||
) -> Option<AccountSelection<'a>> {
|
||||
let preferred = if let Some(preferred) = context.preferred_account.as_ref() {
|
||||
match candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
{
|
||||
Some(candidate) => match self.scheduling_blocker(candidate, context) {
|
||||
None => {
|
||||
return Some(AccountSelection {
|
||||
candidate,
|
||||
preferred: PreferredAccountSelection::Hit,
|
||||
});
|
||||
}
|
||||
Some(blocker) => PreferredAccountSelection::Blocked(blocker),
|
||||
},
|
||||
None => PreferredAccountSelection::Missing,
|
||||
}
|
||||
} else {
|
||||
PreferredAccountSelection::NotRequested
|
||||
};
|
||||
|
||||
let mut eligible = candidates
|
||||
.iter()
|
||||
.filter(|candidate| eligible(candidate, context))
|
||||
.filter(|candidate| self.scheduling_blocker(candidate, context).is_none())
|
||||
.collect::<Vec<_>>();
|
||||
if eligible.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(preferred) = context.preferred_account.as_ref()
|
||||
&& let Some(candidate) = eligible
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
&& (context.policy.strategy() == RotationStrategy::Sticky
|
||||
|| (context.policy.strategy() == RotationStrategy::Smart
|
||||
&& !smart_preference_should_escape(&candidate.signals)))
|
||||
{
|
||||
return Some(candidate);
|
||||
}
|
||||
|
||||
match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
}),
|
||||
let candidate = match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
RotationStrategy::RoundRobin => {
|
||||
eligible.sort_by_key(|candidate| candidate.account.id().clone());
|
||||
let index = context.round_robin_cursor as usize % eligible.len();
|
||||
return Some(eligible[index]);
|
||||
eligible.get(index).copied()?
|
||||
}
|
||||
RotationStrategy::Smart => {
|
||||
return select_smart_candidate(&eligible, context.round_robin_cursor);
|
||||
select_smart_candidate(&eligible, context.round_robin_cursor)?
|
||||
}
|
||||
RotationStrategy::Sticky => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
@@ -959,9 +1077,44 @@ impl AccountSelector {
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
};
|
||||
Some(AccountSelection {
|
||||
candidate,
|
||||
preferred,
|
||||
})
|
||||
}
|
||||
|
||||
fn scheduling_blocker(
|
||||
&self,
|
||||
candidate: &AccountCandidate,
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<AccountSchedulingBlocker> {
|
||||
if !context.availability.bypasses_local_availability()
|
||||
&& !candidate.account.is_schedulable(context.now)
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
}
|
||||
eligible.into_iter().next()
|
||||
if context.excluded_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::Excluded);
|
||||
}
|
||||
if candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get() {
|
||||
return Some(AccountSchedulingBlocker::ConcurrencyLimit);
|
||||
}
|
||||
if candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_some_and(|last_started| {
|
||||
!context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::RequestInterval);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -969,8 +1122,6 @@ const SMART_LOAD_WEIGHT: f64 = 1.0;
|
||||
const SMART_QUOTA_WEIGHT: f64 = 0.8;
|
||||
const SMART_FAILURE_WEIGHT: f64 = 1.0;
|
||||
const SMART_LATENCY_WEIGHT: f64 = 0.5;
|
||||
const SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS: u16 = 5_000;
|
||||
const SMART_PREFERENCE_LATENCY_ESCAPE_MS: u64 = 15_000;
|
||||
|
||||
struct SmartNormalization {
|
||||
max_in_flight: u32,
|
||||
@@ -1091,34 +1242,6 @@ fn higher_is_better(value: u64, minimum: u64, maximum: u64) -> f64 {
|
||||
(value.saturating_sub(minimum) as f64 / (maximum - minimum) as f64).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn smart_preference_should_escape(signals: &AccountRuntimeSignals) -> bool {
|
||||
signals
|
||||
.failure_rate_basis_points
|
||||
.is_some_and(|rate| rate > SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS)
|
||||
|| signals
|
||||
.first_output_latency_ms
|
||||
.is_some_and(|latency| latency > SMART_PREFERENCE_LATENCY_ESCAPE_MS)
|
||||
}
|
||||
|
||||
fn eligible(candidate: &AccountCandidate, context: &AccountSelectionContext) -> bool {
|
||||
if !candidate.account.is_schedulable(context.now)
|
||||
|| context.excluded_accounts.contains(candidate.account.id())
|
||||
|| candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_none_or(|last_started| {
|
||||
context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
}
|
||||
|
||||
/// Credential 值对象构造错误。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
pub enum CredentialError {
|
||||
|
||||
@@ -17,7 +17,10 @@ use crate::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
|
||||
};
|
||||
use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeResult};
|
||||
use crate::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
AccountProbeUpstreamResponse,
|
||||
};
|
||||
use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CoordinatedEvent, EngineError,
|
||||
@@ -36,7 +39,7 @@ use crate::routing::{
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const CONTINUATION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const REDIS_COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
@@ -233,15 +236,6 @@ impl DefaultExecutionService {
|
||||
request.client.policy.authorize().map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
|
||||
})?;
|
||||
if !request.client.snapshot.contains_public_model_for_provider(
|
||||
&request.public_model,
|
||||
request.client.policy.provider_kind(),
|
||||
) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = started_at
|
||||
.checked_add(MODEL_REQUEST_DEADLINE)
|
||||
@@ -260,7 +254,7 @@ impl DefaultExecutionService {
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => result.ok().flatten(),
|
||||
@@ -398,14 +392,33 @@ impl DefaultExecutionService {
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let blocked_providers = match self.circuits.decision(provider_kind).await.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"provider health state is temporarily unavailable",
|
||||
)
|
||||
})? {
|
||||
ProviderCircuitDecision::Allow => BTreeSet::new(),
|
||||
ProviderCircuitDecision::BlockedUntil(_) => BTreeSet::from([provider_kind.clone()]),
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
_ = timeout => None,
|
||||
};
|
||||
let blocked_providers = match decision {
|
||||
Some(Ok(ProviderCircuitDecision::Allow)) => BTreeSet::new(),
|
||||
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
|
||||
BTreeSet::from([provider_kind.clone()])
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
%error,
|
||||
"Provider circuit 读取失败,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
"Provider circuit 读取超时,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
};
|
||||
Ok(RoutingContext {
|
||||
provider_kind: Some(provider_kind.clone()),
|
||||
@@ -416,7 +429,7 @@ impl DefaultExecutionService {
|
||||
async fn probe_inner(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> Result<AccountProbeResult, GatewayError> {
|
||||
) -> Result<AccountProbeResult, AccountProbeError> {
|
||||
let AccountProbeRequest {
|
||||
account_id,
|
||||
provider_kind,
|
||||
@@ -477,11 +490,11 @@ impl DefaultExecutionService {
|
||||
};
|
||||
let mut session = match self
|
||||
.probe_coordinator
|
||||
.start(
|
||||
.start_diagnostic(
|
||||
new_request,
|
||||
operation,
|
||||
plan,
|
||||
Some(account_id),
|
||||
account_id,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
@@ -525,26 +538,55 @@ impl DefaultExecutionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。
|
||||
/// 探测失败先记录脱敏分类事实,再把请求局部的原始上游响应交给认证管理端。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> GatewayError {
|
||||
) -> AccountProbeError {
|
||||
let upstream_response = match error {
|
||||
EngineError::Provider(provider_error) => provider_error
|
||||
.client_visible_upstream_response()
|
||||
.map(AccountProbeUpstreamResponse::from_client_response),
|
||||
_ => None,
|
||||
};
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let _ = self
|
||||
let latency = started_at.elapsed().unwrap_or_default();
|
||||
let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
upstream_model = observed.upstream_model.as_str(),
|
||||
failure_kind = provider_error.kind().as_str(),
|
||||
send_state = ?provider_error.send_state(),
|
||||
upstream_status = ?provider_error.upstream_status(),
|
||||
provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()),
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
if let Err(store_error) = self
|
||||
.observations
|
||||
.record_probe_failure(ProbeFailure {
|
||||
provider_kind: observed.provider_kind.clone(),
|
||||
account_id: observed.account_id.clone(),
|
||||
upstream_model_id: observed.upstream_model.clone(),
|
||||
error: provider_error.clone(),
|
||||
latency: started_at.elapsed().unwrap_or_default(),
|
||||
latency,
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
operation = "record_probe_failure",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
error_kind = ?store_error.kind(),
|
||||
"账号连接测试观测写入失败,测试结果不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
gateway_error_from_engine(error)
|
||||
AccountProbeError::new(gateway_error_from_engine(error), upstream_response)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -651,7 +693,7 @@ impl AccountProbe for DefaultExecutionService {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move { self.probe_inner(request).await })
|
||||
}
|
||||
}
|
||||
@@ -664,10 +706,13 @@ struct AdmissionLease {
|
||||
|
||||
impl AdmissionLease {
|
||||
async fn release(self) {
|
||||
let _ = self
|
||||
if let Err(error) = self
|
||||
.port
|
||||
.release(&self.client_api_key_id, &self.model_request_id)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(%error, "Client admission 释放失败,依赖租约 TTL 收敛");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -742,13 +787,7 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
self.continuation_recorded = true;
|
||||
let record = self.continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
_ = record => {},
|
||||
_ = timeout => {},
|
||||
}
|
||||
record_native_continuation(self.continuation.as_ref(), pin).await;
|
||||
}
|
||||
|
||||
async fn finalize_detached(&mut self) {
|
||||
@@ -756,8 +795,10 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
core.cancel();
|
||||
if !core.is_finalized() {
|
||||
let _ = core.cancel_and_finalize().await;
|
||||
if !core.is_finalized()
|
||||
&& let Err(error) = core.cancel_and_finalize().await
|
||||
{
|
||||
tracing::warn!(%error, "Detached execution 终态收敛失败");
|
||||
}
|
||||
let pending = core
|
||||
.provider_attempt_outcomes()
|
||||
@@ -878,7 +919,32 @@ async fn publish_provider_attempt_outcomes(
|
||||
}
|
||||
Some(_) => continue,
|
||||
};
|
||||
let _ = result;
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(
|
||||
provider = outcome.provider_kind().as_str(),
|
||||
%error,
|
||||
"Provider circuit feedback 写入失败,数据面不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "Continuation affinity 写入失败,后续请求将退化为外部续接");
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!("Continuation affinity 后台写入超时,已丢弃本次亲和记录");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -232,6 +232,7 @@ pub struct AccountAttemptContext {
|
||||
required_account: Option<ProviderAccountId>,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
credential_recovery_attempted: bool,
|
||||
diagnostic_required_account: bool,
|
||||
}
|
||||
|
||||
impl AccountAttemptContext {
|
||||
@@ -246,6 +247,25 @@ impl AccountAttemptContext {
|
||||
required_account,
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 为管理端对固定账号的上游诊断创建上下文。
|
||||
///
|
||||
/// 诊断只绕过本地的被动可用性投影,不能换号,并继续受账号租约保护。
|
||||
#[must_use]
|
||||
pub const fn diagnostic(
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
required_account: ProviderAccountId,
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
excluded_accounts,
|
||||
required_account: Some(required_account),
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,6 +295,12 @@ impl AccountAttemptContext {
|
||||
pub const fn credential_recovery_attempted(&self) -> bool {
|
||||
self.credential_recovery_attempted
|
||||
}
|
||||
|
||||
/// 此尝试是否要由真实上游而非本地投影确认固定账号的可用性。
|
||||
#[must_use]
|
||||
pub const fn is_diagnostic_required_account(&self) -> bool {
|
||||
self.diagnostic_required_account
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求中 Provider 账号绑定状态的唯一归属。
|
||||
@@ -474,6 +500,12 @@ impl AttemptContext {
|
||||
self.account.credential_recovery_attempted()
|
||||
}
|
||||
|
||||
/// 管理端固定账号诊断会跳过本地可用性投影,保留全部租约约束。
|
||||
#[must_use]
|
||||
pub const fn is_diagnostic_required_account(&self) -> bool {
|
||||
self.account.is_diagnostic_required_account()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn continuation(&self) -> Option<&ContinuationBinding> {
|
||||
self.continuation.as_ref()
|
||||
@@ -583,6 +615,7 @@ pub struct ModelRequestFinalization {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub error: Option<GatewayError>,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 管理端账号连通性测试的真实执行链端口。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::error::GatewayError;
|
||||
use crate::error::{ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind};
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
use crate::{engine::credential::ProviderAccountId, operation::Operation};
|
||||
|
||||
@@ -19,9 +22,111 @@ pub struct AccountProbeResult {
|
||||
pub text: Vec<String>,
|
||||
}
|
||||
|
||||
/// 仅供当前管理端连接测试展示的原始上游失败响应。
|
||||
///
|
||||
/// 正文不进入 `Debug`、日志或持久化;Core 只在探测终态从原始 Provider 错误显式复制。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct AccountProbeUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl AccountProbeUpstreamResponse {
|
||||
pub(crate) fn from_client_response(response: &ClientVisibleUpstreamResponse) -> Self {
|
||||
Self {
|
||||
status: response.status(),
|
||||
content_type: response.content_type().map(<[u8]>::to_vec),
|
||||
body: response.body().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AccountProbeUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountProbeUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端连接测试的终态错误。
|
||||
///
|
||||
/// `gateway` 保留稳定分类,`upstream_response` 只面向本次认证管理请求展示源响应。
|
||||
#[derive(Debug)]
|
||||
pub struct AccountProbeError {
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
}
|
||||
|
||||
impl AccountProbeError {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
) -> Self {
|
||||
Self {
|
||||
gateway,
|
||||
upstream_response,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> GatewayErrorKind {
|
||||
self.gateway.kind()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_message(&self) -> &str {
|
||||
self.gateway.client_message()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_type(&self) -> Option<&str> {
|
||||
self.gateway.client_error_type()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_code(&self) -> Option<&str> {
|
||||
self.gateway.client_error_code()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_response(&self) -> Option<&AccountProbeUpstreamResponse> {
|
||||
self.upstream_response.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayError> for AccountProbeError {
|
||||
fn from(gateway: GatewayError) -> Self {
|
||||
Self::new(gateway, None)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AccountProbe: Send + Sync {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>>;
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>>;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::engine::credential::{
|
||||
};
|
||||
use crate::engine::{AttemptContext, UpstreamSendState};
|
||||
use crate::error::{
|
||||
IdentifierError, ProviderError, ProviderErrorKind, SafeUpstreamValue, validate_text,
|
||||
IdentifierError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, validate_text,
|
||||
};
|
||||
use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::operation::Operation;
|
||||
@@ -125,7 +125,7 @@ pub struct ProviderCallMetadata {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
resource: ProviderResource,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<OpaqueUpstreamValue>,
|
||||
transport: UpstreamTransport,
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 设置 adapter 已分类为非 bearer 的 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
@@ -186,7 +186,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 返回安全上游 request ID。
|
||||
#[must_use]
|
||||
pub const fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::engine::UpstreamSendState;
|
||||
use crate::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
/// 应用层标识不满足约束。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -174,22 +176,18 @@ pub enum ContinuationFailure {
|
||||
HistoryUnavailable,
|
||||
}
|
||||
|
||||
/// Adapter 已明确分类为非 bearer、可用于诊断的上游值。
|
||||
/// Adapter 已明确分类为非 bearer 的不透明上游值。
|
||||
///
|
||||
/// Core 不解释或校验内容,只通过自定义 [`Debug`] 避免它意外进入日志。协议 adapter
|
||||
/// 可在原客户端响应中读取原值。
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct SafeUpstreamValue(String);
|
||||
pub struct OpaqueUpstreamValue(String);
|
||||
|
||||
impl SafeUpstreamValue {
|
||||
const MAX_BYTES: usize = 256;
|
||||
|
||||
/// 校验并创建诊断值。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空值、超长值或控制字符会返回 [`IdentifierError`]。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, Self::MAX_BYTES, false, None)?;
|
||||
Ok(Self(value))
|
||||
impl OpaqueUpstreamValue {
|
||||
/// 原样保存上游值。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
/// 返回已经由 adapter 安全分类的原值。
|
||||
@@ -199,9 +197,9 @@ impl SafeUpstreamValue {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for SafeUpstreamValue {
|
||||
impl fmt::Debug for OpaqueUpstreamValue {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("SafeUpstreamValue(<redacted-from-Debug>)")
|
||||
formatter.write_str("OpaqueUpstreamValue(<redacted-from-Debug>)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,31 +210,23 @@ impl fmt::Debug for SafeUpstreamValue {
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamError {
|
||||
message: String,
|
||||
code: Option<SafeUpstreamValue>,
|
||||
error_type: Option<SafeUpstreamValue>,
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
error_type: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamError {
|
||||
const MAX_MESSAGE_BYTES: usize = 8_192;
|
||||
|
||||
/// 从结构化上游 `message`、`code` 和 `type` 创建客户端可见错误。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// message 为空、过长或含控制字符时返回错误;可选 code/type 不满足安全
|
||||
/// 约束时会被省略,不会把整段上游 body 重新暴露。
|
||||
/// 原样保存结构化上游 `message`、`code` 和 `type`。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
message: impl Into<String>,
|
||||
code: Option<String>,
|
||||
error_type: Option<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let message = message.into();
|
||||
validate_text(&message, Self::MAX_MESSAGE_BYTES, false, None)?;
|
||||
Ok(Self {
|
||||
message,
|
||||
code: code.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
error_type: error_type.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
})
|
||||
) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
code: code.map(OpaqueUpstreamValue::new),
|
||||
error_type: error_type.map(OpaqueUpstreamValue::new),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 message。
|
||||
@@ -248,13 +238,13 @@ impl ClientVisibleUpstreamError {
|
||||
/// 返回原上游的结构化 code。
|
||||
#[must_use]
|
||||
pub fn code(&self) -> Option<&str> {
|
||||
self.code.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.code.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 type。
|
||||
#[must_use]
|
||||
pub fn error_type(&self) -> Option<&str> {
|
||||
self.error_type.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.error_type.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,28 +254,110 @@ impl fmt::Debug for ClientVisibleUpstreamError {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
/// [`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象才能把响应交给
|
||||
/// 原客户端协议,或由认证管理端的账号连接测试显式复制到请求局部结果。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
headers: Vec<ProviderResponseHeader>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamResponse {
|
||||
/// 保存 transport 实际收到的状态码、Content-Type 原值和正文。
|
||||
#[must_use]
|
||||
pub fn new(status: u16, content_type: Option<Vec<u8>>, body: Bytes) -> Self {
|
||||
Self {
|
||||
status,
|
||||
content_type,
|
||||
headers: Vec::new(),
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
/// 附加 Provider 已剔除账号绑定与 framing 信息的普通响应头。
|
||||
#[must_use]
|
||||
pub fn with_headers(mut self, headers: Vec<ProviderResponseHeader>) -> Self {
|
||||
self.headers = headers;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.headers
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClientVisibleUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ClientVisibleUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("header_count", &self.headers.len())
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 单次 Provider 调用的稳定错误。
|
||||
///
|
||||
/// 该类型不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// 稳定诊断字段不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// code/request ID/response ID。Adapter 若捕获到可能含 secret 的上下文,只能调用
|
||||
/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记。
|
||||
#[derive(Clone)]
|
||||
///
|
||||
/// 唯一例外是 [`ProviderError::with_atomic_client_events`]:它只承载尚未交付客户端、
|
||||
/// 必须与本错误一起完成重试判断的协议事件。Core 会在任何持久化或日志记录前取走
|
||||
/// 这些事件,不能把它们当作诊断上下文使用。
|
||||
pub struct ProviderError {
|
||||
kind: ProviderErrorKind,
|
||||
send_state: UpstreamSendState,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_code: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_response_id: Option<SafeUpstreamValue>,
|
||||
upstream_values: Option<Box<ProviderErrorUpstreamValues>>,
|
||||
retry_after: Option<Duration>,
|
||||
continuation_failure: Option<ContinuationFailure>,
|
||||
replay_safe: bool,
|
||||
pre_delivery_retry: bool,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
sensitive_context_redacted: bool,
|
||||
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
|
||||
client_visible_upstream_response: Option<Box<ClientVisibleUpstreamResponse>>,
|
||||
atomic_client_events: Option<Box<AtomicClientEvents>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorUpstreamValues {
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
response_id: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AtomicClientEvents(Vec<ProviderEvent>);
|
||||
|
||||
impl ProviderError {
|
||||
/// 创建 Provider 错误。
|
||||
#[must_use]
|
||||
@@ -294,16 +366,17 @@ impl ProviderError {
|
||||
kind,
|
||||
send_state,
|
||||
upstream_status: None,
|
||||
upstream_code: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
upstream_values: None,
|
||||
retry_after: None,
|
||||
continuation_failure: None,
|
||||
replay_safe: false,
|
||||
pre_delivery_retry: false,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
sensitive_context_redacted: false,
|
||||
client_visible_upstream_error: None,
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,25 +391,30 @@ impl ProviderError {
|
||||
|
||||
/// 附加 adapter 已分类为安全的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn with_upstream_code(mut self, code: SafeUpstreamValue) -> Self {
|
||||
self.upstream_code = Some(code);
|
||||
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().code = Some(code);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_response_id(mut self, response_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_response_id = Some(response_id);
|
||||
pub fn with_upstream_response_id(mut self, response_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().response_id = Some(response_id);
|
||||
self
|
||||
}
|
||||
|
||||
fn upstream_values_mut(&mut self) -> &mut ProviderErrorUpstreamValues {
|
||||
self.upstream_values
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorUpstreamValues::default()))
|
||||
}
|
||||
|
||||
/// 附加 Provider 建议的冷却时间。
|
||||
#[must_use]
|
||||
pub const fn with_retry_after(mut self, retry_after: Duration) -> Self {
|
||||
@@ -358,6 +436,17 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复。
|
||||
///
|
||||
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
|
||||
/// Provider 选择了“客户端无感恢复优先”的传输策略。Core 在 continuation、
|
||||
/// 指定账号或下游已经进入提交状态时必须忽略它。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_retry(mut self) -> Self {
|
||||
self.pre_delivery_retry = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次。
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -379,6 +468,27 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加仅可返回给当前请求方的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn with_client_visible_upstream_response(
|
||||
mut self,
|
||||
response: ClientVisibleUpstreamResponse,
|
||||
) -> Self {
|
||||
self.client_visible_upstream_response = Some(Box::new(response));
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加必须先由 Core 判断重试、再决定丢弃或交付的客户端协议事件。
|
||||
///
|
||||
/// Provider 只能放入当前 attempt 尚未交付的事件;调用方应设置明确的有界缓冲,
|
||||
/// 并保证失败事件本身位于这批事件的末尾。
|
||||
#[must_use]
|
||||
pub fn with_atomic_client_events(mut self, events: Vec<ProviderEvent>) -> Self {
|
||||
self.atomic_client_events =
|
||||
(!events.is_empty()).then(|| Box::new(AtomicClientEvents(events)));
|
||||
self
|
||||
}
|
||||
|
||||
/// 丢弃敏感正文,只记录“发生过脱敏”这一事实。
|
||||
#[must_use]
|
||||
pub fn redact_sensitive_context(mut self, _sensitive_context: impl AsRef<str>) -> Self {
|
||||
@@ -406,20 +516,26 @@ impl ProviderError {
|
||||
|
||||
/// 返回安全分类的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn upstream_code(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_code.as_ref()
|
||||
pub fn upstream_code(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.code.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
pub fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.request_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn upstream_response_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_response_id.as_ref()
|
||||
pub fn upstream_response_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.response_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Provider 建议的冷却时间。
|
||||
@@ -440,6 +556,12 @@ impl ProviderError {
|
||||
self.replay_safe
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否允许在首个客户端事件前换号恢复传输失败。
|
||||
#[must_use]
|
||||
pub const fn allows_pre_delivery_retry(&self) -> bool {
|
||||
self.pre_delivery_retry
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否已完成凭据恢复并要求原账号重放。
|
||||
#[must_use]
|
||||
pub const fn retries_same_account(&self) -> bool {
|
||||
@@ -462,6 +584,50 @@ impl ProviderError {
|
||||
pub fn client_visible_upstream_error(&self) -> Option<&ClientVisibleUpstreamError> {
|
||||
self.client_visible_upstream_error.as_deref()
|
||||
}
|
||||
|
||||
/// 返回只供当前协议响应使用的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn client_visible_upstream_response(&self) -> Option<&ClientVisibleUpstreamResponse> {
|
||||
self.client_visible_upstream_response.as_deref()
|
||||
}
|
||||
|
||||
/// 取走只供本次重试/提交决策使用的原子客户端事件。
|
||||
///
|
||||
/// Core 必须在克隆错误、记录中间失败或构造终态前调用本方法,避免原始 wire
|
||||
/// 进入诊断或持久化对象。
|
||||
#[must_use]
|
||||
pub fn take_atomic_client_events(&mut self) -> Vec<ProviderEvent> {
|
||||
self.atomic_client_events
|
||||
.take()
|
||||
.map_or_else(Vec::new, |events| events.0)
|
||||
}
|
||||
|
||||
/// 返回是否携带尚未提交的原子客户端事件;不暴露事件内容。
|
||||
#[must_use]
|
||||
pub fn has_atomic_client_events(&self) -> bool {
|
||||
self.atomic_client_events.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ProviderError {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
kind: self.kind,
|
||||
send_state: self.send_state,
|
||||
upstream_status: self.upstream_status,
|
||||
upstream_values: self.upstream_values.clone(),
|
||||
retry_after: self.retry_after,
|
||||
continuation_failure: self.continuation_failure,
|
||||
replay_safe: self.replay_safe,
|
||||
pre_delivery_retry: self.pre_delivery_retry,
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
retry_same_account: self.retry_same_account,
|
||||
sensitive_context_redacted: self.sensitive_context_redacted,
|
||||
client_visible_upstream_error: self.client_visible_upstream_error.clone(),
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderError {
|
||||
@@ -473,25 +639,20 @@ impl fmt::Debug for ProviderError {
|
||||
.field("upstream_status", &self.upstream_status)
|
||||
.field(
|
||||
"upstream_code",
|
||||
&self.upstream_code.as_ref().map(|_| "<classified-safe>"),
|
||||
&self.upstream_code().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_request_id",
|
||||
&self
|
||||
.upstream_request_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_request_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_response_id",
|
||||
&self
|
||||
.upstream_response_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_response_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field("retry_after", &self.retry_after)
|
||||
.field("continuation_failure", &self.continuation_failure)
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
"credential_recovery_required",
|
||||
&self.credential_recovery_required,
|
||||
@@ -508,6 +669,20 @@ impl fmt::Debug for ProviderError {
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"client_visible_upstream_response",
|
||||
&self
|
||||
.client_visible_upstream_response
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"atomic_client_events",
|
||||
&self
|
||||
.atomic_client_events
|
||||
.as_ref()
|
||||
.map_or(0, |events| events.0.len()),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use thiserror::Error;
|
||||
|
||||
use crate::accounting::{CalculatedCost, ProviderReportedCost, Usage};
|
||||
use crate::engine::provider::UpstreamTransport;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::error::{IdentifierError, OpaqueUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
|
||||
/// 一次响应的稳定元数据。
|
||||
@@ -199,7 +199,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名或显式事件名为空、过长或含控制字符时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。上游事件名保持不透明。
|
||||
pub fn json(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -212,7 +212,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名、显式事件名或 SSE ID 不满足 wire 安全约束时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。SSE 元数据保持不透明。
|
||||
pub fn json_with_sse_metadata(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -222,15 +222,6 @@ impl ProtocolWireEvent {
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let protocol = protocol.into();
|
||||
validate_text(&protocol, 64, true, None)?;
|
||||
if let Some(event_type) = event_type.as_deref() {
|
||||
validate_text(event_type, 256, true, None)?;
|
||||
}
|
||||
if sse_id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id.contains(['\0', '\r', '\n']))
|
||||
{
|
||||
return Err(IdentifierError::ControlCharacter);
|
||||
}
|
||||
Ok(Self {
|
||||
protocol,
|
||||
event_type,
|
||||
@@ -440,24 +431,21 @@ impl fmt::Debug for ProviderResponseMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 已筛选、可由协议 adapter 再次按白名单表达的响应头。
|
||||
/// Provider 已筛选、可由协议 adapter 尝试表达的不透明响应头。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResponseHeader {
|
||||
name: String,
|
||||
value: SafeUpstreamValue,
|
||||
value: Bytes,
|
||||
}
|
||||
|
||||
impl ProviderResponseHeader {
|
||||
/// 创建规范化的小写响应头;非 ASCII token 名称直接拒绝。
|
||||
/// 保存 Provider 交付的原始名称和值;HTTP 可表达性由最终 adapter 判断。
|
||||
#[must_use]
|
||||
pub fn new(name: impl Into<String>, value: SafeUpstreamValue) -> Option<Self> {
|
||||
let name = name.into().trim().to_ascii_lowercase();
|
||||
let valid = !name.is_empty()
|
||||
&& name.len() <= 64
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-');
|
||||
valid.then_some(Self { name, value })
|
||||
pub fn new(name: impl Into<String>, value: Bytes) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -466,7 +454,7 @@ impl ProviderResponseHeader {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn value(&self) -> &SafeUpstreamValue {
|
||||
pub const fn value(&self) -> &Bytes {
|
||||
&self.value
|
||||
}
|
||||
}
|
||||
@@ -475,7 +463,7 @@ impl fmt::Debug for ProviderResponseHeader {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderResponseHeader")
|
||||
.field("name", &self.name)
|
||||
.field("name", &"<redacted>")
|
||||
.field("value", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
@@ -488,7 +476,8 @@ pub struct ProviderResponseObservation {
|
||||
http_version: Option<UpstreamHttpVersion>,
|
||||
websocket_pool: Option<WebSocketPoolKind>,
|
||||
status_code: Option<u16>,
|
||||
request_id: Option<SafeUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
service_tier: Option<String>,
|
||||
timings: ProviderResponseTimings,
|
||||
client_headers: Vec<ProviderResponseHeader>,
|
||||
provider_metadata: Option<ProviderResponseMetadata>,
|
||||
@@ -503,6 +492,7 @@ impl ProviderResponseObservation {
|
||||
websocket_pool: None,
|
||||
status_code: None,
|
||||
request_id: None,
|
||||
service_tier: None,
|
||||
timings: ProviderResponseTimings::default(),
|
||||
client_headers: Vec::new(),
|
||||
provider_metadata: None,
|
||||
@@ -528,11 +518,36 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 Provider 在响应中确认的服务档位。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 档位为空、超过 64 字节或包含控制字符时返回错误。
|
||||
pub fn try_with_service_tier(
|
||||
mut self,
|
||||
service_tier: impl Into<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let service_tier = service_tier.into();
|
||||
validate_text(&service_tier, 64, false, None)?;
|
||||
self.service_tier = Some(service_tier);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// 附加合法服务档位;不适合持久化的上游观测值会被忽略。
|
||||
#[must_use]
|
||||
pub fn with_service_tier_if_valid(mut self, service_tier: impl Into<String>) -> Self {
|
||||
let service_tier = service_tier.into();
|
||||
if validate_text(&service_tier, 64, false, None).is_ok() {
|
||||
self.service_tier = Some(service_tier);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_timings(mut self, timings: ProviderResponseTimings) -> Self {
|
||||
self.timings = timings;
|
||||
@@ -573,10 +588,15 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.request_id.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn service_tier(&self) -> Option<&str> {
|
||||
self.service_tier.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn timings(&self) -> ProviderResponseTimings {
|
||||
self.timings
|
||||
|
||||
@@ -302,7 +302,7 @@ impl GenerateRequest {
|
||||
self.body()
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|response_id| !response_id.is_empty())
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// 返回客户端是否请求了图片生成工具。
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::engine::credential::{
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use crate::error::{IdentifierError, validate_text};
|
||||
use crate::routing::ProviderKind;
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
@@ -527,6 +527,80 @@ impl ProviderCooldown {
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum ProviderCooldownScope {
|
||||
/// 只阻止同一账号调用指定上游模型。
|
||||
UpstreamModel(UpstreamModelId),
|
||||
}
|
||||
|
||||
impl ProviderCooldownScope {
|
||||
#[must_use]
|
||||
pub const fn upstream_model(model: UpstreamModelId) -> Self {
|
||||
Self::UpstreamModel(model)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UpstreamModel(_) => "model",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn value(&self) -> &str {
|
||||
match self {
|
||||
Self::UpstreamModel(model) => model.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不进入账号持久状态的账号+作用域 cooldown。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderScopedCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
}
|
||||
|
||||
impl ProviderScopedCooldown {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
scope,
|
||||
until,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &ProviderCooldownScope {
|
||||
&self.scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCooldownPort: Send + Sync {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
@@ -543,6 +617,24 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>;
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -575,32 +667,21 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按账号稳定扰动刷新提前量;相同账号与过期时间跨进程得到同一执行时刻。
|
||||
/// 按配置的精确提前量计算刷新时刻。
|
||||
pub fn next_attempt_at(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
_account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
let base_margin_seconds = self.margin.as_secs();
|
||||
if base_margin_seconds == 0 {
|
||||
if self.margin >= remaining {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
|
||||
let factor = stable_factor(account_id.as_str(), "normal", 850, 1_150);
|
||||
let jittered_seconds = base_margin_seconds
|
||||
.saturating_mul(u64::from(factor))
|
||||
.saturating_add(500)
|
||||
/ 1_000;
|
||||
if jittered_seconds >= remaining.as_secs() {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
let lead = jittered_seconds.max(1);
|
||||
access_token_expires_at
|
||||
.checked_sub(Duration::from_secs(lead))
|
||||
.checked_sub(self.margin)
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
@@ -778,35 +859,75 @@ pub enum OAuthPendingPutOutcome {
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
/// OAuth 回调处理取得临时 flow 的独占权结果。
|
||||
///
|
||||
/// 与一次性消费不同,Provider 在上游换取令牌失败时可以释放 claim,让同一 flow
|
||||
/// 使用新的回调地址重试;只有完整校验成功后才会消费 flow。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub enum OAuthPendingTakeOutcome {
|
||||
Taken(OpaqueProviderData),
|
||||
pub enum OAuthPendingClaimOutcome {
|
||||
Claimed(OpaqueProviderData),
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
InProgress,
|
||||
}
|
||||
|
||||
impl fmt::Debug for OAuthPendingTakeOutcome {
|
||||
impl fmt::Debug for OAuthPendingClaimOutcome {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Taken(_) => formatter.write_str("Taken([PROVIDER-OWNED])"),
|
||||
Self::Claimed(_) => formatter.write_str("Claimed([PROVIDER-OWNED])"),
|
||||
Self::NotFound => formatter.write_str("NotFound"),
|
||||
Self::OwnerMismatch => formatter.write_str("OwnerMismatch"),
|
||||
Self::InProgress => formatter.write_str("InProgress"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingReleaseOutcome {
|
||||
Released,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum OAuthPendingConsumeOutcome {
|
||||
Consumed,
|
||||
NotFound,
|
||||
OwnerMismatch,
|
||||
ClaimMismatch,
|
||||
}
|
||||
|
||||
pub trait OAuthPendingFlowPort: Send + Sync {
|
||||
fn put_if_absent(
|
||||
&self,
|
||||
flow: NewOAuthPendingFlow,
|
||||
) -> BoxFuture<'_, Result<OAuthPendingPutOutcome, ProviderStoreError>>;
|
||||
|
||||
fn take_if_owner<'a>(
|
||||
fn claim_if_owner<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingTakeOutcome, ProviderStoreError>>;
|
||||
claim: &'a OAuthPendingBinding,
|
||||
claim_ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingClaimOutcome, ProviderStoreError>>;
|
||||
|
||||
fn release_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingReleaseOutcome, ProviderStoreError>>;
|
||||
|
||||
fn consume_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> BoxFuture<'a, Result<OAuthPendingConsumeOutcome, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Provider 只能按能力取用端口,无法取得 Redis client 或 repository 集合。
|
||||
|
||||
@@ -89,6 +89,18 @@ impl UpstreamModelId {
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
/// 从未命中显式映射的客户端模型名构造同名上游模型。
|
||||
///
|
||||
/// 目录和配置仍使用 [`Self::new`] 的内部标识约束;客户端 wire 只要求非空,
|
||||
/// 具体模型字符串是否可用由绑定 Provider 决定。
|
||||
pub fn from_client_wire(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
if value.is_empty() {
|
||||
return Err(IdentifierError::Empty);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
|
||||
@@ -488,15 +488,11 @@ impl RuntimeSnapshot {
|
||||
|
||||
#[must_use]
|
||||
pub fn mapped_model(&self, requested: &str) -> String {
|
||||
let original = requested.trim();
|
||||
let original = requested;
|
||||
let mut current = original.to_owned();
|
||||
let mut seen = BTreeSet::new();
|
||||
for _ in 0..20 {
|
||||
let Some(target) = self
|
||||
.model_mappings
|
||||
.get(¤t)
|
||||
.map(|target| target.trim())
|
||||
else {
|
||||
let Some(target) = self.model_mappings.get(¤t).map(String::as_str) else {
|
||||
return current;
|
||||
};
|
||||
if !seen.insert(current.clone()) || seen.contains(target) {
|
||||
@@ -529,8 +525,14 @@ impl RuntimeSnapshot {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let upstream_model = UpstreamModelId::new(self.mapped_model(public_model.as_str()))
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let requested_model = public_model.as_str();
|
||||
let mapped_model = self.mapped_model(requested_model);
|
||||
let upstream_model = if self.model_mappings.contains_key(requested_model) {
|
||||
UpstreamModelId::new(mapped_model)
|
||||
} else {
|
||||
UpstreamModelId::from_client_wire(mapped_model)
|
||||
}
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let emulated_features = match self
|
||||
.provider_models
|
||||
.get(provider)
|
||||
|
||||
@@ -2,13 +2,12 @@ use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::error::SafeUpstreamValue;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
fn pin() -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new("response-private").expect("valid response"),
|
||||
SafeUpstreamValue::new("upstream-private").expect("valid upstream response"),
|
||||
PreviousResponseId::new("response-private"),
|
||||
PreviousResponseId::new("upstream-private"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
ProviderAccountId::new("acct_codex").expect("valid account"),
|
||||
)
|
||||
@@ -23,13 +22,20 @@ fn native_pin_debug_should_redact_previous_response_id() {
|
||||
|
||||
#[test]
|
||||
fn external_binding_debug_should_redact_previous_response_id() {
|
||||
let binding = ContinuationBinding::External(
|
||||
PreviousResponseId::new("external-private").expect("valid response"),
|
||||
);
|
||||
let binding = ContinuationBinding::External(PreviousResponseId::new("external-private"));
|
||||
|
||||
assert!(!format!("{binding:?}").contains("external-private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_response_id_preserves_any_codex_string_without_debug_disclosure() {
|
||||
let value = format!("resp_{}\0opaque", "x".repeat(257));
|
||||
let response_id = PreviousResponseId::new(value.clone());
|
||||
|
||||
assert_eq!(response_id.as_str(), value);
|
||||
assert!(!format!("{response_id:?}").contains(&value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_pin_should_reject_different_account() {
|
||||
assert!(!pin().matches(
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::executor::block_on;
|
||||
|
||||
@@ -28,8 +29,8 @@ use gateway_core::engine::{
|
||||
ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ContinuationFailure, GatewayErrorKind, ProviderError, ProviderErrorKind, SafeUpstreamValue,
|
||||
StoreError,
|
||||
ClientVisibleUpstreamResponse, ContinuationFailure, GatewayErrorKind, OpaqueUpstreamValue,
|
||||
ProviderError, ProviderErrorKind, StoreError, StoreErrorKind,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -65,6 +66,7 @@ struct FinalState {
|
||||
upstream_transport: Option<String>,
|
||||
http_version: Option<String>,
|
||||
websocket_pool: Option<String>,
|
||||
service_tier: Option<String>,
|
||||
upstream_request_id: Option<String>,
|
||||
upstream_status_code: Option<u16>,
|
||||
transport_decision_wait_ms: Option<u64>,
|
||||
@@ -97,11 +99,37 @@ struct StoreState {
|
||||
#[derive(Default)]
|
||||
struct FakeStore {
|
||||
state: Mutex<StoreState>,
|
||||
failures: BTreeSet<StoreWriteFailure>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum StoreWriteFailure {
|
||||
Create,
|
||||
Commit,
|
||||
Finalize,
|
||||
}
|
||||
|
||||
impl FakeStore {
|
||||
fn failing(failure: StoreWriteFailure) -> Self {
|
||||
Self {
|
||||
state: Mutex::new(StoreState::default()),
|
||||
failures: BTreeSet::from([failure]),
|
||||
}
|
||||
}
|
||||
|
||||
fn fails(&self, failure: StoreWriteFailure) -> Result<(), StoreError> {
|
||||
if self.failures.contains(&failure) {
|
||||
Err(StoreError::new(StoreErrorKind::Unavailable))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionStore for FakeStore {
|
||||
async fn create_model_request(&self, _request: NewModelRequest) -> Result<(), StoreError> {
|
||||
self.fails(StoreWriteFailure::Create)?;
|
||||
self.state.lock().expect("store lock").created += 1;
|
||||
Ok(())
|
||||
}
|
||||
@@ -134,6 +162,7 @@ impl ExecutionStore for FakeStore {
|
||||
_committed_at: SystemTime,
|
||||
client_status_code: Option<u16>,
|
||||
) -> Result<(), StoreError> {
|
||||
self.fails(StoreWriteFailure::Commit)?;
|
||||
let mut state = self.state.lock().expect("store lock");
|
||||
state.commits += 1;
|
||||
state.committed_statuses.push(client_status_code);
|
||||
@@ -172,6 +201,7 @@ impl ExecutionStore for FakeStore {
|
||||
&self,
|
||||
finalization: ModelRequestFinalization,
|
||||
) -> Result<(), StoreError> {
|
||||
self.fails(StoreWriteFailure::Finalize)?;
|
||||
self.state
|
||||
.lock()
|
||||
.expect("store lock")
|
||||
@@ -194,6 +224,7 @@ impl ExecutionStore for FakeStore {
|
||||
upstream_transport: finalization.upstream_transport,
|
||||
http_version: finalization.http_version,
|
||||
websocket_pool: finalization.websocket_pool,
|
||||
service_tier: finalization.service_tier,
|
||||
upstream_request_id: finalization.upstream_request_id,
|
||||
upstream_status_code: finalization.upstream_status_code,
|
||||
transport_decision_wait_ms: finalization.timings.transport_decision_wait_ms,
|
||||
@@ -291,15 +322,11 @@ impl Provider for ScriptedProvider {
|
||||
},
|
||||
UpstreamTransport::new("http_sse").expect("transport"),
|
||||
)
|
||||
.with_upstream_request_id(
|
||||
SafeUpstreamValue::new("upstream-request").expect("safe request id"),
|
||||
);
|
||||
.with_upstream_request_id(OpaqueUpstreamValue::new("upstream-request"));
|
||||
Ok(ProviderStream::new(
|
||||
metadata,
|
||||
Box::pin(futures::stream::iter(
|
||||
items
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::from)),
|
||||
items.into_iter().map(canonical_provider_event),
|
||||
)),
|
||||
(),
|
||||
))
|
||||
@@ -318,12 +345,8 @@ impl Provider for ScriptedProvider {
|
||||
Ok(ProviderStream::new(
|
||||
metadata,
|
||||
Box::pin(
|
||||
futures::stream::iter(
|
||||
items
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::from)),
|
||||
)
|
||||
.chain(futures::stream::pending()),
|
||||
futures::stream::iter(items.into_iter().map(canonical_provider_event))
|
||||
.chain(futures::stream::pending()),
|
||||
),
|
||||
(),
|
||||
))
|
||||
@@ -399,6 +422,75 @@ fn complete_stream(total_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Provid
|
||||
events
|
||||
}
|
||||
|
||||
fn canonical_provider_event(
|
||||
event: Result<GatewayEvent, ProviderError>,
|
||||
) -> Result<ProviderEvent, ProviderError> {
|
||||
event.map(ProviderEvent::canonical)
|
||||
}
|
||||
|
||||
fn atomic_response_failed(response_id: &str, marker: &str) -> ProviderError {
|
||||
let started = ProviderEvent::canonical_with_wire(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
response_id,
|
||||
"gpt-5",
|
||||
))],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": response_id, "model": "gpt-5", "status": "in_progress"}
|
||||
}),
|
||||
)
|
||||
.expect("started wire"),
|
||||
);
|
||||
let failed = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"status": "failed",
|
||||
"error": {"code": "rate_limit_exceeded", "message": marker}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("failed wire"),
|
||||
);
|
||||
ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_status(429)
|
||||
.with_replay_safe()
|
||||
.with_atomic_client_events(vec![started, failed])
|
||||
}
|
||||
|
||||
fn bare_atomic_response_failed(response_id: &str) -> ProviderError {
|
||||
let failed = ProviderEvent::canonical_with_wire(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
response_id,
|
||||
"fallback-model",
|
||||
))],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"status": "failed",
|
||||
"error": {"code": "rate_limit_exceeded", "message": "bare failure"}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("bare failed wire"),
|
||||
);
|
||||
ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_status(429)
|
||||
.with_replay_safe()
|
||||
.with_atomic_client_events(vec![failed])
|
||||
}
|
||||
|
||||
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
|
||||
let mut usage = Usage::new();
|
||||
usage.input_tokens = Some(12);
|
||||
@@ -484,6 +576,17 @@ fn coordinator(
|
||||
Arc<ScriptedProvider>,
|
||||
) {
|
||||
let store = Arc::new(FakeStore::default());
|
||||
coordinator_with_store(scripts, store)
|
||||
}
|
||||
|
||||
fn coordinator_with_store(
|
||||
scripts: Vec<Script>,
|
||||
store: Arc<FakeStore>,
|
||||
) -> (
|
||||
AttemptCoordinator<FakeStore>,
|
||||
Arc<FakeStore>,
|
||||
Arc<ScriptedProvider>,
|
||||
) {
|
||||
let provider = Arc::new(ScriptedProvider::new(scripts));
|
||||
let mut registry = ProviderRegistry::builder();
|
||||
registry
|
||||
@@ -587,6 +690,170 @@ fn success_updates_one_model_request_and_persists_usage() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_store_failures_never_block_successful_delivery() {
|
||||
for failure in [
|
||||
StoreWriteFailure::Create,
|
||||
StoreWriteFailure::Commit,
|
||||
StoreWriteFailure::Finalize,
|
||||
] {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let store = Arc::new(FakeStore::failing(failure));
|
||||
let (coordinator, _, provider) = coordinator_with_store(
|
||||
vec![Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: complete_stream(None),
|
||||
}],
|
||||
store,
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let events = block_on(session.collect_uncommitted())
|
||||
.unwrap_or_else(|error| panic!("{failure:?} blocked collection: {error}"));
|
||||
assert_eq!(events.len(), 2);
|
||||
block_on(session.commit_downstream(Some(200)))
|
||||
.unwrap_or_else(|error| panic!("{failure:?} blocked commit: {error}"));
|
||||
assert!(session.is_finalized());
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terminal_provider_error_preserves_request_local_upstream_response() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let raw_body = Bytes::from_static(b"{\"upstream\":\"unchanged\"}\x00");
|
||||
let provider_error =
|
||||
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
422,
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
raw_body.clone(),
|
||||
));
|
||||
let (coordinator, _, _) = coordinator(vec![Script::Error(provider_error)]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let error = block_on(session.collect_uncommitted()).expect_err("provider rejection");
|
||||
let EngineError::Provider(error) = error else {
|
||||
panic!("expected provider error");
|
||||
};
|
||||
let response = error
|
||||
.client_visible_upstream_response()
|
||||
.expect("request-local upstream response");
|
||||
|
||||
assert_eq!(response.status(), 422);
|
||||
assert_eq!(response.body(), &raw_body);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_preserves_opaque_upstream_response_id_and_builds_native_pin() {
|
||||
let response_id = format!("resp_{}\0opaque", "x".repeat(4_096));
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![
|
||||
Ok(GatewayEvent::Started(ResponseMeta::new(
|
||||
response_id.clone(),
|
||||
"gpt-5",
|
||||
))),
|
||||
Ok(GatewayEvent::Completed(ResponseMeta::new(
|
||||
response_id.clone(),
|
||||
"gpt-5",
|
||||
))),
|
||||
],
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
block_on(session.collect_uncommitted()).expect("collect opaque response");
|
||||
let provider_state = ProviderSessionState::new("openai", Map::new()).expect("provider state");
|
||||
let pin = session
|
||||
.native_continuation_pin(&provider_state)
|
||||
.expect("native continuation pin");
|
||||
assert_eq!(pin.previous_response_id().as_str(), response_id);
|
||||
assert_eq!(pin.upstream_response_id().as_str(), response_id);
|
||||
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(
|
||||
state.finalizations[0].client_response_id,
|
||||
Some(response_id.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
state.finalizations[0].upstream_response_id,
|
||||
Some(response_id)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_identity_change_and_empty_terminal_id_are_observational() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::Stream {
|
||||
account_id: "acct_one",
|
||||
items: vec![
|
||||
Ok(GatewayEvent::Started(ResponseMeta::new(
|
||||
"response-created",
|
||||
"gpt-5",
|
||||
))),
|
||||
Ok(GatewayEvent::Completed(ResponseMeta::new("", "gpt-5"))),
|
||||
],
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
block_on(session.collect_uncommitted()).expect("identity observation cannot fail delivery");
|
||||
let provider_state = ProviderSessionState::new("openai", Map::new()).expect("provider state");
|
||||
let pin = session
|
||||
.native_continuation_pin(&provider_state)
|
||||
.expect("empty opaque response ID remains recordable");
|
||||
assert_eq!(pin.previous_response_id().as_str(), "");
|
||||
assert_eq!(pin.upstream_response_id().as_str(), "");
|
||||
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(
|
||||
state.finalizations[0].client_response_id.as_deref(),
|
||||
Some("")
|
||||
);
|
||||
assert_eq!(
|
||||
state.finalizations[0].upstream_response_id.as_deref(),
|
||||
Some("")
|
||||
);
|
||||
}
|
||||
|
||||
fn finalized_image_request(image_output_tokens: Option<u64>) -> FinalState {
|
||||
let operation = image_generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
@@ -669,7 +936,9 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
.with_http_version(UpstreamHttpVersion::Http2)
|
||||
.with_websocket_pool(WebSocketPoolKind::New)
|
||||
.with_status_code(200)
|
||||
.with_request_id(SafeUpstreamValue::new("upstream-observed").expect("request id"))
|
||||
.with_request_id(OpaqueUpstreamValue::new("upstream-observed"))
|
||||
.try_with_service_tier("priority")
|
||||
.expect("service tier")
|
||||
.with_timings(ProviderResponseTimings {
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
@@ -681,7 +950,7 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
items.extend(
|
||||
complete_stream(Some(21))
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::canonical)),
|
||||
.map(canonical_provider_event),
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::ObservedStream {
|
||||
account_id: "acct_observed",
|
||||
@@ -706,6 +975,7 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
assert_eq!(finalization.upstream_transport.as_deref(), Some("http_sse"));
|
||||
assert_eq!(finalization.http_version.as_deref(), Some("HTTP/2"));
|
||||
assert_eq!(finalization.websocket_pool.as_deref(), Some("new"));
|
||||
assert_eq!(finalization.service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(
|
||||
finalization.upstream_request_id.as_deref(),
|
||||
Some("upstream-observed")
|
||||
@@ -717,6 +987,48 @@ fn response_observation_is_persisted_but_never_delivered() {
|
||||
assert_eq!(finalization.first_event_ms, Some(17));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mismatched_response_observation_should_not_interrupt_client_events() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let first = ProviderEvent::observation(ProviderResponseObservation::new(
|
||||
UpstreamTransport::new("websocket").expect("first transport"),
|
||||
));
|
||||
let mismatched = ProviderEvent::observation(ProviderResponseObservation::new(
|
||||
UpstreamTransport::new("http_sse").expect("mismatched transport"),
|
||||
));
|
||||
let mut items = vec![Ok(first), Ok(mismatched)];
|
||||
items.extend(
|
||||
complete_stream(None)
|
||||
.into_iter()
|
||||
.map(canonical_provider_event),
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::ObservedStream {
|
||||
account_id: "acct_observation_mismatch",
|
||||
items,
|
||||
}]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let events = block_on(session.collect_uncommitted()).expect("client events survive");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
assert_eq!(events.len(), 2);
|
||||
assert_eq!(
|
||||
store.state.lock().expect("store lock").finalizations[0]
|
||||
.upstream_transport
|
||||
.as_deref(),
|
||||
Some("websocket")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_observation_should_persist_provider_metadata_and_output_timings() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
@@ -741,7 +1053,7 @@ fn response_observation_should_persist_provider_metadata_and_output_timings() {
|
||||
items.extend(
|
||||
complete_stream(Some(21))
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::canonical)),
|
||||
.map(canonical_provider_event),
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![Script::ObservedStream {
|
||||
account_id: "acct_observed",
|
||||
@@ -881,7 +1193,7 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
)
|
||||
.with_http_version(UpstreamHttpVersion::Http11)
|
||||
.with_status_code(503)
|
||||
.with_request_id(SafeUpstreamValue::new("discarded-request").expect("request id"))
|
||||
.with_request_id(OpaqueUpstreamValue::new("discarded-request"))
|
||||
.with_timings(ProviderResponseTimings {
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
@@ -895,12 +1207,12 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
)
|
||||
.with_http_version(UpstreamHttpVersion::Http2)
|
||||
.with_status_code(200)
|
||||
.with_request_id(SafeUpstreamValue::new("winning-request").expect("request id"));
|
||||
.with_request_id(OpaqueUpstreamValue::new("winning-request"));
|
||||
let mut second_items = vec![Ok(ProviderEvent::observation(second_observation))];
|
||||
second_items.extend(
|
||||
complete_stream(None)
|
||||
.into_iter()
|
||||
.map(|event| event.map(ProviderEvent::canonical)),
|
||||
.map(canonical_provider_event),
|
||||
);
|
||||
let (coordinator, store, _) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
@@ -1029,8 +1341,8 @@ fn authenticated_native_continuation_reaches_every_attempt_context() {
|
||||
items: complete_stream(None),
|
||||
}]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id").expect("previous response"),
|
||||
SafeUpstreamValue::new("provider-native-id").expect("upstream response"),
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1086,8 +1398,8 @@ fn streaming_native_continuation_delivers_started_event_before_later_output() {
|
||||
items: complete_stream(None),
|
||||
}]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response").expect("previous response"),
|
||||
SafeUpstreamValue::new("upstream-response").expect("upstream response"),
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1162,8 +1474,8 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
},
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id").expect("previous response"),
|
||||
SafeUpstreamValue::new("provider-native-id").expect("upstream response"),
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1202,7 +1514,7 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, _, provider) = coordinator(vec![Script::Stream {
|
||||
@@ -1210,11 +1522,11 @@ fn required_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
items: complete_stream(None),
|
||||
}]);
|
||||
let required = ProviderAccountId::new("acct_required").expect("account id");
|
||||
let mut session = block_on(coordinator.start(
|
||||
let mut session = block_on(coordinator.start_diagnostic(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
Some(required.clone()),
|
||||
required.clone(),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
@@ -1225,6 +1537,7 @@ fn required_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 1);
|
||||
assert_eq!(contexts[0].required_account(), Some(&required));
|
||||
assert!(contexts[0].is_diagnostic_required_account());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1709,9 +2022,7 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_upstream_code(
|
||||
SafeUpstreamValue::new("rate_limit_exceeded").expect("provider code"),
|
||||
)
|
||||
.with_upstream_code(OpaqueUpstreamValue::new("rate_limit_exceeded"))
|
||||
.with_retry_after(Duration::from_secs(30))
|
||||
.with_replay_safe())],
|
||||
},
|
||||
@@ -1739,7 +2050,7 @@ fn rate_limited_account_exhaustion_survives_a_later_empty_selection() {
|
||||
assert_eq!(error.kind(), ProviderErrorKind::RateLimited);
|
||||
assert_eq!(error.upstream_status(), Some(429));
|
||||
assert_eq!(
|
||||
error.upstream_code().map(SafeUpstreamValue::as_str),
|
||||
error.upstream_code().map(OpaqueUpstreamValue::as_str),
|
||||
Some("rate_limit_exceeded")
|
||||
);
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(30)));
|
||||
@@ -1836,11 +2147,282 @@ fn explicit_429_after_structural_event_should_retry_before_commit() {
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_response_failed_should_rotate_account_before_stream_commit() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(atomic_response_failed(
|
||||
"response-first-failed",
|
||||
"first account failure",
|
||||
))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let first = block_on(session.next_event())
|
||||
.expect("second account should produce the first delivery")
|
||||
.expect("started event");
|
||||
let delivered = first.into_provider_events();
|
||||
assert!(delivered.iter().flat_map(ProviderEvent::canonical_facts).any(
|
||||
|event| matches!(event, GatewayEvent::Started(meta) if meta.response_id() == "response-1")
|
||||
));
|
||||
assert!(
|
||||
!delivered
|
||||
.iter()
|
||||
.filter_map(ProviderEvent::wire_event)
|
||||
.any(|wire| {
|
||||
wire.data().pointer("/response/id").and_then(Value::as_str)
|
||||
== Some("response-first-failed")
|
||||
})
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit second account");
|
||||
while block_on(session.next_event())
|
||||
.expect("successful stream")
|
||||
.is_some()
|
||||
{}
|
||||
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.commits, 1);
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_atomic_response_failed_should_rotate_before_the_first_delivery() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(bare_atomic_response_failed("response-bare-failed"))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let first = block_on(session.next_event())
|
||||
.expect("second account should replace the bare failure")
|
||||
.expect("second account started event");
|
||||
assert!(first
|
||||
.into_provider_events()
|
||||
.iter()
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.any(|event| matches!(event, GatewayEvent::Started(meta) if meta.response_id() == "response-1")));
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit second account");
|
||||
while block_on(session.next_event())
|
||||
.expect("successful stream")
|
||||
.is_some()
|
||||
{}
|
||||
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_atomic_response_failed_should_deliver_only_the_last_failure_once() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let scripts = (1..=route_plan.max_attempts().get())
|
||||
.map(|attempt| {
|
||||
let response_id = format!("response-attempt-{attempt}-failed");
|
||||
let marker = if attempt == route_plan.max_attempts().get() {
|
||||
"last upstream failure".to_owned()
|
||||
} else {
|
||||
format!("discarded failure {attempt}")
|
||||
};
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_failed",
|
||||
items: vec![Err(atomic_response_failed(&response_id, &marker))],
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let expected_attempts = route_plan.max_attempts().get();
|
||||
let expected_last_response = format!("response-attempt-{expected_attempts}-failed");
|
||||
let (coordinator, store, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let terminal = block_on(session.next_event())
|
||||
.expect("last failure remains deliverable")
|
||||
.expect("terminal failure batch");
|
||||
assert_eq!(
|
||||
terminal.commit_requirement(),
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
);
|
||||
let events = terminal.into_provider_events();
|
||||
let failures = events
|
||||
.iter()
|
||||
.filter_map(ProviderEvent::wire_event)
|
||||
.filter(|wire| wire.event_type() == Some("response.failed"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert_eq!(
|
||||
failures[0]
|
||||
.data()
|
||||
.pointer("/response/id")
|
||||
.and_then(Value::as_str),
|
||||
Some(expected_last_response.as_str())
|
||||
);
|
||||
assert!(
|
||||
failures[0]
|
||||
.data()
|
||||
.to_string()
|
||||
.contains("last upstream failure")
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit original terminal failure");
|
||||
let error = block_on(session.next_event()).expect_err("typed failure finalizes the request");
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
assert!(session.is_finalized());
|
||||
|
||||
assert_eq!(
|
||||
provider.contexts.lock().expect("contexts lock").len(),
|
||||
usize::try_from(expected_attempts).expect("attempt count fits usize")
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(
|
||||
state.intermediate_failures,
|
||||
usize::try_from(expected_attempts - 1).expect("attempt count fits usize")
|
||||
);
|
||||
assert_eq!(state.commits, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Incomplete);
|
||||
assert!(state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_selection_after_atomic_failure_should_deliver_the_last_upstream_wire() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(bare_atomic_response_failed("response-only-failed"))],
|
||||
},
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let terminal = block_on(session.next_event())
|
||||
.expect("last account failure remains deliverable")
|
||||
.expect("terminal failure batch");
|
||||
let events = terminal.into_provider_events();
|
||||
let failure = events
|
||||
.iter()
|
||||
.filter_map(ProviderEvent::wire_event)
|
||||
.find(|wire| wire.event_type() == Some("response.failed"))
|
||||
.expect("last upstream response.failed");
|
||||
assert_eq!(
|
||||
failure
|
||||
.data()
|
||||
.pointer("/response/id")
|
||||
.and_then(Value::as_str),
|
||||
Some("response-only-failed")
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit terminal failure");
|
||||
assert!(matches!(
|
||||
block_on(session.next_event()),
|
||||
Err(EngineError::Provider(_))
|
||||
));
|
||||
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Incomplete);
|
||||
assert_eq!(
|
||||
state.finalizations[0].client_response_id.as_deref(),
|
||||
Some("response-only-failed")
|
||||
);
|
||||
assert_eq!(
|
||||
state.finalizations[0].upstream_response_id.as_deref(),
|
||||
Some("response-only-failed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_response_failed_keeps_collect_uncommitted_error_semantics() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![Script::ObservedStream {
|
||||
account_id: "acct_required",
|
||||
items: vec![Err(atomic_response_failed(
|
||||
"response-required-failed",
|
||||
"required account failure",
|
||||
))],
|
||||
}]);
|
||||
let required = ProviderAccountId::new("acct_required").expect("required account");
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
Some(required),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("non-streaming collection must keep returning the typed failure");
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
assert!(session.is_finalized());
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 1);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.commits, 0);
|
||||
assert_eq!(state.intermediate_failures, 0);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_explicit_429_is_not_retried() {
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id").expect("previous response"),
|
||||
SafeUpstreamValue::new("provider-native-id").expect("upstream response"),
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_first").expect("account"),
|
||||
);
|
||||
@@ -1861,9 +2443,8 @@ fn native_continuation_explicit_429_is_not_retried() {
|
||||
|
||||
#[test]
|
||||
fn external_continuation_explicit_429_is_not_retried() {
|
||||
let continuation = ContinuationBinding::External(
|
||||
PreviousResponseId::new("external-provider-response").expect("previous response"),
|
||||
);
|
||||
let opaque_response_id = format!("resp_{}\0opaque", "x".repeat(257));
|
||||
let continuation = ContinuationBinding::External(PreviousResponseId::new(opaque_response_id));
|
||||
let (store, provider) = terminal_non_idempotent_failure(
|
||||
vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
@@ -1930,6 +2511,100 @@ fn ambiguous_send_state_stops_retry() {
|
||||
assert!(finalization.latency_ms.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_delivery_transport_failure_rotates_account_for_non_idempotent_generation() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)
|
||||
.with_pre_delivery_retry())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("second account succeeds before delivery");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert!(
|
||||
contexts[1]
|
||||
.excluded_accounts()
|
||||
.contains(&ProviderAccountId::new("acct_first").expect("account id"))
|
||||
);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_delivery_retry_marker_is_ignored_after_downstream_commit() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![
|
||||
Ok(GatewayEvent::Started(ResponseMeta::new(
|
||||
"response-visible",
|
||||
"gpt-5",
|
||||
))),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_retry(),
|
||||
),
|
||||
],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let first = block_on(session.next_event())
|
||||
.expect("first event")
|
||||
.expect("visible event");
|
||||
assert_eq!(
|
||||
first.commit_requirement(),
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit first event");
|
||||
let error = block_on(session.next_event()).expect_err("committed output cannot be replayed");
|
||||
|
||||
assert!(matches!(error, EngineError::Provider(_)));
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 1);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 0);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Incomplete);
|
||||
assert!(state.finalizations[0].committed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_event_before_replay_safe_failure_should_switch_account_before_commit() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
|
||||
@@ -5,10 +5,11 @@ use std::time::{Duration, SystemTime};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountCandidate, AccountFeedbackStats,
|
||||
AccountQuotaSignals, AccountRuntimeSignals, AccountSelectionContext, AccountSelectionPolicy,
|
||||
AccountSelector, CredentialCasUpdate, CredentialRevision, OpaqueProviderData,
|
||||
PlaintextCredential, ProviderAccount, ProviderAccountId, ProviderAccountUpdate,
|
||||
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
|
||||
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
|
||||
RotationStrategy,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -51,6 +52,7 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
preferred_account: None,
|
||||
round_robin_cursor: 0,
|
||||
availability: AccountAvailabilityPolicy::Enforce,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +74,18 @@ fn opaque_provider_data_should_not_expose_quota_values_in_debug() {
|
||||
assert!(!format!("{quota:?}").contains("private-window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_account_identity_debug_should_redact_upstream_ids() {
|
||||
let identity = ProviderAccountIdentity::new(
|
||||
"private-user-id".to_owned(),
|
||||
Some("private-account-id".to_owned()),
|
||||
);
|
||||
let debug = format!("{identity:?}");
|
||||
|
||||
assert!(!debug.contains("private-user-id"));
|
||||
assert!(!debug.contains("private-account-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn availability_should_round_trip_all_database_values() {
|
||||
assert_eq!(
|
||||
@@ -80,6 +94,55 @@ fn availability_should_round_trip_all_database_values() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let mut context = context(RotationStrategy::Sticky);
|
||||
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
|
||||
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(std::slice::from_ref(&exhausted), &context)
|
||||
.map(|selection| selection.candidate().account.id()),
|
||||
Some(exhausted.account.id())
|
||||
);
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
.select(std::slice::from_ref(&disabled), &context)
|
||||
.map(|selection| selection.candidate().account.id()),
|
||||
Some(disabled.account.id())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account =
|
||||
@@ -119,7 +182,7 @@ fn smart_selector_should_prefer_lower_inflight_count() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_idle");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_idle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -136,15 +199,13 @@ fn smart_selector_should_balance_signals_instead_of_using_lexicographic_load() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_healthy");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_keep_healthy_preferred_account_at_escape_boundary() {
|
||||
fn smart_selector_should_report_a_schedulable_preferred_account_hit() {
|
||||
let fallback = candidate("acct_fallback", 0, Some(100));
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(1));
|
||||
preferred.signals.failure_rate_basis_points = Some(5_000);
|
||||
preferred.signals.first_output_latency_ms = Some(15_000);
|
||||
let preferred = candidate("acct_preferred", 0, Some(1));
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
selection.preferred_account = Some(preferred_id);
|
||||
@@ -154,16 +215,22 @@ fn smart_selector_should_keep_healthy_preferred_account_at_escape_boundary() {
|
||||
.select(&candidates, &selection)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_preferred");
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
("acct_preferred", PreferredAccountSelection::Hit)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_escape_preferred_account_after_failure_threshold() {
|
||||
fn smart_selector_should_keep_preferred_account_despite_failure_signals() {
|
||||
let mut fallback = candidate("acct_fallback", 0, Some(100));
|
||||
fallback.signals.failure_rate_basis_points = Some(0);
|
||||
fallback.signals.first_output_latency_ms = Some(100);
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(100));
|
||||
preferred.signals.failure_rate_basis_points = Some(5_001);
|
||||
preferred.signals.failure_rate_basis_points = Some(10_000);
|
||||
preferred.signals.first_output_latency_ms = Some(100);
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
@@ -172,19 +239,19 @@ fn smart_selector_should_escape_preferred_account_after_failure_threshold() {
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
.expect("preferred candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_preferred");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_escape_preferred_account_after_latency_threshold() {
|
||||
fn smart_selector_should_keep_preferred_account_despite_latency_signals() {
|
||||
let mut fallback = candidate("acct_fallback", 0, Some(100));
|
||||
fallback.signals.failure_rate_basis_points = Some(0);
|
||||
fallback.signals.first_output_latency_ms = Some(100);
|
||||
let mut preferred = candidate("acct_preferred", 0, Some(100));
|
||||
preferred.signals.failure_rate_basis_points = Some(0);
|
||||
preferred.signals.first_output_latency_ms = Some(15_001);
|
||||
preferred.signals.first_output_latency_ms = Some(120_000);
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
selection.preferred_account = Some(preferred_id);
|
||||
@@ -192,9 +259,9 @@ fn smart_selector_should_escape_preferred_account_after_latency_threshold() {
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
.expect("preferred candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_preferred");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -210,7 +277,16 @@ fn smart_selector_should_escape_preferred_account_at_concurrency_limit() {
|
||||
.select(&candidates, &selection)
|
||||
.expect("fallback candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
(
|
||||
"acct_fallback",
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::ConcurrencyLimit),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -296,7 +372,10 @@ fn smart_selector_should_use_provider_quota_rank_after_load_is_equal() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_high_quota");
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_high_quota"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -312,7 +391,7 @@ fn quota_reset_selector_should_prefer_known_earliest_window() {
|
||||
.select(&candidates, &context(RotationStrategy::QuotaResetPriority))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_earlier");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_earlier");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -325,7 +404,7 @@ fn smart_selector_should_prefer_known_quota_over_unknown_after_load_is_equal() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_known");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_known");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -338,7 +417,7 @@ fn selector_should_reject_account_at_concurrency_limit() {
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_available");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -360,7 +439,7 @@ fn selector_should_enforce_request_interval_until_boundary() {
|
||||
.select(&candidates, &context)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_ready");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_ready");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -377,7 +456,7 @@ fn sticky_selector_should_fall_back_when_requested_account_is_excluded() {
|
||||
.select(&candidates, &context)
|
||||
.expect("fallback candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_fallback");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_fallback");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -397,11 +476,38 @@ fn round_robin_selector_should_be_stable_for_unsorted_candidates() {
|
||||
.expect("second candidate available");
|
||||
|
||||
assert_eq!(
|
||||
(first.account.id().as_str(), second.account.id().as_str()),
|
||||
(
|
||||
first.candidate().account.id().as_str(),
|
||||
second.candidate().account.id().as_str(),
|
||||
),
|
||||
("acct_first", "acct_second")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_robin_selector_should_honor_a_schedulable_preferred_account_before_the_cursor() {
|
||||
let candidates = vec![
|
||||
candidate("acct_first", 0, None),
|
||||
candidate("acct_second", 0, None),
|
||||
];
|
||||
let mut context = context(RotationStrategy::RoundRobin);
|
||||
context.round_robin_cursor = 0;
|
||||
context.preferred_account =
|
||||
Some(ProviderAccountId::new("acct_second").expect("preferred account"));
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.expect("preferred account available");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
selected.candidate().account.id().as_str(),
|
||||
selected.preferred(),
|
||||
),
|
||||
("acct_second", PreferredAccountSelection::Hit)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_should_honor_request_local_exclusion() {
|
||||
let candidates = vec![
|
||||
@@ -416,7 +522,7 @@ fn selector_should_honor_request_local_exclusion() {
|
||||
.select(&candidates, &context)
|
||||
.expect("second account available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -432,7 +538,7 @@ fn sticky_selector_should_prefer_requested_account() {
|
||||
.select(&candidates, &context)
|
||||
.expect("sticky account available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -447,7 +553,7 @@ fn sticky_selector_without_request_binding_should_reuse_latest_account() {
|
||||
.select(&candidates, &context(RotationStrategy::Sticky))
|
||||
.expect("sticky candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_latest");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_latest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -462,7 +568,7 @@ fn round_robin_selector_should_use_frozen_cursor() {
|
||||
.select(&candidates, &context)
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.account.id().as_str(), "acct_second");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,10 +4,11 @@ use std::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, SystemTime},
|
||||
time::{Duration, Instant, SystemTime},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{executor::block_on, future::BoxFuture};
|
||||
use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
@@ -20,8 +21,9 @@ use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ClientApiKeyUsageSink, DefaultExecutionService, ExecutionService, ProviderCircuitDecision,
|
||||
ProviderCircuitError, ProviderCircuitPort, provider_failure_affects_circuit,
|
||||
ClientApiKeyUsageSink, ClientTransport, DefaultExecutionService, ExecutionRequestMetadata,
|
||||
ExecutionService, ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
StartExecution, provider_failure_affects_circuit,
|
||||
};
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::{
|
||||
@@ -32,12 +34,15 @@ use gateway_core::engine::{
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError,
|
||||
StoreErrorKind,
|
||||
};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, RuntimeSnapshot,
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
use serde_json::json;
|
||||
@@ -105,10 +110,54 @@ fn probe_failures_should_be_observable_without_a_model_request_row() {
|
||||
.expect_err("the provider rejects every probe");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
let upstream = error
|
||||
.upstream_response()
|
||||
.expect("probe must preserve its request-local upstream response");
|
||||
assert_eq!(upstream.status(), 502);
|
||||
assert_eq!(
|
||||
upstream.content_type(),
|
||||
Some(b"application/json".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
upstream.body(),
|
||||
&Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
);
|
||||
assert!(!format!("{error:?}").contains("source upstream failure"));
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_observation_store_failure_preserves_the_provider_error() {
|
||||
let store = Arc::new(TrackingExecutionStore {
|
||||
fail_probe_observation: AtomicBool::new(true),
|
||||
..TrackingExecutionStore::default()
|
||||
});
|
||||
let providers = ProviderRegistry::new([Arc::new(FailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry");
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(probe_snapshot()),
|
||||
store.clone(),
|
||||
providers,
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
|
||||
let error = block_on(service.probe(AccountProbeRequest {
|
||||
account_id: ProviderAccountId::new("acct_probe").expect("account ID"),
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
upstream_model: UpstreamModelId::new("gpt-probe").expect("model ID"),
|
||||
operation: probe_operation(),
|
||||
}))
|
||||
.expect_err("the provider error must survive observation failure");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert!(store.probe_failures().is_empty());
|
||||
}
|
||||
|
||||
struct FailingProvider;
|
||||
|
||||
#[async_trait]
|
||||
@@ -132,10 +181,14 @@ impl Provider for FailingProvider {
|
||||
_request: ProviderRequest,
|
||||
_context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
502,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +212,111 @@ fn successful_authentication_should_record_client_key_usage() {
|
||||
assert_eq!(usage.recorded(), vec!["key_usage_test".to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_store_failure_should_fail_open_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(FailingDecisionCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("recoverable circuit state must not reject the request");
|
||||
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slow_circuit_store_should_time_out_and_fail_open_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(PendingDecisionCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
let started_at = Instant::now();
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("slow recoverable circuit state must not reject the request");
|
||||
|
||||
assert!(started_at.elapsed() < Duration::from_secs(2));
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
let model = "gpt-future-not-in-catalog";
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::from_client_wire(model).expect("client model"),
|
||||
operation: start_operation_for_model(model),
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("stale model catalog must not reject the request");
|
||||
|
||||
assert!(!started.session.is_finalized());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingClientApiKeyUsage {
|
||||
ids: Mutex<Vec<String>>,
|
||||
@@ -183,6 +341,7 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
|
||||
struct TrackingExecutionStore {
|
||||
touched: AtomicBool,
|
||||
probe_failures: Mutex<Vec<String>>,
|
||||
fail_probe_observation: AtomicBool,
|
||||
}
|
||||
|
||||
impl TrackingExecutionStore {
|
||||
@@ -240,6 +399,9 @@ impl ExecutionStore for TrackingExecutionStore {
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
if self.fail_probe_observation.load(Ordering::SeqCst) {
|
||||
return Err(StoreError::new(StoreErrorKind::Unavailable));
|
||||
}
|
||||
self.probe_failures
|
||||
.lock()
|
||||
.expect("probe failures lock")
|
||||
@@ -309,6 +471,56 @@ impl ProviderCircuitPort for UnusedCircuits {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailingDecisionCircuits;
|
||||
|
||||
impl ProviderCircuitPort for FailingDecisionCircuits {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
Box::pin(async { Err(ProviderCircuitError) })
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingDecisionCircuits;
|
||||
|
||||
impl ProviderCircuitPort for PendingDecisionCircuits {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
Box::pin(futures::future::pending())
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
struct UnusedContinuation;
|
||||
|
||||
impl NativeContinuationPort for UnusedContinuation {
|
||||
@@ -371,6 +583,34 @@ fn client_snapshot() -> RuntimeSnapshot {
|
||||
.expect("client snapshot")
|
||||
}
|
||||
|
||||
fn start_snapshot() -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new("openai").expect("provider kind");
|
||||
let capabilities =
|
||||
ModelCapabilities::new(BTreeSet::from([OperationKind::Generate]), Some(16_000));
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
std::num::NonZeroU32::new(1).expect("concurrency"),
|
||||
Duration::from_millis(1),
|
||||
),
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
UpstreamModelId::new("gpt-start").expect("model ID"),
|
||||
capabilities,
|
||||
)],
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_start_test").expect("client API key ID"),
|
||||
PlaintextClientApiKey::new("sk_start_test").expect("plaintext client API key"),
|
||||
provider,
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
)
|
||||
.expect("start snapshot")
|
||||
}
|
||||
|
||||
fn probe_operation() -> Operation {
|
||||
let body = json!({
|
||||
"model": "gpt-probe",
|
||||
@@ -381,3 +621,18 @@ fn probe_operation() -> Operation {
|
||||
.expect("OpenAI payload"),
|
||||
))
|
||||
}
|
||||
|
||||
fn start_operation() -> Operation {
|
||||
start_operation_for_model("gpt-start")
|
||||
}
|
||||
|
||||
fn start_operation_for_model(model: &str) -> Operation {
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"input": [{"type": "message", "role": "user", "content": "ping"}],
|
||||
});
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body.as_object().expect("request object").clone())
|
||||
.expect("OpenAI payload"),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::engine::UpstreamSendState;
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, GatewayError, ProviderError, ProviderErrorKind, SafeUpstreamValue,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, GatewayError, OpaqueUpstreamValue,
|
||||
ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn provider_error_debug_should_not_expose_sensitive_context() {
|
||||
@@ -15,13 +19,22 @@ fn provider_error_debug_should_not_expose_sensitive_context() {
|
||||
#[test]
|
||||
fn provider_error_debug_should_not_print_classified_upstream_values() {
|
||||
let diagnostic = "request-visible-only-through-explicit-accessor";
|
||||
let value = SafeUpstreamValue::new(diagnostic).expect("test diagnostic is valid");
|
||||
let value = OpaqueUpstreamValue::new(diagnostic);
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_upstream_request_id(value);
|
||||
|
||||
assert!(!format!("{error:?}").contains(diagnostic));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_upstream_value_should_preserve_arbitrary_text_without_logging_it() {
|
||||
let original = format!("\0{}\n", "x".repeat(9_000));
|
||||
let value = OpaqueUpstreamValue::new(original.clone());
|
||||
|
||||
assert_eq!(value.as_str(), original);
|
||||
assert!(!format!("{value:?}").contains(&original));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_replay_proof_should_default_to_false() {
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent);
|
||||
@@ -37,18 +50,82 @@ fn provider_error_replay_proof_should_be_explicit() {
|
||||
assert!(error.replay_is_safe());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_atomic_client_events_should_be_take_only_and_debug_redacted() {
|
||||
let marker = "atomic-wire-must-not-enter-debug";
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({"type": "response.failed", "message": marker}),
|
||||
)
|
||||
.expect("atomic wire"),
|
||||
);
|
||||
let mut error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_atomic_client_events(vec![event]);
|
||||
|
||||
assert!(error.has_atomic_client_events());
|
||||
assert!(!format!("{error:?}").contains(marker));
|
||||
assert_eq!(error.take_atomic_client_events().len(), 1);
|
||||
assert!(!error.has_atomic_client_events());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_clone_should_drop_request_local_upstream_response() {
|
||||
let marker = Bytes::from_static(b"raw-client-response-only");
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
429,
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
marker.clone(),
|
||||
)
|
||||
.with_headers(vec![ProviderResponseHeader::new(
|
||||
"x-future-error",
|
||||
Bytes::from_static(b"opaque-header-value"),
|
||||
)]),
|
||||
);
|
||||
|
||||
let cloned = error.clone();
|
||||
|
||||
assert_eq!(
|
||||
error
|
||||
.client_visible_upstream_response()
|
||||
.map(ClientVisibleUpstreamResponse::body),
|
||||
Some(&marker)
|
||||
);
|
||||
assert!(cloned.client_visible_upstream_response().is_none());
|
||||
assert!(!format!("{error:?}").contains("raw-client-response-only"));
|
||||
assert!(!format!("{error:?}").contains("opaque-header-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_error_clone_should_drop_atomic_client_events() {
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({"type": "response.failed"}),
|
||||
)
|
||||
.expect("atomic wire"),
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_atomic_client_events(vec![event]);
|
||||
let cloned = error.clone();
|
||||
|
||||
assert!(error.has_atomic_client_events());
|
||||
assert!(!cloned.has_atomic_client_events());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnostics() {
|
||||
let message = "Your Codex quota is exhausted";
|
||||
let error = ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
message,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
);
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
message,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
));
|
||||
let gateway = GatewayError::from_provider(&error);
|
||||
|
||||
assert_eq!(
|
||||
@@ -60,3 +137,20 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
|
||||
assert_eq!(gateway.client_error_type(), Some("rate_limit_error"));
|
||||
assert!(!format!("{gateway:?}").contains(message));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_visible_upstream_error_should_preserve_opaque_structured_fields() {
|
||||
let message = format!("\0{}\n", "m".repeat(9_000));
|
||||
let code = format!("\0{}", "c".repeat(300));
|
||||
let error_type = String::new();
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
message.clone(),
|
||||
Some(code.clone()),
|
||||
Some(error_type.clone()),
|
||||
);
|
||||
|
||||
assert_eq!(detail.message(), message);
|
||||
assert_eq!(detail.code(), Some(code.as_str()));
|
||||
assert_eq!(detail.error_type(), Some(error_type.as_str()));
|
||||
assert!(!format!("{detail:?}").contains(&message));
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::engine::provider::UpstreamTransport;
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, EventSequenceError, EventSequenceValidator, GatewayEvent,
|
||||
ProtocolWireEvent, ProviderEvent, ProviderResponseObservation, ReasoningDelta, ResponseMeta,
|
||||
TextDelta, ToolCallDelta,
|
||||
ProtocolWireEvent, ProviderEvent, ProviderResponseHeader, ProviderResponseObservation,
|
||||
ReasoningDelta, ResponseMeta, TextDelta, ToolCallDelta,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -21,6 +22,28 @@ fn provider_event_keeps_large_wire_and_observation_payloads_behind_one_indirecti
|
||||
assert!(!observation.has_client_event());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_response_header_should_preserve_opaque_name_and_bytes_without_debug_disclosure() {
|
||||
let name = format!("x-future-{}\0", "n".repeat(512));
|
||||
let value = Bytes::from_static(b"\xffopaque-response-header-secret");
|
||||
let header = ProviderResponseHeader::new(name.clone(), value.clone());
|
||||
|
||||
assert_eq!(header.name(), name);
|
||||
assert_eq!(header.value(), &value);
|
||||
let rendered = format!("{header:?}");
|
||||
assert!(!rendered.contains(&name));
|
||||
assert!(!rendered.contains("opaque-response-header-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_observed_service_tier_should_be_ignored() {
|
||||
let observation =
|
||||
ProviderResponseObservation::new(UpstreamTransport::new("http_sse").expect("transport"))
|
||||
.with_service_tier_if_valid(format!("priority\0{}", "x".repeat(128)));
|
||||
|
||||
assert_eq!(observation.service_tier(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_wire_event_should_preserve_sse_metadata_without_exposing_id_in_debug() {
|
||||
let wire = ProtocolWireEvent::json_with_sse_metadata(
|
||||
@@ -37,6 +60,23 @@ fn protocol_wire_event_should_preserve_sse_metadata_without_exposing_id_in_debug
|
||||
assert!(!format!("{wire:?}").contains("upstream-event-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_wire_event_should_preserve_opaque_sse_metadata() {
|
||||
let event_type = format!("\0{}", "event".repeat(80));
|
||||
let sse_id = "id\0with\r\ncontrols".to_owned();
|
||||
let wire = ProtocolWireEvent::json_with_sse_metadata(
|
||||
"openai",
|
||||
Some(event_type.clone()),
|
||||
json!({"type": "future.event"}),
|
||||
Some(sse_id.clone()),
|
||||
None,
|
||||
)
|
||||
.expect("internal protocol name is valid");
|
||||
|
||||
assert_eq!(wire.event_type(), Some(event_type.as_str()));
|
||||
assert_eq!(wire.sse_id(), Some(sse_id.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validator_should_accept_started_content_delta_completed_sequence() {
|
||||
let mut validator = EventSequenceValidator::new();
|
||||
|
||||
@@ -57,7 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let account = ProviderAccountId::new("acct_stable").expect("valid account");
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
@@ -70,13 +70,7 @@ fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
let first = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh");
|
||||
let second = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh again");
|
||||
|
||||
assert_eq!(first, second);
|
||||
assert!(first > observed_at);
|
||||
assert!(first < expires_at);
|
||||
assert_eq!(first, expires_at - Duration::from_secs(3_600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -190,21 +190,36 @@ fn global_mapping_should_apply_to_every_provider() {
|
||||
#[test]
|
||||
fn unmapped_model_should_pass_through_unchanged() {
|
||||
let snapshot = snapshot();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("future-openai-model").expect("model"),
|
||||
&operation(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("unknown model remains transparent");
|
||||
for requested in [
|
||||
"future-openai-model".to_owned(),
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
"future\0model".to_owned(),
|
||||
" future-openai-model ".to_owned(),
|
||||
] {
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::from_client_wire(requested.clone()).expect("client model"),
|
||||
&operation(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("unknown model remains transparent");
|
||||
|
||||
assert_eq!(
|
||||
plan.candidates()[0].upstream_model().as_str(),
|
||||
"future-openai-model"
|
||||
);
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), requested);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_mapping_should_use_exact_client_keys() {
|
||||
let snapshot = snapshot().with_model_mappings(BTreeMap::from([(
|
||||
" exact-alias ".to_owned(),
|
||||
"gpt-5.5".to_owned(),
|
||||
)]));
|
||||
|
||||
assert_eq!(snapshot.mapped_model(" exact-alias "), "gpt-5.5");
|
||||
assert_eq!(snapshot.mapped_model("exact-alias"), "exact-alias");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -13,7 +13,7 @@ path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
uuid = { version = "1.23.3", features = ["v4"] }
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ pub fn retry_after_seconds_from_value(value: &Value) -> Option<u64> {
|
||||
.or_else(|| {
|
||||
error
|
||||
.get("resets_in_seconds")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(positive_u64)
|
||||
.filter(|seconds| *seconds > 0)
|
||||
})
|
||||
.or_else(|| retry_after_seconds_from_resets_at(error))
|
||||
@@ -317,11 +317,7 @@ pub fn is_rate_limit_header_name(name: &str) -> bool {
|
||||
|
||||
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
|
||||
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
if value
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|event_type| event_type != "codex.rate_limits")
|
||||
{
|
||||
if value.get("type").and_then(Value::as_str) != Some("codex.rate_limits") {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -396,6 +392,12 @@ pub fn rate_limits_to_header_pairs(rate_limits: &ParsedRateLimits) -> Vec<(Strin
|
||||
&format!("{prefix}-secondary"),
|
||||
details.secondary,
|
||||
);
|
||||
if let Some(allowed) = details.allowed {
|
||||
headers.push((format!("{prefix}-allowed"), allowed.to_string()));
|
||||
}
|
||||
if let Some(limit_reached) = details.limit_reached {
|
||||
headers.push((format!("{prefix}-limit-reached"), limit_reached.to_string()));
|
||||
}
|
||||
if let Some(limit_name) = &details.limit_name {
|
||||
headers.push((format!("{prefix}-limit-name"), limit_name.clone()));
|
||||
}
|
||||
@@ -441,7 +443,7 @@ fn nested_number_field(value: &Value, path: &[&str]) -> Option<u64> {
|
||||
}
|
||||
|
||||
fn retry_after_seconds_from_resets_at(error: &Value) -> Option<u64> {
|
||||
let resets_at = error.get("resets_at").and_then(Value::as_u64)?;
|
||||
let resets_at = error.get("resets_at").and_then(positive_u64)?;
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
@@ -449,6 +451,13 @@ fn retry_after_seconds_from_resets_at(error: &Value) -> Option<u64> {
|
||||
(resets_at > now).then_some(resets_at - now)
|
||||
}
|
||||
|
||||
fn positive_u64(value: &Value) -> Option<u64> {
|
||||
value
|
||||
.as_u64()
|
||||
.or_else(|| value.as_str().and_then(|value| value.parse::<u64>().ok()))
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
fn retry_after_seconds_from_rate_limit_message(error: &Value) -> Option<u64> {
|
||||
let code = error
|
||||
.get("code")
|
||||
|
||||
@@ -18,10 +18,11 @@ pub struct SseEvent {
|
||||
pub retry: Option<u64>,
|
||||
}
|
||||
|
||||
/// 保留完整原始字节与旁路解析结果的单个 SSE 帧。
|
||||
/// 保留原始字节与旁路解析结果的 SSE wire 片段。
|
||||
///
|
||||
/// 这用于需要原样转发的 transport;`events` 只供旁路观测,不能反过来重建
|
||||
/// `raw`。
|
||||
/// 通常一个片段就是完整 SSE 帧。单帧超过旁路观测预算时会拆成多个原始片段,
|
||||
/// 此时 `events` 为空;透明 transport 必须按顺序交付 `raw`,不能从 `events`
|
||||
/// 反向重建或否决原始数据。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct SseFrame {
|
||||
raw: Vec<u8>,
|
||||
@@ -62,7 +63,7 @@ impl fmt::Debug for SseFrame {
|
||||
}
|
||||
}
|
||||
|
||||
/// 单事件缓冲上限。
|
||||
/// 单事件旁路解析缓冲上限。
|
||||
pub const MAX_SSE_EVENT_BUFFER_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// SSE 流结束标记帧。
|
||||
@@ -75,6 +76,9 @@ pub struct SseEventDecoder {
|
||||
/// `pending` 中已确认不含帧分隔符的前缀长度。新数据到达时从该边界
|
||||
/// 回退分隔符最大跨界宽度继续扫描,大帧跨多个 chunk 不会被反复重扫。
|
||||
scanned: usize,
|
||||
/// 超大未完成帧已开始直接交付;在遇到帧分隔符前只保留跨 chunk 扫描尾部,
|
||||
/// 不再尝试解析该帧。
|
||||
opaque_frame: bool,
|
||||
}
|
||||
|
||||
impl SseEventDecoder {
|
||||
@@ -129,28 +133,35 @@ impl SseEventDecoder {
|
||||
/// 与 [`Self::push`] 相比,此方法仅应由需要原样转发的 transport 使用;普通
|
||||
/// 解析路径无需为原始帧额外分配内存。旁路解析失败时仍保留原始帧,并以空
|
||||
/// event 列表表示不可观测,避免观测失败截断客户端字节流。
|
||||
pub fn push_frames(&mut self, chunk: &[u8]) -> Result<Vec<SseFrame>, SseError> {
|
||||
pub fn push_frames(&mut self, chunk: &[u8]) -> Vec<SseFrame> {
|
||||
self.pending.extend_from_slice(chunk);
|
||||
let mut frames = Vec::new();
|
||||
if self.opaque_frame && !self.finish_opaque_frame(&mut frames) {
|
||||
self.flush_opaque_prefix(&mut frames);
|
||||
return frames;
|
||||
}
|
||||
let mut consumed = 0usize;
|
||||
|
||||
while let Some(end) = self.next_frame_end(consumed) {
|
||||
let raw = self.pending[consumed..end].to_vec();
|
||||
let events = std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.unwrap_or_default();
|
||||
let events = if raw.len() > MAX_SSE_EVENT_BUFFER_BYTES {
|
||||
Vec::new()
|
||||
} else {
|
||||
std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
frames.push(SseFrame::new(raw, events));
|
||||
consumed = end;
|
||||
}
|
||||
|
||||
self.commit_consumed(consumed);
|
||||
if self.pending.len() > MAX_SSE_EVENT_BUFFER_BYTES {
|
||||
return Err(SseError::BufferExceeded {
|
||||
max_bytes: MAX_SSE_EVENT_BUFFER_BYTES,
|
||||
});
|
||||
self.opaque_frame = true;
|
||||
self.flush_opaque_prefix(&mut frames);
|
||||
}
|
||||
Ok(frames)
|
||||
frames
|
||||
}
|
||||
|
||||
/// 流结束时解析尚未带空行分隔符的最后一帧。
|
||||
@@ -168,17 +179,53 @@ impl SseEventDecoder {
|
||||
/// 在流结束时返回未带空行分隔符的最后一帧及其旁路解析结果。
|
||||
///
|
||||
/// 与 [`Self::push_frames`] 一致,旁路解析失败不会丢弃原始帧。
|
||||
pub fn finish_frames(&mut self) -> Result<Vec<SseFrame>, SseError> {
|
||||
pub fn finish_frames(&mut self) -> Vec<SseFrame> {
|
||||
self.scanned = 0;
|
||||
if self.pending.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
self.opaque_frame = false;
|
||||
return Vec::new();
|
||||
}
|
||||
let raw = std::mem::take(&mut self.pending);
|
||||
let events = std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.unwrap_or_default();
|
||||
Ok(vec![SseFrame::new(raw, events)])
|
||||
let events = if self.opaque_frame || raw.len() > MAX_SSE_EVENT_BUFFER_BYTES {
|
||||
Vec::new()
|
||||
} else {
|
||||
std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
self.opaque_frame = false;
|
||||
vec![SseFrame::new(raw, events)]
|
||||
}
|
||||
|
||||
/// 继续透明交付一个已超出观测预算的帧,并在分隔符后恢复普通解析。
|
||||
fn finish_opaque_frame(&mut self, frames: &mut Vec<SseFrame>) -> bool {
|
||||
let Some((position, separator_len)) = sse_frame_separator_bytes(&self.pending) else {
|
||||
return false;
|
||||
};
|
||||
let end = position + separator_len;
|
||||
let remaining = self.pending.split_off(end);
|
||||
let raw = std::mem::replace(&mut self.pending, remaining);
|
||||
frames.push(SseFrame::new(raw, Vec::new()));
|
||||
self.opaque_frame = false;
|
||||
self.scanned = 0;
|
||||
true
|
||||
}
|
||||
|
||||
/// 仅保留分隔符可能跨 chunk 的最大前缀宽度,其余原始字节立即释放给下游。
|
||||
fn flush_opaque_prefix(&mut self, frames: &mut Vec<SseFrame>) {
|
||||
const SEPARATOR_CROSS_CHUNK_BYTES: usize = 3;
|
||||
let emit_len = self
|
||||
.pending
|
||||
.len()
|
||||
.saturating_sub(SEPARATOR_CROSS_CHUNK_BYTES);
|
||||
if emit_len == 0 {
|
||||
return;
|
||||
}
|
||||
let tail = self.pending.split_off(emit_len);
|
||||
let raw = std::mem::replace(&mut self.pending, tail);
|
||||
self.scanned = 0;
|
||||
frames.push(SseFrame::new(raw, Vec::new()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -366,12 +413,12 @@ pub fn encode_sse_event_with_metadata(
|
||||
retry: Option<u64>,
|
||||
) -> String {
|
||||
let mut frame = String::new();
|
||||
if let Some(id) = id {
|
||||
if let Some(id) = id.filter(|id| sse_id_is_representable(id)) {
|
||||
frame.push_str("id: ");
|
||||
frame.push_str(id);
|
||||
frame.push('\n');
|
||||
}
|
||||
if !event.is_empty() {
|
||||
if !event.is_empty() && sse_field_value_is_single_line(event) {
|
||||
frame.push_str("event: ");
|
||||
frame.push_str(event);
|
||||
frame.push('\n');
|
||||
@@ -390,6 +437,14 @@ pub fn encode_sse_event_with_metadata(
|
||||
frame
|
||||
}
|
||||
|
||||
fn sse_field_value_is_single_line(value: &str) -> bool {
|
||||
!value.contains(['\r', '\n'])
|
||||
}
|
||||
|
||||
fn sse_id_is_representable(value: &str) -> bool {
|
||||
sse_field_value_is_single_line(value) && !value.contains('\0')
|
||||
}
|
||||
|
||||
fn track_event_buffer_bytes(current_bytes: &mut usize, line: &str) -> Result<(), SseError> {
|
||||
let line_separator_bytes = usize::from(*current_bytes != 0);
|
||||
*current_bytes = current_bytes
|
||||
|
||||
@@ -226,6 +226,32 @@ fn retry_after_should_read_structured_response_delay() {
|
||||
assert_eq!(retry_after_seconds_from_body(&body), Some(45));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_string_reset_delay() {
|
||||
let body = json!({
|
||||
"error": {"type": "usage_limit_reached", "resets_in_seconds": "46"}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(retry_after_seconds_from_body(&body), Some(46));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_string_reset_timestamp() {
|
||||
let reset_at = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock")
|
||||
.as_secs()
|
||||
+ 60;
|
||||
let body = json!({
|
||||
"error": {"type": "usage_limit_reached", "resets_at": reset_at.to_string()}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let retry_after = retry_after_seconds_from_body(&body).expect("retry-after");
|
||||
assert!((59..=60).contains(&retry_after));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_nested_error_delay() {
|
||||
let body = json!({
|
||||
@@ -414,6 +440,8 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
"metered_limit_name": "codex-other",
|
||||
"limit_name": "Other requests",
|
||||
"rate_limits": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary": {
|
||||
"used_percent": 33,
|
||||
"window_minutes": 60,
|
||||
@@ -424,7 +452,8 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
"has_credits": true,
|
||||
"unlimited": false,
|
||||
"balance": "9.25"
|
||||
}
|
||||
},
|
||||
"rate_limit_reached_type": "usage_limit_reached"
|
||||
});
|
||||
|
||||
let parsed = parse_rate_limits_event(&event).expect("rate-limit event");
|
||||
@@ -434,29 +463,39 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
(
|
||||
parsed.active_limit.as_deref(),
|
||||
details.limit_name.as_deref(),
|
||||
details.allowed,
|
||||
details.limit_reached,
|
||||
details.primary.expect("primary window").window_minutes,
|
||||
parsed.plan_type.as_deref(),
|
||||
parsed.credits.expect("credits").balance,
|
||||
parsed.rate_limit_reached_type.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("codex_other"),
|
||||
Some("Other requests"),
|
||||
Some(false),
|
||||
Some(true),
|
||||
Some(60),
|
||||
Some("team"),
|
||||
Some("9.25".to_owned()),
|
||||
Some("usage_limit_reached"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rate_limits_event_should_reject_unrelated_event_type() {
|
||||
assert_eq!(
|
||||
parse_rate_limits_event(&json!({
|
||||
for event in [
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"rate_limits": {"primary": {"used_percent": 100}}
|
||||
})),
|
||||
None
|
||||
);
|
||||
}),
|
||||
json!({
|
||||
"rate_limits": {"primary": {"used_percent": 100}}
|
||||
}),
|
||||
] {
|
||||
assert_eq!(parse_rate_limits_event(&event), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -468,9 +507,15 @@ fn rate_limit_header_round_trip_should_preserve_wire_facts() {
|
||||
"300".to_owned(),
|
||||
),
|
||||
("x-codex-active-limit".to_owned(), "codex".to_owned()),
|
||||
("x-codex-allowed".to_owned(), "false".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
|
||||
("x-codex-plan-type".to_owned(), "plus".to_owned()),
|
||||
(
|
||||
"x-codex-rate-limit-reached-type".to_owned(),
|
||||
"usage_limit_reached".to_owned(),
|
||||
),
|
||||
];
|
||||
let parsed = parse_rate_limit_headers(&source).expect("initial parse");
|
||||
let encoded = rate_limits_to_header_pairs(&parsed);
|
||||
|
||||
@@ -135,13 +135,8 @@ fn raw_frame_decoder_should_preserve_exact_frame_across_chunk_boundaries() {
|
||||
let raw = b": keep-alive\r\nid: evt_1\r\nevent: response.future\r\nretry: 250\r\ndata: { \"type\": \"response.future\", \"opaque\": true }\r\n\r\n";
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
|
||||
assert!(
|
||||
decoder
|
||||
.push_frames(&raw[..37])
|
||||
.expect("partial raw frame")
|
||||
.is_empty()
|
||||
);
|
||||
let frames = decoder.push_frames(&raw[37..]).expect("complete raw frame");
|
||||
assert!(decoder.push_frames(&raw[..37]).is_empty());
|
||||
let frames = decoder.push_frames(&raw[37..]);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].raw(), raw);
|
||||
@@ -161,15 +156,11 @@ fn raw_frame_decoder_should_preserve_unparseable_frames_for_transparent_transpor
|
||||
let non_utf8 = b"data: \xff\n\n";
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
|
||||
let frames = decoder
|
||||
.push_frames(invalid_retry)
|
||||
.expect("raw frame remains deliverable");
|
||||
let frames = decoder.push_frames(invalid_retry);
|
||||
assert_eq!(frames[0].raw(), invalid_retry);
|
||||
assert!(frames[0].events().is_empty());
|
||||
|
||||
let frames = decoder
|
||||
.push_frames(non_utf8)
|
||||
.expect("non-UTF-8 raw frame remains deliverable");
|
||||
let frames = decoder.push_frames(non_utf8);
|
||||
assert_eq!(frames[0].raw(), non_utf8);
|
||||
assert!(frames[0].events().is_empty());
|
||||
}
|
||||
@@ -179,15 +170,8 @@ fn raw_frame_decoder_finish_should_preserve_unparseable_trailing_frame() {
|
||||
let raw = b"retry: later\ndata: opaque";
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
|
||||
assert!(
|
||||
decoder
|
||||
.push_frames(raw)
|
||||
.expect("unfinished raw frame remains buffered")
|
||||
.is_empty()
|
||||
);
|
||||
let frames = decoder
|
||||
.finish_frames()
|
||||
.expect("trailing raw frame remains deliverable");
|
||||
assert!(decoder.push_frames(raw).is_empty());
|
||||
let frames = decoder.finish_frames();
|
||||
|
||||
assert_eq!(frames[0].raw(), raw);
|
||||
assert!(frames[0].events().is_empty());
|
||||
@@ -232,6 +216,32 @@ fn incremental_decoder_should_accept_exact_limit_and_reject_one_byte_over() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_frame_decoder_should_stream_oversized_unfinished_frame_and_resume_observation() {
|
||||
let oversized = vec![b'x'; MAX_SSE_EVENT_BUFFER_BYTES + 1];
|
||||
let mut decoder = SseEventDecoder::default();
|
||||
let mut raw = decoder
|
||||
.push_frames(&oversized)
|
||||
.into_iter()
|
||||
.flat_map(|frame| frame.into_parts().0)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(!raw.is_empty());
|
||||
let tail = decoder.push_frames(b"\n\n");
|
||||
assert!(tail.iter().all(|frame| frame.events().is_empty()));
|
||||
raw.extend(tail.into_iter().flat_map(|frame| frame.into_parts().0));
|
||||
assert_eq!(raw.len(), oversized.len() + 2);
|
||||
assert_eq!(&raw[..oversized.len()], oversized.as_slice());
|
||||
assert_eq!(&raw[oversized.len()..], b"\n\n");
|
||||
|
||||
let observed = decoder.push_frames(b"event: response.future\ndata: {\"ok\":true}\n\n");
|
||||
assert_eq!(observed.len(), 1);
|
||||
assert_eq!(
|
||||
observed[0].events()[0].event.as_deref(),
|
||||
Some("response.future")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_round_trip_multiline_data() {
|
||||
let frame = encode_sse_event("response.output_text.delta", "first\nsecond");
|
||||
@@ -267,6 +277,20 @@ fn encoder_should_round_trip_sse_id_and_retry() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_omit_metadata_that_sse_cannot_represent_as_one_field() {
|
||||
let frame = encode_sse_event_with_metadata(
|
||||
"future\nevent",
|
||||
r#"{"type":"future\nevent"}"#,
|
||||
Some("id\0with\nline"),
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(!frame.contains("event:"));
|
||||
assert!(!frame.contains("id:"));
|
||||
assert!(frame.contains(r#"data: {"type":"future\nevent"}"#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_failed_event_should_match_openai_failure_shape() {
|
||||
let frame = response_failed_sse_event("server_error", "stream_disconnected", "closed early");
|
||||
|
||||
@@ -20,7 +20,7 @@ gateway-core = { path = "../gateway-core" }
|
||||
hex = "0.4.3"
|
||||
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json", "macros", "migrate"] }
|
||||
thiserror = "2.0.18"
|
||||
|
||||
@@ -21,7 +21,7 @@ use gateway_core::CoreStorePorts;
|
||||
use gateway_core::health::{HealthProbe, HealthState};
|
||||
use gateway_core::provider_ports::ProviderStorePorts;
|
||||
use gateway_core::task::{
|
||||
ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerDisabledReason, WorkerId,
|
||||
DaemonRestartPolicy, ScheduledTask, WorkerContribution, WorkerCycleContext, WorkerId,
|
||||
WorkerKind, WorkerLeaderLeasePort, WorkerLeaseRequest, WorkerRegistration, WorkerRunnable,
|
||||
WorkerSchedule, WorkerTaskError,
|
||||
};
|
||||
@@ -318,28 +318,39 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
}),
|
||||
);
|
||||
|
||||
let execution = Arc::new(postgres::PgExecutionStore::new(pool.clone()));
|
||||
let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone()));
|
||||
let (execution, execution_writer) =
|
||||
postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository));
|
||||
let execution = Arc::new(execution);
|
||||
let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone()));
|
||||
let core_ports = CoreStorePorts::new(
|
||||
execution.clone(),
|
||||
(
|
||||
Arc::new(redis::RedisClientAdmissionRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?),
|
||||
Arc::new(postgres::PgClientAdmissionRecoveryRepository::new(
|
||||
pool.clone(),
|
||||
)),
|
||||
),
|
||||
let admissions: Arc<dyn gateway_core::engine::admission::ClientAdmissionPort> = Arc::new(
|
||||
redis::RedisClientAdmissionRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?,
|
||||
);
|
||||
let circuits: Arc<dyn gateway_core::engine::execution::ProviderCircuitPort> =
|
||||
Arc::new(redis::RedisProviderCircuitRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
redis::ProviderCircuitPolicy::default(),
|
||||
)?),
|
||||
)?);
|
||||
// Continuation affinity 是下一轮请求的路由事实,Core 必须直接等待 Redis 确认。
|
||||
let continuation: Arc<dyn gateway_core::engine::continuation::NativeContinuationPort> =
|
||||
Arc::new(redis::RedisNativeContinuationRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?),
|
||||
)?);
|
||||
let (admissions, admission_release_writer) =
|
||||
redis::BufferedClientAdmissionPort::new(admissions);
|
||||
let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits);
|
||||
let core_ports = CoreStorePorts::new(
|
||||
execution,
|
||||
(
|
||||
Arc::new(admissions),
|
||||
Arc::new(postgres::PgClientAdmissionRecoveryRepository::new(
|
||||
pool.clone(),
|
||||
)),
|
||||
),
|
||||
Arc::new(circuits),
|
||||
continuation,
|
||||
(
|
||||
Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())),
|
||||
Arc::new(redis::RedisRuntimeChangeRepository::new(
|
||||
@@ -373,7 +384,13 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
connection: redis_connection,
|
||||
}),
|
||||
];
|
||||
let worker_contributions = store_worker_contributions(execution, retention)?;
|
||||
let worker_contributions = store_worker_contributions(
|
||||
execution_repository,
|
||||
execution_writer,
|
||||
admission_release_writer,
|
||||
circuit_feedback_writer,
|
||||
retention,
|
||||
)?;
|
||||
Ok(StoreBundle {
|
||||
admin_ports,
|
||||
core_ports,
|
||||
@@ -386,12 +403,24 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
|
||||
fn store_worker_contributions(
|
||||
execution: Arc<postgres::PgExecutionStore>,
|
||||
execution_writer: postgres::ExecutionObservationWriter<postgres::PgExecutionStore>,
|
||||
admission_release_writer: redis::ClientAdmissionReleaseWriter,
|
||||
circuit_feedback_writer: redis::ProviderCircuitFeedbackWriter,
|
||||
retention: Arc<postgres::PgRetentionRepository>,
|
||||
) -> StoreResult<Vec<WorkerContribution>> {
|
||||
let stale_id = WorkerId::try_new(WorkerKind::StaleModelRequestRecovery, "postgres")
|
||||
.map_err(worker_definition_error)?;
|
||||
let retention_id =
|
||||
WorkerId::try_new(WorkerKind::Retention, "postgres").map_err(worker_definition_error)?;
|
||||
let ops_flush_id =
|
||||
WorkerId::try_new(WorkerKind::OpsFlush, "postgres").map_err(worker_definition_error)?;
|
||||
let admission_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_admission")
|
||||
.map_err(worker_definition_error)?;
|
||||
let circuit_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_circuit")
|
||||
.map_err(worker_definition_error)?;
|
||||
let ops_flush_restart =
|
||||
DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(worker_definition_error)?;
|
||||
Ok(vec![
|
||||
WorkerContribution::Registration(scheduled_worker(
|
||||
stale_id,
|
||||
@@ -403,10 +432,36 @@ fn store_worker_contributions(
|
||||
Duration::from_secs(60 * 60),
|
||||
Box::new(RetentionTask { retention }),
|
||||
)?),
|
||||
WorkerContribution::Disabled {
|
||||
kind: WorkerKind::OpsFlush,
|
||||
reason: WorkerDisabledReason::NoBufferedOpsEvents,
|
||||
},
|
||||
WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
ops_flush_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart: ops_flush_restart,
|
||||
task: Box::new(execution_writer),
|
||||
},
|
||||
)
|
||||
.map_err(worker_definition_error)?,
|
||||
),
|
||||
WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
admission_flush_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart: ops_flush_restart,
|
||||
task: Box::new(admission_release_writer),
|
||||
},
|
||||
)
|
||||
.map_err(worker_definition_error)?,
|
||||
),
|
||||
WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
circuit_flush_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart: ops_flush_restart,
|
||||
task: Box::new(circuit_feedback_writer),
|
||||
},
|
||||
)
|
||||
.map_err(worker_definition_error)?,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
@@ -224,6 +224,7 @@ pub struct ModelRequestFinalization {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub provider_metadata_json: Option<Value>,
|
||||
pub error_kind: Option<String>,
|
||||
pub provider_error_code: Option<String>,
|
||||
@@ -276,6 +277,12 @@ impl ModelRequestFinalization {
|
||||
{
|
||||
return Err(invalid("websocket pool must be new or reuse"));
|
||||
}
|
||||
if let Some(service_tier) = self.service_tier.as_deref() {
|
||||
require_nonempty(ENTITY, "service_tier", service_tier)?;
|
||||
if service_tier.len() > 64 || service_tier.chars().any(char::is_control) {
|
||||
return Err(invalid("service tier is invalid"));
|
||||
}
|
||||
}
|
||||
if self
|
||||
.provider_metadata_json
|
||||
.as_ref()
|
||||
@@ -573,7 +580,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
latency_ms = $35, completed_at = $36,
|
||||
upstream_transport = coalesce($37, upstream_transport),
|
||||
http_version = coalesce($38, http_version), websocket_pool = $39,
|
||||
provider_observation_json = $40
|
||||
service_tier = $40, provider_observation_json = $41
|
||||
where id = $1 and outcome = 'running'",
|
||||
)
|
||||
.bind(&finalization.model_request_id)
|
||||
@@ -586,9 +593,9 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
.bind(finalization.downstream_committed_at)
|
||||
.bind(finalization.client_status_code.map(i32::from))
|
||||
.bind(finalization.upstream_status_code.map(i32::from))
|
||||
.bind(finalization.client_response_id)
|
||||
.bind(finalization.client_response_id.map(String::into_bytes))
|
||||
.bind(finalization.upstream_request_id)
|
||||
.bind(finalization.upstream_response_id)
|
||||
.bind(finalization.upstream_response_id.map(String::into_bytes))
|
||||
.bind(finalization.error_kind)
|
||||
.bind(finalization.provider_error_code)
|
||||
.bind(finalization.error_message)
|
||||
@@ -660,6 +667,7 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
.bind(finalization.upstream_transport)
|
||||
.bind(finalization.http_version)
|
||||
.bind(finalization.websocket_pool)
|
||||
.bind(finalization.service_tier)
|
||||
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
@@ -831,18 +839,6 @@ impl ExecutionStore for PgExecutionStore {
|
||||
let latency_ms = u64::try_from(failure.latency.as_millis())
|
||||
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = failure.provider_kind.as_str(),
|
||||
account_id = failure.account_id.as_str(),
|
||||
upstream_model = failure.upstream_model_id.as_str(),
|
||||
failure_kind = error.kind().as_str(),
|
||||
send_state = ?error.send_state(),
|
||||
upstream_status = ?error.upstream_status(),
|
||||
provider_error_code = ?provider_error_code,
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
super::OpsEventRepository::append_ops_event(
|
||||
&super::PgOpsEventRepository::new(self.pool.clone()),
|
||||
super::OpsEvent {
|
||||
@@ -927,6 +923,7 @@ impl ExecutionStore for PgExecutionStore {
|
||||
upstream_transport: finalization.upstream_transport,
|
||||
http_version: finalization.http_version,
|
||||
websocket_pool: finalization.websocket_pool,
|
||||
service_tier: finalization.service_tier,
|
||||
provider_metadata_json,
|
||||
error_kind,
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
|
||||
@@ -0,0 +1,699 @@
|
||||
//! 数据面执行观测的非阻塞 PostgreSQL 写入队列。
|
||||
|
||||
use std::mem::size_of;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord, CancellationToken, ExecutionStore, IntermediateFailure,
|
||||
ModelRequestFinalization, ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport,
|
||||
UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{ProviderError, StoreError};
|
||||
use gateway_core::task::{DaemonTask, WorkerTaskError};
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
const DEFAULT_QUEUE_CAPACITY: usize = 4_096;
|
||||
const DEFAULT_QUEUE_BYTE_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
const SHUTDOWN_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// 执行观测缓冲区的进程内累计状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ExecutionBufferStats {
|
||||
/// 尚未完成落库的队列项和当前写入项数量。
|
||||
pub queued_items: usize,
|
||||
/// 尚未完成落库的观测对象估算字节数。
|
||||
pub queued_bytes: usize,
|
||||
/// 进程启动后成功入队的累计数量。
|
||||
pub enqueued_total: u64,
|
||||
/// 因队列、字节预算或关闭排空超时丢弃的累计数量。
|
||||
pub dropped_total: u64,
|
||||
/// 已成功写入底层 Store 的累计数量。
|
||||
pub persisted_total: u64,
|
||||
/// 底层 Store 返回失败的累计数量。
|
||||
pub write_failure_total: u64,
|
||||
}
|
||||
|
||||
struct ExecutionBufferState {
|
||||
maximum_queued_bytes: usize,
|
||||
queued_items: AtomicUsize,
|
||||
queued_bytes: AtomicUsize,
|
||||
enqueued_total: AtomicU64,
|
||||
dropped_total: AtomicU64,
|
||||
persisted_total: AtomicU64,
|
||||
write_failure_total: AtomicU64,
|
||||
}
|
||||
|
||||
impl ExecutionBufferState {
|
||||
fn new(maximum_queued_bytes: NonZeroUsize) -> Self {
|
||||
Self {
|
||||
maximum_queued_bytes: maximum_queued_bytes.get(),
|
||||
queued_items: AtomicUsize::new(0),
|
||||
queued_bytes: AtomicUsize::new(0),
|
||||
enqueued_total: AtomicU64::new(0),
|
||||
dropped_total: AtomicU64::new(0),
|
||||
persisted_total: AtomicU64::new(0),
|
||||
write_failure_total: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn reserve(&self, bytes: usize) -> bool {
|
||||
let reserved = self
|
||||
.queued_bytes
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
current
|
||||
.checked_add(bytes)
|
||||
.filter(|next| *next <= self.maximum_queued_bytes)
|
||||
})
|
||||
.is_ok();
|
||||
if reserved {
|
||||
self.queued_items.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
reserved
|
||||
}
|
||||
|
||||
fn release(&self, bytes: usize) {
|
||||
self.queued_bytes.fetch_sub(bytes, Ordering::AcqRel);
|
||||
self.queued_items.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn record_enqueued(&self) {
|
||||
saturating_increment(&self.enqueued_total, 1);
|
||||
}
|
||||
|
||||
fn record_dropped(&self, count: usize) {
|
||||
saturating_increment(
|
||||
&self.dropped_total,
|
||||
u64::try_from(count).unwrap_or(u64::MAX),
|
||||
);
|
||||
}
|
||||
|
||||
fn record_persisted(&self) {
|
||||
saturating_increment(&self.persisted_total, 1);
|
||||
}
|
||||
|
||||
fn record_write_failure(&self) {
|
||||
saturating_increment(&self.write_failure_total, 1);
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> ExecutionBufferStats {
|
||||
ExecutionBufferStats {
|
||||
queued_items: self.queued_items.load(Ordering::Acquire),
|
||||
queued_bytes: self.queued_bytes.load(Ordering::Acquire),
|
||||
enqueued_total: self.enqueued_total.load(Ordering::Acquire),
|
||||
dropped_total: self.dropped_total.load(Ordering::Acquire),
|
||||
persisted_total: self.persisted_total.load(Ordering::Acquire),
|
||||
write_failure_total: self.write_failure_total.load(Ordering::Acquire),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn saturating_increment(counter: &AtomicU64, increment: u64) {
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_add(increment))
|
||||
});
|
||||
}
|
||||
|
||||
/// 将数据面观测写入转换为有界、非阻塞的进程内命令。
|
||||
///
|
||||
/// 队列满、worker 尚未启动或已经退出时只丢弃观测并记录告警;协议数据面不会
|
||||
/// 等待 PostgreSQL,也不会看到 Store 错误。启动恢复仍直接访问底层 Store。
|
||||
pub struct BufferedExecutionStore<S: ?Sized> {
|
||||
inner: Arc<S>,
|
||||
sender: mpsc::Sender<QueuedExecutionObservation>,
|
||||
state: Arc<ExecutionBufferState>,
|
||||
}
|
||||
|
||||
impl<S: ?Sized> BufferedExecutionStore<S> {
|
||||
#[must_use]
|
||||
pub fn new(inner: Arc<S>) -> (Self, ExecutionObservationWriter<S>) {
|
||||
Self::with_capacity(
|
||||
inner,
|
||||
NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_capacity(
|
||||
inner: Arc<S>,
|
||||
capacity: NonZeroUsize,
|
||||
) -> (Self, ExecutionObservationWriter<S>) {
|
||||
Self::with_limits(
|
||||
inner,
|
||||
capacity,
|
||||
NonZeroUsize::new(DEFAULT_QUEUE_BYTE_CAPACITY)
|
||||
.expect("queue byte capacity is non-zero"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_limits(
|
||||
inner: Arc<S>,
|
||||
capacity: NonZeroUsize,
|
||||
maximum_queued_bytes: NonZeroUsize,
|
||||
) -> (Self, ExecutionObservationWriter<S>) {
|
||||
let (sender, receiver) = mpsc::channel(capacity.get());
|
||||
let state = Arc::new(ExecutionBufferState::new(maximum_queued_bytes));
|
||||
(
|
||||
Self {
|
||||
inner: Arc::clone(&inner),
|
||||
sender,
|
||||
state: Arc::clone(&state),
|
||||
},
|
||||
ExecutionObservationWriter {
|
||||
inner,
|
||||
receiver: Mutex::new(receiver),
|
||||
state,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> ExecutionBufferStats {
|
||||
self.state.snapshot()
|
||||
}
|
||||
|
||||
fn enqueue(&self, write: ExecutionObservationWrite) {
|
||||
let estimated_bytes = write
|
||||
.estimated_bytes()
|
||||
.saturating_add(write.request_id().map_or(0, str::len))
|
||||
.max(1);
|
||||
if !self.state.reserve(estimated_bytes) {
|
||||
self.state.record_dropped(1);
|
||||
let stats = self.state.snapshot();
|
||||
tracing::warn!(
|
||||
operation = write.operation(),
|
||||
request_id = ?write.request_id(),
|
||||
reason = "byte_capacity",
|
||||
estimated_bytes,
|
||||
maximum_queued_bytes = self.state.maximum_queued_bytes,
|
||||
dropped_total = stats.dropped_total,
|
||||
"执行观测队列字节预算不足,已丢弃本次写入"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let queued =
|
||||
QueuedExecutionObservation::new(write, estimated_bytes, Arc::clone(&self.state));
|
||||
match self.sender.try_send(queued) {
|
||||
Ok(()) => self.state.record_enqueued(),
|
||||
Err(error) => {
|
||||
let (reason, queued) = match error {
|
||||
mpsc::error::TrySendError::Full(queued) => ("full", queued),
|
||||
mpsc::error::TrySendError::Closed(queued) => ("closed", queued),
|
||||
};
|
||||
let operation = queued.operation();
|
||||
let request_id = queued.request_id().map(ToOwned::to_owned);
|
||||
drop(queued);
|
||||
self.state.record_dropped(1);
|
||||
let stats = self.state.snapshot();
|
||||
tracing::warn!(
|
||||
operation,
|
||||
request_id = ?request_id,
|
||||
reason,
|
||||
queued_items = stats.queued_items,
|
||||
queued_bytes = stats.queued_bytes,
|
||||
dropped_total = stats.dropped_total,
|
||||
"执行观测队列不可用,已丢弃本次写入"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<S> ExecutionStore for BufferedExecutionStore<S>
|
||||
where
|
||||
S: ExecutionStore + ?Sized,
|
||||
{
|
||||
async fn create_model_request(&self, request: NewModelRequest) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::Create(Box::new(request)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::Attempt(Box::new(attempt)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_model_request_with_attempt(
|
||||
&self,
|
||||
request: NewModelRequest,
|
||||
attempt: AttemptRecord,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::CreateWithAttempt(Box::new((
|
||||
request, attempt,
|
||||
))));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
state: UpstreamSendState,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::MarkSendState {
|
||||
request_id: request_id.clone(),
|
||||
state,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_downstream_committed(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
committed_at: SystemTime,
|
||||
client_status_code: Option<u16>,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::MarkDownstreamCommitted {
|
||||
request_id: request_id.clone(),
|
||||
committed_at,
|
||||
client_status_code,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_client_status(
|
||||
&self,
|
||||
request_id: &ModelRequestId,
|
||||
client_status_code: u16,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::RecordClientStatus {
|
||||
request_id: request_id.clone(),
|
||||
client_status_code,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_intermediate_failure(
|
||||
&self,
|
||||
failure: IntermediateFailure,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::IntermediateFailure(Box::new(
|
||||
failure,
|
||||
)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::ProbeFailure(Box::new(failure)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn finalize_model_request(
|
||||
&self,
|
||||
finalization: ModelRequestFinalization,
|
||||
) -> Result<(), StoreError> {
|
||||
self.enqueue(ExecutionObservationWrite::Finalize(Box::new(finalization)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recover_expired(&self, now: SystemTime) -> Result<RecoveryReport, StoreError> {
|
||||
self.inner.recover_expired(now).await
|
||||
}
|
||||
}
|
||||
|
||||
/// 由 Host 监督的单消费者,保证同一请求的观测命令按入队顺序落库。
|
||||
pub struct ExecutionObservationWriter<S: ?Sized> {
|
||||
inner: Arc<S>,
|
||||
receiver: Mutex<mpsc::Receiver<QueuedExecutionObservation>>,
|
||||
state: Arc<ExecutionBufferState>,
|
||||
}
|
||||
|
||||
impl<S: ?Sized> ExecutionObservationWriter<S> {
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> ExecutionBufferStats {
|
||||
self.state.snapshot()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> DaemonTask for ExecutionObservationWriter<S>
|
||||
where
|
||||
S: ExecutionStore + ?Sized,
|
||||
{
|
||||
fn run(
|
||||
&self,
|
||||
cancellation: CancellationToken,
|
||||
) -> futures::future::BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut receiver = self.receiver.lock().await;
|
||||
loop {
|
||||
let queued = tokio::select! {
|
||||
() = cancellation.cancelled() => {
|
||||
drain_on_shutdown(
|
||||
&mut receiver,
|
||||
self.inner.as_ref(),
|
||||
self.state.as_ref(),
|
||||
).await;
|
||||
return Ok(());
|
||||
},
|
||||
queued = receiver.recv() => queued,
|
||||
};
|
||||
let Some(queued) = queued else {
|
||||
return Err(WorkerTaskError::safe("execution observation queue closed"));
|
||||
};
|
||||
persist_queued(queued, self.inner.as_ref()).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn persist_queued<S>(mut queued: QueuedExecutionObservation, store: &S)
|
||||
where
|
||||
S: ExecutionStore + ?Sized,
|
||||
{
|
||||
let operation = queued.operation();
|
||||
let request_id = queued.request_id().map(ToOwned::to_owned);
|
||||
let state = Arc::clone(&queued.state);
|
||||
let Some(write) = queued.take_write() else {
|
||||
state.record_write_failure();
|
||||
tracing::error!(
|
||||
operation,
|
||||
request_id = ?request_id,
|
||||
"执行观测队列内部状态无效,已丢弃本次写入"
|
||||
);
|
||||
return;
|
||||
};
|
||||
// Store 写入没有统一幂等键;超时可能表示已提交,不能在这里盲目重试并
|
||||
// 制造重复 ops_events。失败会被计数并丢弃,数据面始终不等待补偿。
|
||||
match write.persist(store).await {
|
||||
Ok(()) => state.record_persisted(),
|
||||
Err(error) => {
|
||||
state.record_write_failure();
|
||||
let stats = state.snapshot();
|
||||
tracing::warn!(
|
||||
operation,
|
||||
request_id = ?request_id,
|
||||
error_kind = ?error.kind(),
|
||||
write_failure_total = stats.write_failure_total,
|
||||
"执行观测写入失败,数据面不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn drain_on_shutdown<S>(
|
||||
receiver: &mut mpsc::Receiver<QueuedExecutionObservation>,
|
||||
store: &S,
|
||||
state: &ExecutionBufferState,
|
||||
) where
|
||||
S: ExecutionStore + ?Sized,
|
||||
{
|
||||
receiver.close();
|
||||
let queued_at_shutdown = receiver.len();
|
||||
let started_at = Instant::now();
|
||||
let mut dropped = 0_usize;
|
||||
|
||||
while let Some(queued) = receiver.recv().await {
|
||||
let remaining = SHUTDOWN_DRAIN_TIMEOUT.saturating_sub(started_at.elapsed());
|
||||
if remaining.is_zero() {
|
||||
drop(queued);
|
||||
dropped = dropped.saturating_add(1);
|
||||
break;
|
||||
}
|
||||
if tokio::time::timeout(remaining, persist_queued(queued, store))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
dropped = dropped.saturating_add(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
while let Ok(queued) = receiver.try_recv() {
|
||||
drop(queued);
|
||||
dropped = dropped.saturating_add(1);
|
||||
}
|
||||
|
||||
if dropped > 0 {
|
||||
state.record_dropped(dropped);
|
||||
let stats = state.snapshot();
|
||||
tracing::warn!(
|
||||
queued_at_shutdown,
|
||||
dropped,
|
||||
dropped_total = stats.dropped_total,
|
||||
drain_timeout_ms =
|
||||
u64::try_from(SHUTDOWN_DRAIN_TIMEOUT.as_millis()).unwrap_or(u64::MAX),
|
||||
"执行观测队列关闭排空超时,剩余写入已丢弃"
|
||||
);
|
||||
} else if queued_at_shutdown > 0 {
|
||||
tracing::info!(queued_at_shutdown, "执行观测队列已在关闭前排空");
|
||||
}
|
||||
}
|
||||
|
||||
struct QueuedExecutionObservation {
|
||||
write: Option<ExecutionObservationWrite>,
|
||||
operation: &'static str,
|
||||
request_id: Option<String>,
|
||||
estimated_bytes: usize,
|
||||
state: Arc<ExecutionBufferState>,
|
||||
}
|
||||
|
||||
impl QueuedExecutionObservation {
|
||||
fn new(
|
||||
write: ExecutionObservationWrite,
|
||||
estimated_bytes: usize,
|
||||
state: Arc<ExecutionBufferState>,
|
||||
) -> Self {
|
||||
let operation = write.operation();
|
||||
let request_id = write.request_id().map(ToOwned::to_owned);
|
||||
Self {
|
||||
write: Some(write),
|
||||
operation,
|
||||
request_id,
|
||||
estimated_bytes,
|
||||
state,
|
||||
}
|
||||
}
|
||||
|
||||
fn operation(&self) -> &'static str {
|
||||
self.operation
|
||||
}
|
||||
|
||||
fn request_id(&self) -> Option<&str> {
|
||||
self.request_id.as_deref()
|
||||
}
|
||||
|
||||
fn take_write(&mut self) -> Option<ExecutionObservationWrite> {
|
||||
self.write.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for QueuedExecutionObservation {
|
||||
fn drop(&mut self) {
|
||||
self.state.release(self.estimated_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
enum ExecutionObservationWrite {
|
||||
Create(Box<NewModelRequest>),
|
||||
Attempt(Box<AttemptRecord>),
|
||||
CreateWithAttempt(Box<(NewModelRequest, AttemptRecord)>),
|
||||
MarkSendState {
|
||||
request_id: ModelRequestId,
|
||||
state: UpstreamSendState,
|
||||
},
|
||||
MarkDownstreamCommitted {
|
||||
request_id: ModelRequestId,
|
||||
committed_at: SystemTime,
|
||||
client_status_code: Option<u16>,
|
||||
},
|
||||
RecordClientStatus {
|
||||
request_id: ModelRequestId,
|
||||
client_status_code: u16,
|
||||
},
|
||||
IntermediateFailure(Box<IntermediateFailure>),
|
||||
ProbeFailure(Box<ProbeFailure>),
|
||||
Finalize(Box<ModelRequestFinalization>),
|
||||
}
|
||||
|
||||
impl ExecutionObservationWrite {
|
||||
const fn operation(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Create(_) => "create_model_request",
|
||||
Self::Attempt(_) => "record_attempt",
|
||||
Self::CreateWithAttempt(_) => "create_model_request_with_attempt",
|
||||
Self::MarkSendState { .. } => "mark_send_state",
|
||||
Self::MarkDownstreamCommitted { .. } => "mark_downstream_committed",
|
||||
Self::RecordClientStatus { .. } => "record_client_status",
|
||||
Self::IntermediateFailure(_) => "record_intermediate_failure",
|
||||
Self::ProbeFailure(_) => "record_probe_failure",
|
||||
Self::Finalize(_) => "finalize_model_request",
|
||||
}
|
||||
}
|
||||
|
||||
fn request_id(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::Create(request) => Some(request.id.as_str()),
|
||||
Self::Attempt(attempt) => Some(attempt.request_id.as_str()),
|
||||
Self::CreateWithAttempt(write) => Some(write.0.id.as_str()),
|
||||
Self::MarkSendState { request_id, .. }
|
||||
| Self::MarkDownstreamCommitted { request_id, .. }
|
||||
| Self::RecordClientStatus { request_id, .. } => Some(request_id.as_str()),
|
||||
Self::IntermediateFailure(failure) => Some(failure.request_id.as_str()),
|
||||
Self::ProbeFailure(_) => None,
|
||||
Self::Finalize(finalization) => Some(finalization.request_id.as_str()),
|
||||
}
|
||||
}
|
||||
|
||||
fn estimated_bytes(&self) -> usize {
|
||||
size_of::<Self>().saturating_add(match self {
|
||||
Self::Create(request) => new_request_bytes(request),
|
||||
Self::Attempt(attempt) => attempt_bytes(attempt),
|
||||
Self::CreateWithAttempt(write) => {
|
||||
new_request_bytes(&write.0).saturating_add(attempt_bytes(&write.1))
|
||||
}
|
||||
Self::MarkSendState { request_id, .. }
|
||||
| Self::MarkDownstreamCommitted { request_id, .. }
|
||||
| Self::RecordClientStatus { request_id, .. } => request_id.as_str().len(),
|
||||
Self::IntermediateFailure(failure) => text_bytes([
|
||||
Some(failure.request_id.as_str()),
|
||||
Some(failure.provider_kind.as_str()),
|
||||
failure.account_id.as_ref().map(|value| value.as_str()),
|
||||
Some(failure.upstream_model_id.as_str()),
|
||||
failure.upstream_request_id.as_deref(),
|
||||
])
|
||||
.saturating_add(provider_error_bytes(&failure.error)),
|
||||
Self::ProbeFailure(failure) => text_bytes([
|
||||
Some(failure.provider_kind.as_str()),
|
||||
Some(failure.account_id.as_str()),
|
||||
Some(failure.upstream_model_id.as_str()),
|
||||
])
|
||||
.saturating_add(provider_error_bytes(&failure.error)),
|
||||
Self::Finalize(finalization) => {
|
||||
let error_bytes = finalization.error.as_ref().map_or(0, |error| {
|
||||
text_bytes([
|
||||
Some(error.client_message()),
|
||||
error.client_error_code(),
|
||||
error.client_error_type(),
|
||||
])
|
||||
});
|
||||
text_bytes([
|
||||
Some(finalization.request_id.as_str()),
|
||||
finalization.client_response_id.as_deref(),
|
||||
finalization.upstream_request_id.as_deref(),
|
||||
finalization.upstream_response_id.as_deref(),
|
||||
finalization.upstream_transport.as_deref(),
|
||||
finalization.http_version.as_deref(),
|
||||
finalization.websocket_pool.as_deref(),
|
||||
finalization.service_tier.as_deref(),
|
||||
finalization.provider_metadata_json.as_deref(),
|
||||
finalization.provider_error_code.as_deref(),
|
||||
])
|
||||
.saturating_add(error_bytes)
|
||||
.saturating_add(size_of::<ModelRequestFinalization>())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn persist<S>(self, store: &S) -> Result<(), StoreError>
|
||||
where
|
||||
S: ExecutionStore + ?Sized,
|
||||
{
|
||||
match self {
|
||||
Self::Create(request) => store.create_model_request(*request).await,
|
||||
Self::Attempt(attempt) => store.record_attempt(*attempt).await,
|
||||
Self::CreateWithAttempt(write) => {
|
||||
let (request, attempt) = *write;
|
||||
store
|
||||
.create_model_request_with_attempt(request, attempt)
|
||||
.await
|
||||
}
|
||||
Self::MarkSendState { request_id, state } => {
|
||||
store.mark_send_state(&request_id, state).await
|
||||
}
|
||||
Self::MarkDownstreamCommitted {
|
||||
request_id,
|
||||
committed_at,
|
||||
client_status_code,
|
||||
} => {
|
||||
store
|
||||
.mark_downstream_committed(&request_id, committed_at, client_status_code)
|
||||
.await
|
||||
}
|
||||
Self::RecordClientStatus {
|
||||
request_id,
|
||||
client_status_code,
|
||||
} => {
|
||||
store
|
||||
.record_client_status(&request_id, client_status_code)
|
||||
.await
|
||||
}
|
||||
Self::IntermediateFailure(failure) => store.record_intermediate_failure(*failure).await,
|
||||
Self::ProbeFailure(failure) => store.record_probe_failure(*failure).await,
|
||||
Self::Finalize(finalization) => store.finalize_model_request(*finalization).await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_request_bytes(request: &NewModelRequest) -> usize {
|
||||
size_of::<NewModelRequest>().saturating_add(text_bytes([
|
||||
Some(request.id.as_str()),
|
||||
request
|
||||
.client_api_key_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
Some(request.client_api_key_ref.as_str()),
|
||||
Some(request.protocol.as_str()),
|
||||
Some(request.endpoint.as_str()),
|
||||
Some(request.client_transport.as_str()),
|
||||
Some(request.requested_model.as_str()),
|
||||
request.user_agent.as_deref(),
|
||||
request.reasoning_effort.as_deref(),
|
||||
request.reasoning_preset.as_deref(),
|
||||
request.request_kind.as_deref(),
|
||||
request.subagent_kind.as_deref(),
|
||||
]))
|
||||
}
|
||||
|
||||
fn attempt_bytes(attempt: &AttemptRecord) -> usize {
|
||||
size_of::<AttemptRecord>().saturating_add(text_bytes([
|
||||
Some(attempt.request_id.as_str()),
|
||||
Some(attempt.provider_kind.as_str()),
|
||||
attempt
|
||||
.provider_account_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
attempt
|
||||
.provider_account_ref
|
||||
.as_ref()
|
||||
.map(|value| value.as_str()),
|
||||
Some(attempt.upstream_model_id.as_str()),
|
||||
Some(attempt.upstream_transport.as_str()),
|
||||
attempt.http_version.as_deref(),
|
||||
]))
|
||||
}
|
||||
|
||||
fn provider_error_bytes(error: &ProviderError) -> usize {
|
||||
let mut bytes = text_bytes([
|
||||
error.upstream_code().map(|value| value.as_str()),
|
||||
error.upstream_request_id().map(|value| value.as_str()),
|
||||
error.upstream_response_id().map(|value| value.as_str()),
|
||||
]);
|
||||
if let Some(client_error) = error.client_visible_upstream_error() {
|
||||
bytes = bytes.saturating_add(text_bytes([
|
||||
Some(client_error.message()),
|
||||
client_error.code(),
|
||||
client_error.error_type(),
|
||||
]));
|
||||
}
|
||||
if let Some(response) = error.client_visible_upstream_response() {
|
||||
bytes = bytes
|
||||
.saturating_add(response.body().len())
|
||||
.saturating_add(response.content_type().map_or(0, <[u8]>::len));
|
||||
for header in response.headers() {
|
||||
bytes = bytes
|
||||
.saturating_add(header.name().len())
|
||||
.saturating_add(header.value().len());
|
||||
}
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn text_bytes<const N: usize>(values: [Option<&str>; N]) -> usize {
|
||||
values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.fold(0, |total, value| total.saturating_add(value.len()))
|
||||
}
|
||||
@@ -11,6 +11,7 @@ mod admin_security_audit;
|
||||
mod admission_recovery;
|
||||
mod client_keys;
|
||||
mod execution;
|
||||
mod execution_buffer;
|
||||
mod observability;
|
||||
mod ops_events;
|
||||
mod provider_accounts;
|
||||
@@ -22,6 +23,7 @@ pub use admin_security_audit::*;
|
||||
pub use admission_recovery::*;
|
||||
pub use client_keys::*;
|
||||
pub use execution::*;
|
||||
pub use execution_buffer::*;
|
||||
pub use observability::*;
|
||||
pub use ops_events::*;
|
||||
pub use provider_accounts::*;
|
||||
|
||||
@@ -125,7 +125,6 @@ impl UsageRecordFilter {
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.outcome.as_deref(), "outcome filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(self.response_id.as_deref(), "response ID filter"),
|
||||
(
|
||||
self.upstream_request_id.as_deref(),
|
||||
"upstream request ID filter",
|
||||
@@ -189,7 +188,6 @@ impl OpsErrorFilter {
|
||||
(self.operation.as_deref(), "operation filter"),
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(self.response_id.as_deref(), "response ID filter"),
|
||||
(
|
||||
self.upstream_request_id.as_deref(),
|
||||
"upstream request ID filter",
|
||||
@@ -371,6 +369,7 @@ pub struct CalculatedUsageBillingFact {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub provider_kind: String,
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -523,6 +522,7 @@ pub struct UsageRecord {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub upstream_send_state: String,
|
||||
@@ -1380,6 +1380,7 @@ fn admin_calculated_usage_billing_fact(
|
||||
bucket_start: fact.bucket_start,
|
||||
provider_kind: fact.provider_kind,
|
||||
upstream_model_id: fact.upstream_model_id,
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
@@ -1455,6 +1456,7 @@ fn admin_usage_record(record: UsageRecord) -> AdminStoreResult<admin_observabili
|
||||
upstream_transport: record.upstream_transport,
|
||||
http_version: record.http_version,
|
||||
websocket_pool: record.websocket_pool,
|
||||
service_tier: record.service_tier,
|
||||
provider_metadata_json: record.provider_metadata_json,
|
||||
attempt_count: record.attempt_count,
|
||||
upstream_send_state: record.upstream_send_state,
|
||||
@@ -1814,7 +1816,7 @@ async fn calculated_usage_billing_facts(
|
||||
query.push_bind(granularity.sql_interval());
|
||||
query.push(
|
||||
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
|
||||
mr.provider_kind, mr.upstream_model_id,
|
||||
mr.provider_kind, mr.upstream_model_id, mr.service_tier,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
|
||||
mr.cost_currency, mr.cost_amount::text as amount
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
@@ -1990,27 +1992,35 @@ async fn provider_account_metrics(
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"select count(*)::bigint as total,
|
||||
"with normalized as (
|
||||
select enabled,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (
|
||||
where not (
|
||||
enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
)
|
||||
)::bigint as unavailable,
|
||||
count(*) filter (
|
||||
where enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
)::bigint as active,
|
||||
count(*) filter (
|
||||
where availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
)::bigint as expired,
|
||||
count(*) filter (where availability = 'quota_exhausted')::bigint
|
||||
as quota_exhausted,
|
||||
count(*) filter (where not enabled)::bigint as disabled,
|
||||
count(*) filter (where availability = 'banned')::bigint as banned
|
||||
from provider_accounts",
|
||||
count(*) filter (where status in ('expired', 'disabled', 'banned'))::bigint
|
||||
as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
from normalized",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_one(pool)
|
||||
@@ -2033,11 +2043,29 @@ async fn active_provider_account_ids(
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<String>> {
|
||||
sqlx::query(
|
||||
"select id
|
||||
from provider_accounts
|
||||
where enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
"with normalized as (
|
||||
select id,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select id
|
||||
from normalized
|
||||
where status = 'active'
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
@@ -2234,9 +2262,9 @@ fn push_usage_filter(query: &mut QueryBuilder<Postgres>, filter: &UsageRecordFil
|
||||
}
|
||||
if let Some(value) = &filter.response_id {
|
||||
query.push(format!(" and ({alias}.client_response_id = "));
|
||||
query.push_bind(value.clone());
|
||||
query.push_bind(value.as_bytes().to_vec());
|
||||
query.push(format!(" or {alias}.upstream_response_id = "));
|
||||
query.push_bind(value.clone());
|
||||
query.push_bind(value.as_bytes().to_vec());
|
||||
query.push(")");
|
||||
}
|
||||
if let Some(value) = &filter.upstream_request_id {
|
||||
@@ -2248,7 +2276,7 @@ fn push_usage_filter(query: &mut QueryBuilder<Postgres>, filter: &UsageRecordFil
|
||||
" and lower(concat_ws(' ', {alias}.id, {alias}.client_api_key_ref,
|
||||
{alias}.requested_model_id, {alias}.provider_kind,
|
||||
{alias}.provider_account_ref, {alias}.upstream_model_id,
|
||||
{alias}.client_response_id, {alias}.upstream_request_id,
|
||||
encode({alias}.client_response_id, 'escape'), {alias}.upstream_request_id,
|
||||
{alias}.error_kind, {alias}.provider_error_code,
|
||||
{alias}.error_message)) like "
|
||||
));
|
||||
@@ -2611,7 +2639,7 @@ const USAGE_RECORD_SELECT: &str =
|
||||
pa.name as provider_account_name, pa.email as provider_account_email,
|
||||
pa.authentication_kind as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport, mr.http_version, mr.websocket_pool,
|
||||
mr.provider_observation_json,
|
||||
mr.service_tier, mr.provider_observation_json,
|
||||
mr.attempt_count, mr.upstream_send_state, mr.downstream_committed_at,
|
||||
mr.outcome, mr.client_status_code, mr.upstream_status_code,
|
||||
mr.client_response_id, mr.upstream_request_id, mr.upstream_response_id,
|
||||
@@ -3055,7 +3083,6 @@ fn push_ops_filter(statement: &mut QueryBuilder<Postgres>, filter: &OpsErrorFilt
|
||||
("provider_kind", &filter.provider_kind),
|
||||
("operation", &filter.operation),
|
||||
("upstream_transport", &filter.transport),
|
||||
("client_response_id", &filter.response_id),
|
||||
("upstream_request_id", &filter.upstream_request_id),
|
||||
("failure_kind", &filter.failure_kind),
|
||||
] {
|
||||
@@ -3064,6 +3091,10 @@ fn push_ops_filter(statement: &mut QueryBuilder<Postgres>, filter: &OpsErrorFilt
|
||||
statement.push_bind(value.clone());
|
||||
}
|
||||
}
|
||||
if let Some(value) = &filter.response_id {
|
||||
statement.push(" and e.client_response_id = ");
|
||||
statement.push_bind(value.as_bytes().to_vec());
|
||||
}
|
||||
if let Some(model) = &filter.model {
|
||||
statement.push(" and e.upstream_model_id = ");
|
||||
statement.push_bind(model.clone());
|
||||
@@ -3106,6 +3137,7 @@ fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<UsageRecord
|
||||
upstream_transport: get(row, "upstream_transport")?,
|
||||
http_version: get(row, "http_version")?,
|
||||
websocket_pool: get(row, "websocket_pool")?,
|
||||
service_tier: get(row, "service_tier")?,
|
||||
provider_metadata_json: get::<Option<serde_json::Value>>(row, "provider_observation_json")?
|
||||
.map(|value| serde_json::to_string(&value))
|
||||
.transpose()
|
||||
@@ -3116,9 +3148,9 @@ fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<UsageRecord
|
||||
outcome: get(row, "outcome")?,
|
||||
client_status_code: optional_status(row, "client_status_code")?,
|
||||
upstream_status_code: optional_status(row, "upstream_status_code")?,
|
||||
client_response_id: get(row, "client_response_id")?,
|
||||
client_response_id: opaque_response_id(row, "client_response_id")?,
|
||||
upstream_request_id: get(row, "upstream_request_id")?,
|
||||
upstream_response_id: get(row, "upstream_response_id")?,
|
||||
upstream_response_id: opaque_response_id(row, "upstream_response_id")?,
|
||||
error_kind: get(row, "error_kind")?,
|
||||
provider_error_code: get(row, "provider_error_code")?,
|
||||
error_message: get(row, "error_message")?,
|
||||
@@ -3177,7 +3209,7 @@ fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<OpsErrorRecord
|
||||
client_status_code: optional_status(row, "client_status_code")?,
|
||||
upstream_status_code: optional_status(row, "upstream_status_code")?,
|
||||
provider_error_code: get(row, "provider_error_code")?,
|
||||
client_response_id: get(row, "client_response_id")?,
|
||||
client_response_id: opaque_response_id(row, "client_response_id")?,
|
||||
upstream_request_id: get(row, "upstream_request_id")?,
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
message: get(row, "message")?,
|
||||
@@ -3201,6 +3233,7 @@ fn calculated_usage_billing_fact_from_row(
|
||||
bucket_start: get(row, "bucket_start")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
service_tier: get(row, "service_tier")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
@@ -3218,6 +3251,16 @@ fn optional_decimal(
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn opaque_response_id(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> StoreResult<Option<String>> {
|
||||
get::<Option<Vec<u8>>>(row, column)?
|
||||
.map(String::from_utf8)
|
||||
.transpose()
|
||||
.map_err(|_| invalid(column))
|
||||
}
|
||||
|
||||
fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> {
|
||||
let mut unique = BTreeSet::new();
|
||||
for account_id in account_ids {
|
||||
|
||||
@@ -38,8 +38,8 @@ use gateway_core::engine::credential::{
|
||||
CredentialRevision as CoreCredentialRevision, LoadedCredential,
|
||||
NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential,
|
||||
ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId,
|
||||
ProviderAccountStore, ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
ProviderAccountIdentity, ProviderAccountStore,
|
||||
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -125,6 +125,26 @@ fn parse_availability(value: &str) -> StoreResult<AccountAvailability> {
|
||||
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
|
||||
}
|
||||
|
||||
fn validate_availability_cooldown(
|
||||
availability: AccountAvailability,
|
||||
has_cooldown_until: bool,
|
||||
) -> StoreResult<()> {
|
||||
if availability == AccountAvailability::Cooldown && !has_cooldown_until {
|
||||
return Err(invalid("cooldown availability requires cooldown_until"));
|
||||
}
|
||||
if has_cooldown_until
|
||||
&& !matches!(
|
||||
availability,
|
||||
AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted
|
||||
)
|
||||
{
|
||||
return Err(invalid(
|
||||
"cooldown_until is only allowed for cooldown or quota_exhausted availability",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountSummary {
|
||||
pub id: String,
|
||||
@@ -224,11 +244,7 @@ impl NewProviderAccount {
|
||||
if !self.has_refresh_token && self.next_refresh_at.is_some() {
|
||||
return Err(invalid("next_refresh_at requires a refresh token"));
|
||||
}
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown_until must be present exactly for cooldown availability",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
validate_object_size(
|
||||
"provider_credentials_json",
|
||||
&self.provider_credentials_json,
|
||||
@@ -296,6 +312,7 @@ impl ImportProviderAccounts {
|
||||
pub struct RotateProviderAccount {
|
||||
pub scope: ProviderAccountAdminScope,
|
||||
pub profile: UpdateProviderAccount,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub credential: ProviderCredentialUpdate,
|
||||
pub audit: AdminAuditEvent,
|
||||
}
|
||||
@@ -306,6 +323,7 @@ impl fmt::Debug for RotateProviderAccount {
|
||||
.debug_struct("RotateProviderAccount")
|
||||
.field("scope", &self.scope)
|
||||
.field("profile", &self.profile)
|
||||
.field("replacement_identity", &self.replacement_identity)
|
||||
.field("credential", &self.credential)
|
||||
.field("audit", &self.audit)
|
||||
.finish()
|
||||
@@ -378,11 +396,7 @@ pub struct ProviderAccountStateUpdate {
|
||||
impl ProviderAccountStateUpdate {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown availability and cooldown_until must agree",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -390,11 +404,7 @@ impl ProviderAccountStateUpdate {
|
||||
impl ProviderAccountObservation {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown availability and cooldown_until must agree",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and quota_observed_at must agree"));
|
||||
}
|
||||
@@ -612,8 +622,14 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
update.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = $3, availability_reason = $4, cooldown_until = $5,
|
||||
availability_observed_at = $6, updated_at = greatest(now(), $6)
|
||||
set availability = case when enabled then $3 else availability end,
|
||||
availability_reason = case when enabled then $4 else availability_reason end,
|
||||
cooldown_until = case when enabled then $5 else cooldown_until end,
|
||||
availability_observed_at = case
|
||||
when enabled then $6
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $6) else updated_at end
|
||||
where id = $1 and credential_revision = $2",
|
||||
)
|
||||
.bind(update.account_id)
|
||||
@@ -755,6 +771,12 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
command.scope.validate()?;
|
||||
require_nonempty(ENTITY, "account_id", &command.profile.id)?;
|
||||
require_nonempty(ENTITY, "name", &command.profile.name)?;
|
||||
if let Some(identity) = &command.replacement_identity {
|
||||
require_nonempty(ENTITY, "upstream_user_id", identity.upstream_user_id())?;
|
||||
if let Some(account_id) = identity.upstream_account_id() {
|
||||
require_nonempty(ENTITY, "upstream_account_id", account_id)?;
|
||||
}
|
||||
}
|
||||
if command.profile.id != command.credential.account_id {
|
||||
return Err(invalid("rotated profile and credential account IDs differ"));
|
||||
}
|
||||
@@ -770,6 +792,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
&mut transaction,
|
||||
&command.scope,
|
||||
&command.profile,
|
||||
command.replacement_identity.as_ref(),
|
||||
&command.credential,
|
||||
)
|
||||
.await?;
|
||||
@@ -1040,6 +1063,7 @@ impl PgAdminAccountStore {
|
||||
email: prepared.email,
|
||||
plan_type: prepared.plan_type,
|
||||
},
|
||||
replacement_identity: prepared.replacement_identity,
|
||||
credential: ProviderCredentialUpdate {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
expected_revision: store_revision(prepared.expected_credential_revision)?,
|
||||
@@ -1564,7 +1588,11 @@ fn admin_account_status(
|
||||
AdminAccountStatus::Disabled
|
||||
} else if account.availability == AccountAvailability::Banned {
|
||||
AdminAccountStatus::Banned
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted {
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == AccountAvailability::Expired
|
||||
|| account.availability == AccountAvailability::Invalid
|
||||
@@ -1971,8 +1999,13 @@ async fn rotate_provider_account_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
scope: &ProviderAccountAdminScope,
|
||||
profile: &UpdateProviderAccount,
|
||||
replacement_identity: Option<&ProviderAccountIdentity>,
|
||||
update: &ProviderCredentialUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
let replace_identity = replacement_identity.is_some();
|
||||
let upstream_user_id = replacement_identity.map(ProviderAccountIdentity::upstream_user_id);
|
||||
let upstream_account_id =
|
||||
replacement_identity.and_then(ProviderAccountIdentity::upstream_account_id);
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update provider_accounts
|
||||
set name = $4,
|
||||
@@ -1983,6 +2016,28 @@ async fn rotate_provider_account_in_transaction(
|
||||
has_refresh_token = $8,
|
||||
access_token_expires_at = $9,
|
||||
next_refresh_at = $10,
|
||||
upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end,
|
||||
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
|
||||
availability = case
|
||||
when not enabled then availability
|
||||
when availability <> 'quota_exhausted' then 'ready'
|
||||
else availability
|
||||
end,
|
||||
availability_reason = case
|
||||
when not enabled then availability_reason
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else availability_reason
|
||||
end,
|
||||
cooldown_until = case
|
||||
when not enabled then cooldown_until
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else cooldown_until
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when not enabled then availability_observed_at
|
||||
when availability <> 'quota_exhausted' then now()
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = now()
|
||||
where id = $1 and provider_kind = $2
|
||||
and credential_revision = $3
|
||||
@@ -1998,9 +2053,25 @@ async fn rotate_provider_account_in_transaction(
|
||||
.bind(update.has_refresh_token)
|
||||
.bind(update.access_token_expires_at)
|
||||
.bind(update.next_refresh_at)
|
||||
.bind(replace_identity)
|
||||
.bind(upstream_user_id)
|
||||
.bind(upstream_account_id)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("rotate provider account in admin transaction"))?
|
||||
.map_err(|error| {
|
||||
if error
|
||||
.as_database_error()
|
||||
.is_some_and(sqlx::error::DatabaseError::is_unique_violation)
|
||||
{
|
||||
StoreError::Conflict {
|
||||
entity: ENTITY,
|
||||
id: update.account_id.clone(),
|
||||
kind: ConflictKind::InvalidTransition,
|
||||
}
|
||||
} else {
|
||||
postgres_unavailable("rotate provider account in admin transaction")
|
||||
}
|
||||
})?
|
||||
.ok_or_else(|| StoreError::Conflict {
|
||||
entity: ENTITY,
|
||||
id: update.account_id.clone(),
|
||||
@@ -2191,15 +2262,23 @@ impl ProviderAccountStore for PgProviderAccountRepository {
|
||||
&self,
|
||||
account: &CoreProviderAccountId,
|
||||
expected_revision: CoreCredentialRevision,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
let loaded = self.load_current_credential(account).await?;
|
||||
if loaded.account.revision().get() != expected_revision.get() {
|
||||
return Err(CoreStoreError::new(CoreStoreErrorKind::Conflict));
|
||||
}
|
||||
Ok(loaded)
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &CoreProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
let record = self
|
||||
.load_provider_account(account.as_str())
|
||||
.await
|
||||
.map_err(core_store_error)?
|
||||
.ok_or_else(|| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
||||
if record.summary.credential_revision.get() != expected_revision.get() {
|
||||
return Err(CoreStoreError::new(CoreStoreErrorKind::Conflict));
|
||||
}
|
||||
Ok(LoadedCredential {
|
||||
account: core_account_from_summary(record.summary)?,
|
||||
credential: PlaintextCredential::new(record.provider_credentials_json.fields().clone()),
|
||||
|
||||
@@ -225,6 +225,21 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
update: &RuntimeSettingsUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
update.validate()?;
|
||||
let refresh_margin_seconds =
|
||||
i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?;
|
||||
let previous_refresh_margin_seconds = sqlx::query_scalar::<_, i64>(
|
||||
"select refresh_margin_seconds
|
||||
from runtime_settings
|
||||
where id = 1
|
||||
for update",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("lock runtime settings for update"))?
|
||||
.ok_or_else(|| StoreError::NotFound {
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update runtime_settings
|
||||
set config_revision = config_revision + 1,
|
||||
@@ -243,7 +258,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
returning config_revision",
|
||||
)
|
||||
.bind(update.admin_api_key.as_deref())
|
||||
.bind(i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?)
|
||||
.bind(refresh_margin_seconds)
|
||||
.bind(i64::from(update.refresh_concurrency))
|
||||
.bind(i64::from(update.max_concurrent_per_account))
|
||||
.bind(i64::try_from(update.request_interval_ms).map_err(|_| invalid_numeric())?)
|
||||
@@ -259,9 +274,35 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
if previous_refresh_margin_seconds != refresh_margin_seconds {
|
||||
reschedule_provider_refreshes_in_transaction(transaction, refresh_margin_seconds).await?;
|
||||
}
|
||||
Revision::new(u64::try_from(next).map_err(|_| invalid_numeric())?)
|
||||
}
|
||||
|
||||
async fn reschedule_provider_refreshes_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
refresh_margin_seconds: i64,
|
||||
) -> StoreResult<()> {
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set next_refresh_at = case
|
||||
when access_token_expires_at <= now()
|
||||
or extract(epoch from (access_token_expires_at - now())) <= $1
|
||||
then now()
|
||||
else access_token_expires_at - ($1::double precision * interval '1 second')
|
||||
end,
|
||||
updated_at = now()
|
||||
where has_refresh_token
|
||||
and access_token_expires_at is not null",
|
||||
)
|
||||
.bind(refresh_margin_seconds)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("reschedule provider credential refreshes"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn bump_config_revision_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> StoreResult<Revision> {
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
//! 可由 TTL 或后续反馈收敛的 Redis 写副作用队列。
|
||||
//!
|
||||
//! Continuation affinity 决定下一轮 Provider 与账号,必须在响应 ID 可复用前
|
||||
//! 获得 Redis 确认,因此不属于本模块的可丢失副作用。
|
||||
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::future::{BoxFuture, ready};
|
||||
use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
ClientAdmissionRequest, ClientAdmissionRestoreResult,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
};
|
||||
use gateway_core::engine::{CancellationToken, ModelRequestId};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_core::task::{DaemonTask, WorkerTaskError};
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
const DEFAULT_QUEUE_CAPACITY: usize = 4_096;
|
||||
|
||||
/// 准入读取保持强一致;终态释放只做有界入队,Redis 失败由 worker 吸收。
|
||||
#[derive(Clone)]
|
||||
pub struct BufferedClientAdmissionPort {
|
||||
inner: Arc<dyn ClientAdmissionPort>,
|
||||
sender: mpsc::Sender<AdmissionRelease>,
|
||||
}
|
||||
|
||||
impl BufferedClientAdmissionPort {
|
||||
#[must_use]
|
||||
pub fn new(inner: Arc<dyn ClientAdmissionPort>) -> (Self, ClientAdmissionReleaseWriter) {
|
||||
Self::with_capacity(
|
||||
inner,
|
||||
NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_capacity(
|
||||
inner: Arc<dyn ClientAdmissionPort>,
|
||||
capacity: NonZeroUsize,
|
||||
) -> (Self, ClientAdmissionReleaseWriter) {
|
||||
let (sender, receiver) = mpsc::channel(capacity.get());
|
||||
(
|
||||
Self {
|
||||
inner: Arc::clone(&inner),
|
||||
sender,
|
||||
},
|
||||
ClientAdmissionReleaseWriter {
|
||||
inner,
|
||||
receiver: Mutex::new(receiver),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn enqueue(&self, release: AdmissionRelease) -> bool {
|
||||
match self.sender.try_send(release) {
|
||||
Ok(()) => true,
|
||||
Err(error) => {
|
||||
let reason = match error {
|
||||
mpsc::error::TrySendError::Full(_) => "full",
|
||||
mpsc::error::TrySendError::Closed(_) => "closed",
|
||||
};
|
||||
tracing::warn!(
|
||||
operation = "release_client_admission",
|
||||
reason,
|
||||
"Redis 协调队列不可用,已丢弃本次准入释放;租约将由 TTL 收敛"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientAdmissionPort for BufferedClientAdmissionPort {
|
||||
fn admit(
|
||||
&self,
|
||||
request: ClientAdmissionRequest,
|
||||
) -> BoxFuture<'_, Result<ClientAdmissionDecision, ClientAdmissionError>> {
|
||||
self.inner.admit(request)
|
||||
}
|
||||
|
||||
fn release<'a>(
|
||||
&'a self,
|
||||
client_api_key_id: &'a ClientApiKeyId,
|
||||
model_request_id: &'a ModelRequestId,
|
||||
) -> BoxFuture<'a, Result<bool, ClientAdmissionError>> {
|
||||
let enqueued = self.enqueue(AdmissionRelease {
|
||||
client_api_key_id: client_api_key_id.clone(),
|
||||
model_request_id: model_request_id.clone(),
|
||||
});
|
||||
Box::pin(ready(Ok(enqueued)))
|
||||
}
|
||||
|
||||
fn restore(
|
||||
&self,
|
||||
recovery: ClientAdmissionRecovery,
|
||||
) -> BoxFuture<'_, Result<ClientAdmissionRestoreResult, ClientAdmissionError>> {
|
||||
self.inner.restore(recovery)
|
||||
}
|
||||
}
|
||||
|
||||
struct AdmissionRelease {
|
||||
client_api_key_id: ClientApiKeyId,
|
||||
model_request_id: ModelRequestId,
|
||||
}
|
||||
|
||||
pub struct ClientAdmissionReleaseWriter {
|
||||
inner: Arc<dyn ClientAdmissionPort>,
|
||||
receiver: Mutex<mpsc::Receiver<AdmissionRelease>>,
|
||||
}
|
||||
|
||||
impl DaemonTask for ClientAdmissionReleaseWriter {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut receiver = self.receiver.lock().await;
|
||||
loop {
|
||||
let release = tokio::select! {
|
||||
() = cancellation.cancelled() => return Ok(()),
|
||||
release = receiver.recv() => release,
|
||||
};
|
||||
let Some(release) = release else {
|
||||
return Err(WorkerTaskError::safe(
|
||||
"client admission release queue closed",
|
||||
));
|
||||
};
|
||||
if let Err(error) = self
|
||||
.inner
|
||||
.release(&release.client_api_key_id, &release.model_request_id)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(%error, "Client admission 后台释放失败,依赖租约 TTL 收敛");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Circuit decision 仍直读 Redis;success/failure feedback 只做有界入队。
|
||||
#[derive(Clone)]
|
||||
pub struct BufferedProviderCircuitPort {
|
||||
inner: Arc<dyn ProviderCircuitPort>,
|
||||
sender: mpsc::Sender<CircuitFeedback>,
|
||||
}
|
||||
|
||||
impl BufferedProviderCircuitPort {
|
||||
#[must_use]
|
||||
pub fn new(inner: Arc<dyn ProviderCircuitPort>) -> (Self, ProviderCircuitFeedbackWriter) {
|
||||
Self::with_capacity(
|
||||
inner,
|
||||
NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_capacity(
|
||||
inner: Arc<dyn ProviderCircuitPort>,
|
||||
capacity: NonZeroUsize,
|
||||
) -> (Self, ProviderCircuitFeedbackWriter) {
|
||||
let (sender, receiver) = mpsc::channel(capacity.get());
|
||||
(
|
||||
Self {
|
||||
inner: Arc::clone(&inner),
|
||||
sender,
|
||||
},
|
||||
ProviderCircuitFeedbackWriter {
|
||||
inner,
|
||||
receiver: Mutex::new(receiver),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn enqueue(&self, feedback: CircuitFeedback) {
|
||||
if let Err(error) = self.sender.try_send(feedback) {
|
||||
let reason = match error {
|
||||
mpsc::error::TrySendError::Full(_) => "full",
|
||||
mpsc::error::TrySendError::Closed(_) => "closed",
|
||||
};
|
||||
tracing::warn!(
|
||||
operation = "record_provider_circuit_feedback",
|
||||
reason,
|
||||
"Redis 协调队列不可用,已丢弃本次 circuit feedback"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCircuitPort for BufferedProviderCircuitPort {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
self.inner.decision(provider_kind)
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
self.enqueue(CircuitFeedback::Failure(provider_kind.clone()));
|
||||
Box::pin(ready(Ok(())))
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
self.enqueue(CircuitFeedback::Success(provider_kind.clone()));
|
||||
Box::pin(ready(Ok(())))
|
||||
}
|
||||
}
|
||||
|
||||
enum CircuitFeedback {
|
||||
Failure(ProviderKind),
|
||||
Success(ProviderKind),
|
||||
}
|
||||
|
||||
pub struct ProviderCircuitFeedbackWriter {
|
||||
inner: Arc<dyn ProviderCircuitPort>,
|
||||
receiver: Mutex<mpsc::Receiver<CircuitFeedback>>,
|
||||
}
|
||||
|
||||
impl DaemonTask for ProviderCircuitFeedbackWriter {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut receiver = self.receiver.lock().await;
|
||||
loop {
|
||||
let feedback = tokio::select! {
|
||||
() = cancellation.cancelled() => return Ok(()),
|
||||
feedback = receiver.recv() => feedback,
|
||||
};
|
||||
let Some(feedback) = feedback else {
|
||||
return Err(WorkerTaskError::safe(
|
||||
"provider circuit feedback queue closed",
|
||||
));
|
||||
};
|
||||
let result = match feedback {
|
||||
CircuitFeedback::Failure(provider) => {
|
||||
self.inner.observe_failure(&provider).await
|
||||
}
|
||||
CircuitFeedback::Success(provider) => {
|
||||
self.inner.observe_success(&provider).await
|
||||
}
|
||||
};
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "Provider circuit feedback 后台写入失败");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,8 @@ use gateway_core::{
|
||||
},
|
||||
error::StoreError as CoreStoreError,
|
||||
provider_ports::{
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
ProviderCooldown, ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown,
|
||||
ProviderStoreError, ProviderStoreErrorKind,
|
||||
},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
@@ -104,24 +105,35 @@ impl RedisCredentialCooldownRepository {
|
||||
let fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
|
||||
Ok(format!("{}:account:{fingerprint}:cooldown", self.namespace))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
async fn cache_credential_cooldown(&self, cooldown: &CredentialCooldown) -> StoreResult<bool> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
&cooldown.provider_account_id,
|
||||
)?;
|
||||
let until_ms = cooldown.cooldown_until.timestamp_millis();
|
||||
fn scoped_key(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
scope: &ProviderCooldownScope,
|
||||
) -> StoreResult<String> {
|
||||
let account_fingerprint = resource_fingerprint("credential cooldown", provider_account_id)?;
|
||||
let scope_fingerprint = resource_fingerprint("credential cooldown scope", scope.value())?;
|
||||
Ok(format!(
|
||||
"{}:account:{account_fingerprint}:cooldown:{}:{scope_fingerprint}",
|
||||
self.namespace,
|
||||
scope.kind(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn cache_at_key(
|
||||
&self,
|
||||
key: String,
|
||||
credential_revision: Revision,
|
||||
cooldown_until: DateTime<Utc>,
|
||||
) -> StoreResult<bool> {
|
||||
let until_ms = cooldown_until.timestamp_millis();
|
||||
if until_ms <= 0 {
|
||||
return Err(invalid("cooldown expiry must be positive"));
|
||||
}
|
||||
let mut connection = self.connection.clone();
|
||||
let written = Script::new(WRITE_SCRIPT)
|
||||
.key(self.key(&cooldown.provider_account_id)?)
|
||||
.arg(cooldown.credential_revision.get())
|
||||
.key(key)
|
||||
.arg(credential_revision.get())
|
||||
.arg(until_ms)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
@@ -129,18 +141,10 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
Ok(written == 1)
|
||||
}
|
||||
|
||||
async fn read_credential_cooldown(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
) -> StoreResult<Option<CredentialCooldown>> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
provider_account_id,
|
||||
)?;
|
||||
async fn read_at_key(&self, key: String) -> StoreResult<Option<(Revision, DateTime<Utc>)>> {
|
||||
let mut connection = self.connection.clone();
|
||||
let (present, revision, until_ms): (i64, String, String) = Script::new(READ_SCRIPT)
|
||||
.key(self.key(provider_account_id)?)
|
||||
.key(key)
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("read credential cooldown"))?;
|
||||
@@ -155,11 +159,59 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| invalid("cached cooldown expiry is invalid"))?;
|
||||
let cooldown_until = DateTime::from_timestamp_millis(until_ms)
|
||||
.ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
|
||||
Ok(Some(CredentialCooldown {
|
||||
provider_account_id: provider_account_id.to_owned(),
|
||||
credential_revision: Revision::new(revision)?,
|
||||
cooldown_until,
|
||||
}))
|
||||
Ok(Some((Revision::new(revision)?, cooldown_until)))
|
||||
}
|
||||
|
||||
async fn invalidate_at_key(
|
||||
&self,
|
||||
key: String,
|
||||
through_revision: Revision,
|
||||
) -> StoreResult<bool> {
|
||||
let mut connection = self.connection.clone();
|
||||
let removed = Script::new(INVALIDATE_SCRIPT)
|
||||
.key(key)
|
||||
.arg(through_revision.get())
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("invalidate credential cooldown"))?;
|
||||
Ok(removed == 1)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
async fn cache_credential_cooldown(&self, cooldown: &CredentialCooldown) -> StoreResult<bool> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
&cooldown.provider_account_id,
|
||||
)?;
|
||||
self.cache_at_key(
|
||||
self.key(&cooldown.provider_account_id)?,
|
||||
cooldown.credential_revision,
|
||||
cooldown.cooldown_until,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_credential_cooldown(
|
||||
&self,
|
||||
provider_account_id: &str,
|
||||
) -> StoreResult<Option<CredentialCooldown>> {
|
||||
require_nonempty(
|
||||
"credential cooldown",
|
||||
"provider_account_id",
|
||||
provider_account_id,
|
||||
)?;
|
||||
self.read_at_key(self.key(provider_account_id)?)
|
||||
.await
|
||||
.map(|value| {
|
||||
value.map(|(credential_revision, cooldown_until)| CredentialCooldown {
|
||||
provider_account_id: provider_account_id.to_owned(),
|
||||
credential_revision,
|
||||
cooldown_until,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async fn invalidate_credential_cooldown(
|
||||
@@ -172,14 +224,8 @@ impl CredentialCooldownRepository for RedisCredentialCooldownRepository {
|
||||
"provider_account_id",
|
||||
provider_account_id,
|
||||
)?;
|
||||
let mut connection = self.connection.clone();
|
||||
let removed = Script::new(INVALIDATE_SCRIPT)
|
||||
.key(self.key(provider_account_id)?)
|
||||
.arg(through_revision.get())
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
self.invalidate_at_key(self.key(provider_account_id)?, through_revision)
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("invalidate credential cooldown"))?;
|
||||
Ok(removed == 1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +287,69 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
|
||||
.map_err(|_| provider_unavailable("clear credential cooldown"))
|
||||
})
|
||||
}
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
cooldown: ProviderScopedCooldown,
|
||||
) -> futures::future::BoxFuture<'_, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let revision = Revision::new(cooldown.credential_revision().get())
|
||||
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?;
|
||||
self.cache_at_key(
|
||||
self.scoped_key(cooldown.account_id().as_str(), cooldown.scope())
|
||||
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?,
|
||||
revision,
|
||||
cooldown.until().into(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("cache scoped credential cooldown"))
|
||||
})
|
||||
}
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
) -> futures::future::BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
self.read_at_key(
|
||||
self.scoped_key(account_id.as_str(), scope)
|
||||
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("read scoped credential cooldown"))?
|
||||
.map(|(revision, until)| {
|
||||
Ok(ProviderScopedCooldown::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(revision.get())
|
||||
.map_err(|_| provider_invalid("decode scoped credential cooldown"))?,
|
||||
scope.clone(),
|
||||
until.into(),
|
||||
))
|
||||
})
|
||||
.transpose()
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> futures::future::BoxFuture<'a, Result<bool, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let revision = Revision::new(through_revision.get())
|
||||
.map_err(|_| provider_invalid("encode scoped cooldown revision"))?;
|
||||
self.invalidate_at_key(
|
||||
self.scoped_key(account_id.as_str(), scope)
|
||||
.map_err(|_| provider_invalid("encode scoped credential cooldown"))?,
|
||||
revision,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("clear scoped credential cooldown"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号状态写入到可丢失 cooldown cache 的桥接 adapter。
|
||||
@@ -348,6 +457,13 @@ impl ProviderAccountStore for CooldownCachingProviderAccountStore {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
self.authoritative.load_current_credential(account).await
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
@@ -388,6 +504,12 @@ impl ProviderAccountStore for CooldownCachingProviderAccountStore {
|
||||
let availability = change.availability;
|
||||
let cooldown_until = change.cooldown_until;
|
||||
self.authoritative.apply_state_change(change).await?;
|
||||
let Some(account) = self.authoritative.get_account(&account_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.mirror(
|
||||
&account_id,
|
||||
credential_revision,
|
||||
|
||||
@@ -8,6 +8,7 @@ use sha2::{Digest, Sha256};
|
||||
use async_trait::async_trait;
|
||||
|
||||
mod client_admission;
|
||||
mod coordination_buffer;
|
||||
mod credential_cooldown;
|
||||
mod credential_leases;
|
||||
mod credential_state;
|
||||
@@ -20,6 +21,7 @@ mod runtime_change;
|
||||
pub(crate) mod worker_lease;
|
||||
|
||||
pub use client_admission::*;
|
||||
pub use coordination_buffer::*;
|
||||
pub use credential_cooldown::*;
|
||||
pub use credential_leases::*;
|
||||
pub use credential_state::*;
|
||||
|
||||
@@ -12,7 +12,6 @@ use gateway_core::engine::continuation::{
|
||||
NativeContinuationStoreError, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::error::SafeUpstreamValue;
|
||||
use gateway_core::operation::ProviderSessionState;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use redis::aio::ConnectionManager;
|
||||
@@ -25,6 +24,7 @@ use super::{namespace, resource_fingerprint};
|
||||
const CONTINUATION_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
const MAX_CONTINUATIONS: usize = 65_536;
|
||||
const MAX_RECORD_BYTES: usize = 64 * 1024;
|
||||
const MAX_INDEX_CLEANUP_PER_RECORD: usize = 64;
|
||||
|
||||
const RECORD_SCRIPT: &str = r#"
|
||||
local entry_key = KEYS[1]
|
||||
@@ -36,12 +36,13 @@ local ttl_seconds = tonumber(ARGV[4])
|
||||
local cutoff = tonumber(ARGV[5])
|
||||
local max_entries = tonumber(ARGV[6])
|
||||
local entry_prefix = ARGV[7]
|
||||
local cleanup_limit = tonumber(ARGV[8])
|
||||
|
||||
redis.call('SET', entry_key, payload, 'EX', ttl_seconds)
|
||||
redis.call('ZADD', index_key, score, entry_id)
|
||||
redis.call('EXPIRE', index_key, ttl_seconds)
|
||||
|
||||
local stale = redis.call('ZRANGEBYSCORE', index_key, '-inf', cutoff)
|
||||
local stale = redis.call('ZRANGEBYSCORE', index_key, '-inf', cutoff, 'LIMIT', 0, cleanup_limit)
|
||||
for _, stale_id in ipairs(stale) do
|
||||
redis.call('UNLINK', entry_prefix .. '{' .. stale_id .. '}')
|
||||
redis.call('ZREM', index_key, stale_id)
|
||||
@@ -49,6 +50,9 @@ end
|
||||
|
||||
local count = redis.call('ZCARD', index_key)
|
||||
local excess = count - max_entries
|
||||
if excess > cleanup_limit then
|
||||
excess = cleanup_limit
|
||||
end
|
||||
if excess > 0 then
|
||||
local oldest = redis.call('ZRANGE', index_key, 0, excess - 1)
|
||||
for _, old_id in ipairs(oldest) do
|
||||
@@ -110,6 +114,9 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
previous_response_id: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
Box::pin(async move {
|
||||
if previous_response_id.as_str().is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let fingerprint = self.entry_fingerprint(previous_response_id.as_str())?;
|
||||
let mut connection = self.connection.clone();
|
||||
let payload = redis::cmd("GET")
|
||||
@@ -128,6 +135,9 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
pin: NativeContinuationPin,
|
||||
) -> BoxFuture<'a, Result<(), NativeContinuationStoreError>> {
|
||||
Box::pin(async move {
|
||||
if pin.previous_response_id().as_str().is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let fingerprint = self.entry_fingerprint(pin.previous_response_id().as_str())?;
|
||||
let payload = encode_pin(&pin)?;
|
||||
if payload.len() > MAX_RECORD_BYTES {
|
||||
@@ -153,6 +163,7 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
.arg(cutoff)
|
||||
.arg(MAX_CONTINUATIONS)
|
||||
.arg(self.entry_prefix())
|
||||
.arg(MAX_INDEX_CLEANUP_PER_RECORD)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
@@ -186,8 +197,7 @@ fn decode_pin(
|
||||
serde_json::from_str(payload).map_err(|_| NativeContinuationStoreError)?;
|
||||
let provider = ProviderKind::new(wire.provider).map_err(|_| NativeContinuationStoreError)?;
|
||||
let account = ProviderAccountId::new(wire.account).map_err(|_| NativeContinuationStoreError)?;
|
||||
let upstream_response_id = SafeUpstreamValue::new(wire.upstream_response_id)
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
let upstream_response_id = PreviousResponseId::new(wire.upstream_response_id);
|
||||
let scope = parse_scope(&wire.scope)?;
|
||||
let mut pin = NativeContinuationPin::new(
|
||||
previous_response_id.clone(),
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::provider_ports::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingFlowPort, OAuthPendingPutOutcome,
|
||||
OAuthPendingTakeOutcome, ProviderStoreError, ProviderStoreErrorKind,
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome, ProviderStoreError,
|
||||
ProviderStoreErrorKind,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use redis::{Script, aio::ConnectionManager};
|
||||
@@ -31,7 +32,7 @@ redis.call('PEXPIREAT', KEYS[1], expires_at_ms)
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const TAKE_SCRIPT: &str = r#"
|
||||
const CLAIM_SCRIPT: &str = r#"
|
||||
local function equal_bytes(left, right)
|
||||
local different = math.abs(string.len(left) - string.len(right))
|
||||
local length = math.max(string.len(left), string.len(right))
|
||||
@@ -51,14 +52,82 @@ end
|
||||
if not equal_bytes(owner, ARGV[1]) then
|
||||
return {-1, ''}
|
||||
end
|
||||
local clock = redis.call('TIME')
|
||||
local now_ms = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000)
|
||||
local existing_claim = redis.call('HGET', KEYS[1], 'claim_fingerprint')
|
||||
if existing_claim ~= false then
|
||||
local claim_expires_at = redis.call('HGET', KEYS[1], 'claim_expires_at_epoch_millis')
|
||||
if claim_expires_at ~= false and tonumber(claim_expires_at) > now_ms then
|
||||
return {-2, ''}
|
||||
end
|
||||
redis.call('HDEL', KEYS[1], 'claim_fingerprint', 'claim_expires_at_epoch_millis')
|
||||
end
|
||||
local payload = redis.call('HGET', KEYS[1], 'provider_payload')
|
||||
if payload == false then
|
||||
return redis.error_reply('OAuth pending flow payload is missing')
|
||||
end
|
||||
redis.call('DEL', KEYS[1])
|
||||
redis.call('HSET', KEYS[1],
|
||||
'claim_fingerprint', ARGV[2],
|
||||
'claim_expires_at_epoch_millis', now_ms + tonumber(ARGV[3]))
|
||||
return {1, payload}
|
||||
"#;
|
||||
|
||||
const RELEASE_CLAIM_SCRIPT: &str = r#"
|
||||
local function equal_bytes(left, right)
|
||||
local different = math.abs(string.len(left) - string.len(right))
|
||||
local length = math.max(string.len(left), string.len(right))
|
||||
for index = 1, length do
|
||||
local left_byte = string.byte(left, index) or 0
|
||||
local right_byte = string.byte(right, index) or 0
|
||||
if left_byte ~= right_byte then
|
||||
different = different + 1
|
||||
end
|
||||
end
|
||||
return different == 0
|
||||
end
|
||||
local owner = redis.call('HGET', KEYS[1], 'owner_fingerprint')
|
||||
if owner == false then
|
||||
return 0
|
||||
end
|
||||
if not equal_bytes(owner, ARGV[1]) then
|
||||
return -1
|
||||
end
|
||||
local claim = redis.call('HGET', KEYS[1], 'claim_fingerprint')
|
||||
if claim == false or not equal_bytes(claim, ARGV[2]) then
|
||||
return -2
|
||||
end
|
||||
redis.call('HDEL', KEYS[1], 'claim_fingerprint', 'claim_expires_at_epoch_millis')
|
||||
return 1
|
||||
"#;
|
||||
|
||||
const CONSUME_CLAIM_SCRIPT: &str = r#"
|
||||
local function equal_bytes(left, right)
|
||||
local different = math.abs(string.len(left) - string.len(right))
|
||||
local length = math.max(string.len(left), string.len(right))
|
||||
for index = 1, length do
|
||||
local left_byte = string.byte(left, index) or 0
|
||||
local right_byte = string.byte(right, index) or 0
|
||||
if left_byte ~= right_byte then
|
||||
different = different + 1
|
||||
end
|
||||
end
|
||||
return different == 0
|
||||
end
|
||||
local owner = redis.call('HGET', KEYS[1], 'owner_fingerprint')
|
||||
if owner == false then
|
||||
return 0
|
||||
end
|
||||
if not equal_bytes(owner, ARGV[1]) then
|
||||
return -1
|
||||
end
|
||||
local claim = redis.call('HGET', KEYS[1], 'claim_fingerprint')
|
||||
if claim == false or not equal_bytes(claim, ARGV[2]) then
|
||||
return -2
|
||||
end
|
||||
redis.call('DEL', KEYS[1])
|
||||
return 1
|
||||
"#;
|
||||
|
||||
/// OAuth pending flow 的 Redis 原子持久化能力。
|
||||
#[derive(Clone)]
|
||||
pub struct RedisOAuthPendingFlowRepository {
|
||||
@@ -118,39 +187,115 @@ impl OAuthPendingFlowPort for RedisOAuthPendingFlowRepository {
|
||||
})
|
||||
}
|
||||
|
||||
fn take_if_owner<'a>(
|
||||
fn claim_if_owner<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
) -> futures::future::BoxFuture<'a, Result<OAuthPendingTakeOutcome, ProviderStoreError>> {
|
||||
claim: &'a OAuthPendingBinding,
|
||||
claim_ttl: Duration,
|
||||
) -> futures::future::BoxFuture<'a, Result<OAuthPendingClaimOutcome, ProviderStoreError>> {
|
||||
Box::pin(async move {
|
||||
let key = self.key(provider_kind, flow);
|
||||
let owner = provider_scoped_fingerprint(provider_kind, owner);
|
||||
let claim = provider_scoped_fingerprint(provider_kind, claim);
|
||||
let claim_ttl = ttl_millis(claim_ttl)
|
||||
.map_err(|_| provider_invalid("encode OAuth pending claim TTL"))?;
|
||||
let mut connection = self.connection.clone();
|
||||
let (status, payload): (i64, Vec<u8>) = Script::new(TAKE_SCRIPT)
|
||||
let (status, payload): (i64, Vec<u8>) = Script::new(CLAIM_SCRIPT)
|
||||
.key(key)
|
||||
.arg(owner)
|
||||
.arg(claim)
|
||||
.arg(claim_ttl)
|
||||
.invoke_async(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("take OAuth pending flow"))?;
|
||||
.map_err(|_| provider_unavailable("claim OAuth pending flow"))?;
|
||||
match status {
|
||||
0 => Ok(OAuthPendingTakeOutcome::NotFound),
|
||||
-1 => Ok(OAuthPendingTakeOutcome::OwnerMismatch),
|
||||
0 => Ok(OAuthPendingClaimOutcome::NotFound),
|
||||
-1 => Ok(OAuthPendingClaimOutcome::OwnerMismatch),
|
||||
-2 => Ok(OAuthPendingClaimOutcome::InProgress),
|
||||
1 => {
|
||||
let value: serde_json::Value = serde_json::from_slice(&payload)
|
||||
.map_err(|_| provider_invalid("decode OAuth pending payload"))?;
|
||||
let serde_json::Value::Object(fields) = value else {
|
||||
return Err(provider_invalid("decode OAuth pending payload"));
|
||||
};
|
||||
Ok(OAuthPendingTakeOutcome::Taken(
|
||||
Ok(OAuthPendingClaimOutcome::Claimed(
|
||||
gateway_core::engine::credential::OpaqueProviderData::new(fields),
|
||||
))
|
||||
}
|
||||
_ => Err(provider_invalid("decode OAuth pending outcome")),
|
||||
_ => Err(provider_invalid("decode OAuth pending claim outcome")),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn release_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> futures::future::BoxFuture<'a, Result<OAuthPendingReleaseOutcome, ProviderStoreError>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
let key = self.key(provider_kind, flow);
|
||||
let owner = provider_scoped_fingerprint(provider_kind, owner);
|
||||
let claim = provider_scoped_fingerprint(provider_kind, claim);
|
||||
let mut connection = self.connection.clone();
|
||||
let status = Script::new(RELEASE_CLAIM_SCRIPT)
|
||||
.key(key)
|
||||
.arg(owner)
|
||||
.arg(claim)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("release OAuth pending claim"))?;
|
||||
Ok(match status {
|
||||
0 => OAuthPendingReleaseOutcome::NotFound,
|
||||
-1 => OAuthPendingReleaseOutcome::OwnerMismatch,
|
||||
-2 => OAuthPendingReleaseOutcome::ClaimMismatch,
|
||||
1 => OAuthPendingReleaseOutcome::Released,
|
||||
_ => {
|
||||
return Err(provider_invalid(
|
||||
"decode OAuth pending claim release outcome",
|
||||
));
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn consume_claim<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
flow: &'a OAuthPendingBinding,
|
||||
owner: &'a OAuthPendingBinding,
|
||||
claim: &'a OAuthPendingBinding,
|
||||
) -> futures::future::BoxFuture<'a, Result<OAuthPendingConsumeOutcome, ProviderStoreError>>
|
||||
{
|
||||
Box::pin(async move {
|
||||
let key = self.key(provider_kind, flow);
|
||||
let owner = provider_scoped_fingerprint(provider_kind, owner);
|
||||
let claim = provider_scoped_fingerprint(provider_kind, claim);
|
||||
let mut connection = self.connection.clone();
|
||||
let status = Script::new(CONSUME_CLAIM_SCRIPT)
|
||||
.key(key)
|
||||
.arg(owner)
|
||||
.arg(claim)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| provider_unavailable("consume OAuth pending claim"))?;
|
||||
Ok(match status {
|
||||
0 => OAuthPendingConsumeOutcome::NotFound,
|
||||
-1 => OAuthPendingConsumeOutcome::OwnerMismatch,
|
||||
-2 => OAuthPendingConsumeOutcome::ClaimMismatch,
|
||||
1 => OAuthPendingConsumeOutcome::Consumed,
|
||||
_ => {
|
||||
return Err(provider_invalid(
|
||||
"decode OAuth pending claim consume outcome",
|
||||
));
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_scoped_fingerprint(
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
ClientAdmissionRequest, ClientAdmissionRestoreResult,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
};
|
||||
use gateway_core::engine::{CancellationToken, ModelRequestId};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_store::redis::{BufferedClientAdmissionPort, BufferedProviderCircuitPort};
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingCoordination {
|
||||
operations: Mutex<Vec<&'static str>>,
|
||||
}
|
||||
|
||||
impl RecordingCoordination {
|
||||
fn record(&self, operation: &'static str) {
|
||||
self.operations
|
||||
.lock()
|
||||
.expect("coordination operations lock")
|
||||
.push(operation);
|
||||
}
|
||||
|
||||
fn operations(&self) -> Vec<&'static str> {
|
||||
self.operations
|
||||
.lock()
|
||||
.expect("coordination operations lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientAdmissionPort for RecordingCoordination {
|
||||
fn admit(
|
||||
&self,
|
||||
_: ClientAdmissionRequest,
|
||||
) -> BoxFuture<'_, Result<ClientAdmissionDecision, ClientAdmissionError>> {
|
||||
Box::pin(async { Ok(ClientAdmissionDecision::Granted) })
|
||||
}
|
||||
|
||||
fn release<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientApiKeyId,
|
||||
_: &'a ModelRequestId,
|
||||
) -> BoxFuture<'a, Result<bool, ClientAdmissionError>> {
|
||||
Box::pin(async move {
|
||||
self.record("admission");
|
||||
Ok(true)
|
||||
})
|
||||
}
|
||||
|
||||
fn restore(
|
||||
&self,
|
||||
_: ClientAdmissionRecovery,
|
||||
) -> BoxFuture<'_, Result<ClientAdmissionRestoreResult, ClientAdmissionError>> {
|
||||
Box::pin(async { Ok(ClientAdmissionRestoreResult::default()) })
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderCircuitPort for RecordingCoordination {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<ProviderCircuitDecision, ProviderCircuitError>> {
|
||||
Box::pin(async { Ok(ProviderCircuitDecision::Allow) })
|
||||
}
|
||||
|
||||
fn observe_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async move {
|
||||
self.record("circuit_failure");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn observe_success<'a>(
|
||||
&'a self,
|
||||
_: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<(), ProviderCircuitError>> {
|
||||
Box::pin(async move {
|
||||
self.record("circuit_success");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_recoverable_coordination_queues_should_drop_writes_without_waiting() {
|
||||
let inner = Arc::new(RecordingCoordination::default());
|
||||
let (admissions, _admission_writer) = BufferedClientAdmissionPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(1).expect("capacity"),
|
||||
);
|
||||
let (circuits, _circuit_writer) = BufferedProviderCircuitPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(1).expect("capacity"),
|
||||
);
|
||||
let client = ClientApiKeyId::new("key_buffer_test").expect("client key");
|
||||
let request = ModelRequestId::new("req_buffer_test").expect("request ID");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
|
||||
tokio::time::timeout(Duration::from_millis(50), async {
|
||||
admissions
|
||||
.release(&client, &request)
|
||||
.await
|
||||
.expect("first admission enqueue");
|
||||
admissions
|
||||
.release(&client, &request)
|
||||
.await
|
||||
.expect("full admission queue remains fail-open");
|
||||
circuits
|
||||
.observe_success(&provider)
|
||||
.await
|
||||
.expect("first circuit enqueue");
|
||||
circuits
|
||||
.observe_failure(&provider)
|
||||
.await
|
||||
.expect("full circuit queue remains fail-open");
|
||||
})
|
||||
.await
|
||||
.expect("coordination enqueue must never wait for Redis");
|
||||
|
||||
assert!(inner.operations().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redis_coordination_writers_should_flush_each_side_effect() {
|
||||
let inner = Arc::new(RecordingCoordination::default());
|
||||
let (admissions, admission_writer) = BufferedClientAdmissionPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let (circuits, circuit_writer) = BufferedProviderCircuitPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let client = ClientApiKeyId::new("key_writer_test").expect("client key");
|
||||
let request = ModelRequestId::new("req_writer_test").expect("request ID");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
admissions
|
||||
.release(&client, &request)
|
||||
.await
|
||||
.expect("enqueue admission release");
|
||||
circuits
|
||||
.observe_success(&provider)
|
||||
.await
|
||||
.expect("enqueue circuit feedback");
|
||||
let cancellation = CancellationToken::new();
|
||||
let tasks = [
|
||||
spawn_writer(Arc::new(admission_writer), cancellation.clone()),
|
||||
spawn_writer(Arc::new(circuit_writer), cancellation.clone()),
|
||||
];
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if inner.operations().len() == 2 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writers should drain queued coordination writes");
|
||||
cancellation.cancel();
|
||||
for task in tasks {
|
||||
task.await
|
||||
.expect("coordination writer task")
|
||||
.expect("coordination writer cancellation");
|
||||
}
|
||||
|
||||
let mut operations = inner.operations();
|
||||
operations.sort_unstable();
|
||||
assert_eq!(operations, ["admission", "circuit_success"]);
|
||||
}
|
||||
|
||||
fn spawn_writer<T>(
|
||||
writer: Arc<T>,
|
||||
cancellation: CancellationToken,
|
||||
) -> tokio::task::JoinHandle<Result<(), gateway_core::task::WorkerTaskError>>
|
||||
where
|
||||
T: gateway_core::task::DaemonTask + 'static,
|
||||
{
|
||||
tokio::spawn(async move { writer.run(cancellation).await })
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::engine::{
|
||||
AttemptRecord, CancellationToken, ExecutionStore, IntermediateFailure,
|
||||
ModelRequestFinalization, ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport,
|
||||
UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
OpaqueUpstreamValue, ProviderError, ProviderErrorKind, StoreError, StoreErrorKind,
|
||||
};
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
use gateway_core::task::DaemonTask as _;
|
||||
use gateway_store::postgres::BufferedExecutionStore;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingStore {
|
||||
operations: Mutex<Vec<&'static str>>,
|
||||
fail_once: Mutex<Option<&'static str>>,
|
||||
}
|
||||
|
||||
impl RecordingStore {
|
||||
fn failing_once(operation: &'static str) -> Self {
|
||||
Self {
|
||||
operations: Mutex::new(Vec::new()),
|
||||
fail_once: Mutex::new(Some(operation)),
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&self, operation: &'static str) -> Result<(), StoreError> {
|
||||
self.operations
|
||||
.lock()
|
||||
.expect("operations lock")
|
||||
.push(operation);
|
||||
let mut fail_once = self.fail_once.lock().expect("failure lock");
|
||||
if fail_once.as_ref().is_some_and(|value| *value == operation) {
|
||||
*fail_once = None;
|
||||
return Err(StoreError::new(StoreErrorKind::Unavailable));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionStore for RecordingStore {
|
||||
async fn create_model_request(&self, _: NewModelRequest) -> Result<(), StoreError> {
|
||||
self.record("create")
|
||||
}
|
||||
|
||||
async fn record_attempt(&self, _: AttemptRecord) -> Result<(), StoreError> {
|
||||
self.record("attempt")
|
||||
}
|
||||
|
||||
async fn mark_send_state(
|
||||
&self,
|
||||
_: &ModelRequestId,
|
||||
_: UpstreamSendState,
|
||||
) -> Result<(), StoreError> {
|
||||
self.record("send")
|
||||
}
|
||||
|
||||
async fn mark_downstream_committed(
|
||||
&self,
|
||||
_: &ModelRequestId,
|
||||
_: SystemTime,
|
||||
_: Option<u16>,
|
||||
) -> Result<(), StoreError> {
|
||||
self.record("commit")
|
||||
}
|
||||
|
||||
async fn record_client_status(&self, _: &ModelRequestId, _: u16) -> Result<(), StoreError> {
|
||||
self.record("status")
|
||||
}
|
||||
|
||||
async fn record_intermediate_failure(&self, _: IntermediateFailure) -> Result<(), StoreError> {
|
||||
self.record("intermediate_failure")
|
||||
}
|
||||
|
||||
async fn record_probe_failure(&self, _: ProbeFailure) -> Result<(), StoreError> {
|
||||
self.record("probe_failure")
|
||||
}
|
||||
|
||||
async fn finalize_model_request(&self, _: ModelRequestFinalization) -> Result<(), StoreError> {
|
||||
self.record("finalize")
|
||||
}
|
||||
|
||||
async fn recover_expired(&self, _: SystemTime) -> Result<RecoveryReport, StoreError> {
|
||||
self.record("recover")?;
|
||||
Ok(RecoveryReport::default())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_observation_queue_never_waits_for_the_database() {
|
||||
let inner = Arc::new(RecordingStore::default());
|
||||
let (store, _writer) = BufferedExecutionStore::with_capacity(
|
||||
Arc::clone(&inner),
|
||||
NonZeroUsize::new(1).expect("capacity"),
|
||||
);
|
||||
let request_id = ModelRequestId::new("req_queue_full").expect("request id");
|
||||
|
||||
tokio::time::timeout(
|
||||
Duration::from_millis(50),
|
||||
store.mark_send_state(&request_id, UpstreamSendState::Sent),
|
||||
)
|
||||
.await
|
||||
.expect("first enqueue must not wait")
|
||||
.expect("first enqueue is fail-open");
|
||||
tokio::time::timeout(
|
||||
Duration::from_millis(50),
|
||||
store.record_client_status(&request_id, 200),
|
||||
)
|
||||
.await
|
||||
.expect("full queue must not wait")
|
||||
.expect("full queue is fail-open");
|
||||
|
||||
assert!(inner.operations.lock().expect("operations lock").is_empty());
|
||||
assert_eq!(store.stats().queued_items, 1);
|
||||
assert_eq!(store.stats().dropped_total, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_writer_persists_commands_in_enqueue_order() {
|
||||
let inner = Arc::new(RecordingStore::default());
|
||||
let (store, writer) = BufferedExecutionStore::with_capacity(
|
||||
Arc::clone(&inner),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let request_id = ModelRequestId::new("req_queue_order").expect("request id");
|
||||
store
|
||||
.mark_send_state(&request_id, UpstreamSendState::Sent)
|
||||
.await
|
||||
.expect("enqueue send state");
|
||||
store
|
||||
.record_client_status(&request_id, 200)
|
||||
.await
|
||||
.expect("enqueue client status");
|
||||
|
||||
let cancellation = CancellationToken::new();
|
||||
let writer = Arc::new(writer);
|
||||
let task = tokio::spawn({
|
||||
let writer = Arc::clone(&writer);
|
||||
let cancellation = cancellation.clone();
|
||||
async move { writer.run(cancellation).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if inner.operations.lock().expect("operations lock").len() == 2 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writer should drain queued observations");
|
||||
cancellation.cancel();
|
||||
task.await
|
||||
.expect("writer task")
|
||||
.expect("writer cancellation");
|
||||
|
||||
assert_eq!(
|
||||
*inner.operations.lock().expect("operations lock"),
|
||||
["send", "status"]
|
||||
);
|
||||
assert_eq!(store.stats().persisted_total, 2);
|
||||
assert_eq!(store.stats().queued_items, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_writer_continues_after_a_database_write_failure() {
|
||||
let inner = Arc::new(RecordingStore::failing_once("send"));
|
||||
let (store, writer) = BufferedExecutionStore::with_capacity(
|
||||
Arc::clone(&inner),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let request_id = ModelRequestId::new("req_queue_failure").expect("request id");
|
||||
store
|
||||
.mark_send_state(&request_id, UpstreamSendState::Sent)
|
||||
.await
|
||||
.expect("enqueue failed write");
|
||||
store
|
||||
.record_client_status(&request_id, 200)
|
||||
.await
|
||||
.expect("enqueue following write");
|
||||
|
||||
let cancellation = CancellationToken::new();
|
||||
cancellation.cancel();
|
||||
writer
|
||||
.run(cancellation)
|
||||
.await
|
||||
.expect("shutdown drains observations");
|
||||
|
||||
assert_eq!(
|
||||
*inner.operations.lock().expect("operations lock"),
|
||||
["send", "status"]
|
||||
);
|
||||
assert_eq!(store.stats().write_failure_total, 1);
|
||||
assert_eq!(store.stats().persisted_total, 1);
|
||||
assert_eq!(store.stats().queued_items, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_drains_observations_already_accepted_by_the_queue() {
|
||||
let inner = Arc::new(RecordingStore::default());
|
||||
let (store, writer) = BufferedExecutionStore::with_capacity(
|
||||
Arc::clone(&inner),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let request_id = ModelRequestId::new("req_queue_shutdown").expect("request id");
|
||||
store
|
||||
.mark_send_state(&request_id, UpstreamSendState::Sent)
|
||||
.await
|
||||
.expect("enqueue send state");
|
||||
store
|
||||
.record_client_status(&request_id, 200)
|
||||
.await
|
||||
.expect("enqueue client status");
|
||||
|
||||
let cancellation = CancellationToken::new();
|
||||
cancellation.cancel();
|
||||
writer
|
||||
.run(cancellation)
|
||||
.await
|
||||
.expect("shutdown drains observations");
|
||||
|
||||
assert_eq!(
|
||||
*inner.operations.lock().expect("operations lock"),
|
||||
["send", "status"]
|
||||
);
|
||||
assert_eq!(store.stats().persisted_total, 2);
|
||||
assert_eq!(store.stats().dropped_total, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_byte_budget_drops_payload_without_waiting_for_the_database() {
|
||||
let inner = Arc::new(RecordingStore::default());
|
||||
let (store, _writer) = BufferedExecutionStore::with_limits(
|
||||
Arc::clone(&inner),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
NonZeroUsize::new(1_024).expect("byte capacity"),
|
||||
);
|
||||
let provider_error =
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_upstream_code(OpaqueUpstreamValue::new("x".repeat(2_048)));
|
||||
let failure = ProbeFailure {
|
||||
provider_kind: ProviderKind::new("openai").expect("provider"),
|
||||
account_id: ProviderAccountId::new("acct_byte_budget").expect("account"),
|
||||
upstream_model_id: UpstreamModelId::new("gpt-byte-budget").expect("model"),
|
||||
error: provider_error,
|
||||
latency: Duration::from_millis(1),
|
||||
};
|
||||
|
||||
tokio::time::timeout(
|
||||
Duration::from_millis(50),
|
||||
store.record_probe_failure(failure),
|
||||
)
|
||||
.await
|
||||
.expect("byte budget must not wait")
|
||||
.expect("byte budget is fail-open");
|
||||
|
||||
assert!(inner.operations.lock().expect("operations lock").is_empty());
|
||||
assert_eq!(store.stats().queued_items, 0);
|
||||
assert_eq!(store.stats().dropped_total, 1);
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
mod coordination_buffer;
|
||||
mod execution_buffer;
|
||||
mod postgres;
|
||||
mod redis;
|
||||
mod support;
|
||||
|
||||
@@ -250,15 +250,24 @@ async fn core_adapter_should_persist_calculated_cost_exactly() {
|
||||
|
||||
let mut finalization = successful_core_finalization("req_calculated_cost");
|
||||
finalization.websocket_pool = Some("reuse".to_owned());
|
||||
finalization.service_tier = Some("priority".to_owned());
|
||||
finalization.cost = CalculatedCost::from_usd_ticks(12_345)
|
||||
.expect("calculated cost")
|
||||
.into_estimate();
|
||||
ExecutionStore::finalize_model_request(&repository, finalization)
|
||||
.await
|
||||
.expect("persist calculated cost");
|
||||
let persisted: (String, String, String, String, String, String) = sqlx::query_as(
|
||||
let persisted: (
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select cost_source, cost_amount::text, cost_currency, upstream_transport, http_version,
|
||||
websocket_pool
|
||||
websocket_pool, service_tier
|
||||
from model_requests where id = 'req_calculated_cost'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
@@ -274,6 +283,7 @@ async fn core_adapter_should_persist_calculated_cost_exactly() {
|
||||
"websocket".to_owned(),
|
||||
"HTTP/2".to_owned(),
|
||||
"reuse".to_owned(),
|
||||
Some("priority".to_owned()),
|
||||
)
|
||||
);
|
||||
database.close().await;
|
||||
@@ -376,6 +386,7 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
|
||||
upstream_transport: Some("websocket".to_owned()),
|
||||
http_version: Some("HTTP/2".to_owned()),
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
@@ -388,6 +399,37 @@ fn successful_core_finalization(id: &str) -> CoreModelRequestFinalization {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn core_adapter_persists_opaque_response_ids_as_bytes() {
|
||||
let Some(database) = TestDatabase::create("execution_opaque_response_id").await else {
|
||||
return;
|
||||
};
|
||||
seed_running_request(&database.pool, "req_opaque_response_id")
|
||||
.await
|
||||
.expect("seed model request");
|
||||
let store = PgExecutionStore::new(database.pool.clone());
|
||||
let response_id = format!("resp_{}\0opaque", "x".repeat(4_096));
|
||||
let mut finalization = successful_core_finalization("req_opaque_response_id");
|
||||
finalization.client_response_id = Some(response_id.clone());
|
||||
finalization.upstream_response_id = Some(response_id.clone());
|
||||
|
||||
ExecutionStore::finalize_model_request(&store, finalization)
|
||||
.await
|
||||
.expect("persist opaque response IDs");
|
||||
|
||||
let persisted: (Option<Vec<u8>>, Option<Vec<u8>>) = sqlx::query_as(
|
||||
"select client_response_id, upstream_response_id
|
||||
from model_requests where id = 'req_opaque_response_id'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load opaque response IDs");
|
||||
assert_eq!(persisted.0.as_deref(), Some(response_id.as_bytes()));
|
||||
assert_eq!(persisted.1.as_deref(), Some(response_id.as_bytes()));
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn core_adapter_should_persist_only_object_provider_observation() {
|
||||
let Some(database) = TestDatabase::create("execution_provider_observation").await else {
|
||||
|
||||
@@ -58,6 +58,30 @@ impl TestDatabase {
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply terminal migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0002_drop_opaque_response_id_index.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply opaque response ID migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0003_store_opaque_response_ids_as_bytes.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply opaque response storage migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0004_add_model_request_service_tier.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply model request service tier migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0005_allow_quota_exhausted_cooldown.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply quota exhausted cooldown migration");
|
||||
migration.commit().await.expect("commit terminal migration");
|
||||
Some(Self {
|
||||
admin,
|
||||
@@ -80,7 +104,7 @@ impl TestDatabase {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_migrate_should_apply_initial_migration_once_and_reopen_cleanly() {
|
||||
async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly() {
|
||||
let Some(database_url) = crate::support::test_env("CPR_TEST_DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
@@ -104,7 +128,7 @@ async fn connect_and_migrate_should_apply_initial_migration_once_and_reopen_clea
|
||||
.to_string();
|
||||
let first = connect_and_migrate(&isolated_url, gateway_store::StorePoolConfig::default())
|
||||
.await
|
||||
.expect("apply initial migration through production migrator");
|
||||
.expect("apply migrations through production migrator");
|
||||
let first_table_count = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from information_schema.tables where table_schema = 'public'",
|
||||
)
|
||||
@@ -121,6 +145,28 @@ async fn connect_and_migrate_should_apply_initial_migration_once_and_reopen_clea
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("count successful migrations");
|
||||
let response_id_types = sqlx::query_scalar::<_, String>(
|
||||
"select data_type
|
||||
from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'model_requests'
|
||||
and column_name in ('client_response_id', 'upstream_response_id')
|
||||
order by column_name",
|
||||
)
|
||||
.fetch_all(&second)
|
||||
.await
|
||||
.expect("load opaque response ID column types");
|
||||
let raw_response_id_index_exists = sqlx::query_scalar::<_, bool>(
|
||||
"select exists (
|
||||
select 1
|
||||
from pg_indexes
|
||||
where schemaname = 'public'
|
||||
and indexname = 'model_requests_client_response_uq'
|
||||
)",
|
||||
)
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("check removed raw response ID index");
|
||||
second.close().await;
|
||||
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
@@ -131,16 +177,25 @@ async fn connect_and_migrate_should_apply_initial_migration_once_and_reopen_clea
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (8, 1));
|
||||
assert_eq!((first_table_count, migration_count), (8, 5));
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initial_migration_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = include_str!("../../../../migrations/0001_initial.sql")
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = [
|
||||
include_str!("../../../../migrations/0001_initial.sql"),
|
||||
include_str!("../../../../migrations/0002_drop_opaque_response_id_index.sql"),
|
||||
include_str!("../../../../migrations/0003_store_opaque_response_ids_as_bytes.sql"),
|
||||
include_str!("../../../../migrations/0004_add_model_request_service_tier.sql"),
|
||||
include_str!("../../../../migrations/0005_allow_quota_exhausted_cooldown.sql"),
|
||||
]
|
||||
.into_iter()
|
||||
.flat_map(str::lines)
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
|
||||
assert_eq!(transaction_statements, 0);
|
||||
}
|
||||
|
||||
@@ -62,6 +62,55 @@ fn postgres_admin_observability_adapter_implements_terminal_port() {
|
||||
assert_port::<PgAdminObservabilityStore>();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
let Some(database) = TestDatabase::create("observability_opaque_response_id").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now)
|
||||
.await
|
||||
.expect("seed observability facts");
|
||||
let response_id = format!("resp_{}\0opaque", "x".repeat(4_096));
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set client_response_id = $1, upstream_response_id = $1
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.bind(response_id.as_bytes().to_vec())
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("persist opaque response IDs");
|
||||
let range = ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1))
|
||||
.expect("observability range");
|
||||
let repository = PgObservabilityRepository::new(database.pool.clone());
|
||||
|
||||
let records = repository
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
response_id: Some(response_id.clone()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1).expect("page"),
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("filter opaque response ID");
|
||||
assert_eq!(records.total, 1);
|
||||
assert_eq!(
|
||||
records.items[0].client_response_id.as_deref(),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
records.items[0].upstream_response_id.as_deref(),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_account_metrics_should_partition_available_and_unavailable_accounts() {
|
||||
let Some(database) = TestDatabase::create("dashboard_account_metrics").await else {
|
||||
@@ -73,16 +122,27 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability,
|
||||
availability_reason, cooldown_until,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_available', 'openai', 'available', 'user-available', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_usage_limit', 'openai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', $1, $1, $1)",
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
|
||||
null, null, $1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
@@ -105,10 +165,11 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
metrics.active,
|
||||
metrics.unavailable,
|
||||
metrics.expired,
|
||||
metrics.quota_exhausted,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
),
|
||||
(4, 3, 1, 3, 1, 1, 1),
|
||||
(6, 5, 2, 3, 1, 1, 1, 1),
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -138,6 +199,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
assert_eq!(facts[0].upstream_model_id, "gpt-5.5");
|
||||
assert_eq!(facts[0].input_tokens, Some(800));
|
||||
assert_eq!(facts[0].output_tokens, Some(200));
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
let store = PgAdminObservabilityStore::new(database.pool.clone(), None);
|
||||
@@ -150,6 +212,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
.await
|
||||
.expect("admin calculated usage billing facts");
|
||||
assert_eq!(facts.len(), 1);
|
||||
assert_eq!(facts[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(facts[0].total.amount.as_str(), "1.25");
|
||||
|
||||
database.close().await;
|
||||
@@ -197,6 +260,10 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
1,
|
||||
);
|
||||
assert_eq!(dashboard.recent_requests.len(), 1);
|
||||
assert_eq!(
|
||||
dashboard.recent_requests[0].service_tier.as_deref(),
|
||||
Some("priority")
|
||||
);
|
||||
assert_eq!(
|
||||
dashboard.recent_requests[0].outcome,
|
||||
admin_observability::RequestOutcome::Succeeded,
|
||||
@@ -270,6 +337,10 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
assert_eq!(first_page.total, 1);
|
||||
assert_eq!(first_page.items.len(), 1);
|
||||
assert!(first_page.next_cursor.is_none());
|
||||
assert_eq!(
|
||||
first_page.items[0].service_tier.as_deref(),
|
||||
Some("priority")
|
||||
);
|
||||
|
||||
let filtered = store
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
@@ -296,6 +367,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
.await
|
||||
.expect("fully filtered usage page");
|
||||
assert_eq!(filtered.total, 1);
|
||||
assert_eq!(filtered.items[0].service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(
|
||||
filtered.items[0].upstream_request_id.as_deref(),
|
||||
Some("upstream_req_success")
|
||||
@@ -328,6 +400,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
detail.request.outcome,
|
||||
admin_observability::RequestOutcome::Succeeded
|
||||
);
|
||||
assert_eq!(detail.request.service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(detail.attempts.len(), 1);
|
||||
assert_eq!(
|
||||
detail.attempts[0].outcome,
|
||||
@@ -507,6 +580,10 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
);
|
||||
assert_eq!(dashboard.recent_requests.len(), 1);
|
||||
assert_eq!(dashboard.recent_requests[0].id, "req_observe_success");
|
||||
assert_eq!(
|
||||
dashboard.recent_requests[0].service_tier.as_deref(),
|
||||
Some("priority")
|
||||
);
|
||||
assert_eq!(
|
||||
dashboard
|
||||
.trend
|
||||
@@ -581,6 +658,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
assert_eq!(
|
||||
(
|
||||
successful_image.websocket_pool.as_deref(),
|
||||
successful_image.service_tier.as_deref(),
|
||||
successful_image
|
||||
.provider_account_authentication_kind
|
||||
.as_deref(),
|
||||
@@ -591,6 +669,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
),
|
||||
(
|
||||
Some("reuse"),
|
||||
Some("priority"),
|
||||
Some("oauth"),
|
||||
Some(31),
|
||||
Some(9),
|
||||
@@ -726,6 +805,7 @@ async fn seed_observability_facts(
|
||||
client_transport, requested_model_id,
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, websocket_pool,
|
||||
service_tier,
|
||||
attempt_count,
|
||||
upstream_send_state, downstream_committed_at, outcome, client_status_code,
|
||||
upstream_status_code, client_response_id, upstream_request_id, upstream_response_id,
|
||||
@@ -738,7 +818,7 @@ async fn seed_observability_facts(
|
||||
'req_observe_success', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'acct_observe', 'upstream-model',
|
||||
'http_sse', 'reuse', 1, 'sent', $1 - interval '19 minutes', 'succeeded', 200, 200,
|
||||
'http_sse', 'reuse', 'priority', 1, 'sent', $1 - interval '19 minutes', 'succeeded', 200, 200,
|
||||
'resp_observe_success', 'upstream_req_success', 'upstream_resp_success',
|
||||
100, 20, 40, 3, 5, 31, 9, 120, true, true,
|
||||
'provider_reported', 1.25, 'USD', 120, 900,
|
||||
@@ -800,13 +880,13 @@ async fn seed_calculated_billing_facts(
|
||||
client_transport, requested_model_id, provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, attempt_count,
|
||||
upstream_send_state, downstream_committed_at, outcome, client_status_code,
|
||||
input_tokens, output_tokens, cached_tokens, cache_write_tokens, total_tokens,
|
||||
input_tokens, output_tokens, cached_tokens, cache_write_tokens, total_tokens, service_tier,
|
||||
cost_source, cost_amount, cost_currency, started_at, deadline_at, completed_at
|
||||
) values (
|
||||
'req_observe_calculated', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe', 'acct_observe', 'gpt-5.5',
|
||||
'http_sse', 1, 'sent', $1 - interval '29 minutes', 'succeeded', 200,
|
||||
800, 200, 0, 0, 1000, 'calculated', 1.25, 'USD',
|
||||
800, 200, 0, 0, 1000, 'priority', 'calculated', 1.25, 'USD',
|
||||
$1 - interval '30 minutes', $1 + interval '10 minutes', $1 - interval '29 minutes'
|
||||
)",
|
||||
)
|
||||
|
||||
@@ -17,9 +17,10 @@ use gateway_admin::{
|
||||
ports::store::AccountStore,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
|
||||
OpaqueProviderData, PlaintextCredential, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, ProviderAccountId,
|
||||
ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_store::{
|
||||
@@ -769,6 +770,25 @@ async fn core_refresh_cas_updates_profile_and_credential_under_one_revision() {
|
||||
assert_eq!(row.3["access_token"], "after-secret");
|
||||
assert_eq!(row.4, 2);
|
||||
|
||||
assert!(
|
||||
repository
|
||||
.load_credential(
|
||||
&account_id,
|
||||
CredentialRevision::new(1).expect("stale credential revision"),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let current = repository
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.expect("load current credential without a caller revision");
|
||||
assert_eq!(current.account.revision().get(), 2);
|
||||
assert_eq!(
|
||||
current.credential.expose_to_provider()["access_token"],
|
||||
"after-secret"
|
||||
);
|
||||
|
||||
let stale = CredentialCasUpdate::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(1).expect("stale credential revision"),
|
||||
@@ -879,6 +899,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: wrong_scope,
|
||||
profile: profile("acct_admin_a", "wrong scope"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update("acct_admin_a", 1, "wrong-scope-secret"),
|
||||
audit: audit("audit_wrong_scope", "rotate", "acct_admin_a"),
|
||||
})
|
||||
@@ -893,10 +914,24 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
));
|
||||
assert_eq!(current_revision(&database.pool).await, 2);
|
||||
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = 'expired', availability_reason = 'credential_expired'
|
||||
where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed stale credential state");
|
||||
|
||||
let rotation = repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: scope.clone(),
|
||||
profile: profile("acct_admin_a", "rotated account"),
|
||||
replacement_identity: Some(ProviderAccountIdentity::new(
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)),
|
||||
credential: credential_update("acct_admin_a", 1, "rotated-secret"),
|
||||
audit: audit("audit_account_rotate", "rotate", "acct_admin_a"),
|
||||
})
|
||||
@@ -904,6 +939,71 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.expect("rotate provider account");
|
||||
assert_eq!(rotation.config_revision.get(), 3);
|
||||
assert_eq!(rotation.credential_revision.get(), 2);
|
||||
let restored: (
|
||||
String,
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select availability, availability_reason, cooldown_until,
|
||||
upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load restored account state");
|
||||
assert_eq!(
|
||||
restored,
|
||||
(
|
||||
"ready".to_owned(),
|
||||
None,
|
||||
None,
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)
|
||||
);
|
||||
|
||||
let identity_conflict = repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: scope.clone(),
|
||||
profile: profile("acct_admin_a", "must roll back"),
|
||||
replacement_identity: Some(ProviderAccountIdentity::new(
|
||||
"user-admin-b".to_owned(),
|
||||
None,
|
||||
)),
|
||||
credential: credential_update("acct_admin_a", 2, "must-not-persist"),
|
||||
audit: audit(
|
||||
"audit_account_identity_conflict",
|
||||
"reauthorize",
|
||||
"acct_admin_a",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect_err("upstream identity collision must roll back rotation");
|
||||
assert!(matches!(
|
||||
identity_conflict,
|
||||
StoreError::Conflict {
|
||||
kind: ConflictKind::InvalidTransition,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(current_revision(&database.pool).await, 3);
|
||||
let unchanged: (String, serde_json::Value, i64, String, Option<String>) = sqlx::query_as(
|
||||
"select name, provider_credentials_json, credential_revision,
|
||||
upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load account after identity collision");
|
||||
assert_eq!(unchanged.0, "rotated account");
|
||||
assert_eq!(unchanged.1["access_token"], "rotated-secret");
|
||||
assert_eq!(unchanged.2, 2);
|
||||
assert_eq!(unchanged.3, "user-admin-rebound");
|
||||
assert_eq!(unchanged.4.as_deref(), Some("workspace-admin-rebound"));
|
||||
|
||||
let revision = repository
|
||||
.set_provider_account_enabled_admin(SetProviderAccountEnabled {
|
||||
@@ -946,6 +1046,135 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
let Some(database) = TestDatabase::create("provider_account_rotation_quota").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
|
||||
exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
exhausted.availability_reason = Some("quota_exhausted".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![exhausted],
|
||||
audit: audit(
|
||||
"audit_rotation_quota_import",
|
||||
"import",
|
||||
"acct_rotation_quota",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("import exhausted account");
|
||||
|
||||
repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope,
|
||||
profile: profile("acct_rotation_quota", "reauthorized account"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update("acct_rotation_quota", 1, "reauthorized-secret"),
|
||||
audit: audit(
|
||||
"audit_rotation_quota_rotate",
|
||||
"reauthorize",
|
||||
"acct_rotation_quota",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("rotate exhausted account credential");
|
||||
|
||||
let availability: (String, Option<String>) = sqlx::query_as(
|
||||
"select availability, availability_reason from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(
|
||||
availability,
|
||||
(
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let Some(database) = TestDatabase::create("provider_account_disabled_refresh").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
let account_id = ProviderAccountId::new("acct_disabled_refresh").expect("account ID");
|
||||
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
|
||||
disabled.enabled = false;
|
||||
disabled.availability = AccountAvailability::Expired;
|
||||
disabled.availability_reason = Some("credential_expired".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![disabled],
|
||||
audit: audit(
|
||||
"audit_disabled_refresh_import",
|
||||
"import",
|
||||
account_id.as_str(),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("import disabled account");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope,
|
||||
profile: profile(account_id.as_str(), "refreshed disabled account"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update(account_id.as_str(), 1, "disabled-refreshed-secret"),
|
||||
audit: audit(
|
||||
"audit_disabled_refresh_rotate",
|
||||
"refresh",
|
||||
account_id.as_str(),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("refresh disabled account credential");
|
||||
|
||||
let current: (bool, String, Option<String>, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, availability_reason,
|
||||
provider_credentials_json, credential_revision
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load disabled account after refresh writes");
|
||||
assert!(!current.0);
|
||||
assert_eq!(current.1, "expired");
|
||||
assert_eq!(current.2.as_deref(), Some("credential_expired"));
|
||||
assert_eq!(current.3["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.4, 2);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
NewProviderAccount {
|
||||
id: id.to_owned(),
|
||||
@@ -1105,7 +1334,15 @@ fn cooldown_observation_requires_expiry() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_mismatched_cooldown_runtime_facts() {
|
||||
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
|
||||
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
|
||||
imported.availability = AccountAvailability::QuotaExhausted;
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_cooldown_until_for_ready_runtime_facts() {
|
||||
let mut imported = account("acct_invalid_cooldown", "user-invalid-cooldown");
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_err());
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use gateway_store::postgres::RuntimeSettingsUpdate;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{
|
||||
PgRuntimeSettingsRepository, RuntimeSettingsRepository, RuntimeSettingsUpdate,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_keep_account_rotation_global() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
use super::TestDatabase;
|
||||
|
||||
fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettingsUpdate {
|
||||
RuntimeSettingsUpdate {
|
||||
admin_api_key: None,
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_margin_seconds,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
@@ -18,24 +22,110 @@ fn runtime_settings_keep_account_rotation_global() {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_keep_account_rotation_global() {
|
||||
let settings = settings_with_margin(3_600);
|
||||
assert!(settings.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_reject_invalid_model_mapping() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
admin_api_key: None,
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
rotation_strategy: "smart".to_owned(),
|
||||
model_mappings: BTreeMap::from([("".to_owned(), "gpt-5.5".to_owned())]),
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
..settings_with_margin(3_600)
|
||||
};
|
||||
|
||||
assert!(settings.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_margin_change_should_reschedule_existing_refreshable_accounts() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_reschedule").await else {
|
||||
return;
|
||||
};
|
||||
let expires_at = timestamp_micros(Utc::now() + TimeDelta::hours(2));
|
||||
insert_refreshable_account(
|
||||
&database.pool,
|
||||
"acct_refresh_margin_changed",
|
||||
expires_at,
|
||||
expires_at - TimeDelta::hours(1),
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(1_800))
|
||||
.await
|
||||
.expect("update refresh margin");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_changed").await,
|
||||
expires_at - TimeDelta::seconds(1_800)
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unchanged_refresh_margin_should_preserve_existing_retry_schedule() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_unchanged").await else {
|
||||
return;
|
||||
};
|
||||
let expires_at = timestamp_micros(Utc::now() + TimeDelta::hours(2));
|
||||
let retry_at = timestamp_micros(Utc::now() + TimeDelta::minutes(5));
|
||||
insert_refreshable_account(
|
||||
&database.pool,
|
||||
"acct_refresh_margin_unchanged",
|
||||
expires_at,
|
||||
retry_at,
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(3_600))
|
||||
.await
|
||||
.expect("update unrelated runtime settings");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_unchanged").await,
|
||||
retry_at
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn insert_refreshable_account(
|
||||
pool: &sqlx::PgPool,
|
||||
account_id: &str,
|
||||
expires_at: DateTime<Utc>,
|
||||
next_refresh_at: DateTime<Utc>,
|
||||
) {
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, has_refresh_token, access_token_expires_at,
|
||||
next_refresh_at, availability, availability_observed_at, created_at, updated_at
|
||||
) values ($1, 'openai', $1, $1, 'oauth', '{}'::jsonb, true, $2, $3,
|
||||
'ready', now(), now(), now())",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(expires_at)
|
||||
.bind(next_refresh_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("insert refreshable account");
|
||||
}
|
||||
|
||||
async fn account_next_refresh_at(pool: &sqlx::PgPool, account_id: &str) -> DateTime<Utc> {
|
||||
sqlx::query_scalar("select next_refresh_at from provider_accounts where id = $1")
|
||||
.bind(account_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("load next refresh time")
|
||||
}
|
||||
|
||||
fn timestamp_micros(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
DateTime::from_timestamp_micros(value.timestamp_micros()).expect("valid test timestamp")
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ use gateway_core::{
|
||||
QuotaObservation, QuotaWriteOutcome,
|
||||
},
|
||||
error::{StoreError as CoreStoreError, StoreErrorKind},
|
||||
provider_ports::{ProviderCooldownPort, ProviderCooldownScope, ProviderScopedCooldown},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
use gateway_store::{
|
||||
@@ -49,9 +50,13 @@ impl ProviderAccountStore for CooldownTestAccountStore {
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not get one account")
|
||||
Ok(self
|
||||
.accounts
|
||||
.iter()
|
||||
.find(|candidate| candidate.id() == account)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
@@ -73,6 +78,13 @@ impl ProviderAccountStore for CooldownTestAccountStore {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not load credentials")
|
||||
}
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
_update: CredentialCasUpdate,
|
||||
@@ -182,10 +194,26 @@ fn composition_fixture(
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
composition_fixture_with_accounts(
|
||||
vec![ready_test_account("acct_cooldown_composition")],
|
||||
fail_state_write,
|
||||
fail_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn composition_fixture_with_accounts(
|
||||
accounts: Vec<ProviderAccount>,
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts: Vec::new(),
|
||||
accounts,
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write,
|
||||
});
|
||||
@@ -241,6 +269,31 @@ async fn cooldown_composition_keeps_postgres_terminal_when_redis_fails() {
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_state_write_does_not_change_runtime_cooldown() {
|
||||
let disabled = test_provider_account("acct_cooldown_composition").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
);
|
||||
let (store, cache, calls) = composition_fixture_with_accounts(vec![disabled], false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
assert!(
|
||||
cache
|
||||
.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_state_invalidates_same_revision_cooldown_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
@@ -523,6 +576,109 @@ async fn credential_cooldown_read_removes_expired_grace_key() {
|
||||
assert!(namespace_keys(&mut connection, &namespace).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_cooldown_isolated_by_model_and_revision_fenced() {
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let account_id = ProviderAccountId::new("acct_scoped_cooldown").expect("valid account ID");
|
||||
let revision = CredentialRevision::new(2).expect("positive revision");
|
||||
let model_a = ProviderCooldownScope::upstream_model(
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
);
|
||||
let model_b = ProviderCooldownScope::upstream_model(
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.6").expect("model"),
|
||||
);
|
||||
let until_a: SystemTime = millisecond_precision(Utc::now() + Duration::seconds(30)).into();
|
||||
let until_b: SystemTime = millisecond_precision(Utc::now() + Duration::seconds(60)).into();
|
||||
|
||||
assert!(
|
||||
repository
|
||||
.put_scoped_if_later(ProviderScopedCooldown::new(
|
||||
account_id.clone(),
|
||||
revision,
|
||||
model_a.clone(),
|
||||
until_a,
|
||||
))
|
||||
.await
|
||||
.expect("cache model A cooldown")
|
||||
);
|
||||
assert!(
|
||||
!repository
|
||||
.put_scoped_if_later(ProviderScopedCooldown::new(
|
||||
account_id.clone(),
|
||||
CredentialRevision::new(1).expect("stale revision"),
|
||||
model_a.clone(),
|
||||
until_b,
|
||||
))
|
||||
.await
|
||||
.expect("reject stale model A cooldown")
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.put_scoped_if_later(ProviderScopedCooldown::new(
|
||||
account_id.clone(),
|
||||
revision,
|
||||
model_b.clone(),
|
||||
until_b,
|
||||
))
|
||||
.await
|
||||
.expect("cache model B cooldown")
|
||||
);
|
||||
assert_eq!(
|
||||
repository
|
||||
.read_scoped(&account_id, &model_a)
|
||||
.await
|
||||
.expect("read model A cooldown")
|
||||
.expect("model A cooldown")
|
||||
.until(),
|
||||
until_a
|
||||
);
|
||||
assert_eq!(
|
||||
repository
|
||||
.read_scoped(&account_id, &model_b)
|
||||
.await
|
||||
.expect("read model B cooldown")
|
||||
.expect("model B cooldown")
|
||||
.until(),
|
||||
until_b
|
||||
);
|
||||
assert!(
|
||||
!repository
|
||||
.clear_scoped(
|
||||
&account_id,
|
||||
&model_a,
|
||||
CredentialRevision::new(1).expect("stale revision"),
|
||||
)
|
||||
.await
|
||||
.expect("fence stale model A invalidation")
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.clear_scoped(&account_id, &model_a, revision)
|
||||
.await
|
||||
.expect("clear model A cooldown")
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.read_scoped(&account_id, &model_a)
|
||||
.await
|
||||
.expect("read cleared model A cooldown")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
repository
|
||||
.read_scoped(&account_id, &model_b)
|
||||
.await
|
||||
.expect("read retained model B cooldown")
|
||||
.is_some()
|
||||
);
|
||||
let keys = namespace_keys(&mut connection, &namespace).await;
|
||||
assert_eq!(keys.len(), 1);
|
||||
assert!(!keys[0].contains(account_id.as_str()));
|
||||
assert!(!keys[0].contains(model_b.value()));
|
||||
}
|
||||
|
||||
fn cooldown(account_id: &str, revision: u64, seconds: i64) -> CredentialCooldown {
|
||||
CredentialCooldown {
|
||||
provider_account_id: account_id.to_owned(),
|
||||
|
||||
@@ -2,7 +2,6 @@ use gateway_core::engine::continuation::{
|
||||
NativeContinuationPin, NativeContinuationPort, NativeContinuationScope, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::error::SafeUpstreamValue;
|
||||
use gateway_core::operation::ProviderSessionState;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_store::redis::RedisNativeContinuationRepository;
|
||||
@@ -16,7 +15,7 @@ async fn native_continuation_round_trips_opaque_state_without_exposing_response_
|
||||
return;
|
||||
};
|
||||
let previous_response_id =
|
||||
PreviousResponseId::new("resp-client-visible-secret").expect("previous response ID");
|
||||
PreviousResponseId::new(format!("resp_{}\0client-visible-secret", "x".repeat(257)));
|
||||
let state = ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([(
|
||||
@@ -30,7 +29,7 @@ async fn native_continuation_round_trips_opaque_state_without_exposing_response_
|
||||
.record(
|
||||
NativeContinuationPin::new(
|
||||
previous_response_id.clone(),
|
||||
SafeUpstreamValue::new("resp-upstream-handle").expect("upstream response ID"),
|
||||
previous_response_id.clone(),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_primary").expect("account"),
|
||||
)
|
||||
@@ -47,7 +46,7 @@ async fn native_continuation_round_trips_opaque_state_without_exposing_response_
|
||||
.expect("stored continuation");
|
||||
assert_eq!(
|
||||
resolved.upstream_response_id().as_str(),
|
||||
"resp-upstream-handle"
|
||||
previous_response_id.as_str()
|
||||
);
|
||||
assert_eq!(resolved.account().as_str(), "acct_primary");
|
||||
assert_eq!(resolved.session_state(), Some(&state));
|
||||
@@ -55,7 +54,7 @@ async fn native_continuation_round_trips_opaque_state_without_exposing_response_
|
||||
let keys = namespace_keys(&mut connection, &namespace).await;
|
||||
assert!(
|
||||
keys.iter()
|
||||
.all(|key| !key.contains("resp-client-visible-secret"))
|
||||
.all(|key| !key.contains("client-visible-secret"))
|
||||
);
|
||||
let entry_key = keys
|
||||
.iter()
|
||||
@@ -71,14 +70,40 @@ async fn native_continuation_round_trips_opaque_state_without_exposing_response_
|
||||
delete_namespace_keys(&mut connection, &namespace).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_opaque_continuation_is_a_store_miss_without_a_redis_record() {
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let response_id = PreviousResponseId::new("");
|
||||
|
||||
assert!(
|
||||
repository
|
||||
.resolve(&response_id)
|
||||
.await
|
||||
.expect("resolve")
|
||||
.is_none()
|
||||
);
|
||||
repository
|
||||
.record(NativeContinuationPin::new(
|
||||
response_id.clone(),
|
||||
response_id,
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_primary").expect("account"),
|
||||
))
|
||||
.await
|
||||
.expect("empty opaque handle is not a store error");
|
||||
assert!(namespace_keys(&mut connection, &namespace).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_continuation_rejects_provider_state_for_a_different_provider() {
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
let pin = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("resp-mismatched-state").expect("previous response ID"),
|
||||
SafeUpstreamValue::new("resp-upstream-handle").expect("upstream response ID"),
|
||||
PreviousResponseId::new("resp-mismatched-state"),
|
||||
PreviousResponseId::new("resp-upstream-handle"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_primary").expect("account"),
|
||||
)
|
||||
@@ -90,6 +115,51 @@ async fn native_continuation_rejects_provider_state_for_a_different_provider() {
|
||||
assert!(namespace_keys(&mut connection, &namespace).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_continuation_record_should_bound_index_cleanup_work() {
|
||||
const STALE_ENTRY_COUNT: usize = 100;
|
||||
const CLEANUP_BATCH: usize = 64;
|
||||
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
repository
|
||||
.record(pin("resp-cleanup-anchor"))
|
||||
.await
|
||||
.expect("record cleanup anchor");
|
||||
let index_key = namespace_keys(&mut connection, &namespace)
|
||||
.await
|
||||
.into_iter()
|
||||
.find(|key| key.ends_with(":global:index"))
|
||||
.expect("continuation index key");
|
||||
let mut seed = redis::cmd("ZADD");
|
||||
seed.arg(&index_key);
|
||||
for index in 0..STALE_ENTRY_COUNT {
|
||||
seed.arg(0).arg(format!("stale-{index}"));
|
||||
}
|
||||
assert_eq!(
|
||||
seed.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.expect("seed stale continuation index entries"),
|
||||
i64::try_from(STALE_ENTRY_COUNT).expect("stale entry count fits i64")
|
||||
);
|
||||
|
||||
repository
|
||||
.record(pin("resp-cleanup-trigger"))
|
||||
.await
|
||||
.expect("record continuation with bounded cleanup");
|
||||
|
||||
assert_eq!(
|
||||
redis::cmd("ZCARD")
|
||||
.arg(index_key)
|
||||
.query_async::<usize>(&mut connection)
|
||||
.await
|
||||
.expect("count continuation index entries"),
|
||||
STALE_ENTRY_COUNT + 2 - CLEANUP_BATCH
|
||||
);
|
||||
delete_namespace_keys(&mut connection, &namespace).await;
|
||||
}
|
||||
|
||||
async fn repository() -> Option<(RedisNativeContinuationRepository, ConnectionManager, String)> {
|
||||
let redis_url = crate::support::test_env("CPR_TEST_REDIS_URL")?;
|
||||
let client = redis::Client::open(redis_url).expect("valid CPR_TEST_REDIS_URL");
|
||||
@@ -103,6 +173,15 @@ async fn repository() -> Option<(RedisNativeContinuationRepository, ConnectionMa
|
||||
Some((repository, connection, namespace))
|
||||
}
|
||||
|
||||
fn pin(response_id: &str) -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new(response_id),
|
||||
PreviousResponseId::new(response_id),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_primary").expect("account"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn namespace_keys(connection: &mut ConnectionManager, namespace: &str) -> Vec<String> {
|
||||
redis::cmd("KEYS")
|
||||
.arg(format!("{namespace}:*"))
|
||||
|
||||
@@ -3,8 +3,8 @@ use std::{sync::Arc, time::Duration};
|
||||
use gateway_core::{
|
||||
engine::credential::OpaqueProviderData,
|
||||
provider_ports::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingFlowPort, OAuthPendingPutOutcome,
|
||||
OAuthPendingTakeOutcome,
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome,
|
||||
OAuthPendingConsumeOutcome, OAuthPendingFlowPort, OAuthPendingPutOutcome,
|
||||
},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
@@ -23,6 +23,7 @@ async fn codex_pending_flow_wrong_owner_does_not_consume_it() {
|
||||
OAuthPendingBinding::try_new(format!("flow-{}", Uuid::new_v4())).expect("flow binding");
|
||||
let owner = OAuthPendingBinding::try_new("owner-a").expect("owner binding");
|
||||
let other_owner = OAuthPendingBinding::try_new("owner-b").expect("other owner binding");
|
||||
let claim = OAuthPendingBinding::try_new("claim-a").expect("claim binding");
|
||||
let mut payload = serde_json::Map::new();
|
||||
payload.insert(
|
||||
"state".to_owned(),
|
||||
@@ -77,24 +78,37 @@ async fn codex_pending_flow_wrong_owner_does_not_consume_it() {
|
||||
assert!(ttl > 0 && ttl <= 60_000);
|
||||
assert_eq!(
|
||||
repository
|
||||
.take_if_owner(&provider, &flow, &other_owner)
|
||||
.claim_if_owner(
|
||||
&provider,
|
||||
&flow,
|
||||
&other_owner,
|
||||
&claim,
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await
|
||||
.expect("reject wrong owner"),
|
||||
OAuthPendingTakeOutcome::OwnerMismatch
|
||||
OAuthPendingClaimOutcome::OwnerMismatch
|
||||
);
|
||||
assert!(matches!(
|
||||
repository
|
||||
.take_if_owner(&provider, &flow, &owner)
|
||||
.claim_if_owner(&provider, &flow, &owner, &claim, Duration::from_secs(30),)
|
||||
.await
|
||||
.expect("consume right owner"),
|
||||
OAuthPendingTakeOutcome::Taken(_)
|
||||
.expect("claim right owner"),
|
||||
OAuthPendingClaimOutcome::Claimed(_)
|
||||
));
|
||||
assert_eq!(
|
||||
repository
|
||||
.take_if_owner(&provider, &flow, &owner)
|
||||
.consume_claim(&provider, &flow, &owner, &claim)
|
||||
.await
|
||||
.expect("consume right owner claim"),
|
||||
OAuthPendingConsumeOutcome::Consumed
|
||||
);
|
||||
assert_eq!(
|
||||
repository
|
||||
.claim_if_owner(&provider, &flow, &owner, &claim, Duration::from_secs(30),)
|
||||
.await
|
||||
.expect("flow is one shot"),
|
||||
OAuthPendingTakeOutcome::NotFound
|
||||
OAuthPendingClaimOutcome::NotFound
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ rustls-native-certs = "0.8.4"
|
||||
rustls-pki-types = "1.14.1"
|
||||
secrecy = "0.10.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
|
||||
@@ -31,8 +31,9 @@ use gateway_core::engine::credential::{
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_core::provider_ports::{
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingFlowPort, OAuthPendingPutOutcome,
|
||||
OAuthPendingTakeOutcome, ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
NewOAuthPendingFlow, OAuthPendingBinding, OAuthPendingClaimOutcome, OAuthPendingConsumeOutcome,
|
||||
OAuthPendingFlowPort, OAuthPendingPutOutcome, OAuthPendingReleaseOutcome,
|
||||
ProviderRuntimePolicyPort, ProviderStoreError, ProviderStoreErrorKind,
|
||||
};
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
@@ -47,10 +48,11 @@ use crate::credential::{
|
||||
CodexAccountQuotaSnapshot, CodexCredentialAdmin, CodexCredentialAdminError,
|
||||
CodexCredentialAdminService, CodexCredentialCatalogError, CodexCredentialCatalogService,
|
||||
CodexCredentialQuotaError, CodexCredentialQuotaService, CodexOAuthAdmin, CodexOAuthAdminError,
|
||||
CodexOAuthPendingStore, CodexOAuthPendingStoreError, CodexPendingAuthorization,
|
||||
CodexQuotaWindowKind, CodexQuotaWindowRole, CompleteCodexOAuthAuthorization,
|
||||
CompletedCodexOAuthCredential, ExportManagedCodexCredential, RotateManagedCodexCredential,
|
||||
StartCodexOAuthAuthorization, StoredCodexPendingAuthorization, refresh_time,
|
||||
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
|
||||
CodexPendingAuthorization, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential, ExportManagedCodexCredential,
|
||||
RotateManagedCodexCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
refresh_time,
|
||||
};
|
||||
use crate::transport::CodexWebSocketPool;
|
||||
use crate::transport::openai_billing_breakdown;
|
||||
@@ -59,7 +61,7 @@ use crate::transport::profile::{
|
||||
};
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const PENDING_DOCUMENT_SCHEMA_VERSION: u64 = 2;
|
||||
const PENDING_DOCUMENT_SCHEMA_VERSION: u64 = 3;
|
||||
const MAX_ACCESS_TOKEN_BYTES: usize = 16 * 1024;
|
||||
const MAX_REFRESH_TOKEN_BYTES: usize = 64 * 1024;
|
||||
|
||||
@@ -144,7 +146,7 @@ impl OpenAiAdminProvider {
|
||||
}
|
||||
let current = self
|
||||
.accounts
|
||||
.load_credential(existing.id(), existing.revision())
|
||||
.load_current_credential(existing.id())
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
incoming.credential = CodexCredentialCodec::preserve_installation_id(
|
||||
@@ -166,6 +168,19 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
self.websocket_pool.evict_account(account_id.as_str()).await;
|
||||
}
|
||||
|
||||
async fn account_facts_changed(&self, account_ids: &[ProviderAccountId]) {
|
||||
if account_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Err(error) = self.catalog.invalidate() {
|
||||
tracing::warn!(
|
||||
account_count = account_ids.len(),
|
||||
error = %error,
|
||||
"OpenAI model catalog invalidation failed after account commit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
upstream_model: &UpstreamModelId,
|
||||
@@ -214,7 +229,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
output_tokens,
|
||||
input.cached_tokens.unwrap_or_default(),
|
||||
input.cache_write_tokens.unwrap_or_default(),
|
||||
None,
|
||||
input.service_tier.as_deref(),
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -316,29 +331,37 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
);
|
||||
})
|
||||
.map_err(map_oauth_error)?;
|
||||
let credential = match completed.credential {
|
||||
CompletedCodexOAuthCredential::Create(credential) => {
|
||||
PreparedAuthorizationCredential::Create(prepared_create(credential, Utc::now())?)
|
||||
let (mutation, completed_credential, authorization_guard) = completed.into_parts();
|
||||
let credential = match (mutation.target(), completed_credential) {
|
||||
(
|
||||
AuthorizationMutationTarget::Create { .. },
|
||||
CompletedCodexOAuthCredential::Create(credential),
|
||||
) => {
|
||||
prepared_create(credential, Utc::now()).map(PreparedAuthorizationCredential::Create)
|
||||
}
|
||||
CompletedCodexOAuthCredential::Reauthorize(credential) => {
|
||||
let provider_kind = match completed.mutation.target() {
|
||||
AuthorizationMutationTarget::Reauthorize { .. } => {
|
||||
completed.mutation.provider_kind().clone()
|
||||
}
|
||||
AuthorizationMutationTarget::Create { .. } => {
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Internal));
|
||||
}
|
||||
};
|
||||
PreparedAuthorizationCredential::Reauthorize(prepared_rotation(
|
||||
credential,
|
||||
provider_kind,
|
||||
)?)
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize { .. },
|
||||
CompletedCodexOAuthCredential::Reauthorize(credential),
|
||||
) => prepared_rotation(credential, mutation.provider_kind().clone())
|
||||
.map(PreparedAuthorizationCredential::Reauthorize),
|
||||
_ => Err(provider_admin_error(ProviderAdminErrorKind::Internal)),
|
||||
};
|
||||
let credential = match credential {
|
||||
Ok(credential) => credential,
|
||||
Err(error) => {
|
||||
if let Err(settlement_error) = authorization_guard.abort().await {
|
||||
tracing::warn!(
|
||||
request_id = %request_id,
|
||||
settlement_error = %settlement_error,
|
||||
"OpenAI OAuth claim release failed after credential preparation"
|
||||
);
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Unavailable));
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
Ok(PreparedAuthorizationCommit {
|
||||
pending: completed.mutation,
|
||||
credential,
|
||||
})
|
||||
Ok(PreparedAuthorizationCommit::new(mutation, credential)
|
||||
.with_authorization_guard(authorization_guard))
|
||||
}
|
||||
|
||||
async fn prepare_rotation(
|
||||
@@ -346,16 +369,11 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
command: PrepareCredentialRotation,
|
||||
) -> Result<PreparedCredentialRotation, ProviderAdminError> {
|
||||
validate_account_record(&command.account, &self.provider_kind)?;
|
||||
if command.account.credential_revision != command.expected_credential_revision {
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Conflict));
|
||||
}
|
||||
let account_id = ProviderAccountId::new(command.account.id.clone())
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let expected_revision = CredentialRevision::new(command.expected_credential_revision.get())
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let current = self
|
||||
.accounts
|
||||
.load_credential(&account_id, expected_revision)
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
if !account_matches_record(¤t.account, &command.account) {
|
||||
@@ -415,11 +433,17 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
validate_account_record(&command.account, &self.provider_kind)?;
|
||||
let account_id = ProviderAccountId::new(command.account.id.clone())
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let revision = CredentialRevision::new(command.account.credential_revision.get())
|
||||
.map_err(|_| provider_admin_error(ProviderAdminErrorKind::Invalid))?;
|
||||
let current = self
|
||||
.accounts
|
||||
.load_current_credential(&account_id)
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
if !account_matches_record(¤t.account, &command.account) {
|
||||
return Err(provider_admin_error(ProviderAdminErrorKind::Conflict));
|
||||
}
|
||||
let prepared = self
|
||||
.credentials
|
||||
.manual_refresh(account_id, revision)
|
||||
.manual_refresh(current)
|
||||
.await
|
||||
.map_err(map_credential_admin_error)?;
|
||||
prepared_rotation(prepared, command.account.provider_kind)
|
||||
@@ -562,7 +586,7 @@ fn prepared_rotation(
|
||||
prepared: crate::credential::PreparedCodexCredentialRotation,
|
||||
provider_kind: ProviderKind,
|
||||
) -> Result<PreparedCredentialRotation, ProviderAdminError> {
|
||||
let (profile, credential, guard) = prepared.into_parts();
|
||||
let (profile, credential, replacement_identity, guard) = prepared.into_parts();
|
||||
let (
|
||||
account_id,
|
||||
expected_revision,
|
||||
@@ -582,6 +606,7 @@ fn prepared_rotation(
|
||||
account_id,
|
||||
provider_kind,
|
||||
expected_credential_revision,
|
||||
replacement_identity,
|
||||
name: profile.name,
|
||||
email: profile.email,
|
||||
plan_type: profile.plan_type,
|
||||
@@ -707,20 +732,9 @@ fn dashboard_desktop_release(
|
||||
fn account_matches_record(account: &ProviderAccount, record: &AccountRecord) -> bool {
|
||||
account.id().as_str() == record.id
|
||||
&& account.provider() == &record.provider_kind
|
||||
&& account.revision().get() == record.credential_revision.get()
|
||||
&& account.name() == record.name
|
||||
&& account.email() == record.email.as_deref()
|
||||
&& account.upstream_user_id() == record.upstream_user_id
|
||||
&& account.upstream_account_id() == record.upstream_account_id.as_deref()
|
||||
&& account.plan_type() == record.plan_type.as_deref()
|
||||
&& account.authentication_kind() == record.authentication_kind
|
||||
&& account.enabled() == record.enabled
|
||||
&& account.availability() == record.availability
|
||||
&& account.cooldown_until().map(DateTime::<Utc>::from) == record.cooldown_until
|
||||
&& account.access_token_expires_at().map(DateTime::<Utc>::from)
|
||||
== record.access_token_expires_at
|
||||
&& account.next_refresh_at().map(DateTime::<Utc>::from) == record.next_refresh_at
|
||||
&& account.has_refresh_token() == record.has_refresh_token
|
||||
}
|
||||
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
@@ -896,21 +910,86 @@ impl CodexOAuthPendingStore for OpenAiOAuthPendingStore {
|
||||
}
|
||||
}
|
||||
|
||||
async fn take(
|
||||
async fn claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
) -> Result<Option<CodexPendingAuthorization>, CodexOAuthPendingStoreError> {
|
||||
claim_ref: &str,
|
||||
claim_ttl: std::time::Duration,
|
||||
) -> Result<CodexOAuthPendingClaimOutcome, CodexOAuthPendingStoreError> {
|
||||
let flow = binding(flow_id)?;
|
||||
let owner = binding(owner_ref)?;
|
||||
let claim = binding(claim_ref)?;
|
||||
let outcome = self
|
||||
.port
|
||||
.claim_if_owner(&self.provider_kind, &flow, &owner, &claim, claim_ttl)
|
||||
.await
|
||||
.map_err(map_pending_store_error)?;
|
||||
match outcome {
|
||||
OAuthPendingClaimOutcome::Claimed(payload) => match decode_pending(payload) {
|
||||
Ok(pending) => Ok(CodexOAuthPendingClaimOutcome::Claimed(Box::new(pending))),
|
||||
Err(error) => match self
|
||||
.port
|
||||
.release_claim(&self.provider_kind, &flow, &owner, &claim)
|
||||
.await
|
||||
.map_err(map_pending_store_error)?
|
||||
{
|
||||
OAuthPendingReleaseOutcome::Released => Err(error),
|
||||
OAuthPendingReleaseOutcome::NotFound
|
||||
| OAuthPendingReleaseOutcome::OwnerMismatch
|
||||
| OAuthPendingReleaseOutcome::ClaimMismatch => {
|
||||
Err(CodexOAuthPendingStoreError::Unavailable)
|
||||
}
|
||||
},
|
||||
},
|
||||
OAuthPendingClaimOutcome::NotFound | OAuthPendingClaimOutcome::OwnerMismatch => {
|
||||
Ok(CodexOAuthPendingClaimOutcome::NotFound)
|
||||
}
|
||||
OAuthPendingClaimOutcome::InProgress => Ok(CodexOAuthPendingClaimOutcome::InProgress),
|
||||
}
|
||||
}
|
||||
|
||||
async fn release_claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
claim_ref: &str,
|
||||
) -> Result<bool, CodexOAuthPendingStoreError> {
|
||||
let flow = binding(flow_id)?;
|
||||
let owner = binding(owner_ref)?;
|
||||
let claim = binding(claim_ref)?;
|
||||
match self
|
||||
.port
|
||||
.take_if_owner(&self.provider_kind, &flow, &owner)
|
||||
.release_claim(&self.provider_kind, &flow, &owner, &claim)
|
||||
.await
|
||||
.map_err(map_pending_store_error)?
|
||||
{
|
||||
OAuthPendingTakeOutcome::Taken(payload) => decode_pending(payload).map(Some),
|
||||
OAuthPendingTakeOutcome::NotFound | OAuthPendingTakeOutcome::OwnerMismatch => Ok(None),
|
||||
OAuthPendingReleaseOutcome::Released => Ok(true),
|
||||
OAuthPendingReleaseOutcome::NotFound
|
||||
| OAuthPendingReleaseOutcome::OwnerMismatch
|
||||
| OAuthPendingReleaseOutcome::ClaimMismatch => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
async fn consume_claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
claim_ref: &str,
|
||||
) -> Result<bool, CodexOAuthPendingStoreError> {
|
||||
let flow = binding(flow_id)?;
|
||||
let owner = binding(owner_ref)?;
|
||||
let claim = binding(claim_ref)?;
|
||||
match self
|
||||
.port
|
||||
.consume_claim(&self.provider_kind, &flow, &owner, &claim)
|
||||
.await
|
||||
.map_err(map_pending_store_error)?
|
||||
{
|
||||
OAuthPendingConsumeOutcome::Consumed => Ok(true),
|
||||
OAuthPendingConsumeOutcome::NotFound
|
||||
| OAuthPendingConsumeOutcome::OwnerMismatch
|
||||
| OAuthPendingConsumeOutcome::ClaimMismatch => Ok(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -927,7 +1006,6 @@ struct PendingDocument {
|
||||
nonce: String,
|
||||
code_verifier: String,
|
||||
reauthorization_account_id: Option<String>,
|
||||
reauthorization_credential_revision: Option<u64>,
|
||||
mutation: PendingMutationDocument,
|
||||
}
|
||||
|
||||
@@ -944,13 +1022,8 @@ struct PendingMutationDocument {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
enum PendingTargetDocument {
|
||||
Create {
|
||||
name: String,
|
||||
},
|
||||
Reauthorize {
|
||||
account_id: String,
|
||||
expected_credential_revision: u64,
|
||||
},
|
||||
Create { name: String },
|
||||
Reauthorize { account_id: String },
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -994,15 +1067,9 @@ fn encode_pending(pending: &CodexPendingAuthorization) -> Map<String, Value> {
|
||||
);
|
||||
document.insert(
|
||||
"reauthorization_account_id".to_owned(),
|
||||
pending.reauthorization().map_or(Value::Null, |target| {
|
||||
Value::String(target.account_id().to_string())
|
||||
}),
|
||||
);
|
||||
document.insert(
|
||||
"reauthorization_credential_revision".to_owned(),
|
||||
pending.reauthorization().map_or(Value::Null, |target| {
|
||||
Value::Number(Number::from(target.credential_revision().get()))
|
||||
}),
|
||||
pending
|
||||
.reauthorization()
|
||||
.map_or(Value::Null, |target| Value::String(target.to_string())),
|
||||
);
|
||||
document.insert(
|
||||
"mutation".to_owned(),
|
||||
@@ -1043,19 +1110,12 @@ fn encode_target(target: &AuthorizationMutationTarget) -> Map<String, Value> {
|
||||
document.insert("kind".to_owned(), Value::String("create".to_owned()));
|
||||
document.insert("name".to_owned(), Value::String(name.clone()));
|
||||
}
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => {
|
||||
AuthorizationMutationTarget::Reauthorize { account_id } => {
|
||||
document.insert("kind".to_owned(), Value::String("reauthorize".to_owned()));
|
||||
document.insert(
|
||||
"account_id".to_owned(),
|
||||
Value::String(account_id.to_string()),
|
||||
);
|
||||
document.insert(
|
||||
"expected_credential_revision".to_owned(),
|
||||
Value::Number(Number::from(expected_credential_revision.get())),
|
||||
);
|
||||
}
|
||||
}
|
||||
document
|
||||
@@ -1096,7 +1156,6 @@ fn decode_pending(
|
||||
nonce: SecretString::from(document.nonce),
|
||||
code_verifier: SecretString::from(document.code_verifier),
|
||||
reauthorization_account_id: document.reauthorization_account_id,
|
||||
reauthorization_credential_revision: document.reauthorization_credential_revision,
|
||||
mutation: decode_mutation(document.mutation)?,
|
||||
})
|
||||
}
|
||||
@@ -1111,15 +1170,12 @@ fn decode_mutation(
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?;
|
||||
let target = match document.target {
|
||||
PendingTargetDocument::Create { name } => AuthorizationMutationTarget::Create { name },
|
||||
PendingTargetDocument::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?,
|
||||
expected_credential_revision: Revision::new(expected_credential_revision)
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?,
|
||||
},
|
||||
PendingTargetDocument::Reauthorize { account_id } => {
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?,
|
||||
}
|
||||
}
|
||||
};
|
||||
let actor = match document.owner {
|
||||
PendingOwnerDocument::AdminSession { admin_user_id } => {
|
||||
@@ -1222,10 +1278,10 @@ fn map_credential_admin_error(error: CodexCredentialAdminError) -> ProviderAdmin
|
||||
| Error::AccountBanned
|
||||
| Error::IdentityRejected => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound => ProviderAdminErrorKind::NotFound,
|
||||
Error::RevisionConflict | Error::RefreshLeaseUnavailable | Error::RefreshAmbiguous => {
|
||||
Error::RefreshLeaseUnavailable | Error::RefreshAmbiguous => {
|
||||
ProviderAdminErrorKind::Conflict
|
||||
}
|
||||
Error::StoreUnavailable | Error::RefreshUnavailable | Error::IdentityUnavailable => {
|
||||
Error::RefreshUnavailable | Error::IdentityUnavailable => {
|
||||
ProviderAdminErrorKind::Unavailable
|
||||
}
|
||||
})
|
||||
@@ -1237,8 +1293,6 @@ const fn credential_admin_error_code(error: CodexCredentialAdminError) -> &'stat
|
||||
CodexCredentialAdminError::IdentityMismatch => "identity_mismatch",
|
||||
CodexCredentialAdminError::InvalidCredential => "invalid_credential",
|
||||
CodexCredentialAdminError::NotFound => "not_found",
|
||||
CodexCredentialAdminError::RevisionConflict => "revision_conflict",
|
||||
CodexCredentialAdminError::StoreUnavailable => "store_unavailable",
|
||||
CodexCredentialAdminError::MissingRefreshToken => "missing_refresh_token",
|
||||
CodexCredentialAdminError::RefreshLeaseUnavailable => "refresh_lease_unavailable",
|
||||
CodexCredentialAdminError::RefreshRejected => "refresh_rejected",
|
||||
@@ -1272,9 +1326,11 @@ fn log_import_failure(stage: &'static str, error: &'static str) {
|
||||
fn map_oauth_error(error: CodexOAuthAdminError) -> ProviderAdminError {
|
||||
use CodexOAuthAdminError as Error;
|
||||
provider_admin_error(match error {
|
||||
Error::InvalidInput | Error::UpstreamRejected | Error::Credential => {
|
||||
ProviderAdminErrorKind::Invalid
|
||||
}
|
||||
Error::InvalidInput
|
||||
| Error::CallbackRejected
|
||||
| Error::TokenRejected
|
||||
| Error::IdentityRejected
|
||||
| Error::Credential => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound | Error::FlowExpired => ProviderAdminErrorKind::NotFound,
|
||||
Error::Conflict | Error::Ambiguous => ProviderAdminErrorKind::Conflict,
|
||||
Error::UpstreamUnavailable | Error::StorageUnavailable => {
|
||||
@@ -1289,7 +1345,9 @@ const fn oauth_error_code(error: &CodexOAuthAdminError) -> &'static str {
|
||||
CodexOAuthAdminError::NotFound => "not_found",
|
||||
CodexOAuthAdminError::Conflict => "conflict",
|
||||
CodexOAuthAdminError::FlowExpired => "flow_expired",
|
||||
CodexOAuthAdminError::UpstreamRejected => "upstream_rejected",
|
||||
CodexOAuthAdminError::CallbackRejected => "callback_rejected",
|
||||
CodexOAuthAdminError::TokenRejected => "token_rejected",
|
||||
CodexOAuthAdminError::IdentityRejected => "identity_rejected",
|
||||
CodexOAuthAdminError::UpstreamUnavailable => "upstream_unavailable",
|
||||
CodexOAuthAdminError::Ambiguous => "ambiguous",
|
||||
CodexOAuthAdminError::StorageUnavailable => "storage_unavailable",
|
||||
@@ -1303,9 +1361,10 @@ fn map_quota_error(error: CodexCredentialQuotaError) -> ProviderAdminError {
|
||||
Error::InvalidCredentialData => ProviderAdminErrorKind::Invalid,
|
||||
Error::NotFound => ProviderAdminErrorKind::NotFound,
|
||||
Error::RevisionConflict => ProviderAdminErrorKind::Conflict,
|
||||
Error::Repository(_) | Error::Store | Error::Upstream => {
|
||||
ProviderAdminErrorKind::Unavailable
|
||||
}
|
||||
Error::CredentialRefreshRequired
|
||||
| Error::Repository(_)
|
||||
| Error::Store
|
||||
| Error::Upstream => ProviderAdminErrorKind::Unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ use std::time::SystemTime;
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
|
||||
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountUpdate,
|
||||
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
|
||||
ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::error::StoreErrorKind;
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
|
||||
ProviderRefreshCapacityRequest, ProviderRefreshLeaseRequest, ProviderRefreshPolicy,
|
||||
@@ -28,7 +28,6 @@ use thiserror::Error;
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
};
|
||||
use super::repository::CodexCredentialRepository;
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{RefreshFailure, TokenRefresher};
|
||||
use super::types::{
|
||||
@@ -42,32 +41,6 @@ const CODEX_CREDENTIAL_SCHEMA_VERSION: u32 = 1;
|
||||
const MAX_BATCH: usize = 200;
|
||||
const MAX_IMPORT_DOCUMENT_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
const CPR_CONTAINER_KEYS: &[&str] = &["sourceFormat", "source_format", "accounts"];
|
||||
const CPR_ACCOUNT_KEYS: &[&str] = &[
|
||||
"id",
|
||||
"email",
|
||||
"accountId",
|
||||
"account_id",
|
||||
"userId",
|
||||
"user_id",
|
||||
"label",
|
||||
"planType",
|
||||
"plan_type",
|
||||
"token",
|
||||
"at",
|
||||
"accessToken",
|
||||
"access_token",
|
||||
"refreshToken",
|
||||
"refresh_token",
|
||||
"accessTokenExpiresAt",
|
||||
"access_token_expires_at",
|
||||
"status",
|
||||
"addedAt",
|
||||
"added_at",
|
||||
"updatedAt",
|
||||
"updated_at",
|
||||
];
|
||||
|
||||
pub struct ImportCodexOAuthCredential {
|
||||
pub account_id: String,
|
||||
pub name: String,
|
||||
@@ -133,7 +106,6 @@ enum ParsedCodexAuthentication {
|
||||
OAuth {
|
||||
access_token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
id_token: Option<String>,
|
||||
},
|
||||
AgentIdentity {
|
||||
runtime_id: String,
|
||||
@@ -148,7 +120,6 @@ impl fmt::Debug for ParsedCodexAuthentication {
|
||||
Self::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token,
|
||||
} => formatter
|
||||
.debug_struct("OAuth")
|
||||
.field("access_token", &access_token.as_ref().map(|_| "<redacted>"))
|
||||
@@ -156,7 +127,6 @@ impl fmt::Debug for ParsedCodexAuthentication {
|
||||
"refresh_token",
|
||||
&refresh_token.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.field("id_token", &id_token.as_ref().map(|_| "<redacted>"))
|
||||
.finish(),
|
||||
Self::AgentIdentity {
|
||||
runtime_id: _,
|
||||
@@ -317,6 +287,7 @@ impl std::fmt::Debug for RotateManagedCodexCredential {
|
||||
pub struct PreparedCodexCredentialRotation {
|
||||
pub profile: ProviderAccountUpdate,
|
||||
pub credential: CredentialCasUpdate,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
refresh_guards: Option<ProviderRefreshGuards>,
|
||||
}
|
||||
|
||||
@@ -331,6 +302,7 @@ impl std::fmt::Debug for PreparedCodexCredentialRotation {
|
||||
.debug_struct("PreparedCodexCredentialRotation")
|
||||
.field("profile", &self.profile)
|
||||
.field("credential", &self.credential)
|
||||
.field("replacement_identity", &self.replacement_identity)
|
||||
.field(
|
||||
"refresh_guards",
|
||||
&self.refresh_guards.as_ref().map(|_| "<held>"),
|
||||
@@ -347,11 +319,13 @@ impl PreparedCodexCredentialRotation {
|
||||
) -> (
|
||||
ProviderAccountUpdate,
|
||||
CredentialCasUpdate,
|
||||
Option<ProviderAccountIdentity>,
|
||||
PreparedCodexCredentialRotationGuard,
|
||||
) {
|
||||
(
|
||||
self.profile,
|
||||
self.credential,
|
||||
self.replacement_identity,
|
||||
PreparedCodexCredentialRotationGuard(self.refresh_guards),
|
||||
)
|
||||
}
|
||||
@@ -379,10 +353,6 @@ pub enum CodexCredentialAdminError {
|
||||
InvalidCredential,
|
||||
#[error("Codex account was not found")]
|
||||
NotFound,
|
||||
#[error("Codex credential revision is stale")]
|
||||
RevisionConflict,
|
||||
#[error("Codex account store is unavailable")]
|
||||
StoreUnavailable,
|
||||
#[error("Codex account has no refresh token")]
|
||||
MissingRefreshToken,
|
||||
#[error("Codex refresh lease is unavailable")]
|
||||
@@ -525,6 +495,22 @@ impl CodexCredentialAdmin {
|
||||
pub fn prepare_rotation(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
self.prepare_oauth_rotation(input, false)
|
||||
}
|
||||
|
||||
/// 完整 OAuth 授权按目标数据 ID 替换凭据与上游账号身份。
|
||||
pub(crate) fn prepare_reauthorization(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
self.prepare_oauth_rotation(input, true)
|
||||
}
|
||||
|
||||
fn prepare_oauth_rotation(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
replace_identity: bool,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
let access_token_expires_at =
|
||||
required_time(input.verified_account.access_token_expires_at)?;
|
||||
@@ -535,14 +521,32 @@ impl CodexCredentialAdmin {
|
||||
.ok_or(CodexCredentialAdminError::IdentityMismatch)?;
|
||||
if input.current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| input.current.account.authentication_kind() != CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
|| input.current.account.upstream_account_id()
|
||||
!= Some(input.verified_account.chatgpt_account_id.as_str())
|
||||
|| input.current.account.upstream_user_id() != input.verified_account.chatgpt_user_id
|
||||
|| oauth.principal.oauth_subject != input.verified_account.oauth_subject
|
||||
|| oauth.principal.poid != input.verified_account.poid
|
||||
|| (!replace_identity
|
||||
&& (input.current.account.upstream_account_id()
|
||||
!= Some(input.verified_account.chatgpt_account_id.as_str())
|
||||
|| input.current.account.upstream_user_id()
|
||||
!= input.verified_account.chatgpt_user_id
|
||||
|| oauth.principal.oauth_subject != input.verified_account.oauth_subject
|
||||
|| oauth.principal.poid != input.verified_account.poid))
|
||||
{
|
||||
return Err(CodexCredentialAdminError::IdentityMismatch);
|
||||
}
|
||||
let replacement_identity = replace_identity.then(|| {
|
||||
ProviderAccountIdentity::new(
|
||||
input.verified_account.chatgpt_user_id.clone(),
|
||||
Some(input.verified_account.chatgpt_account_id.clone()),
|
||||
)
|
||||
});
|
||||
if replace_identity {
|
||||
oauth
|
||||
.principal
|
||||
.oauth_subject
|
||||
.clone_from(&input.verified_account.oauth_subject);
|
||||
oauth
|
||||
.principal
|
||||
.poid
|
||||
.clone_from(&input.verified_account.poid);
|
||||
}
|
||||
oauth.access_token = input.secret.access_token.expose_secret().to_owned();
|
||||
oauth.refresh_token = input
|
||||
.secret
|
||||
@@ -575,14 +579,14 @@ impl CodexCredentialAdmin {
|
||||
Ok(PreparedCodexCredentialRotation {
|
||||
profile,
|
||||
credential,
|
||||
replacement_identity,
|
||||
refresh_guards: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 有状态的 Codex 手工刷新边界;只读取 Store 并准备 CAS,不自行持久化。
|
||||
/// 有状态的 Codex 手工刷新边界;消费调用方刚读取的当前 credential 并准备 CAS。
|
||||
pub struct CodexCredentialAdminService {
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
@@ -593,7 +597,6 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CodexCredentialAdminService")
|
||||
.field("repository", &"CodexCredentialRepository")
|
||||
.field("refresher", &"TokenRefresher")
|
||||
.field("verifier", &"CodexAccountIdentityVerifier")
|
||||
.field("leases", &"ProviderLeasePort")
|
||||
@@ -604,14 +607,12 @@ impl fmt::Debug for CodexCredentialAdminService {
|
||||
|
||||
impl CodexCredentialAdminService {
|
||||
pub fn new(
|
||||
repository: CodexCredentialRepository,
|
||||
refresher: Arc<dyn TokenRefresher>,
|
||||
verifier: Arc<dyn CodexAccountIdentityVerifier>,
|
||||
leases: Arc<dyn ProviderLeasePort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
refresher,
|
||||
verifier,
|
||||
leases,
|
||||
@@ -622,19 +623,11 @@ impl CodexCredentialAdminService {
|
||||
/// 官方 RT exchange + AT 身份验证;结果由 App 在同一 revision/audit 事务中提交。
|
||||
pub async fn manual_refresh(
|
||||
&self,
|
||||
account_id: ProviderAccountId,
|
||||
expected_revision: CredentialRevision,
|
||||
current: LoadedCredential,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
let current = self
|
||||
.repository
|
||||
.store()
|
||||
.load_credential(&account_id, expected_revision)
|
||||
.await
|
||||
.map_err(map_admin_store_error)?;
|
||||
if current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| current.account.id() != &account_id
|
||||
|| current.account.revision() != expected_revision
|
||||
{
|
||||
let account_id = current.account.id().clone();
|
||||
let expected_revision = current.account.revision();
|
||||
if current.account.provider().as_str() != PROVIDER_NAME {
|
||||
return Err(CodexCredentialAdminError::NotFound);
|
||||
}
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
@@ -720,7 +713,7 @@ impl CodexCredentialAdminService {
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
/// 按文档结构严格判别账号布局,并归一到唯一 `NewProviderAccount` 写入路径。
|
||||
/// 从 OAuth token 对补全账号事实,并归一到唯一 `NewProviderAccount` 写入路径。
|
||||
pub async fn prepare_import_document(
|
||||
&self,
|
||||
payload: Value,
|
||||
@@ -741,19 +734,18 @@ impl CodexCredentialAdminService {
|
||||
let mut accounts = Vec::with_capacity(candidates.len());
|
||||
let mut policy = None;
|
||||
for candidate in candidates {
|
||||
let account_id = candidate
|
||||
.id
|
||||
.clone()
|
||||
.filter(|id| ProviderAccountId::new(id.clone()).is_ok())
|
||||
.unwrap_or_else(|| format!("acct_{}", uuid::Uuid::now_v7().simple()));
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (enabled, availability) = import_runtime_state(candidate.status.as_deref())?;
|
||||
let mut prepared = match &candidate.authentication {
|
||||
ParsedCodexAuthentication::OAuth { .. } => {
|
||||
let (secret, verified_account) =
|
||||
self.verify_import_candidate(&candidate).await?;
|
||||
let prepared = match &candidate.authentication {
|
||||
ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
} => {
|
||||
let account_id = format!("acct_{}", uuid::Uuid::now_v7().simple());
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (secret, verified_account) = self
|
||||
.verify_import_tokens(access_token.clone(), refresh_token.clone())
|
||||
.await?;
|
||||
if !upstream_identities.insert((
|
||||
verified_account.chatgpt_user_id.clone(),
|
||||
verified_account.chatgpt_account_id.clone(),
|
||||
@@ -780,19 +772,29 @@ impl CodexCredentialAdminService {
|
||||
)?;
|
||||
CodexCredentialAdmin.prepare_import(ImportCodexOAuthCredential {
|
||||
account_id,
|
||||
name: candidate
|
||||
.name
|
||||
name: verified_account
|
||||
.email
|
||||
.clone()
|
||||
.filter(|name| !name.trim().is_empty())
|
||||
.unwrap_or_else(|| "Codex OAuth".to_owned()),
|
||||
secret,
|
||||
verified_account,
|
||||
next_refresh_at,
|
||||
enabled,
|
||||
enabled: true,
|
||||
})?
|
||||
}
|
||||
ParsedCodexAuthentication::AgentIdentity { .. } => {
|
||||
let account =
|
||||
let account_id = candidate
|
||||
.id
|
||||
.clone()
|
||||
.filter(|id| ProviderAccountId::new(id.clone()).is_ok())
|
||||
.unwrap_or_else(|| format!("acct_{}", uuid::Uuid::now_v7().simple()));
|
||||
if !account_ids.insert(account_id.clone()) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let (enabled, availability) =
|
||||
import_runtime_state(candidate.status.as_deref())?;
|
||||
let mut account =
|
||||
self.prepare_agent_identity_import(&candidate, account_id, enabled)?;
|
||||
if !upstream_identities.insert((
|
||||
account.account.upstream_user_id().to_owned(),
|
||||
@@ -804,29 +806,23 @@ impl CodexCredentialAdminService {
|
||||
)) {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
account.account =
|
||||
account
|
||||
.account
|
||||
.with_runtime_state(enabled, availability, None);
|
||||
account
|
||||
}
|
||||
};
|
||||
prepared.account = prepared
|
||||
.account
|
||||
.with_runtime_state(enabled, availability, None);
|
||||
accounts.push(prepared);
|
||||
}
|
||||
Ok(PreparedCodexAccountImport { accounts })
|
||||
}
|
||||
|
||||
async fn verify_import_candidate(
|
||||
async fn verify_import_tokens(
|
||||
&self,
|
||||
candidate: &ParsedCodexImportAccount,
|
||||
access_token: Option<String>,
|
||||
refresh_token: Option<String>,
|
||||
) -> Result<(CodexOAuthSecret, CodexAccountProfile), CodexCredentialAdminError> {
|
||||
let ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token,
|
||||
} = &candidate.authentication
|
||||
else {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
};
|
||||
let access_token = access_token
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
@@ -835,26 +831,14 @@ impl CodexCredentialAdminService {
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
.filter(|token| !token.is_empty());
|
||||
let id_token = id_token
|
||||
.as_deref()
|
||||
.map(normalize_bearer)
|
||||
.filter(|token| !token.is_empty());
|
||||
let expectation = CodexIdentityExpectation::imported(
|
||||
candidate.chatgpt_account_id.clone(),
|
||||
candidate.chatgpt_user_id.clone(),
|
||||
)
|
||||
.map_err(|_| CodexCredentialAdminError::InvalidInput)?;
|
||||
if let Some(access_token) = access_token {
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(access_token),
|
||||
refresh_token: refresh_token.clone().map(SecretString::from),
|
||||
id_token: id_token.clone().map(SecretString::from),
|
||||
id_token: None,
|
||||
};
|
||||
match self.verifier.verify(&secret, &expectation).await {
|
||||
Ok(verification) => {
|
||||
let profile = verification.into_complete().map_err(map_identity_error)?;
|
||||
return Ok((secret, profile));
|
||||
}
|
||||
match self.verifier.verify_import(&secret).await {
|
||||
Ok(profile) => return Ok((secret, profile)),
|
||||
Err(CodexIdentityVerificationError::Unavailable) => {
|
||||
return Err(CodexCredentialAdminError::IdentityUnavailable);
|
||||
}
|
||||
@@ -873,17 +857,19 @@ impl CodexCredentialAdminService {
|
||||
if tokens.access_token.trim().is_empty() || tokens.expires_in.is_zero() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
// 导入时已经消费来源 RT;只有上游明确轮换返回的新 RT 才能继续保存。
|
||||
let secret = CodexOAuthSecret {
|
||||
access_token: SecretString::from(tokens.access_token),
|
||||
refresh_token: tokens.refresh_token.map(SecretString::from),
|
||||
id_token: id_token.map(SecretString::from),
|
||||
// RT 未轮换时仍保留导入提供的 RT,保证本次补全不会丢失后续刷新能力。
|
||||
refresh_token: tokens
|
||||
.refresh_token
|
||||
.map(SecretString::from)
|
||||
.or_else(|| Some(SecretString::from(refresh_token))),
|
||||
id_token: None,
|
||||
};
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify(&secret, &expectation)
|
||||
.verify_import(&secret)
|
||||
.await
|
||||
.and_then(super::identity::CodexIdentityVerification::into_complete)
|
||||
.map_err(map_identity_error)?;
|
||||
Ok((secret, profile))
|
||||
}
|
||||
@@ -1026,17 +1012,6 @@ fn china_rfc3339(value: DateTime<Utc>) -> String {
|
||||
.to_rfc3339()
|
||||
}
|
||||
|
||||
fn map_admin_store_error(error: gateway_core::error::StoreError) -> CodexCredentialAdminError {
|
||||
match error.kind() {
|
||||
StoreErrorKind::Conflict => CodexCredentialAdminError::RevisionConflict,
|
||||
StoreErrorKind::Unavailable => CodexCredentialAdminError::StoreUnavailable,
|
||||
StoreErrorKind::InvalidState | StoreErrorKind::InvalidData => {
|
||||
CodexCredentialAdminError::NotFound
|
||||
}
|
||||
_ => CodexCredentialAdminError::StoreUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
const fn map_refresh_failure(error: RefreshFailure) -> CodexCredentialAdminError {
|
||||
match error {
|
||||
RefreshFailure::InvalidGrant => CodexCredentialAdminError::RefreshRejected,
|
||||
@@ -1058,67 +1033,29 @@ const fn map_identity_error(error: CodexIdentityVerificationError) -> CodexCrede
|
||||
fn parse_import_document(
|
||||
payload: &Value,
|
||||
) -> Result<Vec<ParsedCodexImportAccount>, CodexCredentialAdminError> {
|
||||
let shape = if looks_like_agent_identity_document(payload) {
|
||||
ImportDocumentShape::AgentIdentity
|
||||
} else if looks_like_credential_bundle(payload) {
|
||||
ImportDocumentShape::CredentialBundle
|
||||
} else if looks_like_auth_document(payload) {
|
||||
ImportDocumentShape::AuthDocument
|
||||
} else {
|
||||
ImportDocumentShape::Native
|
||||
};
|
||||
if shape == ImportDocumentShape::Native
|
||||
&& payload.get("accounts").is_some()
|
||||
&& payload.as_object().is_none_or(|object| {
|
||||
object
|
||||
.keys()
|
||||
.any(|key| !CPR_CONTAINER_KEYS.contains(&key.as_str()))
|
||||
})
|
||||
{
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let values = import_account_values(payload)?;
|
||||
let mut accounts = Vec::new();
|
||||
for value in values {
|
||||
// CPR 导出允许 OAuth 与 Agent Identity 共存;按账号选择 Provider
|
||||
// 解析器,避免一个 Agent Identity 条目把整份文档误判成单一格式。
|
||||
let account_shape = if looks_like_agent_identity_account(value) {
|
||||
ImportDocumentShape::AgentIdentity
|
||||
} else if shape == ImportDocumentShape::AgentIdentity {
|
||||
if value.get("credentials").is_some() || value.get("tokens").is_some() {
|
||||
ImportDocumentShape::CredentialBundle
|
||||
} else if value
|
||||
.as_object()
|
||||
.is_some_and(auth_document_provider_is_openai)
|
||||
{
|
||||
ImportDocumentShape::AuthDocument
|
||||
} else {
|
||||
ImportDocumentShape::Native
|
||||
}
|
||||
} else {
|
||||
shape
|
||||
};
|
||||
let parsed = match account_shape {
|
||||
ImportDocumentShape::Native => Some(parse_native_account(value)?),
|
||||
ImportDocumentShape::AgentIdentity => Some(parse_agent_identity_account(value)?),
|
||||
ImportDocumentShape::CredentialBundle => parse_credential_bundle_account(value)?,
|
||||
ImportDocumentShape::AuthDocument => parse_auth_document_account(value)?,
|
||||
};
|
||||
if let Some(parsed) = parsed {
|
||||
accounts.push(parsed);
|
||||
for value in import_account_values(payload)? {
|
||||
if looks_like_agent_identity_account(value) {
|
||||
accounts.push(parse_agent_identity_account(value)?);
|
||||
continue;
|
||||
}
|
||||
if !is_openai_oauth_candidate(value) {
|
||||
continue;
|
||||
}
|
||||
accounts.push(ParsedCodexImportAccount {
|
||||
id: None,
|
||||
name: None,
|
||||
email: None,
|
||||
plan_type: None,
|
||||
chatgpt_account_id: None,
|
||||
chatgpt_user_id: None,
|
||||
authentication: parse_oauth_import_tokens(value)?,
|
||||
status: None,
|
||||
});
|
||||
}
|
||||
Ok(accounts)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum ImportDocumentShape {
|
||||
Native,
|
||||
AgentIdentity,
|
||||
CredentialBundle,
|
||||
AuthDocument,
|
||||
}
|
||||
|
||||
fn parse_agent_identity_account(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
@@ -1189,274 +1126,44 @@ fn import_account_values(payload: &Value) -> Result<Vec<&Value>, CodexCredential
|
||||
Ok(vec![payload])
|
||||
}
|
||||
|
||||
fn parse_native_account(
|
||||
fn parse_oauth_import_tokens(
|
||||
value: &Value,
|
||||
) -> Result<ParsedCodexImportAccount, CodexCredentialAdminError> {
|
||||
if value.get("accounts").is_some() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
) -> Result<ParsedCodexAuthentication, CodexCredentialAdminError> {
|
||||
let mut access_token = None;
|
||||
let mut refresh_token = None;
|
||||
let mut pending = vec![value];
|
||||
while let Some(current) = pending.pop() {
|
||||
match current {
|
||||
Value::Object(object) => {
|
||||
for (key, value) in object {
|
||||
let token = match key.as_str() {
|
||||
"access_token" | "accessToken" | "at" | "token" => &mut access_token,
|
||||
"refresh_token" | "refreshToken" | "rt" => &mut refresh_token,
|
||||
_ => {
|
||||
pending.push(value);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let value = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or(CodexCredentialAdminError::InvalidCredential)?;
|
||||
if token.replace(value.to_owned()).is_some() {
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::Array(values) => pending.extend(values),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let account = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?;
|
||||
if account
|
||||
.keys()
|
||||
.any(|key| !CPR_ACCOUNT_KEYS.contains(&key.as_str()))
|
||||
{
|
||||
return Err(CodexCredentialAdminError::InvalidInput);
|
||||
}
|
||||
let access_token = first_string(value, &["token", "at", "accessToken", "access_token"]);
|
||||
let refresh_token = first_string(value, &["refreshToken", "refresh_token"]);
|
||||
if access_token.is_none() && refresh_token.is_none() {
|
||||
return Err(CodexCredentialAdminError::InvalidCredential);
|
||||
}
|
||||
Ok(ParsedCodexImportAccount {
|
||||
id: first_string(value, &["id"]),
|
||||
name: first_string(value, &["label", "email"]),
|
||||
email: first_string(value, &["email"]),
|
||||
plan_type: first_string(value, &["planType", "plan_type"]),
|
||||
chatgpt_account_id: first_string(value, &["accountId", "account_id"]),
|
||||
chatgpt_user_id: first_string(value, &["userId", "user_id"]),
|
||||
authentication: ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token: None,
|
||||
},
|
||||
status: first_string(value, &["status"]),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_credential_bundle_account(
|
||||
value: &Value,
|
||||
) -> Result<Option<ParsedCodexImportAccount>, CodexCredentialAdminError> {
|
||||
let account = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?;
|
||||
if let Some(credentials) = value.get("credentials") {
|
||||
if account
|
||||
.get("platform")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|value| !value.eq_ignore_ascii_case("openai"))
|
||||
|| account
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|value| !value.eq_ignore_ascii_case("oauth"))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let access_token = first_path_string(
|
||||
credentials,
|
||||
&[
|
||||
&["access_token"],
|
||||
&["accessToken"],
|
||||
&["at"],
|
||||
&["token", "access_token"],
|
||||
&["token", "accessToken"],
|
||||
&["token", "at"],
|
||||
],
|
||||
);
|
||||
let refresh_token = first_path_string(
|
||||
credentials,
|
||||
&[
|
||||
&["refresh_token"],
|
||||
&["refreshToken"],
|
||||
&["rt"],
|
||||
&["token", "refresh_token"],
|
||||
&["token", "refreshToken"],
|
||||
&["token", "rt"],
|
||||
],
|
||||
);
|
||||
if access_token.is_none() && refresh_token.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(Some(ParsedCodexImportAccount {
|
||||
id: first_string(value, &["id"]),
|
||||
name: first_string(value, &["label", "name"]),
|
||||
email: first_path_string(credentials, &[&["email"], &["user", "email"]]),
|
||||
plan_type: first_path_string(
|
||||
credentials,
|
||||
&[&["plan_type"], &["planType"], &["plan", "type"]],
|
||||
),
|
||||
chatgpt_account_id: first_path_string(
|
||||
credentials,
|
||||
&[
|
||||
&["chatgpt_account_id"],
|
||||
&["chatgptAccountId"],
|
||||
&["account_id"],
|
||||
&["accountId"],
|
||||
&["account", "id"],
|
||||
&["account", "account_id"],
|
||||
&["account", "chatgpt_account_id"],
|
||||
],
|
||||
),
|
||||
chatgpt_user_id: first_path_string(
|
||||
credentials,
|
||||
&[
|
||||
&["chatgpt_user_id"],
|
||||
&["chatgptUserId"],
|
||||
&["user_id"],
|
||||
&["userId"],
|
||||
&["user", "id"],
|
||||
&["account", "user_id"],
|
||||
],
|
||||
),
|
||||
authentication: ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token: first_path_string(credentials, &[&["id_token"], &["idToken"]]),
|
||||
},
|
||||
status: first_string(value, &["status"]),
|
||||
}));
|
||||
}
|
||||
let access_token = first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["tokens", "access_token"],
|
||||
&["tokens", "accessToken"],
|
||||
&["tokens", "at"],
|
||||
&["access_token"],
|
||||
&["accessToken"],
|
||||
&["token"],
|
||||
&["at"],
|
||||
],
|
||||
);
|
||||
let refresh_token = first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["tokens", "refresh_token"],
|
||||
&["tokens", "refreshToken"],
|
||||
&["tokens", "rt"],
|
||||
&["refresh_token"],
|
||||
&["refreshToken"],
|
||||
&["rt"],
|
||||
],
|
||||
);
|
||||
if access_token.is_none() && refresh_token.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(ParsedCodexImportAccount {
|
||||
id: first_string(value, &["id"]),
|
||||
name: first_path_string(value, &[&["label"], &["name"], &["user", "name"]]),
|
||||
email: first_path_string(value, &[&["email"], &["user", "email"]]),
|
||||
plan_type: first_path_string(value, &[&["plan_type"], &["planType"], &["plan", "type"]]),
|
||||
chatgpt_account_id: first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["chatgpt_account_id"],
|
||||
&["chatgptAccountId"],
|
||||
&["account_id"],
|
||||
&["accountId"],
|
||||
&["account", "id"],
|
||||
&["account", "account_id"],
|
||||
&["account", "chatgpt_account_id"],
|
||||
],
|
||||
),
|
||||
chatgpt_user_id: first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["chatgpt_user_id"],
|
||||
&["chatgptUserId"],
|
||||
&["user_id"],
|
||||
&["userId"],
|
||||
&["user", "id"],
|
||||
&["account", "user_id"],
|
||||
],
|
||||
),
|
||||
authentication: ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token: first_path_string(
|
||||
value,
|
||||
&[&["tokens", "id_token"], &["id_token"], &["idToken"]],
|
||||
),
|
||||
},
|
||||
status: first_string(value, &["status"]),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_auth_document_account(
|
||||
value: &Value,
|
||||
) -> Result<Option<ParsedCodexImportAccount>, CodexCredentialAdminError> {
|
||||
let account = value
|
||||
.as_object()
|
||||
.ok_or(CodexCredentialAdminError::InvalidInput)?;
|
||||
if !auth_document_provider_is_openai(account) {
|
||||
return Ok(None);
|
||||
}
|
||||
let access_token = first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["access_token"],
|
||||
&["accessToken"],
|
||||
&["at"],
|
||||
&["token", "access_token"],
|
||||
&["token", "accessToken"],
|
||||
&["token", "at"],
|
||||
&["token"],
|
||||
],
|
||||
);
|
||||
let refresh_token = first_path_string(
|
||||
value,
|
||||
&[
|
||||
&["refresh_token"],
|
||||
&["refreshToken"],
|
||||
&["rt"],
|
||||
&["token", "refresh_token"],
|
||||
&["token", "refreshToken"],
|
||||
&["token", "rt"],
|
||||
],
|
||||
);
|
||||
if access_token.is_none() && refresh_token.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(ParsedCodexImportAccount {
|
||||
id: None,
|
||||
name: first_string(value, &["label", "name", "email"]),
|
||||
email: first_string(value, &["email"]),
|
||||
plan_type: first_string(value, &["plan_type", "planType"]),
|
||||
chatgpt_account_id: first_string(
|
||||
value,
|
||||
&[
|
||||
"chatgpt_account_id",
|
||||
"chatgptAccountId",
|
||||
"account_id",
|
||||
"accountId",
|
||||
],
|
||||
),
|
||||
chatgpt_user_id: first_string(
|
||||
value,
|
||||
&["chatgpt_user_id", "chatgptUserId", "user_id", "userId"],
|
||||
),
|
||||
authentication: ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
id_token: first_string(value, &["id_token", "idToken"]),
|
||||
},
|
||||
status: account
|
||||
.get("disabled")
|
||||
.and_then(Value::as_bool)
|
||||
.filter(|disabled| *disabled)
|
||||
.map(|_| "disabled".to_owned()),
|
||||
}))
|
||||
}
|
||||
|
||||
fn looks_like_credential_bundle(value: &Value) -> bool {
|
||||
value.get("exported_at").is_some()
|
||||
|| value.get("proxies").is_some()
|
||||
|| import_account_values(value).is_ok_and(|accounts| {
|
||||
accounts.iter().any(|account| {
|
||||
account.get("credentials").is_some()
|
||||
|| account.get("tokens").is_some()
|
||||
|| account.get("cachedQuota").is_some()
|
||||
|| account.get("cached_quota").is_some()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn looks_like_agent_identity_document(value: &Value) -> bool {
|
||||
import_account_values(value).is_ok_and(|accounts| {
|
||||
accounts
|
||||
.iter()
|
||||
.any(|account| looks_like_agent_identity_account(account))
|
||||
Ok(ParsedCodexAuthentication::OAuth {
|
||||
access_token,
|
||||
refresh_token,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1471,26 +1178,24 @@ fn looks_like_agent_identity_account(value: &Value) -> bool {
|
||||
|| first_string(identity, &["agent_runtime_id", "agentRuntimeId"]).is_some()
|
||||
}
|
||||
|
||||
fn looks_like_auth_document(value: &Value) -> bool {
|
||||
import_account_values(value).is_ok_and(|accounts| {
|
||||
accounts.iter().any(|account| {
|
||||
account
|
||||
.as_object()
|
||||
.is_some_and(auth_document_provider_is_openai)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn auth_document_provider_is_openai(account: &serde_json::Map<String, Value>) -> bool {
|
||||
account
|
||||
.get("type")
|
||||
fn is_openai_oauth_candidate(value: &Value) -> bool {
|
||||
let Some(account) = value.as_object() else {
|
||||
return false;
|
||||
};
|
||||
if let Some(provider) = account
|
||||
.get("platform")
|
||||
.or_else(|| account.get("provider"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.is_some_and(|provider| {
|
||||
provider.eq_ignore_ascii_case("openai")
|
||||
// CLIProxyAPI 的 OpenAI Codex OAuth 文件使用 `type: codex`。
|
||||
|| provider.eq_ignore_ascii_case("codex")
|
||||
{
|
||||
return provider.eq_ignore_ascii_case("openai") || provider.eq_ignore_ascii_case("codex");
|
||||
}
|
||||
account
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_none_or(|kind| {
|
||||
kind.eq_ignore_ascii_case("openai")
|
||||
|| kind.eq_ignore_ascii_case("codex")
|
||||
|| kind.eq_ignore_ascii_case("oauth")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1502,23 +1207,6 @@ fn first_string(value: &Value, keys: &[&str]) -> Option<String> {
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn first_path_string(value: &Value, paths: &[&[&str]]) -> Option<String> {
|
||||
paths
|
||||
.iter()
|
||||
.find_map(|path| value_at_path(value, path).and_then(Value::as_str))
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn value_at_path<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> {
|
||||
let mut current = value;
|
||||
for key in path {
|
||||
current = current.get(*key)?;
|
||||
}
|
||||
Some(current)
|
||||
}
|
||||
|
||||
fn normalize_bearer(value: &str) -> String {
|
||||
let value = value.trim();
|
||||
value
|
||||
|
||||
@@ -347,7 +347,7 @@ impl CodexAgentIdentityTaskService {
|
||||
self.load_current(account_id).await
|
||||
}
|
||||
|
||||
async fn load_current(
|
||||
pub(crate) async fn load_current(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<PreparedCodexRuntimeCredential, CodexAgentIdentityError> {
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialRevision, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
AccountAvailability, OpaqueProviderData, ProviderAccount, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::engine::provider::ProviderCatalogGeneration;
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -96,13 +96,7 @@ impl CodexPlanCatalog {
|
||||
pub struct CodexCredentialCatalogSnapshot {
|
||||
observed_at: SystemTime,
|
||||
models: Vec<CodexCatalogModel>,
|
||||
account_models: BTreeMap<ProviderAccountId, CodexAccountEntitlement>,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
struct CodexAccountEntitlement {
|
||||
revision: CredentialRevision,
|
||||
models: Vec<String>,
|
||||
scope_models: BTreeMap<CodexCatalogScope, Vec<String>>,
|
||||
}
|
||||
|
||||
impl CodexCredentialCatalogSnapshot {
|
||||
@@ -116,16 +110,12 @@ impl CodexCredentialCatalogSnapshot {
|
||||
&self.models
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn account_models(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
revision: CredentialRevision,
|
||||
) -> Option<&[String]> {
|
||||
self.account_models
|
||||
.get(account_id)
|
||||
.filter(|entry| entry.revision == revision)
|
||||
.map(|entry| entry.models.as_slice())
|
||||
account: &ProviderAccount,
|
||||
) -> Result<Option<&[String]>, CodexCredentialCatalogError> {
|
||||
let scope = CodexCatalogScope::for_account(account)?;
|
||||
Ok(self.scope_models.get(&scope).map(Vec::as_slice))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +125,7 @@ impl fmt::Debug for CodexCredentialCatalogSnapshot {
|
||||
.debug_struct("CodexCredentialCatalogSnapshot")
|
||||
.field("observed_at", &self.observed_at)
|
||||
.field("model_count", &self.models.len())
|
||||
.field("account_count", &self.account_models.len())
|
||||
.field("scope_count", &self.scope_models.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -173,8 +163,6 @@ struct CatalogCacheState {
|
||||
}
|
||||
|
||||
struct FetchedAccountModels {
|
||||
account_id: ProviderAccountId,
|
||||
revision: CredentialRevision,
|
||||
models: Vec<CodexCatalogModel>,
|
||||
etag: Option<String>,
|
||||
}
|
||||
@@ -246,30 +234,30 @@ impl CodexCredentialCatalogService {
|
||||
.cloned())
|
||||
}
|
||||
|
||||
/// 读取单账号当前仍新鲜的模型 entitlement;不触发网络。
|
||||
/// 读取账号当前已验证套餐的模型 entitlement;不触发网络。
|
||||
pub fn cached_account_models(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
revision: CredentialRevision,
|
||||
account: &ProviderAccount,
|
||||
) -> Result<Option<Vec<String>>, CodexCredentialCatalogError> {
|
||||
Ok(self.cached()?.and_then(|snapshot| {
|
||||
snapshot
|
||||
.account_models(account_id, revision)
|
||||
.map(<[String]>::to_vec)
|
||||
}))
|
||||
let Some(snapshot) = self.cached()? else {
|
||||
return Ok(None);
|
||||
};
|
||||
snapshot
|
||||
.account_models(account)
|
||||
.map(|models| models.map(<[String]>::to_vec))
|
||||
}
|
||||
|
||||
pub fn observed_model_support(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
revision: CredentialRevision,
|
||||
account: &ProviderAccount,
|
||||
model: &str,
|
||||
) -> Result<Option<bool>, CodexCredentialCatalogError> {
|
||||
Ok(self.cached()?.and_then(|snapshot| {
|
||||
snapshot
|
||||
.account_models(account_id, revision)
|
||||
.map(|models| models.iter().any(|candidate| candidate == model))
|
||||
}))
|
||||
let Some(snapshot) = self.cached()? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(snapshot.account_models(account)?.is_some_and(
|
||||
|models| models.iter().any(|candidate| candidate == model),
|
||||
)))
|
||||
}
|
||||
|
||||
/// 优先读取套餐目录 cache;缺失时才用当前账号所属套餐的有限候选集实时填充。
|
||||
@@ -353,13 +341,13 @@ impl CodexCredentialCatalogService {
|
||||
);
|
||||
let mut union = BTreeMap::<String, CodexCatalogModel>::new();
|
||||
let mut union_order = Vec::new();
|
||||
let mut account_models = BTreeMap::new();
|
||||
let mut scope_models = BTreeMap::new();
|
||||
let mut etags = Vec::new();
|
||||
for (scope, candidates) in groups {
|
||||
let fetched = self.fetch_scope_models(&client, candidates).await?;
|
||||
let entitlement = model_ids(&fetched.models);
|
||||
self.replace_plan_catalog(&CodexPlanCatalog::new(
|
||||
scope,
|
||||
scope.clone(),
|
||||
Utc::now(),
|
||||
entitlement.clone(),
|
||||
))
|
||||
@@ -379,13 +367,7 @@ impl CodexCredentialCatalogService {
|
||||
Entry::Occupied(_) => {}
|
||||
}
|
||||
}
|
||||
account_models.insert(
|
||||
fetched.account_id,
|
||||
CodexAccountEntitlement {
|
||||
revision: fetched.revision,
|
||||
models: entitlement,
|
||||
},
|
||||
);
|
||||
scope_models.insert(scope, entitlement);
|
||||
etags.extend(fetched.etag);
|
||||
}
|
||||
let observed_at = SystemTime::now();
|
||||
@@ -395,7 +377,7 @@ impl CodexCredentialCatalogService {
|
||||
.into_iter()
|
||||
.filter_map(|id| union.remove(&id))
|
||||
.collect(),
|
||||
account_models,
|
||||
scope_models,
|
||||
};
|
||||
Ok(FetchedCatalog { snapshot, etags })
|
||||
}
|
||||
@@ -502,8 +484,6 @@ impl CodexCredentialCatalogService {
|
||||
}
|
||||
let snapshot = result.map_err(|_| CodexCredentialCatalogError::Upstream)?;
|
||||
Ok(FetchedAccountModels {
|
||||
account_id: prepared.account.id().clone(),
|
||||
revision: prepared.account.revision(),
|
||||
models: snapshot.models().to_vec(),
|
||||
etag: snapshot.etag().map(str::to_owned),
|
||||
})
|
||||
@@ -782,7 +762,7 @@ fn same_catalog(
|
||||
left: &CodexCredentialCatalogSnapshot,
|
||||
right: &CodexCredentialCatalogSnapshot,
|
||||
) -> bool {
|
||||
left.models == right.models && left.account_models == right.account_models
|
||||
left.models == right.models && left.scope_models == right.scope_models
|
||||
}
|
||||
|
||||
fn validate_response_etag(etag: &str) -> Result<(), CodexCredentialCatalogError> {
|
||||
|
||||
@@ -116,6 +116,24 @@ impl CodexSignedIdentity {
|
||||
pub const fn access_token_expires_at(&self) -> DateTime<Utc> {
|
||||
self.access_token_expires_at
|
||||
}
|
||||
|
||||
fn into_import_profile(self) -> Result<CodexAccountProfile, CodexIdentityVerificationError> {
|
||||
let chatgpt_account_id = self
|
||||
.claimed_account_id
|
||||
.ok_or(CodexIdentityVerificationError::Rejected)?;
|
||||
let chatgpt_user_id = self
|
||||
.claimed_user_id
|
||||
.ok_or(CodexIdentityVerificationError::Rejected)?;
|
||||
Ok(CodexAccountProfile {
|
||||
email: self.email,
|
||||
oauth_subject: self.oauth_subject,
|
||||
poid: self.poid,
|
||||
chatgpt_account_id,
|
||||
chatgpt_user_id,
|
||||
plan_type: self.plan_type,
|
||||
access_token_expires_at: Some(self.access_token_expires_at),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 导入文档或现有 credential 对认证结果施加的一致性约束。
|
||||
@@ -180,6 +198,27 @@ impl CodexIdentityExpectation {
|
||||
})
|
||||
}
|
||||
|
||||
/// 重新授权只绑定公共稳定账号事实;新 OAuth subject/poid 在 OIDC
|
||||
/// 验证后随凭据一起更新。
|
||||
pub fn reauthorization(
|
||||
chatgpt_account_id: String,
|
||||
chatgpt_user_id: String,
|
||||
installation_id: String,
|
||||
) -> Result<Self, CodexIdentityVerificationError> {
|
||||
if !valid_identity(&chatgpt_account_id)
|
||||
|| !valid_identity(&chatgpt_user_id)
|
||||
|| !valid_installation_id(&installation_id)
|
||||
{
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(Self {
|
||||
chatgpt_account_id: Some(chatgpt_account_id),
|
||||
chatgpt_user_id: Some(chatgpt_user_id),
|
||||
installation_id: Some(installation_id),
|
||||
..Self::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn chatgpt_account_id(&self) -> Option<&str> {
|
||||
self.chatgpt_account_id.as_deref()
|
||||
@@ -256,6 +295,19 @@ pub trait CodexAccountIdentityVerifier: Send + Sync {
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError>;
|
||||
|
||||
/// 为无状态导入补全 OAuth 账号事实。
|
||||
///
|
||||
/// 导入不信任导出文件中的身份投影;实现必须只使用已验签的 token
|
||||
/// 与官方认证接口返回的事实生成可持久化 profile。
|
||||
async fn verify_import(
|
||||
&self,
|
||||
secret: &CodexOAuthSecret,
|
||||
) -> Result<CodexAccountProfile, CodexIdentityVerificationError> {
|
||||
self.verify(secret, &CodexIdentityExpectation::default())
|
||||
.await?
|
||||
.into_complete()
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
secret: &CodexOAuthSecret,
|
||||
@@ -580,6 +632,31 @@ impl CodexAccountIdentityVerifier for CodexAccountIdentityService {
|
||||
self.complete(secret, signed, expectation).await
|
||||
}
|
||||
|
||||
async fn verify_import(
|
||||
&self,
|
||||
secret: &CodexOAuthSecret,
|
||||
) -> Result<CodexAccountProfile, CodexIdentityVerificationError> {
|
||||
let signed = self.signed.verify_access(secret).await?;
|
||||
let expectation = CodexIdentityExpectation::imported(
|
||||
signed.claimed_account_id.clone(),
|
||||
signed.claimed_user_id.clone(),
|
||||
)?;
|
||||
let fallback = signed.clone().into_import_profile().ok();
|
||||
match self.accounts.fetch(secret, &expectation).await {
|
||||
Ok(account) => match complete_profile(signed, account, &expectation) {
|
||||
Ok(profile) => Ok(profile),
|
||||
Err(_) => fallback.ok_or(CodexIdentityVerificationError::Rejected),
|
||||
},
|
||||
// `/wham/usage` 是导入时的补全来源,不是 token 有效性的裁决者。AT
|
||||
// 已完成官方验签且带有完整账号声明时,接口拒绝、暂时不可用或返回默认
|
||||
// workspace 的冲突投影,都不能阻止恢复该 OAuth 账号。
|
||||
Err(
|
||||
CodexIdentityVerificationError::Rejected
|
||||
| CodexIdentityVerificationError::Unavailable,
|
||||
) => fallback.ok_or(CodexIdentityVerificationError::Rejected),
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_authorization(
|
||||
&self,
|
||||
secret: &CodexOAuthSecret,
|
||||
@@ -591,7 +668,22 @@ impl CodexAccountIdentityVerifier for CodexAccountIdentityService {
|
||||
.signed
|
||||
.verify_authorization(secret, id_token, expected_nonce)
|
||||
.await?;
|
||||
self.complete(secret, signed, expectation).await
|
||||
let fallback = signed_profile(signed.clone(), expectation).ok();
|
||||
match self.accounts.fetch(secret, expectation).await {
|
||||
Ok(account) => match complete_profile(signed, account, expectation) {
|
||||
Ok(profile) => Ok(CodexIdentityVerification::Complete(profile)),
|
||||
Err(_) => fallback
|
||||
.map(CodexIdentityVerification::Complete)
|
||||
.ok_or(CodexIdentityVerificationError::Rejected),
|
||||
},
|
||||
Err(CodexIdentityVerificationError::Rejected) => fallback
|
||||
.map(CodexIdentityVerification::Complete)
|
||||
.ok_or(CodexIdentityVerificationError::Rejected),
|
||||
Err(CodexIdentityVerificationError::Unavailable) => Ok(match fallback {
|
||||
Some(profile) => CodexIdentityVerification::Complete(profile),
|
||||
None => CodexIdentityVerification::SignedOnly(signed),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -829,6 +921,30 @@ fn complete_profile(
|
||||
})
|
||||
}
|
||||
|
||||
fn signed_profile(
|
||||
signed: CodexSignedIdentity,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexAccountProfile, CodexIdentityVerificationError> {
|
||||
let profile = signed.into_import_profile()?;
|
||||
if expectation
|
||||
.oauth_subject
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != profile.oauth_subject)
|
||||
|| matches!(&expectation.poid, PoidExpectation::Exact(expected) if expected != &profile.poid)
|
||||
|| expectation
|
||||
.chatgpt_account_id
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_account_id)
|
||||
|| expectation
|
||||
.chatgpt_user_id
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != profile.chatgpt_user_id)
|
||||
{
|
||||
return Err(CodexIdentityVerificationError::Rejected);
|
||||
}
|
||||
Ok(profile)
|
||||
}
|
||||
|
||||
fn authenticated_account(
|
||||
value: &Value,
|
||||
) -> Result<CodexAuthenticatedAccount, CodexIdentityVerificationError> {
|
||||
|
||||
@@ -47,7 +47,7 @@ pub use identity::{
|
||||
};
|
||||
pub use oauth::{
|
||||
CodexOAuthAdmin, CodexOAuthAdminError, CodexOAuthAdminService, CodexOAuthAuthorizationStarted,
|
||||
CodexOAuthPendingStore, CodexOAuthPendingStoreError, CodexOAuthReauthorizationTarget,
|
||||
CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore, CodexOAuthPendingStoreError,
|
||||
CodexPendingAuthorization, CompleteCodexOAuthAuthorization, CompletedCodexOAuthAuthorization,
|
||||
CompletedCodexOAuthCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
};
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
//! Codex Authorization Code + PKCE/OIDC 管理流。
|
||||
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::{fmt, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_admin::model::provider_credentials::{
|
||||
AuthorizationMutationTarget, AuthorizationOwner, PendingAuthorizationMutation,
|
||||
use gateway_admin::model::{
|
||||
AdminError,
|
||||
provider_credentials::{
|
||||
AuthorizationCommitGuard, AuthorizationMutationTarget, AuthorizationOwner,
|
||||
PendingAuthorizationMutation,
|
||||
},
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
CredentialRevision, NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
NewProviderAccount, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use gateway_core::provider_ports::ProviderRuntimePolicyPort;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
@@ -26,7 +29,6 @@ use super::admin::{
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{
|
||||
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
|
||||
OFFICIAL_CODEX_OAUTH_CLIENT_ID, OFFICIAL_CODEX_REDIRECT_URI,
|
||||
@@ -35,6 +37,7 @@ use super::token_client::{
|
||||
const AUTHORIZATION_ENDPOINT: &str = "https://auth.openai.com/oauth/authorize";
|
||||
const AUTHORIZATION_SCOPE: &str = "openid profile email offline_access";
|
||||
const AUTHORIZATION_TTL: TimeDelta = TimeDelta::minutes(10);
|
||||
const AUTHORIZATION_CLAIM_TTL: Duration = Duration::from_secs(90);
|
||||
const MAX_CALLBACK_BYTES: usize = 16 * 1024;
|
||||
const MAX_TEXT_BYTES: usize = 512;
|
||||
|
||||
@@ -43,24 +46,6 @@ pub struct StartCodexOAuthAuthorization {
|
||||
pub mutation: PendingAuthorizationMutation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CodexOAuthReauthorizationTarget {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
}
|
||||
|
||||
impl CodexOAuthReauthorizationTarget {
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct CodexOAuthAuthorizationStarted {
|
||||
pub flow_id: String,
|
||||
@@ -89,6 +74,7 @@ pub struct CompleteCodexOAuthAuthorization {
|
||||
pub struct CompletedCodexOAuthAuthorization<T> {
|
||||
pub mutation: PendingAuthorizationMutation,
|
||||
pub credential: T,
|
||||
authorization_guard: Box<dyn AuthorizationCommitGuard>,
|
||||
}
|
||||
|
||||
/// OAuth exchange 后唯一的 credential preparation 结果。
|
||||
@@ -116,6 +102,18 @@ impl<T> fmt::Debug for CompletedCodexOAuthAuthorization<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> CompletedCodexOAuthAuthorization<T> {
|
||||
pub(crate) fn into_parts(
|
||||
self,
|
||||
) -> (
|
||||
PendingAuthorizationMutation,
|
||||
T,
|
||||
Box<dyn AuthorizationCommitGuard>,
|
||||
) {
|
||||
(self.mutation, self.credential, self.authorization_guard)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for CompleteCodexOAuthAuthorization {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
@@ -136,7 +134,7 @@ pub struct CodexPendingAuthorization {
|
||||
state: SecretString,
|
||||
nonce: SecretString,
|
||||
code_verifier: SecretString,
|
||||
reauthorization: Option<CodexOAuthReauthorizationTarget>,
|
||||
reauthorization: Option<ProviderAccountId>,
|
||||
mutation: PendingAuthorizationMutation,
|
||||
}
|
||||
|
||||
@@ -150,7 +148,6 @@ pub struct StoredCodexPendingAuthorization {
|
||||
pub nonce: SecretString,
|
||||
pub code_verifier: SecretString,
|
||||
pub reauthorization_account_id: Option<String>,
|
||||
pub reauthorization_credential_revision: Option<u64>,
|
||||
pub mutation: PendingAuthorizationMutation,
|
||||
}
|
||||
|
||||
@@ -158,19 +155,11 @@ impl CodexPendingAuthorization {
|
||||
pub fn from_stored(
|
||||
input: StoredCodexPendingAuthorization,
|
||||
) -> Result<Self, CodexOAuthPendingStoreError> {
|
||||
let reauthorization = match (
|
||||
input.reauthorization_account_id,
|
||||
input.reauthorization_credential_revision,
|
||||
) {
|
||||
(None, None) => None,
|
||||
(Some(account_id), Some(revision)) => Some(CodexOAuthReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?,
|
||||
credential_revision: CredentialRevision::new(revision)
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?,
|
||||
}),
|
||||
_ => return Err(CodexOAuthPendingStoreError::InvalidValue),
|
||||
};
|
||||
let reauthorization = input
|
||||
.reauthorization_account_id
|
||||
.map(ProviderAccountId::new)
|
||||
.transpose()
|
||||
.map_err(|_| CodexOAuthPendingStoreError::InvalidValue)?;
|
||||
let pending = Self {
|
||||
flow_id: input.flow_id,
|
||||
owner_ref: input.owner_ref,
|
||||
@@ -239,7 +228,7 @@ impl CodexPendingAuthorization {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn reauthorization(&self) -> Option<&CodexOAuthReauthorizationTarget> {
|
||||
pub const fn reauthorization(&self) -> Option<&ProviderAccountId> {
|
||||
self.reauthorization.as_ref()
|
||||
}
|
||||
|
||||
@@ -284,11 +273,33 @@ pub trait CodexOAuthPendingStore: Send + Sync {
|
||||
pending: &CodexPendingAuthorization,
|
||||
) -> Result<(), CodexOAuthPendingStoreError>;
|
||||
|
||||
async fn take(
|
||||
async fn claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
) -> Result<Option<CodexPendingAuthorization>, CodexOAuthPendingStoreError>;
|
||||
claim_ref: &str,
|
||||
claim_ttl: Duration,
|
||||
) -> Result<CodexOAuthPendingClaimOutcome, CodexOAuthPendingStoreError>;
|
||||
|
||||
async fn release_claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
claim_ref: &str,
|
||||
) -> Result<bool, CodexOAuthPendingStoreError>;
|
||||
|
||||
async fn consume_claim(
|
||||
&self,
|
||||
owner_ref: &str,
|
||||
flow_id: &str,
|
||||
claim_ref: &str,
|
||||
) -> Result<bool, CodexOAuthPendingStoreError>;
|
||||
}
|
||||
|
||||
pub enum CodexOAuthPendingClaimOutcome {
|
||||
Claimed(Box<CodexPendingAuthorization>),
|
||||
NotFound,
|
||||
InProgress,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, thiserror::Error)]
|
||||
@@ -301,8 +312,12 @@ pub enum CodexOAuthAdminError {
|
||||
Conflict,
|
||||
#[error("Codex OAuth flow expired")]
|
||||
FlowExpired,
|
||||
#[error("Codex OAuth upstream rejected the operation")]
|
||||
UpstreamRejected,
|
||||
#[error("Codex OAuth callback was rejected")]
|
||||
CallbackRejected,
|
||||
#[error("Codex OAuth token exchange was rejected")]
|
||||
TokenRejected,
|
||||
#[error("Codex OAuth identity verification was rejected")]
|
||||
IdentityRejected,
|
||||
#[error("Codex OAuth upstream is unavailable")]
|
||||
UpstreamUnavailable,
|
||||
#[error("Codex OAuth exchange send state is ambiguous")]
|
||||
@@ -336,6 +351,46 @@ pub struct CodexOAuthAdminService {
|
||||
oauth_client_id: String,
|
||||
}
|
||||
|
||||
struct CodexOAuthAuthorizationCommitGuard {
|
||||
pending: Arc<dyn CodexOAuthPendingStore>,
|
||||
owner_ref: String,
|
||||
flow_id: String,
|
||||
claim_ref: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AuthorizationCommitGuard for CodexOAuthAuthorizationCommitGuard {
|
||||
async fn commit(self: Box<Self>) -> Result<(), AdminError> {
|
||||
let consumed = self
|
||||
.pending
|
||||
.consume_claim(&self.owner_ref, &self.flow_id, &self.claim_ref)
|
||||
.await
|
||||
.map_err(map_authorization_settlement_error)?;
|
||||
if consumed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AdminError::conflict(
|
||||
"OpenAI OAuth pending claim is no longer current",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn abort(self: Box<Self>) -> Result<(), AdminError> {
|
||||
let released = self
|
||||
.pending
|
||||
.release_claim(&self.owner_ref, &self.flow_id, &self.claim_ref)
|
||||
.await
|
||||
.map_err(map_authorization_settlement_error)?;
|
||||
if released {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AdminError::unavailable(
|
||||
"OpenAI OAuth pending claim could not be released",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexOAuthAdminService {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
@@ -372,15 +427,10 @@ impl CodexOAuthAdmin for CodexOAuthAdminService {
|
||||
) -> Result<CodexOAuthAuthorizationStarted, CodexOAuthAdminError> {
|
||||
let (name, reauthorization) = match command.mutation.target() {
|
||||
AuthorizationMutationTarget::Create { name } => (name.clone(), None),
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => {
|
||||
let expected_revision = CredentialRevision::new(expected_credential_revision.get())
|
||||
.map_err(|_| CodexOAuthAdminError::InvalidInput)?;
|
||||
AuthorizationMutationTarget::Reauthorize { account_id } => {
|
||||
let current = self
|
||||
.store
|
||||
.load_credential(account_id, expected_revision)
|
||||
.load_current_credential(account_id)
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
if current.account.provider().as_str() != "openai"
|
||||
@@ -388,13 +438,7 @@ impl CodexOAuthAdmin for CodexOAuthAdminService {
|
||||
{
|
||||
return Err(CodexOAuthAdminError::NotFound);
|
||||
}
|
||||
(
|
||||
current.account.name().to_owned(),
|
||||
Some(CodexOAuthReauthorizationTarget {
|
||||
account_id: account_id.clone(),
|
||||
credential_revision: expected_revision,
|
||||
}),
|
||||
)
|
||||
(current.account.name().to_owned(), Some(account_id.clone()))
|
||||
}
|
||||
};
|
||||
self.start_pending(command.mutation, name, reauthorization)
|
||||
@@ -412,23 +456,50 @@ impl CodexOAuthAdmin for CodexOAuthAdminService {
|
||||
{
|
||||
return Err(CodexOAuthAdminError::InvalidInput);
|
||||
}
|
||||
let (pending, mut secret, id_token) = self.exchange_pending(command).await?;
|
||||
let claim_ref = random_secret()?;
|
||||
let pending = self.claim_pending(&command, &claim_ref).await?;
|
||||
let completed = self
|
||||
.complete_claimed_authorization(pending, command.callback_url.expose_secret())
|
||||
.await;
|
||||
match completed {
|
||||
Ok((mutation, credential)) => Ok(CompletedCodexOAuthAuthorization {
|
||||
mutation,
|
||||
credential,
|
||||
authorization_guard: Box::new(CodexOAuthAuthorizationCommitGuard {
|
||||
pending: Arc::clone(&self.pending),
|
||||
owner_ref: command.owner_ref,
|
||||
flow_id: command.flow_id,
|
||||
claim_ref,
|
||||
}),
|
||||
}),
|
||||
Err(error) => {
|
||||
self.release_claim(&command, &claim_ref).await?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexOAuthAdminService {
|
||||
async fn complete_claimed_authorization(
|
||||
&self,
|
||||
pending: CodexPendingAuthorization,
|
||||
callback_url: &str,
|
||||
) -> Result<(PendingAuthorizationMutation, CompletedCodexOAuthCredential), CodexOAuthAdminError>
|
||||
{
|
||||
let (mut secret, id_token) = self.exchange_pending(&pending, callback_url).await?;
|
||||
let mutation = pending.mutation.clone();
|
||||
let current = if let Some(target) = pending.reauthorization() {
|
||||
let current = self
|
||||
.store
|
||||
.load_credential(target.account_id(), target.credential_revision())
|
||||
.load_current_credential(target)
|
||||
.await
|
||||
.map_err(map_store_error)?;
|
||||
Some(current)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let expectation = current
|
||||
.as_ref()
|
||||
.map(oauth_identity_expectation)
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let expectation = CodexIdentityExpectation::default();
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_authorization(&secret, &id_token, &pending.nonce, &expectation)
|
||||
@@ -451,7 +522,7 @@ impl CodexOAuthAdmin for CodexOAuthAdminService {
|
||||
.map_err(map_admin_error)?;
|
||||
CompletedCodexOAuthCredential::Reauthorize(
|
||||
self.credentials
|
||||
.prepare_rotation(RotateManagedCodexCredential {
|
||||
.prepare_reauthorization(RotateManagedCodexCredential {
|
||||
current,
|
||||
secret,
|
||||
verified_account: profile,
|
||||
@@ -483,10 +554,7 @@ impl CodexOAuthAdmin for CodexOAuthAdminService {
|
||||
.map_err(map_admin_error)?,
|
||||
)
|
||||
};
|
||||
Ok(CompletedCodexOAuthAuthorization {
|
||||
mutation,
|
||||
credential,
|
||||
})
|
||||
Ok((mutation, credential))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -495,7 +563,7 @@ impl CodexOAuthAdminService {
|
||||
&self,
|
||||
mutation: PendingAuthorizationMutation,
|
||||
name: String,
|
||||
reauthorization: Option<CodexOAuthReauthorizationTarget>,
|
||||
reauthorization: Option<ProviderAccountId>,
|
||||
) -> Result<CodexOAuthAuthorizationStarted, CodexOAuthAdminError> {
|
||||
if !valid_text(&name) {
|
||||
return Err(CodexOAuthAdminError::InvalidInput);
|
||||
@@ -514,11 +582,7 @@ impl CodexOAuthAdminService {
|
||||
state: SecretString::from(random_secret()?),
|
||||
nonce: SecretString::from(random_secret()?),
|
||||
code_verifier: SecretString::from(random_secret()?),
|
||||
reauthorization_account_id: reauthorization
|
||||
.as_ref()
|
||||
.map(|target| target.account_id().to_string()),
|
||||
reauthorization_credential_revision: reauthorization
|
||||
.map(|target| target.credential_revision().get()),
|
||||
reauthorization_account_id: reauthorization.as_ref().map(ToString::to_string),
|
||||
mutation,
|
||||
})
|
||||
.map_err(map_pending_error)?;
|
||||
@@ -533,38 +597,70 @@ impl CodexOAuthAdminService {
|
||||
})
|
||||
}
|
||||
|
||||
async fn exchange_pending(
|
||||
async fn claim_pending(
|
||||
&self,
|
||||
command: CompleteCodexOAuthAuthorization,
|
||||
) -> Result<
|
||||
(
|
||||
CodexPendingAuthorization,
|
||||
super::types::CodexOAuthSecret,
|
||||
SecretString,
|
||||
),
|
||||
CodexOAuthAdminError,
|
||||
> {
|
||||
command: &CompleteCodexOAuthAuthorization,
|
||||
claim_ref: &str,
|
||||
) -> Result<CodexPendingAuthorization, CodexOAuthAdminError> {
|
||||
if !valid_text(&command.owner_ref)
|
||||
|| !valid_text(&command.flow_id)
|
||||
|| !valid_secret(claim_ref)
|
||||
|| command.callback_url.expose_secret().len() > MAX_CALLBACK_BYTES
|
||||
{
|
||||
return Err(CodexOAuthAdminError::InvalidInput);
|
||||
}
|
||||
let pending = self
|
||||
let pending = match self
|
||||
.pending
|
||||
.take(&command.owner_ref, &command.flow_id)
|
||||
.claim(
|
||||
&command.owner_ref,
|
||||
&command.flow_id,
|
||||
claim_ref,
|
||||
AUTHORIZATION_CLAIM_TTL,
|
||||
)
|
||||
.await
|
||||
.map_err(map_pending_error)?
|
||||
.ok_or(CodexOAuthAdminError::NotFound)?;
|
||||
{
|
||||
CodexOAuthPendingClaimOutcome::Claimed(pending) => *pending,
|
||||
CodexOAuthPendingClaimOutcome::NotFound => return Err(CodexOAuthAdminError::NotFound),
|
||||
CodexOAuthPendingClaimOutcome::InProgress => {
|
||||
return Err(CodexOAuthAdminError::Conflict);
|
||||
}
|
||||
};
|
||||
if pending.expires_at <= Utc::now() {
|
||||
self.release_claim(command, claim_ref).await?;
|
||||
return Err(CodexOAuthAdminError::FlowExpired);
|
||||
}
|
||||
let (code, callback_state) = callback_parts(command.callback_url.expose_secret())?;
|
||||
Ok(pending)
|
||||
}
|
||||
|
||||
async fn release_claim(
|
||||
&self,
|
||||
command: &CompleteCodexOAuthAuthorization,
|
||||
claim_ref: &str,
|
||||
) -> Result<(), CodexOAuthAdminError> {
|
||||
if self
|
||||
.pending
|
||||
.release_claim(&command.owner_ref, &command.flow_id, claim_ref)
|
||||
.await
|
||||
.map_err(map_pending_error)?
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(CodexOAuthAdminError::StorageUnavailable)
|
||||
}
|
||||
}
|
||||
|
||||
async fn exchange_pending(
|
||||
&self,
|
||||
pending: &CodexPendingAuthorization,
|
||||
callback_url: &str,
|
||||
) -> Result<(super::types::CodexOAuthSecret, SecretString), CodexOAuthAdminError> {
|
||||
let (code, callback_state) = callback_parts(callback_url)?;
|
||||
if !constant_time_equal(
|
||||
pending.state.expose_secret().as_bytes(),
|
||||
callback_state.expose_secret().as_bytes(),
|
||||
) {
|
||||
return Err(CodexOAuthAdminError::UpstreamRejected);
|
||||
return Err(CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
let tokens = self
|
||||
.exchanger
|
||||
@@ -574,30 +670,10 @@ impl CodexOAuthAdminService {
|
||||
})
|
||||
.await
|
||||
.map_err(map_exchange_error)?;
|
||||
Ok((pending, tokens.secret, tokens.id_token))
|
||||
Ok((tokens.secret, tokens.id_token))
|
||||
}
|
||||
}
|
||||
|
||||
fn oauth_identity_expectation(
|
||||
current: &gateway_core::engine::credential::LoadedCredential,
|
||||
) -> Result<CodexIdentityExpectation, CodexOAuthAdminError> {
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
let account_id = current
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexOAuthAdminError::Credential)?;
|
||||
let principal = runtime.principal.ok_or(CodexOAuthAdminError::Credential)?;
|
||||
CodexIdentityExpectation::current(
|
||||
principal.oauth_subject,
|
||||
principal.poid,
|
||||
account_id.to_owned(),
|
||||
current.account.upstream_user_id().to_owned(),
|
||||
runtime.installation_id,
|
||||
)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)
|
||||
}
|
||||
|
||||
fn authorization_url(
|
||||
pending: &CodexPendingAuthorization,
|
||||
oauth_client_id: &str,
|
||||
@@ -622,14 +698,14 @@ fn authorization_url(
|
||||
}
|
||||
|
||||
fn callback_parts(value: &str) -> Result<(SecretString, SecretString), CodexOAuthAdminError> {
|
||||
let url = Url::parse(value).map_err(|_| CodexOAuthAdminError::UpstreamRejected)?;
|
||||
let url = Url::parse(value).map_err(|_| CodexOAuthAdminError::CallbackRejected)?;
|
||||
if url.scheme() != "http"
|
||||
|| url.host_str() != Some("localhost")
|
||||
|| url.port() != Some(1455)
|
||||
|| url.path() != "/auth/callback"
|
||||
|| url.fragment().is_some()
|
||||
{
|
||||
return Err(CodexOAuthAdminError::UpstreamRejected);
|
||||
return Err(CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
let mut code = None;
|
||||
let mut state = None;
|
||||
@@ -646,7 +722,7 @@ fn callback_parts(value: &str) -> Result<(SecretString, SecretString), CodexOAut
|
||||
let state = state.filter(|value| valid_secret(value));
|
||||
match (code, state) {
|
||||
(Some(code), Some(state)) => Ok((SecretString::from(code), SecretString::from(state))),
|
||||
_ => Err(CodexOAuthAdminError::UpstreamRejected),
|
||||
_ => Err(CodexOAuthAdminError::CallbackRejected),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -655,7 +731,7 @@ fn set_unique_callback_parameter(
|
||||
value: String,
|
||||
) -> Result<(), CodexOAuthAdminError> {
|
||||
if target.replace(value).is_some() {
|
||||
return Err(CodexOAuthAdminError::UpstreamRejected);
|
||||
return Err(CodexOAuthAdminError::CallbackRejected);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -700,15 +776,8 @@ fn pending_mutation_matches(pending: &CodexPendingAuthorization) -> bool {
|
||||
}
|
||||
match (pending.mutation.target(), pending.reauthorization.as_ref()) {
|
||||
(AuthorizationMutationTarget::Create { name }, None) => name == &pending.name,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
},
|
||||
Some(target),
|
||||
) => {
|
||||
account_id == target.account_id()
|
||||
&& expected_credential_revision.get() == target.credential_revision().get()
|
||||
(AuthorizationMutationTarget::Reauthorize { account_id }, Some(target)) => {
|
||||
account_id == target
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
@@ -722,9 +791,23 @@ fn map_pending_error(error: CodexOAuthPendingStoreError) -> CodexOAuthAdminError
|
||||
}
|
||||
}
|
||||
|
||||
fn map_authorization_settlement_error(error: CodexOAuthPendingStoreError) -> AdminError {
|
||||
match error {
|
||||
CodexOAuthPendingStoreError::InvalidValue => {
|
||||
AdminError::internal("OpenAI OAuth pending claim is invalid")
|
||||
}
|
||||
CodexOAuthPendingStoreError::Conflict => {
|
||||
AdminError::conflict("OpenAI OAuth pending claim conflicts with current state")
|
||||
}
|
||||
CodexOAuthPendingStoreError::Unavailable => {
|
||||
AdminError::unavailable("OpenAI OAuth pending claim store is unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_exchange_error(error: AuthorizationCodeExchangeError) -> CodexOAuthAdminError {
|
||||
match error {
|
||||
AuthorizationCodeExchangeError::Rejected => CodexOAuthAdminError::UpstreamRejected,
|
||||
AuthorizationCodeExchangeError::Rejected => CodexOAuthAdminError::TokenRejected,
|
||||
AuthorizationCodeExchangeError::Unavailable => CodexOAuthAdminError::UpstreamUnavailable,
|
||||
AuthorizationCodeExchangeError::Ambiguous => CodexOAuthAdminError::Ambiguous,
|
||||
}
|
||||
@@ -732,7 +815,7 @@ fn map_exchange_error(error: AuthorizationCodeExchangeError) -> CodexOAuthAdminE
|
||||
|
||||
fn map_identity_error(error: CodexIdentityVerificationError) -> CodexOAuthAdminError {
|
||||
match error {
|
||||
CodexIdentityVerificationError::Rejected => CodexOAuthAdminError::UpstreamRejected,
|
||||
CodexIdentityVerificationError::Rejected => CodexOAuthAdminError::IdentityRejected,
|
||||
CodexIdentityVerificationError::Unavailable => CodexOAuthAdminError::UpstreamUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,10 @@ const MAX_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60 * 60);
|
||||
const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const QUOTA_RESET_RECHECK_GRACE: Duration = Duration::from_secs(30);
|
||||
// ready 账号主要依赖真实请求的被动观察;这里只提供低频、定额的状态兜底。
|
||||
const QUOTA_READY_REFRESH_MAX_AGE: Duration = Duration::from_secs(2 * 60 * 60);
|
||||
const QUOTA_READY_REFRESH_BATCH_SIZE: usize = 8;
|
||||
|
||||
/// OpenAI Provider 主动额度刷新的调度策略。
|
||||
///
|
||||
@@ -175,11 +179,19 @@ impl CodexQuotaFact {
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_fact_is_active_exhaustion(fact: CodexQuotaFact, now: SystemTime) -> bool {
|
||||
fact.exhausted()
|
||||
&& fact
|
||||
.resets_at()
|
||||
.map(SystemTime::from)
|
||||
.is_none_or(|reset_at| reset_at > now)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct CodexQuotaSyncSummary {
|
||||
pub updated: u64,
|
||||
pub exhausted: u64,
|
||||
pub invalid: u64,
|
||||
pub banned: u64,
|
||||
pub cooldown: u64,
|
||||
pub transient: u64,
|
||||
pub stale: u64,
|
||||
@@ -196,6 +208,8 @@ impl CodexQuotaSyncSummary {
|
||||
pub enum CodexCredentialQuotaError {
|
||||
#[error("Codex quota response is invalid")]
|
||||
InvalidCredentialData,
|
||||
#[error("Codex OAuth access token must be refreshed before querying quota")]
|
||||
CredentialRefreshRequired,
|
||||
#[error(transparent)]
|
||||
Repository(#[from] CredentialRepositoryError),
|
||||
#[error("provider account store is unavailable")]
|
||||
@@ -235,6 +249,8 @@ struct CodexQuotaProjectionState {
|
||||
next_version: u64,
|
||||
entries: BTreeMap<ProviderAccountId, CodexQuotaSchedulingEntry>,
|
||||
last_periodic_refresh_at: BTreeMap<ProviderAccountId, Instant>,
|
||||
last_exhausted_reset_recheck: BTreeMap<ProviderAccountId, CodexQuotaResetMarker>,
|
||||
ready_refresh_cursor: Option<ProviderAccountId>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -245,6 +261,24 @@ struct CodexQuotaSchedulingEntry {
|
||||
signals: Option<AccountQuotaSignals>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct CodexQuotaResetMarker {
|
||||
revision: CredentialRevision,
|
||||
reset_at: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CodexQuotaRefreshKind {
|
||||
Exhausted { reset_at: Option<SystemTime> },
|
||||
Ready,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CodexQuotaRefreshCandidate {
|
||||
account: ProviderAccount,
|
||||
kind: CodexQuotaRefreshKind,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CodexQuotaHydrationTarget {
|
||||
account: ProviderAccount,
|
||||
@@ -270,6 +304,38 @@ enum CodexQuotaFetchError {
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CodexQuotaStateTransition {
|
||||
Banned { reason: &'static str },
|
||||
Exhausted,
|
||||
ResettableLimit { until: Option<SystemTime> },
|
||||
}
|
||||
|
||||
impl CodexQuotaStateTransition {
|
||||
const fn availability(self) -> AccountAvailability {
|
||||
match self {
|
||||
Self::Banned { .. } => AccountAvailability::Banned,
|
||||
Self::Exhausted => AccountAvailability::QuotaExhausted,
|
||||
Self::ResettableLimit { .. } => AccountAvailability::QuotaExhausted,
|
||||
}
|
||||
}
|
||||
|
||||
const fn reason(self) -> &'static str {
|
||||
match self {
|
||||
Self::Banned { reason } => reason,
|
||||
Self::Exhausted => "quota_exhausted",
|
||||
Self::ResettableLimit { .. } => "quota_rate_limited",
|
||||
}
|
||||
}
|
||||
|
||||
const fn cooldown_until(self) -> Option<SystemTime> {
|
||||
match self {
|
||||
Self::ResettableLimit { until } => until,
|
||||
Self::Banned { .. } | Self::Exhausted => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexQuotaSchedulingProjection {
|
||||
fn hydration_targets(&self, accounts: &[ProviderAccount]) -> Vec<CodexQuotaHydrationTarget> {
|
||||
let state = self
|
||||
@@ -395,17 +461,18 @@ impl CodexQuotaSchedulingProjection {
|
||||
fn reserve_periodic_refreshes(
|
||||
&self,
|
||||
accounts: Vec<ProviderAccount>,
|
||||
observations: Option<&BTreeMap<ProviderAccountId, QuotaObservation>>,
|
||||
now: SystemTime,
|
||||
) -> Vec<ProviderAccount> {
|
||||
let candidates = accounts
|
||||
.into_iter()
|
||||
.filter(|account| eligible_quota_sync_account(account, now))
|
||||
.filter_map(|account| quota_refresh_candidate(account, observations, now))
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_ids = candidates
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
.map(|candidate| candidate.account.id().clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let now = Instant::now();
|
||||
let refreshed_at = Instant::now();
|
||||
let mut state = self
|
||||
.state
|
||||
.write()
|
||||
@@ -413,26 +480,202 @@ impl CodexQuotaSchedulingProjection {
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.retain(|account_id, _| candidate_ids.contains(account_id));
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|account| {
|
||||
let due = state
|
||||
.last_periodic_refresh_at
|
||||
.get(account.id())
|
||||
.is_none_or(|last| {
|
||||
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
|
||||
});
|
||||
if due {
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), now);
|
||||
state
|
||||
.last_exhausted_reset_recheck
|
||||
.retain(|account_id, _| candidate_ids.contains(account_id));
|
||||
|
||||
let mut reserved = Vec::new();
|
||||
let mut ready = Vec::new();
|
||||
for candidate in candidates {
|
||||
match candidate.kind {
|
||||
CodexQuotaRefreshKind::Exhausted { reset_at } => {
|
||||
if reserve_exhausted_quota_refresh(
|
||||
&mut state,
|
||||
&candidate.account,
|
||||
reset_at,
|
||||
now,
|
||||
refreshed_at,
|
||||
) {
|
||||
reserved.push(candidate.account);
|
||||
}
|
||||
}
|
||||
due
|
||||
})
|
||||
.collect()
|
||||
CodexQuotaRefreshKind::Ready => ready.push(candidate.account),
|
||||
}
|
||||
}
|
||||
|
||||
ready.sort_by(|left, right| left.id().cmp(right.id()));
|
||||
let ready = ready
|
||||
.into_iter()
|
||||
.filter(|account| periodic_quota_refresh_due(&state, account.id(), refreshed_at))
|
||||
.collect::<Vec<_>>();
|
||||
let ready_len = ready.len();
|
||||
if ready_len == 0 {
|
||||
return reserved;
|
||||
}
|
||||
let start = state
|
||||
.ready_refresh_cursor
|
||||
.as_ref()
|
||||
.and_then(|cursor| ready.iter().position(|account| account.id() > cursor))
|
||||
.unwrap_or(0);
|
||||
let ready = ready
|
||||
.into_iter()
|
||||
.cycle()
|
||||
.skip(start)
|
||||
.take(ready_len.min(QUOTA_READY_REFRESH_BATCH_SIZE))
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(last) = ready.last() {
|
||||
state.ready_refresh_cursor = Some(last.id().clone());
|
||||
}
|
||||
for account in &ready {
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
}
|
||||
reserved.extend(ready);
|
||||
reserved
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_refresh_candidate(
|
||||
account: ProviderAccount,
|
||||
observations: Option<&BTreeMap<ProviderAccountId, QuotaObservation>>,
|
||||
now: SystemTime,
|
||||
) -> Option<CodexQuotaRefreshCandidate> {
|
||||
if !eligible_periodic_quota_refresh(&account, now) {
|
||||
return None;
|
||||
}
|
||||
let observation = observations.and_then(|observations| observations.get(account.id()));
|
||||
let snapshot = quota_snapshot_for_account(&account, observation);
|
||||
let kind = match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => CodexQuotaRefreshKind::Exhausted {
|
||||
reset_at: account
|
||||
.cooldown_until()
|
||||
.or_else(|| quota_reset_at_from_snapshot(snapshot.as_ref())),
|
||||
},
|
||||
AccountAvailability::Cooldown
|
||||
if snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.fact().exhausted()) =>
|
||||
{
|
||||
CodexQuotaRefreshKind::Exhausted {
|
||||
reset_at: quota_reset_at_from_snapshot(snapshot.as_ref()),
|
||||
}
|
||||
}
|
||||
AccountAvailability::Ready
|
||||
if observations.is_some() && quota_observation_is_stale(&account, observation, now) =>
|
||||
{
|
||||
CodexQuotaRefreshKind::Ready
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
Some(CodexQuotaRefreshCandidate { account, kind })
|
||||
}
|
||||
|
||||
fn quota_snapshot_for_account(
|
||||
account: &ProviderAccount,
|
||||
observation: Option<&QuotaObservation>,
|
||||
) -> Option<CodexAccountQuotaSnapshot> {
|
||||
observation
|
||||
.filter(|observation| observation.expected_revision == account.revision())
|
||||
.and_then(quota_snapshot_from_observation)
|
||||
}
|
||||
|
||||
fn quota_reset_at_from_snapshot(
|
||||
snapshot: Option<&CodexAccountQuotaSnapshot>,
|
||||
) -> Option<SystemTime> {
|
||||
snapshot
|
||||
.and_then(|snapshot| snapshot.fact().resets_at())
|
||||
.map(SystemTime::from)
|
||||
}
|
||||
|
||||
fn quota_observation_confirms_active_exhaustion(
|
||||
account: &ProviderAccount,
|
||||
observation: Option<&QuotaObservation>,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
quota_snapshot_for_account(account, observation)
|
||||
.is_some_and(|snapshot| quota_fact_is_active_exhaustion(snapshot.fact(), now))
|
||||
}
|
||||
|
||||
fn quota_observation_is_stale(
|
||||
account: &ProviderAccount,
|
||||
observation: Option<&QuotaObservation>,
|
||||
now: SystemTime,
|
||||
) -> bool {
|
||||
let Some(observation) = observation.filter(|observation| {
|
||||
observation.expected_revision == account.revision()
|
||||
&& quota_snapshot_from_observation(observation).is_some()
|
||||
}) else {
|
||||
return true;
|
||||
};
|
||||
observation
|
||||
.observed_at
|
||||
.and_then(|observed_at| now.duration_since(observed_at).ok())
|
||||
.is_none_or(|age| age >= QUOTA_READY_REFRESH_MAX_AGE)
|
||||
}
|
||||
|
||||
fn reserve_exhausted_quota_refresh(
|
||||
state: &mut CodexQuotaProjectionState,
|
||||
account: &ProviderAccount,
|
||||
reset_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
refreshed_at: Instant,
|
||||
) -> bool {
|
||||
if let Some(reset_at) = reset_at {
|
||||
if !quota_reset_recheck_ready(reset_at, now) {
|
||||
return false;
|
||||
}
|
||||
let same_reset_already_checked = state
|
||||
.last_exhausted_reset_recheck
|
||||
.get(account.id())
|
||||
.is_some_and(|marker| {
|
||||
marker.revision == account.revision() && marker.reset_at == reset_at
|
||||
});
|
||||
if same_reset_already_checked
|
||||
&& !periodic_quota_refresh_due(state, account.id(), refreshed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
state.last_exhausted_reset_recheck.insert(
|
||||
account.id().clone(),
|
||||
CodexQuotaResetMarker {
|
||||
revision: account.revision(),
|
||||
reset_at,
|
||||
},
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if !periodic_quota_refresh_due(state, account.id(), refreshed_at) {
|
||||
return false;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
true
|
||||
}
|
||||
|
||||
fn quota_reset_recheck_ready(reset_at: SystemTime, now: SystemTime) -> bool {
|
||||
reset_at
|
||||
.checked_add(QUOTA_RESET_RECHECK_GRACE)
|
||||
.is_some_and(|ready_at| ready_at <= now)
|
||||
}
|
||||
|
||||
fn periodic_quota_refresh_due(
|
||||
state: &CodexQuotaProjectionState,
|
||||
account_id: &ProviderAccountId,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.get(account_id)
|
||||
.is_none_or(|last| {
|
||||
now.saturating_duration_since(*last) >= QUOTA_PERIODIC_REFRESH_MIN_INTERVAL
|
||||
})
|
||||
}
|
||||
|
||||
fn insert_projection_entry(
|
||||
state: &mut CodexQuotaProjectionState,
|
||||
account_id: ProviderAccountId,
|
||||
@@ -516,10 +759,31 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
|
||||
pub async fn synchronize(&self) -> Result<CodexQuotaSyncSummary, CodexCredentialQuotaError> {
|
||||
let accounts = self.repository.list_for_provider().await?;
|
||||
let mut accounts = self.repository.list_for_provider().await?;
|
||||
let mut summary = CodexQuotaSyncSummary::default();
|
||||
let now = SystemTime::now();
|
||||
let accounts = self.scheduling.reserve_periodic_refreshes(accounts, now);
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
.collect::<Vec<_>>();
|
||||
let observations = match self.store.get_quotas(&account_ids).await {
|
||||
Ok(observations) => Some(
|
||||
observations
|
||||
.into_iter()
|
||||
.map(|observation| (observation.account_id.clone(), observation))
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
),
|
||||
Err(error) => {
|
||||
// 持久观察不可用时不盲扫 ready;保留已耗尽账号的既有恢复轮询。
|
||||
tracing::warn!(error = %error, "OpenAI quota refresh skipped ready-account scan");
|
||||
None
|
||||
}
|
||||
};
|
||||
self.reconcile_persisted_exhaustion(&mut accounts, observations.as_ref(), now)
|
||||
.await;
|
||||
let accounts =
|
||||
self.scheduling
|
||||
.reserve_periodic_refreshes(accounts, observations.as_ref(), now);
|
||||
if accounts.is_empty() {
|
||||
return Ok(summary);
|
||||
}
|
||||
@@ -552,25 +816,24 @@ impl CodexCredentialQuotaService {
|
||||
summary.transient += 1;
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
let (availability, reason, cooldown) = classify_error(&error, observed_at);
|
||||
match availability {
|
||||
Some(AccountAvailability::Invalid | AccountAvailability::Banned) => {
|
||||
summary.invalid += 1;
|
||||
}
|
||||
Some(AccountAvailability::QuotaExhausted) => summary.exhausted += 1,
|
||||
Some(AccountAvailability::Cooldown) => summary.cooldown += 1,
|
||||
_ => {
|
||||
summary.transient += 1;
|
||||
continue;
|
||||
let Some(transition) = quota_state_transition(&error, observed_at) else {
|
||||
summary.transient += 1;
|
||||
continue;
|
||||
};
|
||||
match transition {
|
||||
CodexQuotaStateTransition::Banned { .. } => summary.banned += 1,
|
||||
CodexQuotaStateTransition::Exhausted
|
||||
| CodexQuotaStateTransition::ResettableLimit { .. } => {
|
||||
summary.exhausted += 1;
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(
|
||||
&account,
|
||||
availability.expect("classified availability"),
|
||||
reason.map(str::to_owned),
|
||||
cooldown,
|
||||
transition.availability(),
|
||||
Some(transition.reason().to_owned()),
|
||||
transition.cooldown_until(),
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
@@ -580,6 +843,70 @@ impl CodexCredentialQuotaService {
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
async fn reconcile_persisted_exhaustion(
|
||||
&self,
|
||||
accounts: &mut [ProviderAccount],
|
||||
observations: Option<&BTreeMap<ProviderAccountId, QuotaObservation>>,
|
||||
observed_at: SystemTime,
|
||||
) {
|
||||
let Some(observations) = observations else {
|
||||
return;
|
||||
};
|
||||
for account in accounts {
|
||||
if account.availability() != AccountAvailability::Cooldown
|
||||
|| !quota_observation_confirms_active_exhaustion(
|
||||
account,
|
||||
observations.get(account.id()),
|
||||
observed_at,
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Ok(Some(current)) = self.store.get_account(account.id()).await else {
|
||||
continue;
|
||||
};
|
||||
if current.revision() != account.revision()
|
||||
|| current.availability() != AccountAvailability::Cooldown
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let snapshot = match self.read_snapshot_for(¤t).await {
|
||||
Ok(Some(snapshot))
|
||||
if quota_fact_is_active_exhaustion(snapshot.fact(), observed_at) =>
|
||||
{
|
||||
snapshot
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let cooldown_until =
|
||||
quota_reset_at_from_snapshot(Some(&snapshot)).or_else(|| current.cooldown_until());
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI persisted quota exhaustion reconciliation failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let enabled = current.enabled();
|
||||
*account = current.with_runtime_state(
|
||||
enabled,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
cooldown_until,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析并 revision-fenced 落库单账号的 Provider quota JSON。
|
||||
async fn apply_fetched_quota(
|
||||
&self,
|
||||
@@ -621,10 +948,10 @@ impl CodexCredentialQuotaService {
|
||||
} else {
|
||||
summary.updated += 1;
|
||||
}
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
fact.exhausted(),
|
||||
fact,
|
||||
SystemTime::now(),
|
||||
) {
|
||||
let _ = self
|
||||
@@ -633,7 +960,7 @@ impl CodexCredentialQuotaService {
|
||||
¤t,
|
||||
availability,
|
||||
fact.exhausted().then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
@@ -687,10 +1014,10 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
self.scheduling
|
||||
.observe(current.id().clone(), current.revision(), observed_at, fact);
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
fact.exhausted(),
|
||||
fact,
|
||||
observed_at,
|
||||
) {
|
||||
self.repository
|
||||
@@ -698,7 +1025,7 @@ impl CodexCredentialQuotaService {
|
||||
¤t,
|
||||
availability,
|
||||
fact.exhausted().then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
@@ -717,6 +1044,17 @@ impl CodexCredentialQuotaService {
|
||||
.await?
|
||||
.filter(|account| account.provider().as_str() == "openai")
|
||||
.ok_or(CodexCredentialQuotaError::NotFound)?;
|
||||
self.read_snapshot_for(&account).await
|
||||
}
|
||||
|
||||
async fn read_snapshot_for(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
) -> Result<Option<CodexAccountQuotaSnapshot>, CodexCredentialQuotaError> {
|
||||
if account.provider().as_str() != "openai" {
|
||||
return Err(CodexCredentialQuotaError::NotFound);
|
||||
}
|
||||
let account_id = account.id();
|
||||
let Some(observation) = self
|
||||
.store
|
||||
.get_quotas(std::slice::from_ref(account_id))
|
||||
@@ -763,12 +1101,15 @@ impl CodexCredentialQuotaService {
|
||||
.await?
|
||||
.filter(|account| account.provider().as_str() == "openai")
|
||||
.ok_or(CodexCredentialQuotaError::NotFound)?;
|
||||
let observed_at = SystemTime::now();
|
||||
if !access_token_is_current(&account, observed_at) {
|
||||
return Err(CodexCredentialQuotaError::CredentialRefreshRequired);
|
||||
}
|
||||
let client = CodexBackendClient::new(
|
||||
self.http.clone(),
|
||||
self.base_url.clone(),
|
||||
self.profile.clone(),
|
||||
);
|
||||
let observed_at = SystemTime::now();
|
||||
let FetchedCodexQuota { account, value } =
|
||||
match self.fetch_usage_with_recovery(&client, &account).await {
|
||||
Ok(fetched) => fetched,
|
||||
@@ -779,15 +1120,14 @@ impl CodexCredentialQuotaService {
|
||||
return Err(CodexCredentialQuotaError::Upstream);
|
||||
}
|
||||
Err(CodexQuotaFetchError::Upstream { account, error }) => {
|
||||
let (availability, reason, cooldown) = classify_error(&error, observed_at);
|
||||
if let Some(availability) = availability {
|
||||
if let Some(transition) = quota_state_transition(&error, observed_at) {
|
||||
let _ = self
|
||||
.repository
|
||||
.apply_state(
|
||||
&account,
|
||||
availability,
|
||||
reason.map(str::to_owned),
|
||||
cooldown,
|
||||
transition.availability(),
|
||||
Some(transition.reason().to_owned()),
|
||||
transition.cooldown_until(),
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
@@ -832,10 +1172,10 @@ impl CodexCredentialQuotaService {
|
||||
snapshot.observed_at(),
|
||||
snapshot.fact(),
|
||||
);
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
snapshot.fact().exhausted(),
|
||||
snapshot.fact(),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
self.repository
|
||||
@@ -846,7 +1186,7 @@ impl CodexCredentialQuotaService {
|
||||
.fact()
|
||||
.exhausted()
|
||||
.then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
@@ -918,12 +1258,14 @@ async fn fetch_usage_once(
|
||||
.map_err(CodexQuotaFetchAttemptError::Upstream)
|
||||
}
|
||||
|
||||
fn eligible_quota_sync_account(account: &ProviderAccount, now: SystemTime) -> bool {
|
||||
account.enabled()
|
||||
&& account
|
||||
.access_token_expires_at()
|
||||
.is_none_or(|expires_at| expires_at > now)
|
||||
&& account.availability() == AccountAvailability::QuotaExhausted
|
||||
fn eligible_periodic_quota_refresh(account: &ProviderAccount, now: SystemTime) -> bool {
|
||||
account.enabled() && access_token_is_current(account, now)
|
||||
}
|
||||
|
||||
fn access_token_is_current(account: &ProviderAccount, now: SystemTime) -> bool {
|
||||
account
|
||||
.access_token_expires_at()
|
||||
.is_none_or(|expires_at| expires_at > now)
|
||||
}
|
||||
|
||||
fn merge_passive_quota(
|
||||
@@ -1052,41 +1394,40 @@ fn passive_rate_limit_window(window: RateLimitWindow) -> Map<String, Value> {
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn quota_success_availability(
|
||||
fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
exhausted: bool,
|
||||
fact: CodexQuotaFact,
|
||||
now: SystemTime,
|
||||
) -> Option<AccountAvailability> {
|
||||
match current {
|
||||
) -> Option<(AccountAvailability, Option<SystemTime>)> {
|
||||
if fact.exhausted() {
|
||||
let observed_cooldown = fact.resets_at().map(SystemTime::from).or(cooldown_until);
|
||||
return (current != AccountAvailability::QuotaExhausted
|
||||
|| cooldown_until != observed_cooldown)
|
||||
.then_some((AccountAvailability::QuotaExhausted, observed_cooldown));
|
||||
}
|
||||
|
||||
let availability = match current {
|
||||
AccountAvailability::Invalid
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned => None,
|
||||
AccountAvailability::QuotaExhausted => (!exhausted).then_some(AccountAvailability::Ready),
|
||||
AccountAvailability::Ready => exhausted.then_some(AccountAvailability::QuotaExhausted),
|
||||
AccountAvailability::Cooldown if exhausted => Some(AccountAvailability::QuotaExhausted),
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted => Some(AccountAvailability::Ready),
|
||||
AccountAvailability::Ready => None,
|
||||
AccountAvailability::Cooldown
|
||||
if cooldown_until.is_some_and(|cooldown_until| cooldown_until <= now) =>
|
||||
{
|
||||
Some(AccountAvailability::Ready)
|
||||
}
|
||||
AccountAvailability::Cooldown => None,
|
||||
AccountAvailability::Unknown => Some(if exhausted {
|
||||
AccountAvailability::QuotaExhausted
|
||||
} else {
|
||||
AccountAvailability::Ready
|
||||
}),
|
||||
}
|
||||
};
|
||||
availability.map(|availability| (availability, None))
|
||||
}
|
||||
|
||||
fn classify_error(
|
||||
fn quota_state_transition(
|
||||
error: &CodexClientError,
|
||||
observed_at: SystemTime,
|
||||
) -> (
|
||||
Option<AccountAvailability>,
|
||||
Option<&'static str>,
|
||||
Option<SystemTime>,
|
||||
) {
|
||||
) -> Option<CodexQuotaStateTransition> {
|
||||
let CodexClientError::Upstream {
|
||||
status,
|
||||
retry_after_seconds,
|
||||
@@ -1094,38 +1435,27 @@ fn classify_error(
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
return (None, None, None);
|
||||
return None;
|
||||
};
|
||||
if *status == StatusCode::PAYMENT_REQUIRED && is_deactivated_workspace(body) {
|
||||
return (
|
||||
Some(AccountAvailability::Banned),
|
||||
Some("quota_deactivated_workspace"),
|
||||
None,
|
||||
);
|
||||
return Some(CodexQuotaStateTransition::Banned {
|
||||
reason: "quota_deactivated_workspace",
|
||||
});
|
||||
}
|
||||
match *status {
|
||||
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (
|
||||
Some(AccountAvailability::Invalid),
|
||||
Some("quota_auth_rejected"),
|
||||
None,
|
||||
),
|
||||
StatusCode::PAYMENT_REQUIRED => (
|
||||
Some(AccountAvailability::QuotaExhausted),
|
||||
Some("quota_exhausted"),
|
||||
None,
|
||||
),
|
||||
StatusCode::PAYMENT_REQUIRED => Some(CodexQuotaStateTransition::Exhausted),
|
||||
StatusCode::TOO_MANY_REQUESTS => {
|
||||
let duration = Duration::from_secs(
|
||||
retry_after_seconds.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN.as_secs()),
|
||||
)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
(
|
||||
Some(AccountAvailability::Cooldown),
|
||||
Some("quota_rate_limited"),
|
||||
observed_at.checked_add(duration),
|
||||
)
|
||||
Some(CodexQuotaStateTransition::ResettableLimit {
|
||||
until: observed_at.checked_add(duration),
|
||||
})
|
||||
}
|
||||
_ => (None, None, None),
|
||||
// quota/usage 接口只能产生额度域状态;鉴权拒绝不具备判定 RT
|
||||
// 永久失效的证据,终态只能由 OAuth refresh/身份校验状态机写入。
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1557,7 +1887,6 @@ impl QuotaAggregate {
|
||||
self.remaining_percent
|
||||
.map_or(remaining, |current| current.min(remaining)),
|
||||
);
|
||||
self.exhausted |= used >= 100.0;
|
||||
}
|
||||
if let Some(reset) = object.get("reset_at") {
|
||||
let seconds = reset
|
||||
|
||||
@@ -28,6 +28,16 @@ const MAX_REFRESH_BATCH: u32 = 1_000;
|
||||
/// 连续失败计数窗口;每次瞬态失败刷新该 TTL,静默满窗后计数过期归零。
|
||||
const REFRESH_BACKOFF_WINDOW: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
fn refresh_due_at(account: &ProviderAccount, now: SystemTime) -> Option<SystemTime> {
|
||||
if let Some(expires_at) = account.access_token_expires_at()
|
||||
&& expires_at <= now
|
||||
{
|
||||
// AT 已过期时不能继续等待持久退避;后续额度/请求探测都必须先用 RT 续出可用 AT。
|
||||
return Some(expires_at);
|
||||
}
|
||||
account.next_refresh_at().filter(|next| *next <= now)
|
||||
}
|
||||
|
||||
pub struct DueCodexCredential {
|
||||
pub account: ProviderAccount,
|
||||
pub secret: CodexOAuthSecret,
|
||||
@@ -246,11 +256,9 @@ impl CodexCredentialRefreshService {
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
)
|
||||
&& (account.availability() != AccountAvailability::Cooldown
|
||||
|| account.cooldown_until().is_some_and(|until| until <= now))
|
||||
&& account.next_refresh_at().is_some_and(|next| next <= now)
|
||||
&& refresh_due_at(account, now).is_some()
|
||||
});
|
||||
accounts.sort_by_key(|account| (account.next_refresh_at(), account.id().clone()));
|
||||
accounts.sort_by_key(|account| (refresh_due_at(account, now), account.id().clone()));
|
||||
accounts.truncate(MAX_REFRESH_BATCH as usize);
|
||||
let mut due = Vec::with_capacity(accounts.len());
|
||||
let mut failures = Vec::new();
|
||||
|
||||
@@ -237,6 +237,9 @@ impl CodexCredentialRepository {
|
||||
cooldown_until: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
|
||||
@@ -6,8 +6,9 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountCandidate, AccountFeedbackStats, AccountRuntimeSignals,
|
||||
AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId,
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
|
||||
AccountRuntimeSignals, AccountSchedulingBlocker, AccountSelectionContext, AccountSelector,
|
||||
PreferredAccountSelection, ProviderAccount, ProviderAccountId,
|
||||
};
|
||||
use gateway_core::engine::{AttemptContext, ContinuationAttempt};
|
||||
use gateway_core::provider_ports::{
|
||||
@@ -55,6 +56,11 @@ pub enum CodexAccountFailure {
|
||||
Banned,
|
||||
/// 账号信用额度已耗尽。
|
||||
QuotaExhausted,
|
||||
/// 当前用量窗口已耗尽;到重置时间后可自动恢复。
|
||||
UsageLimitExhausted {
|
||||
/// 上游返回的窗口重置时长。
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
/// 账号触发临时用量限制。
|
||||
RateLimited {
|
||||
/// 上游明确返回的最短冷却时长。
|
||||
@@ -117,6 +123,118 @@ enum SessionAffinityLookup {
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AffinityEscapeReason {
|
||||
HardUnavailable,
|
||||
QuotaExhausted,
|
||||
Cooldown,
|
||||
LeaseSaturated,
|
||||
PinnedAccount,
|
||||
SelectionInvariant,
|
||||
}
|
||||
|
||||
impl AffinityEscapeReason {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::HardUnavailable => "hard_unavailable",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Cooldown => "cooldown",
|
||||
Self::LeaseSaturated => "lease_saturated",
|
||||
Self::PinnedAccount => "pinned_account",
|
||||
Self::SelectionInvariant => "selection_invariant",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct AffinitySelection {
|
||||
bound_account: Option<ProviderAccountId>,
|
||||
preferred_account: Option<ProviderAccountId>,
|
||||
escape_reason: Option<AffinityEscapeReason>,
|
||||
}
|
||||
|
||||
impl AffinitySelection {
|
||||
fn preferred(account_id: ProviderAccountId) -> Self {
|
||||
Self {
|
||||
bound_account: Some(account_id.clone()),
|
||||
preferred_account: Some(account_id),
|
||||
escape_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn escaped(account_id: ProviderAccountId, reason: AffinityEscapeReason) -> Self {
|
||||
Self {
|
||||
bound_account: Some(account_id),
|
||||
preferred_account: None,
|
||||
escape_reason: Some(reason),
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_account(&self) -> Option<&ProviderAccountId> {
|
||||
self.bound_account.as_ref()
|
||||
}
|
||||
|
||||
fn preferred_account(&self) -> Option<&ProviderAccountId> {
|
||||
self.preferred_account.as_ref()
|
||||
}
|
||||
|
||||
fn escape(&mut self, reason: AffinityEscapeReason) {
|
||||
if self.bound_account.is_some() && self.escape_reason.is_none() {
|
||||
self.escape_reason = Some(reason);
|
||||
self.preferred_account = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_preferred_selection(&mut self, selection: PreferredAccountSelection) {
|
||||
if self.bound_account.is_none() || self.escape_reason.is_some() {
|
||||
return;
|
||||
}
|
||||
match selection {
|
||||
PreferredAccountSelection::Hit => {}
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::ConcurrencyLimit) => {
|
||||
self.escape(AffinityEscapeReason::LeaseSaturated);
|
||||
}
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::RequestInterval) => {
|
||||
self.escape(AffinityEscapeReason::Cooldown);
|
||||
}
|
||||
PreferredAccountSelection::Blocked(
|
||||
AccountSchedulingBlocker::LocalAvailability | AccountSchedulingBlocker::Excluded,
|
||||
)
|
||||
| PreferredAccountSelection::Missing => {
|
||||
self.escape(AffinityEscapeReason::HardUnavailable);
|
||||
}
|
||||
PreferredAccountSelection::NotRequested => {
|
||||
self.escape(AffinityEscapeReason::SelectionInvariant);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_lease_busy(&mut self, account_id: &ProviderAccountId) {
|
||||
if self.bound_account.as_ref() == Some(account_id) {
|
||||
self.escape(AffinityEscapeReason::LeaseSaturated);
|
||||
}
|
||||
}
|
||||
|
||||
fn telemetry(&self, selected_account: &ProviderAccountId) -> AffinityTelemetry {
|
||||
AffinityTelemetry {
|
||||
affinity_hit: self.bound_account.as_ref() == Some(selected_account)
|
||||
&& self.escape_reason.is_none(),
|
||||
escape_reason: self.escape_reason,
|
||||
account_switch: self
|
||||
.bound_account
|
||||
.as_ref()
|
||||
.is_some_and(|bound| bound != selected_account),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct AffinityTelemetry {
|
||||
affinity_hit: bool,
|
||||
escape_reason: Option<AffinityEscapeReason>,
|
||||
account_switch: bool,
|
||||
}
|
||||
|
||||
impl CodexCredentialSelector {
|
||||
#[must_use]
|
||||
// 选择器显式持有各能力边界,避免把 Provider 私有服务重新包装成通用容器。
|
||||
@@ -162,22 +280,23 @@ impl CodexCredentialSelector {
|
||||
request: &SelectCodexCredential<'_>,
|
||||
cyber_policy_session_key: Option<&ProviderSessionAffinityKey>,
|
||||
) -> Result<CodexCredentialLease, CredentialSelectionError> {
|
||||
let diagnostic = request.attempt.is_diagnostic_required_account();
|
||||
let accounts = self.repository.list_for_provider().await?;
|
||||
let accounts = accounts
|
||||
.into_iter()
|
||||
.filter(|account| {
|
||||
account.provider() == &self.provider_kind
|
||||
&& !matches!(
|
||||
self.catalog.observed_model_support(
|
||||
account.id(),
|
||||
account.revision(),
|
||||
request.upstream_model,
|
||||
),
|
||||
Ok(Some(false))
|
||||
)
|
||||
&& (diagnostic || {
|
||||
let observed_support = self
|
||||
.catalog
|
||||
.observed_model_support(account, request.upstream_model);
|
||||
matches!(observed_support, Ok(None | Some(true)))
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
if !diagnostic {
|
||||
self.quota.prepare_scheduling(&accounts).await;
|
||||
}
|
||||
let account_ids = accounts
|
||||
.iter()
|
||||
.map(|account| account.id().clone())
|
||||
@@ -223,23 +342,15 @@ impl CodexCredentialSelector {
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let scheduling_candidates = quota_exhausted_candidates.as_deref().unwrap_or(&candidates);
|
||||
let affinity_account = match request.session_affinity_key {
|
||||
Some(key) => match self.lookup_session_affinity(key).await {
|
||||
SessionAffinityLookup::Bound(account_id)
|
||||
if scheduling_candidates.iter().any(|candidate| {
|
||||
candidate.account.id() == &account_id
|
||||
&& candidate.account.is_schedulable(SystemTime::now())
|
||||
}) =>
|
||||
{
|
||||
Some(account_id)
|
||||
}
|
||||
SessionAffinityLookup::Bound(_) => {
|
||||
self.clear_session_affinity(key).await;
|
||||
None
|
||||
}
|
||||
SessionAffinityLookup::Missing | SessionAffinityLookup::Unavailable => None,
|
||||
},
|
||||
None => None,
|
||||
let mut affinity = if diagnostic {
|
||||
AffinitySelection::default()
|
||||
} else {
|
||||
self.resolve_session_affinity(
|
||||
request.session_affinity_key,
|
||||
&candidates,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
let cyber_policy_scope = self
|
||||
.prepare_cyber_policy_scope(cyber_policy_session_key)
|
||||
@@ -280,7 +391,16 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
}
|
||||
let preferred = pinned_account.or(affinity_account);
|
||||
if pinned_account
|
||||
.as_ref()
|
||||
.zip(affinity.bound_account())
|
||||
.is_some_and(|(pinned, bound)| pinned != bound)
|
||||
{
|
||||
affinity.escape(AffinityEscapeReason::PinnedAccount);
|
||||
}
|
||||
let preferred = pinned_account
|
||||
.clone()
|
||||
.or_else(|| affinity.preferred_account().cloned());
|
||||
let mut shortest_retry = None;
|
||||
|
||||
loop {
|
||||
@@ -290,8 +410,13 @@ impl CodexCredentialSelector {
|
||||
excluded_accounts: excluded.clone(),
|
||||
preferred_account: preferred.clone(),
|
||||
round_robin_cursor,
|
||||
availability: if diagnostic {
|
||||
AccountAvailabilityPolicy::BypassForDiagnostic
|
||||
} else {
|
||||
AccountAvailabilityPolicy::Enforce
|
||||
},
|
||||
};
|
||||
let Some(selected) = AccountSelector.select(scheduling_candidates, &context) else {
|
||||
let Some(selection) = AccountSelector.select(scheduling_candidates, &context) else {
|
||||
return match shortest_retry {
|
||||
Some(retry_after) => Err(CredentialSelectionError::CapacityUnavailable {
|
||||
retry_after: Some(retry_after),
|
||||
@@ -299,11 +424,14 @@ impl CodexCredentialSelector {
|
||||
None => Err(CredentialSelectionError::NoEligibleCredential),
|
||||
};
|
||||
};
|
||||
affinity.observe_preferred_selection(selection.preferred());
|
||||
let selected = selection.candidate();
|
||||
let account = candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == selected.account.id())
|
||||
.map(|candidate| candidate.account.clone())
|
||||
.ok_or(CredentialSelectionError::InvalidCredential)?;
|
||||
let allows_account_state_mutation = !diagnostic || account.enabled();
|
||||
let policy = request.attempt.account_selection_policy();
|
||||
match self
|
||||
.leases
|
||||
@@ -320,18 +448,33 @@ impl CodexCredentialSelector {
|
||||
.await?
|
||||
{
|
||||
ProviderLeaseAcquisition::Busy { retry_after } => {
|
||||
affinity.observe_lease_busy(account.id());
|
||||
shortest_retry = minimum_duration(shortest_retry, retry_after);
|
||||
excluded.insert(account.id().clone());
|
||||
}
|
||||
ProviderLeaseAcquisition::Acquired(guard) => {
|
||||
let affinity_telemetry = affinity.telemetry(account.id());
|
||||
tracing::info!(
|
||||
request_id = %request.attempt.request_id(),
|
||||
attempt_index = request.attempt.attempt_index().get(),
|
||||
rotation_strategy = policy.strategy().as_str(),
|
||||
account_id = %account.id(),
|
||||
affinity_hit = affinity_telemetry.affinity_hit,
|
||||
escape_reason = affinity_telemetry
|
||||
.escape_reason
|
||||
.map_or("", AffinityEscapeReason::as_str),
|
||||
account_switch = affinity_telemetry.account_switch,
|
||||
"OpenAI account selected"
|
||||
);
|
||||
let (account, runtime) = if account.authentication_kind()
|
||||
== CODEX_AUTHENTICATION_KIND_AGENT_IDENTITY
|
||||
{
|
||||
let prepared = self
|
||||
.agent_identity
|
||||
.prepare(&account)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::InvalidCredential)?;
|
||||
let prepared = if allows_account_state_mutation {
|
||||
self.agent_identity.prepare(&account).await
|
||||
} else {
|
||||
self.agent_identity.load_current(account.id()).await
|
||||
}
|
||||
.map_err(|_| CredentialSelectionError::InvalidCredential)?;
|
||||
(prepared.account, prepared.credential)
|
||||
} else {
|
||||
let runtime = self.repository.load_runtime_credential(&account).await?;
|
||||
@@ -359,6 +502,8 @@ impl CodexCredentialSelector {
|
||||
authentication: runtime.authentication,
|
||||
cookies,
|
||||
cyber_policy_scope,
|
||||
allows_account_state_mutation,
|
||||
affinity_telemetry,
|
||||
_guard: guard,
|
||||
});
|
||||
}
|
||||
@@ -366,6 +511,34 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_session_affinity(
|
||||
&self,
|
||||
key: Option<&ProviderSessionAffinityKey>,
|
||||
candidates: &[AccountCandidate],
|
||||
now: SystemTime,
|
||||
) -> AffinitySelection {
|
||||
let Some(key) = key else {
|
||||
return AffinitySelection::default();
|
||||
};
|
||||
let SessionAffinityLookup::Bound(account_id) = self.lookup_session_affinity(key).await
|
||||
else {
|
||||
return AffinitySelection::default();
|
||||
};
|
||||
let Some(candidate) = candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == &account_id)
|
||||
else {
|
||||
self.clear_session_affinity(key).await;
|
||||
return AffinitySelection::escaped(account_id, AffinityEscapeReason::HardUnavailable);
|
||||
};
|
||||
if candidate.account.is_schedulable(now) {
|
||||
return AffinitySelection::preferred(account_id);
|
||||
}
|
||||
let reason = affinity_unavailable_reason(&candidate.account, now);
|
||||
self.clear_session_affinity(key).await;
|
||||
AffinitySelection::escaped(account_id, reason)
|
||||
}
|
||||
|
||||
async fn lookup_session_affinity(
|
||||
&self,
|
||||
key: &ProviderSessionAffinityKey,
|
||||
@@ -534,14 +707,20 @@ impl CodexCredentialSelector {
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::UsageLimitExhausted { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(
|
||||
account,
|
||||
retry_after,
|
||||
"usage_limit_exhausted",
|
||||
now,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
let until = now.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN));
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("rate_limited".to_owned()),
|
||||
until,
|
||||
CookieRecovery::None,
|
||||
)
|
||||
return self
|
||||
.record_resettable_limit_failure(account, retry_after, "rate_limited", now)
|
||||
.await;
|
||||
}
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after } => {
|
||||
let delay = self.cloudflare_challenge_delay(account.id(), now, retry_after);
|
||||
@@ -579,11 +758,52 @@ impl CodexCredentialSelector {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_resettable_limit_failure(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
reason: &'static str,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let current = self.current_account(account.id()).await?;
|
||||
if current.revision() != account.revision() {
|
||||
return Err(CredentialRepositoryError::RevisionConflict.into());
|
||||
}
|
||||
let cooldown_until =
|
||||
observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN));
|
||||
let cooldown_until = match (current.availability(), current.cooldown_until()) {
|
||||
(AccountAvailability::QuotaExhausted, Some(existing)) => {
|
||||
cooldown_until.map_or(Some(existing), |until| Some(existing.max(until)))
|
||||
}
|
||||
_ => cooldown_until,
|
||||
};
|
||||
let reason = if current.availability() == AccountAvailability::QuotaExhausted
|
||||
&& current.cooldown_until().is_none()
|
||||
{
|
||||
Some("quota_exhausted".to_owned())
|
||||
} else {
|
||||
Some(reason.to_owned())
|
||||
};
|
||||
// 一次真实 429 证明上游已经接受当前凭据,可纠正旧的认证/封禁终态。
|
||||
// resettable 用量限制沿用 v2 的 quota lock 语义,并保留/延长复核时间。
|
||||
self.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn record_success(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
session_affinity_key: Option<&ProviderSessionAffinityKey>,
|
||||
) {
|
||||
self.restore_recoverable_account_state(account).await;
|
||||
self.risk_recovery
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
@@ -626,6 +846,44 @@ impl CodexCredentialSelector {
|
||||
}
|
||||
}
|
||||
|
||||
async fn restore_recoverable_account_state(&self, account: &ProviderAccount) {
|
||||
let Ok(current) = self.current_account(account.id()).await else {
|
||||
return;
|
||||
};
|
||||
// QuotaExhausted 只能由结构化额度事实解除;一次成功响应本身不能证明额度已经恢复。
|
||||
if current.provider() != &self.provider_kind
|
||||
|| current.revision() != account.revision()
|
||||
|| !current.enabled()
|
||||
|| !matches!(
|
||||
current.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
None,
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %account.id(),
|
||||
error = %error,
|
||||
"OpenAI account state recovery after successful upstream response failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn current_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -764,6 +1022,25 @@ fn quota_exhausted_scheduling_projection(account: &ProviderAccount) -> ProviderA
|
||||
account.clone()
|
||||
}
|
||||
|
||||
fn affinity_unavailable_reason(account: &ProviderAccount, now: SystemTime) -> AffinityEscapeReason {
|
||||
if !account.enabled()
|
||||
|| account
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
return AffinityEscapeReason::HardUnavailable;
|
||||
}
|
||||
match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => AffinityEscapeReason::QuotaExhausted,
|
||||
AccountAvailability::Cooldown => AffinityEscapeReason::Cooldown,
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Ready
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => AffinityEscapeReason::HardUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
fn active_risk_recovery<'a>(
|
||||
recovery: &'a mut HashMap<String, RiskRecoveryState>,
|
||||
account_id: &str,
|
||||
@@ -801,6 +1078,8 @@ pub struct CodexCredentialLease {
|
||||
cookies: Vec<RuntimeCodexCookie>,
|
||||
installation_id: String,
|
||||
cyber_policy_scope: Option<CodexCyberPolicyScope>,
|
||||
allows_account_state_mutation: bool,
|
||||
affinity_telemetry: AffinityTelemetry,
|
||||
_guard: Box<dyn ProviderLeaseGuard>,
|
||||
}
|
||||
|
||||
@@ -834,6 +1113,30 @@ impl CodexCredentialLease {
|
||||
pub(crate) const fn cyber_policy_scope(&self) -> Option<&CodexCyberPolicyScope> {
|
||||
self.cyber_policy_scope.as_ref()
|
||||
}
|
||||
|
||||
/// 禁用账号的管理端诊断必须只返回真实上游结果,不能回写账号侧状态。
|
||||
#[must_use]
|
||||
pub(crate) const fn allows_account_state_mutation(&self) -> bool {
|
||||
self.allows_account_state_mutation
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn affinity_hit(&self) -> bool {
|
||||
self.affinity_telemetry.affinity_hit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn escape_reason(&self) -> Option<&'static str> {
|
||||
match self.affinity_telemetry.escape_reason {
|
||||
Some(reason) => Some(reason.as_str()),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_switch(&self) -> bool {
|
||||
self.affinity_telemetry.account_switch
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexCredentialLease {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user