mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-12 00:06:23 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5671a57dc8 | ||
|
|
166e8754a2 | ||
|
|
fbaf6c1bdd | ||
|
|
7cdc2dba6f | ||
|
|
a321ccf737 | ||
|
|
4533ae8850 | ||
|
|
7542ed365e | ||
|
|
bf4da7a3e7 | ||
|
|
0fe354b083 | ||
|
|
6d0c583599 | ||
|
|
2e861c03c4 | ||
|
|
d5fc778ea7 | ||
|
|
404cd41ff4 | ||
|
|
851f473b88 |
@@ -21,15 +21,16 @@
|
||||
|
||||
## 能力
|
||||
|
||||
| 领域 | 实现 |
|
||||
| -------- | --------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI SSE 与 WebSocket 均原样转发上游字节;xAI 在 `providers/xai` 内做 Grok 与 Responses 协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因 |
|
||||
| 领域 | 实现 |
|
||||
| -------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -93,6 +94,11 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
|
||||
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
|
||||
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
|
||||
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
|
||||
[账号接口](docs/api.md#5-账号)。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
|
||||
@@ -145,7 +151,7 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
## 运维
|
||||
|
||||
```bash
|
||||
# 升级
|
||||
# 实例升级
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build
|
||||
|
||||
@@ -179,7 +185,8 @@ release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。
|
||||
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
|
||||
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
@@ -212,6 +212,9 @@ pub enum AccountConnectionTestEvent {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
provider_error_type: Option<String>,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::fmt;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId},
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
};
|
||||
|
||||
@@ -496,6 +496,7 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
|
||||
@@ -452,12 +452,33 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_status: status,
|
||||
}))
|
||||
.collect(),
|
||||
Err(error) => vec![AccountConnectionTestEvent::Failed {
|
||||
message: error.client_message().to_owned(),
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
account_status: status,
|
||||
}],
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
.upstream_response()
|
||||
.map(gateway_core::engine::probe::AccountProbeUpstreamResponse::status);
|
||||
let upstream_content_type = error
|
||||
.upstream_response()
|
||||
.and_then(|response| response.content_type())
|
||||
.and_then(|value| std::str::from_utf8(value).ok())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_body = error
|
||||
.upstream_response()
|
||||
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
|
||||
let message = upstream_body
|
||||
.as_deref()
|
||||
.filter(|body| !body.is_empty())
|
||||
.unwrap_or_else(|| error.client_message())
|
||||
.to_owned();
|
||||
vec![AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
.flat_map(futures::stream::iter);
|
||||
@@ -468,22 +489,25 @@ impl AccountsService for DefaultAccountsService {
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
match account.availability {
|
||||
crate::model::accounts::AccountAvailability::Banned => AccountStatus::Banned,
|
||||
crate::model::accounts::AccountAvailability::QuotaExhausted => {
|
||||
AccountStatus::QuotaExhausted
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid => AccountStatus::Expired,
|
||||
_ if account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now) =>
|
||||
{
|
||||
AccountStatus::Expired
|
||||
}
|
||||
_ => AccountStatus::Active,
|
||||
}
|
||||
AccountStatus::Active
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -492,14 +492,14 @@ impl DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 逐条尽力把 calculated 总额升级为完整分解;单条脏数据(非法 Provider kind
|
||||
/// 或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
/// 逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
|
||||
for record in records {
|
||||
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if source != "calculated" {
|
||||
if !matches!(source.as_str(), "calculated" | "provider_reported") {
|
||||
continue;
|
||||
}
|
||||
let (Some(provider), Some(upstream_model_id)) = (
|
||||
|
||||
@@ -13,8 +13,7 @@ use crate::{
|
||||
CredentialDeletionResult, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutation,
|
||||
CredentialMutationResult, CredentialPage, ImportCredentials, PrepareCredentialImport,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCredential, ProviderQuotaRequest,
|
||||
RotateCredential, StartAuthorization,
|
||||
PrepareCredentialRotation, ProviderQuotaRequest, RotateCredential, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{provider::ProviderAdmin, store::AccountStore},
|
||||
@@ -195,10 +194,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI authorization",
|
||||
)
|
||||
.await?;
|
||||
let creates_account = matches!(
|
||||
&prepared.credential,
|
||||
PreparedAuthorizationCredential::Create(_)
|
||||
);
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
@@ -210,13 +205,11 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
if creates_account {
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
operation::{GenerateRequest, Operation, ProtocolPayload},
|
||||
@@ -155,6 +155,7 @@ impl FakeProviderAdmin {
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision"),
|
||||
replacement_identity: None,
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
@@ -794,12 +795,42 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
&& error_type == "invalid_request_error"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.enabled = false;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -891,6 +922,36 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -1141,7 +1202,7 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
@@ -1156,7 +1217,7 @@ impl AccountProbe for FailingAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"included usage exhausted",
|
||||
@@ -1167,7 +1228,8 @@ impl AccountProbe for FailingAccountProbe {
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
)
|
||||
.with_client_visible_upstream_error(detail))
|
||||
.with_client_visible_upstream_error(detail)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ use gateway_admin::{
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
policy::ClientApiKeyId,
|
||||
@@ -647,12 +647,13 @@ impl AccountProbe for UnavailableProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"test account probe is unavailable",
|
||||
))
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,9 +327,19 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let invalid_kind = "x".repeat(65);
|
||||
store.replace_usage_records(vec![
|
||||
calculated_total_record("request_invalid_kind", Some(&invalid_kind), now),
|
||||
calculated_total_record("request_unregistered_kind", Some("anthropic"), now),
|
||||
calculated_total_record("request_enriched", Some("openai"), now),
|
||||
total_record(
|
||||
"request_invalid_kind",
|
||||
Some(&invalid_kind),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record(
|
||||
"request_unregistered_kind",
|
||||
Some("anthropic"),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record("request_enriched", Some("openai"), "calculated", now),
|
||||
]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
@@ -354,6 +364,30 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_usage_records(vec![total_record(
|
||||
"request_provider_reported",
|
||||
Some("openai"),
|
||||
"provider_reported",
|
||||
now,
|
||||
)]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("usage records");
|
||||
|
||||
assert!(matches!(
|
||||
page.items[0].billing,
|
||||
Some(UsageBilling::Calculated(_))
|
||||
));
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
@@ -567,9 +601,10 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
}
|
||||
}
|
||||
|
||||
fn calculated_total_record(
|
||||
fn total_record(
|
||||
id: &str,
|
||||
provider_kind: Option<&str>,
|
||||
source: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageRecord {
|
||||
UsageRecord {
|
||||
@@ -613,11 +648,11 @@ fn calculated_total_record(
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
cost_source: "calculated".to_owned(),
|
||||
cost_source: source.to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Total {
|
||||
source: "calculated".to_owned(),
|
||||
source: source.to_owned(),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
|
||||
@@ -411,8 +411,10 @@ async fn openai_reauthorization_should_commit_after_credential_revision_advances
|
||||
"store.commit_authorization",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
assert_eq!(store.audit_requests(), ["oauth-complete-openai"]);
|
||||
}
|
||||
|
||||
|
||||
@@ -531,12 +531,18 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
message,
|
||||
provider_error_code,
|
||||
provider_error_type,
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
"providerErrorCode": provider_error_code,
|
||||
"providerErrorType": provider_error_type,
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -337,6 +337,9 @@ mod actions {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
provider_error_type: Some("invalid_request_error".to_owned()),
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
},
|
||||
]
|
||||
@@ -368,6 +371,9 @@ mod actions {
|
||||
"error": "upstream unavailable",
|
||||
"providerErrorCode": "usage_exhausted",
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "active"
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -58,9 +58,8 @@ use gateway_api::admin::AdminSessionState;
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::{
|
||||
ConfigRevision, ProviderKind, PublicModelId, UpstreamModelId, snapshot::SnapshotControl,
|
||||
@@ -675,7 +674,7 @@ impl AccountProbe for NoopProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async { panic!("unused account probe") })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,46 @@ impl fmt::Display for ProviderAccountId {
|
||||
}
|
||||
}
|
||||
|
||||
/// credential 轮换时可选替换的上游账号身份。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountIdentity {
|
||||
upstream_user_id: String,
|
||||
upstream_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderAccountIdentity {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderAccountIdentity")
|
||||
.field("upstream_user_id", &"<redacted>")
|
||||
.field(
|
||||
"upstream_account_id",
|
||||
&self.upstream_account_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAccountIdentity {
|
||||
#[must_use]
|
||||
pub const fn new(upstream_user_id: String, upstream_account_id: Option<String>) -> Self {
|
||||
Self {
|
||||
upstream_user_id,
|
||||
upstream_account_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_account_id(&self) -> Option<&str> {
|
||||
self.upstream_account_id.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// `provider_accounts.credential_revision` 的正数 CAS revision。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct CredentialRevision(NonZeroU64);
|
||||
|
||||
@@ -17,7 +17,10 @@ use crate::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
|
||||
};
|
||||
use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeResult};
|
||||
use crate::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
AccountProbeUpstreamResponse,
|
||||
};
|
||||
use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CoordinatedEvent, EngineError,
|
||||
@@ -426,7 +429,7 @@ impl DefaultExecutionService {
|
||||
async fn probe_inner(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> Result<AccountProbeResult, GatewayError> {
|
||||
) -> Result<AccountProbeResult, AccountProbeError> {
|
||||
let AccountProbeRequest {
|
||||
account_id,
|
||||
provider_kind,
|
||||
@@ -535,14 +538,19 @@ impl DefaultExecutionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先记录完整分类事实,再仅把 Provider 显式标记为客户端可见的
|
||||
/// 结构化 message/code/type 带入协议错误。
|
||||
/// 探测失败先记录脱敏分类事实,再把请求局部的原始上游响应交给认证管理端。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> GatewayError {
|
||||
) -> AccountProbeError {
|
||||
let upstream_response = match error {
|
||||
EngineError::Provider(provider_error) => provider_error
|
||||
.client_visible_upstream_response()
|
||||
.map(AccountProbeUpstreamResponse::from_client_response),
|
||||
_ => None,
|
||||
};
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let latency = started_at.elapsed().unwrap_or_default();
|
||||
let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
@@ -578,7 +586,7 @@ impl DefaultExecutionService {
|
||||
);
|
||||
}
|
||||
}
|
||||
gateway_error_from_engine(error)
|
||||
AccountProbeError::new(gateway_error_from_engine(error), upstream_response)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -685,7 +693,7 @@ impl AccountProbe for DefaultExecutionService {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move { self.probe_inner(request).await })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 管理端账号连通性测试的真实执行链端口。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::error::GatewayError;
|
||||
use crate::error::{ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind};
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
use crate::{engine::credential::ProviderAccountId, operation::Operation};
|
||||
|
||||
@@ -19,9 +22,111 @@ pub struct AccountProbeResult {
|
||||
pub text: Vec<String>,
|
||||
}
|
||||
|
||||
/// 仅供当前管理端连接测试展示的原始上游失败响应。
|
||||
///
|
||||
/// 正文不进入 `Debug`、日志或持久化;Core 只在探测终态从原始 Provider 错误显式复制。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct AccountProbeUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl AccountProbeUpstreamResponse {
|
||||
pub(crate) fn from_client_response(response: &ClientVisibleUpstreamResponse) -> Self {
|
||||
Self {
|
||||
status: response.status(),
|
||||
content_type: response.content_type().map(<[u8]>::to_vec),
|
||||
body: response.body().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AccountProbeUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountProbeUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端连接测试的终态错误。
|
||||
///
|
||||
/// `gateway` 保留稳定分类,`upstream_response` 只面向本次认证管理请求展示源响应。
|
||||
#[derive(Debug)]
|
||||
pub struct AccountProbeError {
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
}
|
||||
|
||||
impl AccountProbeError {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
) -> Self {
|
||||
Self {
|
||||
gateway,
|
||||
upstream_response,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> GatewayErrorKind {
|
||||
self.gateway.kind()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_message(&self) -> &str {
|
||||
self.gateway.client_message()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_type(&self) -> Option<&str> {
|
||||
self.gateway.client_error_type()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_code(&self) -> Option<&str> {
|
||||
self.gateway.client_error_code()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_response(&self) -> Option<&AccountProbeUpstreamResponse> {
|
||||
self.upstream_response.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayError> for AccountProbeError {
|
||||
fn from(gateway: GatewayError) -> Self {
|
||||
Self::new(gateway, None)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AccountProbe: Send + Sync {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>>;
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>>;
|
||||
}
|
||||
|
||||
@@ -256,9 +256,9 @@ impl fmt::Debug for ClientVisibleUpstreamError {
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志、探针或持久化。它刻意不实现
|
||||
/// [`Clone`];[`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象
|
||||
/// 才能把响应交给协议 adapter。
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
/// [`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象才能把响应交给
|
||||
/// 原客户端协议,或由认证管理端的账号连接测试显式复制到请求局部结果。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamResponse {
|
||||
status: u16,
|
||||
|
||||
@@ -667,32 +667,21 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按账号稳定扰动刷新提前量;相同账号与过期时间跨进程得到同一执行时刻。
|
||||
/// 按配置的精确提前量计算刷新时刻。
|
||||
pub fn next_attempt_at(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
_account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
let base_margin_seconds = self.margin.as_secs();
|
||||
if base_margin_seconds == 0 {
|
||||
if self.margin >= remaining {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
|
||||
let factor = stable_factor(account_id.as_str(), "normal", 850, 1_150);
|
||||
let jittered_seconds = base_margin_seconds
|
||||
.saturating_mul(u64::from(factor))
|
||||
.saturating_add(500)
|
||||
/ 1_000;
|
||||
if jittered_seconds >= remaining.as_secs() {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
let lead = jittered_seconds.max(1);
|
||||
access_token_expires_at
|
||||
.checked_sub(Duration::from_secs(lead))
|
||||
.checked_sub(self.margin)
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,8 @@ use gateway_core::engine::credential::{
|
||||
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountUpdate, RotationStrategy,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
|
||||
RotationStrategy,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
@@ -73,6 +74,18 @@ fn opaque_provider_data_should_not_expose_quota_values_in_debug() {
|
||||
assert!(!format!("{quota:?}").contains("private-window"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_account_identity_debug_should_redact_upstream_ids() {
|
||||
let identity = ProviderAccountIdentity::new(
|
||||
"private-user-id".to_owned(),
|
||||
Some("private-account-id".to_owned()),
|
||||
);
|
||||
let debug = format!("{identity:?}");
|
||||
|
||||
assert!(!debug.contains("private-user-id"));
|
||||
assert!(!debug.contains("private-account-id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn availability_should_round_trip_all_database_values() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{executor::block_on, future::BoxFuture};
|
||||
use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
@@ -34,7 +35,8 @@ use gateway_core::engine::{
|
||||
ModelRequestId, NewModelRequest, ProbeFailure, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError, StoreErrorKind,
|
||||
ClientVisibleUpstreamResponse, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError,
|
||||
StoreErrorKind,
|
||||
};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
@@ -108,6 +110,19 @@ fn probe_failures_should_be_observable_without_a_model_request_row() {
|
||||
.expect_err("the provider rejects every probe");
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
let upstream = error
|
||||
.upstream_response()
|
||||
.expect("probe must preserve its request-local upstream response");
|
||||
assert_eq!(upstream.status(), 502);
|
||||
assert_eq!(
|
||||
upstream.content_type(),
|
||||
Some(b"application/json".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
upstream.body(),
|
||||
&Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
);
|
||||
assert!(!format!("{error:?}").contains("source upstream failure"));
|
||||
assert!(!store.touched.load(Ordering::SeqCst));
|
||||
assert_eq!(store.probe_failures(), vec!["transport".to_owned()]);
|
||||
}
|
||||
@@ -166,10 +181,14 @@ impl Provider for FailingProvider {
|
||||
_request: ProviderRequest,
|
||||
_context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
502,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"message":"source upstream failure"}}"#),
|
||||
)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let account = ProviderAccountId::new("acct_stable").expect("valid account");
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
@@ -70,13 +70,7 @@ fn refresh_schedule_is_stable_and_never_uses_query_time_margin() {
|
||||
let first = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh");
|
||||
let second = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh again");
|
||||
|
||||
assert_eq!(first, second);
|
||||
assert!(first > observed_at);
|
||||
assert!(first < expires_at);
|
||||
assert_eq!(first, expires_at - Duration::from_secs(3_600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -151,7 +151,7 @@ pub fn retry_after_seconds_from_value(value: &Value) -> Option<u64> {
|
||||
.or_else(|| {
|
||||
error
|
||||
.get("resets_in_seconds")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(positive_u64)
|
||||
.filter(|seconds| *seconds > 0)
|
||||
})
|
||||
.or_else(|| retry_after_seconds_from_resets_at(error))
|
||||
@@ -392,6 +392,12 @@ pub fn rate_limits_to_header_pairs(rate_limits: &ParsedRateLimits) -> Vec<(Strin
|
||||
&format!("{prefix}-secondary"),
|
||||
details.secondary,
|
||||
);
|
||||
if let Some(allowed) = details.allowed {
|
||||
headers.push((format!("{prefix}-allowed"), allowed.to_string()));
|
||||
}
|
||||
if let Some(limit_reached) = details.limit_reached {
|
||||
headers.push((format!("{prefix}-limit-reached"), limit_reached.to_string()));
|
||||
}
|
||||
if let Some(limit_name) = &details.limit_name {
|
||||
headers.push((format!("{prefix}-limit-name"), limit_name.clone()));
|
||||
}
|
||||
@@ -437,7 +443,7 @@ fn nested_number_field(value: &Value, path: &[&str]) -> Option<u64> {
|
||||
}
|
||||
|
||||
fn retry_after_seconds_from_resets_at(error: &Value) -> Option<u64> {
|
||||
let resets_at = error.get("resets_at").and_then(Value::as_u64)?;
|
||||
let resets_at = error.get("resets_at").and_then(positive_u64)?;
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
@@ -445,6 +451,13 @@ fn retry_after_seconds_from_resets_at(error: &Value) -> Option<u64> {
|
||||
(resets_at > now).then_some(resets_at - now)
|
||||
}
|
||||
|
||||
fn positive_u64(value: &Value) -> Option<u64> {
|
||||
value
|
||||
.as_u64()
|
||||
.or_else(|| value.as_str().and_then(|value| value.parse::<u64>().ok()))
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
fn retry_after_seconds_from_rate_limit_message(error: &Value) -> Option<u64> {
|
||||
let code = error
|
||||
.get("code")
|
||||
|
||||
@@ -226,6 +226,32 @@ fn retry_after_should_read_structured_response_delay() {
|
||||
assert_eq!(retry_after_seconds_from_body(&body), Some(45));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_string_reset_delay() {
|
||||
let body = json!({
|
||||
"error": {"type": "usage_limit_reached", "resets_in_seconds": "46"}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(retry_after_seconds_from_body(&body), Some(46));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_string_reset_timestamp() {
|
||||
let reset_at = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock")
|
||||
.as_secs()
|
||||
+ 60;
|
||||
let body = json!({
|
||||
"error": {"type": "usage_limit_reached", "resets_at": reset_at.to_string()}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let retry_after = retry_after_seconds_from_body(&body).expect("retry-after");
|
||||
assert!((59..=60).contains(&retry_after));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_after_should_read_nested_error_delay() {
|
||||
let body = json!({
|
||||
@@ -414,6 +440,8 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
"metered_limit_name": "codex-other",
|
||||
"limit_name": "Other requests",
|
||||
"rate_limits": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary": {
|
||||
"used_percent": 33,
|
||||
"window_minutes": 60,
|
||||
@@ -424,7 +452,8 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
"has_credits": true,
|
||||
"unlimited": false,
|
||||
"balance": "9.25"
|
||||
}
|
||||
},
|
||||
"rate_limit_reached_type": "usage_limit_reached"
|
||||
});
|
||||
|
||||
let parsed = parse_rate_limits_event(&event).expect("rate-limit event");
|
||||
@@ -434,16 +463,22 @@ fn parse_rate_limits_event_should_extract_websocket_metered_limit() {
|
||||
(
|
||||
parsed.active_limit.as_deref(),
|
||||
details.limit_name.as_deref(),
|
||||
details.allowed,
|
||||
details.limit_reached,
|
||||
details.primary.expect("primary window").window_minutes,
|
||||
parsed.plan_type.as_deref(),
|
||||
parsed.credits.expect("credits").balance,
|
||||
parsed.rate_limit_reached_type.as_deref(),
|
||||
),
|
||||
(
|
||||
Some("codex_other"),
|
||||
Some("Other requests"),
|
||||
Some(false),
|
||||
Some(true),
|
||||
Some(60),
|
||||
Some("team"),
|
||||
Some("9.25".to_owned()),
|
||||
Some("usage_limit_reached"),
|
||||
)
|
||||
);
|
||||
}
|
||||
@@ -472,9 +507,15 @@ fn rate_limit_header_round_trip_should_preserve_wire_facts() {
|
||||
"300".to_owned(),
|
||||
),
|
||||
("x-codex-active-limit".to_owned(), "codex".to_owned()),
|
||||
("x-codex-allowed".to_owned(), "false".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-has-credits".to_owned(), "true".to_owned()),
|
||||
("x-codex-credits-unlimited".to_owned(), "false".to_owned()),
|
||||
("x-codex-plan-type".to_owned(), "plus".to_owned()),
|
||||
(
|
||||
"x-codex-rate-limit-reached-type".to_owned(),
|
||||
"usage_limit_reached".to_owned(),
|
||||
),
|
||||
];
|
||||
let parsed = parse_rate_limit_headers(&source).expect("initial parse");
|
||||
let encoded = rate_limits_to_header_pairs(&parsed);
|
||||
|
||||
@@ -332,6 +332,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
REDIS_NAMESPACE,
|
||||
redis::ProviderCircuitPolicy::default(),
|
||||
)?);
|
||||
// Continuation affinity 是下一轮请求的路由事实,Core 必须直接等待 Redis 确认。
|
||||
let continuation: Arc<dyn gateway_core::engine::continuation::NativeContinuationPort> =
|
||||
Arc::new(redis::RedisNativeContinuationRepository::new(
|
||||
redis_connection.clone(),
|
||||
@@ -340,8 +341,6 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
let (admissions, admission_release_writer) =
|
||||
redis::BufferedClientAdmissionPort::new(admissions);
|
||||
let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits);
|
||||
let (continuation, continuation_writer) =
|
||||
redis::BufferedNativeContinuationPort::new(continuation);
|
||||
let core_ports = CoreStorePorts::new(
|
||||
execution,
|
||||
(
|
||||
@@ -351,7 +350,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
)),
|
||||
),
|
||||
Arc::new(circuits),
|
||||
Arc::new(continuation),
|
||||
continuation,
|
||||
(
|
||||
Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())),
|
||||
Arc::new(redis::RedisRuntimeChangeRepository::new(
|
||||
@@ -390,7 +389,6 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
execution_writer,
|
||||
admission_release_writer,
|
||||
circuit_feedback_writer,
|
||||
continuation_writer,
|
||||
retention,
|
||||
)?;
|
||||
Ok(StoreBundle {
|
||||
@@ -408,7 +406,6 @@ fn store_worker_contributions(
|
||||
execution_writer: postgres::ExecutionObservationWriter<postgres::PgExecutionStore>,
|
||||
admission_release_writer: redis::ClientAdmissionReleaseWriter,
|
||||
circuit_feedback_writer: redis::ProviderCircuitFeedbackWriter,
|
||||
continuation_writer: redis::NativeContinuationWriter,
|
||||
retention: Arc<postgres::PgRetentionRepository>,
|
||||
) -> StoreResult<Vec<WorkerContribution>> {
|
||||
let stale_id = WorkerId::try_new(WorkerKind::StaleModelRequestRecovery, "postgres")
|
||||
@@ -421,8 +418,6 @@ fn store_worker_contributions(
|
||||
.map_err(worker_definition_error)?;
|
||||
let circuit_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_circuit")
|
||||
.map_err(worker_definition_error)?;
|
||||
let continuation_flush_id = WorkerId::try_new(WorkerKind::OpsFlush, "redis_continuation")
|
||||
.map_err(worker_definition_error)?;
|
||||
let ops_flush_restart =
|
||||
DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(worker_definition_error)?;
|
||||
@@ -467,16 +462,6 @@ fn store_worker_contributions(
|
||||
)
|
||||
.map_err(worker_definition_error)?,
|
||||
),
|
||||
WorkerContribution::Registration(
|
||||
WorkerRegistration::try_new(
|
||||
continuation_flush_id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart: ops_flush_restart,
|
||||
task: Box::new(continuation_writer),
|
||||
},
|
||||
)
|
||||
.map_err(worker_definition_error)?,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
@@ -1992,27 +1992,35 @@ async fn provider_account_metrics(
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"select count(*)::bigint as total,
|
||||
"with normalized as (
|
||||
select enabled,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (
|
||||
where not (
|
||||
enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
)
|
||||
)::bigint as unavailable,
|
||||
count(*) filter (
|
||||
where enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
)::bigint as active,
|
||||
count(*) filter (
|
||||
where availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
)::bigint as expired,
|
||||
count(*) filter (where availability = 'quota_exhausted')::bigint
|
||||
as quota_exhausted,
|
||||
count(*) filter (where not enabled)::bigint as disabled,
|
||||
count(*) filter (where availability = 'banned')::bigint as banned
|
||||
from provider_accounts",
|
||||
count(*) filter (where status in ('expired', 'disabled', 'banned'))::bigint
|
||||
as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
from normalized",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_one(pool)
|
||||
@@ -2035,11 +2043,29 @@ async fn active_provider_account_ids(
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<String>> {
|
||||
sqlx::query(
|
||||
"select id
|
||||
from provider_accounts
|
||||
where enabled and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
"with normalized as (
|
||||
select id,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted'
|
||||
or (
|
||||
availability = 'cooldown'
|
||||
and availability_reason = 'usage_limit_exhausted'
|
||||
and (cooldown_until is null or cooldown_until > $1)
|
||||
)
|
||||
then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select id
|
||||
from normalized
|
||||
where status = 'active'
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
|
||||
@@ -38,8 +38,8 @@ use gateway_core::engine::credential::{
|
||||
CredentialRevision as CoreCredentialRevision, LoadedCredential,
|
||||
NewProviderAccount as CoreNewProviderAccount, OpaqueProviderData, PlaintextCredential,
|
||||
ProviderAccount as CoreProviderAccount, ProviderAccountId as CoreProviderAccountId,
|
||||
ProviderAccountStore, ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
ProviderAccountIdentity, ProviderAccountStore,
|
||||
ProviderAccountUpdate as CoreProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
@@ -125,6 +125,26 @@ fn parse_availability(value: &str) -> StoreResult<AccountAvailability> {
|
||||
AccountAvailability::parse(value).ok_or_else(|| invalid("unknown availability value"))
|
||||
}
|
||||
|
||||
fn validate_availability_cooldown(
|
||||
availability: AccountAvailability,
|
||||
has_cooldown_until: bool,
|
||||
) -> StoreResult<()> {
|
||||
if availability == AccountAvailability::Cooldown && !has_cooldown_until {
|
||||
return Err(invalid("cooldown availability requires cooldown_until"));
|
||||
}
|
||||
if has_cooldown_until
|
||||
&& !matches!(
|
||||
availability,
|
||||
AccountAvailability::Cooldown | AccountAvailability::QuotaExhausted
|
||||
)
|
||||
{
|
||||
return Err(invalid(
|
||||
"cooldown_until is only allowed for cooldown or quota_exhausted availability",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountSummary {
|
||||
pub id: String,
|
||||
@@ -224,11 +244,7 @@ impl NewProviderAccount {
|
||||
if !self.has_refresh_token && self.next_refresh_at.is_some() {
|
||||
return Err(invalid("next_refresh_at requires a refresh token"));
|
||||
}
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown_until must be present exactly for cooldown availability",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
validate_object_size(
|
||||
"provider_credentials_json",
|
||||
&self.provider_credentials_json,
|
||||
@@ -296,6 +312,7 @@ impl ImportProviderAccounts {
|
||||
pub struct RotateProviderAccount {
|
||||
pub scope: ProviderAccountAdminScope,
|
||||
pub profile: UpdateProviderAccount,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub credential: ProviderCredentialUpdate,
|
||||
pub audit: AdminAuditEvent,
|
||||
}
|
||||
@@ -306,6 +323,7 @@ impl fmt::Debug for RotateProviderAccount {
|
||||
.debug_struct("RotateProviderAccount")
|
||||
.field("scope", &self.scope)
|
||||
.field("profile", &self.profile)
|
||||
.field("replacement_identity", &self.replacement_identity)
|
||||
.field("credential", &self.credential)
|
||||
.field("audit", &self.audit)
|
||||
.finish()
|
||||
@@ -378,11 +396,7 @@ pub struct ProviderAccountStateUpdate {
|
||||
impl ProviderAccountStateUpdate {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown availability and cooldown_until must agree",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -390,11 +404,7 @@ impl ProviderAccountStateUpdate {
|
||||
impl ProviderAccountObservation {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
require_nonempty(ENTITY, "account_id", &self.account_id)?;
|
||||
if (self.availability == AccountAvailability::Cooldown) != self.cooldown_until.is_some() {
|
||||
return Err(invalid(
|
||||
"cooldown availability and cooldown_until must agree",
|
||||
));
|
||||
}
|
||||
validate_availability_cooldown(self.availability, self.cooldown_until.is_some())?;
|
||||
if self.provider_quota_json.is_some() != self.quota_observed_at.is_some() {
|
||||
return Err(invalid("quota JSON and quota_observed_at must agree"));
|
||||
}
|
||||
@@ -612,8 +622,14 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
|
||||
update.validate()?;
|
||||
let result = sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = $3, availability_reason = $4, cooldown_until = $5,
|
||||
availability_observed_at = $6, updated_at = greatest(now(), $6)
|
||||
set availability = case when enabled then $3 else availability end,
|
||||
availability_reason = case when enabled then $4 else availability_reason end,
|
||||
cooldown_until = case when enabled then $5 else cooldown_until end,
|
||||
availability_observed_at = case
|
||||
when enabled then $6
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = case when enabled then greatest(now(), $6) else updated_at end
|
||||
where id = $1 and credential_revision = $2",
|
||||
)
|
||||
.bind(update.account_id)
|
||||
@@ -755,6 +771,12 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
command.scope.validate()?;
|
||||
require_nonempty(ENTITY, "account_id", &command.profile.id)?;
|
||||
require_nonempty(ENTITY, "name", &command.profile.name)?;
|
||||
if let Some(identity) = &command.replacement_identity {
|
||||
require_nonempty(ENTITY, "upstream_user_id", identity.upstream_user_id())?;
|
||||
if let Some(account_id) = identity.upstream_account_id() {
|
||||
require_nonempty(ENTITY, "upstream_account_id", account_id)?;
|
||||
}
|
||||
}
|
||||
if command.profile.id != command.credential.account_id {
|
||||
return Err(invalid("rotated profile and credential account IDs differ"));
|
||||
}
|
||||
@@ -770,6 +792,7 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
|
||||
&mut transaction,
|
||||
&command.scope,
|
||||
&command.profile,
|
||||
command.replacement_identity.as_ref(),
|
||||
&command.credential,
|
||||
)
|
||||
.await?;
|
||||
@@ -1040,6 +1063,7 @@ impl PgAdminAccountStore {
|
||||
email: prepared.email,
|
||||
plan_type: prepared.plan_type,
|
||||
},
|
||||
replacement_identity: prepared.replacement_identity,
|
||||
credential: ProviderCredentialUpdate {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
expected_revision: store_revision(prepared.expected_credential_revision)?,
|
||||
@@ -1564,7 +1588,11 @@ fn admin_account_status(
|
||||
AdminAccountStatus::Disabled
|
||||
} else if account.availability == AccountAvailability::Banned {
|
||||
AdminAccountStatus::Banned
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted {
|
||||
} else if account.availability == AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AdminAccountStatus::QuotaExhausted
|
||||
} else if account.availability == AccountAvailability::Expired
|
||||
|| account.availability == AccountAvailability::Invalid
|
||||
@@ -1971,8 +1999,13 @@ async fn rotate_provider_account_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
scope: &ProviderAccountAdminScope,
|
||||
profile: &UpdateProviderAccount,
|
||||
replacement_identity: Option<&ProviderAccountIdentity>,
|
||||
update: &ProviderCredentialUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
let replace_identity = replacement_identity.is_some();
|
||||
let upstream_user_id = replacement_identity.map(ProviderAccountIdentity::upstream_user_id);
|
||||
let upstream_account_id =
|
||||
replacement_identity.and_then(ProviderAccountIdentity::upstream_account_id);
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update provider_accounts
|
||||
set name = $4,
|
||||
@@ -1983,6 +2016,28 @@ async fn rotate_provider_account_in_transaction(
|
||||
has_refresh_token = $8,
|
||||
access_token_expires_at = $9,
|
||||
next_refresh_at = $10,
|
||||
upstream_user_id = case when $11::boolean then $12::text else upstream_user_id end,
|
||||
upstream_account_id = case when $11::boolean then $13::text else upstream_account_id end,
|
||||
availability = case
|
||||
when not enabled then availability
|
||||
when availability <> 'quota_exhausted' then 'ready'
|
||||
else availability
|
||||
end,
|
||||
availability_reason = case
|
||||
when not enabled then availability_reason
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else availability_reason
|
||||
end,
|
||||
cooldown_until = case
|
||||
when not enabled then cooldown_until
|
||||
when availability <> 'quota_exhausted' then null
|
||||
else cooldown_until
|
||||
end,
|
||||
availability_observed_at = case
|
||||
when not enabled then availability_observed_at
|
||||
when availability <> 'quota_exhausted' then now()
|
||||
else availability_observed_at
|
||||
end,
|
||||
updated_at = now()
|
||||
where id = $1 and provider_kind = $2
|
||||
and credential_revision = $3
|
||||
@@ -1998,9 +2053,25 @@ async fn rotate_provider_account_in_transaction(
|
||||
.bind(update.has_refresh_token)
|
||||
.bind(update.access_token_expires_at)
|
||||
.bind(update.next_refresh_at)
|
||||
.bind(replace_identity)
|
||||
.bind(upstream_user_id)
|
||||
.bind(upstream_account_id)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("rotate provider account in admin transaction"))?
|
||||
.map_err(|error| {
|
||||
if error
|
||||
.as_database_error()
|
||||
.is_some_and(sqlx::error::DatabaseError::is_unique_violation)
|
||||
{
|
||||
StoreError::Conflict {
|
||||
entity: ENTITY,
|
||||
id: update.account_id.clone(),
|
||||
kind: ConflictKind::InvalidTransition,
|
||||
}
|
||||
} else {
|
||||
postgres_unavailable("rotate provider account in admin transaction")
|
||||
}
|
||||
})?
|
||||
.ok_or_else(|| StoreError::Conflict {
|
||||
entity: ENTITY,
|
||||
id: update.account_id.clone(),
|
||||
|
||||
@@ -225,6 +225,21 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
update: &RuntimeSettingsUpdate,
|
||||
) -> StoreResult<Revision> {
|
||||
update.validate()?;
|
||||
let refresh_margin_seconds =
|
||||
i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?;
|
||||
let previous_refresh_margin_seconds = sqlx::query_scalar::<_, i64>(
|
||||
"select refresh_margin_seconds
|
||||
from runtime_settings
|
||||
where id = 1
|
||||
for update",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("lock runtime settings for update"))?
|
||||
.ok_or_else(|| StoreError::NotFound {
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
let next = sqlx::query_scalar::<_, i64>(
|
||||
"update runtime_settings
|
||||
set config_revision = config_revision + 1,
|
||||
@@ -243,7 +258,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
returning config_revision",
|
||||
)
|
||||
.bind(update.admin_api_key.as_deref())
|
||||
.bind(i64::try_from(update.refresh_margin_seconds).map_err(|_| invalid_numeric())?)
|
||||
.bind(refresh_margin_seconds)
|
||||
.bind(i64::from(update.refresh_concurrency))
|
||||
.bind(i64::from(update.max_concurrent_per_account))
|
||||
.bind(i64::try_from(update.request_interval_ms).map_err(|_| invalid_numeric())?)
|
||||
@@ -259,9 +274,35 @@ pub(crate) async fn update_runtime_settings_in_transaction(
|
||||
entity: "runtime settings",
|
||||
id: "1".to_owned(),
|
||||
})?;
|
||||
if previous_refresh_margin_seconds != refresh_margin_seconds {
|
||||
reschedule_provider_refreshes_in_transaction(transaction, refresh_margin_seconds).await?;
|
||||
}
|
||||
Revision::new(u64::try_from(next).map_err(|_| invalid_numeric())?)
|
||||
}
|
||||
|
||||
async fn reschedule_provider_refreshes_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
refresh_margin_seconds: i64,
|
||||
) -> StoreResult<()> {
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set next_refresh_at = case
|
||||
when access_token_expires_at <= now()
|
||||
or extract(epoch from (access_token_expires_at - now())) <= $1
|
||||
then now()
|
||||
else access_token_expires_at - ($1::double precision * interval '1 second')
|
||||
end,
|
||||
updated_at = now()
|
||||
where has_refresh_token
|
||||
and access_token_expires_at is not null",
|
||||
)
|
||||
.bind(refresh_margin_seconds)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("reschedule provider credential refreshes"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn bump_config_revision_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> StoreResult<Revision> {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
//! 数据面 Redis 写副作用的有界非阻塞队列。
|
||||
//! 可由 TTL 或后续反馈收敛的 Redis 写副作用队列。
|
||||
//!
|
||||
//! Continuation affinity 决定下一轮 Provider 与账号,必须在响应 ID 可复用前
|
||||
//! 获得 Redis 确认,因此不属于本模块的可丢失副作用。
|
||||
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
@@ -8,9 +11,6 @@ use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
ClientAdmissionRequest, ClientAdmissionRestoreResult,
|
||||
};
|
||||
use gateway_core::engine::continuation::{
|
||||
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
};
|
||||
@@ -252,94 +252,3 @@ impl DaemonTask for ProviderCircuitFeedbackWriter {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Continuation resolve 仍直读 Redis;新 pin 只做有界入队。
|
||||
#[derive(Clone)]
|
||||
pub struct BufferedNativeContinuationPort {
|
||||
inner: Arc<dyn NativeContinuationPort>,
|
||||
sender: mpsc::Sender<NativeContinuationPin>,
|
||||
}
|
||||
|
||||
impl BufferedNativeContinuationPort {
|
||||
#[must_use]
|
||||
pub fn new(inner: Arc<dyn NativeContinuationPort>) -> (Self, NativeContinuationWriter) {
|
||||
Self::with_capacity(
|
||||
inner,
|
||||
NonZeroUsize::new(DEFAULT_QUEUE_CAPACITY).expect("queue capacity is non-zero"),
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_capacity(
|
||||
inner: Arc<dyn NativeContinuationPort>,
|
||||
capacity: NonZeroUsize,
|
||||
) -> (Self, NativeContinuationWriter) {
|
||||
let (sender, receiver) = mpsc::channel(capacity.get());
|
||||
(
|
||||
Self {
|
||||
inner: Arc::clone(&inner),
|
||||
sender,
|
||||
},
|
||||
NativeContinuationWriter {
|
||||
inner,
|
||||
receiver: Mutex::new(receiver),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn enqueue(&self, pin: NativeContinuationPin) {
|
||||
if let Err(error) = self.sender.try_send(pin) {
|
||||
let reason = match error {
|
||||
mpsc::error::TrySendError::Full(_) => "full",
|
||||
mpsc::error::TrySendError::Closed(_) => "closed",
|
||||
};
|
||||
tracing::warn!(
|
||||
operation = "record_native_continuation",
|
||||
reason,
|
||||
"Redis 协调队列不可用,已丢弃本次 continuation affinity"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeContinuationPort for BufferedNativeContinuationPort {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
previous_response_id: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
self.inner.resolve(previous_response_id)
|
||||
}
|
||||
|
||||
fn record<'a>(
|
||||
&'a self,
|
||||
pin: NativeContinuationPin,
|
||||
) -> BoxFuture<'a, Result<(), NativeContinuationStoreError>> {
|
||||
self.enqueue(pin);
|
||||
Box::pin(ready(Ok(())))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NativeContinuationWriter {
|
||||
inner: Arc<dyn NativeContinuationPort>,
|
||||
receiver: Mutex<mpsc::Receiver<NativeContinuationPin>>,
|
||||
}
|
||||
|
||||
impl DaemonTask for NativeContinuationWriter {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let mut receiver = self.receiver.lock().await;
|
||||
loop {
|
||||
let pin = tokio::select! {
|
||||
() = cancellation.cancelled() => return Ok(()),
|
||||
pin = receiver.recv() => pin,
|
||||
};
|
||||
let Some(pin) = pin else {
|
||||
return Err(WorkerTaskError::safe("native continuation queue closed"));
|
||||
};
|
||||
if let Err(error) = self.inner.record(pin).await {
|
||||
tracing::warn!(%error, "Continuation affinity 后台写入失败");
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -504,6 +504,12 @@ impl ProviderAccountStore for CooldownCachingProviderAccountStore {
|
||||
let availability = change.availability;
|
||||
let cooldown_until = change.cooldown_until;
|
||||
self.authoritative.apply_state_change(change).await?;
|
||||
let Some(account) = self.authoritative.get_account(&account_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.mirror(
|
||||
&account_id,
|
||||
credential_revision,
|
||||
|
||||
@@ -24,6 +24,7 @@ use super::{namespace, resource_fingerprint};
|
||||
const CONTINUATION_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
const MAX_CONTINUATIONS: usize = 65_536;
|
||||
const MAX_RECORD_BYTES: usize = 64 * 1024;
|
||||
const MAX_INDEX_CLEANUP_PER_RECORD: usize = 64;
|
||||
|
||||
const RECORD_SCRIPT: &str = r#"
|
||||
local entry_key = KEYS[1]
|
||||
@@ -35,12 +36,13 @@ local ttl_seconds = tonumber(ARGV[4])
|
||||
local cutoff = tonumber(ARGV[5])
|
||||
local max_entries = tonumber(ARGV[6])
|
||||
local entry_prefix = ARGV[7]
|
||||
local cleanup_limit = tonumber(ARGV[8])
|
||||
|
||||
redis.call('SET', entry_key, payload, 'EX', ttl_seconds)
|
||||
redis.call('ZADD', index_key, score, entry_id)
|
||||
redis.call('EXPIRE', index_key, ttl_seconds)
|
||||
|
||||
local stale = redis.call('ZRANGEBYSCORE', index_key, '-inf', cutoff)
|
||||
local stale = redis.call('ZRANGEBYSCORE', index_key, '-inf', cutoff, 'LIMIT', 0, cleanup_limit)
|
||||
for _, stale_id in ipairs(stale) do
|
||||
redis.call('UNLINK', entry_prefix .. '{' .. stale_id .. '}')
|
||||
redis.call('ZREM', index_key, stale_id)
|
||||
@@ -48,6 +50,9 @@ end
|
||||
|
||||
local count = redis.call('ZCARD', index_key)
|
||||
local excess = count - max_entries
|
||||
if excess > cleanup_limit then
|
||||
excess = cleanup_limit
|
||||
end
|
||||
if excess > 0 then
|
||||
local oldest = redis.call('ZRANGE', index_key, 0, excess - 1)
|
||||
for _, old_id in ipairs(oldest) do
|
||||
@@ -158,6 +163,7 @@ impl NativeContinuationPort for RedisNativeContinuationRepository {
|
||||
.arg(cutoff)
|
||||
.arg(MAX_CONTINUATIONS)
|
||||
.arg(self.entry_prefix())
|
||||
.arg(MAX_INDEX_CLEANUP_PER_RECORD)
|
||||
.invoke_async::<i64>(&mut connection)
|
||||
.await
|
||||
.map_err(|_| NativeContinuationStoreError)?;
|
||||
|
||||
@@ -7,19 +7,13 @@ use gateway_core::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionError, ClientAdmissionPort, ClientAdmissionRecovery,
|
||||
ClientAdmissionRequest, ClientAdmissionRestoreResult,
|
||||
};
|
||||
use gateway_core::engine::continuation::{
|
||||
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::engine::execution::{
|
||||
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
};
|
||||
use gateway_core::engine::{CancellationToken, ModelRequestId};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_store::redis::{
|
||||
BufferedClientAdmissionPort, BufferedNativeContinuationPort, BufferedProviderCircuitPort,
|
||||
};
|
||||
use gateway_store::redis::{BufferedClientAdmissionPort, BufferedProviderCircuitPort};
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingCoordination {
|
||||
@@ -98,27 +92,8 @@ impl ProviderCircuitPort for RecordingCoordination {
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeContinuationPort for RecordingCoordination {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
_: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
}
|
||||
|
||||
fn record<'a>(
|
||||
&'a self,
|
||||
_: NativeContinuationPin,
|
||||
) -> BoxFuture<'a, Result<(), NativeContinuationStoreError>> {
|
||||
Box::pin(async move {
|
||||
self.record("continuation");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_redis_coordination_queues_should_drop_writes_without_waiting() {
|
||||
async fn full_recoverable_coordination_queues_should_drop_writes_without_waiting() {
|
||||
let inner = Arc::new(RecordingCoordination::default());
|
||||
let (admissions, _admission_writer) = BufferedClientAdmissionPort::with_capacity(
|
||||
inner.clone(),
|
||||
@@ -128,10 +103,6 @@ async fn full_redis_coordination_queues_should_drop_writes_without_waiting() {
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(1).expect("capacity"),
|
||||
);
|
||||
let (continuation, _continuation_writer) = BufferedNativeContinuationPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(1).expect("capacity"),
|
||||
);
|
||||
let client = ClientApiKeyId::new("key_buffer_test").expect("client key");
|
||||
let request = ModelRequestId::new("req_buffer_test").expect("request ID");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
@@ -153,14 +124,6 @@ async fn full_redis_coordination_queues_should_drop_writes_without_waiting() {
|
||||
.observe_failure(&provider)
|
||||
.await
|
||||
.expect("full circuit queue remains fail-open");
|
||||
continuation
|
||||
.record(continuation_pin())
|
||||
.await
|
||||
.expect("first continuation enqueue");
|
||||
continuation
|
||||
.record(continuation_pin())
|
||||
.await
|
||||
.expect("full continuation queue remains fail-open");
|
||||
})
|
||||
.await
|
||||
.expect("coordination enqueue must never wait for Redis");
|
||||
@@ -179,10 +142,6 @@ async fn redis_coordination_writers_should_flush_each_side_effect() {
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let (continuation, continuation_writer) = BufferedNativeContinuationPort::with_capacity(
|
||||
inner.clone(),
|
||||
NonZeroUsize::new(8).expect("capacity"),
|
||||
);
|
||||
let client = ClientApiKeyId::new("key_writer_test").expect("client key");
|
||||
let request = ModelRequestId::new("req_writer_test").expect("request ID");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
@@ -194,20 +153,14 @@ async fn redis_coordination_writers_should_flush_each_side_effect() {
|
||||
.observe_success(&provider)
|
||||
.await
|
||||
.expect("enqueue circuit feedback");
|
||||
continuation
|
||||
.record(continuation_pin())
|
||||
.await
|
||||
.expect("enqueue continuation");
|
||||
|
||||
let cancellation = CancellationToken::new();
|
||||
let tasks = [
|
||||
spawn_writer(Arc::new(admission_writer), cancellation.clone()),
|
||||
spawn_writer(Arc::new(circuit_writer), cancellation.clone()),
|
||||
spawn_writer(Arc::new(continuation_writer), cancellation.clone()),
|
||||
];
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if inner.operations().len() == 3 {
|
||||
if inner.operations().len() == 2 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
@@ -224,7 +177,7 @@ async fn redis_coordination_writers_should_flush_each_side_effect() {
|
||||
|
||||
let mut operations = inner.operations();
|
||||
operations.sort_unstable();
|
||||
assert_eq!(operations, ["admission", "circuit_success", "continuation"]);
|
||||
assert_eq!(operations, ["admission", "circuit_success"]);
|
||||
}
|
||||
|
||||
fn spawn_writer<T>(
|
||||
@@ -236,12 +189,3 @@ where
|
||||
{
|
||||
tokio::spawn(async move { writer.run(cancellation).await })
|
||||
}
|
||||
|
||||
fn continuation_pin() -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new("resp_buffer_test"),
|
||||
PreviousResponseId::new("resp_buffer_test"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_buffer_test").expect("account"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -76,6 +76,12 @@ impl TestDatabase {
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply model request service tier migration");
|
||||
sqlx::raw_sql(include_str!(
|
||||
"../../../../migrations/0005_allow_quota_exhausted_cooldown.sql"
|
||||
))
|
||||
.execute(&mut *migration)
|
||||
.await
|
||||
.expect("apply quota exhausted cooldown migration");
|
||||
migration.commit().await.expect("commit terminal migration");
|
||||
Some(Self {
|
||||
admin,
|
||||
@@ -171,7 +177,7 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
.expect("drop migration test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!((first_table_count, migration_count), (8, 4));
|
||||
assert_eq!((first_table_count, migration_count), (8, 5));
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
}
|
||||
@@ -183,6 +189,7 @@ fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
include_str!("../../../../migrations/0002_drop_opaque_response_id_index.sql"),
|
||||
include_str!("../../../../migrations/0003_store_opaque_response_ids_as_bytes.sql"),
|
||||
include_str!("../../../../migrations/0004_add_model_request_service_tier.sql"),
|
||||
include_str!("../../../../migrations/0005_allow_quota_exhausted_cooldown.sql"),
|
||||
]
|
||||
.into_iter()
|
||||
.flat_map(str::lines)
|
||||
|
||||
@@ -122,16 +122,27 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, credential_revision, has_refresh_token,
|
||||
access_token_expires_at, enabled, availability,
|
||||
availability_reason, cooldown_until,
|
||||
availability_observed_at, created_at, updated_at
|
||||
) values
|
||||
('acct_available', 'openai', 'available', 'user-available', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_expired', 'openai', 'expired', 'user-expired', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 - interval '1 day', true, 'ready',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_rate_cooldown', 'openai', 'rate-cooldown', 'user-rate-cooldown', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'rate_limited', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_usage_limit', 'openai', 'usage-limit', 'user-usage-limit', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'cooldown',
|
||||
'usage_limit_exhausted', $1 + interval '2 hours', $1, $1, $1),
|
||||
('acct_banned', 'xai', 'banned', 'user-banned', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned', $1, $1, $1),
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', true, 'banned',
|
||||
null, null, $1, $1, $1),
|
||||
('acct_disabled', 'xai', 'disabled', 'user-disabled', 'oauth',
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready', $1, $1, $1)",
|
||||
'{}'::jsonb, 1, false, $1 + interval '1 day', false, 'ready',
|
||||
null, null, $1, $1, $1)",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&database.pool)
|
||||
@@ -154,10 +165,11 @@ async fn dashboard_account_metrics_should_partition_available_and_unavailable_ac
|
||||
metrics.active,
|
||||
metrics.unavailable,
|
||||
metrics.expired,
|
||||
metrics.quota_exhausted,
|
||||
metrics.disabled,
|
||||
metrics.banned,
|
||||
),
|
||||
(4, 3, 1, 3, 1, 1, 1),
|
||||
(6, 5, 2, 3, 1, 1, 1, 1),
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -17,9 +17,10 @@ use gateway_admin::{
|
||||
ports::store::AccountStore,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasOutcome, CredentialCasUpdate, CredentialRevision,
|
||||
OpaqueProviderData, PlaintextCredential, ProviderAccountId, ProviderAccountStore,
|
||||
ProviderAccountUpdate, QuotaObservation, QuotaWriteOutcome,
|
||||
AccountAvailability, AccountStateChange, CredentialCasOutcome, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, ProviderAccountId,
|
||||
ProviderAccountIdentity, ProviderAccountStore, ProviderAccountUpdate, QuotaObservation,
|
||||
QuotaWriteOutcome,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_store::{
|
||||
@@ -898,6 +899,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: wrong_scope,
|
||||
profile: profile("acct_admin_a", "wrong scope"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update("acct_admin_a", 1, "wrong-scope-secret"),
|
||||
audit: audit("audit_wrong_scope", "rotate", "acct_admin_a"),
|
||||
})
|
||||
@@ -912,10 +914,24 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
));
|
||||
assert_eq!(current_revision(&database.pool).await, 2);
|
||||
|
||||
sqlx::query(
|
||||
"update provider_accounts
|
||||
set availability = 'expired', availability_reason = 'credential_expired'
|
||||
where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("seed stale credential state");
|
||||
|
||||
let rotation = repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: scope.clone(),
|
||||
profile: profile("acct_admin_a", "rotated account"),
|
||||
replacement_identity: Some(ProviderAccountIdentity::new(
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)),
|
||||
credential: credential_update("acct_admin_a", 1, "rotated-secret"),
|
||||
audit: audit("audit_account_rotate", "rotate", "acct_admin_a"),
|
||||
})
|
||||
@@ -923,6 +939,71 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
.expect("rotate provider account");
|
||||
assert_eq!(rotation.config_revision.get(), 3);
|
||||
assert_eq!(rotation.credential_revision.get(), 2);
|
||||
let restored: (
|
||||
String,
|
||||
Option<String>,
|
||||
Option<chrono::DateTime<Utc>>,
|
||||
String,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
"select availability, availability_reason, cooldown_until,
|
||||
upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load restored account state");
|
||||
assert_eq!(
|
||||
restored,
|
||||
(
|
||||
"ready".to_owned(),
|
||||
None,
|
||||
None,
|
||||
"user-admin-rebound".to_owned(),
|
||||
Some("workspace-admin-rebound".to_owned()),
|
||||
)
|
||||
);
|
||||
|
||||
let identity_conflict = repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope: scope.clone(),
|
||||
profile: profile("acct_admin_a", "must roll back"),
|
||||
replacement_identity: Some(ProviderAccountIdentity::new(
|
||||
"user-admin-b".to_owned(),
|
||||
None,
|
||||
)),
|
||||
credential: credential_update("acct_admin_a", 2, "must-not-persist"),
|
||||
audit: audit(
|
||||
"audit_account_identity_conflict",
|
||||
"reauthorize",
|
||||
"acct_admin_a",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect_err("upstream identity collision must roll back rotation");
|
||||
assert!(matches!(
|
||||
identity_conflict,
|
||||
StoreError::Conflict {
|
||||
kind: ConflictKind::InvalidTransition,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(current_revision(&database.pool).await, 3);
|
||||
let unchanged: (String, serde_json::Value, i64, String, Option<String>) = sqlx::query_as(
|
||||
"select name, provider_credentials_json, credential_revision,
|
||||
upstream_user_id, upstream_account_id
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_admin_a")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load account after identity collision");
|
||||
assert_eq!(unchanged.0, "rotated account");
|
||||
assert_eq!(unchanged.1["access_token"], "rotated-secret");
|
||||
assert_eq!(unchanged.2, 2);
|
||||
assert_eq!(unchanged.3, "user-admin-rebound");
|
||||
assert_eq!(unchanged.4.as_deref(), Some("workspace-admin-rebound"));
|
||||
|
||||
let revision = repository
|
||||
.set_provider_account_enabled_admin(SetProviderAccountEnabled {
|
||||
@@ -965,6 +1046,135 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_credential_rotation_preserves_quota_exhaustion() {
|
||||
let Some(database) = TestDatabase::create("provider_account_rotation_quota").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
let mut exhausted = account("acct_rotation_quota", "user-rotation-quota");
|
||||
exhausted.availability = AccountAvailability::QuotaExhausted;
|
||||
exhausted.availability_reason = Some("quota_exhausted".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![exhausted],
|
||||
audit: audit(
|
||||
"audit_rotation_quota_import",
|
||||
"import",
|
||||
"acct_rotation_quota",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("import exhausted account");
|
||||
|
||||
repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope,
|
||||
profile: profile("acct_rotation_quota", "reauthorized account"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update("acct_rotation_quota", 1, "reauthorized-secret"),
|
||||
audit: audit(
|
||||
"audit_rotation_quota_rotate",
|
||||
"reauthorize",
|
||||
"acct_rotation_quota",
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("rotate exhausted account credential");
|
||||
|
||||
let availability: (String, Option<String>) = sqlx::query_as(
|
||||
"select availability, availability_reason from provider_accounts where id = $1",
|
||||
)
|
||||
.bind("acct_rotation_quota")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load exhausted account after credential rotation");
|
||||
assert_eq!(
|
||||
availability,
|
||||
(
|
||||
"quota_exhausted".to_owned(),
|
||||
Some("quota_exhausted".to_owned()),
|
||||
),
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_preserves_user_state_during_refresh_writes() {
|
||||
let Some(database) = TestDatabase::create("provider_account_disabled_refresh").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let scope = ProviderAccountAdminScope {
|
||||
provider_kind: "openai".to_owned(),
|
||||
};
|
||||
let account_id = ProviderAccountId::new("acct_disabled_refresh").expect("account ID");
|
||||
let mut disabled = account(account_id.as_str(), "user-disabled-refresh");
|
||||
disabled.enabled = false;
|
||||
disabled.availability = AccountAvailability::Expired;
|
||||
disabled.availability_reason = Some("credential_expired".to_owned());
|
||||
repository
|
||||
.import_provider_accounts(ImportProviderAccounts {
|
||||
scope: scope.clone(),
|
||||
accounts: vec![disabled],
|
||||
audit: audit(
|
||||
"audit_disabled_refresh_import",
|
||||
"import",
|
||||
account_id.as_str(),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("import disabled account");
|
||||
|
||||
repository
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account_id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("credential revision"),
|
||||
availability: AccountAvailability::Ready,
|
||||
reason: None,
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
repository
|
||||
.rotate_provider_account(RotateProviderAccount {
|
||||
scope,
|
||||
profile: profile(account_id.as_str(), "refreshed disabled account"),
|
||||
replacement_identity: None,
|
||||
credential: credential_update(account_id.as_str(), 1, "disabled-refreshed-secret"),
|
||||
audit: audit(
|
||||
"audit_disabled_refresh_rotate",
|
||||
"refresh",
|
||||
account_id.as_str(),
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("refresh disabled account credential");
|
||||
|
||||
let current: (bool, String, Option<String>, serde_json::Value, i64) = sqlx::query_as(
|
||||
"select enabled, availability, availability_reason,
|
||||
provider_credentials_json, credential_revision
|
||||
from provider_accounts where id = $1",
|
||||
)
|
||||
.bind(account_id.as_str())
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("load disabled account after refresh writes");
|
||||
assert!(!current.0);
|
||||
assert_eq!(current.1, "expired");
|
||||
assert_eq!(current.2.as_deref(), Some("credential_expired"));
|
||||
assert_eq!(current.3["access_token"], "disabled-refreshed-secret");
|
||||
assert_eq!(current.4, 2);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
fn account(id: &str, upstream_user_id: &str) -> NewProviderAccount {
|
||||
NewProviderAccount {
|
||||
id: id.to_owned(),
|
||||
@@ -1124,7 +1334,15 @@ fn cooldown_observation_requires_expiry() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_mismatched_cooldown_runtime_facts() {
|
||||
fn quota_exhausted_account_accepts_cooldown_runtime_facts() {
|
||||
let mut imported = account("acct_quota_cooldown", "user-quota-cooldown");
|
||||
imported.availability = AccountAvailability::QuotaExhausted;
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_account_rejects_cooldown_until_for_ready_runtime_facts() {
|
||||
let mut imported = account("acct_invalid_cooldown", "user-invalid-cooldown");
|
||||
imported.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
assert!(imported.validate().is_err());
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use gateway_store::postgres::RuntimeSettingsUpdate;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{
|
||||
PgRuntimeSettingsRepository, RuntimeSettingsRepository, RuntimeSettingsUpdate,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_keep_account_rotation_global() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
use super::TestDatabase;
|
||||
|
||||
fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettingsUpdate {
|
||||
RuntimeSettingsUpdate {
|
||||
admin_api_key: None,
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_margin_seconds,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
@@ -18,24 +22,110 @@ fn runtime_settings_keep_account_rotation_global() {
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_keep_account_rotation_global() {
|
||||
let settings = settings_with_margin(3_600);
|
||||
assert!(settings.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_settings_reject_invalid_model_mapping() {
|
||||
let settings = RuntimeSettingsUpdate {
|
||||
admin_api_key: None,
|
||||
refresh_margin_seconds: 3_600,
|
||||
refresh_concurrency: 2,
|
||||
max_concurrent_per_account: 3,
|
||||
request_interval_ms: 50,
|
||||
rotation_strategy: "smart".to_owned(),
|
||||
model_mappings: BTreeMap::from([("".to_owned(), "gpt-5.5".to_owned())]),
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 30,
|
||||
audit_retention_days: 90,
|
||||
..settings_with_margin(3_600)
|
||||
};
|
||||
|
||||
assert!(settings.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_margin_change_should_reschedule_existing_refreshable_accounts() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_reschedule").await else {
|
||||
return;
|
||||
};
|
||||
let expires_at = timestamp_micros(Utc::now() + TimeDelta::hours(2));
|
||||
insert_refreshable_account(
|
||||
&database.pool,
|
||||
"acct_refresh_margin_changed",
|
||||
expires_at,
|
||||
expires_at - TimeDelta::hours(1),
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(1_800))
|
||||
.await
|
||||
.expect("update refresh margin");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_changed").await,
|
||||
expires_at - TimeDelta::seconds(1_800)
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unchanged_refresh_margin_should_preserve_existing_retry_schedule() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_unchanged").await else {
|
||||
return;
|
||||
};
|
||||
let expires_at = timestamp_micros(Utc::now() + TimeDelta::hours(2));
|
||||
let retry_at = timestamp_micros(Utc::now() + TimeDelta::minutes(5));
|
||||
insert_refreshable_account(
|
||||
&database.pool,
|
||||
"acct_refresh_margin_unchanged",
|
||||
expires_at,
|
||||
retry_at,
|
||||
)
|
||||
.await;
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(3_600))
|
||||
.await
|
||||
.expect("update unrelated runtime settings");
|
||||
|
||||
assert_eq!(
|
||||
account_next_refresh_at(&database.pool, "acct_refresh_margin_unchanged").await,
|
||||
retry_at
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn insert_refreshable_account(
|
||||
pool: &sqlx::PgPool,
|
||||
account_id: &str,
|
||||
expires_at: DateTime<Utc>,
|
||||
next_refresh_at: DateTime<Utc>,
|
||||
) {
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id, provider_kind, name, upstream_user_id, authentication_kind,
|
||||
provider_credentials_json, has_refresh_token, access_token_expires_at,
|
||||
next_refresh_at, availability, availability_observed_at, created_at, updated_at
|
||||
) values ($1, 'openai', $1, $1, 'oauth', '{}'::jsonb, true, $2, $3,
|
||||
'ready', now(), now(), now())",
|
||||
)
|
||||
.bind(account_id)
|
||||
.bind(expires_at)
|
||||
.bind(next_refresh_at)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("insert refreshable account");
|
||||
}
|
||||
|
||||
async fn account_next_refresh_at(pool: &sqlx::PgPool, account_id: &str) -> DateTime<Utc> {
|
||||
sqlx::query_scalar("select next_refresh_at from provider_accounts where id = $1")
|
||||
.bind(account_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("load next refresh time")
|
||||
}
|
||||
|
||||
fn timestamp_micros(value: DateTime<Utc>) -> DateTime<Utc> {
|
||||
DateTime::from_timestamp_micros(value.timestamp_micros()).expect("valid test timestamp")
|
||||
}
|
||||
|
||||
@@ -50,9 +50,13 @@ impl ProviderAccountStore for CooldownTestAccountStore {
|
||||
|
||||
async fn get_account(
|
||||
&self,
|
||||
_account: &ProviderAccountId,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<Option<ProviderAccount>, CoreStoreError> {
|
||||
unreachable!("cooldown tests do not get one account")
|
||||
Ok(self
|
||||
.accounts
|
||||
.iter()
|
||||
.find(|candidate| candidate.id() == account)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_accounts(&self) -> Result<Vec<ProviderAccount>, CoreStoreError> {
|
||||
@@ -190,10 +194,26 @@ fn composition_fixture(
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
composition_fixture_with_accounts(
|
||||
vec![ready_test_account("acct_cooldown_composition")],
|
||||
fail_state_write,
|
||||
fail_cache,
|
||||
)
|
||||
}
|
||||
|
||||
fn composition_fixture_with_accounts(
|
||||
accounts: Vec<ProviderAccount>,
|
||||
fail_state_write: bool,
|
||||
fail_cache: bool,
|
||||
) -> (
|
||||
CooldownCachingProviderAccountStore,
|
||||
Arc<CooldownTestCache>,
|
||||
Arc<Mutex<Vec<&'static str>>>,
|
||||
) {
|
||||
let calls = Arc::new(Mutex::new(Vec::new()));
|
||||
let authoritative = Arc::new(CooldownTestAccountStore {
|
||||
accounts: Vec::new(),
|
||||
accounts,
|
||||
calls: Arc::clone(&calls),
|
||||
fail_state_write,
|
||||
});
|
||||
@@ -249,6 +269,31 @@ async fn cooldown_composition_keeps_postgres_terminal_when_redis_fails() {
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres", "redis"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_account_state_write_does_not_change_runtime_cooldown() {
|
||||
let disabled = test_provider_account("acct_cooldown_composition").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::Ready,
|
||||
None,
|
||||
);
|
||||
let (store, cache, calls) = composition_fixture_with_accounts(vec![disabled], false, false);
|
||||
|
||||
store
|
||||
.apply_state_change(cooldown_state_change(60))
|
||||
.await
|
||||
.expect("disabled state write is a no-op");
|
||||
|
||||
assert_eq!(*calls.lock().expect("calls lock"), ["postgres"]);
|
||||
assert!(cache.cached.lock().expect("cached lock").is_empty());
|
||||
assert!(
|
||||
cache
|
||||
.invalidated
|
||||
.lock()
|
||||
.expect("invalidated lock")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_state_invalidates_same_revision_cooldown_after_postgres_commit() {
|
||||
let (store, cache, calls) = composition_fixture(false, false);
|
||||
|
||||
@@ -115,6 +115,51 @@ async fn native_continuation_rejects_provider_state_for_a_different_provider() {
|
||||
assert!(namespace_keys(&mut connection, &namespace).await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_continuation_record_should_bound_index_cleanup_work() {
|
||||
const STALE_ENTRY_COUNT: usize = 100;
|
||||
const CLEANUP_BATCH: usize = 64;
|
||||
|
||||
let Some((repository, mut connection, namespace)) = repository().await else {
|
||||
return;
|
||||
};
|
||||
repository
|
||||
.record(pin("resp-cleanup-anchor"))
|
||||
.await
|
||||
.expect("record cleanup anchor");
|
||||
let index_key = namespace_keys(&mut connection, &namespace)
|
||||
.await
|
||||
.into_iter()
|
||||
.find(|key| key.ends_with(":global:index"))
|
||||
.expect("continuation index key");
|
||||
let mut seed = redis::cmd("ZADD");
|
||||
seed.arg(&index_key);
|
||||
for index in 0..STALE_ENTRY_COUNT {
|
||||
seed.arg(0).arg(format!("stale-{index}"));
|
||||
}
|
||||
assert_eq!(
|
||||
seed.query_async::<i64>(&mut connection)
|
||||
.await
|
||||
.expect("seed stale continuation index entries"),
|
||||
i64::try_from(STALE_ENTRY_COUNT).expect("stale entry count fits i64")
|
||||
);
|
||||
|
||||
repository
|
||||
.record(pin("resp-cleanup-trigger"))
|
||||
.await
|
||||
.expect("record continuation with bounded cleanup");
|
||||
|
||||
assert_eq!(
|
||||
redis::cmd("ZCARD")
|
||||
.arg(index_key)
|
||||
.query_async::<usize>(&mut connection)
|
||||
.await
|
||||
.expect("count continuation index entries"),
|
||||
STALE_ENTRY_COUNT + 2 - CLEANUP_BATCH
|
||||
);
|
||||
delete_namespace_keys(&mut connection, &namespace).await;
|
||||
}
|
||||
|
||||
async fn repository() -> Option<(RedisNativeContinuationRepository, ConnectionManager, String)> {
|
||||
let redis_url = crate::support::test_env("CPR_TEST_REDIS_URL")?;
|
||||
let client = redis::Client::open(redis_url).expect("valid CPR_TEST_REDIS_URL");
|
||||
@@ -128,6 +173,15 @@ async fn repository() -> Option<(RedisNativeContinuationRepository, ConnectionMa
|
||||
Some((repository, connection, namespace))
|
||||
}
|
||||
|
||||
fn pin(response_id: &str) -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new(response_id),
|
||||
PreviousResponseId::new(response_id),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_primary").expect("account"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn namespace_keys(connection: &mut ConnectionManager, namespace: &str) -> Vec<String> {
|
||||
redis::cmd("KEYS")
|
||||
.arg(format!("{namespace}:*"))
|
||||
|
||||
@@ -586,7 +586,7 @@ fn prepared_rotation(
|
||||
prepared: crate::credential::PreparedCodexCredentialRotation,
|
||||
provider_kind: ProviderKind,
|
||||
) -> Result<PreparedCredentialRotation, ProviderAdminError> {
|
||||
let (profile, credential, guard) = prepared.into_parts();
|
||||
let (profile, credential, replacement_identity, guard) = prepared.into_parts();
|
||||
let (
|
||||
account_id,
|
||||
expected_revision,
|
||||
@@ -606,6 +606,7 @@ fn prepared_rotation(
|
||||
account_id,
|
||||
provider_kind,
|
||||
expected_credential_revision,
|
||||
replacement_identity,
|
||||
name: profile.name,
|
||||
email: profile.email,
|
||||
plan_type: profile.plan_type,
|
||||
|
||||
@@ -11,7 +11,8 @@ use std::time::SystemTime;
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, CredentialCasUpdate, CredentialRevision, LoadedCredential,
|
||||
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountUpdate,
|
||||
NewProviderAccount, ProviderAccount, ProviderAccountId, ProviderAccountIdentity,
|
||||
ProviderAccountUpdate,
|
||||
};
|
||||
use gateway_core::provider_ports::{
|
||||
ProviderLeaseAcquisition, ProviderLeaseGuard, ProviderLeasePort, ProviderLeaseRequest,
|
||||
@@ -286,6 +287,7 @@ impl std::fmt::Debug for RotateManagedCodexCredential {
|
||||
pub struct PreparedCodexCredentialRotation {
|
||||
pub profile: ProviderAccountUpdate,
|
||||
pub credential: CredentialCasUpdate,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
refresh_guards: Option<ProviderRefreshGuards>,
|
||||
}
|
||||
|
||||
@@ -300,6 +302,7 @@ impl std::fmt::Debug for PreparedCodexCredentialRotation {
|
||||
.debug_struct("PreparedCodexCredentialRotation")
|
||||
.field("profile", &self.profile)
|
||||
.field("credential", &self.credential)
|
||||
.field("replacement_identity", &self.replacement_identity)
|
||||
.field(
|
||||
"refresh_guards",
|
||||
&self.refresh_guards.as_ref().map(|_| "<held>"),
|
||||
@@ -316,11 +319,13 @@ impl PreparedCodexCredentialRotation {
|
||||
) -> (
|
||||
ProviderAccountUpdate,
|
||||
CredentialCasUpdate,
|
||||
Option<ProviderAccountIdentity>,
|
||||
PreparedCodexCredentialRotationGuard,
|
||||
) {
|
||||
(
|
||||
self.profile,
|
||||
self.credential,
|
||||
self.replacement_identity,
|
||||
PreparedCodexCredentialRotationGuard(self.refresh_guards),
|
||||
)
|
||||
}
|
||||
@@ -494,7 +499,7 @@ impl CodexCredentialAdmin {
|
||||
self.prepare_oauth_rotation(input, false)
|
||||
}
|
||||
|
||||
/// 完整 OAuth 授权允许认证主体投影更新,但不允许目标账号或用户发生变化。
|
||||
/// 完整 OAuth 授权按目标数据 ID 替换凭据与上游账号身份。
|
||||
pub(crate) fn prepare_reauthorization(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
@@ -505,7 +510,7 @@ impl CodexCredentialAdmin {
|
||||
fn prepare_oauth_rotation(
|
||||
&self,
|
||||
input: RotateManagedCodexCredential,
|
||||
replace_principal: bool,
|
||||
replace_identity: bool,
|
||||
) -> Result<PreparedCodexCredentialRotation, CodexCredentialAdminError> {
|
||||
let access_token_expires_at =
|
||||
required_time(input.verified_account.access_token_expires_at)?;
|
||||
@@ -516,16 +521,23 @@ impl CodexCredentialAdmin {
|
||||
.ok_or(CodexCredentialAdminError::IdentityMismatch)?;
|
||||
if input.current.account.provider().as_str() != PROVIDER_NAME
|
||||
|| input.current.account.authentication_kind() != CODEX_AUTHENTICATION_KIND_OAUTH
|
||||
|| input.current.account.upstream_account_id()
|
||||
!= Some(input.verified_account.chatgpt_account_id.as_str())
|
||||
|| input.current.account.upstream_user_id() != input.verified_account.chatgpt_user_id
|
||||
|| (!replace_principal
|
||||
&& (oauth.principal.oauth_subject != input.verified_account.oauth_subject
|
||||
|| (!replace_identity
|
||||
&& (input.current.account.upstream_account_id()
|
||||
!= Some(input.verified_account.chatgpt_account_id.as_str())
|
||||
|| input.current.account.upstream_user_id()
|
||||
!= input.verified_account.chatgpt_user_id
|
||||
|| oauth.principal.oauth_subject != input.verified_account.oauth_subject
|
||||
|| oauth.principal.poid != input.verified_account.poid))
|
||||
{
|
||||
return Err(CodexCredentialAdminError::IdentityMismatch);
|
||||
}
|
||||
if replace_principal {
|
||||
let replacement_identity = replace_identity.then(|| {
|
||||
ProviderAccountIdentity::new(
|
||||
input.verified_account.chatgpt_user_id.clone(),
|
||||
Some(input.verified_account.chatgpt_account_id.clone()),
|
||||
)
|
||||
});
|
||||
if replace_identity {
|
||||
oauth
|
||||
.principal
|
||||
.oauth_subject
|
||||
@@ -567,6 +579,7 @@ impl CodexCredentialAdmin {
|
||||
Ok(PreparedCodexCredentialRotation {
|
||||
profile,
|
||||
credential,
|
||||
replacement_identity,
|
||||
refresh_guards: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ use super::admin::{
|
||||
use super::identity::{
|
||||
CodexAccountIdentityVerifier, CodexIdentityExpectation, CodexIdentityVerificationError,
|
||||
};
|
||||
use super::security::CodexCredentialCodec;
|
||||
use super::token_client::{
|
||||
AuthorizationCodeExchangeError, AuthorizationCodeExchanger, AuthorizationCodeGrant,
|
||||
OFFICIAL_CODEX_OAUTH_CLIENT_ID, OFFICIAL_CODEX_REDIRECT_URI,
|
||||
@@ -500,11 +499,7 @@ impl CodexOAuthAdminService {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let expectation = current
|
||||
.as_ref()
|
||||
.map(oauth_identity_expectation)
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
let expectation = CodexIdentityExpectation::default();
|
||||
let profile = self
|
||||
.verifier
|
||||
.verify_authorization(&secret, &id_token, &pending.nonce, &expectation)
|
||||
@@ -679,26 +674,6 @@ impl CodexOAuthAdminService {
|
||||
}
|
||||
}
|
||||
|
||||
fn oauth_identity_expectation(
|
||||
current: &gateway_core::engine::credential::LoadedCredential,
|
||||
) -> Result<CodexIdentityExpectation, CodexOAuthAdminError> {
|
||||
let runtime = CodexCredentialCodec::decode(¤t.credential)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)?;
|
||||
if runtime.authentication.oauth().is_none() {
|
||||
return Err(CodexOAuthAdminError::Credential);
|
||||
}
|
||||
let account_id = current
|
||||
.account
|
||||
.upstream_account_id()
|
||||
.ok_or(CodexOAuthAdminError::Credential)?;
|
||||
CodexIdentityExpectation::reauthorization(
|
||||
account_id.to_owned(),
|
||||
current.account.upstream_user_id().to_owned(),
|
||||
runtime.installation_id,
|
||||
)
|
||||
.map_err(|_| CodexOAuthAdminError::Credential)
|
||||
}
|
||||
|
||||
fn authorization_url(
|
||||
pending: &CodexPendingAuthorization,
|
||||
oauth_client_id: &str,
|
||||
|
||||
@@ -30,6 +30,7 @@ const MAX_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60 * 60);
|
||||
const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const QUOTA_RESET_RECHECK_GRACE: Duration = Duration::from_secs(30);
|
||||
// ready 账号主要依赖真实请求的被动观察;这里只提供低频、定额的状态兜底。
|
||||
const QUOTA_READY_REFRESH_MAX_AGE: Duration = Duration::from_secs(2 * 60 * 60);
|
||||
const QUOTA_READY_REFRESH_BATCH_SIZE: usize = 8;
|
||||
@@ -307,7 +308,7 @@ enum CodexQuotaFetchError {
|
||||
enum CodexQuotaStateTransition {
|
||||
Banned { reason: &'static str },
|
||||
Exhausted,
|
||||
Cooldown { until: Option<SystemTime> },
|
||||
ResettableLimit { until: Option<SystemTime> },
|
||||
}
|
||||
|
||||
impl CodexQuotaStateTransition {
|
||||
@@ -315,7 +316,7 @@ impl CodexQuotaStateTransition {
|
||||
match self {
|
||||
Self::Banned { .. } => AccountAvailability::Banned,
|
||||
Self::Exhausted => AccountAvailability::QuotaExhausted,
|
||||
Self::Cooldown { .. } => AccountAvailability::Cooldown,
|
||||
Self::ResettableLimit { .. } => AccountAvailability::QuotaExhausted,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -323,13 +324,13 @@ impl CodexQuotaStateTransition {
|
||||
match self {
|
||||
Self::Banned { reason } => reason,
|
||||
Self::Exhausted => "quota_exhausted",
|
||||
Self::Cooldown { .. } => "quota_rate_limited",
|
||||
Self::ResettableLimit { .. } => "quota_rate_limited",
|
||||
}
|
||||
}
|
||||
|
||||
const fn cooldown_until(self) -> Option<SystemTime> {
|
||||
match self {
|
||||
Self::Cooldown { until } => until,
|
||||
Self::ResettableLimit { until } => until,
|
||||
Self::Banned { .. } | Self::Exhausted => None,
|
||||
}
|
||||
}
|
||||
@@ -547,7 +548,9 @@ fn quota_refresh_candidate(
|
||||
let snapshot = quota_snapshot_for_account(&account, observation);
|
||||
let kind = match account.availability() {
|
||||
AccountAvailability::QuotaExhausted => CodexQuotaRefreshKind::Exhausted {
|
||||
reset_at: quota_reset_at_from_snapshot(snapshot.as_ref()),
|
||||
reset_at: account
|
||||
.cooldown_until()
|
||||
.or_else(|| quota_reset_at_from_snapshot(snapshot.as_ref())),
|
||||
},
|
||||
AccountAvailability::Cooldown
|
||||
if snapshot
|
||||
@@ -618,22 +621,24 @@ fn reserve_exhausted_quota_refresh(
|
||||
now: SystemTime,
|
||||
refreshed_at: Instant,
|
||||
) -> bool {
|
||||
let reset_recheck_due = reset_at.is_some_and(|reset_at| {
|
||||
reset_at <= now
|
||||
&& state
|
||||
.last_exhausted_reset_recheck
|
||||
.get(account.id())
|
||||
.is_none_or(|marker| {
|
||||
marker.revision != account.revision() || marker.reset_at != reset_at
|
||||
})
|
||||
});
|
||||
if !reset_recheck_due && !periodic_quota_refresh_due(state, account.id(), refreshed_at) {
|
||||
return false;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
if let Some(reset_at) = reset_at.filter(|reset_at| *reset_at <= now) {
|
||||
if let Some(reset_at) = reset_at {
|
||||
if !quota_reset_recheck_ready(reset_at, now) {
|
||||
return false;
|
||||
}
|
||||
let same_reset_already_checked = state
|
||||
.last_exhausted_reset_recheck
|
||||
.get(account.id())
|
||||
.is_some_and(|marker| {
|
||||
marker.revision == account.revision() && marker.reset_at == reset_at
|
||||
});
|
||||
if same_reset_already_checked
|
||||
&& !periodic_quota_refresh_due(state, account.id(), refreshed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
state.last_exhausted_reset_recheck.insert(
|
||||
account.id().clone(),
|
||||
CodexQuotaResetMarker {
|
||||
@@ -641,10 +646,23 @@ fn reserve_exhausted_quota_refresh(
|
||||
reset_at,
|
||||
},
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if !periodic_quota_refresh_due(state, account.id(), refreshed_at) {
|
||||
return false;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
true
|
||||
}
|
||||
|
||||
fn quota_reset_recheck_ready(reset_at: SystemTime, now: SystemTime) -> bool {
|
||||
reset_at
|
||||
.checked_add(QUOTA_RESET_RECHECK_GRACE)
|
||||
.is_some_and(|ready_at| ready_at <= now)
|
||||
}
|
||||
|
||||
fn periodic_quota_refresh_due(
|
||||
state: &CodexQuotaProjectionState,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -802,16 +820,12 @@ impl CodexCredentialQuotaService {
|
||||
summary.transient += 1;
|
||||
continue;
|
||||
};
|
||||
let preserve_exhaustion = account.availability()
|
||||
== AccountAvailability::QuotaExhausted
|
||||
&& transition.availability() == AccountAvailability::Cooldown;
|
||||
match transition {
|
||||
CodexQuotaStateTransition::Banned { .. } => summary.banned += 1,
|
||||
CodexQuotaStateTransition::Exhausted => summary.exhausted += 1,
|
||||
CodexQuotaStateTransition::Cooldown { .. } => summary.cooldown += 1,
|
||||
}
|
||||
if preserve_exhaustion {
|
||||
continue;
|
||||
CodexQuotaStateTransition::Exhausted
|
||||
| CodexQuotaStateTransition::ResettableLimit { .. } => {
|
||||
summary.exhausted += 1;
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
.repository
|
||||
@@ -856,19 +870,23 @@ impl CodexCredentialQuotaService {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !matches!(
|
||||
self.has_active_exhaustion(¤t, observed_at).await,
|
||||
Ok(true)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let snapshot = match self.read_snapshot_for(¤t).await {
|
||||
Ok(Some(snapshot))
|
||||
if quota_fact_is_active_exhaustion(snapshot.fact(), observed_at) =>
|
||||
{
|
||||
snapshot
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let cooldown_until =
|
||||
quota_reset_at_from_snapshot(Some(&snapshot)).or_else(|| current.cooldown_until());
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await
|
||||
@@ -881,8 +899,11 @@ impl CodexCredentialQuotaService {
|
||||
continue;
|
||||
}
|
||||
let enabled = current.enabled();
|
||||
*account =
|
||||
current.with_runtime_state(enabled, AccountAvailability::QuotaExhausted, None);
|
||||
*account = current.with_runtime_state(
|
||||
enabled,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
cooldown_until,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -927,10 +948,10 @@ impl CodexCredentialQuotaService {
|
||||
} else {
|
||||
summary.updated += 1;
|
||||
}
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
fact.exhausted(),
|
||||
fact,
|
||||
SystemTime::now(),
|
||||
) {
|
||||
let _ = self
|
||||
@@ -939,7 +960,7 @@ impl CodexCredentialQuotaService {
|
||||
¤t,
|
||||
availability,
|
||||
fact.exhausted().then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
@@ -993,10 +1014,10 @@ impl CodexCredentialQuotaService {
|
||||
}
|
||||
self.scheduling
|
||||
.observe(current.id().clone(), current.revision(), observed_at, fact);
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
fact.exhausted(),
|
||||
fact,
|
||||
observed_at,
|
||||
) {
|
||||
self.repository
|
||||
@@ -1004,7 +1025,7 @@ impl CodexCredentialQuotaService {
|
||||
¤t,
|
||||
availability,
|
||||
fact.exhausted().then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
@@ -1026,19 +1047,6 @@ impl CodexCredentialQuotaService {
|
||||
self.read_snapshot_for(&account).await
|
||||
}
|
||||
|
||||
pub(super) async fn has_active_exhaustion(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
now: SystemTime,
|
||||
) -> Result<bool, CodexCredentialQuotaError> {
|
||||
match self.read_snapshot_for(account).await {
|
||||
Ok(snapshot) => Ok(snapshot
|
||||
.is_some_and(|snapshot| quota_fact_is_active_exhaustion(snapshot.fact(), now))),
|
||||
Err(CodexCredentialQuotaError::RevisionConflict) => Ok(false),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_snapshot_for(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
@@ -1164,10 +1172,10 @@ impl CodexCredentialQuotaService {
|
||||
snapshot.observed_at(),
|
||||
snapshot.fact(),
|
||||
);
|
||||
if let Some(availability) = quota_success_availability(
|
||||
if let Some((availability, cooldown_until)) = quota_success_state(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
snapshot.fact().exhausted(),
|
||||
snapshot.fact(),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
self.repository
|
||||
@@ -1178,7 +1186,7 @@ impl CodexCredentialQuotaService {
|
||||
.fact()
|
||||
.exhausted()
|
||||
.then_some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
@@ -1386,31 +1394,34 @@ fn passive_rate_limit_window(window: RateLimitWindow) -> Map<String, Value> {
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn quota_success_availability(
|
||||
fn quota_success_state(
|
||||
current: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
exhausted: bool,
|
||||
fact: CodexQuotaFact,
|
||||
now: SystemTime,
|
||||
) -> Option<AccountAvailability> {
|
||||
match current {
|
||||
) -> Option<(AccountAvailability, Option<SystemTime>)> {
|
||||
if fact.exhausted() {
|
||||
let observed_cooldown = fact.resets_at().map(SystemTime::from).or(cooldown_until);
|
||||
return (current != AccountAvailability::QuotaExhausted
|
||||
|| cooldown_until != observed_cooldown)
|
||||
.then_some((AccountAvailability::QuotaExhausted, observed_cooldown));
|
||||
}
|
||||
|
||||
let availability = match current {
|
||||
AccountAvailability::Invalid
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned => None,
|
||||
AccountAvailability::QuotaExhausted => (!exhausted).then_some(AccountAvailability::Ready),
|
||||
AccountAvailability::Ready => exhausted.then_some(AccountAvailability::QuotaExhausted),
|
||||
AccountAvailability::Cooldown if exhausted => Some(AccountAvailability::QuotaExhausted),
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted => Some(AccountAvailability::Ready),
|
||||
AccountAvailability::Ready => None,
|
||||
AccountAvailability::Cooldown
|
||||
if cooldown_until.is_some_and(|cooldown_until| cooldown_until <= now) =>
|
||||
{
|
||||
Some(AccountAvailability::Ready)
|
||||
}
|
||||
AccountAvailability::Cooldown => None,
|
||||
AccountAvailability::Unknown => Some(if exhausted {
|
||||
AccountAvailability::QuotaExhausted
|
||||
} else {
|
||||
AccountAvailability::Ready
|
||||
}),
|
||||
}
|
||||
};
|
||||
availability.map(|availability| (availability, None))
|
||||
}
|
||||
|
||||
fn quota_state_transition(
|
||||
@@ -1438,7 +1449,7 @@ fn quota_state_transition(
|
||||
retry_after_seconds.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN.as_secs()),
|
||||
)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
Some(CodexQuotaStateTransition::Cooldown {
|
||||
Some(CodexQuotaStateTransition::ResettableLimit {
|
||||
until: observed_at.checked_add(duration),
|
||||
})
|
||||
}
|
||||
@@ -1876,7 +1887,6 @@ impl QuotaAggregate {
|
||||
self.remaining_percent
|
||||
.map_or(remaining, |current| current.min(remaining)),
|
||||
);
|
||||
self.exhausted |= used >= 100.0;
|
||||
}
|
||||
if let Some(reset) = object.get("reset_at") {
|
||||
let seconds = reset
|
||||
|
||||
@@ -256,8 +256,6 @@ impl CodexCredentialRefreshService {
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid
|
||||
)
|
||||
&& (account.availability() != AccountAvailability::Cooldown
|
||||
|| account.cooldown_until().is_some_and(|until| until <= now))
|
||||
&& refresh_due_at(account, now).is_some()
|
||||
});
|
||||
accounts.sort_by_key(|account| (refresh_due_at(account, now), account.id().clone()));
|
||||
|
||||
@@ -237,6 +237,9 @@ impl CodexCredentialRepository {
|
||||
cooldown_until: Option<SystemTime>,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialRepositoryError> {
|
||||
if !account.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
|
||||
@@ -56,6 +56,11 @@ pub enum CodexAccountFailure {
|
||||
Banned,
|
||||
/// 账号信用额度已耗尽。
|
||||
QuotaExhausted,
|
||||
/// 当前用量窗口已耗尽;到重置时间后可自动恢复。
|
||||
UsageLimitExhausted {
|
||||
/// 上游返回的窗口重置时长。
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
/// 账号触发临时用量限制。
|
||||
RateLimited {
|
||||
/// 上游明确返回的最短冷却时长。
|
||||
@@ -702,9 +707,19 @@ impl CodexCredentialSelector {
|
||||
None,
|
||||
CookieRecovery::None,
|
||||
),
|
||||
CodexAccountFailure::UsageLimitExhausted { retry_after } => {
|
||||
return self
|
||||
.record_resettable_limit_failure(
|
||||
account,
|
||||
retry_after,
|
||||
"usage_limit_exhausted",
|
||||
now,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
CodexAccountFailure::RateLimited { retry_after } => {
|
||||
return self
|
||||
.record_rate_limit_failure(account, retry_after, now)
|
||||
.record_resettable_limit_failure(account, retry_after, "rate_limited", now)
|
||||
.await;
|
||||
}
|
||||
CodexAccountFailure::CloudflareChallenge { retry_after } => {
|
||||
@@ -743,45 +758,42 @@ impl CodexCredentialSelector {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_rate_limit_failure(
|
||||
async fn record_resettable_limit_failure(
|
||||
&self,
|
||||
account: &ProviderAccount,
|
||||
retry_after: Option<Duration>,
|
||||
reason: &'static str,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<(), CredentialSelectionError> {
|
||||
let current = self.current_account(account.id()).await?;
|
||||
if current.revision() != account.revision() {
|
||||
return Err(CredentialRepositoryError::RevisionConflict.into());
|
||||
}
|
||||
if matches!(
|
||||
current.availability(),
|
||||
AccountAvailability::Invalid
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::QuotaExhausted
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
let quota_exhausted = self
|
||||
.quota
|
||||
.has_active_exhaustion(¤t, observed_at)
|
||||
.await
|
||||
.map_err(|_| CredentialSelectionError::Store)?;
|
||||
let (availability, reason, cooldown_until) = if quota_exhausted {
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some("quota_exhausted".to_owned()),
|
||||
None,
|
||||
)
|
||||
} else {
|
||||
(
|
||||
AccountAvailability::Cooldown,
|
||||
Some("rate_limited".to_owned()),
|
||||
observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)),
|
||||
)
|
||||
let cooldown_until =
|
||||
observed_at.checked_add(retry_after.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN));
|
||||
let cooldown_until = match (current.availability(), current.cooldown_until()) {
|
||||
(AccountAvailability::QuotaExhausted, Some(existing)) => {
|
||||
cooldown_until.map_or(Some(existing), |until| Some(existing.max(until)))
|
||||
}
|
||||
_ => cooldown_until,
|
||||
};
|
||||
let reason = if current.availability() == AccountAvailability::QuotaExhausted
|
||||
&& current.cooldown_until().is_none()
|
||||
{
|
||||
Some("quota_exhausted".to_owned())
|
||||
} else {
|
||||
Some(reason.to_owned())
|
||||
};
|
||||
// 一次真实 429 证明上游已经接受当前凭据,可纠正旧的认证/封禁终态。
|
||||
// resettable 用量限制沿用 v2 的 quota lock 语义,并保留/延长复核时间。
|
||||
self.repository
|
||||
.apply_state(¤t, availability, reason, cooldown_until, observed_at)
|
||||
.apply_state(
|
||||
¤t,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -847,6 +859,8 @@ impl CodexCredentialSelector {
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
)
|
||||
{
|
||||
return;
|
||||
|
||||
@@ -68,6 +68,7 @@ use crate::transport::request::{
|
||||
scope_request_to_account,
|
||||
};
|
||||
use crate::transport::session::CodexSessionIdentity;
|
||||
use crate::transport::usage::normalize_service_tier;
|
||||
use crate::transport::websocket::{CodexWebSocketExchangeError, PreviousResponseUnavailableReason};
|
||||
use crate::transport::{
|
||||
CODEX_RESPONSES_PATH, CodexAccountSelectionTelemetry, CodexBackendClient,
|
||||
@@ -822,8 +823,9 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
// 下游按字节转发上游原文,避免 serde 往返改写数值/精度(大整数→f64、logprobs 等)。
|
||||
// WS 帧由 reducer 以 encode_sse_event(&event, raw) 逐字节内嵌上游原始 JSON
|
||||
// (transport/protocol/websocket.rs),push_frames 抽出的 data 即上游原文。
|
||||
let mut decoder =
|
||||
CodexCanonicalDecoder::new(upstream_model.as_str()).with_raw_sse_passthrough();
|
||||
let mut decoder = CodexCanonicalDecoder::new(upstream_model.as_str())
|
||||
.with_requested_service_tier(request.service_tier())
|
||||
.with_raw_sse_passthrough();
|
||||
let mut pre_commit_events = PreCommitClientEvents::new(stream_commit_policy);
|
||||
loop {
|
||||
let Some(stream_deadline) = remaining(context.deadline()) else {
|
||||
@@ -895,7 +897,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
};
|
||||
pre_commit_events.observe_chunk(chunk_len);
|
||||
let service_tier_changed = observation_state
|
||||
.observe_service_tier(decoder.response_service_tier());
|
||||
.observe_upstream_service_tier(decoder.response_service_tier());
|
||||
let terminal_failure = canonical_failure.map(|(error, semantic_output_seen)| {
|
||||
let atomic_upstream_failure = matches!(&error, CodexCanonicalError::Upstream(_));
|
||||
(
|
||||
@@ -990,8 +992,8 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
)
|
||||
});
|
||||
let timing_signals = decoder.take_timing_signals();
|
||||
let service_tier_changed =
|
||||
observation_state.observe_service_tier(decoder.response_service_tier());
|
||||
let service_tier_changed = observation_state
|
||||
.observe_upstream_service_tier(decoder.response_service_tier());
|
||||
let timing_changed = observation_state
|
||||
.observe_timing_signals(timing_signals, context.timing_started_at());
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
@@ -1070,7 +1072,8 @@ struct OpenAiResponseObservationState {
|
||||
requested_model: String,
|
||||
stream: bool,
|
||||
compact: bool,
|
||||
service_tier: Option<String>,
|
||||
requested_service_tier: Option<String>,
|
||||
upstream_service_tier: Option<String>,
|
||||
rate_limit_headers: Vec<(String, String)>,
|
||||
account_id: String,
|
||||
attempt_index: u32,
|
||||
@@ -1102,8 +1105,8 @@ impl OpenAiResponseObservationState {
|
||||
requested_model: request.model().to_owned(),
|
||||
stream: request.stream(),
|
||||
compact: semantics.compact,
|
||||
// 请求档位保留在 requestSummary;一等观测只接受响应确认的实际档位。
|
||||
service_tier: None,
|
||||
requested_service_tier: normalize_service_tier(request.service_tier()),
|
||||
upstream_service_tier: None,
|
||||
rate_limit_headers: selected_observation_headers(&response.rate_limit_headers),
|
||||
account_id: account_id.to_owned(),
|
||||
attempt_index,
|
||||
@@ -1127,7 +1130,7 @@ impl OpenAiResponseObservationState {
|
||||
if let Some(metadata) = self.provider_metadata() {
|
||||
observation = observation.with_provider_metadata(metadata);
|
||||
}
|
||||
if let Some(service_tier) = self.service_tier.as_deref() {
|
||||
if let Some(service_tier) = self.effective_service_tier() {
|
||||
observation = observation.with_service_tier_if_valid(service_tier.to_owned());
|
||||
}
|
||||
Some(observation)
|
||||
@@ -1158,17 +1161,23 @@ impl OpenAiResponseObservationState {
|
||||
changed
|
||||
}
|
||||
|
||||
fn observe_service_tier(&mut self, service_tier: Option<&str>) -> bool {
|
||||
let Some(service_tier) = service_tier else {
|
||||
fn observe_upstream_service_tier(&mut self, service_tier: Option<&str>) -> bool {
|
||||
let Some(service_tier) = normalize_service_tier(service_tier) else {
|
||||
return false;
|
||||
};
|
||||
if self.service_tier.as_deref() == Some(service_tier) {
|
||||
if self.upstream_service_tier.as_deref() == Some(service_tier.as_str()) {
|
||||
return false;
|
||||
}
|
||||
self.service_tier = Some(service_tier.to_owned());
|
||||
self.upstream_service_tier = Some(service_tier);
|
||||
true
|
||||
}
|
||||
|
||||
fn effective_service_tier(&self) -> Option<&str> {
|
||||
self.requested_service_tier
|
||||
.as_deref()
|
||||
.or(self.upstream_service_tier.as_deref())
|
||||
}
|
||||
|
||||
fn merge_rate_limit_headers(&mut self, updates: &[(String, String)]) -> bool {
|
||||
let mut changed = false;
|
||||
for (name, value) in selected_observation_headers(updates) {
|
||||
@@ -1245,9 +1254,21 @@ impl OpenAiResponseObservationState {
|
||||
)),
|
||||
);
|
||||
metadata.insert("requestSummary".to_owned(), self.request_summary.clone());
|
||||
if let Some(service_tier) = &self.service_tier {
|
||||
if let Some(service_tier) = self.effective_service_tier() {
|
||||
metadata.insert(
|
||||
"serviceTier".to_owned(),
|
||||
Value::String(service_tier.to_owned()),
|
||||
);
|
||||
}
|
||||
if let Some(service_tier) = &self.requested_service_tier {
|
||||
metadata.insert(
|
||||
"requestedServiceTier".to_owned(),
|
||||
Value::String(service_tier.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(service_tier) = &self.upstream_service_tier {
|
||||
metadata.insert(
|
||||
"upstreamServiceTier".to_owned(),
|
||||
Value::String(service_tier.clone()),
|
||||
);
|
||||
}
|
||||
@@ -2191,6 +2212,7 @@ const fn provider_error_kind(category: CodexFailureCategory) -> ProviderErrorKin
|
||||
CodexFailureCategory::IdentityVerificationRequired | CodexFailureCategory::Banned => {
|
||||
ProviderErrorKind::PermissionDenied
|
||||
}
|
||||
CodexFailureCategory::UsageLimitExhausted => ProviderErrorKind::QuotaExhausted,
|
||||
CodexFailureCategory::RateLimited => ProviderErrorKind::RateLimited,
|
||||
CodexFailureCategory::QuotaExhausted => ProviderErrorKind::QuotaExhausted,
|
||||
CodexFailureCategory::CloudflareChallenge
|
||||
@@ -2221,6 +2243,11 @@ fn account_failure(
|
||||
Some(CodexAccountFailure::IdentityVerificationRequired)
|
||||
}
|
||||
CodexFailureCategory::Banned => Some(CodexAccountFailure::Banned),
|
||||
CodexFailureCategory::UsageLimitExhausted => {
|
||||
Some(CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: retry_after_seconds.map(Duration::from_secs),
|
||||
})
|
||||
}
|
||||
CodexFailureCategory::RateLimited => Some(CodexAccountFailure::RateLimited {
|
||||
retry_after: retry_after_seconds.map(Duration::from_secs),
|
||||
}),
|
||||
|
||||
@@ -21,10 +21,9 @@ use thiserror::Error;
|
||||
use super::protocol::responses::{
|
||||
ResponseEventSignals, ResponsesSseFailure, response_event_signals,
|
||||
};
|
||||
use super::usage::openai_billing_breakdown;
|
||||
use super::usage::{normalize_service_tier, openai_billing_breakdown};
|
||||
|
||||
const CONTENTS_PER_OUTPUT: u32 = 1_024;
|
||||
const MAX_SERVICE_TIER_BYTES: usize = 64;
|
||||
|
||||
/// 单 attempt 的增量 Responses decoder。
|
||||
///
|
||||
@@ -42,6 +41,7 @@ pub struct CodexCanonicalDecoder {
|
||||
reasoning_output_seen: BTreeSet<u32>,
|
||||
usage_emitted: bool,
|
||||
semantic_output_seen: bool,
|
||||
requested_service_tier: Option<String>,
|
||||
response_service_tier: Option<String>,
|
||||
timing_signals: ResponseEventSignals,
|
||||
raw_sse_passthrough: bool,
|
||||
@@ -137,6 +137,7 @@ impl CodexCanonicalDecoder {
|
||||
reasoning_output_seen: BTreeSet::new(),
|
||||
usage_emitted: false,
|
||||
semantic_output_seen: false,
|
||||
requested_service_tier: None,
|
||||
response_service_tier: None,
|
||||
timing_signals: ResponseEventSignals::default(),
|
||||
raw_sse_passthrough: false,
|
||||
@@ -152,6 +153,13 @@ impl CodexCanonicalDecoder {
|
||||
self
|
||||
}
|
||||
|
||||
/// 请求显式指定的档位优先用于计费;响应档位只在请求未指定时兜底。
|
||||
#[must_use]
|
||||
pub fn with_requested_service_tier(mut self, service_tier: Option<&str>) -> Self {
|
||||
self.requested_service_tier = normalize_service_tier(service_tier);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn push(&mut self, chunk: &[u8]) -> CodexCanonicalOutcome {
|
||||
self.timing_signals = ResponseEventSignals::default();
|
||||
if self.raw_sse_passthrough {
|
||||
@@ -308,19 +316,14 @@ impl CodexCanonicalDecoder {
|
||||
}
|
||||
|
||||
fn observe_response_service_tier(&mut self, value: &Value) {
|
||||
let Some(service_tier) = response_object(value)
|
||||
.and_then(|response| response.get("service_tier"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= MAX_SERVICE_TIER_BYTES
|
||||
&& !value.chars().any(char::is_control)
|
||||
})
|
||||
else {
|
||||
let Some(service_tier) = normalize_service_tier(
|
||||
response_object(value)
|
||||
.and_then(|response| response.get("service_tier"))
|
||||
.and_then(Value::as_str),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
self.response_service_tier = Some(service_tier.to_ascii_lowercase());
|
||||
self.response_service_tier = Some(service_tier);
|
||||
}
|
||||
|
||||
/// 构造下发的 wire event。
|
||||
@@ -836,9 +839,9 @@ impl CodexCanonicalDecoder {
|
||||
.filter(|model| !model.is_empty())
|
||||
.unwrap_or(&self.fallback_model)
|
||||
.to_owned();
|
||||
let service_tier = response
|
||||
.get("service_tier")
|
||||
.and_then(Value::as_str)
|
||||
let service_tier = self
|
||||
.requested_service_tier
|
||||
.as_deref()
|
||||
.or(self.response_service_tier.as_deref());
|
||||
if let Some(breakdown) = usage
|
||||
.filter(|usage| billable_usage_is_complete(response, *usage))
|
||||
|
||||
@@ -3,7 +3,8 @@ use std::{sync::Arc, time::Instant};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
events::retry_after_seconds_from_body,
|
||||
events::{self, retry_after_seconds_from_body},
|
||||
sse::{SseEventDecoder, SseFrame},
|
||||
};
|
||||
use reqwest::{
|
||||
Client, Response as ReqwestResponse,
|
||||
@@ -130,13 +131,14 @@ impl CodexBackendClient {
|
||||
});
|
||||
}
|
||||
|
||||
let rate_limit_header_updates = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
body: http_sse_stream(response),
|
||||
body: http_sse_stream(response, Arc::clone(&rate_limit_header_updates)),
|
||||
transport: CodexBackendTransport::HttpSse,
|
||||
turn_state,
|
||||
set_cookie_headers,
|
||||
rate_limit_headers,
|
||||
rate_limit_header_updates: None,
|
||||
rate_limit_header_updates: Some(rate_limit_header_updates),
|
||||
turn_state_update: None,
|
||||
websocket_pool_decision: None,
|
||||
diagnostics,
|
||||
@@ -594,20 +596,74 @@ fn websocket_connection_profile(headers: &HeaderMap) -> String {
|
||||
.join("\0")
|
||||
}
|
||||
|
||||
fn http_sse_stream(response: ReqwestResponse) -> CodexBackendSseStream {
|
||||
fn http_sse_stream(
|
||||
response: ReqwestResponse,
|
||||
rate_limit_updates: CodexRateLimitHeaderUpdates,
|
||||
) -> CodexBackendSseStream {
|
||||
let stream: CodexBackendSseStream =
|
||||
Box::pin(response.bytes_stream().map_err(CodexClientError::Http));
|
||||
Box::pin(futures::stream::unfold(Some(stream), |stream| async move {
|
||||
let mut stream = stream?;
|
||||
match tokio::time::timeout(UPSTREAM_STREAM_IDLE_TIMEOUT, stream.next()).await {
|
||||
Ok(Some(chunk)) => Some((chunk, Some(stream))),
|
||||
Ok(None) => None,
|
||||
Err(_) => Some((
|
||||
Err(CodexClientError::StreamIdleTimeout {
|
||||
timeout: UPSTREAM_STREAM_IDLE_TIMEOUT,
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
let stream: CodexBackendSseStream =
|
||||
Box::pin(futures::stream::unfold(Some(stream), |stream| async move {
|
||||
let mut stream = stream?;
|
||||
match tokio::time::timeout(UPSTREAM_STREAM_IDLE_TIMEOUT, stream.next()).await {
|
||||
Ok(Some(chunk)) => Some((chunk, Some(stream))),
|
||||
Ok(None) => None,
|
||||
Err(_) => Some((
|
||||
Err(CodexClientError::StreamIdleTimeout {
|
||||
timeout: UPSTREAM_STREAM_IDLE_TIMEOUT,
|
||||
}),
|
||||
None,
|
||||
)),
|
||||
}
|
||||
}));
|
||||
observe_http_sse_rate_limits(stream, rate_limit_updates)
|
||||
}
|
||||
|
||||
fn observe_http_sse_rate_limits(
|
||||
stream: CodexBackendSseStream,
|
||||
updates: CodexRateLimitHeaderUpdates,
|
||||
) -> CodexBackendSseStream {
|
||||
Box::pin(futures::stream::unfold(
|
||||
(stream, SseEventDecoder::default(), updates),
|
||||
|(mut stream, mut decoder, updates)| async move {
|
||||
match stream.next().await {
|
||||
Some(chunk) => {
|
||||
if let Ok(bytes) = &chunk {
|
||||
append_http_sse_rate_limit_updates(decoder.push_frames(bytes), &updates)
|
||||
.await;
|
||||
}
|
||||
Some((chunk, (stream, decoder, updates)))
|
||||
}
|
||||
None => {
|
||||
append_http_sse_rate_limit_updates(decoder.finish_frames(), &updates).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
async fn append_http_sse_rate_limit_updates(
|
||||
frames: Vec<SseFrame>,
|
||||
updates: &CodexRateLimitHeaderUpdates,
|
||||
) {
|
||||
let mut headers = Vec::new();
|
||||
for frame in frames {
|
||||
for event in frame.events() {
|
||||
if event
|
||||
.event
|
||||
.as_deref()
|
||||
.is_some_and(|event| event != "codex.rate_limits")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(rate_limits) = events::parse_rate_limits_event_raw(&event.data) else {
|
||||
continue;
|
||||
};
|
||||
headers.extend(events::rate_limits_to_header_pairs(&rate_limits));
|
||||
}
|
||||
}))
|
||||
}
|
||||
if !headers.is_empty() {
|
||||
updates.lock().await.extend(headers);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,8 @@ pub enum CodexFailureCategory {
|
||||
CredentialExpired,
|
||||
IdentityVerificationRequired,
|
||||
Banned,
|
||||
/// 当前用量窗口已经耗尽,但存在自动恢复的重置时间。
|
||||
UsageLimitExhausted,
|
||||
RateLimited,
|
||||
QuotaExhausted,
|
||||
CloudflareChallenge,
|
||||
@@ -217,6 +219,7 @@ impl CodexUpstreamFailure {
|
||||
| CodexFailureCategory::CredentialExpired
|
||||
| CodexFailureCategory::IdentityVerificationRequired
|
||||
| CodexFailureCategory::Banned
|
||||
| CodexFailureCategory::UsageLimitExhausted
|
||||
| CodexFailureCategory::RateLimited
|
||||
| CodexFailureCategory::QuotaExhausted
|
||||
| CodexFailureCategory::CloudflareChallenge
|
||||
@@ -424,24 +427,36 @@ fn classify_upstream_failure(
|
||||
if status == Some(StatusCode::NOT_FOUND) && body.trim().is_empty() {
|
||||
return CodexFailureCategory::CloudflarePathBlocked;
|
||||
}
|
||||
if [code.as_str(), error_type.as_str()]
|
||||
.into_iter()
|
||||
.any(is_quota_signal)
|
||||
|| is_quota_message(&message)
|
||||
|| is_quota_message(&body)
|
||||
let structured_failure_signals = [code.as_str(), error_type.as_str()];
|
||||
if structured_failure_signals.into_iter().any(is_quota_signal)
|
||||
|| status == Some(StatusCode::PAYMENT_REQUIRED)
|
||||
{
|
||||
return CodexFailureCategory::QuotaExhausted;
|
||||
}
|
||||
if [code.as_str(), error_type.as_str()]
|
||||
if structured_failure_signals
|
||||
.into_iter()
|
||||
.any(is_usage_limit_signal)
|
||||
{
|
||||
return CodexFailureCategory::UsageLimitExhausted;
|
||||
}
|
||||
if structured_failure_signals
|
||||
.into_iter()
|
||||
.any(is_rate_limit_signal)
|
||||
|| is_rate_limit_message(&message)
|
||||
|| is_rate_limit_message(&body)
|
||||
|| status == Some(StatusCode::TOO_MANY_REQUESTS)
|
||||
{
|
||||
return CodexFailureCategory::RateLimited;
|
||||
}
|
||||
if is_usage_limit_message(&message) || is_usage_limit_message(&body) {
|
||||
return CodexFailureCategory::UsageLimitExhausted;
|
||||
}
|
||||
if is_quota_message(&message) || is_quota_message(&body) {
|
||||
return CodexFailureCategory::QuotaExhausted;
|
||||
}
|
||||
if is_rate_limit_message(&message) || is_rate_limit_message(&body) {
|
||||
return CodexFailureCategory::RateLimited;
|
||||
}
|
||||
if status == Some(StatusCode::TOO_MANY_REQUESTS) {
|
||||
return CodexFailureCategory::RateLimited;
|
||||
}
|
||||
if is_upstream_overload(&code) || is_upstream_overload(&message) {
|
||||
return CodexFailureCategory::Unavailable;
|
||||
}
|
||||
@@ -532,18 +547,22 @@ fn is_expired_credential(value: &str) -> bool {
|
||||
|| value.contains("invalid api key")
|
||||
}
|
||||
|
||||
fn is_rate_limit_signal(value: &str) -> bool {
|
||||
fn is_usage_limit_signal(value: &str) -> bool {
|
||||
matches!(
|
||||
value,
|
||||
"usage_limit_reached"
|
||||
| "rate_limit_exceeded"
|
||||
| "rate_limit_reached"
|
||||
| "rate_limit_error"
|
||||
| "workspace_owner_usage_limit_reached"
|
||||
| "workspace_member_usage_limit_reached"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_rate_limit_signal(value: &str) -> bool {
|
||||
matches!(
|
||||
value,
|
||||
"rate_limit_exceeded" | "rate_limit_reached" | "rate_limit_error"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_quota_signal(value: &str) -> bool {
|
||||
matches!(
|
||||
value,
|
||||
@@ -557,7 +576,11 @@ fn is_quota_signal(value: &str) -> bool {
|
||||
}
|
||||
|
||||
fn is_rate_limit_message(value: &str) -> bool {
|
||||
value.contains("rate limit") || value.contains("usage limit")
|
||||
value.contains("rate limit")
|
||||
}
|
||||
|
||||
fn is_usage_limit_message(value: &str) -> bool {
|
||||
value.contains("usage limit")
|
||||
}
|
||||
|
||||
fn is_quota_message(value: &str) -> bool {
|
||||
|
||||
@@ -298,10 +298,13 @@ fn model_matches_rule(model: &str, rule: &str) -> bool {
|
||||
.is_some_and(|suffix| matches!(suffix.as_bytes().first(), Some(b'-' | b'.' | b':')))
|
||||
}
|
||||
|
||||
fn normalize_service_tier(service_tier: Option<&str>) -> Option<String> {
|
||||
/// 规范化请求或响应携带的服务档位,供观测与计费共用。
|
||||
pub(crate) fn normalize_service_tier(service_tier: Option<&str>) -> Option<String> {
|
||||
service_tier
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.filter(|value| {
|
||||
!value.is_empty() && value.len() <= 64 && !value.chars().any(char::is_control)
|
||||
})
|
||||
.map(str::to_ascii_lowercase)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,10 @@ use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::executor::block_on;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountFeedbackStats, AccountSelectionPolicy,
|
||||
AccountStateChange, OpaqueProviderData, ProviderAccountId, ProviderAccountStore as _,
|
||||
QuotaObservation, RotationStrategy,
|
||||
AccountStateChange, ProviderAccountId, ProviderAccountStore as _, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ModelRequestId, RequestAttemptContext,
|
||||
@@ -24,7 +22,6 @@ use provider_openai::credential::{
|
||||
};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use secrecy::ExposeSecret;
|
||||
use serde_json::json;
|
||||
use url::Url;
|
||||
|
||||
use crate::support::{
|
||||
@@ -121,7 +118,7 @@ fn selector_with_runtime(
|
||||
os_version: "6.8".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "selector-contract".to_owned(),
|
||||
verified_at: Utc::now(),
|
||||
verified_at: chrono::Utc::now(),
|
||||
});
|
||||
let http = reqwest::Client::builder().build().expect("HTTP client");
|
||||
let agent_identity = agent_identity_service(store);
|
||||
@@ -731,7 +728,7 @@ fn credential_expired_failure_marks_unified_account_expired() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_marks_account_cooldown_with_a_cooldown_deadline() {
|
||||
fn rate_limited_failure_marks_quota_exhausted_with_a_reset_deadline() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
@@ -757,7 +754,7 @@ fn rate_limited_failure_marks_account_cooldown_with_a_cooldown_deadline() {
|
||||
.expect("record rate-limit failure");
|
||||
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
@@ -765,31 +762,47 @@ fn rate_limited_failure_marks_account_cooldown_with_a_cooldown_deadline() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_limit_exhaustion_marks_quota_exhausted_without_usage_probe() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
selector
|
||||
.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::UsageLimitExhausted {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("record usage-limit exhaustion");
|
||||
|
||||
let account = store.account("acct_primary").expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let reset_at = Utc::now().timestamp() + 24 * 60 * 60;
|
||||
block_on(
|
||||
store.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": reset_at}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
}),
|
||||
)
|
||||
.expect("persist exhausted quota");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("quota_exhausted".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
@@ -807,33 +820,54 @@ fn rate_limited_failure_does_not_downgrade_persisted_quota_exhaustion() {
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_ignores_exhaustion_after_its_reset_time() {
|
||||
fn rate_limited_failure_keeps_longer_existing_quota_deadline() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let existing_deadline = SystemTime::now() + Duration::from_secs(300);
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(existing_deadline),
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("mark exhausted");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
&account,
|
||||
CodexAccountFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(30)),
|
||||
},
|
||||
))
|
||||
.expect("record shorter rate-limit failure");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.cooldown_until(),
|
||||
Some(existing_deadline)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_does_not_consult_stale_quota_snapshot() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
let reset_at = Utc::now().timestamp() - 1;
|
||||
block_on(
|
||||
store.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
quota: Some(OpaqueProviderData::new(
|
||||
json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": reset_at}
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.expect("quota object")
|
||||
.clone(),
|
||||
)),
|
||||
observed_at: Some(SystemTime::now()),
|
||||
}),
|
||||
)
|
||||
.expect("persist expired quota exhaustion");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_failure(
|
||||
@@ -849,12 +883,13 @@ fn rate_limited_failure_ignores_exhaustion_after_its_reset_time() {
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::Cooldown
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert_eq!(store.quota_reads(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failure_preserves_terminal_account_state() {
|
||||
fn authenticated_rate_limit_recovers_stale_authentication_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
@@ -883,10 +918,45 @@ fn rate_limited_failure_preserves_terminal_account_state() {
|
||||
.account("acct_primary")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::Invalid
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_upstream_response_recovers_non_quota_terminal_states() {
|
||||
for stale in [
|
||||
AccountAvailability::Expired,
|
||||
AccountAvailability::Invalid,
|
||||
AccountAvailability::Banned,
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_primary", "at-primary");
|
||||
let account = store.account("acct_primary").expect("account");
|
||||
block_on(store.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: stale,
|
||||
reason: Some("stale_observation".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
}))
|
||||
.expect("seed stale state");
|
||||
let current = store.account("acct_primary").expect("stale account");
|
||||
let selector = selector(&store, Arc::new(TestLeaseCoordinator::default()));
|
||||
|
||||
block_on(selector.record_success(¤t, None));
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_primary")
|
||||
.expect("recovered account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready,
|
||||
"stale state {stale:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -926,7 +996,7 @@ fn rate_limited_failures_for_distinct_accounts_do_not_conflict() {
|
||||
.expect("persisted second account")
|
||||
.availability(),
|
||||
],
|
||||
[AccountAvailability::Cooldown; 2]
|
||||
[AccountAvailability::QuotaExhausted; 2]
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -600,10 +600,10 @@ async fn reauthorization_uses_revision_advanced_after_start_for_prepared_rotatio
|
||||
);
|
||||
}
|
||||
|
||||
struct RebindingVerifier;
|
||||
struct ReplacementIdentityVerifier;
|
||||
|
||||
#[async_trait]
|
||||
impl CodexAccountIdentityVerifier for RebindingVerifier {
|
||||
impl CodexAccountIdentityVerifier for ReplacementIdentityVerifier {
|
||||
async fn verify(
|
||||
&self,
|
||||
_: &CodexOAuthSecret,
|
||||
@@ -617,8 +617,9 @@ impl CodexAccountIdentityVerifier for RebindingVerifier {
|
||||
_: &CodexOAuthSecret,
|
||||
_: &SecretString,
|
||||
_: &SecretString,
|
||||
_: &CodexIdentityExpectation,
|
||||
expectation: &CodexIdentityExpectation,
|
||||
) -> Result<CodexIdentityVerification, CodexIdentityVerificationError> {
|
||||
assert_eq!(expectation, &CodexIdentityExpectation::default());
|
||||
Ok(CodexIdentityVerification::Complete(profile(
|
||||
"chatgpt-different-owner",
|
||||
)))
|
||||
@@ -699,7 +700,7 @@ async fn reauthorization_updates_changed_free_principal_for_same_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reauthorization_rejects_identity_rebinding() {
|
||||
async fn reauthorization_rebinds_target_record_to_authorized_identity() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
@@ -711,7 +712,7 @@ async fn reauthorization_rejects_identity_rebinding() {
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let service = service_with_store(store, Arc::new(RebindingVerifier));
|
||||
let service = service_with_store(store, Arc::new(ReplacementIdentityVerifier));
|
||||
let started = service
|
||||
.start_authorization(StartCodexOAuthAuthorization {
|
||||
mutation: reauthorization_mutation("request-rebind", "acct_oauth_rebind"),
|
||||
@@ -723,7 +724,7 @@ async fn reauthorization_rejects_identity_rebinding() {
|
||||
.query_pairs()
|
||||
.find_map(|(key, value)| (key == "state").then(|| value.into_owned()))
|
||||
.expect("state");
|
||||
let error = service
|
||||
let completed = service
|
||||
.complete_authorization(CompleteCodexOAuthAuthorization {
|
||||
owner_ref: owner_ref(),
|
||||
flow_id: started.flow_id,
|
||||
@@ -732,6 +733,24 @@ async fn reauthorization_rejects_identity_rebinding() {
|
||||
)),
|
||||
})
|
||||
.await
|
||||
.expect_err("identity rebind");
|
||||
assert_eq!(error, CodexOAuthAdminError::Credential);
|
||||
.expect("complete identity replacement");
|
||||
let CompletedCodexOAuthCredential::Reauthorize(prepared) = completed.credential else {
|
||||
panic!("expected prepared reauthorization");
|
||||
};
|
||||
assert_eq!(
|
||||
prepared.credential.account_id().as_str(),
|
||||
"acct_oauth_rebind"
|
||||
);
|
||||
let replacement = prepared
|
||||
.replacement_identity
|
||||
.as_ref()
|
||||
.expect("replacement identity");
|
||||
assert_eq!(
|
||||
replacement.upstream_account_id(),
|
||||
Some("chatgpt-different-owner")
|
||||
);
|
||||
assert_eq!(
|
||||
replacement.upstream_user_id(),
|
||||
"user-chatgpt-different-owner"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -55,6 +55,36 @@ fn parser_treats_any_confirmed_provider_limit_as_exhausted() {
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_keeps_full_percent_as_display_without_provider_exhaustion_signal() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid full-percent quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(0));
|
||||
assert!(!fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_treats_confirmed_limit_reached_as_exhausted_even_below_full_percent() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 98, "reset_at": 1_900_000_000}
|
||||
}
|
||||
}))
|
||||
.expect("valid provider-confirmed quota");
|
||||
|
||||
assert_eq!(fact.remaining_percent(), Some(2));
|
||||
assert!(fact.exhausted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parser_does_not_infer_exhaustion_from_unknown_credit_fields() {
|
||||
let fact = parse_codex_quota_usage(&json!({
|
||||
@@ -181,6 +211,14 @@ fn blocked_network_quota_service(store: &Arc<MemoryAccountStore>) -> CodexCreden
|
||||
}
|
||||
|
||||
async fn create_account(store: &Arc<MemoryAccountStore>, account_id: &str) {
|
||||
create_account_with_enabled(store, account_id, true).await;
|
||||
}
|
||||
|
||||
async fn create_account_with_enabled(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
account_id: &str,
|
||||
enabled: bool,
|
||||
) {
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: account_id.to_owned(),
|
||||
@@ -188,7 +226,7 @@ async fn create_account(store: &Arc<MemoryAccountStore>, account_id: &str) {
|
||||
secret: secret(&format!("token-{account_id}")),
|
||||
verified_account: profile(&format!("chatgpt-{account_id}")),
|
||||
next_refresh_at: Some(Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
enabled,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -361,12 +399,66 @@ async fn passive_rate_limit_headers_update_quota_and_account_state_with_revision
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn passive_full_percent_with_explicit_allowance_keeps_account_ready() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_passive_full_allowed").await;
|
||||
let account = store.account("acct_passive_full_allowed").expect("account");
|
||||
let service = quota_service(&store);
|
||||
let headers = vec![
|
||||
("x-codex-active-limit".to_owned(), "codex".to_owned()),
|
||||
("x-codex-primary-used-percent".to_owned(), "100".to_owned()),
|
||||
(
|
||||
"x-codex-primary-reset-at".to_owned(),
|
||||
"1900000000".to_owned(),
|
||||
),
|
||||
("x-codex-allowed".to_owned(), "true".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "false".to_owned()),
|
||||
];
|
||||
|
||||
assert!(
|
||||
service
|
||||
.synchronize_passive_headers(&account, &headers)
|
||||
.await
|
||||
.expect("passive quota")
|
||||
);
|
||||
let current = store
|
||||
.account("acct_passive_full_allowed")
|
||||
.expect("current account");
|
||||
let snapshot = service
|
||||
.read_account(account.id())
|
||||
.await
|
||||
.expect("read quota")
|
||||
.expect("quota snapshot");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
current.availability(),
|
||||
current.cooldown_until(),
|
||||
snapshot.fact().remaining_percent(),
|
||||
snapshot.fact().exhausted(),
|
||||
),
|
||||
(AccountAvailability::Ready, None, Some(0), false)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_updates_account_state_from_provider_response() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_manual_quota_state";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
reason: Some("stale_expired_state".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed stale expired state");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
@@ -438,6 +530,61 @@ async fn manual_quota_refresh_updates_account_state_from_provider_response() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_refresh_preserves_disabled_account_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_disabled_quota_state";
|
||||
create_account_with_enabled(&store, account_id, false).await;
|
||||
let account = store.account(account_id).expect("created disabled account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::Expired,
|
||||
reason: Some("disabled_expired_state".to_owned()),
|
||||
cooldown_until: None,
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("seed disabled account state");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header(
|
||||
"authorization",
|
||||
format!("Bearer token-{account_id}"),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": true,
|
||||
"limit_reached": false,
|
||||
"primary_window": {"used_percent": 20, "reset_at": 1_900_000_000}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("refresh disabled account quota");
|
||||
|
||||
let current = store
|
||||
.account(account_id)
|
||||
.expect("disabled account after quota refresh");
|
||||
assert!(!current.enabled());
|
||||
assert_eq!(current.availability(), AccountAvailability::Expired);
|
||||
assert!(store.has_quota(account_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_quota_auth_rejection_does_not_invalidate_refreshable_credential() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -697,12 +844,18 @@ async fn periodic_quota_synchronization_repairs_cooldown_with_active_exhausted_q
|
||||
|
||||
service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
let reconciled = store
|
||||
.account("acct_periodic_stuck_cooldown")
|
||||
.expect("reconciled account");
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_periodic_stuck_cooldown")
|
||||
.expect("reconciled account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
(reconciled.availability(), reconciled.cooldown_until()),
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some(
|
||||
SystemTime::UNIX_EPOCH
|
||||
+ Duration::from_secs(u64::try_from(reset_at).expect("positive reset")),
|
||||
),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -741,12 +894,14 @@ async fn periodic_quota_synchronization_does_not_downgrade_exhaustion_on_rate_li
|
||||
|
||||
service.synchronize().await.expect("quota synchronization");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_periodic_exhausted")
|
||||
.expect("persisted account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
let account = store
|
||||
.account("acct_periodic_exhausted")
|
||||
.expect("persisted account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -877,6 +1032,60 @@ async fn periodic_quota_synchronization_attempts_quota_exhausted_accounts() {
|
||||
assert_eq!(summary.transient, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_waits_for_quota_cooldown_deadline_with_grace() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_periodic_exhausted_cooldown").await;
|
||||
let account = store
|
||||
.account("acct_periodic_exhausted_cooldown")
|
||||
.expect("created account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_sub(Duration::from_secs(29)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted with recent cooldown");
|
||||
let service = blocked_network_quota_service(&store);
|
||||
|
||||
let waiting = service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("quota synchronization before grace");
|
||||
|
||||
assert_eq!(waiting, CodexQuotaSyncSummary::default());
|
||||
let current = store
|
||||
.account("acct_periodic_exhausted_cooldown")
|
||||
.expect("current account");
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: current.id().clone(),
|
||||
expected_revision: current.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: SystemTime::now().checked_sub(Duration::from_secs(31)),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted past grace");
|
||||
|
||||
let due = service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("quota synchronization after grace");
|
||||
let repeated = service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("quota synchronization repeated");
|
||||
|
||||
assert_eq!(due.transient, 1);
|
||||
assert_eq!(repeated, CodexQuotaSyncSummary::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_throttles_the_same_exhausted_account() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -955,8 +1164,8 @@ async fn periodic_quota_synchronization_rechecks_an_exhausted_account_once_when_
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let first = service.synchronize().await.expect("initial refresh");
|
||||
assert_eq!(first.transient, 1);
|
||||
let waiting = service.synchronize().await.expect("initial cooldown wait");
|
||||
assert_eq!(waiting, CodexQuotaSyncSummary::default());
|
||||
store
|
||||
.compare_and_swap_quota(QuotaObservation {
|
||||
account_id: account.id().clone(),
|
||||
@@ -987,6 +1196,77 @@ async fn periodic_quota_synchronization_rechecks_an_exhausted_account_once_when_
|
||||
assert_eq!(repeated, CodexQuotaSyncSummary::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_quota_synchronization_reschedules_a_confirmed_new_reset() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_periodic_new_reset";
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("created account");
|
||||
let initial_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(1);
|
||||
let confirmed_reset = SystemTime::UNIX_EPOCH + Duration::from_secs(2);
|
||||
store
|
||||
.apply_state_change(AccountStateChange {
|
||||
account_id: account.id().clone(),
|
||||
expected_revision: account.revision(),
|
||||
availability: AccountAvailability::QuotaExhausted,
|
||||
reason: Some("usage_limit_exhausted".to_owned()),
|
||||
cooldown_until: Some(initial_reset),
|
||||
observed_at: SystemTime::now(),
|
||||
})
|
||||
.await
|
||||
.expect("mark account exhausted with due reset");
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"rate_limit": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary_window": {"used_percent": 100, "reset_at": 2}
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let service = quota_service_with_base_url(
|
||||
&store,
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client"),
|
||||
server.uri(),
|
||||
);
|
||||
|
||||
let first = service.synchronize().await.expect("first reset recheck");
|
||||
let persisted_reset = store
|
||||
.account(account_id)
|
||||
.expect("account after first recheck")
|
||||
.cooldown_until();
|
||||
let second = service.synchronize().await.expect("new reset recheck");
|
||||
let repeated = service.synchronize().await.expect("same reset throttled");
|
||||
let request_count = server
|
||||
.received_requests()
|
||||
.await
|
||||
.expect("received requests")
|
||||
.len();
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
first.exhausted,
|
||||
persisted_reset,
|
||||
second.exhausted,
|
||||
repeated,
|
||||
request_count,
|
||||
),
|
||||
(
|
||||
1,
|
||||
Some(confirmed_reset),
|
||||
1,
|
||||
CodexQuotaSyncSummary::default(),
|
||||
2,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_quota_json_for_one_account_does_not_abort_the_batch() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -665,6 +665,74 @@ async fn expired_access_token_refreshes_even_when_next_refresh_is_deferred() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_access_token_refreshes_during_active_cooldown() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
seed_due_account(&store).await;
|
||||
let original = store.account("acct_refresh").expect("seeded account");
|
||||
let mut expired_profile = profile("chatgpt-acct_refresh");
|
||||
expired_profile.access_token_expires_at =
|
||||
Some(chrono::Utc::now() - chrono::Duration::minutes(5));
|
||||
store
|
||||
.repository()
|
||||
.rotate_oauth_secret(RotateCodexCredential {
|
||||
account_id: "acct_refresh".to_owned(),
|
||||
expected_credential_revision: original.revision().get(),
|
||||
secret: secret("old-access"),
|
||||
verified_account: expired_profile,
|
||||
next_refresh_at: Some(chrono::Utc::now() + chrono::Duration::hours(12)),
|
||||
})
|
||||
.await
|
||||
.expect("seed expired access token with deferred refresh");
|
||||
let deferred = store.account("acct_refresh").expect("deferred account");
|
||||
let cooldown_until = SystemTime::now() + Duration::from_secs(12 * 60 * 60);
|
||||
store
|
||||
.repository()
|
||||
.apply_state(
|
||||
&deferred,
|
||||
AccountAvailability::Cooldown,
|
||||
Some("rate_limited".to_owned()),
|
||||
Some(cooldown_until),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await
|
||||
.expect("mark account cooling down");
|
||||
let refresher = Arc::new(Refresher::new(Ok(success_tokens())));
|
||||
let service = CodexCredentialRefreshService::new(
|
||||
store.repository(),
|
||||
refresher.clone(),
|
||||
Arc::new(VerifiedIdentity),
|
||||
Arc::new(RefreshLeases {
|
||||
available: true,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
}),
|
||||
Arc::new(CountingCredentialState::default()),
|
||||
runtime_policy(),
|
||||
);
|
||||
|
||||
let outcomes = service.refresh_due().await.expect("refresh due");
|
||||
|
||||
assert!(matches!(
|
||||
outcomes.as_slice(),
|
||||
[CodexCredentialRefreshOutcome::Refreshed {
|
||||
account_id,
|
||||
credential_revision: 3,
|
||||
}] if account_id == "acct_refresh"
|
||||
));
|
||||
assert_eq!(
|
||||
refresher.seen.lock().expect("seen tokens lock").as_slice(),
|
||||
["rt-old-access"]
|
||||
);
|
||||
let refreshed = store.account("acct_refresh").expect("refreshed account");
|
||||
assert_eq!(refreshed.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(refreshed.cooldown_until(), Some(cooldown_until));
|
||||
assert!(
|
||||
refreshed
|
||||
.access_token_expires_at()
|
||||
.is_some_and(|expires_at| expires_at > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unavailable_refresh_lease_prevents_duplicate_token_exchange() {
|
||||
let (store, refresher, service) = setup(Ok(success_tokens()), false).await;
|
||||
|
||||
@@ -7,7 +7,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountFeedbackStats, ProviderAccountId,
|
||||
AccountAvailability, AccountFeedbackStats, ProviderAccountId, ProviderAccountStore as _,
|
||||
};
|
||||
use gateway_core::engine::provider::{Provider as _, ProviderRequest};
|
||||
use gateway_core::engine::{
|
||||
@@ -29,14 +29,14 @@ use provider_openai::credential::{
|
||||
use provider_openai::transport::CodexWebSocketPool;
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use provider_openai::{CodexProvider, OFFICIAL_CODEX_BASE_URL};
|
||||
use serde_json::{Map, json};
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::oneshot,
|
||||
time::timeout,
|
||||
};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::support::{
|
||||
@@ -827,13 +827,23 @@ async fn completed_response_persists_session_affinity_before_stream_consumer_sto
|
||||
.expect("prepare provider stream");
|
||||
|
||||
let mut observed_service_tier = None;
|
||||
let mut upstream_service_tier = None;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("provider event");
|
||||
if let Some(service_tier) = event
|
||||
.response_observation()
|
||||
.and_then(|observation| observation.service_tier())
|
||||
{
|
||||
observed_service_tier = Some(service_tier.to_owned());
|
||||
if let Some(observation) = event.response_observation() {
|
||||
if let Some(service_tier) = observation.service_tier() {
|
||||
observed_service_tier = Some(service_tier.to_owned());
|
||||
}
|
||||
upstream_service_tier = observation
|
||||
.provider_metadata()
|
||||
.and_then(|metadata| serde_json::from_str::<Value>(metadata.as_json()).ok())
|
||||
.and_then(|metadata| {
|
||||
metadata
|
||||
.get("upstreamServiceTier")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.or(upstream_service_tier);
|
||||
}
|
||||
if event
|
||||
.canonical_facts()
|
||||
@@ -846,11 +856,12 @@ async fn completed_response_persists_session_affinity_before_stream_consumer_sto
|
||||
drop(stream);
|
||||
|
||||
assert_eq!(affinity.binding_count(), 1);
|
||||
assert_eq!(observed_service_tier.as_deref(), Some("default"));
|
||||
assert_eq!(observed_service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(upstream_service_tier.as_deref(), Some("default"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_failed_before_semantic_output_is_atomic_and_persists_cooldown() {
|
||||
async fn response_failed_before_semantic_output_is_atomic_and_persists_quota_lock() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_atomic_failure";
|
||||
create_account(&store, account_id).await;
|
||||
@@ -895,7 +906,7 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_cooldown(
|
||||
vec!["response.created", "response.failed"]
|
||||
);
|
||||
let account = store.account(account_id).expect("rate-limited account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
@@ -906,6 +917,68 @@ async fn response_failed_before_semantic_output_is_atomic_and_persists_cooldown(
|
||||
let _ = server.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_limit_failure_marks_quota_exhausted_without_usage_probe() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_usage_limit_request_path";
|
||||
create_account(&store, account_id).await;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.and(header("authorization", format!("Bearer at-{account_id}")))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(0)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_usage_limit_confirm\",\"model\":\"gpt-5.4\",\"status\":\"in_progress\"}}\n\n",
|
||||
"event: response.failed\n",
|
||||
"data: {\"type\":\"response.failed\",\"status_code\":429,\"retry_after_seconds\":86400,\"response\":{\"id\":\"resp_usage_limit_confirm\",\"status\":\"failed\",\"error\":{\"code\":\"usage_limit_reached\",\"message\":\"usage limit reached\"}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
context("req_usage_limit_confirm", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
let failure = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("usage-limit failure must surface a typed failure"),
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(failure.kind(), ProviderErrorKind::QuotaExhausted);
|
||||
let account = store.account(account_id).expect("usage-limit account");
|
||||
assert_eq!(account.availability(), AccountAvailability::QuotaExhausted);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert!(!store.has_quota(account_id));
|
||||
let requests = server.received_requests().await.expect("received requests");
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/api/codex/usage")
|
||||
.count(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_request_should_forward_created_before_later_failed_chunk() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1207,8 +1280,8 @@ async fn response_failed_after_semantic_output_is_exposed_and_not_replay_safe()
|
||||
async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_state() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_disabled_diagnostic";
|
||||
create_account_with_enabled(&store, account_id, false).await;
|
||||
let account = store.account(account_id).expect("disabled test account");
|
||||
create_account(&store, account_id).await;
|
||||
let account = store.account(account_id).expect("test account");
|
||||
store
|
||||
.repository()
|
||||
.apply_state(
|
||||
@@ -1220,6 +1293,10 @@ async fn disabled_account_diagnostic_uses_upstream_without_persisting_account_st
|
||||
)
|
||||
.await
|
||||
.expect("seed quota-exhausted state");
|
||||
store
|
||||
.set_enabled(account.id(), false)
|
||||
.await
|
||||
.expect("disable test account");
|
||||
|
||||
let affinity = Arc::new(MemorySessionAffinity::default());
|
||||
let server = MockServer::start().await;
|
||||
@@ -1323,6 +1400,76 @@ async fn successful_response_preserves_quota_exhaustion_from_rate_limit_headers(
|
||||
assert!(store.has_quota(account_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_http_sse_rate_limit_event_persists_structured_exhaustion() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = "acct_http_sse_exhausted";
|
||||
create_account(&store, account_id).await;
|
||||
let reset_at = 1_900_000_000_u64;
|
||||
let rate_limit_event = json!({
|
||||
"type": "codex.rate_limits",
|
||||
"rate_limits": {
|
||||
"allowed": false,
|
||||
"limit_reached": true,
|
||||
"primary": {
|
||||
"used_percent": 42,
|
||||
"window_minutes": 300,
|
||||
"reset_at": reset_at,
|
||||
},
|
||||
},
|
||||
});
|
||||
let completed_event = json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_http_sse_exhausted",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
});
|
||||
let body = format!(
|
||||
"event: codex.rate_limits\ndata: {rate_limit_event}\n\nevent: response.completed\ndata: {completed_event}\n\n"
|
||||
);
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(body),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
context("req_http_sse_exhausted", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("successful upstream response");
|
||||
}
|
||||
|
||||
let account = store
|
||||
.account(account_id)
|
||||
.expect("account after HTTP SSE response");
|
||||
assert_eq!(
|
||||
(
|
||||
account.availability(),
|
||||
account.cooldown_until(),
|
||||
store.has_quota(account_id),
|
||||
),
|
||||
(
|
||||
AccountAvailability::QuotaExhausted,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(reset_at)),
|
||||
true,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_observation_reads_openai_metadata_without_changing_the_operation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -203,18 +203,40 @@ fn decoder_should_emit_calculated_cost_for_complete_known_model_usage() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_keep_response_service_tier_when_terminal_event_omits_it() {
|
||||
fn decoder_should_prefer_requested_service_tier_over_response_tier_for_billing() {
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_fast_cost\",\"model\":\"gpt-5.4\",\"service_tier\":\"priority\"}}\n\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_fast_cost\",\"model\":\"gpt-5.4\",\"service_tier\":\"default\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fast_cost\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"usage\":{\"input_tokens\":100,\"output_tokens\":10,\"input_tokens_details\":{\"cached_tokens\":25,\"cache_write_tokens\":0},\"total_tokens\":110}}}\n\n",
|
||||
);
|
||||
let mut decoder = CodexCanonicalDecoder::new("fallback");
|
||||
let mut decoder =
|
||||
CodexCanonicalDecoder::new("fallback").with_requested_service_tier(Some("priority"));
|
||||
let events = decoder
|
||||
.push(body.as_bytes())
|
||||
.expect("canonical priority response");
|
||||
|
||||
assert_eq!(decoder.response_service_tier(), Some("default"));
|
||||
assert!(canonical_facts(&events).into_iter().any(|event| matches!(
|
||||
event,
|
||||
GatewayEvent::CalculatedCost(cost)
|
||||
if cost.total().amount().scaled() == 6_875_000
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_use_response_service_tier_when_request_omits_it() {
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_response_tier_cost\",\"model\":\"gpt-5.4\",\"service_tier\":\"priority\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_response_tier_cost\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"usage\":{\"input_tokens\":100,\"output_tokens\":10,\"input_tokens_details\":{\"cached_tokens\":25,\"cache_write_tokens\":0},\"total_tokens\":110}}}\n\n",
|
||||
);
|
||||
let mut decoder = CodexCanonicalDecoder::new("fallback");
|
||||
let events = decoder
|
||||
.push(body.as_bytes())
|
||||
.expect("canonical response-tier priority response");
|
||||
|
||||
assert_eq!(decoder.response_service_tier(), Some("priority"));
|
||||
assert!(canonical_facts(&events).into_iter().any(|event| matches!(
|
||||
event,
|
||||
@@ -223,6 +245,31 @@ fn decoder_should_keep_response_service_tier_when_terminal_event_omits_it() {
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_decoder_should_prefer_requested_service_tier_over_response_tier_for_billing() {
|
||||
let created = websocket_event_to_sse_frame(
|
||||
r#"{"type":"response.created","response":{"id":"resp_ws_fast_cost","model":"gpt-5.4","service_tier":"default"}}"#,
|
||||
)
|
||||
.expect("created frame");
|
||||
let completed = websocket_event_to_sse_frame(
|
||||
r#"{"type":"response.completed","response":{"id":"resp_ws_fast_cost","model":"gpt-5.4","status":"completed","usage":{"input_tokens":100,"output_tokens":10,"input_tokens_details":{"cached_tokens":25,"cache_write_tokens":0},"total_tokens":110}}}"#,
|
||||
)
|
||||
.expect("completed frame");
|
||||
let mut decoder = CodexCanonicalDecoder::new("fallback")
|
||||
.with_requested_service_tier(Some("priority"))
|
||||
.with_raw_sse_passthrough();
|
||||
let events = decoder
|
||||
.push(format!("{created}{completed}").as_bytes())
|
||||
.expect("canonical priority WebSocket response");
|
||||
|
||||
assert_eq!(decoder.response_service_tier(), Some("default"));
|
||||
assert!(canonical_facts(&events).into_iter().any(|event| matches!(
|
||||
event,
|
||||
GatewayEvent::CalculatedCost(cost)
|
||||
if cost.total().amount().scaled() == 6_875_000
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_normalize_text_usage_and_completion() {
|
||||
let body = concat!(
|
||||
|
||||
@@ -156,7 +156,93 @@ fn structured_quota_signal_takes_precedence_over_http_429() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structured_usage_limit_type_is_resettable_quota_exhaustion() {
|
||||
assert_eq!(
|
||||
classify(
|
||||
429,
|
||||
r#"{"error":{"code":null,"type":"usage_limit_reached","message":"The usage limit has been reached"}}"#
|
||||
),
|
||||
CodexFailureCategory::UsageLimitExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_limit_message_without_a_type_is_resettable_quota_exhaustion() {
|
||||
assert_eq!(
|
||||
classify(
|
||||
429,
|
||||
r#"{"error":{"message":"The usage limit has been reached"}}"#
|
||||
),
|
||||
CodexFailureCategory::UsageLimitExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bare_http_429_remains_a_temporary_rate_limit() {
|
||||
assert_eq!(classify(429, ""), CodexFailureCategory::RateLimited);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub2api_openai_account_failure_matrix_is_preserved() {
|
||||
let cases = [
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"message":"Organization has been disabled"}}"#,
|
||||
CodexFailureCategory::Banned,
|
||||
),
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"message":"Identity verification is required"}}"#,
|
||||
CodexFailureCategory::IdentityVerificationRequired,
|
||||
),
|
||||
(
|
||||
400,
|
||||
r#"{"error":{"message":"invalid request payload"}}"#,
|
||||
CodexFailureCategory::InvalidRequest,
|
||||
),
|
||||
(
|
||||
401,
|
||||
r#"{"error":{"code":"token_revoked","message":"token has been revoked"}}"#,
|
||||
CodexFailureCategory::CredentialExpired,
|
||||
),
|
||||
(
|
||||
401,
|
||||
r#"{"detail":"Unauthorized"}"#,
|
||||
CodexFailureCategory::CredentialExpired,
|
||||
),
|
||||
(
|
||||
402,
|
||||
r#"{"detail":{"code":"deactivated_workspace","message":"workspace disabled"}}"#,
|
||||
CodexFailureCategory::Banned,
|
||||
),
|
||||
(
|
||||
402,
|
||||
r#"{"error":{"message":"payment required"}}"#,
|
||||
CodexFailureCategory::QuotaExhausted,
|
||||
),
|
||||
(
|
||||
403,
|
||||
r#"{"error":{"message":"access forbidden"}}"#,
|
||||
CodexFailureCategory::PermissionDenied,
|
||||
),
|
||||
(
|
||||
429,
|
||||
r#"{"error":{"type":"usage_limit_reached","message":"quota window reached"}}"#,
|
||||
CodexFailureCategory::UsageLimitExhausted,
|
||||
),
|
||||
(
|
||||
529,
|
||||
r#"{"error":{"type":"server_error","message":"overloaded"}}"#,
|
||||
CodexFailureCategory::Unavailable,
|
||||
),
|
||||
];
|
||||
|
||||
for (status, body, expected) in cases {
|
||||
assert_eq!(
|
||||
classify(status, body),
|
||||
expected,
|
||||
"status={status} body={body}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,6 +269,54 @@ async fn codex_backend_client_should_parse_retry_after_from_rate_limit_error_bod
|
||||
assert_eq!(retry_after_seconds, Some(12));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn codex_backend_http_sse_should_capture_structured_rate_limit_event_updates() {
|
||||
let server = wiremock::MockServer::start().await;
|
||||
wiremock::Mock::given(wiremock::matchers::method("POST"))
|
||||
.and(wiremock::matchers::path("/codex/responses"))
|
||||
.respond_with(
|
||||
wiremock::ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(concat!(
|
||||
"event: codex.rate_limits\n",
|
||||
"data: {\"type\":\"codex.rate_limits\",\"rate_limits\":{\"allowed\":true,\"limit_reached\":false,\"primary\":{\"used_percent\":100,\"window_minutes\":300,\"reset_at\":1893456300}}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_http_rate_limits\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
|
||||
)),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
server.uri(),
|
||||
test_wire_profile(),
|
||||
);
|
||||
let mut request = codex_request("gpt-5.5", "", Vec::new());
|
||||
request.force_http_sse = true;
|
||||
|
||||
let response = client
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_http_rate_limit_event", Some("acct")),
|
||||
)
|
||||
.await
|
||||
.expect("HTTP SSE response");
|
||||
let structured = response
|
||||
.rate_limit_headers
|
||||
.iter()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "x-codex-allowed" | "x-codex-limit-reached"))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
structured,
|
||||
vec![
|
||||
("x-codex-allowed".to_owned(), "true".to_owned()),
|
||||
("x-codex-limit-reached".to_owned(), "false".to_owned()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn codex_backend_http_sse_should_fail_after_five_minutes_without_stream_data() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -942,6 +942,7 @@ fn prepared_rotation(
|
||||
provider_kind,
|
||||
expected_credential_revision: Revision::new(expected_revision.get())
|
||||
.map_err(|_| provider_error(ProviderAdminErrorKind::Internal))?,
|
||||
replacement_identity: None,
|
||||
name: profile.name,
|
||||
email: profile.email,
|
||||
plan_type: profile.plan_type,
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use chrono::Utc;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAvailability, AccountAvailabilityPolicy, AccountCandidate, AccountFeedbackStats,
|
||||
AccountSelectionContext, AccountSelector, ProviderAccount, ProviderAccountId,
|
||||
@@ -17,7 +17,7 @@ use gateway_core::provider_ports::{
|
||||
use gateway_core::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
use super::catalog::{GrokCatalogScope, GrokCredentialCatalogCache, GrokCredentialQuotaService};
|
||||
use super::repository::GrokCredentialRepository;
|
||||
use super::repository::{GrokCredentialRepository, GrokCredentialRepositoryError};
|
||||
use super::types::{GrokCredentialAvailability, UpdateGrokCredentialState};
|
||||
use crate::{
|
||||
GrokCredentialFailure, GrokCredentialFeedbackFuture, GrokSessionBinding, GrokSessionSelection,
|
||||
@@ -29,7 +29,6 @@ const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const MAX_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const STREAM_INTERRUPTION_COOLDOWN: Duration = Duration::from_secs(30);
|
||||
const UNAUTHORIZED_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const PAYMENT_REQUIRED_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const MODEL_ACCESS_DENIED_COOLDOWN: Duration = Duration::from_secs(5 * 60);
|
||||
const MODEL_QUOTA_COOLDOWN: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const MAX_MODEL_COOLDOWN: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
@@ -369,6 +368,66 @@ impl GrokAccountSessionSelector {
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn persist_account_failure(
|
||||
&self,
|
||||
session: &SelectedGrokSession,
|
||||
availability: GrokCredentialAvailability,
|
||||
reason: &'static str,
|
||||
cooldown_until: Option<DateTime<Utc>>,
|
||||
observed_at: chrono::DateTime<Utc>,
|
||||
) {
|
||||
let update = |expected_revision| UpdateGrokCredentialState {
|
||||
account_id: session.account_id().clone(),
|
||||
expected_revision,
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
};
|
||||
match self
|
||||
.repository
|
||||
.update_state(&update(session.credential_revision()))
|
||||
.await
|
||||
{
|
||||
Ok(()) => return,
|
||||
Err(
|
||||
GrokCredentialRepositoryError::Conflict
|
||||
| GrokCredentialRepositoryError::StaleCredentialRevision,
|
||||
) => {}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %session.account_id(),
|
||||
error = %error,
|
||||
"xAI account failure state write failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let current = match self.repository.load_current(session.account_id()).await {
|
||||
Ok(current) if current.account.enabled() => current,
|
||||
Ok(_) => return,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %session.account_id(),
|
||||
error = %error,
|
||||
"xAI account failure state reload failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Err(error) = self
|
||||
.repository
|
||||
.update_state(&update(current.account.revision()))
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
account_id = %session.account_id(),
|
||||
error = %error,
|
||||
"xAI account failure state retry failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for GrokAccountSessionSelector {
|
||||
@@ -415,15 +474,11 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
"upstream_free_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::PaymentRequired { retry_after } => {
|
||||
let retry_after = bounded_cooldown(
|
||||
retry_after,
|
||||
PAYMENT_REQUIRED_COOLDOWN,
|
||||
MAX_RATE_LIMIT_COOLDOWN,
|
||||
);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
return;
|
||||
}
|
||||
GrokCredentialFailure::PaymentRequired { .. } => (
|
||||
GrokCredentialAvailability::QuotaExhausted,
|
||||
"upstream_payment_quota_exhausted",
|
||||
None,
|
||||
),
|
||||
GrokCredentialFailure::ModelQuotaExhausted {
|
||||
upstream_model,
|
||||
retry_after,
|
||||
@@ -452,26 +507,31 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
.unwrap_or(DEFAULT_RATE_LIMIT_COOLDOWN)
|
||||
.min(MAX_RATE_LIMIT_COOLDOWN);
|
||||
self.record_runtime_cooldown(session, retry_after).await;
|
||||
return;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"upstream_rate_limited",
|
||||
persistent_cooldown_until(observed_at, retry_after),
|
||||
)
|
||||
}
|
||||
GrokCredentialFailure::StreamInterrupted => {
|
||||
self.record_runtime_cooldown(session, STREAM_INTERRUPTION_COOLDOWN)
|
||||
.await;
|
||||
return;
|
||||
(
|
||||
GrokCredentialAvailability::Cooldown,
|
||||
"upstream_stream_interrupted",
|
||||
persistent_cooldown_until(observed_at, STREAM_INTERRUPTION_COOLDOWN),
|
||||
)
|
||||
}
|
||||
};
|
||||
let (availability, reason, cooldown_until) = persistent;
|
||||
let _ = self
|
||||
.repository
|
||||
.update_state(&UpdateGrokCredentialState {
|
||||
account_id: session.account_id().clone(),
|
||||
expected_revision: session.credential_revision(),
|
||||
availability,
|
||||
availability_reason: Some(reason.to_owned()),
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
})
|
||||
.await;
|
||||
self.persist_account_failure(
|
||||
session,
|
||||
availability,
|
||||
reason,
|
||||
cooldown_until,
|
||||
observed_at,
|
||||
)
|
||||
.await;
|
||||
})
|
||||
}
|
||||
|
||||
@@ -493,7 +553,6 @@ impl GrokSessionSelector for GrokAccountSessionSelector {
|
||||
account.availability(),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::Cooldown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
)
|
||||
{
|
||||
@@ -540,6 +599,15 @@ fn bounded_cooldown(
|
||||
.min(maximum)
|
||||
}
|
||||
|
||||
fn persistent_cooldown_until(
|
||||
observed_at: DateTime<Utc>,
|
||||
duration: Duration,
|
||||
) -> Option<DateTime<Utc>> {
|
||||
chrono::Duration::from_std(duration)
|
||||
.ok()
|
||||
.map(|duration| observed_at + duration)
|
||||
}
|
||||
|
||||
fn insert_runtime_cooldown(
|
||||
cooldowns: &mut std::collections::BTreeMap<ProviderAccountId, RuntimeCooldown>,
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -4,6 +4,7 @@ mod admin;
|
||||
pub mod config;
|
||||
pub mod credential;
|
||||
mod provider;
|
||||
mod reasoning_replay;
|
||||
pub mod transport;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
@@ -5,7 +5,6 @@ use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use futures::{StreamExt, future::BoxFuture};
|
||||
use gateway_core::engine::continuation::ContinuationBinding;
|
||||
use gateway_core::engine::credential::{
|
||||
@@ -48,9 +47,13 @@ use crate::credential::{
|
||||
GrokCredentialRecovery, GrokCredentialRecoveryOutcome, GrokCredentialRefreshOutcome,
|
||||
GrokCredentialRefreshService, GrokQuotaError,
|
||||
};
|
||||
use crate::reasoning_replay::{
|
||||
GrokReasoningReplay, GrokReasoningReplayCapture, GrokReasoningReplayKey,
|
||||
valid_reasoning_ciphertext,
|
||||
};
|
||||
use crate::transport::canonical::GrokCanonicalDecoder;
|
||||
use crate::transport::config::XAI_PROVIDER_NAME;
|
||||
use crate::transport::headers::{GrokClientIdentity, GrokHeader, build_grok_headers};
|
||||
use crate::transport::headers::{GrokClientIdentity, build_grok_headers};
|
||||
use crate::transport::profile::{GROK_CLI_RELEASE_POLL_INTERVAL, GrokCliReleaseService};
|
||||
use crate::transport::{
|
||||
GROK_RESPONSES_URL, GrokCompactionDecodeError, GrokCompactionRequest,
|
||||
@@ -66,8 +69,6 @@ const HTTP_SSE_TRANSPORT: &str = "http_sse";
|
||||
const DEFAULT_GROK_MODEL: &str = "grok-4.5";
|
||||
const XAI_SESSION_STATE_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
const XAI_SESSION_OUTPUT_LIMIT: usize = 4_096;
|
||||
const MIN_REASONING_CIPHERTEXT_BYTES: usize = 50;
|
||||
const MIN_REASONING_CIPHERTEXT_ENTROPY: f64 = 0.85;
|
||||
const REASONING_DECODE_FAILED_CODE: &str = "reasoning_decode_failed";
|
||||
const RESPONSE_NOT_FOUND_CODE: &str = "not_found";
|
||||
|
||||
@@ -82,6 +83,7 @@ pub struct GrokBuildProvider {
|
||||
credential_recovery: Arc<dyn GrokCredentialRecovery>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
client_identity: GrokClientIdentity,
|
||||
reasoning_replay: GrokReasoningReplay,
|
||||
wire_profile: XaiWireProfileState,
|
||||
responses_url: Url,
|
||||
}
|
||||
@@ -105,6 +107,7 @@ impl GrokBuildProvider {
|
||||
credential_recovery,
|
||||
account_feedback,
|
||||
client_identity: GrokClientIdentity::new(),
|
||||
reasoning_replay: GrokReasoningReplay::new(),
|
||||
wire_profile,
|
||||
responses_url,
|
||||
})
|
||||
@@ -217,6 +220,38 @@ impl GrokBuildProvider {
|
||||
selected.account_id(),
|
||||
request_input.as_slice(),
|
||||
)?;
|
||||
let inherited_replay_session_id = context
|
||||
.continuation()
|
||||
.is_none()
|
||||
.then(|| {
|
||||
previous_session
|
||||
.as_ref()
|
||||
.and_then(|state| state.session_id.as_deref())
|
||||
})
|
||||
.flatten();
|
||||
let reasoning_replay_key = upstream_request
|
||||
.reasoning_replay_session_id()
|
||||
.or(inherited_replay_session_id)
|
||||
.and_then(|session_id| {
|
||||
self.reasoning_replay.key(
|
||||
candidate.upstream_model().as_str(),
|
||||
session_id,
|
||||
selected.account_id().as_str(),
|
||||
)
|
||||
});
|
||||
if !upstream_request.has_previous_response_id()
|
||||
&& let (Some(key), Some(input)) = (
|
||||
reasoning_replay_key.as_ref(),
|
||||
upstream_request.replay_input_items(),
|
||||
)
|
||||
&& let Some(input) = self.reasoning_replay.apply(key, &input)
|
||||
{
|
||||
upstream_request
|
||||
.set_replay_input(input)
|
||||
.map_err(map_request_error)?;
|
||||
}
|
||||
let reasoning_replay_capture =
|
||||
reasoning_replay_key.map(|key| self.reasoning_replay.capture(key));
|
||||
let session_capture = (!matches!(
|
||||
context.continuation(),
|
||||
Some(ContinuationBinding::Pinned(_) | ContinuationBinding::External(_))
|
||||
@@ -244,6 +279,7 @@ impl GrokBuildProvider {
|
||||
context,
|
||||
session: Arc::clone(&selected),
|
||||
session_capture,
|
||||
reasoning_replay_capture,
|
||||
},
|
||||
);
|
||||
let stream = ProviderStream::new(metadata, events, selected);
|
||||
@@ -274,7 +310,7 @@ impl GrokBuildProvider {
|
||||
.map(|previous| ProviderAccountId::new(previous.account_id.clone()))
|
||||
.transpose()
|
||||
.map_err(|_| protocol_not_sent())?;
|
||||
let upstream_session_id = previous_session
|
||||
let inherited_session_id = previous_session
|
||||
.as_ref()
|
||||
.and_then(|previous| previous.session_id.clone());
|
||||
let upstream_request = GrokCompactionRequest::encode(
|
||||
@@ -283,6 +319,12 @@ impl GrokBuildProvider {
|
||||
context.client_api_key_ref(),
|
||||
)
|
||||
.map_err(map_request_error)?;
|
||||
let explicit_replay_session_id = upstream_request
|
||||
.reasoning_replay_session_id()
|
||||
.map(str::to_owned);
|
||||
let upstream_session_id = inherited_session_id
|
||||
.clone()
|
||||
.or_else(|| explicit_replay_session_id.clone());
|
||||
let selected = Arc::new(
|
||||
select_grok_session(
|
||||
self.selector.as_ref(),
|
||||
@@ -293,6 +335,16 @@ impl GrokBuildProvider {
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
let reasoning_replay_key = explicit_replay_session_id
|
||||
.as_deref()
|
||||
.or(inherited_session_id.as_deref())
|
||||
.and_then(|session_id| {
|
||||
self.reasoning_replay.key(
|
||||
candidate.upstream_model().as_str(),
|
||||
session_id,
|
||||
selected.account_id().as_str(),
|
||||
)
|
||||
});
|
||||
let allows_account_state_mutation = selected.allows_account_state_mutation();
|
||||
let metadata = provider_call_metadata(candidate, &selected)?;
|
||||
let events = cold_compaction_http_sse_stream(
|
||||
@@ -308,6 +360,8 @@ impl GrokBuildProvider {
|
||||
upstream_session_id,
|
||||
context,
|
||||
session: Arc::clone(&selected),
|
||||
reasoning_replay: self.reasoning_replay.clone(),
|
||||
reasoning_replay_key,
|
||||
},
|
||||
);
|
||||
let stream = ProviderStream::new(metadata, events, selected);
|
||||
@@ -715,46 +769,6 @@ fn portable_output_item(mut item: Value, strip_opaque: bool) -> Option<Value> {
|
||||
Some(item)
|
||||
}
|
||||
|
||||
fn valid_reasoning_ciphertext(value: &str) -> bool {
|
||||
if value.is_empty()
|
||||
|| value != value.trim()
|
||||
|| value.len() > XAI_SESSION_STATE_MAX_BYTES
|
||||
|| value.starts_with("gAAAA")
|
||||
|| value.contains('=')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Ok(decoded) = base64::engine::general_purpose::STANDARD_NO_PAD.decode(value) else {
|
||||
return false;
|
||||
};
|
||||
decoded.len() >= MIN_REASONING_CIPHERTEXT_BYTES
|
||||
&& byte_entropy_ratio(&decoded) >= MIN_REASONING_CIPHERTEXT_ENTROPY
|
||||
}
|
||||
|
||||
fn byte_entropy_ratio(value: &[u8]) -> f64 {
|
||||
if value.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut counts = [0_u32; 256];
|
||||
for byte in value {
|
||||
counts[usize::from(*byte)] += 1;
|
||||
}
|
||||
let size = value.len() as f64;
|
||||
let entropy = counts
|
||||
.into_iter()
|
||||
.filter(|count| *count > 0)
|
||||
.map(|count| {
|
||||
let probability = f64::from(count) / size;
|
||||
-probability * probability.log2()
|
||||
})
|
||||
.sum::<f64>();
|
||||
let symbols = value.len().min(256);
|
||||
if symbols <= 1 {
|
||||
return 0.0;
|
||||
}
|
||||
entropy / (symbols as f64).log2()
|
||||
}
|
||||
|
||||
fn has_readable_reasoning(item: &Map<String, Value>) -> bool {
|
||||
["summary", "content"].into_iter().any(|field| {
|
||||
item.get(field)
|
||||
@@ -898,6 +912,7 @@ struct GrokStreamAttempt {
|
||||
context: AttemptContext,
|
||||
session: Arc<SelectedGrokSession>,
|
||||
session_capture: Option<GrokSessionCapture>,
|
||||
reasoning_replay_capture: Option<GrokReasoningReplayCapture>,
|
||||
}
|
||||
|
||||
struct GrokCompactionStreamAttempt {
|
||||
@@ -910,6 +925,8 @@ struct GrokCompactionStreamAttempt {
|
||||
upstream_session_id: Option<String>,
|
||||
context: AttemptContext,
|
||||
session: Arc<SelectedGrokSession>,
|
||||
reasoning_replay: GrokReasoningReplay,
|
||||
reasoning_replay_key: Option<GrokReasoningReplayKey>,
|
||||
}
|
||||
|
||||
struct AcceptedGrokInference {
|
||||
@@ -1044,9 +1061,11 @@ fn cold_compaction_http_sse_stream(
|
||||
upstream_session_id,
|
||||
context,
|
||||
session,
|
||||
reasoning_replay,
|
||||
reasoning_replay_key,
|
||||
} = attempt;
|
||||
Box::pin(async_stream::try_stream! {
|
||||
let mut headers = build_grok_headers(
|
||||
let headers = build_grok_headers(
|
||||
&wire_profile,
|
||||
&session,
|
||||
&client_identity,
|
||||
@@ -1055,12 +1074,6 @@ fn cold_compaction_http_sse_stream(
|
||||
None,
|
||||
&upstream_model,
|
||||
);
|
||||
if let Some(upstream_session_id) = upstream_session_id.as_deref() {
|
||||
headers.push(GrokHeader::sensitive(
|
||||
"x-grok-session-id",
|
||||
crate::credential::SecretValue::new(upstream_session_id.to_owned()),
|
||||
));
|
||||
}
|
||||
let body = request.to_json_bytes().map_err(map_request_error)?;
|
||||
let inference_request = GrokInferenceRequest::new(
|
||||
responses_url,
|
||||
@@ -1147,6 +1160,9 @@ fn cold_compaction_http_sse_stream(
|
||||
if upstream_completed && session.allows_account_state_mutation() {
|
||||
selector.record_success(&session).await;
|
||||
}
|
||||
if let Some(key) = reasoning_replay_key.as_ref() {
|
||||
reasoning_replay.clear(key);
|
||||
}
|
||||
yield ProviderEvent::canonical_with_wire(vec![GatewayEvent::Started(started)], created);
|
||||
yield ProviderEvent::wire(output_done);
|
||||
let mut terminal_facts = facts.accounting;
|
||||
@@ -1245,6 +1261,7 @@ fn cold_http_sse_stream(
|
||||
context,
|
||||
session,
|
||||
mut session_capture,
|
||||
mut reasoning_replay_capture,
|
||||
} = attempt;
|
||||
Box::pin(async_stream::try_stream! {
|
||||
if context.cancellation().is_cancelled() {
|
||||
@@ -1364,6 +1381,9 @@ fn cold_http_sse_stream(
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
attach_xai_session_update(&mut events, &mut session_capture)?;
|
||||
if let Some(capture) = reasoning_replay_capture.as_mut() {
|
||||
capture.observe(&events);
|
||||
}
|
||||
if completed && session.allows_account_state_mutation() {
|
||||
selector.record_success(&session).await;
|
||||
}
|
||||
@@ -1389,6 +1409,9 @@ fn cold_http_sse_stream(
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)));
|
||||
attach_xai_session_update(&mut final_events, &mut session_capture)?;
|
||||
if let Some(capture) = reasoning_replay_capture.as_mut() {
|
||||
capture.observe(&final_events);
|
||||
}
|
||||
if completed && session.allows_account_state_mutation() {
|
||||
selector.record_success(&session).await;
|
||||
}
|
||||
@@ -1871,7 +1894,9 @@ const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
|
||||
const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
|
||||
const OAUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const QUOTA_CATALOG_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
// grok2api 对 Free/未知额度耗尽采用 24 小时恢复探测;当前公共账号状态没有
|
||||
// 独立的 paid period-end 字段,因此 Build 账号统一使用同一保守下限。
|
||||
const QUOTA_PERIODIC_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const CLI_RELEASE_WORKER_OWNER: &str = "xai-cli-release";
|
||||
|
||||
pub(crate) fn worker_contributions(
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
//! Grok Build 的账号隔离 reasoning replay。
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::Engine as _;
|
||||
use gateway_core::event::ProviderEvent;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const REASONING_REPLAY_TTL: Duration = Duration::from_secs(60 * 60);
|
||||
const REASONING_REPLAY_MAX_ENTRIES: usize = 1_024;
|
||||
const REASONING_REPLAY_MAX_BYTES: usize = 8 * 1024 * 1024;
|
||||
const REASONING_REPLAY_MAX_OUTPUT_ITEMS: usize = 4_096;
|
||||
const MIN_REASONING_CIPHERTEXT_BYTES: usize = 50;
|
||||
const MIN_REASONING_CIPHERTEXT_ENTROPY: f64 = 0.85;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub(crate) struct GrokReasoningReplayKey {
|
||||
model: String,
|
||||
session_id: String,
|
||||
account_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GrokReasoningReplay {
|
||||
state: Arc<Mutex<GrokReasoningReplayState>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct GrokReasoningReplayState {
|
||||
entries: BTreeMap<GrokReasoningReplayKey, GrokReasoningReplayEntry>,
|
||||
}
|
||||
|
||||
struct GrokReasoningReplayEntry {
|
||||
stored_at: Instant,
|
||||
items: Vec<Value>,
|
||||
}
|
||||
|
||||
pub(crate) struct GrokReasoningReplayCapture {
|
||||
replay: GrokReasoningReplay,
|
||||
key: GrokReasoningReplayKey,
|
||||
output_items: BTreeMap<u32, Value>,
|
||||
captured_bytes: usize,
|
||||
truncated: bool,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl Default for GrokReasoningReplay {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl GrokReasoningReplay {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(GrokReasoningReplayState::default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn key(
|
||||
&self,
|
||||
model: &str,
|
||||
session_id: &str,
|
||||
account_id: &str,
|
||||
) -> Option<GrokReasoningReplayKey> {
|
||||
let model = model.trim();
|
||||
let session_id = session_id.trim();
|
||||
let account_id = account_id.trim();
|
||||
if model.is_empty() || session_id.is_empty() || account_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(GrokReasoningReplayKey {
|
||||
model: model.to_owned(),
|
||||
session_id: session_id.to_owned(),
|
||||
account_id: account_id.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn apply(
|
||||
&self,
|
||||
key: &GrokReasoningReplayKey,
|
||||
input: &[Value],
|
||||
) -> Option<Vec<Value>> {
|
||||
if input.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let cached = self.read(key)?;
|
||||
let filtered = filter_replay_items_for_input(input, cached)?;
|
||||
Some(insert_replay_items(input, &filtered))
|
||||
}
|
||||
|
||||
pub(crate) fn capture(&self, key: GrokReasoningReplayKey) -> GrokReasoningReplayCapture {
|
||||
GrokReasoningReplayCapture {
|
||||
replay: self.clone(),
|
||||
key,
|
||||
output_items: BTreeMap::new(),
|
||||
captured_bytes: 0,
|
||||
truncated: false,
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn clear(&self, key: &GrokReasoningReplayKey) {
|
||||
self.lock_state().entries.remove(key);
|
||||
}
|
||||
|
||||
fn read(&self, key: &GrokReasoningReplayKey) -> Option<Vec<Value>> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.lock_state();
|
||||
prune_expired(&mut state, now);
|
||||
state.entries.get(key).map(|entry| entry.items.clone())
|
||||
}
|
||||
|
||||
fn store_output(&self, key: GrokReasoningReplayKey, output: Vec<Value>) {
|
||||
let normalized = match normalize_replay_items(output) {
|
||||
ReplayNormalization::Store(items) => items,
|
||||
ReplayNormalization::Delete => {
|
||||
self.clear(&key);
|
||||
return;
|
||||
}
|
||||
ReplayNormalization::Oversized => return,
|
||||
};
|
||||
let now = Instant::now();
|
||||
let mut state = self.lock_state();
|
||||
prune_expired(&mut state, now);
|
||||
if !state.entries.contains_key(&key)
|
||||
&& state.entries.len() >= REASONING_REPLAY_MAX_ENTRIES
|
||||
&& let Some(oldest) = state
|
||||
.entries
|
||||
.iter()
|
||||
.min_by_key(|(_, entry)| entry.stored_at)
|
||||
.map(|(key, _)| key.clone())
|
||||
{
|
||||
state.entries.remove(&oldest);
|
||||
}
|
||||
state.entries.insert(
|
||||
key,
|
||||
GrokReasoningReplayEntry {
|
||||
stored_at: now,
|
||||
items: normalized,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn lock_state(&self) -> MutexGuard<'_, GrokReasoningReplayState> {
|
||||
match self.state.lock() {
|
||||
Ok(state) => state,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GrokReasoningReplayCapture {
|
||||
pub(crate) fn observe(&mut self, events: &[ProviderEvent]) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
let mut completed = false;
|
||||
for event in events {
|
||||
let Some(wire) = event.wire_event() else {
|
||||
continue;
|
||||
};
|
||||
let payload = wire.data();
|
||||
let event_type = wire
|
||||
.event_type()
|
||||
.or_else(|| payload.get("type").and_then(Value::as_str));
|
||||
if event_type == Some("response.output_item.done")
|
||||
&& let Some(item) = payload.get("item").cloned()
|
||||
{
|
||||
let index = payload
|
||||
.get("output_index")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.unwrap_or_else(|| u32::try_from(self.output_items.len()).unwrap_or(u32::MAX));
|
||||
self.capture_item(index, item);
|
||||
}
|
||||
if event_type == Some("response.completed") {
|
||||
capture_terminal_output(payload, self);
|
||||
completed = true;
|
||||
}
|
||||
}
|
||||
if completed {
|
||||
self.finished = true;
|
||||
if self.truncated {
|
||||
self.output_items.clear();
|
||||
return;
|
||||
}
|
||||
let output = std::mem::take(&mut self.output_items)
|
||||
.into_values()
|
||||
.collect();
|
||||
self.replay.store_output(self.key.clone(), output);
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_item(&mut self, index: u32, item: Value) {
|
||||
if self.truncated || self.output_items.contains_key(&index) {
|
||||
return;
|
||||
}
|
||||
if self.output_items.len() >= REASONING_REPLAY_MAX_OUTPUT_ITEMS {
|
||||
self.truncate();
|
||||
return;
|
||||
}
|
||||
let Some(captured_bytes) = serde_json::to_vec(&item)
|
||||
.ok()
|
||||
.and_then(|encoded| self.captured_bytes.checked_add(encoded.len()))
|
||||
.filter(|captured_bytes| *captured_bytes <= REASONING_REPLAY_MAX_BYTES)
|
||||
else {
|
||||
self.truncate();
|
||||
return;
|
||||
};
|
||||
self.captured_bytes = captured_bytes;
|
||||
self.output_items.insert(index, item);
|
||||
}
|
||||
|
||||
fn truncate(&mut self) {
|
||||
self.truncated = true;
|
||||
self.output_items.clear();
|
||||
self.captured_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
fn capture_terminal_output(payload: &Value, capture: &mut GrokReasoningReplayCapture) {
|
||||
let Some(output) = payload
|
||||
.get("response")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|response| response.get("output"))
|
||||
.and_then(Value::as_array)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if output.len() > REASONING_REPLAY_MAX_OUTPUT_ITEMS {
|
||||
capture.truncate();
|
||||
return;
|
||||
}
|
||||
for (index, item) in output
|
||||
.iter()
|
||||
.take(REASONING_REPLAY_MAX_OUTPUT_ITEMS)
|
||||
.enumerate()
|
||||
{
|
||||
let Some(index) = u32::try_from(index).ok() else {
|
||||
break;
|
||||
};
|
||||
capture.capture_item(index, item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
fn prune_expired(state: &mut GrokReasoningReplayState, now: Instant) {
|
||||
state
|
||||
.entries
|
||||
.retain(|_, entry| now.saturating_duration_since(entry.stored_at) < REASONING_REPLAY_TTL);
|
||||
}
|
||||
|
||||
enum ReplayNormalization {
|
||||
Store(Vec<Value>),
|
||||
Delete,
|
||||
Oversized,
|
||||
}
|
||||
|
||||
fn normalize_replay_items(items: Vec<Value>) -> ReplayNormalization {
|
||||
let mut normalized = Vec::new();
|
||||
let mut has_anchor = false;
|
||||
for item in items {
|
||||
let Some(item) = normalize_replay_item(item) else {
|
||||
continue;
|
||||
};
|
||||
if matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("reasoning" | "function_call" | "custom_tool_call")
|
||||
) {
|
||||
has_anchor = true;
|
||||
}
|
||||
normalized.push(item);
|
||||
}
|
||||
if !has_anchor || normalized.is_empty() {
|
||||
return ReplayNormalization::Delete;
|
||||
}
|
||||
match serde_json::to_vec(&normalized) {
|
||||
Ok(encoded) if encoded.len() <= REASONING_REPLAY_MAX_BYTES => {
|
||||
ReplayNormalization::Store(normalized)
|
||||
}
|
||||
_ => ReplayNormalization::Oversized,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_replay_item(item: Value) -> Option<Value> {
|
||||
let object = item.as_object()?;
|
||||
match object.get("type").and_then(Value::as_str)?.trim() {
|
||||
"reasoning" => normalize_reasoning_item(object),
|
||||
"message" => normalize_assistant_message_item(object),
|
||||
"function_call" => normalize_function_call_item(object),
|
||||
"custom_tool_call" => normalize_custom_tool_call_item(object),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_reasoning_item(item: &Map<String, Value>) -> Option<Value> {
|
||||
let encrypted = item.get("encrypted_content")?.as_str()?;
|
||||
if !valid_reasoning_ciphertext(encrypted) {
|
||||
return None;
|
||||
}
|
||||
Some(Value::Object(Map::from_iter([
|
||||
("type".to_owned(), Value::String("reasoning".to_owned())),
|
||||
("summary".to_owned(), Value::Array(Vec::new())),
|
||||
("content".to_owned(), Value::Null),
|
||||
(
|
||||
"encrypted_content".to_owned(),
|
||||
Value::String(encrypted.to_owned()),
|
||||
),
|
||||
])))
|
||||
}
|
||||
|
||||
fn normalize_assistant_message_item(item: &Map<String, Value>) -> Option<Value> {
|
||||
if !item
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let parts = item
|
||||
.get("content")?
|
||||
.as_array()?
|
||||
.iter()
|
||||
.filter_map(normalize_assistant_part)
|
||||
.collect::<Vec<_>>();
|
||||
if parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Value::Object(Map::from_iter([
|
||||
("type".to_owned(), Value::String("message".to_owned())),
|
||||
("role".to_owned(), Value::String("assistant".to_owned())),
|
||||
("content".to_owned(), Value::Array(parts)),
|
||||
])))
|
||||
}
|
||||
|
||||
fn normalize_assistant_part(part: &Value) -> Option<Value> {
|
||||
let part = part.as_object()?;
|
||||
match part.get("type").and_then(Value::as_str)?.trim() {
|
||||
"output_text" => Some(Value::Object(Map::from_iter([
|
||||
("type".to_owned(), Value::String("output_text".to_owned())),
|
||||
(
|
||||
"text".to_owned(),
|
||||
Value::String(part.get("text")?.as_str()?.to_owned()),
|
||||
),
|
||||
]))),
|
||||
"refusal" => Some(Value::Object(Map::from_iter([
|
||||
("type".to_owned(), Value::String("refusal".to_owned())),
|
||||
(
|
||||
"refusal".to_owned(),
|
||||
Value::String(part.get("refusal")?.as_str()?.to_owned()),
|
||||
),
|
||||
]))),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_function_call_item(item: &Map<String, Value>) -> Option<Value> {
|
||||
let call_id = nonempty_string(item, "call_id")?;
|
||||
let name = nonempty_string(item, "name")?;
|
||||
let arguments = item.get("arguments")?.as_str()?.to_owned();
|
||||
Some(Value::Object(Map::from_iter([
|
||||
("type".to_owned(), Value::String("function_call".to_owned())),
|
||||
("call_id".to_owned(), Value::String(call_id)),
|
||||
("name".to_owned(), Value::String(name)),
|
||||
("arguments".to_owned(), Value::String(arguments)),
|
||||
])))
|
||||
}
|
||||
|
||||
fn normalize_custom_tool_call_item(item: &Map<String, Value>) -> Option<Value> {
|
||||
let call_id = nonempty_string(item, "call_id")?;
|
||||
let name = nonempty_string(item, "name")?;
|
||||
let input = item.get("input")?.clone();
|
||||
let status = item
|
||||
.get("status")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or("completed")
|
||||
.to_owned();
|
||||
Some(Value::Object(Map::from_iter([
|
||||
(
|
||||
"type".to_owned(),
|
||||
Value::String("custom_tool_call".to_owned()),
|
||||
),
|
||||
("status".to_owned(), Value::String(status)),
|
||||
("call_id".to_owned(), Value::String(call_id)),
|
||||
("name".to_owned(), Value::String(name)),
|
||||
("input".to_owned(), input),
|
||||
])))
|
||||
}
|
||||
|
||||
fn nonempty_string(item: &Map<String, Value>, field: &str) -> Option<String> {
|
||||
item.get(field)
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn filter_replay_items_for_input(input: &[Value], cached: Vec<Value>) -> Option<Vec<Value>> {
|
||||
let last_assistant = input.iter().rev().find_map(assistant_message);
|
||||
let cached_assistant = cached.iter().find_map(assistant_message);
|
||||
let assistant_matches = match (last_assistant, cached_assistant) {
|
||||
(Some(current), Some(previous)) => {
|
||||
if !assistant_content_equal(current, previous) {
|
||||
return None;
|
||||
}
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
let mut existing_calls = BTreeSet::new();
|
||||
let mut existing_outputs = BTreeMap::new();
|
||||
let mut existing_encrypted = BTreeSet::new();
|
||||
for item in input {
|
||||
let Some(item) = item.as_object() else {
|
||||
continue;
|
||||
};
|
||||
match item.get("type").and_then(Value::as_str).map(str::trim) {
|
||||
Some("reasoning") => {
|
||||
if let Some(encrypted) = item
|
||||
.get("encrypted_content")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
existing_encrypted.insert(encrypted.to_owned());
|
||||
}
|
||||
}
|
||||
Some("function_call_output" | "custom_tool_call_output") => {
|
||||
if let Some(call_id) = item.get("call_id").and_then(Value::as_str) {
|
||||
for candidate in comparable_call_ids(call_id) {
|
||||
existing_outputs.insert(candidate, call_id.trim().to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(kind @ ("function_call" | "custom_tool_call")) => {
|
||||
if let Some(call_id) = item.get("call_id").and_then(Value::as_str) {
|
||||
existing_calls.extend(replay_call_keys(kind, call_id));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let mut filtered = Vec::new();
|
||||
for mut item in cached {
|
||||
let kind = item
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
match kind.as_str() {
|
||||
"reasoning" => {
|
||||
if item
|
||||
.get("encrypted_content")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|encrypted| existing_encrypted.contains(encrypted))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
"message" if assistant_matches => continue,
|
||||
"message" => {}
|
||||
"function_call" | "custom_tool_call" => {
|
||||
let Some(call_id) = item.get("call_id").and_then(Value::as_str) else {
|
||||
continue;
|
||||
};
|
||||
let keys = replay_call_keys(&kind, call_id);
|
||||
if keys.is_empty() || keys.iter().any(|key| existing_calls.contains(key)) {
|
||||
continue;
|
||||
}
|
||||
let output_call_id = comparable_call_ids(call_id)
|
||||
.into_iter()
|
||||
.find_map(|candidate| existing_outputs.get(&candidate).cloned());
|
||||
let Some(output_call_id) = output_call_id else {
|
||||
continue;
|
||||
};
|
||||
existing_calls.extend(keys);
|
||||
if output_call_id != call_id
|
||||
&& let Some(item) = item.as_object_mut()
|
||||
{
|
||||
item.insert("call_id".to_owned(), Value::String(output_call_id));
|
||||
}
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
filtered.push(item);
|
||||
}
|
||||
(!filtered.is_empty()).then_some(filtered)
|
||||
}
|
||||
|
||||
fn assistant_message(item: &Value) -> Option<&Map<String, Value>> {
|
||||
let item = item.as_object()?;
|
||||
let item_type = item
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or_default();
|
||||
let role = item.get("role")?.as_str()?.trim();
|
||||
((item_type.is_empty() || item_type == "message") && role.eq_ignore_ascii_case("assistant"))
|
||||
.then_some(item)
|
||||
}
|
||||
|
||||
fn assistant_content_equal(left: &Map<String, Value>, right: &Map<String, Value>) -> bool {
|
||||
let Some(left) = assistant_parts(left.get("content")) else {
|
||||
return false;
|
||||
};
|
||||
let Some(right) = assistant_parts(right.get("content")) else {
|
||||
return false;
|
||||
};
|
||||
left == right
|
||||
}
|
||||
|
||||
fn assistant_parts(content: Option<&Value>) -> Option<Vec<(String, String)>> {
|
||||
match content? {
|
||||
Value::String(text) => Some(vec![("output_text".to_owned(), text.clone())]),
|
||||
Value::Array(parts) => {
|
||||
let mut result = Vec::new();
|
||||
for part in parts {
|
||||
let part = part.as_object()?;
|
||||
let kind = part.get("type")?.as_str()?.trim();
|
||||
let value = match kind {
|
||||
"output_text" => part.get("text")?.as_str()?,
|
||||
"refusal" => part.get("refusal")?.as_str()?,
|
||||
_ => return None,
|
||||
};
|
||||
result.push((kind.to_owned(), value.to_owned()));
|
||||
}
|
||||
(!result.is_empty()).then_some(result)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn comparable_call_ids(call_id: &str) -> Vec<String> {
|
||||
let call_id = call_id.trim();
|
||||
if call_id.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
if let Some(upstream) = call_id
|
||||
.strip_prefix("toolu_")
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
return vec![call_id.to_owned(), upstream.to_owned()];
|
||||
}
|
||||
vec![call_id.to_owned(), format!("toolu_{call_id}")]
|
||||
}
|
||||
|
||||
fn replay_call_keys(item_type: &str, call_id: &str) -> Vec<String> {
|
||||
if !matches!(item_type, "function_call" | "custom_tool_call") {
|
||||
return Vec::new();
|
||||
}
|
||||
comparable_call_ids(call_id)
|
||||
.into_iter()
|
||||
.map(|call_id| format!("{item_type}\0{call_id}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn insert_replay_items(input: &[Value], replay: &[Value]) -> Vec<Value> {
|
||||
let insert_at = replay_insert_index(input, replay);
|
||||
let mut result = Vec::new();
|
||||
for (index, item) in input.iter().enumerate() {
|
||||
if index == insert_at {
|
||||
result.extend(replay.iter().cloned());
|
||||
}
|
||||
result.push(item.clone());
|
||||
}
|
||||
if insert_at == input.len() {
|
||||
result.extend(replay.iter().cloned());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn replay_insert_index(input: &[Value], replay: &[Value]) -> usize {
|
||||
let replay_call_ids = replay
|
||||
.iter()
|
||||
.filter_map(Value::as_object)
|
||||
.filter(|item| {
|
||||
matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("function_call" | "custom_tool_call")
|
||||
)
|
||||
})
|
||||
.filter_map(|item| item.get("call_id").and_then(Value::as_str))
|
||||
.flat_map(comparable_call_ids)
|
||||
.collect::<BTreeSet<_>>();
|
||||
if !replay_call_ids.is_empty() {
|
||||
for (index, item) in input.iter().enumerate() {
|
||||
let Some(item) = item.as_object() else {
|
||||
continue;
|
||||
};
|
||||
if !matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("function_call_output" | "custom_tool_call_output")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let call_ids = item
|
||||
.get("call_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(comparable_call_ids)
|
||||
.unwrap_or_default();
|
||||
if call_ids.is_empty()
|
||||
|| call_ids
|
||||
.iter()
|
||||
.any(|call_id| replay_call_ids.contains(call_id))
|
||||
{
|
||||
return index;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(index) = input
|
||||
.iter()
|
||||
.rposition(|item| assistant_message(item).is_some())
|
||||
{
|
||||
return index;
|
||||
}
|
||||
input
|
||||
.iter()
|
||||
.position(should_insert_replay_before)
|
||||
.unwrap_or(input.len())
|
||||
}
|
||||
|
||||
fn should_insert_replay_before(item: &Value) -> bool {
|
||||
let Some(item) = item.as_object() else {
|
||||
return true;
|
||||
};
|
||||
let item_type = item
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or_default();
|
||||
let role = item
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
if role.is_empty() || (!item_type.is_empty() && item_type != "message") {
|
||||
return true;
|
||||
}
|
||||
!matches!(role.as_str(), "system" | "developer")
|
||||
}
|
||||
|
||||
pub(crate) fn valid_reasoning_ciphertext(value: &str) -> bool {
|
||||
if value.is_empty()
|
||||
|| value != value.trim()
|
||||
|| value.len() > REASONING_REPLAY_MAX_BYTES
|
||||
|| value.starts_with("gAAAA")
|
||||
|| value.contains('=')
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Ok(decoded) = base64::engine::general_purpose::STANDARD_NO_PAD.decode(value) else {
|
||||
return false;
|
||||
};
|
||||
decoded.len() >= MIN_REASONING_CIPHERTEXT_BYTES
|
||||
&& byte_entropy_ratio(&decoded) >= MIN_REASONING_CIPHERTEXT_ENTROPY
|
||||
}
|
||||
|
||||
fn byte_entropy_ratio(value: &[u8]) -> f64 {
|
||||
if value.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut counts = [0_u32; 256];
|
||||
for byte in value {
|
||||
counts[usize::from(*byte)] += 1;
|
||||
}
|
||||
let size = value.len() as f64;
|
||||
let entropy = counts
|
||||
.into_iter()
|
||||
.filter(|count| *count > 0)
|
||||
.map(|count| {
|
||||
let probability = f64::from(count) / size;
|
||||
-probability * probability.log2()
|
||||
})
|
||||
.sum::<f64>();
|
||||
let symbols = value.len().min(256);
|
||||
if symbols <= 1 {
|
||||
return 0.0;
|
||||
}
|
||||
entropy / (symbols as f64).log2()
|
||||
}
|
||||
@@ -42,6 +42,7 @@ If the prior conversation contains a note about files at /tmp/compaction/segment
|
||||
pub struct GrokCompactionRequest {
|
||||
body: Map<String, Value>,
|
||||
affinity: Option<GrokSessionAffinityKey>,
|
||||
reasoning_replay_session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl GrokCompactionRequest {
|
||||
@@ -60,6 +61,8 @@ impl GrokCompactionRequest {
|
||||
upstream_model,
|
||||
client_api_key_ref,
|
||||
)?;
|
||||
let reasoning_replay_session_id =
|
||||
normalized.reasoning_replay_session_id().map(str::to_owned);
|
||||
let mut body = normalized.body().clone();
|
||||
let mut input = normalized.input_items();
|
||||
input.push(summary_prompt_item());
|
||||
@@ -95,6 +98,7 @@ impl GrokCompactionRequest {
|
||||
Ok(Self {
|
||||
body,
|
||||
affinity: normalized.affinity().cloned(),
|
||||
reasoning_replay_session_id,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -110,6 +114,12 @@ impl GrokCompactionRequest {
|
||||
self.affinity.as_ref()
|
||||
}
|
||||
|
||||
/// 返回用于在成功 compact 后清除 reasoning replay 的显式会话身份。
|
||||
#[must_use]
|
||||
pub(crate) fn reasoning_replay_session_id(&self) -> Option<&str> {
|
||||
self.reasoning_replay_session_id.as_deref()
|
||||
}
|
||||
|
||||
/// 序列化上游请求正文。
|
||||
///
|
||||
/// # Errors
|
||||
|
||||
@@ -142,11 +142,11 @@ pub fn build_grok_headers(
|
||||
"x-grok-conv-id",
|
||||
SecretValue::new(upstream_session_id.to_owned()),
|
||||
));
|
||||
headers.push(GrokHeader::sensitive(
|
||||
"x-grok-session-id",
|
||||
SecretValue::new(upstream_session_id.to_owned()),
|
||||
));
|
||||
if let Some(turn_index) = turn_index {
|
||||
headers.push(GrokHeader::sensitive(
|
||||
"x-grok-session-id",
|
||||
SecretValue::new(upstream_session_id.to_owned()),
|
||||
));
|
||||
headers.push(GrokHeader::public("x-grok-turn-idx", turn_index));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ const SESSION_FIELDS: &[&str] = &[
|
||||
"conversation_id",
|
||||
"conversationId",
|
||||
];
|
||||
const PROTOCOL_CONTEXT_SESSION_FIELDS: &[&str] = &["conversation_id", "session_id", "thread_id"];
|
||||
const FOREIGN_CLIENT_METADATA_FIELDS: &[&str] = &["x-openai-subagent"];
|
||||
const MAX_SESSION_SEED_BYTES: usize = 1_024;
|
||||
const GROK_CACHE_ROUTE_TOOLS: &[&str] = &["web_search", "x_search"];
|
||||
@@ -68,6 +69,7 @@ const GROK_INTERNAL_HISTORY_KEYS: &[&str] =
|
||||
pub struct GrokResponsesRequest {
|
||||
body: Map<String, Value>,
|
||||
session_id: Option<String>,
|
||||
reasoning_replay_session_id: Option<String>,
|
||||
affinity: Option<GrokSessionAffinityKey>,
|
||||
response_transform: GrokResponseTransform,
|
||||
}
|
||||
@@ -85,6 +87,23 @@ impl GrokResponsesRequest {
|
||||
self.session_id.as_deref()
|
||||
}
|
||||
|
||||
/// 返回只有显式会话身份才启用的 reasoning replay scope。
|
||||
pub(crate) fn reasoning_replay_session_id(&self) -> Option<&str> {
|
||||
self.reasoning_replay_session_id.as_deref()
|
||||
}
|
||||
|
||||
pub(crate) fn has_previous_response_id(&self) -> bool {
|
||||
self.body
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn replay_input_items(&self) -> Option<Vec<Value>> {
|
||||
self.body.get("input").and_then(Value::as_array).cloned()
|
||||
}
|
||||
|
||||
/// 返回与会话一致、额外绑定模型的账号亲和键。
|
||||
#[must_use]
|
||||
pub const fn affinity(&self) -> Option<&GrokSessionAffinityKey> {
|
||||
@@ -149,6 +168,9 @@ impl GrokResponsesRequest {
|
||||
return;
|
||||
};
|
||||
self.session_id = Some(session_id.to_owned());
|
||||
if self.reasoning_replay_session_id.is_some() {
|
||||
self.reasoning_replay_session_id = Some(session_id.to_owned());
|
||||
}
|
||||
self.affinity = None;
|
||||
self.body.insert(
|
||||
"prompt_cache_key".to_owned(),
|
||||
@@ -159,6 +181,7 @@ impl GrokResponsesRequest {
|
||||
pub(crate) fn clear_session(&mut self) {
|
||||
self.body.remove("prompt_cache_key");
|
||||
self.session_id = None;
|
||||
self.reasoning_replay_session_id = None;
|
||||
self.affinity = None;
|
||||
}
|
||||
|
||||
@@ -207,13 +230,16 @@ impl GrokResponsesRequest {
|
||||
);
|
||||
sanitize_account_identity(&mut body);
|
||||
sanitize_foreign_client_metadata(&mut body);
|
||||
let response_transform = normalize_responses_request(&mut body)?;
|
||||
let mut response_transform = normalize_responses_request(&mut body)?;
|
||||
response_transform.observe_client_cache_tools();
|
||||
if enable_cache_route {
|
||||
enable_grok_prompt_cache_route(&mut body);
|
||||
enable_grok_prompt_cache_route(&mut body, upstream_model, &mut response_transform);
|
||||
}
|
||||
let (session_id, affinity) = identity.map_or((None, None), |(session_id, affinity)| {
|
||||
(Some(session_id), Some(affinity))
|
||||
});
|
||||
let reasoning_replay_session_id =
|
||||
session_seed.is_some().then(|| session_id.clone()).flatten();
|
||||
match session_id.as_ref() {
|
||||
Some(session_id) => {
|
||||
body.insert(
|
||||
@@ -230,6 +256,7 @@ impl GrokResponsesRequest {
|
||||
Ok(Self {
|
||||
body,
|
||||
session_id,
|
||||
reasoning_replay_session_id,
|
||||
affinity,
|
||||
response_transform,
|
||||
})
|
||||
@@ -321,23 +348,61 @@ fn sanitize_foreign_client_metadata(body: &mut Map<String, Value>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn enable_grok_prompt_cache_route(body: &mut Map<String, Value>) {
|
||||
fn enable_grok_prompt_cache_route(
|
||||
body: &mut Map<String, Value>,
|
||||
upstream_model: &str,
|
||||
response: &mut GrokResponseTransform,
|
||||
) {
|
||||
let tools = body
|
||||
.entry("tools".to_owned())
|
||||
.or_insert_with(|| Value::Array(Vec::new()));
|
||||
let Value::Array(tools) = tools else {
|
||||
return;
|
||||
};
|
||||
// 缓存路由不得改变模型可调用的工具集:客户端带工具时无法既保留其
|
||||
// tool_choice 语义又阻止注入的原生搜索被调用,因此直接不注入。
|
||||
if !tools.is_empty() {
|
||||
response.observe_upstream_cache_tools(tools);
|
||||
if is_cache_media_model(upstream_model) || has_tool_type(tools, "image_generation") {
|
||||
return;
|
||||
}
|
||||
for tool in GROK_CACHE_ROUTE_TOOLS {
|
||||
tools.push(json_object([("type", Value::String((*tool).to_owned()))]));
|
||||
if tools.is_empty() {
|
||||
for tool in GROK_CACHE_ROUTE_TOOLS {
|
||||
tools.push(json_object([("type", Value::String((*tool).to_owned()))]));
|
||||
response.mark_injected_cache_tool(tool);
|
||||
}
|
||||
// 无客户端工具时以 none 选中缓存路由,同时不授予搜索能力。
|
||||
body.insert("tool_choice".to_owned(), Value::String("none".to_owned()));
|
||||
} else if !has_tool_type(tools, "x_search") {
|
||||
tools.push(json_object([(
|
||||
"type",
|
||||
Value::String("x_search".to_owned()),
|
||||
)]));
|
||||
response.mark_injected_cache_tool("x_search");
|
||||
}
|
||||
append_x_search_to_allowed_tools(body);
|
||||
}
|
||||
|
||||
fn is_cache_media_model(model: &str) -> bool {
|
||||
let model = model.trim().to_ascii_lowercase();
|
||||
["image", "imagine", "video"]
|
||||
.into_iter()
|
||||
.any(|marker| model.contains(marker))
|
||||
}
|
||||
|
||||
fn append_x_search_to_allowed_tools(body: &mut Map<String, Value>) {
|
||||
let Some(choice) = body.get_mut("tool_choice").and_then(Value::as_object_mut) else {
|
||||
return;
|
||||
};
|
||||
if choice.get("type").and_then(Value::as_str) != Some("allowed_tools") {
|
||||
return;
|
||||
}
|
||||
let Some(allowed) = choice.get_mut("tools").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
if !has_tool_type(allowed, "x_search") {
|
||||
allowed.push(json_object([(
|
||||
"type",
|
||||
Value::String("x_search".to_owned()),
|
||||
)]));
|
||||
}
|
||||
// 禁用工具调用,使原生搜索声明只承担缓存路由作用。
|
||||
body.insert("tool_choice".to_owned(), Value::String("none".to_owned()));
|
||||
}
|
||||
|
||||
fn explicit_session_seed(request: &GenerateRequest, body: &Map<String, Value>) -> Option<String> {
|
||||
@@ -351,6 +416,17 @@ fn explicit_session_seed(request: &GenerateRequest, body: &Map<String, Value>) -
|
||||
.and_then(Value::as_object)
|
||||
.and_then(first_session_value)
|
||||
})
|
||||
.or_else(|| {
|
||||
PROTOCOL_CONTEXT_SESSION_FIELDS.iter().find_map(|field| {
|
||||
request
|
||||
.protocol_payload()
|
||||
.context()
|
||||
.get(*field)
|
||||
.and_then(Value::as_str)
|
||||
.and_then(valid_session_seed)
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn first_session_value(body: &Map<String, Value>) -> Option<String> {
|
||||
@@ -674,6 +750,11 @@ pub(crate) struct GrokResponseTransform {
|
||||
aliases: BTreeMap<String, ToolIdentity>,
|
||||
visible_tools: Vec<Value>,
|
||||
legacy_local_shell: bool,
|
||||
filter_x_search: bool,
|
||||
injected_tool_types: BTreeSet<String>,
|
||||
client_declared_tools: BTreeSet<String>,
|
||||
dropped_output_indexes: BTreeSet<u64>,
|
||||
dropped_item_ids: BTreeSet<String>,
|
||||
stream_calls: BTreeMap<String, StreamCallState>,
|
||||
stream_keys: BTreeMap<String, String>,
|
||||
}
|
||||
@@ -698,7 +779,42 @@ impl GrokTransformedWireEvent {
|
||||
impl GrokResponseTransform {
|
||||
#[must_use]
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.aliases.is_empty() && self.visible_tools.is_empty() && !self.legacy_local_shell
|
||||
self.aliases.is_empty()
|
||||
&& self.visible_tools.is_empty()
|
||||
&& !self.legacy_local_shell
|
||||
&& !self.filter_x_search
|
||||
&& self.injected_tool_types.is_empty()
|
||||
}
|
||||
|
||||
fn observe_client_cache_tools(&mut self) {
|
||||
for tool in &self.visible_tools {
|
||||
let kind = tool.get("type").and_then(Value::as_str).unwrap_or_default();
|
||||
if kind == "x_search" {
|
||||
self.filter_x_search = true;
|
||||
}
|
||||
if matches!(kind, "function" | "custom")
|
||||
&& let Some(name) = tool
|
||||
.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
{
|
||||
self.client_declared_tools.insert(name.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_upstream_cache_tools(&mut self, tools: &[Value]) {
|
||||
if has_tool_type(tools, "x_search") {
|
||||
self.filter_x_search = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_injected_cache_tool(&mut self, kind: &str) {
|
||||
self.injected_tool_types.insert(kind.to_owned());
|
||||
if kind == "x_search" {
|
||||
self.filter_x_search = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rewrite_stream_event(
|
||||
@@ -706,6 +822,20 @@ impl GrokResponseTransform {
|
||||
event_type: &str,
|
||||
mut value: Value,
|
||||
) -> Result<Vec<GrokTransformedWireEvent>, GrokRequestEncodeError> {
|
||||
if self.cache_filter_enabled() {
|
||||
if value
|
||||
.get("item")
|
||||
.is_some_and(|item| self.is_internal_cache_call(item))
|
||||
{
|
||||
self.record_dropped_cache_item(&value);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.filter_cache_envelope(&mut value)?;
|
||||
if self.references_dropped_cache_item(&value) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.compact_cache_output_index(&mut value);
|
||||
}
|
||||
if event_type == "response.output_item.added"
|
||||
&& let Some(item) = value.get("item").and_then(Value::as_object)
|
||||
&& let Some(primary) = self.remember_stream_call(item)
|
||||
@@ -1031,9 +1161,145 @@ impl GrokResponseTransform {
|
||||
])
|
||||
}
|
||||
|
||||
fn cache_filter_enabled(&self) -> bool {
|
||||
self.filter_x_search || !self.injected_tool_types.is_empty()
|
||||
}
|
||||
|
||||
fn is_internal_cache_call(&self, item: &Value) -> bool {
|
||||
let Some(item) = item.as_object() else {
|
||||
return false;
|
||||
};
|
||||
let kind = string_field(item, "type").trim();
|
||||
if kind == "web_search_call" {
|
||||
return self.injected_tool_types.contains("web_search");
|
||||
}
|
||||
if !matches!(kind, "custom_tool_call" | "function_call") {
|
||||
return false;
|
||||
}
|
||||
if string_field(item, "call_id").trim().starts_with("xs_call") {
|
||||
return true;
|
||||
}
|
||||
let name = string_field(item, "name").trim();
|
||||
matches!(
|
||||
name,
|
||||
"x_user_search" | "x_semantic_search" | "x_keyword_search" | "x_thread_fetch"
|
||||
) && string_field(item, "namespace").trim().is_empty()
|
||||
&& !self.client_declared_tools.contains(name)
|
||||
}
|
||||
|
||||
fn record_dropped_cache_item(&mut self, payload: &Value) {
|
||||
if let Some(index) = payload.get("output_index").and_then(Value::as_u64) {
|
||||
self.dropped_output_indexes.insert(index);
|
||||
}
|
||||
let Some(item) = payload.get("item").and_then(Value::as_object) else {
|
||||
return;
|
||||
};
|
||||
for field in ["id", "call_id"] {
|
||||
if let Some(value) = item
|
||||
.get(field)
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
self.dropped_item_ids.insert(value.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn references_dropped_cache_item(&self, payload: &Value) -> bool {
|
||||
if payload
|
||||
.get("output_index")
|
||||
.and_then(Value::as_u64)
|
||||
.is_some_and(|index| self.dropped_output_indexes.contains(&index))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
["item_id", "call_id"].into_iter().any(|field| {
|
||||
payload
|
||||
.get(field)
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| self.dropped_item_ids.contains(value))
|
||||
})
|
||||
}
|
||||
|
||||
fn compact_cache_output_index(&self, payload: &mut Value) {
|
||||
let Some(index) = payload.get("output_index").and_then(Value::as_u64) else {
|
||||
return;
|
||||
};
|
||||
let removed_before = self
|
||||
.dropped_output_indexes
|
||||
.iter()
|
||||
.filter(|dropped| **dropped < index)
|
||||
.count() as u64;
|
||||
if removed_before > 0
|
||||
&& let Some(payload) = payload.as_object_mut()
|
||||
{
|
||||
payload.insert(
|
||||
"output_index".to_owned(),
|
||||
Value::from(index.saturating_sub(removed_before)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn filter_cache_envelope(&self, payload: &mut Value) -> Result<(), GrokRequestEncodeError> {
|
||||
let Some(payload) = payload.as_object_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
self.filter_cache_output(payload)?;
|
||||
self.filter_injected_cache_tools(payload)?;
|
||||
if let Some(response) = payload.get_mut("response").and_then(Value::as_object_mut) {
|
||||
self.filter_cache_output(response)?;
|
||||
self.filter_injected_cache_tools(response)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn filter_cache_output(
|
||||
&self,
|
||||
envelope: &mut Map<String, Value>,
|
||||
) -> Result<(), GrokRequestEncodeError> {
|
||||
let Some(output) = envelope.get_mut("output") else {
|
||||
return Ok(());
|
||||
};
|
||||
let output = output
|
||||
.as_array_mut()
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
output.retain(|item| !self.is_internal_cache_call(item));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn filter_injected_cache_tools(
|
||||
&self,
|
||||
envelope: &mut Map<String, Value>,
|
||||
) -> Result<(), GrokRequestEncodeError> {
|
||||
if self.injected_tool_types.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(tools) = envelope.get_mut("tools") else {
|
||||
return Ok(());
|
||||
};
|
||||
let tools = tools
|
||||
.as_array_mut()
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
tools.retain(|tool| {
|
||||
tool.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_none_or(|kind| !self.injected_tool_types.contains(kind.trim()))
|
||||
});
|
||||
if tools.is_empty() {
|
||||
envelope.remove("tools");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_visible_tools(&self, response: &mut Map<String, Value>) {
|
||||
if response.contains_key("tools") {
|
||||
response.insert("tools".to_owned(), Value::Array(self.visible_tools.clone()));
|
||||
if self.visible_tools.is_empty() {
|
||||
response.remove("tools");
|
||||
} else {
|
||||
response.insert("tools".to_owned(), Value::Array(self.visible_tools.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1125,6 +1391,8 @@ impl fmt::Debug for GrokResponseTransform {
|
||||
.field("alias_count", &self.aliases.len())
|
||||
.field("visible_tool_count", &self.visible_tools.len())
|
||||
.field("legacy_local_shell", &self.legacy_local_shell)
|
||||
.field("filter_x_search", &self.filter_x_search)
|
||||
.field("injected_tool_types", &self.injected_tool_types)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -1982,6 +2250,9 @@ impl ToolNormalizer {
|
||||
]),
|
||||
);
|
||||
}
|
||||
"allowed_tools" => {
|
||||
self.normalize_allowed_tools_choice(payload, object, normalized_tools)?;
|
||||
}
|
||||
"function" => self.normalize_function_tool_choice(payload, object)?,
|
||||
_ => {
|
||||
if let Some(hosted_kind) = normalize_hosted_tool_choice_kind(&kind) {
|
||||
@@ -2004,6 +2275,94 @@ impl ToolNormalizer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_allowed_tools_choice(
|
||||
&self,
|
||||
payload: &mut Map<String, Value>,
|
||||
mut object: Map<String, Value>,
|
||||
normalized_tools: &[Value],
|
||||
) -> Result<(), GrokRequestEncodeError> {
|
||||
if !matches!(
|
||||
object.get("mode").and_then(Value::as_str),
|
||||
Some("auto" | "required")
|
||||
) {
|
||||
return Err(GrokRequestEncodeError::InvalidRequestNormalization);
|
||||
}
|
||||
let allowed = object
|
||||
.get_mut("tools")
|
||||
.and_then(Value::as_array_mut)
|
||||
.filter(|tools| !tools.is_empty())
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
for allowed_tool in allowed {
|
||||
self.normalize_allowed_tool_reference(allowed_tool, normalized_tools)?;
|
||||
}
|
||||
payload.insert("tool_choice".to_owned(), Value::Object(object));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_allowed_tool_reference(
|
||||
&self,
|
||||
allowed_tool: &mut Value,
|
||||
normalized_tools: &[Value],
|
||||
) -> Result<(), GrokRequestEncodeError> {
|
||||
let object = allowed_tool
|
||||
.as_object_mut()
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
let kind = string_field(object, "type").trim().to_owned();
|
||||
match kind.as_str() {
|
||||
"function" => {
|
||||
let choice = match object.get_mut("function") {
|
||||
Some(Value::Object(function)) => function,
|
||||
Some(_) => return Err(GrokRequestEncodeError::InvalidRequestNormalization),
|
||||
None => object,
|
||||
};
|
||||
rewrite_namespace_choice(choice, &self.identity_aliases)?;
|
||||
let name = string_field(choice, "name").trim();
|
||||
if name.is_empty() || !has_named_tool(normalized_tools, "function", name) {
|
||||
return Err(GrokRequestEncodeError::InvalidRequestNormalization);
|
||||
}
|
||||
}
|
||||
"custom" => {
|
||||
let identity = ToolIdentity::new(
|
||||
ToolKind::Custom,
|
||||
string_field(object, "namespace").trim(),
|
||||
string_field(object, "name").trim(),
|
||||
);
|
||||
let alias = self
|
||||
.identity_aliases
|
||||
.get(&identity)
|
||||
.cloned()
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
object.insert("type".to_owned(), Value::String("function".to_owned()));
|
||||
object.insert("name".to_owned(), Value::String(alias));
|
||||
object.remove("namespace");
|
||||
}
|
||||
"apply_patch" | "tool_search" => {
|
||||
let identity = if kind == "apply_patch" {
|
||||
ToolIdentity::new(ToolKind::ApplyPatch, "", "apply_patch")
|
||||
} else {
|
||||
ToolIdentity::new(ToolKind::ToolSearch, "", "tool_search")
|
||||
};
|
||||
let alias = self
|
||||
.identity_aliases
|
||||
.get(&identity)
|
||||
.cloned()
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
object.clear();
|
||||
object.insert("type".to_owned(), Value::String("function".to_owned()));
|
||||
object.insert("name".to_owned(), Value::String(alias));
|
||||
}
|
||||
_ => {
|
||||
let hosted_kind = normalize_hosted_tool_choice_kind(&kind)
|
||||
.ok_or(GrokRequestEncodeError::InvalidRequestNormalization)?;
|
||||
if !has_tool_type(normalized_tools, hosted_kind) {
|
||||
return Err(GrokRequestEncodeError::InvalidRequestNormalization);
|
||||
}
|
||||
object.insert("type".to_owned(), Value::String(hosted_kind.to_owned()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_function_tool_choice(
|
||||
&self,
|
||||
payload: &mut Map<String, Value>,
|
||||
@@ -2080,6 +2439,13 @@ fn has_tool_type(tools: &[Value], kind: &str) -> bool {
|
||||
.any(|tool| tool.pointer("/type").and_then(Value::as_str) == Some(kind))
|
||||
}
|
||||
|
||||
fn has_named_tool(tools: &[Value], kind: &str, name: &str) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
tool.pointer("/type").and_then(Value::as_str) == Some(kind)
|
||||
&& tool.pointer("/name").and_then(Value::as_str) == Some(name)
|
||||
})
|
||||
}
|
||||
|
||||
fn tools_of_type(tools: &[Value], kind: &str) -> Vec<Value> {
|
||||
tools
|
||||
.iter()
|
||||
|
||||
@@ -324,7 +324,7 @@ async fn unauthorized_feedback_records_runtime_cooldown_without_persisting_accou
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cooldown_for_superseded_credential_does_not_block_rotated_account() {
|
||||
async fn account_scoped_cooldown_survives_credential_rotation() {
|
||||
let fixture = SelectorFixture::new(&["revision-fence"]).await;
|
||||
let id = account_id("revision-fence");
|
||||
let session = fixture
|
||||
@@ -369,17 +369,25 @@ async fn cooldown_for_superseded_credential_does_not_block_rotated_account() {
|
||||
CredentialCasOutcome::Updated(revision) if revision.get() == 2
|
||||
));
|
||||
|
||||
let next = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("replacement credential should not inherit cooldown");
|
||||
assert_eq!(next.account_id(), &id);
|
||||
assert!(fixture.cooldowns.cooldown(&id).is_none());
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await,
|
||||
Err(GrokSessionSelectorError::NoEligibleSession)
|
||||
));
|
||||
let account = fixture.store.account(&id).expect("rotated account");
|
||||
assert_eq!(account.revision().get(), 2);
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rate_limit_feedback_records_runtime_cooldown_without_persisting_account_state() {
|
||||
async fn rate_limit_feedback_persists_the_grok2api_cooldown_state() {
|
||||
let fixture = SelectorFixture::new(&["rate-limit", "available"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
@@ -397,7 +405,12 @@ async fn rate_limit_feedback_records_runtime_cooldown_without_persisting_account
|
||||
)
|
||||
.await;
|
||||
let account = fixture.store.account(&selected).expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -413,7 +426,35 @@ async fn rate_limit_feedback_records_runtime_cooldown_without_persisting_account
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_payment_required_feedback_only_records_runtime_account_cooldown() {
|
||||
async fn successful_request_clears_the_persisted_cooldown_state() {
|
||||
let fixture = SelectorFixture::new(&["cooldown-success"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("session");
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(
|
||||
&session,
|
||||
GrokCredentialFailure::RateLimited {
|
||||
retry_after: Some(Duration::from_secs(5)),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
fixture.selector.record_success(&session).await;
|
||||
|
||||
let account = fixture
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert!(account.cooldown_until().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn payment_required_feedback_persists_account_quota_exhaustion() {
|
||||
let fixture = SelectorFixture::new(&["payment-required", "available"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
@@ -437,9 +478,9 @@ async fn unknown_payment_required_feedback_only_records_runtime_account_cooldown
|
||||
.account(&selected)
|
||||
.expect("selected account")
|
||||
.availability(),
|
||||
AccountAvailability::Ready
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_some());
|
||||
assert!(fixture.cooldowns.cooldown(&selected).is_none());
|
||||
let next = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
@@ -538,7 +579,7 @@ async fn model_access_feedback_reports_model_cooldown_without_blocking_the_accou
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interrupted_stream_feedback_records_runtime_cooldown_without_persisting_account_state() {
|
||||
async fn interrupted_stream_feedback_persists_the_grok2api_cooldown_state() {
|
||||
let fixture = SelectorFixture::new(&["stream-interrupted"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
@@ -554,7 +595,12 @@ async fn interrupted_stream_feedback_records_runtime_cooldown_without_persisting
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account");
|
||||
assert_eq!(account.availability(), AccountAvailability::Ready);
|
||||
assert_eq!(account.availability(), AccountAvailability::Cooldown);
|
||||
assert!(
|
||||
account
|
||||
.cooldown_until()
|
||||
.is_some_and(|until| until > SystemTime::now())
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.cooldowns
|
||||
@@ -595,6 +641,75 @@ async fn quota_feedback_uses_common_quota_exhausted_state() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_feedback_should_follow_the_account_across_a_credential_rotation() {
|
||||
let fixture = SelectorFixture::new(&["quota-rotation"]).await;
|
||||
let id = account_id("quota-rotation");
|
||||
let session = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("session");
|
||||
let current = fixture
|
||||
.store
|
||||
.load_credential(&id, CredentialRevision::new(1).expect("revision"))
|
||||
.await
|
||||
.expect("current credential");
|
||||
let prepared = GrokCredentialAdmin
|
||||
.prepare_rotation(&RotateManagedGrokCredential {
|
||||
current,
|
||||
secret: provider_xai::GrokOAuthSecret {
|
||||
access_token: provider_xai::SecretValue::new("new-access"),
|
||||
refresh_token: provider_xai::SecretValue::new("new-refresh"),
|
||||
id_token: None,
|
||||
scope: provider_xai::OFFICIAL_SCOPES.join(" "),
|
||||
},
|
||||
verified_account: crate::support::profile("subject-quota-rotation"),
|
||||
next_refresh_at: chrono::Utc::now() + chrono::Duration::minutes(30),
|
||||
})
|
||||
.expect("rotate");
|
||||
fixture
|
||||
.store
|
||||
.compare_and_swap_credential(prepared.credential)
|
||||
.await
|
||||
.expect("persist rotation");
|
||||
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(&session, GrokCredentialFailure::FreeQuotaExhausted)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
fixture.store.account(&id).expect("account").availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_success_should_not_clear_a_confirmed_quota_exhaustion() {
|
||||
let fixture = SelectorFixture::new(&["quota-success"]).await;
|
||||
let session = fixture
|
||||
.selector
|
||||
.select(fixture.request(BTreeSet::new()))
|
||||
.await
|
||||
.expect("session");
|
||||
fixture
|
||||
.selector
|
||||
.record_failure(&session, GrokCredentialFailure::FreeQuotaExhausted)
|
||||
.await;
|
||||
|
||||
fixture.selector.record_success(&session).await;
|
||||
|
||||
assert_eq!(
|
||||
fixture
|
||||
.store
|
||||
.account(session.account_id())
|
||||
.expect("account")
|
||||
.availability(),
|
||||
AccountAvailability::QuotaExhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn excluded_account_is_never_selected_again() {
|
||||
let fixture = SelectorFixture::new(&["excluded"]).await;
|
||||
|
||||
@@ -106,6 +106,15 @@ const SUCCESS_SSE: &[u8] = concat!(
|
||||
)
|
||||
.as_bytes();
|
||||
|
||||
fn replay_ciphertext(seed: u8) -> String {
|
||||
use base64::Engine as _;
|
||||
|
||||
let bytes = (0_u16..128)
|
||||
.map(|value| (value as u8).wrapping_add(seed))
|
||||
.collect::<Vec<_>>();
|
||||
base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes)
|
||||
}
|
||||
|
||||
fn stateful_sse(encrypted_content: &str) -> Vec<u8> {
|
||||
format!(
|
||||
concat!(
|
||||
@@ -121,6 +130,45 @@ fn stateful_sse(encrypted_content: &str) -> Vec<u8> {
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn stateful_sse_with_assistant(encrypted_content: &str, assistant: &str) -> Vec<u8> {
|
||||
let assistant = serde_json::to_string(assistant).expect("assistant text");
|
||||
format!(
|
||||
concat!(
|
||||
"event: response.created\n",
|
||||
"data: {{\"type\":\"response.created\",\"response\":{{\"id\":\"resp_state\",\"model\":\"grok-4.5\"}}}}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{{\"id\":\"reason_account_bound\",\"type\":\"reasoning\",\"status\":\"completed\",\"summary\":[],\"content\":null,\"encrypted_content\":\"{}\"}}}}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {{\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{{\"id\":\"message_account_bound\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{{\"type\":\"output_text\",\"text\":{assistant}}}]}}}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_state\",\"model\":\"grok-4.5\",\"status\":\"completed\"}}}}\n\n"
|
||||
),
|
||||
encrypted_content,
|
||||
assistant = assistant,
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn stateful_sse_with_custom_tool_call(encrypted_content: &str, input: &str) -> Vec<u8> {
|
||||
let arguments = serde_json::to_string(&json!({"input": input})).expect("tool arguments");
|
||||
let arguments_json = serde_json::to_string(&arguments).expect("arguments JSON string");
|
||||
format!(
|
||||
concat!(
|
||||
"event: response.created\n",
|
||||
"data: {{\"type\":\"response.created\",\"response\":{{\"id\":\"resp_state\",\"model\":\"grok-4.5\"}}}}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{{\"id\":\"reason_account_bound\",\"type\":\"reasoning\",\"status\":\"completed\",\"summary\":[],\"content\":null,\"encrypted_content\":\"{}\"}}}}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {{\"type\":\"response.output_item.done\",\"output_index\":1,\"item\":{{\"id\":\"item_exec\",\"type\":\"function_call\",\"status\":\"completed\",\"call_id\":\"call_exec\",\"name\":\"exec\",\"arguments\":{arguments_json}}}}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_state\",\"model\":\"grok-4.5\",\"status\":\"completed\"}}}}\n\n"
|
||||
),
|
||||
encrypted_content,
|
||||
arguments_json = arguments_json,
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn custom_apply_patch_sse(patch: &str) -> Vec<u8> {
|
||||
let arguments = serde_json::to_string(&json!({"patch": patch})).expect("patch arguments");
|
||||
let arguments_json = serde_json::to_string(&arguments).expect("arguments JSON string");
|
||||
@@ -278,6 +326,54 @@ impl GrokSessionSelector for StubSelector {
|
||||
}
|
||||
}
|
||||
|
||||
struct SequencedAccountSelector {
|
||||
accounts: Mutex<VecDeque<gateway_core::engine::credential::ProviderAccountId>>,
|
||||
}
|
||||
|
||||
impl SequencedAccountSelector {
|
||||
fn new(
|
||||
accounts: impl IntoIterator<Item = gateway_core::engine::credential::ProviderAccountId>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
accounts: Mutex::new(accounts.into_iter().collect()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl GrokSessionSelector for SequencedAccountSelector {
|
||||
fn select(&self, request: GrokSessionSelection) -> GrokSessionSelectorFuture<'_> {
|
||||
let selected = self.accounts.lock().expect("account sequence").pop_front();
|
||||
Box::pin(async move {
|
||||
let id = selected.ok_or(GrokSessionSelectorError::NoEligibleSession)?;
|
||||
if request.excluded_accounts().contains(&id)
|
||||
|| request
|
||||
.required_account()
|
||||
.is_some_and(|required| required != &id)
|
||||
{
|
||||
return Err(GrokSessionSelectorError::NoEligibleSession);
|
||||
}
|
||||
SelectedGrokSession::new(
|
||||
id,
|
||||
CredentialRevision::new(1).expect("revision"),
|
||||
SecretValue::new("oauth-access"),
|
||||
SecretValue::new("verified-user"),
|
||||
Some(SecretValue::new("user@example.com")),
|
||||
GrokSessionBinding::new("acct_provider").expect("binding"),
|
||||
(),
|
||||
)
|
||||
.map_err(|_| GrokSessionSelectorError::InvalidSession)
|
||||
})
|
||||
}
|
||||
|
||||
fn record_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a SelectedGrokSession,
|
||||
_: GrokCredentialFailure,
|
||||
) -> GrokCredentialFeedbackFuture<'a> {
|
||||
Box::pin(async {})
|
||||
}
|
||||
}
|
||||
|
||||
enum InferenceMode {
|
||||
Success,
|
||||
SuccessWithMetrics(GrokInferenceTransportMetrics),
|
||||
@@ -481,7 +577,7 @@ async fn provider_with_recovery(
|
||||
}
|
||||
|
||||
async fn provider_with_catalog_transport(
|
||||
selector: Arc<StubSelector>,
|
||||
selector: Arc<dyn GrokSessionSelector>,
|
||||
transport: Arc<StubInferenceTransport>,
|
||||
recovery: Arc<StubRecovery>,
|
||||
catalog_transport: Arc<dyn GrokModelCatalogTransport>,
|
||||
@@ -569,6 +665,36 @@ fn operation() -> Operation {
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload))
|
||||
}
|
||||
|
||||
fn reasoning_replay_operation(session: &str, input: serde_json::Value) -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("prompt_cache_key".to_owned(), json!(session)),
|
||||
("input".to_owned(), input),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload");
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload))
|
||||
}
|
||||
|
||||
fn contextual_reasoning_replay_operation(session: &str, input: serde_json::Value) -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), input),
|
||||
("tools".to_owned(), json!([{"type":"custom","name":"exec"}])),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload")
|
||||
.with_context(Map::from_iter([(
|
||||
"conversation_id".to_owned(),
|
||||
json!(session),
|
||||
)]));
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload))
|
||||
}
|
||||
|
||||
fn operation_with_reasoning_effort(effort: &str) -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
@@ -621,6 +747,26 @@ fn compaction_operation() -> Operation {
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload))
|
||||
}
|
||||
|
||||
fn compaction_operation_with_prompt_cache(session: &str) -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("prompt_cache_key".to_owned(), json!(session)),
|
||||
(
|
||||
"input".to_owned(),
|
||||
json!([
|
||||
{"type": "message", "role": "user", "content": "history"},
|
||||
{"type": "compaction_trigger"}
|
||||
]),
|
||||
),
|
||||
("stream".to_owned(), json!(true)),
|
||||
]),
|
||||
)
|
||||
.expect("OpenAI payload");
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload))
|
||||
}
|
||||
|
||||
fn compaction_operation_with_state(state: ProviderSessionState) -> Operation {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
@@ -737,6 +883,18 @@ fn context_with_continuation_attempt(
|
||||
context(CancellationToken::new(), Some(continuation)).with_continuation_attempt(attempt)
|
||||
}
|
||||
|
||||
async fn execute_successfully(provider: &GrokBuildProvider, operation: Operation) {
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
provider_request_with_operation("xai", operation),
|
||||
context(CancellationToken::new(), None),
|
||||
)
|
||||
.await
|
||||
.expect("provider stream");
|
||||
let events = stream.by_ref().collect::<Vec<_>>().await;
|
||||
assert!(events.iter().all(Result::is_ok));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_observation_preserves_raw_xai_reasoning_effort() {
|
||||
let provider = provider(StubSelector::success(), StubInferenceTransport::success()).await;
|
||||
@@ -1300,7 +1458,7 @@ async fn explicit_quota_http_429_marks_provider_error_replay_safe() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_http_402_is_retryable_without_account_quota_feedback() {
|
||||
async fn unknown_http_402_is_retryable_with_payment_quota_feedback() {
|
||||
let selector = StubSelector::success();
|
||||
let transport = StubInferenceTransport::error(
|
||||
GrokInferenceTransportError::new(
|
||||
@@ -1750,6 +1908,239 @@ async fn external_previous_response_is_rejected_before_selection() {
|
||||
assert_eq!(selector.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_replay_is_reused_only_for_the_same_explicit_session() {
|
||||
let encrypted_content = replay_ciphertext(0);
|
||||
let transport = StubInferenceTransport::sequence([
|
||||
InferenceMode::SuccessBody(stateful_sse(&encrypted_content)),
|
||||
InferenceMode::Success,
|
||||
InferenceMode::Success,
|
||||
]);
|
||||
let provider = provider(StubSelector::success(), transport.clone()).await;
|
||||
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"session-a",
|
||||
json!([{"type":"message","role":"user","content":"first"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"session-b",
|
||||
json!([{"type":"message","role":"user","content":"other"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"session-a",
|
||||
json!([{"type":"message","role":"user","content":"next"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
let requests = transport.requests.lock().expect("requests");
|
||||
let other: serde_json::Value =
|
||||
serde_json::from_slice(requests[1].body()).expect("other session body");
|
||||
let replayed: serde_json::Value =
|
||||
serde_json::from_slice(requests[2].body()).expect("replayed body");
|
||||
assert!(
|
||||
other["input"]
|
||||
.as_array()
|
||||
.is_some_and(|input| input.iter().all(|item| item["type"] != "reasoning"))
|
||||
);
|
||||
assert_eq!(
|
||||
replayed.pointer("/input/0/encrypted_content"),
|
||||
Some(&json!(encrypted_content))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_replay_should_pair_context_scoped_custom_call_with_its_output() {
|
||||
let encrypted_content = replay_ciphertext(4);
|
||||
let transport = StubInferenceTransport::sequence([
|
||||
InferenceMode::SuccessBody(stateful_sse_with_custom_tool_call(
|
||||
&encrypted_content,
|
||||
"inspect skills",
|
||||
)),
|
||||
InferenceMode::Success,
|
||||
]);
|
||||
let provider = provider(StubSelector::success(), transport.clone()).await;
|
||||
|
||||
execute_successfully(
|
||||
&provider,
|
||||
contextual_reasoning_replay_operation(
|
||||
"conversation-from-header",
|
||||
json!([{"type":"message","role":"user","content":"find skills"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
contextual_reasoning_replay_operation(
|
||||
"conversation-from-header",
|
||||
json!([{
|
||||
"type":"custom_tool_call_output",
|
||||
"call_id":"call_exec",
|
||||
"output":[{"type":"input_text","text":"skills found"}]
|
||||
}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
let requests = transport.requests.lock().expect("requests");
|
||||
let replayed: serde_json::Value =
|
||||
serde_json::from_slice(requests[1].body()).expect("replayed body");
|
||||
let item_types = replayed["input"]
|
||||
.as_array()
|
||||
.expect("replayed input")
|
||||
.iter()
|
||||
.filter_map(|item| item["type"].as_str())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(
|
||||
item_types,
|
||||
["reasoning", "function_call", "function_call_output"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_replay_is_scoped_to_the_selected_account() {
|
||||
let encrypted_content = replay_ciphertext(1);
|
||||
let transport = StubInferenceTransport::sequence([
|
||||
InferenceMode::SuccessBody(stateful_sse(&encrypted_content)),
|
||||
InferenceMode::Success,
|
||||
InferenceMode::Success,
|
||||
]);
|
||||
let selector = SequencedAccountSelector::new([
|
||||
account_id("replay-a"),
|
||||
account_id("replay-b"),
|
||||
account_id("replay-a"),
|
||||
]);
|
||||
let provider = provider_with_catalog_transport(
|
||||
selector,
|
||||
transport.clone(),
|
||||
StubRecovery::new(GrokCredentialRecoveryOutcome::Unavailable),
|
||||
Arc::new(StaticCatalogTransport),
|
||||
)
|
||||
.await;
|
||||
|
||||
for prompt in ["first", "different account", "owner again"] {
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"shared-session",
|
||||
json!([{"type":"message","role":"user","content":prompt}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let requests = transport.requests.lock().expect("requests");
|
||||
let different_account: serde_json::Value =
|
||||
serde_json::from_slice(requests[1].body()).expect("different account request body");
|
||||
let owner_again: serde_json::Value =
|
||||
serde_json::from_slice(requests[2].body()).expect("owner request body");
|
||||
assert!(
|
||||
different_account["input"]
|
||||
.as_array()
|
||||
.is_some_and(|input| input.iter().all(|item| item["type"] != "reasoning"))
|
||||
);
|
||||
assert_eq!(
|
||||
owner_again.pointer("/input/0/encrypted_content"),
|
||||
Some(&json!(encrypted_content))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reasoning_replay_rejects_mismatched_assistant_history() {
|
||||
let encrypted_content = replay_ciphertext(2);
|
||||
let transport = StubInferenceTransport::sequence([
|
||||
InferenceMode::SuccessBody(stateful_sse_with_assistant(
|
||||
&encrypted_content,
|
||||
"cached answer",
|
||||
)),
|
||||
InferenceMode::Success,
|
||||
]);
|
||||
let provider = provider(StubSelector::success(), transport.clone()).await;
|
||||
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"assistant-session",
|
||||
json!([{"type":"message","role":"user","content":"first"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"assistant-session",
|
||||
json!([
|
||||
{"type":"message","role":"user","content":"first"},
|
||||
{"type":"message","role":"assistant","content":[{"type":"output_text","text":"different answer"}]},
|
||||
{"type":"message","role":"user","content":"next"}
|
||||
]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
let requests = transport.requests.lock().expect("requests");
|
||||
let body: serde_json::Value = serde_json::from_slice(requests[1].body()).expect("second body");
|
||||
assert!(
|
||||
body["input"]
|
||||
.as_array()
|
||||
.is_some_and(|input| input.iter().all(|item| item["type"] != "reasoning"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successful_compaction_clears_reasoning_replay_for_the_session() {
|
||||
let encrypted_content = replay_ciphertext(3);
|
||||
let summary = valid_compaction_summary("reasoning replay cleared");
|
||||
let transport = StubInferenceTransport::sequence([
|
||||
InferenceMode::SuccessBody(stateful_sse(&encrypted_content)),
|
||||
InferenceMode::SuccessBody(compaction_sse(&summary, None)),
|
||||
InferenceMode::Success,
|
||||
]);
|
||||
let provider = provider(StubSelector::success(), transport.clone()).await;
|
||||
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"compact-session",
|
||||
json!([{"type":"message","role":"user","content":"first"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
compaction_operation_with_prompt_cache("compact-session"),
|
||||
)
|
||||
.await;
|
||||
execute_successfully(
|
||||
&provider,
|
||||
reasoning_replay_operation(
|
||||
"compact-session",
|
||||
json!([{"type":"message","role":"user","content":"after compact"}]),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
let requests = transport.requests.lock().expect("requests");
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(requests[2].body()).expect("post-compaction body");
|
||||
assert!(
|
||||
body["input"]
|
||||
.as_array()
|
||||
.is_some_and(|input| input.iter().all(|item| item["type"] != "reasoning"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_state_inherits_session_and_recovers_reasoning_on_pinned_account() {
|
||||
use base64::Engine as _;
|
||||
|
||||
@@ -642,6 +642,76 @@ fn decoder_should_preserve_image_and_hosted_tool_items_without_inventing_facts()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_hide_injected_search_calls_and_compact_later_output_indexes() {
|
||||
let request = tool_request(serde_json::json!({
|
||||
"model": "client",
|
||||
"prompt_cache_key": "cache-session",
|
||||
"input": "hello"
|
||||
}));
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_cache\",\"tools\":[{\"type\":\"web_search\"},{\"type\":\"x_search\"}]}}\n\n",
|
||||
"event: response.output_item.added\n",
|
||||
"data: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"id\":\"item_xs\",\"call_id\":\"xs_call_1\",\"name\":\"x_semantic_search\",\"arguments\":\"{}\"}}\n\n",
|
||||
"event: response.function_call_arguments.delta\n",
|
||||
"data: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"item_id\":\"item_xs\",\"call_id\":\"xs_call_1\",\"delta\":\"{}\"}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"id\":\"item_xs\",\"call_id\":\"xs_call_1\",\"name\":\"x_semantic_search\",\"arguments\":\"{}\"}}\n\n",
|
||||
"event: response.output_item.added\n",
|
||||
"data: {\"type\":\"response.output_item.added\",\"output_index\":1,\"item\":{\"type\":\"image_generation_call\",\"id\":\"image_visible\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_cache\",\"status\":\"completed\",\"tools\":[{\"type\":\"web_search\"},{\"type\":\"x_search\"}],\"output\":[{\"type\":\"function_call\",\"call_id\":\"xs_call_1\",\"name\":\"x_semantic_search\",\"arguments\":\"{}\"},{\"type\":\"image_generation_call\",\"id\":\"image_visible\"}]}}\n\n",
|
||||
);
|
||||
|
||||
let events = GrokCanonicalDecoder::for_request("grok-4.5", &request)
|
||||
.push(body.as_bytes())
|
||||
.expect("filtered cache route response");
|
||||
let wire = wire_events(&events);
|
||||
|
||||
assert_eq!(wire.len(), 3);
|
||||
assert_eq!(
|
||||
wire[1].data().get("output_index"),
|
||||
Some(&serde_json::json!(0))
|
||||
);
|
||||
assert_eq!(
|
||||
wire[2].data().pointer("/response/output/0/id"),
|
||||
Some(&serde_json::json!("image_visible"))
|
||||
);
|
||||
assert_eq!(wire[2].data().pointer("/response/output/1"), None);
|
||||
assert_eq!(wire[2].data().pointer("/response/tools"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_an_internal_search_name_declared_by_the_client() {
|
||||
let request = tool_request(serde_json::json!({
|
||||
"model": "client",
|
||||
"prompt_cache_key": "cache-session",
|
||||
"input": "hello",
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"name": "x_user_search",
|
||||
"parameters": {"type": "object"}
|
||||
}]
|
||||
}));
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_declared\"}}\n\n",
|
||||
"event: response.output_item.added\n",
|
||||
"data: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"id\":\"item_client\",\"call_id\":\"call_client\",\"name\":\"x_user_search\",\"arguments\":\"{}\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_declared\",\"status\":\"completed\"}}\n\n",
|
||||
);
|
||||
|
||||
let events = GrokCanonicalDecoder::for_request("grok-4.5", &request)
|
||||
.push(body.as_bytes())
|
||||
.expect("client-declared search function");
|
||||
|
||||
assert!(wire_events(&events).iter().any(|event| {
|
||||
event.data().pointer("/item/name") == Some(&serde_json::json!("x_user_search"))
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_tolerate_mismatched_event_identity_and_preserve_the_wire_event() {
|
||||
let body = concat!(
|
||||
|
||||
@@ -91,7 +91,7 @@ fn headers_should_bind_identity_to_the_selected_oauth_account() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_should_keep_openai_cache_requests_out_of_shell_session_mode() {
|
||||
fn headers_should_send_both_stable_session_headers_without_inventing_a_turn_index() {
|
||||
let headers = build_grok_headers(
|
||||
&crate::support::xai_wire_profile(),
|
||||
&selected_session(),
|
||||
@@ -109,7 +109,7 @@ fn headers_should_keep_openai_cache_requests_out_of_shell_session_mode() {
|
||||
};
|
||||
|
||||
assert_eq!(value("x-grok-conv-id"), Some("session-fixture"));
|
||||
assert_eq!(value("x-grok-session-id"), None);
|
||||
assert_eq!(value("x-grok-session-id"), Some("session-fixture"));
|
||||
assert_eq!(value("x-grok-turn-idx"), None);
|
||||
}
|
||||
|
||||
|
||||
@@ -256,7 +256,7 @@ fn explicit_session_should_enable_the_noop_native_cache_route() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_session_should_not_add_invokable_cache_tools_to_client_tool_requests() {
|
||||
fn explicit_session_should_add_only_internal_x_search_to_client_tool_requests() {
|
||||
let request = raw_request(json!({
|
||||
"model": "client",
|
||||
"prompt_cache_key": "conversation-42",
|
||||
@@ -273,15 +273,15 @@ fn explicit_session_should_not_add_invokable_cache_tools_to_client_tool_requests
|
||||
GrokResponsesRequest::encode(&request, "grok-4.5", &client_key()).expect("request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
|
||||
// 缓存路由不得扩大模型可调用的工具集:带客户端工具时不注入原生搜索。
|
||||
assert_eq!(
|
||||
body.pointer("/tools"),
|
||||
Some(&json!([{
|
||||
body.pointer("/tools/0"),
|
||||
Some(&json!({
|
||||
"type": "function",
|
||||
"name": "read_file",
|
||||
"parameters": {"type": "object"}
|
||||
}]))
|
||||
}))
|
||||
);
|
||||
assert_eq!(body.pointer("/tools/1"), Some(&json!({"type": "x_search"})));
|
||||
assert_eq!(body.pointer("/tool_choice"), Some(&json!("auto")));
|
||||
}
|
||||
|
||||
@@ -384,7 +384,7 @@ fn function_parameters_should_report_the_invalid_nullable_root_field() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_session_should_not_inject_cache_tools_after_codex_additional_tools_normalization() {
|
||||
fn explicit_session_should_add_x_search_after_codex_additional_tools_normalization() {
|
||||
let request = raw_request(json!({
|
||||
"model": "client",
|
||||
"prompt_cache_key": "conversation-42",
|
||||
@@ -417,11 +417,38 @@ fn explicit_session_should_not_inject_cache_tools_after_codex_additional_tools_n
|
||||
Some(&json!(["patch"]))
|
||||
);
|
||||
assert_eq!(body.pointer("/tools/1/name"), Some(&json!("read_file")));
|
||||
// 归一化装载的会话工具视同客户端工具:不再追加可被调用的原生搜索。
|
||||
assert_eq!(body.pointer("/tools/2"), None);
|
||||
assert_eq!(body.pointer("/tools/2"), Some(&json!({"type": "x_search"})));
|
||||
assert_eq!(body.pointer("/tool_choice"), Some(&json!("auto")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_session_should_add_x_search_to_allowed_tools_choice() {
|
||||
let request = raw_request(json!({
|
||||
"model": "client",
|
||||
"prompt_cache_key": "conversation-42",
|
||||
"input": "first",
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"name": "read_file",
|
||||
"parameters": {"type": "object"}
|
||||
}],
|
||||
"tool_choice": {
|
||||
"type": "allowed_tools",
|
||||
"mode": "auto",
|
||||
"tools": [{"type": "function", "name": "read_file"}]
|
||||
}
|
||||
}));
|
||||
|
||||
let encoded =
|
||||
GrokResponsesRequest::encode(&request, "grok-4.5", &client_key()).expect("request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
|
||||
assert_eq!(
|
||||
body.pointer("/tool_choice/tools/1"),
|
||||
Some(&json!({"type": "x_search"}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_session_should_not_duplicate_existing_native_cache_tools() {
|
||||
let request = raw_request(json!({
|
||||
|
||||
@@ -2,3 +2,4 @@
|
||||
0ae3945c02461347ab8dcfc64821027cd2366505778e0450d71bd82772c4e99d 0002_drop_opaque_response_id_index.sql
|
||||
e608ed89991f9d6317491714447a00a61ede90caca4fb030eb30695654afb664 0003_store_opaque_response_ids_as_bytes.sql
|
||||
df4e02cf2335733326c12429ced4cbd0b35b8a22b3fc24981c3c0549477d7c43 0004_add_model_request_service_tier.sql
|
||||
fe8f36913c6db7053148db450f05e008dab6cdb9ca8cdcfcb22d97ad9a87bf49 0005_allow_quota_exhausted_cooldown.sql
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
alter table provider_accounts
|
||||
drop constraint provider_accounts_cooldown_ck,
|
||||
add constraint provider_accounts_cooldown_ck check (
|
||||
(
|
||||
availability != 'cooldown'
|
||||
or cooldown_until is not null
|
||||
)
|
||||
and (
|
||||
cooldown_until is null
|
||||
or availability in ('cooldown', 'quota_exhausted')
|
||||
)
|
||||
);
|
||||
+9
-3
@@ -77,9 +77,9 @@ PostgreSQL/Redis 启动密码。日常校验使用 `config --quiet`。
|
||||
进程退出终止。drain 结束后才关停后台 worker,预算为
|
||||
`host.worker_shutdown_timeout_seconds`(默认 30 秒),两段预算按最坏情况串联。
|
||||
|
||||
Compose 的 `stop_grace_period` 为 30 秒,覆盖空闲与常规负载下的关停;若存量长连接把两段
|
||||
预算全部用满(合计 60 秒),Docker 会在宽限期结束时 SIGKILL。要求最坏情况下也完整走完
|
||||
drain 与 worker 收尾时,把 `stop_grace_period` 调到大于两个超时之和。
|
||||
Compose 的 `stop_grace_period` 为 75 秒,覆盖默认 30 秒 HTTP drain、30 秒 worker 收尾和额外调度
|
||||
余量。若调大任一应用超时,也必须把 `stop_grace_period` 调到大于两段超时之和;否则 Docker 会在
|
||||
宽限期结束时 SIGKILL。
|
||||
|
||||
## 本地开发
|
||||
|
||||
@@ -154,6 +154,12 @@ release/publish <version>
|
||||
```
|
||||
|
||||
该脚本负责更新 `release/version.yaml`、创建版本提交和带注释 tag,并原子推送分支与 tag。
|
||||
它不会登录任何服务器,也不会拉取镜像或调用管理端在线更新。
|
||||
|
||||
源码提交、仓库发版和运行实例升级是三种独立状态:本地 commit 不等于 Release,Release/tag 和镜像
|
||||
已生成也不等于实例已升级。判断某项修复是否在线前,应先通过管理端版本接口或容器 image digest
|
||||
确认运行实例的实际 revision;实例只有在执行上面的 Compose pull/up,或成功完成管理端在线更新后
|
||||
才会改变。
|
||||
|
||||
构建元数据仍可作为一次性进程环境传入,不需要 `.env` 文件:
|
||||
|
||||
|
||||
+39
-9
@@ -55,7 +55,7 @@ Admin API 不暴露全局配置版本,mutation 请求也不要求客户端提
|
||||
|
||||
## 3. Responses 与模型目录
|
||||
|
||||
请求正文上限为 16 MiB。
|
||||
Responses HTTP body、WebSocket message 和 frame 不设置网关私有长度上限;协议可接受性由上游决定。
|
||||
|
||||
| 方法 | 路由 | 说明 |
|
||||
| --- | --- | --- |
|
||||
@@ -70,10 +70,12 @@ Admin API 不暴露全局配置版本,mutation 请求也不要求客户端提
|
||||
`GET /v1/models` 默认返回 OpenAI 兼容列表 `{"object": "list", "data": [...]}`;请求携带非空
|
||||
`client_version` query 参数(Codex 客户端)时改为返回 Codex 专用目录合同 `{"models": [...]}`。
|
||||
|
||||
OpenAI 路径保留客户端 Responses wire 语义,上游事件字节在 HTTP SSE 与 WebSocket 两条链路上都
|
||||
原样转发;xAI 是 Grok wire 与 Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成,
|
||||
上游结构化错误的 message/code/type 会透传给客户端,其中内嵌的账号指纹 UUID 已脱敏。模型映射
|
||||
是全局精确映射,未命中时模型名原样交给所属 Provider。
|
||||
OpenAI 路径保留客户端 Responses wire 语义:请求 body 的未知字段和字段顺序保持不变(受控模型
|
||||
映射除外),HTTP SSE 与 WebSocket 的上游业务事件字节原样转发,response ID 按 opaque 值处理而不
|
||||
假设 UUID 或固定长度;OpenAI 上游错误 envelope 和允许下发的 opaque header 值也不由 canonical
|
||||
观测结果重写。xAI 是 Grok wire 与 Responses wire 之间的协议转换层,转换只在 xAI Provider 内完成。
|
||||
上游结构化错误的 message/code/type 会透传给客户端,其中内嵌的账号指纹 UUID 已脱敏。模型映射是
|
||||
全局精确映射,未命中时模型名原样交给所属 Provider。
|
||||
|
||||
## 4. 管理员认证
|
||||
|
||||
@@ -94,17 +96,17 @@ OpenAI 路径保留客户端 Responses wire 语义,上游事件字节在 HTTP
|
||||
| `GET` | `/api/admin/accounts/detail` | `accountId` | 查询账号详情、额度和本地用量 |
|
||||
| `GET` | `/api/admin/accounts/export` | `accountIds`、`confirm=export_sensitive_accounts` | 显式导出最多 200 个账号的敏感 Provider 文档 |
|
||||
| `POST` | `/api/admin/accounts/import` | `{ provider, data }` | 导入或按上游身份更新账号 |
|
||||
| `POST` | `/api/admin/accounts/refresh` | `{ accountId }` | 手工刷新账号 credential |
|
||||
| `POST` | `/api/admin/accounts/refresh` | `{ accountId }` | 手工刷新 OAuth credential(AT/RT),不刷新额度 |
|
||||
| `POST` | `/api/admin/accounts/rotate` | OpenAI rotation 字段 | 手工替换 OpenAI OAuth token |
|
||||
| `POST` | `/api/admin/accounts/enable` | `{ provider, accountId }` | 启用账号 |
|
||||
| `POST` | `/api/admin/accounts/disable` | `{ provider, accountId }` | 禁用账号 |
|
||||
| `POST` | `/api/admin/accounts/delete` | `{ provider, accountIds }` | 批量删除 1–200 个账号 |
|
||||
| `GET` | `/api/admin/accounts/quota` | `accountId` | 读取当前额度,不强制访问上游 |
|
||||
| `POST` | `/api/admin/accounts/quota/refresh` | `{ accountId }` | 立即刷新该账号额度 |
|
||||
| `POST` | `/api/admin/accounts/quota/refresh` | `{ accountId }` | 访问 Provider 并刷新额度,同时同步额度所属状态 |
|
||||
| `GET` | `/api/admin/accounts/models` | `accountId` | 优先读取该 Provider + 套餐的模型 cache,缺失时有限实时拉取 |
|
||||
| `POST` | `/api/admin/accounts/models/refresh` | `{ accountId }` | 强制拉取最新模型并覆盖 cache |
|
||||
| `GET` | `/api/admin/accounts/connection-test` | `accountId`、`modelId` | 通过 SSE 返回实时连接测试事件,不作为业务 Responses 用量记录 |
|
||||
| `POST` | `/api/admin/accounts/oauth/start` | OAuth start 字段 | 创建 OpenAI 或 xAI OAuth flow |
|
||||
| `POST` | `/api/admin/accounts/oauth/start` | `{ provider, name, accountId? }` | 创建 OpenAI 或 xAI OAuth flow;`accountId` 表示重新授权 |
|
||||
| `POST` | `/api/admin/accounts/oauth/complete` | `{ provider, flowId, callbackUrl }` | 消费 OAuth callback 并写入账号 |
|
||||
|
||||
账号列表支持以下稳定值:
|
||||
@@ -163,7 +165,30 @@ OAuth start 使用:
|
||||
}
|
||||
```
|
||||
|
||||
重新授权已有账号时只需提供 `accountId`。
|
||||
重新授权已有账号时,start 请求仍携带 `provider` 和展示用 `name`,只额外提供目标 `accountId`;
|
||||
客户端不得提交 `credentialRevision`、旧 token 身份或其他并发控制字段。complete 请求也不重复提交
|
||||
`accountId`,后端通过 `flowId` 中保存的目标绑定完成授权。
|
||||
|
||||
### OpenAI 身份、额度与状态
|
||||
|
||||
- 导入会校验签名 access token 和稳定账号身份;`/wham/usage` 用于补全账号/套餐事实和额度,
|
||||
不是判断 token 是否有效的唯一依据。
|
||||
- 账号文件导入和首次 OAuth 创建在 credential 提交后立即尝试一次额度观测。观测失败只记录告警,
|
||||
不回滚已提交的账号;重新授权只轮换目标账号 credential,不隐式等同于手工额度刷新。
|
||||
- 重新授权在 start 时绑定目标账号,complete 时重新读取目标的当前 credential revision,并验证上游
|
||||
account/user 身份没有换绑。Free token 缺少 `chatgpt_account_id` 时,后端使用目标账号 header 调用
|
||||
`/wham/usage` 交叉确认,而不是把该账号推断成 K12。
|
||||
- OAuth pending flow 先取得带过期时间的独占 claim,只有账号事务提交成功后才消费。失败会释放 claim,
|
||||
但上游 authorization code 本身通常只能交换一次;已完成过 token exchange 时应重新创建 OAuth flow。
|
||||
- `GET /accounts/quota` 只读取最后一次落库快照;`POST /accounts/quota/refresh` 才访问上游。access token
|
||||
已过期时,额度刷新要求先走 credential 刷新或重新授权,不会拿过期 token 探测额度。
|
||||
- 成功额度观测会 revision-fenced 写入 quota,并在 `active` 与 `quota_exhausted` 等额度所属状态间同步;
|
||||
不会清除 `expired`、`banned` 等 credential/封禁终态。额度接口的 401/403 也不足以判定 refresh token
|
||||
永久失效,credential 终态只由 OAuth refresh 或身份校验链路写入。
|
||||
- 正常 Responses 请求会解析上游响应的 rate-limit headers,合并进同一 quota 快照并同步状态。Free、
|
||||
K12 等套餐共用该状态机;套餐只参与账号展示和按套餐隔离的模型目录 cache,不存在 K12 专属额度路径。
|
||||
- 账号页没有定时静默轮询。手工额度刷新完成后页面会重新读取该账号;请求驱动或后台任务产生的状态
|
||||
变化,需要下一次显式查询账号列表后才会显示。
|
||||
|
||||
## 6. Client Key
|
||||
|
||||
@@ -225,6 +250,11 @@ request/response/upstream ID、outcome 与搜索文本。诊断 `dimension` 可
|
||||
汇总与洞察中的请求数与 outcome 分布覆盖筛选范围内全部请求;token、缓存、延迟与成本聚合仅统计
|
||||
已完整交付客户端的成功响应。
|
||||
|
||||
OpenAI 的 `serviceTier` 只接受上游响应生命周期事件确认的实际 `response.service_tier`;请求里的
|
||||
期望档位只保留在 request summary,不能冒充响应事实。计费展示把 `priority`/`fast` 映射为 `Fast`,
|
||||
`flex` 映射为 `Flex`,缺失或 `default` 映射为 `Default`;未知非空值原样展示。Fast 优先使用模型的
|
||||
priority 价格,缺少专用价格时回退到标准价格的 `2.00x`;Flex 为 `0.50x`,Default 为 `1.00x`。
|
||||
|
||||
## 9. 版本、更新与重启
|
||||
|
||||
| 方法 | 路由 | 主要 query/body | 说明 |
|
||||
|
||||
+50
-33
@@ -53,21 +53,24 @@ API decode/auth
|
||||
-> freeze RuntimeSnapshot
|
||||
-> compile RoutePlan
|
||||
-> admission
|
||||
-> persist model_request
|
||||
-> enqueue model_request observation
|
||||
-> select Provider/account
|
||||
-> persist attempt facts in model_request
|
||||
-> enqueue attempt/send observations
|
||||
-> cross upstream send barrier
|
||||
-> canonical stream
|
||||
-> persist downstream delivery barrier
|
||||
-> terminal observation and accounting
|
||||
-> cross in-memory downstream delivery barrier and enqueue observation
|
||||
-> enqueue terminal observation and accounting
|
||||
```
|
||||
|
||||
核心不变量:
|
||||
|
||||
1. 每个客户端请求只有一条 `model_requests`;attempt 明细保存在其 JSON 事实中,不另建 attempt 表。
|
||||
2. 任何可能到达上游的调用都必须先有持久请求事实。
|
||||
2. 任何可能到达上游的调用都必须先按序提交 request/attempt 观测到有界队列;PostgreSQL 投影是可恢复
|
||||
观测,不是数据面发送的同步前置条件。
|
||||
3. `not_sent`、`sent`、`ambiguous` 是不同的上游发送边界;发送状态是请求级单调水位(`sent` > `ambiguous` > `not_sent`),跨 attempt 只升不降,终态写回不得低于水位;`ambiguous` 不自动重放。
|
||||
4. `downstream_committed_at` 表示网关已作出不可撤回的交付承诺,从该时刻禁止 retry/fallback;它在实际首字节写出前持久化,不宣称字节已经到达客户端。
|
||||
4. 内存中的 downstream commit 表示网关已作出不可撤回的交付承诺,从该时刻禁止 retry/fallback;
|
||||
它在实际首字节写出前越过并按序入队。`downstream_committed_at` 是该事实的 PostgreSQL 观测投影,
|
||||
不宣称字节已经到达客户端,也不反向决定协议交付。
|
||||
5. Provider 每次 `execute` 只能选择一个 credential 并准备一个 cold stream,不得隐藏换号或业务 retry。
|
||||
6. 下游 commit 前,Core 可按冻结策略处理同一 Provider 内的账号 fallback;禁止跨 Provider fallback。
|
||||
7. 本架构不通过隐式连接复用承载业务身份。HTTP client 可安全复用 transport 连接,但账号、credential revision、cookie/session binding 必须显式绑定到本次调用。
|
||||
@@ -84,7 +87,9 @@ Fallback 只允许在同一 Provider 内更换可用账号,不跨 Provider。
|
||||
|
||||
## 5. Continuation
|
||||
|
||||
系统只保存已被上游确认的安全外部 ID:`model_requests.upstream_response_id`,不新增 conversation、transcript、continuation 或 claim 表。
|
||||
系统在 `model_requests.client_response_id` 与 `upstream_response_id` 中记录响应生命周期确认的 ID,
|
||||
两者都按 opaque UTF-8 bytes 保存,不做格式、长度或唯一性推断;不新增 conversation、transcript、
|
||||
continuation 或 claim 表。
|
||||
|
||||
- `store=true`:使用 Provider 持久化的 native handle。
|
||||
- `store=false`:opaque replay state 仅存在于活连接内,不落 PostgreSQL。
|
||||
@@ -107,13 +112,26 @@ Provider 独占以下职责:
|
||||
|
||||
客户端可见正文的立场按 Provider 不同:
|
||||
|
||||
- OpenAI 是逐字节透明代理:上游 wire 是客户端可见的事实来源,HTTP SSE 帧以原始字节原样下发,WebSocket 上游 JSON 文本原样下发;canonical facts 只从同一帧旁路解析,用于观测、亲和与计费。仅网关内部帧(rate limits、metadata)被就地消费,不向客户端转发。
|
||||
- OpenAI 是逐字节透明代理:请求 body 的未知字段和字段顺序保持不变(受控模型映射除外),上游 wire 是客户端可见的事实来源,HTTP SSE 帧以原始字节原样下发,WebSocket 上游 JSON 文本原样下发;response ID 以 opaque bytes 存储,不假设 UUID 或固定长度。canonical facts 只从同一帧旁路解析,用于观测、亲和与计费。仅网关内部帧(rate limits、metadata)被就地消费,不向客户端转发。
|
||||
- xAI 是翻译层:把 Grok wire 解码后重新合成 Responses wire 事件,不承诺字节透明;上游错误按结构化 message/code/type 透出,message 中的 UUID 账号指纹替换为占位符。
|
||||
|
||||
credential 更新使用 `credential_revision` CAS。认证永久失败、封禁、额度耗尽和带截止时间的 cooldown 是账号运行时事实。cooldown 到期后,Core 的有效调度谓词会自然允许账号重新参与选择;Redis cooldown 只是热缓存。成功调用后才可把账号状态重新观测为 `ready`。自动 refresh 的瞬态失败(限流、5xx、超时、畸形响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;只有上游明确的永久错误(`invalid_grant`、封禁)才写入终态失效。
|
||||
credential 更新使用 `credential_revision` CAS。认证永久失败、封禁、额度耗尽和带截止时间的 cooldown 是账号运行时事实。cooldown 到期后,Core 的有效调度谓词会自然允许账号重新参与选择;Redis cooldown 只是热缓存。成功调用或成功额度观测才可把账号状态重新观测为 `ready`。自动 refresh 的瞬态失败(限流、5xx、超时、畸形响应)保留现有凭据并按指数退避推迟下一次刷新,不把账号标记为失效;只有上游明确的永久错误(`invalid_grant`、封禁)才写入终态失效。
|
||||
|
||||
`refresh_token_expires_at` 不是公共 SQL 列或 Core 权威状态。xAI 可在 `provider_credentials_json` 内保存它作为 Provider 私有提示;真正失效以 refresh endpoint 返回的永久错误为准。
|
||||
|
||||
OpenAI 导入、首次 OAuth 和重新授权都先验证签名 token 与稳定账号身份。重新授权的目标绑定只保存
|
||||
目标账号 ID,不接受或冻结客户端提供的 credential revision;complete 阶段重新读取目标账号及其当前
|
||||
revision,并以当前 revision 做最终 CAS。上游账号或用户身份发生换绑时拒绝提交;Free token 缺少
|
||||
账号 claim 时,用目标账号 header 下的 usage 身份做交叉确认。认证主体投影可以随同一上游账号的
|
||||
合法重新授权更新,但目标 upstream account/user 不得改变。
|
||||
|
||||
credential 与 quota 是两套状态机。主动 quota refresh 会拒绝过期 access token;quota 探测的
|
||||
401/403 不具备判定 refresh token 永久失效的证据。成功主动观测和正常推理响应中的 rate-limit header
|
||||
观测都 revision-fenced 写入同一 quota,并转换 readiness、`QuotaExhausted` 或 `Cooldown` 等额度事实;
|
||||
只有明确的 deactivated workspace 才能由额度链路写入 `Banned`,且额度观测不能清除 `Invalid`、
|
||||
`Expired`、`Banned`。Free、K12 等套餐复用该逻辑;`plan_type` 在此不参与额度分支,只作为 Provider
|
||||
事实并用于隔离不同套餐的 catalog cache。
|
||||
|
||||
## 7. 控制面与内部 revision
|
||||
|
||||
Admin API 不要求客户端提交配置 revision,也不向客户端暴露配置 revision。会改变调度快照或安全配置的写入,必须在同一 PostgreSQL 事务中:
|
||||
@@ -147,10 +165,16 @@ Admin API 不要求客户端提交配置 revision,也不向客户端暴露配
|
||||
- credential JSON 以 Provider 自己的 schema 明文保存在 `provider_credentials_json`;日志、Debug、API 和 audit
|
||||
不输出其中的 secret,Provider 只在需要发往上游时把值包装为运行时 secret。
|
||||
- stale recovery 只把超时 `running` 请求收敛为 `incomplete`,不重放业务请求。
|
||||
- `ops_events` 逐条同步持久化,`occurrence_count=1`;OpsFlush worker 以 Disabled 声明贡献(没有需要刷写的内存缓冲),不存在隐藏内存聚合权威。
|
||||
- 数据面 execution observation 通过有界、带总字节预算的进程内队列交给 PostgreSQL OpsFlush worker;
|
||||
入队不等待数据库,队列满、worker 不可用或写入失败时记录累计统计并丢弃该可恢复观测,不能替换
|
||||
客户端可见协议结果。队列不是第二份业务权威,也不做无幂等键的盲目重试。
|
||||
- retention 只删除已满足保留规则的历史事实,不改变运行中请求。
|
||||
|
||||
业务 schema 由 `backend/migrations/` 下的编号迁移按序定义:`0001_initial.sql` 建表,后续迁移只做增量变更。已合入的迁移永久冻结——`.frozen-sha256` 是冻结清单,CI 校验清单相对 PR base 只增不改,启动时 sqlx 重校验已应用迁移的 checksum,不一致直接拒绝启动。规则见 `backend/migrations/README.md`。
|
||||
业务 schema 由 `backend/migrations/` 下的编号迁移按序定义:`0001_initial.sql` 建表,`0002` 删除
|
||||
opaque response ID 唯一索引,`0003` 将 client/upstream response ID 改为无格式假设的 UTF-8 bytes,
|
||||
`0004` 增加响应观测得到的 `model_requests.service_tier`。已合入的迁移永久冻结——
|
||||
`.frozen-sha256` 是冻结清单,CI 校验清单相对 PR base 只增不改,启动时 sqlx 重校验已应用迁移的
|
||||
checksum,不一致直接拒绝启动。规则见 `backend/migrations/README.md`。
|
||||
|
||||
## 9. Redis 终态
|
||||
|
||||
@@ -165,19 +189,24 @@ codex-proxy-rs:oauth-pending:v1:{provider_kind}:{flow_fingerprint}
|
||||
flow_fingerprint = SHA-256(provider_kind || 0x00 || raw_flow_binding)
|
||||
```
|
||||
|
||||
Hash 字段恰好为:
|
||||
创建时的基础 Hash 字段为:
|
||||
|
||||
- `owner_fingerprint`
|
||||
- `expires_at_epoch_seconds`
|
||||
- `provider_payload`
|
||||
|
||||
create/take 使用 Lua 原子执行。owner mismatch 不删除记录;匹配 owner 的 take 返回 payload 并一次性删除。OpenAI TTL 为 10 分钟,xAI TTL 为 30 分钟。
|
||||
complete 先通过 Lua 原子 claim,并临时增加 `claim_fingerprint` 与
|
||||
`claim_expires_at_epoch_millis`。owner mismatch 不删除记录;有效 claim 阻止并发 complete,失败时只有
|
||||
claim owner 可以释放,账号事务提交后才原子删除整个 pending key。OpenAI TTL 为 10 分钟,xAI TTL
|
||||
为 30 分钟。
|
||||
|
||||
## 10. 后台任务与恢复
|
||||
|
||||
HTTP serve/drain 与 worker 运行时都在 `gateway-host`:Host 负责注册校验、leader lease、失败退避、健康暴露与关闭;`apps/gateway` 的 bootstrap 只装配。WorkerContribution 由 Store、Core 与两个 Provider bundle 显式贡献,host bundle 自身不贡献任何 worker:
|
||||
|
||||
- Store:stale execution recovery 与 retention;OpsFlush 以 Disabled 声明贡献。
|
||||
- Store:stale execution recovery、retention,以及 PostgreSQL execution observation、Redis client
|
||||
admission release 和 Provider circuit feedback 三个 OpsFlush daemon。native continuation 直接写 Redis,
|
||||
单次 Lua 记录只清理有界数量的过期/超限索引项,避免无界阻塞。
|
||||
- Core:RuntimeSnapshot revision 周期对账(无 lease,逐副本运行)与 Redis change 长驻订阅(daemon,逐副本运行)。
|
||||
- Provider:credential refresh、quota/catalog 健康,以及模型 etag 与官方版本 release 检查(OpenAI CLI、OpenAI Desktop、xAI CLI)。
|
||||
|
||||
@@ -185,7 +214,7 @@ HTTP serve/drain 与 worker 运行时都在 `gateway-host`:Host 负责注册
|
||||
|
||||
自更新、回滚与进程重启不是 worker:它们是 Admin API 触发的 Host 系统操作,只替换并重启处理该请求的那个副本。
|
||||
|
||||
关闭时序:收到取消信号后,HTTP drain(axum 优雅关闭与游离连接等待)共享同一个从信号时刻起算的绝对截止点,逾期放弃等待存量连接;随后 worker 在独立的 shutdown 超时内 join。
|
||||
关闭时序:收到取消信号后,HTTP drain(axum 优雅关闭与游离连接等待)共享同一个从信号时刻起算的绝对截止点,逾期放弃等待存量连接;随后 worker 在独立的 shutdown 超时内 join。有界观测队列会在该阶段按各自预算 drain,逾期的可恢复观测被计数并丢弃。
|
||||
|
||||
worker 不得通过导入 Admin use case 绕过边界,也不得维护第二份业务状态。
|
||||
|
||||
@@ -196,27 +225,15 @@ worker 不得通过导入 Admin use case 绕过边界,也不得维护第二份
|
||||
- Core 规则使用确定性测试;Provider 使用 contract/fixture 测试。
|
||||
- FK、CAS、revision、recovery 使用真实 PostgreSQL 测试。
|
||||
- admission、lease、cooldown、OAuth pending 使用真实 Redis 测试。
|
||||
- 真实账号测试只从仓库外部文件读取,不打印、复制或提交凭证。
|
||||
|
||||
真实对话测试是显式 ignored 的生产边界测试,必须依次穿过 Provider 导入/必要刷新、实时 catalog、账号 selector、生产 HTTP/SSE transport、canonical 解码、usage 与 completed 终态。xAI 用例允许 OAuth refresh token 轮换,因此必须额外显式授权:
|
||||
|
||||
```bash
|
||||
CODEX_REAL_ACCOUNT_FIXTURE=/outside/cpr-accounts.json \
|
||||
cargo test -p provider-openai --test main \
|
||||
admin::real_openai_conversation_crosses_production_provider_boundaries \
|
||||
-- --ignored --exact
|
||||
|
||||
XAI_REAL_ACCOUNT_FIXTURE=/outside/grok-accounts.json \
|
||||
XAI_ALLOW_DESTRUCTIVE_FIXTURE_REFRESH=1 \
|
||||
cargo test -p provider-xai --test main \
|
||||
admin::real_xai_conversation_crosses_production_provider_boundaries \
|
||||
-- --ignored --exact
|
||||
```
|
||||
- 自动化测试不得发送真实对话或轮换生产 refresh token;仓库不提供破坏性 live fixture。
|
||||
- OpenAI identity 仅保留可选的外部 `CODEX_REAL_ACCOUNT_FILE` 官方 JWKS 签名检查,环境变量未设置时
|
||||
直接跳过;它不经过 selector/transport,也不是数据面验收。
|
||||
|
||||
终态门禁:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
cargo fmt --all -- --check
|
||||
cargo test --workspace --all-targets
|
||||
cargo fmt --check
|
||||
cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
cargo test --test main --locked
|
||||
```
|
||||
|
||||
@@ -39,25 +39,12 @@ interface AccountConnectionTestOptions {
|
||||
onAccountStatus: (accountId: string, status: string) => void
|
||||
}
|
||||
|
||||
// providerErrorCode 到测试弹窗提示文案的映射。
|
||||
const CONNECTION_TEST_ERROR_HINTS: Record<string, string> = {
|
||||
account_cooling_down: '账号冷却中',
|
||||
account_capacity_busy: '账号并发已占满或请求间隔未到,请稍后重试',
|
||||
no_eligible_account: '该账号没有可用于所选模型的凭据',
|
||||
account_selector_unavailable: '账号调度状态暂时不可读,请稍后重试',
|
||||
}
|
||||
|
||||
function connectionTestErrorText(event: any) {
|
||||
const fallback = event.error || '测试连接失败'
|
||||
const hint = CONNECTION_TEST_ERROR_HINTS[event.providerErrorCode]
|
||||
if (!hint)
|
||||
return fallback
|
||||
if (event.providerErrorCode !== 'account_cooling_down')
|
||||
return hint
|
||||
const seconds = /retry in (\d+)s/.exec(fallback)?.[1]
|
||||
return seconds
|
||||
? `${hint},约 ${seconds} 秒后自动恢复(上游流被中断后触发)`
|
||||
: `${hint}(上游流被中断后触发)`
|
||||
if (typeof event.upstreamBody === 'string' && event.upstreamBody.length > 0)
|
||||
return event.upstreamBody
|
||||
if (Number.isInteger(event.upstreamStatus))
|
||||
return `HTTP ${event.upstreamStatus}`
|
||||
return event.error || '测试连接失败'
|
||||
}
|
||||
|
||||
export function useAccountConnectionTest(options: AccountConnectionTestOptions) {
|
||||
@@ -293,8 +280,15 @@ export function useAccountConnectionTest(options: AccountConnectionTestOptions)
|
||||
if (event.type === 'error') {
|
||||
applyAccountStatus(event.accountStatus)
|
||||
connectionTestError.value = connectionTestErrorText(event)
|
||||
const detail = event.providerErrorCode || event.providerErrorType
|
||||
? { code: event.providerErrorCode, type: event.providerErrorType, message: event.error }
|
||||
const detail = event.upstreamStatus || event.upstreamContentType || event.upstreamBody
|
||||
|| event.providerErrorCode || event.providerErrorType
|
||||
? {
|
||||
status: event.upstreamStatus,
|
||||
contentType: event.upstreamContentType,
|
||||
body: event.upstreamBody,
|
||||
code: event.providerErrorCode,
|
||||
type: event.providerErrorType,
|
||||
}
|
||||
: undefined
|
||||
appendConnectionTestLog(connectionTestError.value, 'danger', detail)
|
||||
finishConnectionTest('error')
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.0.21"
|
||||
version: "3.0.26"
|
||||
|
||||
@@ -40,13 +40,19 @@
|
||||
|
||||
## 存储
|
||||
|
||||
- PostgreSQL 业务表只有 `0001_initial.sql` 定义的七张;后续编号迁移只做增量(如 `0002` 的保留期删除索引)。
|
||||
- PostgreSQL 业务表只有 `0001_initial.sql` 定义的七张;后续编号迁移只做增量。当前 `0002` 删除
|
||||
opaque response ID 索引,`0003` 把 response ID 改为 bytes,`0004` 增加响应观测的 service tier。
|
||||
- 已应用迁移按字节冻结:`backend/migrations/.frozen-sha256` 是冻结清单,CI 相对 PR base 做 append-only 校验;schema 变更一律新增编号迁移,规则见 `backend/migrations/README.md`。
|
||||
- `config_revision` 只用于会改变调度快照或安全配置的管理 mutation。
|
||||
- quota、cooldown、catalog generation、自动 refresh 不推进全局 revision。
|
||||
- refresh 只推进账号 `credential_revision`;Redis cooldown 是可丢失热缓存。
|
||||
- OAuth pending 使用 Provider 域隔离 SHA-256 key、固定三字段 Hash 和原子一次消费。
|
||||
- 真实 secret 不进入日志、Debug、fixture、文档或 audit details;测试只能使用合成值。
|
||||
- OAuth pending 使用 Provider 域隔离 SHA-256 key;基础 Hash 保存 owner、过期时间和 Provider payload,
|
||||
complete 先原子 claim,失败释放 claim,账号事务提交后才消费。
|
||||
- 数据面 PostgreSQL execution observation 与 Redis admission release/circuit feedback 使用有界 OpsFlush
|
||||
队列;队列可丢失、可观测且不是权威状态,不得让 Store 失败改写客户端协议结果。native
|
||||
continuation 直接写 Redis,并限制每次记录的索引清理量。
|
||||
- 真实 secret 不进入日志、Debug、fixture、文档或 audit details;默认测试使用合成值,外部 access-token
|
||||
签名检查是唯一显式 opt-in 例外。
|
||||
|
||||
## 前端
|
||||
|
||||
@@ -57,9 +63,11 @@
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
cargo +1.97.0 fmt --all --manifest-path backend/Cargo.toml -- --check
|
||||
cargo +1.97.0 clippy --manifest-path backend/Cargo.toml --all-targets --all-features --locked -- -D warnings
|
||||
cargo +1.97.0 test --manifest-path backend/Cargo.toml --test main --locked
|
||||
cd backend
|
||||
cargo fmt --check
|
||||
cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
cargo test --test main --locked
|
||||
cd ..
|
||||
pnpm --dir frontend format:check
|
||||
pnpm --dir frontend build
|
||||
docker compose -f deploy/compose.yaml config --quiet
|
||||
@@ -69,11 +77,6 @@ docker compose -f deploy/compose.yaml config --quiet
|
||||
`gateway-store` 的 PostgreSQL/Redis 集成测试需要 `CPR_TEST_DATABASE_URL`、
|
||||
`CPR_TEST_REDIS_URL`;本地未设置时静默跳过,CI 缺失则直接失败。
|
||||
|
||||
真实账号测试只从仓库外路径读取,不打印或复制 credential。
|
||||
|
||||
真实数据面验证使用两个显式 ignored 测试:
|
||||
|
||||
- `admin::real_openai_conversation_crosses_production_provider_boundaries`
|
||||
- `admin::real_xai_conversation_crosses_production_provider_boundaries`
|
||||
|
||||
两者覆盖真实 catalog、selector、生产 SSE transport、canonical event、usage 和 completed;xAI 测试还要求 `XAI_ALLOW_DESTRUCTIVE_FIXTURE_REFRESH=1`,因为验证过程可能轮换 refresh token。
|
||||
自动化测试不发送真实对话,也不轮换生产 refresh token。OpenAI identity 有一个可选的
|
||||
`CODEX_REAL_ACCOUNT_FILE` 外部文件签名检查,只验证官方 JWKS,不经过 selector 或数据面;未设置时
|
||||
直接跳过。线上链路验收使用隔离的操作员请求,并按 request ID 对照服务日志和观测记录。
|
||||
|
||||
Reference in New Issue
Block a user