mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2640a03219 | ||
|
|
7d0e9a0a20 | ||
|
|
09982d7221 | ||
|
|
c4db1bb244 | ||
|
|
e7c2f025cc | ||
|
|
4ee294b208 | ||
|
|
93f1a82d3f | ||
|
|
c46f98cd2b | ||
|
|
5c69b1b082 | ||
|
|
6a78fb9232 | ||
|
|
02a03df79a | ||
|
|
2aca8fb79b | ||
|
|
891b227b69 | ||
|
|
04096dddb7 | ||
|
|
3a9b1c479f | ||
|
|
944d3f299e | ||
|
|
56451601fb | ||
|
|
4d53ef6068 | ||
|
|
a62bb196f7 | ||
|
|
7ad0ca48c5 | ||
|
|
e25639c11b | ||
|
|
8abac080a2 | ||
|
|
37cd059531 | ||
|
|
c46910df22 |
@@ -79,7 +79,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -4,6 +4,8 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths-ignore:
|
||||
- 'release/version.yaml'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
@@ -163,6 +163,15 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
upload_frontend: true
|
||||
|
||||
container:
|
||||
needs: [release-info]
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
uses: ./.github/workflows/_container.yml
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
build-docker-binary:
|
||||
name: build-docker-binary (${{ matrix.os }}-${{ matrix.arch }})
|
||||
needs: release-info
|
||||
@@ -497,7 +506,7 @@ jobs:
|
||||
exit-code: '1'
|
||||
|
||||
publish:
|
||||
needs: [release-info, package-assets, build-docker-image]
|
||||
needs: [release-info, package-assets, build-docker-image, container]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
|
||||
Generated
+45
-30
@@ -49,13 +49,13 @@ checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d"
|
||||
|
||||
[[package]]
|
||||
name = "argon2"
|
||||
version = "0.5.3"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
|
||||
checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"blake2",
|
||||
"cpufeatures 0.2.17",
|
||||
"cpufeatures 0.3.0",
|
||||
"password-hash",
|
||||
]
|
||||
|
||||
@@ -116,7 +116,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -206,9 +206,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.142.0"
|
||||
version = "1.144.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
|
||||
checksum = "30dc8bf6baaf7d46336a0ca2c69f223d9b90d7a801fb3e28f7ea17b00dc6b1de"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -331,9 +331,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c1c8a04cb31ba74d0115af5a890bb8c0d48fba64b52812fa13929a6ef0cc83c"
|
||||
checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
@@ -375,9 +375,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.13.1"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "483b858ff67522011c4786310c5cd8fd88d0be7ea3d5f1a48328446300c4269e"
|
||||
checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -599,11 +599,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "blake2"
|
||||
version = "0.10.6"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
|
||||
checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -740,7 +740,7 @@ dependencies = [
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"syn 3.0.3",
|
||||
"syn 3.0.4",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1013,7 +1013,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1318,7 +1317,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1363,6 +1362,7 @@ dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"cron",
|
||||
@@ -1586,6 +1586,11 @@ name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
@@ -2046,11 +2051,11 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.4"
|
||||
version = "0.18.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
|
||||
checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2232,13 +2237,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "password-hash"
|
||||
version = "0.5.0"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
|
||||
checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"getrandom 0.4.3",
|
||||
"phc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2262,6 +2266,17 @@ version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "phc"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"ctutils",
|
||||
"getrandom 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
@@ -2982,7 +2997,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3407,9 +3422,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.3"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||
checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3477,7 +3492,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3913,9 +3928,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.24.1"
|
||||
version = "1.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
|
||||
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
|
||||
@@ -144,7 +144,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"codex_version: '0.147.0-alpha.6.6'",
|
||||
"codex_version: '0.151.0-alpha.7.2'",
|
||||
"codex_version: 'latest'",
|
||||
));
|
||||
}
|
||||
@@ -165,7 +165,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '6415'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '7377'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -12,9 +12,10 @@ name = "main"
|
||||
path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
argon2 = "0.5.3"
|
||||
argon2 = "0.6.0"
|
||||
async-trait = "0.1.89"
|
||||
base64 = "0.23.0"
|
||||
bytes = "1.11.1"
|
||||
chrono = "0.4.45"
|
||||
futures = "0.3.32"
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
|
||||
@@ -38,13 +38,6 @@ impl TimeRange {
|
||||
}
|
||||
}
|
||||
|
||||
/// 观测记录的稳定键集游标。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObservabilityCursor {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
/// 请求结果状态。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RequestOutcome {
|
||||
@@ -130,9 +123,8 @@ pub struct UsageFilter {
|
||||
pub struct UsageQuery {
|
||||
pub range: TimeRange,
|
||||
pub filter: UsageFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub include_total: bool,
|
||||
}
|
||||
|
||||
/// 运维错误过滤条件。
|
||||
@@ -143,6 +135,7 @@ pub struct OpsErrorFilter {
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
@@ -158,9 +151,8 @@ pub struct OpsErrorFilter {
|
||||
pub struct OpsErrorQuery {
|
||||
pub range: TimeRange,
|
||||
pub filter: OpsErrorFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub include_total: bool,
|
||||
}
|
||||
|
||||
/// 用量诊断维度。
|
||||
@@ -667,8 +659,9 @@ pub struct UsageRecord {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsagePage {
|
||||
pub items: Vec<UsageListRecord>,
|
||||
pub total: Option<u64>,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: u16,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
/// 请求中的一次上游尝试或运维事件。
|
||||
@@ -770,6 +763,10 @@ pub struct OpsError {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub protocol: Option<String>,
|
||||
pub client_transport: Option<String>,
|
||||
pub requested_model_id: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -787,6 +784,13 @@ pub struct OpsError {
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub occurrence_count: u32,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub stable_sort_id: String,
|
||||
}
|
||||
@@ -795,8 +799,9 @@ pub struct OpsError {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpsErrorPage {
|
||||
pub items: Vec<OpsError>,
|
||||
pub total: Option<u64>,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: u16,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
/// 仪表盘趋势指标。
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
//! Provider 管理能力交换的中立 Command 与 Result。
|
||||
|
||||
use std::fmt;
|
||||
use std::{fmt, pin::Pin};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use chrono::{DateTime, NaiveDate, Utc};
|
||||
use futures::Stream;
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
@@ -679,6 +681,35 @@ pub struct ProviderProfileStatistics {
|
||||
pub activity_insights: ProviderProfileActivityInsights,
|
||||
}
|
||||
|
||||
/// Provider 头像正文的中立字节流;公共层不解释 MIME,也不限制总字节数。
|
||||
pub type ProviderProfileAvatarStream =
|
||||
Pin<Box<dyn Stream<Item = Result<Bytes, ProviderProfileAvatarStreamError>> + Send + 'static>>;
|
||||
|
||||
/// Provider 头像流在响应开始后的通用失败,不携带上游 URL 或正文。
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("provider profile avatar stream failed")]
|
||||
pub struct ProviderProfileAvatarStreamError;
|
||||
|
||||
/// Provider 已校验并打开的官方头像响应。
|
||||
pub struct ProviderProfileAvatar {
|
||||
pub content_type: Option<String>,
|
||||
pub content_length: Option<u64>,
|
||||
pub etag: Option<String>,
|
||||
pub body: ProviderProfileAvatarStream,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderProfileAvatar {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderProfileAvatar")
|
||||
.field("content_type", &self.content_type)
|
||||
.field("content_length", &self.content_length)
|
||||
.field("etag", &self.etag)
|
||||
.field("body", &"<stream>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 返回的一张安全主动额度重置卡。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResetCredit {
|
||||
|
||||
@@ -17,8 +17,8 @@ use crate::model::provider_credentials::{
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCommit, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, ProviderExport, ProviderExportCredentialInput, ProviderModels,
|
||||
ProviderProfileStatistics, ProviderQuota, ProviderQuotaRequest, ProviderResetCreditResult,
|
||||
ProviderResetCredits,
|
||||
ProviderProfileAvatar, ProviderProfileStatistics, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderResetCreditResult, ProviderResetCredits,
|
||||
};
|
||||
|
||||
/// Provider 管理失败的稳定分类。
|
||||
@@ -157,6 +157,14 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
/// 打开 Provider 官方头像字节流;不支持该能力的 Provider 使用默认拒绝。
|
||||
async fn profile_avatar(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderProfileAvatar, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
/// 查询 Provider 主动额度重置卡;不支持该能力的 Provider 使用默认拒绝。
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
|
||||
@@ -25,8 +25,9 @@ use crate::{
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
ConsumeProviderResetCredit, PrepareCredentialRefresh, ProviderModels,
|
||||
ProviderProfileStatistics, ProviderQuota, ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderResetCreditResult, ProviderResetCredits, QuotaLocalUsageAttribution,
|
||||
ProviderProfileAvatar, ProviderProfileStatistics, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderQuotaWindow, ProviderResetCreditResult, ProviderResetCredits,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -90,6 +91,13 @@ pub trait AccountsService: Send + Sync {
|
||||
Err(AdminError::invalid("当前 Provider 不支持账号统计"))
|
||||
}
|
||||
|
||||
async fn profile_avatar(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderProfileAvatar, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持账号头像"))
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
_context: &MutationContext,
|
||||
@@ -566,6 +574,17 @@ impl AccountsService for DefaultAccountsService {
|
||||
.map_err(|error| map_provider_error(error, "provider profile statistics"))
|
||||
}
|
||||
|
||||
async fn profile_avatar(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderProfileAvatar, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(account_id).await?;
|
||||
provider
|
||||
.profile_avatar(account_id)
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider profile avatar"))
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier, password_hash::SaltString};
|
||||
use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
|
||||
use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -195,9 +195,8 @@ impl AuthService for DefaultAuthService {
|
||||
}
|
||||
|
||||
fn hash_admin_password(password: &str) -> Result<String, AdminError> {
|
||||
let salt = SaltString::generate(&mut OsRng);
|
||||
Argon2::default()
|
||||
.hash_password(password.as_bytes(), &salt)
|
||||
.hash_password(password.as_bytes())
|
||||
.map(|hash| hash.to_string())
|
||||
.map_err(|_| AdminError::internal("管理员密码哈希失败"))
|
||||
}
|
||||
|
||||
@@ -675,13 +675,14 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
.clone())
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
||||
Ok(UsagePage {
|
||||
items,
|
||||
total: Some(total),
|
||||
next_cursor: None,
|
||||
current_page: query.current_page,
|
||||
page_size: query.page_size.get(),
|
||||
total,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -783,9 +784,8 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
UsageQuery {
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(50).expect("page size"),
|
||||
include_total: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,9 @@ use std::{collections::BTreeSet, convert::Infallible, fmt};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::Body,
|
||||
extract::State,
|
||||
http::{HeaderValue, StatusCode, header::CACHE_CONTROL},
|
||||
http::{HeaderName, HeaderValue, StatusCode, header},
|
||||
response::{
|
||||
IntoResponse, Response,
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
@@ -28,10 +29,10 @@ use gateway_admin::model::{
|
||||
AuthorizationStarted, CompleteAuthorization, ConsumeProviderResetCredit,
|
||||
CredentialDeletion, CredentialDeletionResult, CredentialImportResult, CredentialMutation,
|
||||
CredentialMutationResult, ImportCredentials, ProviderDocument, ProviderModels,
|
||||
ProviderProfileActivityInsights, ProviderProfileDailyUsage, ProviderProfileInvocation,
|
||||
ProviderProfileStatistics, ProviderProfileStatisticsSummary, ProviderQuota,
|
||||
ProviderQuotaWindow, ProviderResetCredit, ProviderResetCreditResult, ProviderResetCredits,
|
||||
RotateCredential, StartAuthorization,
|
||||
ProviderProfileActivityInsights, ProviderProfileAvatar, ProviderProfileDailyUsage,
|
||||
ProviderProfileInvocation, ProviderProfileStatistics, ProviderProfileStatisticsSummary,
|
||||
ProviderQuota, ProviderQuotaWindow, ProviderResetCredit, ProviderResetCreditResult,
|
||||
ProviderResetCredits, RotateCredential, StartAuthorization,
|
||||
},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -60,6 +61,7 @@ const MAX_ID_TOKEN_BYTES: usize = 16 * 1024;
|
||||
const MAX_CALLBACK_URL_BYTES: usize = 64 * 1024;
|
||||
const MAX_ACCOUNT_DELETE_BATCH: usize = 200;
|
||||
const MAX_ACCOUNT_GROUP_BATCH: usize = 1000;
|
||||
const MAX_AVATAR_VERSION_BYTES: usize = 32;
|
||||
|
||||
/// 账号列表查询参数。
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
@@ -422,6 +424,33 @@ impl AccountIdQuery {
|
||||
}
|
||||
}
|
||||
|
||||
/// 头像 GET 的固定 query;`version` 只参与浏览器缓存键,不进入上游请求。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountProfileAvatarQuery {
|
||||
pub account_id: String,
|
||||
pub version: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountProfileAvatarQuery {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
if self.version.as_deref().is_some_and(|version| {
|
||||
version.is_empty()
|
||||
|| version.len() > MAX_AVATAR_VERSION_BYTES
|
||||
|| !version.bytes().all(|byte| byte.is_ascii_alphanumeric())
|
||||
}) {
|
||||
return Err(WireValidationError::new("version"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_id(self) -> Result<ProviderAccountId, WireValidationError> {
|
||||
self.validate()?;
|
||||
ProviderAccountId::new(self.account_id).map_err(|_| WireValidationError::new("accountId"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 敏感导出的固定 query;IDs 使用逗号分隔,禁止隐式导出全部账号。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -1291,6 +1320,10 @@ where
|
||||
"/api/admin/accounts/profile-statistics",
|
||||
get(account_profile_statistics::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/accounts/profile-avatar",
|
||||
get(account_profile_avatar::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/accounts/reset-credits",
|
||||
get(account_reset_credits::<S>).post(consume_account_reset_credit::<S>),
|
||||
@@ -1400,7 +1433,7 @@ where
|
||||
let mut response = AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)).into_response();
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
@@ -1646,6 +1679,69 @@ where
|
||||
))
|
||||
}
|
||||
|
||||
async fn account_profile_avatar<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<AccountProfileAvatarQuery>,
|
||||
) -> Result<Response, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let account_id = query.into_id().map_err(map_wire_error)?;
|
||||
let avatar = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.profile_avatar(&account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(profile_avatar_response(avatar))
|
||||
}
|
||||
|
||||
/// 将 Provider 头像流投影为受保护的同源 HTTP 响应。
|
||||
#[must_use]
|
||||
pub fn profile_avatar_response(avatar: ProviderProfileAvatar) -> Response {
|
||||
let ProviderProfileAvatar {
|
||||
content_type,
|
||||
content_length,
|
||||
etag,
|
||||
body,
|
||||
} = avatar;
|
||||
let mut response = Response::new(Body::from_stream(body));
|
||||
*response.status_mut() = StatusCode::OK;
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(
|
||||
header::CONTENT_TYPE,
|
||||
content_type
|
||||
.and_then(|value| HeaderValue::from_str(&value).ok())
|
||||
.unwrap_or_else(|| HeaderValue::from_static("application/octet-stream")),
|
||||
);
|
||||
if let Some(value) =
|
||||
content_length.and_then(|length| HeaderValue::from_str(&length.to_string()).ok())
|
||||
{
|
||||
headers.insert(header::CONTENT_LENGTH, value);
|
||||
}
|
||||
if let Some(value) = etag.and_then(|value| HeaderValue::from_str(&value).ok()) {
|
||||
headers.insert(header::ETAG, value);
|
||||
}
|
||||
headers.insert(
|
||||
header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("private, max-age=3600"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-content-type-options"),
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("cross-origin-resource-policy"),
|
||||
HeaderValue::from_static("same-origin"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("content-security-policy"),
|
||||
HeaderValue::from_static("sandbox; default-src 'none'"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
async fn refresh_account_quota<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
|
||||
@@ -58,6 +58,7 @@ async fn admin_not_found() -> AdminError {
|
||||
async fn no_store(mut response: Response) -> Response {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
.entry(header::CACHE_CONTROL)
|
||||
.or_insert(HeaderValue::from_static("no-store"));
|
||||
response
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use axum::{Router, extract::State, http::StatusCode, response::IntoResponse, routing::get};
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -515,18 +515,13 @@ pub(crate) fn usage_detail_view(detail: domain::UsageDetail) -> UsageRecordDetai
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_page_view(
|
||||
page: domain::UsagePage,
|
||||
) -> Result<PageData<UsageListRecordView>, WireValidationError> {
|
||||
Ok(PageData {
|
||||
pub(crate) fn usage_page_view(page: domain::UsagePage) -> PageData<UsageListRecordView> {
|
||||
PageData {
|
||||
items: page.items.into_iter().map(usage_list_record_view).collect(),
|
||||
current_page: page.current_page,
|
||||
page_size: page.page_size,
|
||||
total: page.total,
|
||||
next_cursor: page
|
||||
.next_cursor
|
||||
.as_ref()
|
||||
.map(encode_observability_cursor)
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
@@ -536,24 +531,45 @@ pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
.or(error.provider_account_name.as_ref())
|
||||
.or(error.provider_account_ref.as_ref())
|
||||
.cloned();
|
||||
let model = error
|
||||
.upstream_model_id
|
||||
.clone()
|
||||
.or_else(|| error.requested_model_id.clone());
|
||||
OpsErrorView {
|
||||
id: error.event_id,
|
||||
request_id: error.request_id,
|
||||
client_api_key_id: error.client_api_key_ref,
|
||||
kind: error.component.clone(),
|
||||
operation: error.operation,
|
||||
protocol: error.protocol,
|
||||
client_transport: error.client_transport,
|
||||
provider: error.provider_kind,
|
||||
authentication_kind: error.provider_account_authentication_kind,
|
||||
account_id: error.provider_account_ref,
|
||||
account_name: error.provider_account_name,
|
||||
account_email: error.provider_account_email,
|
||||
route: error.endpoint.unwrap_or_else(|| "—".to_owned()),
|
||||
model: error.upstream_model_id,
|
||||
model,
|
||||
requested_model: error.requested_model_id,
|
||||
upstream_model: error.upstream_model_id,
|
||||
service_tier: error.service_tier,
|
||||
client_status_code: error.client_status_code.map(i64::from),
|
||||
upstream_status_code: error.upstream_status_code.map(i64::from),
|
||||
transport: error.upstream_transport,
|
||||
attempt_index: error.attempt_index,
|
||||
failure_class: error.failure_kind,
|
||||
provider_error_code: error.provider_error_code,
|
||||
occurrence_count: error.occurrence_count,
|
||||
response_id: error.client_response_id,
|
||||
upstream_request_id: error.upstream_request_id,
|
||||
latency_ms: error.latency_ms,
|
||||
client_ip: error.client_ip,
|
||||
user_agent: error.user_agent,
|
||||
reasoning_effort: error.reasoning_effort,
|
||||
reasoning_preset: error.reasoning_preset,
|
||||
request_kind: error.request_kind,
|
||||
subagent_kind: error.subagent_kind,
|
||||
compact: error.compact,
|
||||
message: error.message,
|
||||
metadata: OpsErrorMetadataView {
|
||||
source: error.source,
|
||||
@@ -566,18 +582,13 @@ pub(crate) fn ops_error_view(error: domain::OpsError) -> OpsErrorView {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ops_page_view(
|
||||
page: domain::OpsErrorPage,
|
||||
) -> Result<PageData<OpsErrorView>, WireValidationError> {
|
||||
Ok(PageData {
|
||||
pub(crate) fn ops_page_view(page: domain::OpsErrorPage) -> PageData<OpsErrorView> {
|
||||
PageData {
|
||||
items: page.items.into_iter().map(ops_error_view).collect(),
|
||||
current_page: page.current_page,
|
||||
page_size: page.page_size,
|
||||
total: page.total,
|
||||
next_cursor: page
|
||||
.next_cursor
|
||||
.as_ref()
|
||||
.map(encode_observability_cursor)
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn trend_point_view(point: domain::TrendPoint) -> TrendPointView {
|
||||
|
||||
@@ -2,14 +2,10 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
use base64::Engine as _;
|
||||
|
||||
/// 观测列表默认页大小。
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小。
|
||||
pub const MAX_PAGE_SIZE: u16 = 100;
|
||||
/// 游标编码在 HTTP 层解码前允许的最大长度。
|
||||
pub const MAX_CURSOR_BYTES: usize = 1024;
|
||||
|
||||
/// Dashboard 查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
@@ -31,8 +27,8 @@ impl DashboardQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub outcome: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
@@ -52,18 +48,17 @@ pub struct UsageQuery {
|
||||
}
|
||||
|
||||
impl UsageQuery {
|
||||
/// 校验分页字段,不改变仓储层的分页 owner。
|
||||
pub fn validate_page_size(&self) -> Result<u16, WireValidationError> {
|
||||
/// 校验 Element Plus 风格的页码分页字段。
|
||||
pub fn validate_pagination(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let current_page = self.current_page.unwrap_or(1);
|
||||
if current_page == 0 {
|
||||
return Err(WireValidationError::new("currentPage"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok(page_size)
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界;编码和排序语义由应用层负责。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
Ok((current_page, page_size))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,8 +100,8 @@ impl DiagnosticsQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct OpsQuery {
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
@@ -128,18 +123,17 @@ pub struct OpsQuery {
|
||||
}
|
||||
|
||||
impl OpsQuery {
|
||||
/// 校验分页字段。
|
||||
pub fn validate_page_size(&self) -> Result<u16, WireValidationError> {
|
||||
/// 校验 Element Plus 风格的页码分页字段。
|
||||
pub fn validate_pagination(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let current_page = self.current_page.unwrap_or(1);
|
||||
if current_page == 0 {
|
||||
return Err(WireValidationError::new("currentPage"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok(page_size)
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
Ok((current_page, page_size))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,13 +242,6 @@ pub(crate) fn require_text(value: &str, field: &'static str) -> Result<(), WireV
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -336,22 +323,19 @@ pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, Wi
|
||||
}
|
||||
|
||||
pub(crate) fn usage_command(query: &UsageQuery) -> Result<domain::UsageQuery, WireValidationError> {
|
||||
let page_size = query.validate_page_size()?;
|
||||
query.validate_cursor()?;
|
||||
let (current_page, page_size) = query.validate_pagination()?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
Ok(domain::UsageQuery {
|
||||
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
|
||||
filter: usage_filter(query)?,
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
current_page,
|
||||
page_size: page_size_value,
|
||||
include_total: query.cursor.is_none(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, WireValidationError> {
|
||||
let page_size = query.validate_page_size()?;
|
||||
query.validate_cursor()?;
|
||||
let (current_page, page_size) = query.validate_pagination()?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
let status_code = parse_status(
|
||||
@@ -367,7 +351,8 @@ pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, Wir
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())),
|
||||
operation: non_empty(query.kind.clone()),
|
||||
endpoint: non_empty(query.route.clone()),
|
||||
model: non_empty(query.model.clone()),
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
@@ -377,47 +362,11 @@ pub(crate) fn ops_command(query: &OpsQuery) -> Result<domain::OpsErrorQuery, Wir
|
||||
status_code,
|
||||
search: non_empty(query.search.clone()),
|
||||
},
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
current_page,
|
||||
page_size: page_size_value,
|
||||
include_total: query.cursor.is_none(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn decode_observability_cursor(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<domain::ObservabilityCursor>, WireValidationError> {
|
||||
value
|
||||
.map(|encoded| {
|
||||
if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
let bytes = URL_SAFE_NO_PAD
|
||||
.decode(encoded)
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
let wire: CursorWire =
|
||||
serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?;
|
||||
if wire.stable_id.trim().is_empty() {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(domain::ObservabilityCursor {
|
||||
observed_at: wire.observed_at,
|
||||
stable_id: wire.stable_id,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn encode_observability_cursor(
|
||||
cursor: &domain::ObservabilityCursor,
|
||||
) -> Result<String, WireValidationError> {
|
||||
let bytes = serde_json::to_vec(&CursorWire {
|
||||
observed_at: cursor.observed_at,
|
||||
stable_id: cursor.stable_id.clone(),
|
||||
})
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
Ok(URL_SAFE_NO_PAD.encode(bytes))
|
||||
}
|
||||
|
||||
pub(crate) fn non_empty(value: Option<String>) -> Option<String> {
|
||||
value
|
||||
.map(|value| value.trim().to_owned())
|
||||
|
||||
@@ -91,7 +91,7 @@ where
|
||||
.usage_records(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = usage_page_view(result).map_err(|_| AdminError::internal())?;
|
||||
let data = usage_page_view(result);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -207,6 +207,6 @@ where
|
||||
.ops_errors(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = ops_page_view(result).map_err(|_| AdminError::internal())?;
|
||||
let data = ops_page_view(result);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -2,21 +2,14 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 观测列表游标的稳定 wire 形状。
|
||||
#[derive(Clone, Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CursorWire {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
/// 观测列表响应数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PageData<T> {
|
||||
pub items: Vec<T>,
|
||||
pub total: Option<u64>,
|
||||
pub next_cursor: Option<String>,
|
||||
pub current_page: u32,
|
||||
pub page_size: u16,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
/// Token 详情展示。
|
||||
@@ -765,19 +758,36 @@ pub struct OpsErrorView {
|
||||
pub request_id: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub operation: String,
|
||||
pub protocol: Option<String>,
|
||||
pub client_transport: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub failure_class: String,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub occurrence_count: u32,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub message: String,
|
||||
pub metadata: OpsErrorMetadataView,
|
||||
pub created_at: DateTime<Utc>,
|
||||
|
||||
@@ -15,8 +15,18 @@ use super::error::RequestDecodeError;
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
/// Responses 请求进入共享解码内核时的下游传输来源。
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum RequestDecodeSource {
|
||||
/// 单次 HTTP 请求,请求头与正文属于同一请求。
|
||||
Http,
|
||||
/// 复用连接上的 WebSocket `response.create` 帧。
|
||||
WebSocketFrame,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OpenAiRequestHeaders {
|
||||
turn_state: Option<String>,
|
||||
@@ -71,10 +81,18 @@ impl OpenAiRequestHeaders {
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_context(&self, use_websocket: Option<bool>) -> Map<String, Value> {
|
||||
fn protocol_context(
|
||||
&self,
|
||||
use_websocket: Option<bool>,
|
||||
frame_turn_metadata: Option<&String>,
|
||||
) -> Map<String, Value> {
|
||||
let mut context = Map::new();
|
||||
insert_protocol_context(&mut context, "turn_state", self.turn_state.as_ref());
|
||||
insert_protocol_context(&mut context, "turn_metadata", self.turn_metadata.as_ref());
|
||||
insert_protocol_context(
|
||||
&mut context,
|
||||
"turn_metadata",
|
||||
frame_turn_metadata.or(self.turn_metadata.as_ref()),
|
||||
);
|
||||
insert_protocol_context(&mut context, "beta_features", self.beta_features.as_ref());
|
||||
insert_protocol_context(&mut context, "version", self.version.as_ref());
|
||||
insert_protocol_context(
|
||||
@@ -304,14 +322,15 @@ pub(super) fn decode_request_inner(
|
||||
let Value::Object(object) = value else {
|
||||
return Err(RequestDecodeError::ExpectedObject);
|
||||
};
|
||||
decode_request_object(object, review, request_headers)
|
||||
decode_request_object(object, review, request_headers, RequestDecodeSource::Http)
|
||||
}
|
||||
|
||||
/// 解码已解析的顶层 object;WebSocket 帧解码复用此内核,避免序列化往返。
|
||||
/// 解码已解析的顶层 object;按下游传输来源恢复连接级协议上下文。
|
||||
pub(super) fn decode_request_object(
|
||||
mut object: Map<String, Value>,
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
source: RequestDecodeSource,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// 仅消费已识别的本地 transport 开关;未知同名值保留给未来上游协议。
|
||||
let use_websocket = object.get("use_websocket").and_then(Value::as_bool);
|
||||
@@ -341,7 +360,11 @@ pub(super) fn decode_request_object(
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
request_headers.apply_subagent(&mut object, review.then_some(REVIEW_SUBAGENT));
|
||||
let protocol_context = request_headers.protocol_context(use_websocket);
|
||||
let frame_turn_metadata = match source {
|
||||
RequestDecodeSource::Http => None,
|
||||
RequestDecodeSource::WebSocketFrame => frame_turn_metadata(&object),
|
||||
};
|
||||
let protocol_context = request_headers.protocol_context(use_websocket, frame_turn_metadata);
|
||||
|
||||
let payload = ProtocolPayload::json_object(OPENAI_PROTOCOL, object)
|
||||
.map_err(|_| RequestDecodeError::CanonicalContract {
|
||||
@@ -362,6 +385,16 @@ pub(super) fn decode_request_object(
|
||||
})
|
||||
}
|
||||
|
||||
fn frame_turn_metadata(body: &Map<String, Value>) -> Option<&String> {
|
||||
body.get("client_metadata")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|metadata| metadata.get(CODEX_TURN_METADATA_KEY))
|
||||
.and_then(|value| match value {
|
||||
Value::String(value) if !value.trim().is_empty() => Some(value),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
.entry("client_metadata".to_owned())
|
||||
@@ -455,10 +488,11 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent
|
||||
// 不能透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent/version
|
||||
// 不能作为不透明头透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
// 设备 attestation/integrity 头只能由官方客户端或网关自身生成,
|
||||
// 客户端注入的 x-oai-attestation / X-OAI-IS 不得透传上游。
|
||||
| "x-oai-attestation"
|
||||
|
||||
@@ -6,7 +6,8 @@ use serde_json::{Map, Value, json};
|
||||
use thiserror::Error;
|
||||
|
||||
use super::super::{
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError, request::OpenAiRequestHeaders,
|
||||
DecodedResponsesRequest, ProtocolErrorBody, RequestDecodeError,
|
||||
request::{OpenAiRequestHeaders, RequestDecodeSource},
|
||||
};
|
||||
|
||||
/// 使用连接级请求头和 Provider 上下文解码官方 `response.create` 文本帧。
|
||||
@@ -40,8 +41,13 @@ fn decode_response_create_inner(
|
||||
if matches!(body.get("stream"), Some(value) if value.as_bool() != Some(true)) {
|
||||
return Err(ResponseCreateFrameError::StreamingRequired);
|
||||
}
|
||||
super::super::request::decode_request_object(body, false, request_headers)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
super::super::request::decode_request_object(
|
||||
body,
|
||||
false,
|
||||
request_headers,
|
||||
RequestDecodeSource::WebSocketFrame,
|
||||
)
|
||||
.map_err(ResponseCreateFrameError::Request)
|
||||
}
|
||||
|
||||
/// `response.create` 帧的稳定安全错误。
|
||||
|
||||
@@ -76,7 +76,7 @@ mod profile_statistics {
|
||||
ProviderProfileActivityInsights, ProviderProfileDailyUsage, ProviderProfileInvocation,
|
||||
ProviderProfileStatistics, ProviderProfileStatisticsSummary,
|
||||
};
|
||||
use gateway_api::admin::accounts::AccountProfileStatisticsData;
|
||||
use gateway_api::admin::accounts::{AccountProfileAvatarQuery, AccountProfileStatisticsData};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
@@ -134,6 +134,74 @@ mod profile_statistics {
|
||||
assert!(value.get("models").is_none());
|
||||
assert!(value.get("estimatedCost").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_avatar_query_accepts_only_bounded_cache_versions() {
|
||||
let valid: AccountProfileAvatarQuery = serde_json::from_value(json!({
|
||||
"accountId": "acct_openai",
|
||||
"version": "k9m2z1"
|
||||
}))
|
||||
.expect("decode avatar query");
|
||||
valid.validate().expect("validate avatar query");
|
||||
|
||||
for version in ["", "contains-dash", "x23456789012345678901234567890123"] {
|
||||
let query: AccountProfileAvatarQuery = serde_json::from_value(json!({
|
||||
"accountId": "acct_openai",
|
||||
"version": version
|
||||
}))
|
||||
.expect("decode invalid avatar query");
|
||||
assert_eq!(
|
||||
query.validate().expect_err("reject version").field(),
|
||||
"version"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<AccountProfileAvatarQuery>(json!({
|
||||
"accountId": "acct_openai",
|
||||
"sourceUrl": "https://example.test/avatar"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod profile_avatar {
|
||||
use axum::body::to_bytes;
|
||||
use bytes::Bytes;
|
||||
use futures::stream;
|
||||
use gateway_admin::model::provider_credentials::ProviderProfileAvatar;
|
||||
use gateway_api::admin::accounts::profile_avatar_response;
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_avatar_response_preserves_stream_metadata_and_private_cache() {
|
||||
let response = profile_avatar_response(ProviderProfileAvatar {
|
||||
content_type: Some("image/svg+xml".to_owned()),
|
||||
content_length: Some(6),
|
||||
etag: Some("\"v1\"".to_owned()),
|
||||
body: Box::pin(stream::iter([Ok(Bytes::from_static(b"avatar"))])),
|
||||
});
|
||||
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(response.headers()["content-type"], "image/svg+xml");
|
||||
assert_eq!(response.headers()["content-length"], "6");
|
||||
assert_eq!(response.headers()["etag"], "\"v1\"");
|
||||
assert_eq!(response.headers()["cache-control"], "private, max-age=3600");
|
||||
assert_eq!(response.headers()["x-content-type-options"], "nosniff");
|
||||
assert_eq!(
|
||||
response.headers()["cross-origin-resource-policy"],
|
||||
"same-origin"
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers()["content-security-policy"],
|
||||
"sandbox; default-src 'none'"
|
||||
);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), 16)
|
||||
.await
|
||||
.expect("avatar response body"),
|
||||
Bytes::from_static(b"avatar")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod batch_update {
|
||||
|
||||
@@ -858,9 +858,10 @@ impl ObservabilityStore for UnusedStore {
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
Ok(UsagePage {
|
||||
total: query.include_total.then_some(items.len() as u64),
|
||||
current_page: query.current_page,
|
||||
page_size: query.page_size.get(),
|
||||
total: items.len() as u64,
|
||||
items,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -888,9 +889,10 @@ impl ObservabilityStore for UnusedStore {
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
let items = self.ops_errors.lock().expect("ops errors").clone();
|
||||
Ok(OpsErrorPage {
|
||||
total: query.include_total.then_some(items.len() as u64),
|
||||
current_page: query.current_page,
|
||||
page_size: query.page_size.get(),
|
||||
total: items.len() as u64,
|
||||
items,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,14 +19,13 @@ fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_bound_page_size_and_cursor() {
|
||||
fn usage_query_should_accept_element_plus_pagination() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"pageSize": 100,
|
||||
"cursor": "opaque"
|
||||
"currentPage": 129,
|
||||
"pageSize": 10
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(query.validate_page_size().unwrap(), 100);
|
||||
assert!(query.validate_cursor().is_ok());
|
||||
assert_eq!(query.validate_pagination().unwrap(), (129, 10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -35,14 +34,19 @@ fn usage_query_should_reject_removed_total_toggle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_removed_offset_page_contract() {
|
||||
fn usage_query_should_reject_removed_cursor_contract() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"cursor": "opaque"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_nonstandard_page_name() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"page": 1})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_page_size().unwrap_err().field(), "pageSize");
|
||||
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::observability::DesktopReleaseStatus;
|
||||
use gateway_api::admin::observability::{
|
||||
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
|
||||
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
|
||||
DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData,
|
||||
TrendKind, TrendPointView, TrendSummaryView,
|
||||
BillingView, CostCoverageView, DashboardAccountRequestBucketView, DashboardAccountUsageView,
|
||||
DashboardDesktopReleaseStatusView, DashboardWireAttributeView, DashboardWireProfileView,
|
||||
DashboardWireTargetView, PageData, TokenDetailsView, TrendData, TrendKind, TrendPointView,
|
||||
TrendSummaryView,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -14,12 +14,15 @@ use serde_json::json;
|
||||
fn usage_page_should_keep_terminal_camel_case_shape() {
|
||||
let data = PageData {
|
||||
items: vec![json!({"id": "request_1"})],
|
||||
total: Some(1),
|
||||
next_cursor: Some("cursor".to_owned()),
|
||||
current_page: 129,
|
||||
page_size: 10,
|
||||
total: 1287,
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["total"], 1);
|
||||
assert_eq!(value["nextCursor"], "cursor");
|
||||
assert_eq!(value["currentPage"], 129);
|
||||
assert_eq!(value["pageSize"], 10);
|
||||
assert_eq!(value["total"], 1287);
|
||||
assert!(value.get("nextCursor").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -154,11 +157,7 @@ fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
|
||||
image_output_tokens_display: "-".to_owned(),
|
||||
total_tokens_display: "3".to_owned(),
|
||||
};
|
||||
let cursor = CursorWire {
|
||||
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
|
||||
stable_id: "request_1".to_owned(),
|
||||
};
|
||||
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
|
||||
let value = serde_json::to_value((&coverage, &token_details)).unwrap();
|
||||
assert!(value.to_string().contains("known"));
|
||||
assert!(!value.to_string().contains("secret"));
|
||||
}
|
||||
@@ -540,6 +539,10 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
protocol: Some("openai".to_owned()),
|
||||
client_transport: Some("http_sse".to_owned()),
|
||||
requested_model_id: Some("gpt-5.4".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
@@ -557,6 +560,13 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
occurrence_count: 1,
|
||||
client_ip: Some("203.0.113.8".to_owned()),
|
||||
user_agent: Some("codex-cli/0.144.0".to_owned()),
|
||||
reasoning_effort: Some("medium".to_owned()),
|
||||
reasoning_preset: None,
|
||||
request_kind: Some("root".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: Some(false),
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
@@ -586,6 +596,11 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
|
||||
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"requestedModel": value["data"]["items"][0]["requestedModel"],
|
||||
"clientTransport": value["data"]["items"][0]["clientTransport"],
|
||||
"clientIp": value["data"]["items"][0]["clientIp"],
|
||||
"userAgent": value["data"]["items"][0]["userAgent"],
|
||||
"providerErrorCode": value["data"]["items"][0]["providerErrorCode"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"provider": "openai",
|
||||
@@ -595,6 +610,11 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"accountLabel": "err@example.invalid",
|
||||
"clientStatusCode": 502,
|
||||
"route": "/v1/responses",
|
||||
"requestedModel": "gpt-5.4",
|
||||
"clientTransport": "http_sse",
|
||||
"clientIp": "203.0.113.8",
|
||||
"userAgent": "codex-cli/0.144.0",
|
||||
"providerErrorCode": "upstream",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -196,7 +196,13 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
HeaderValue::from_static("{\"kind\":\"review\"}"),
|
||||
);
|
||||
headers.insert("conversation-id", HeaderValue::from_static("conversation"));
|
||||
let body = json!({"model": "smart-code", "input": "hello"});
|
||||
let body = json!({
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": "{\"kind\":\"turn\"}"
|
||||
}
|
||||
});
|
||||
|
||||
let decoded = decode_request_with_headers(body.to_string().as_bytes(), &headers)
|
||||
.expect("request should decode");
|
||||
@@ -320,8 +326,8 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
vec![b"future-ascii".to_vec(), b"\x80\xff".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("version"),
|
||||
vec![b"future-v1".to_vec(), b"future-\x80".to_vec()]
|
||||
openai_protocol_context(&decoded).get("version"),
|
||||
Some(&json!("future-v1"))
|
||||
);
|
||||
assert_eq!(
|
||||
values("openai-beta"),
|
||||
@@ -349,6 +355,7 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
"version",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
"x-oai-is-update",
|
||||
|
||||
@@ -9,11 +9,14 @@ use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{HeaderMap, HeaderValue, Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_api::openai::responses::{
|
||||
DecodedResponsesRequest, OpenAiRequestHeaders, ResponseCreateFrameError,
|
||||
decode_response_create_with_context,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
@@ -25,6 +28,7 @@ use gateway_core::event::{
|
||||
};
|
||||
use gateway_core::operation::Operation;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use gateway_protocol::openai::codex_responses_request_semantics;
|
||||
use serde_json::{Value, json};
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as ClientMessage;
|
||||
@@ -34,6 +38,22 @@ use tower::ServiceExt;
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, authenticated_client, models::ModelsExecution};
|
||||
|
||||
fn decode_response_create_with_turn_header(
|
||||
payload: Value,
|
||||
opening_turn_metadata: &'static str,
|
||||
) -> DecodedResponsesRequest {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-codex-turn-metadata",
|
||||
HeaderValue::from_static(opening_turn_metadata),
|
||||
);
|
||||
decode_response_create_with_context(
|
||||
&payload.to_string(),
|
||||
&OpenAiRequestHeaders::from_headers(&headers),
|
||||
)
|
||||
.expect("response.create should decode")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
let decoded = decode_response_create(
|
||||
@@ -118,6 +138,87 @@ fn response_create_should_preserve_compaction_trigger_for_openai() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_prefer_frame_turn_metadata_over_opening_headers() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": r#"{"request_kind":"turn"}"#
|
||||
}
|
||||
}),
|
||||
r#"{"request_kind":"compaction"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("turn"), false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_accept_frame_compaction_after_a_turn_opening_header() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"client_metadata": {
|
||||
"x-codex-turn-metadata": r#"{"request_kind":"compaction"}"#
|
||||
}
|
||||
}),
|
||||
r#"{"request_kind":"turn"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("compaction"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_fall_back_to_opening_turn_metadata_when_the_frame_omits_it() {
|
||||
let decoded = decode_response_create_with_turn_header(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "smart-code",
|
||||
"input": "hello"
|
||||
}),
|
||||
r#"{"request_kind":"compaction"}"#,
|
||||
);
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("Responses must map to Generate");
|
||||
};
|
||||
|
||||
let semantics = codex_responses_request_semantics(
|
||||
request.protocol_payload().body(),
|
||||
request.protocol_payload().context(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(semantics.request_kind.as_deref(), semantics.compact),
|
||||
(Some("compaction"), true)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_reject_explicit_non_streaming_requests() {
|
||||
let error = decode_response_create(
|
||||
|
||||
@@ -12,11 +12,11 @@ use crate::engine::continuation::{
|
||||
};
|
||||
use crate::engine::provider::{Provider, ProviderCallMetadata, ProviderRequest, ProviderStream};
|
||||
use crate::engine::{
|
||||
AccountAttemptContext, AttemptContext, AttemptRecord, AttemptTrigger, CancellationToken,
|
||||
CommitRequirement, ContinuationAttempt, CoordinatedEvent, EngineError, ExecutionOutcome,
|
||||
ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId,
|
||||
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
|
||||
RequestAttemptContext, UpstreamSendState,
|
||||
AccountAttemptContext, AttemptContext, AttemptRecord, AttemptTransport, AttemptTrigger,
|
||||
CancellationToken, CommitRequirement, ContinuationAttempt, CoordinatedEvent, EngineError,
|
||||
ExecutionOutcome, ExecutionStore, GatewayEngine, IntermediateFailure, ModelRequestFinalization,
|
||||
ModelRequestId, ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner,
|
||||
ProviderAttemptOutcome, RequestAttemptContext, UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use crate::event::{
|
||||
@@ -162,10 +162,12 @@ where
|
||||
account_state_owner,
|
||||
cancellation,
|
||||
attempts: 0,
|
||||
routing_attempts: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
transport_recovery: None,
|
||||
current: None,
|
||||
send_state_watermark: UpstreamSendState::NotSent,
|
||||
downstream_committed_at: None,
|
||||
@@ -216,6 +218,13 @@ struct FailureFinalization {
|
||||
retry_after_ms: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingTransportRecovery {
|
||||
account: crate::engine::credential::ProviderAccountId,
|
||||
transport: AttemptTransport,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话。
|
||||
///
|
||||
/// API 只能提交下游 delivery 边界;账号重试、断流终结与
|
||||
@@ -237,7 +246,10 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
account_state_owner: Option<ProviderAccountStateOwner>,
|
||||
cancellation: CancellationToken,
|
||||
/// 所有实际上游 attempt 数;包含同账号传输重试,作为持久化序号。
|
||||
attempts: u32,
|
||||
/// 路由预算只统计正常选号/账号恢复,不被 Provider-owned 传输预算消耗。
|
||||
routing_attempts: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
@@ -245,6 +257,9 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
/// attempt 建立时即被消费,后续可重试错误仍可换号消耗剩余重试预算。
|
||||
/// 与 `required_account`(外部指定、贯穿整个请求)语义不同,不可合并。
|
||||
recovery_account: Option<crate::engine::credential::ProviderAccountId>,
|
||||
/// 上游传输在提交边界前失败后的一次性同账号重试或备用传输钉选。
|
||||
/// 与凭据恢复分开保存,避免把 transport recovery 误记为 OAuth 恢复。
|
||||
transport_recovery: Option<PendingTransportRecovery>,
|
||||
current: Option<CurrentAttempt>,
|
||||
/// 请求级发送状态水位;跨 attempt 单调不降,终态写回不得低于此档。
|
||||
send_state_watermark: UpstreamSendState,
|
||||
@@ -586,9 +601,25 @@ where
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
if self.attempts >= self.plan.max_attempts().get() {
|
||||
let transport_recovery = self.transport_recovery.take();
|
||||
if transport_recovery.is_none() && self.routing_attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
if let Some(recovery) = transport_recovery.as_ref()
|
||||
&& !recovery.delay.is_zero()
|
||||
{
|
||||
match poll_retry_delay(recovery.delay, self.cancellation.clone(), self.deadline).await {
|
||||
RetryDelayBoundary::Elapsed => {}
|
||||
RetryDelayBoundary::Cancelled => {
|
||||
self.finish_interruption(EngineError::Cancelled).await?;
|
||||
return Err(EngineError::Cancelled);
|
||||
}
|
||||
RetryDelayBoundary::Deadline => {
|
||||
self.finish_interruption(EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(candidate) = self.plan.candidates().get(self.candidate_index).cloned() else {
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
@@ -610,14 +641,23 @@ where
|
||||
.checked_add(1)
|
||||
.and_then(NonZeroU32::new)
|
||||
.ok_or(EngineError::EmptyRoutingPlan)?;
|
||||
// recovery 钉选在此被一次性消费,只绑定本次 replay attempt;
|
||||
// 请求局部恢复钉选在此被一次性消费,只绑定本次 replay attempt;
|
||||
// 外部 required_account 每次 attempt 都重新生效。
|
||||
let pinned_account = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => Some(account.clone()),
|
||||
AccountSelection::Scheduled(_) => self
|
||||
.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
let is_transport_recovery = transport_recovery.is_some();
|
||||
let (pinned_account, attempt_transport) = if let Some(recovery) = transport_recovery {
|
||||
(Some(recovery.account), recovery.transport)
|
||||
} else {
|
||||
match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => {
|
||||
(Some(account.clone()), AttemptTransport::Default)
|
||||
}
|
||||
AccountSelection::Scheduled(_) => (
|
||||
self.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
AttemptTransport::Default,
|
||||
),
|
||||
}
|
||||
};
|
||||
let account_context = match &self.account_selection {
|
||||
AccountSelection::Diagnostic(account) => AccountAttemptContext::diagnostic(
|
||||
@@ -646,7 +686,8 @@ where
|
||||
self.continuation.clone(),
|
||||
self.cancellation.clone(),
|
||||
)
|
||||
.with_continuation_attempt(self.continuation_attempt);
|
||||
.with_continuation_attempt(self.continuation_attempt)
|
||||
.with_transport(attempt_transport);
|
||||
let trigger = if self.attempts == 0 {
|
||||
AttemptTrigger::Initial
|
||||
} else {
|
||||
@@ -864,6 +905,9 @@ where
|
||||
}
|
||||
}
|
||||
self.attempts = next_attempt.get();
|
||||
if !is_transport_recovery {
|
||||
self.routing_attempts = self.routing_attempts.saturating_add(1);
|
||||
}
|
||||
self.current = Some(CurrentAttempt {
|
||||
stream,
|
||||
metadata,
|
||||
@@ -1017,30 +1061,50 @@ where
|
||||
attempt_send_state,
|
||||
provider_proved_replay_safe,
|
||||
);
|
||||
let pre_delivery_retry = self.account_selection.required_account().is_none()
|
||||
let account_rotation_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& error.allows_pre_delivery_retry()
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
&& matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::AccountRotation)
|
||||
)
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
let transport_recovery = match error.pre_delivery_retry() {
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportRetry {
|
||||
retry_index,
|
||||
delay,
|
||||
}) if self.downstream_committed_at.is_none() && !self.delivery_pending => {
|
||||
Some((AttemptTransport::Retry(retry_index), delay))
|
||||
}
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransportFallback)
|
||||
if self.downstream_committed_at.is_none() && !self.delivery_pending =>
|
||||
{
|
||||
Some((AttemptTransport::Fallback, Duration::ZERO))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let ordinary_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& provider_proved_replay_safe
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
let same_account_retry = error.retries_same_account()
|
||||
&& provider_proved_replay_safe
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& self.attempts < self.plan.max_attempts().get()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& !self
|
||||
.credential_recovery_attempted_accounts
|
||||
.contains(current.metadata.provider_account_id());
|
||||
let retryable =
|
||||
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
|
||||
let retryable = continuation_retry
|
||||
|| same_account_retry
|
||||
|| ordinary_retry
|
||||
|| account_rotation_retry
|
||||
|| transport_recovery.is_some();
|
||||
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
|
||||
@@ -1053,6 +1117,12 @@ where
|
||||
// 只钉住紧随其后的 replay attempt;replay 再遇可重试错误时,
|
||||
// ordinary/continuation 重试门不受影响,仍可换号。
|
||||
self.recovery_account = Some(account);
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
self.transport_recovery = Some(PendingTransportRecovery {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport,
|
||||
delay,
|
||||
});
|
||||
} else if !continuation_retry {
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
@@ -1130,7 +1200,7 @@ where
|
||||
|| self.delivery_pending
|
||||
|| send_state == UpstreamSendState::Ambiguous
|
||||
|| !provider_proved_replay_safe
|
||||
|| self.attempts >= self.plan.max_attempts().get()
|
||||
|| self.routing_attempts >= self.plan.max_attempts().get()
|
||||
|| self
|
||||
.operation
|
||||
.provider_session_state(current.metadata.provider().as_str())
|
||||
@@ -1611,6 +1681,31 @@ enum ProviderBoundary {
|
||||
Deadline,
|
||||
}
|
||||
|
||||
enum RetryDelayBoundary {
|
||||
Elapsed,
|
||||
Cancelled,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
async fn poll_retry_delay(
|
||||
delay: Duration,
|
||||
cancellation: CancellationToken,
|
||||
deadline: SystemTime,
|
||||
) -> RetryDelayBoundary {
|
||||
let Ok(remaining) = deadline.duration_since(SystemTime::now()) else {
|
||||
return RetryDelayBoundary::Deadline;
|
||||
};
|
||||
let retry_delay = Delay::new(delay).fuse();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let timeout = Delay::new(remaining).fuse();
|
||||
pin_mut!(retry_delay, cancelled, timeout);
|
||||
select_biased! {
|
||||
_ = cancelled => RetryDelayBoundary::Cancelled,
|
||||
_ = timeout => RetryDelayBoundary::Deadline,
|
||||
_ = retry_delay => RetryDelayBoundary::Elapsed,
|
||||
}
|
||||
}
|
||||
|
||||
async fn poll_provider(
|
||||
provider: Arc<dyn Provider>,
|
||||
request: ProviderRequest,
|
||||
|
||||
@@ -106,6 +106,18 @@ pub enum AttemptTrigger {
|
||||
AccountRetry,
|
||||
}
|
||||
|
||||
/// Provider 可为当前 attempt 选择的请求局部传输档位。
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
pub enum AttemptTransport {
|
||||
/// 使用 Provider 的默认传输策略。
|
||||
#[default]
|
||||
Default,
|
||||
/// 固定账号重试 Provider 的首选传输;序号由 Provider 的独立预算驱动。
|
||||
Retry(NonZeroU32),
|
||||
/// 使用 Provider 定义的备用传输。
|
||||
Fallback,
|
||||
}
|
||||
|
||||
impl AttemptTrigger {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
@@ -224,8 +236,8 @@ fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
|
||||
/// 单次 attempt 的账号选择与账号绑定状态事实。
|
||||
///
|
||||
/// `required_account` 只用于管理端 connection test 等必须命中唯一账号的内部请求;
|
||||
/// 一旦设置,Core 与 Provider 都不得换号或切换 target。
|
||||
/// `required_account` 用于管理端 connection test,或请求局部的同账号恢复 attempt;
|
||||
/// 一旦设置,Provider 在本 attempt 内不得换号或切换 target。
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AccountAttemptContext {
|
||||
excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
@@ -425,6 +437,7 @@ pub struct AttemptContext {
|
||||
account: AccountAttemptContext,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
transport: AttemptTransport,
|
||||
cancellation: CancellationToken,
|
||||
}
|
||||
|
||||
@@ -452,6 +465,7 @@ impl AttemptContext {
|
||||
account,
|
||||
continuation,
|
||||
continuation_attempt,
|
||||
transport: AttemptTransport::Default,
|
||||
cancellation,
|
||||
}
|
||||
}
|
||||
@@ -466,6 +480,13 @@ impl AttemptContext {
|
||||
self
|
||||
}
|
||||
|
||||
/// 覆盖本次 attempt 的 Provider 传输档位。
|
||||
#[must_use]
|
||||
pub const fn with_transport(mut self, transport: AttemptTransport) -> Self {
|
||||
self.transport = transport;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> &ModelRequestId {
|
||||
self.request.request_id()
|
||||
@@ -503,7 +524,7 @@ impl AttemptContext {
|
||||
self.account.excluded_accounts()
|
||||
}
|
||||
|
||||
/// 管理端 connection test 等内部请求强制使用的唯一账号。
|
||||
/// 管理端诊断或请求局部恢复 attempt 强制使用的唯一账号。
|
||||
#[must_use]
|
||||
pub const fn required_account(&self) -> Option<&ProviderAccountId> {
|
||||
self.account.required_account()
|
||||
@@ -542,6 +563,12 @@ impl AttemptContext {
|
||||
self.continuation_attempt
|
||||
}
|
||||
|
||||
/// 返回本次 attempt 的 Provider 传输档位。
|
||||
#[must_use]
|
||||
pub const fn transport(&self) -> AttemptTransport {
|
||||
self.transport
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cancellation(&self) -> &CancellationToken {
|
||||
&self.cancellation
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 网关核心使用的稳定错误分类。
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -197,6 +198,22 @@ pub enum ContinuationFailure {
|
||||
HistoryUnavailable,
|
||||
}
|
||||
|
||||
/// Provider 在下游尚未提交事件时请求的隐藏恢复方式。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum PreDeliveryRetry {
|
||||
/// 排除本次账号,按普通调度策略重新选号。
|
||||
AccountRotation,
|
||||
/// 固定本次账号,并按 Provider 给出的序号重试当前传输。
|
||||
SameAccountTransportRetry {
|
||||
/// Provider-owned 传输重试序号,从 1 开始。
|
||||
retry_index: NonZeroU32,
|
||||
/// 发起下一次传输尝试前的退避时长。
|
||||
delay: Duration,
|
||||
},
|
||||
/// 固定本次账号,并要求 Provider 使用备用传输。
|
||||
SameAccountTransportFallback,
|
||||
}
|
||||
|
||||
/// Adapter 已明确分类为非 bearer 的不透明上游值。
|
||||
///
|
||||
/// Core 不解释或校验内容,只通过自定义 [`Debug`] 避免它意外进入日志。协议 adapter
|
||||
@@ -360,7 +377,7 @@ pub struct ProviderError {
|
||||
retry_after: Option<Duration>,
|
||||
continuation_failure: Option<ContinuationFailure>,
|
||||
replay_safe: bool,
|
||||
pre_delivery_retry: bool,
|
||||
pre_delivery_retry: Option<PreDeliveryRetry>,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
sensitive_context_redacted: bool,
|
||||
@@ -390,7 +407,7 @@ impl ProviderError {
|
||||
retry_after: None,
|
||||
continuation_failure: None,
|
||||
replay_safe: false,
|
||||
pre_delivery_retry: false,
|
||||
pre_delivery_retry: None,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
sensitive_context_redacted: false,
|
||||
@@ -456,10 +473,49 @@ impl ProviderError {
|
||||
/// 指定账号或下游已经进入提交状态时必须忽略它。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_retry(mut self) -> Self {
|
||||
self.pre_delivery_retry = true;
|
||||
self.pre_delivery_retry = Some(PreDeliveryRetry::AccountRotation);
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 在首个客户端事件前固定原账号并改用 Provider 备用传输。
|
||||
///
|
||||
/// 与普通换号恢复相同,Core 仍会拒绝 continuation、指定账号、预算耗尽或
|
||||
/// 下游已提交后的隐藏重放。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_transport_fallback(mut self) -> Self {
|
||||
self.set_pre_delivery_transport_fallback();
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地设置同账号备用传输恢复,保留当前错误携带的原客户端响应。
|
||||
pub const fn set_pre_delivery_transport_fallback(&mut self) {
|
||||
self.pre_delivery_retry = Some(PreDeliveryRetry::SameAccountTransportFallback);
|
||||
}
|
||||
|
||||
/// 允许 Core 在首个客户端事件前固定原账号并重试当前 Provider 传输。
|
||||
///
|
||||
/// 重试预算和退避策略由 Provider 所属协议定义;Core 只负责同账号钉选、
|
||||
/// deadline/取消边界以及下游 commit barrier。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_transport_retry(
|
||||
mut self,
|
||||
retry_index: NonZeroU32,
|
||||
delay: Duration,
|
||||
) -> Self {
|
||||
self.set_pre_delivery_transport_retry(retry_index, delay);
|
||||
self
|
||||
}
|
||||
|
||||
/// 原地设置同账号当前传输重试,保留当前错误携带的原客户端响应。
|
||||
pub const fn set_pre_delivery_transport_retry(
|
||||
&mut self,
|
||||
retry_index: NonZeroU32,
|
||||
delay: Duration,
|
||||
) {
|
||||
self.pre_delivery_retry =
|
||||
Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, delay });
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次。
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -561,9 +617,15 @@ impl ProviderError {
|
||||
self.replay_safe
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否允许在首个客户端事件前换号恢复传输失败。
|
||||
/// 返回 Provider 是否允许在首个客户端事件前隐藏恢复传输失败。
|
||||
#[must_use]
|
||||
pub const fn allows_pre_delivery_retry(&self) -> bool {
|
||||
self.pre_delivery_retry.is_some()
|
||||
}
|
||||
|
||||
/// 返回下游提交前由 Provider 选择的隐藏恢复方式。
|
||||
#[must_use]
|
||||
pub const fn pre_delivery_retry(&self) -> Option<PreDeliveryRetry> {
|
||||
self.pre_delivery_retry
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ use gateway_core::engine::provider::{
|
||||
ProviderRegistry, ProviderRequest, ProviderStream, UpstreamTransport,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptCoordinator, AttemptRecord, CancellationToken, CommitRequirement,
|
||||
ContinuationAttempt, EngineError, ExecutionOutcome, ExecutionStore, GatewayEngine,
|
||||
IntermediateFailure, ModelRequestFinalization, ModelRequestId, NewModelRequest,
|
||||
AttemptContext, AttemptCoordinator, AttemptRecord, AttemptTransport, CancellationToken,
|
||||
CommitRequirement, ContinuationAttempt, EngineError, ExecutionOutcome, ExecutionStore,
|
||||
GatewayEngine, IntermediateFailure, ModelRequestFinalization, ModelRequestId, NewModelRequest,
|
||||
ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
@@ -2754,6 +2754,109 @@ fn pre_delivery_transport_failure_rotates_account_for_non_idempotent_generation(
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_delivery_transport_fallback_retries_the_same_account_with_fallback_transport() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)
|
||||
.with_pre_delivery_transport_fallback())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("same-account fallback succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
|
||||
let original = ProviderAccountId::new("acct_first").expect("account id");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(contexts[0].transport(), AttemptTransport::Default);
|
||||
assert_eq!(contexts[1].transport(), AttemptTransport::Fallback);
|
||||
assert_eq!(contexts[1].required_account(), Some(&original));
|
||||
assert!(!contexts[1].excluded_accounts().contains(&original));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_owned_transport_retries_keep_the_same_account_until_fallback() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let retry_one = NonZeroU32::new(1).expect("non-zero retry index");
|
||||
let retry_two = NonZeroU32::new(2).expect("non-zero retry index");
|
||||
let transport_error =
|
||||
|| ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transport_error().with_pre_delivery_transport_retry(retry_one, Duration::ZERO)
|
||||
)],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(
|
||||
transport_error().with_pre_delivery_transport_retry(retry_two, Duration::ZERO)
|
||||
)],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(transport_error().with_pre_delivery_transport_fallback())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("fallback succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit winning response");
|
||||
|
||||
let original = ProviderAccountId::new("acct_first").expect("account id");
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 4);
|
||||
assert_eq!(contexts[0].transport(), AttemptTransport::Default);
|
||||
assert_eq!(contexts[1].transport(), AttemptTransport::Retry(retry_one));
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Retry(retry_two));
|
||||
assert_eq!(contexts[3].transport(), AttemptTransport::Fallback);
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&original));
|
||||
assert!(!context.excluded_accounts().contains(&original));
|
||||
}
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].attempt_count, 4);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_capacity_exhaustion_returns_the_last_retryable_upstream_failure() {
|
||||
let operation = generate_operation();
|
||||
@@ -2890,7 +2993,7 @@ fn pre_delivery_retry_marker_is_ignored_after_downstream_commit() {
|
||||
))),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_retry(),
|
||||
.with_pre_delivery_transport_fallback(),
|
||||
),
|
||||
],
|
||||
},
|
||||
|
||||
@@ -125,9 +125,8 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
outcome: Some("succeeded".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10)?,
|
||||
include_total: false,
|
||||
};
|
||||
// 每条 SQL 独立取一个全局观测槽位,避免整包预留造成队头阻塞。
|
||||
let (totals, (provider_accounts, _)) = futures::try_join!(
|
||||
@@ -148,7 +147,7 @@ impl ObservabilityRepository for PgObservabilityRepository {
|
||||
),
|
||||
self.query_budget
|
||||
.run("load dashboard recent requests", async {
|
||||
Ok(list_usage_records(&self.pool, recent_query).await?.items)
|
||||
list_usage_record_items(&self.pool, &recent_query).await
|
||||
}),
|
||||
)?;
|
||||
Ok(DashboardObservation {
|
||||
|
||||
@@ -30,22 +30,15 @@ pub(crate) fn store_request_outcome(outcome: admin_observability::RequestOutcome
|
||||
outcome.as_str().to_owned()
|
||||
}
|
||||
|
||||
pub(crate) fn store_cursor(
|
||||
cursor: admin_observability::ObservabilityCursor,
|
||||
) -> AdminStoreResult<ObservabilityCursor> {
|
||||
ObservabilityCursor::new(cursor.observed_at, cursor.stable_id).map_err(observability_error)
|
||||
}
|
||||
|
||||
pub(crate) fn store_usage_query(
|
||||
query: admin_observability::UsageQuery,
|
||||
) -> AdminStoreResult<UsageRecordQuery> {
|
||||
Ok(UsageRecordQuery {
|
||||
range: store_range(query.range)?,
|
||||
filter: store_usage_filter(query.filter),
|
||||
cursor: query.cursor.map(store_cursor).transpose()?,
|
||||
current_page: query.current_page,
|
||||
page_size: ObservabilityPageSize::new(query.page_size.get())
|
||||
.map_err(observability_error)?,
|
||||
include_total: query.include_total,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -58,6 +51,7 @@ pub(crate) fn store_ops_error_filter(
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
provider_kind: filter.provider_kind,
|
||||
operation: filter.operation,
|
||||
endpoint: filter.endpoint,
|
||||
model: filter.model,
|
||||
transport: filter.transport,
|
||||
attempt_index: filter.attempt_index,
|
||||
@@ -75,10 +69,9 @@ pub(crate) fn store_ops_error_query(
|
||||
Ok(OpsErrorQuery {
|
||||
range: store_range(query.range)?,
|
||||
filter: store_ops_error_filter(query.filter),
|
||||
cursor: query.cursor.map(store_cursor).transpose()?,
|
||||
current_page: query.current_page,
|
||||
page_size: ObservabilityPageSize::new(query.page_size.get())
|
||||
.map_err(observability_error)?,
|
||||
include_total: query.include_total,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -409,20 +402,12 @@ pub(crate) fn admin_usage_page(
|
||||
.into_iter()
|
||||
.map(admin_usage_list_record)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
current_page: page.current_page,
|
||||
page_size: page.page_size,
|
||||
total: page.total,
|
||||
next_cursor: page.next_cursor.map(admin_cursor),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_cursor(
|
||||
cursor: ObservabilityCursor,
|
||||
) -> admin_observability::ObservabilityCursor {
|
||||
admin_observability::ObservabilityCursor {
|
||||
observed_at: cursor.observed_at,
|
||||
stable_id: cursor.stable_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_list_record(
|
||||
record: UsageListRecord,
|
||||
) -> AdminStoreResult<admin_observability::UsageListRecord> {
|
||||
@@ -701,8 +686,9 @@ pub(crate) fn admin_ops_error_page(
|
||||
) -> AdminStoreResult<admin_observability::OpsErrorPage> {
|
||||
Ok(admin_observability::OpsErrorPage {
|
||||
items: page.items.into_iter().map(admin_ops_error).collect(),
|
||||
current_page: page.current_page,
|
||||
page_size: page.page_size,
|
||||
total: page.total,
|
||||
next_cursor: page.next_cursor.map(admin_cursor),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -715,6 +701,10 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
|
||||
client_api_key_ref: error.client_api_key_ref,
|
||||
component: error.component,
|
||||
operation: error.operation,
|
||||
protocol: error.protocol,
|
||||
client_transport: error.client_transport,
|
||||
requested_model_id: error.requested_model_id,
|
||||
service_tier: error.service_tier,
|
||||
endpoint: error.endpoint,
|
||||
provider_kind: error.provider_kind,
|
||||
provider_account_ref: error.provider_account_ref,
|
||||
@@ -732,6 +722,13 @@ pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::Ops
|
||||
latency_ms: error.latency_ms,
|
||||
message: error.message,
|
||||
occurrence_count: error.occurrence_count,
|
||||
client_ip: error.client_ip,
|
||||
user_agent: error.user_agent,
|
||||
reasoning_effort: error.reasoning_effort,
|
||||
reasoning_preset: error.reasoning_preset,
|
||||
request_kind: error.request_kind,
|
||||
subagent_kind: error.subagent_kind,
|
||||
compact: error.compact,
|
||||
occurred_at: error.occurred_at,
|
||||
stable_sort_id: error.stable_sort_id,
|
||||
}
|
||||
@@ -881,6 +878,10 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
|
||||
client_api_key_ref: get(row, "client_api_key_ref")?,
|
||||
component: get(row, "component")?,
|
||||
operation: get(row, "operation")?,
|
||||
protocol: get(row, "protocol")?,
|
||||
client_transport: get(row, "client_transport")?,
|
||||
requested_model_id: get(row, "requested_model_id")?,
|
||||
service_tier: get(row, "service_tier")?,
|
||||
endpoint: get(row, "endpoint")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
provider_account_ref: get(row, "provider_account_ref")?,
|
||||
@@ -898,6 +899,13 @@ pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<Ops
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
message: get(row, "message")?,
|
||||
occurrence_count: to_u32(get(row, "occurrence_count")?)?,
|
||||
client_ip: get(row, "client_ip")?,
|
||||
user_agent: get(row, "user_agent")?,
|
||||
reasoning_effort: get(row, "reasoning_effort")?,
|
||||
reasoning_preset: get(row, "reasoning_preset")?,
|
||||
request_kind: get(row, "request_kind")?,
|
||||
subagent_kind: get(row, "subagent_kind")?,
|
||||
compact: get(row, "compact")?,
|
||||
occurred_at: get(row, "occurred_at")?,
|
||||
stable_sort_id: get(row, "stable_sort_id")?,
|
||||
})
|
||||
|
||||
@@ -43,21 +43,17 @@ impl ObservabilityPageSize {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObservabilityCursor {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
impl ObservabilityCursor {
|
||||
pub fn new(observed_at: DateTime<Utc>, stable_id: impl Into<String>) -> StoreResult<Self> {
|
||||
let stable_id = stable_id.into();
|
||||
validate_text(&stable_id, MAX_FILTER_BYTES, "cursor ID")?;
|
||||
Ok(Self {
|
||||
observed_at,
|
||||
stable_id,
|
||||
})
|
||||
}
|
||||
pub(crate) fn observability_page_offset(
|
||||
current_page: u32,
|
||||
page_size: ObservabilityPageSize,
|
||||
) -> StoreResult<i64> {
|
||||
let page_index = current_page
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| invalid("current page must be positive"))?;
|
||||
u64::from(page_index)
|
||||
.checked_mul(u64::from(page_size.get()))
|
||||
.and_then(|offset| i64::try_from(offset).ok())
|
||||
.ok_or_else(|| invalid("page offset is too large"))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
@@ -119,9 +115,8 @@ impl UsageRecordFilter {
|
||||
pub struct UsageRecordQuery {
|
||||
pub range: ObservabilityRange,
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
pub include_total: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
@@ -131,6 +126,7 @@ pub struct OpsErrorFilter {
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
@@ -152,6 +148,7 @@ impl OpsErrorFilter {
|
||||
),
|
||||
(self.provider_kind.as_deref(), "provider filter"),
|
||||
(self.operation.as_deref(), "operation filter"),
|
||||
(self.endpoint.as_deref(), "endpoint filter"),
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(
|
||||
@@ -183,9 +180,8 @@ impl OpsErrorFilter {
|
||||
pub struct OpsErrorQuery {
|
||||
pub range: ObservabilityRange,
|
||||
pub filter: OpsErrorFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
pub include_total: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -609,8 +605,9 @@ pub struct UsageRecord {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecordPage {
|
||||
pub items: Vec<UsageListRecord>,
|
||||
pub total: Option<u64>,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: u16,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -699,6 +696,10 @@ pub struct OpsErrorRecord {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub protocol: Option<String>,
|
||||
pub client_transport: Option<String>,
|
||||
pub requested_model_id: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
@@ -716,6 +717,13 @@ pub struct OpsErrorRecord {
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub occurrence_count: u32,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub stable_sort_id: String,
|
||||
}
|
||||
@@ -723,8 +731,9 @@ pub struct OpsErrorRecord {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpsErrorPage {
|
||||
pub items: Vec<OpsErrorRecord>,
|
||||
pub total: Option<u64>,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
pub current_page: u32,
|
||||
pub page_size: u16,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -6,6 +6,7 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.id as event_id, mr.id as request_id,
|
||||
nullif(mr.attempt_count, 0) as attempt_index,
|
||||
mr.client_api_key_ref, 'model_request'::text as component, mr.operation,
|
||||
mr.protocol, mr.client_transport, mr.requested_model_id, mr.service_tier,
|
||||
mr.endpoint, mr.provider_kind, mr.provider_account_ref,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
@@ -17,7 +18,11 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
|
||||
mr.latency_ms,
|
||||
coalesce(mr.error_message, mr.error_kind, 'request failed') as message,
|
||||
1::integer as occurrence_count, mr.completed_at as occurred_at,
|
||||
1::integer as occurrence_count,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact,
|
||||
mr.completed_at as occurred_at,
|
||||
'model_request:' || mr.id as stable_sort_id
|
||||
from model_requests mr
|
||||
where mr.outcome = 'failed'";
|
||||
@@ -25,7 +30,8 @@ where mr.outcome = 'failed'";
|
||||
const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
oe.id as event_id, oe.model_request_id as request_id, oe.attempt_index,
|
||||
mr.client_api_key_ref, oe.component, oe.operation,
|
||||
null::text as endpoint, oe.provider_kind, oe.provider_account_ref,
|
||||
mr.protocol, mr.client_transport, mr.requested_model_id, mr.service_tier,
|
||||
mr.endpoint, oe.provider_kind, oe.provider_account_ref,
|
||||
oe.provider_account_name_snapshot as provider_account_name,
|
||||
oe.provider_account_email_snapshot as provider_account_email,
|
||||
oe.provider_account_authentication_kind_snapshot
|
||||
@@ -33,7 +39,11 @@ const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
oe.upstream_model_id, null::text as upstream_transport, oe.failure_kind,
|
||||
null::integer as client_status_code, oe.status_code as upstream_status_code,
|
||||
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
|
||||
oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at as occurred_at,
|
||||
oe.latency_ms, oe.message, oe.occurrence_count,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact,
|
||||
oe.created_at as occurred_at,
|
||||
'ops_event:' || oe.id as stable_sort_id
|
||||
from ops_events oe
|
||||
left join model_requests mr on mr.id = oe.model_request_id
|
||||
@@ -44,54 +54,32 @@ pub(crate) async fn list_ops_errors(
|
||||
query: OpsErrorQuery,
|
||||
) -> StoreResult<OpsErrorPage> {
|
||||
query.filter.validate()?;
|
||||
let total = if query.include_total {
|
||||
Some(count_ops_errors(pool, query.range, &query.filter).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let total = count_ops_errors(pool, query.range, &query.filter).await?;
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new("select * from (");
|
||||
statement.push(REQUEST_ERROR_SELECT);
|
||||
push_request_error_predicates(
|
||||
&mut statement,
|
||||
query.range,
|
||||
&query.filter,
|
||||
query.cursor.as_ref(),
|
||||
);
|
||||
push_request_error_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(" union all ");
|
||||
statement.push(OPS_EVENT_SELECT);
|
||||
push_ops_event_predicates(
|
||||
&mut statement,
|
||||
query.range,
|
||||
&query.filter,
|
||||
query.cursor.as_ref(),
|
||||
);
|
||||
push_ops_event_predicates(&mut statement, query.range, &query.filter);
|
||||
statement.push(") e order by occurred_at desc, stable_sort_id desc limit ");
|
||||
statement.push_bind(i64::from(query.page_size.get()) + 1);
|
||||
statement.push_bind(i64::from(query.page_size.get()));
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("list ops errors"))?;
|
||||
let mut items = rows
|
||||
let items = rows
|
||||
.iter()
|
||||
.map(ops_error_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let has_more = items.len() > usize::from(query.page_size.get());
|
||||
if has_more {
|
||||
items.pop();
|
||||
}
|
||||
let next_cursor = if has_more {
|
||||
items
|
||||
.last()
|
||||
.map(|item| ObservabilityCursor::new(item.occurred_at, item.stable_sort_id.clone()))
|
||||
.transpose()?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(OpsErrorPage {
|
||||
items,
|
||||
current_page: query.current_page,
|
||||
page_size: query.page_size.get(),
|
||||
total,
|
||||
next_cursor,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -103,11 +91,11 @@ pub(crate) async fn count_ops_errors(
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select coalesce(sum(source_count), 0)::bigint from (select count(*)::bigint as source_count from model_requests mr where mr.outcome = 'failed'",
|
||||
);
|
||||
push_request_error_predicates(&mut statement, range, filter, None);
|
||||
push_request_error_predicates(&mut statement, range, filter);
|
||||
statement.push(
|
||||
" union all select count(*)::bigint as source_count from ops_events oe left join model_requests mr on mr.id = oe.model_request_id where true",
|
||||
);
|
||||
push_ops_event_predicates(&mut statement, range, filter, None);
|
||||
push_ops_event_predicates(&mut statement, range, filter);
|
||||
statement.push(") counts");
|
||||
let total = statement
|
||||
.build_query_scalar::<i64>()
|
||||
@@ -121,7 +109,6 @@ fn push_request_error_predicates(
|
||||
statement: &mut QueryBuilder<Postgres>,
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
cursor: Option<&ObservabilityCursor>,
|
||||
) {
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
for (column, value) in [
|
||||
@@ -130,6 +117,7 @@ fn push_request_error_predicates(
|
||||
("mr.provider_account_ref", &filter.provider_account_ref),
|
||||
("mr.provider_kind", &filter.provider_kind),
|
||||
("mr.operation", &filter.operation),
|
||||
("mr.endpoint", &filter.endpoint),
|
||||
("mr.upstream_transport", &filter.transport),
|
||||
("mr.upstream_request_id", &filter.upstream_request_id),
|
||||
] {
|
||||
@@ -162,19 +150,12 @@ fn push_request_error_predicates(
|
||||
search,
|
||||
);
|
||||
}
|
||||
push_cursor(
|
||||
statement,
|
||||
"mr.completed_at",
|
||||
"'model_request:' || mr.id",
|
||||
cursor,
|
||||
);
|
||||
}
|
||||
|
||||
fn push_ops_event_predicates(
|
||||
statement: &mut QueryBuilder<Postgres>,
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
cursor: Option<&ObservabilityCursor>,
|
||||
) {
|
||||
push_range(statement, "oe.created_at", range);
|
||||
for (column, value) in [
|
||||
@@ -183,6 +164,7 @@ fn push_ops_event_predicates(
|
||||
("oe.provider_account_ref", &filter.provider_account_ref),
|
||||
("oe.provider_kind", &filter.provider_kind),
|
||||
("oe.operation", &filter.operation),
|
||||
("mr.endpoint", &filter.endpoint),
|
||||
("oe.upstream_request_id", &filter.upstream_request_id),
|
||||
("oe.failure_kind", &filter.failure_kind),
|
||||
] {
|
||||
@@ -217,7 +199,6 @@ fn push_ops_event_predicates(
|
||||
search,
|
||||
);
|
||||
}
|
||||
push_cursor(statement, "oe.created_at", "'ops_event:' || oe.id", cursor);
|
||||
}
|
||||
|
||||
fn push_range(statement: &mut QueryBuilder<Postgres>, column: &str, range: ObservabilityRange) {
|
||||
@@ -262,18 +243,3 @@ fn push_prefix_search(statement: &mut QueryBuilder<Postgres>, columns: &[&str],
|
||||
}
|
||||
statement.push(")");
|
||||
}
|
||||
|
||||
fn push_cursor(
|
||||
statement: &mut QueryBuilder<Postgres>,
|
||||
occurred_at: &str,
|
||||
stable_sort_id: &str,
|
||||
cursor: Option<&ObservabilityCursor>,
|
||||
) {
|
||||
if let Some(cursor) = cursor {
|
||||
statement.push(format!(" and ({occurred_at}, {stable_sort_id}) < ("));
|
||||
statement.push_bind(cursor.observed_at);
|
||||
statement.push(", ");
|
||||
statement.push_bind(cursor.stable_id.clone());
|
||||
statement.push(")");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ pub(crate) const USAGE_LIST_RECORD_SELECT: &str =
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
mr.provider_processing_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.account_selection_wait_ms, mr.capacity_used_slots, mr.capacity_total_slots,
|
||||
mr.client_ip::text as client_ip, mr.user_agent,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.subagent_kind, mr.compact,
|
||||
mr.started_at
|
||||
from model_requests mr";
|
||||
@@ -145,7 +145,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
mr.provider_processing_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.account_selection_wait_ms, mr.capacity_used_slots, mr.capacity_total_slots,
|
||||
mr.client_ip::text as client_ip,
|
||||
host(mr.client_ip) as client_ip,
|
||||
mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
mr.subagent_kind, mr.compact, mr.image_generation_requested,
|
||||
mr.image_generation_succeeded, mr.started_at, mr.deadline_at, mr.completed_at
|
||||
@@ -156,11 +156,22 @@ pub(crate) async fn list_usage_records(
|
||||
query: UsageRecordQuery,
|
||||
) -> StoreResult<UsageRecordPage> {
|
||||
query.filter.validate()?;
|
||||
let total = if query.include_total {
|
||||
Some(count_usage_records(pool, query.range, &query.filter).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let total = count_usage_records(pool, query.range, &query.filter).await?;
|
||||
let items = list_usage_record_items(pool, &query).await?;
|
||||
Ok(UsageRecordPage {
|
||||
items,
|
||||
current_page: query.current_page,
|
||||
page_size: query.page_size.get(),
|
||||
total,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn list_usage_record_items(
|
||||
pool: &PgPool,
|
||||
query: &UsageRecordQuery,
|
||||
) -> StoreResult<Vec<UsageListRecord>> {
|
||||
query.filter.validate()?;
|
||||
let offset = observability_page_offset(query.current_page, query.page_size)?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new(USAGE_LIST_RECORD_SELECT);
|
||||
statement.push(" where mr.started_at >= ");
|
||||
statement.push_bind(query.range.start);
|
||||
@@ -168,41 +179,20 @@ pub(crate) async fn list_usage_records(
|
||||
statement.push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
push_usage_filter(&mut statement, &query.filter, "mr");
|
||||
if let Some(cursor) = &query.cursor {
|
||||
statement.push(" and (mr.started_at, mr.id) < (");
|
||||
statement.push_bind(cursor.observed_at);
|
||||
statement.push(", ");
|
||||
statement.push_bind(cursor.stable_id.clone());
|
||||
statement.push(")");
|
||||
}
|
||||
statement.push(" order by mr.started_at desc, mr.id desc limit ");
|
||||
statement.push_bind(i64::from(query.page_size.get()) + 1);
|
||||
statement.push_bind(i64::from(query.page_size.get()));
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("list usage records"))?;
|
||||
let mut items = rows
|
||||
let items = rows
|
||||
.iter()
|
||||
.map(usage_list_record_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let has_more = items.len() > usize::from(query.page_size.get());
|
||||
if has_more {
|
||||
items.pop();
|
||||
}
|
||||
let next_cursor = if has_more {
|
||||
items
|
||||
.last()
|
||||
.map(|item| ObservabilityCursor::new(item.started_at, item.id.clone()))
|
||||
.transpose()?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(UsageRecordPage {
|
||||
items,
|
||||
total,
|
||||
next_cursor,
|
||||
})
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub(crate) async fn count_usage_records(
|
||||
|
||||
@@ -106,13 +106,12 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
response_id: Some(response_id.clone()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("filter opaque response ID");
|
||||
assert_eq!(records.total, Some(1));
|
||||
assert_eq!(records.total, 1);
|
||||
assert_eq!(records.items[0].id, "req_observe_success");
|
||||
let detail = repository
|
||||
.usage_record_detail("req_observe_success")
|
||||
@@ -131,8 +130,8 @@ async fn observability_preserves_and_filters_opaque_response_ids() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_page_should_skip_exact_count_when_total_is_not_requested() {
|
||||
let Some(database) = TestDatabase::create("usage_page_without_total").await else {
|
||||
async fn usage_page_should_always_return_total() {
|
||||
let Some(database) = TestDatabase::create("usage_page_with_total").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
@@ -146,14 +145,15 @@ async fn usage_page_should_skip_exact_count_when_total_is_not_requested() {
|
||||
.list_usage_records(UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter::default(),
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: false,
|
||||
})
|
||||
.await
|
||||
.expect("usage page without total");
|
||||
.expect("usage page with total");
|
||||
|
||||
assert_eq!(page.total, None);
|
||||
assert_eq!(page.total, 1);
|
||||
assert_eq!(page.current_page, 1);
|
||||
assert_eq!(page.page_size, 10);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -176,14 +176,13 @@ async fn usage_search_should_match_literal_prefix_instead_of_substring() {
|
||||
search: Some("observe_success".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("usage substring search");
|
||||
|
||||
assert_eq!(page.total, Some(0));
|
||||
assert_eq!(page.total, 0);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -206,14 +205,13 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
|
||||
search: Some("req%".to_owned()),
|
||||
..OpsErrorFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("ops literal wildcard search");
|
||||
|
||||
assert_eq!(page.total, Some(0));
|
||||
assert_eq!(page.total, 0);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -625,20 +623,34 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
filter: admin_observability::UsageFilter::default(),
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(1).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("first usage page");
|
||||
assert_eq!(first_page.total, Some(1));
|
||||
assert_eq!(first_page.total, 1);
|
||||
assert_eq!(first_page.items.len(), 1);
|
||||
assert!(first_page.next_cursor.is_none());
|
||||
assert_eq!(first_page.current_page, 1);
|
||||
assert_eq!(first_page.page_size, 1);
|
||||
assert_eq!(
|
||||
first_page.items[0].service_tier.as_deref(),
|
||||
Some("priority")
|
||||
);
|
||||
|
||||
let deep_page = store
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
filter: admin_observability::UsageFilter::default(),
|
||||
current_page: 129,
|
||||
page_size: PageSize::new(1).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("direct deep usage page");
|
||||
assert_eq!(deep_page.current_page, 129);
|
||||
assert_eq!(deep_page.page_size, 1);
|
||||
assert_eq!(deep_page.total, 1);
|
||||
assert!(deep_page.items.is_empty());
|
||||
|
||||
let filtered = store
|
||||
.list_usage_records(admin_observability::UsageQuery {
|
||||
range,
|
||||
@@ -657,13 +669,12 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
upstream_request_id: Some("upstream_req_success".to_owned()),
|
||||
search: Some("req_observe_success".to_owned()),
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("fully filtered usage page");
|
||||
assert_eq!(filtered.total, Some(1));
|
||||
assert_eq!(filtered.total, 1);
|
||||
assert_eq!(filtered.items[0].id, "req_observe_success");
|
||||
assert_eq!(filtered.items[0].service_tier.as_deref(), Some("priority"));
|
||||
|
||||
@@ -677,13 +688,12 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
),
|
||||
..admin_observability::UsageFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("other outcome filter should reach PostgreSQL");
|
||||
assert_eq!(other_outcome.total, Some(0));
|
||||
assert_eq!(other_outcome.total, 0);
|
||||
assert!(other_outcome.items.is_empty());
|
||||
|
||||
let detail = store
|
||||
@@ -751,21 +761,36 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
request_id: Some("req_observe_failed".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_observe".to_owned()),
|
||||
operation: Some("responses".to_owned()),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
model: Some("upstream-model".to_owned()),
|
||||
failure_kind: Some("rate_limited".to_owned()),
|
||||
status_code: Some(429),
|
||||
search: Some("req_observe_failed".to_owned()),
|
||||
..admin_observability::OpsErrorFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("admin ops errors");
|
||||
assert_eq!(errors.total, Some(2));
|
||||
assert_eq!(errors.total, 2);
|
||||
assert!(errors.items.iter().all(|item| item.occurred_at <= now));
|
||||
|
||||
let deep_errors = store
|
||||
.list_ops_errors(admin_observability::OpsErrorQuery {
|
||||
range,
|
||||
filter: admin_observability::OpsErrorFilter::default(),
|
||||
current_page: 129,
|
||||
page_size: PageSize::new(1).expect("page size"),
|
||||
})
|
||||
.await
|
||||
.expect("direct deep ops page");
|
||||
assert_eq!(deep_errors.current_page, 129);
|
||||
assert_eq!(deep_errors.page_size, 1);
|
||||
assert_eq!(deep_errors.total, 2);
|
||||
assert!(deep_errors.items.is_empty());
|
||||
|
||||
for filter in [
|
||||
admin_observability::OpsErrorFilter {
|
||||
client_api_key_ref: Some("missing-key".to_owned()),
|
||||
@@ -796,13 +821,12 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
.list_ops_errors(admin_observability::OpsErrorQuery {
|
||||
range,
|
||||
filter,
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("fully forwarded ops filter");
|
||||
assert_eq!(page.total, Some(0));
|
||||
assert_eq!(page.total, 0);
|
||||
}
|
||||
|
||||
database.close().await;
|
||||
@@ -1052,13 +1076,12 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
provider_account_ref: Some("acct_observe".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("usage records");
|
||||
assert_eq!(usage_page.total, Some(1));
|
||||
assert_eq!(usage_page.total, 1);
|
||||
let successful_image = usage_page
|
||||
.items
|
||||
.iter()
|
||||
@@ -1157,13 +1180,12 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range,
|
||||
filter: OpsErrorFilter::default(),
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("ops errors");
|
||||
assert_eq!(errors.total, Some(2));
|
||||
assert_eq!(errors.total, 2);
|
||||
let request_error = errors
|
||||
.items
|
||||
.iter()
|
||||
@@ -1171,6 +1193,15 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("request error");
|
||||
assert_eq!(request_error.client_status_code, Some(502));
|
||||
assert_eq!(request_error.upstream_status_code, Some(429));
|
||||
assert_eq!(request_error.client_ip.as_deref(), Some("203.0.113.9"));
|
||||
assert_eq!(
|
||||
request_error.user_agent.as_deref(),
|
||||
Some("codex-cli/0.144.0")
|
||||
);
|
||||
assert_eq!(
|
||||
request_error.requested_model_id.as_deref(),
|
||||
Some("public-model")
|
||||
);
|
||||
let attempt_error = errors
|
||||
.items
|
||||
.iter()
|
||||
@@ -1178,6 +1209,12 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
.expect("attempt error");
|
||||
assert_eq!(attempt_error.client_status_code, None);
|
||||
assert_eq!(attempt_error.upstream_status_code, Some(429));
|
||||
assert_eq!(attempt_error.endpoint.as_deref(), Some("/v1/responses"));
|
||||
assert_eq!(attempt_error.client_ip.as_deref(), Some("203.0.113.9"));
|
||||
assert_eq!(
|
||||
attempt_error.user_agent.as_deref(),
|
||||
Some("codex-cli/0.144.0")
|
||||
);
|
||||
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1266,7 +1303,7 @@ async fn seed_observability_facts(
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, requested_model_id,
|
||||
client_transport, requested_model_id, service_tier,
|
||||
provider_kind, provider_account_id,
|
||||
provider_account_ref, upstream_model_id, upstream_transport, attempt_count,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
@@ -1275,16 +1312,20 @@ async fn seed_observability_facts(
|
||||
error_kind, provider_error_code, error_message, retry_after_ms,
|
||||
input_tokens, cached_tokens, image_generation_requested,
|
||||
image_generation_succeeded, cost_source, latency_ms,
|
||||
client_ip, user_agent, reasoning_effort, reasoning_preset,
|
||||
request_kind, subagent_kind, compact,
|
||||
started_at, deadline_at, completed_at,
|
||||
routing_scope, routing_group_refs, routing_group_names_snapshot
|
||||
) values (
|
||||
'req_observe_failed', 'key_observe', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', 'public-model', 'openai', 'acct_observe',
|
||||
'http_sse', 'public-model', 'priority', 'openai', 'acct_observe',
|
||||
'acct_observe', 'upstream-model',
|
||||
'http_sse', 2,
|
||||
'primary', 'account@example.invalid', 'oauth',
|
||||
'sent', 'failed', 502, 429, 'rate_limited', 'rate_limit',
|
||||
'upstream limited', 1000, 0, 0, true, false, 'unavailable', 700,
|
||||
'203.0.113.9', 'codex-cli/0.144.0', 'medium', null,
|
||||
'root', null, false,
|
||||
$1 - interval '10 minutes', $1 + interval '20 minutes', $1 - interval '9 minutes',
|
||||
'all', '{}'::text[], '[]'::jsonb
|
||||
)",
|
||||
|
||||
@@ -60,7 +60,7 @@ async fn completed_usage_projections_should_accept_statusless_websocket_but_reje
|
||||
|
||||
assert_eq!(
|
||||
(page.total, request_ids),
|
||||
(Some(1), vec!["req_statusless_websocket"])
|
||||
(1, vec!["req_statusless_websocket"])
|
||||
);
|
||||
|
||||
let overview = repository
|
||||
@@ -399,9 +399,8 @@ async fn ops_errors_should_keep_request_and_event_snapshots_after_account_deleti
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range: range_around(started_at),
|
||||
filter: OpsErrorFilter::default(),
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
})
|
||||
.await
|
||||
.expect("list ops errors after deletion");
|
||||
@@ -886,9 +885,8 @@ fn usage_query(started_at: DateTime<Utc>, filter: UsageRecordFilter) -> UsageRec
|
||||
UsageRecordQuery {
|
||||
range: range_around(started_at),
|
||||
filter,
|
||||
cursor: None,
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).expect("page size"),
|
||||
include_total: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::time::SystemTime;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::TryStreamExt as _;
|
||||
use gateway_admin::model::accounts::AccountRecord;
|
||||
use gateway_admin::model::observability::{
|
||||
CalculatedBillingBreakdown, CurrencyCost, DashboardDesktopRelease, DashboardWireAttribute,
|
||||
@@ -19,9 +20,10 @@ use gateway_admin::model::provider_credentials::{
|
||||
PreparedCredentialCreate, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
PreparedCredentialRotationFacts, ProviderDocument, ProviderExport,
|
||||
ProviderExportCredentialInput, ProviderModel, ProviderModels, ProviderProfileActivityInsights,
|
||||
ProviderProfileDailyUsage, ProviderProfileInvocation, ProviderProfileStatistics,
|
||||
ProviderProfileStatisticsSummary, ProviderQuota, ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderQuotaWindowRole, ProviderResetCredit, ProviderResetCreditResult, ProviderResetCredits,
|
||||
ProviderProfileAvatar, ProviderProfileAvatarStreamError, ProviderProfileDailyUsage,
|
||||
ProviderProfileInvocation, ProviderProfileStatistics, ProviderProfileStatisticsSummary,
|
||||
ProviderQuota, ProviderQuotaRequest, ProviderQuotaWindow, ProviderQuotaWindowRole,
|
||||
ProviderResetCredit, ProviderResetCreditResult, ProviderResetCredits,
|
||||
QuotaLocalUsageAttribution,
|
||||
};
|
||||
use gateway_admin::model::{MutationActor, MutationContext, Revision};
|
||||
@@ -49,10 +51,10 @@ use crate::credential::{
|
||||
CodexCredentialAdminService, CodexCredentialCatalogError, CodexCredentialCatalogService,
|
||||
CodexCredentialProfileService, CodexCredentialQuotaError, CodexCredentialQuotaService,
|
||||
CodexOAuthAdmin, CodexOAuthAdminError, CodexOAuthPendingClaimOutcome, CodexOAuthPendingStore,
|
||||
CodexOAuthPendingStoreError, CodexPendingAuthorization, CodexProfileStatisticsError,
|
||||
CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole, CodexResetCreditsError,
|
||||
CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential, ExportManagedCodexCredential,
|
||||
StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
CodexOAuthPendingStoreError, CodexPendingAuthorization, CodexProfileAvatarError,
|
||||
CodexProfileStatisticsError, CodexQuotaWindow, CodexQuotaWindowKind, CodexQuotaWindowRole,
|
||||
CodexResetCreditsError, CompleteCodexOAuthAuthorization, CompletedCodexOAuthCredential,
|
||||
ExportManagedCodexCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
};
|
||||
use crate::credential::{
|
||||
CodexCredentialCodec, CodexOAuthSecret, oauth_owner_ref, parse_access_token_expiration,
|
||||
@@ -61,7 +63,9 @@ use crate::transport::CodexWebSocketPool;
|
||||
use crate::transport::profile::{
|
||||
CodexDesktopReleaseSnapshot, CodexDesktopReleaseStatus, CodexWireProfile, CodexWireProfileState,
|
||||
};
|
||||
use crate::transport::{CodexProfileStatistics, OpenAiBillingUsage, openai_billing_breakdown};
|
||||
use crate::transport::{
|
||||
CodexProfileAvatar, CodexProfileStatistics, OpenAiBillingUsage, openai_billing_breakdown,
|
||||
};
|
||||
|
||||
const PROVIDER_NAME: &str = "openai";
|
||||
const PENDING_DOCUMENT_SCHEMA_VERSION: u64 = 3;
|
||||
@@ -462,6 +466,18 @@ impl ProviderAdmin for OpenAiAdminProvider {
|
||||
Ok(project_profile_statistics(statistics))
|
||||
}
|
||||
|
||||
async fn profile_avatar(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderProfileAvatar, ProviderAdminError> {
|
||||
self.account(account_id).await?;
|
||||
self.profile_statistics
|
||||
.profile_avatar(account_id)
|
||||
.await
|
||||
.map(project_profile_avatar)
|
||||
.map_err(map_profile_avatar_error)
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -865,6 +881,15 @@ fn project_profile_statistics(statistics: CodexProfileStatistics) -> ProviderPro
|
||||
}
|
||||
}
|
||||
|
||||
fn project_profile_avatar(avatar: CodexProfileAvatar) -> ProviderProfileAvatar {
|
||||
ProviderProfileAvatar {
|
||||
content_type: avatar.content_type,
|
||||
content_length: avatar.content_length,
|
||||
etag: avatar.etag,
|
||||
body: Box::pin(avatar.body.map_err(|_| ProviderProfileAvatarStreamError)),
|
||||
}
|
||||
}
|
||||
|
||||
fn project_reset_credits(
|
||||
credits: crate::transport::CodexRateLimitResetCredits,
|
||||
) -> ProviderResetCredits {
|
||||
@@ -1498,6 +1523,19 @@ fn map_profile_statistics_error(error: CodexProfileStatisticsError) -> ProviderA
|
||||
}
|
||||
}
|
||||
|
||||
fn map_profile_avatar_error(error: CodexProfileAvatarError) -> ProviderAdminError {
|
||||
match error {
|
||||
CodexProfileAvatarError::ProfileStatistics(error) => map_profile_statistics_error(error),
|
||||
CodexProfileAvatarError::Missing => provider_admin_error(ProviderAdminErrorKind::NotFound),
|
||||
CodexProfileAvatarError::InvalidSource | CodexProfileAvatarError::Upstream { .. } => {
|
||||
provider_admin_error(ProviderAdminErrorKind::BadGateway)
|
||||
}
|
||||
CodexProfileAvatarError::TransportUnavailable => {
|
||||
provider_admin_error(ProviderAdminErrorKind::Unavailable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_reset_credits_error(error: CodexResetCreditsError) -> ProviderAdminError {
|
||||
use CodexResetCreditsError as Error;
|
||||
|
||||
|
||||
@@ -18,6 +18,15 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
/// 本服务的流式请求默认重试次数。
|
||||
pub const DEFAULT_STREAM_MAX_RETRIES: u64 = 3;
|
||||
/// 防止错误配置产生无界隐藏重放;与官方 Codex 的硬上限一致。
|
||||
pub const MAX_STREAM_MAX_RETRIES: u64 = 100;
|
||||
|
||||
const fn default_stream_max_retries() -> u64 {
|
||||
DEFAULT_STREAM_MAX_RETRIES
|
||||
}
|
||||
|
||||
/// OpenAI Provider 唯一启动配置。
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -30,6 +39,8 @@ pub struct OpenAiConfig {
|
||||
pub quota: CodexQuotaSettings,
|
||||
#[serde(default)]
|
||||
pub auth: CodexAuthSettings,
|
||||
#[serde(default = "default_stream_max_retries")]
|
||||
pub stream_max_retries: u64,
|
||||
pub wire_profile: CodexWireProfileConfig,
|
||||
#[serde(skip)]
|
||||
identity_secret_path: PathBuf,
|
||||
@@ -88,6 +99,13 @@ impl OpenAiConfig {
|
||||
self.auth.refresh_enabled
|
||||
}
|
||||
|
||||
/// 返回经官方同款硬上限约束后的上游流重试预算。
|
||||
#[must_use]
|
||||
pub fn stream_max_retries(&self) -> u32 {
|
||||
u32::try_from(self.stream_max_retries.min(MAX_STREAM_MAX_RETRIES))
|
||||
.unwrap_or(MAX_STREAM_MAX_RETRIES as u32)
|
||||
}
|
||||
|
||||
pub(crate) fn session_identity(
|
||||
&self,
|
||||
) -> Result<CodexSessionIdentity, CodexSessionIdentityError> {
|
||||
@@ -102,6 +120,7 @@ impl Default for OpenAiConfig {
|
||||
ws_pool: CodexWebSocketPoolSettings::default(),
|
||||
quota: CodexQuotaSettings::default(),
|
||||
auth: CodexAuthSettings::default(),
|
||||
stream_max_retries: DEFAULT_STREAM_MAX_RETRIES,
|
||||
wire_profile: CodexWireProfileConfig::default(),
|
||||
identity_secret_path: PathBuf::new(),
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! OpenAI 根会话锚点到 Store 不透明账号亲和键及诊断上下文的单向派生。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -8,6 +10,7 @@ use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
use crate::transport::request::derive_conversation_anchor;
|
||||
|
||||
const AFFINITY_KEY_HASH_LENGTH: usize = 12;
|
||||
pub(crate) const CODEX_ROOT_SESSION_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
|
||||
/// 一次请求派生出的账号亲和键及其结构化日志上下文。
|
||||
pub(crate) struct CodexSessionAffinity {
|
||||
|
||||
@@ -16,7 +16,8 @@ pub mod token_client;
|
||||
mod types;
|
||||
|
||||
pub(crate) use affinity::{
|
||||
CodexSessionAffinity, derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
CODEX_ROOT_SESSION_TTL, CodexSessionAffinity, derive_codex_cyber_policy_session_key,
|
||||
derive_codex_session_affinity,
|
||||
};
|
||||
pub(crate) use oauth::oauth_owner_ref;
|
||||
pub(crate) use types::parse_access_token_expiration;
|
||||
@@ -38,7 +39,9 @@ pub use oauth::{
|
||||
CodexPendingAuthorization, CompleteCodexOAuthAuthorization, CompletedCodexOAuthAuthorization,
|
||||
CompletedCodexOAuthCredential, StartCodexOAuthAuthorization, StoredCodexPendingAuthorization,
|
||||
};
|
||||
pub use profile_statistics::{CodexCredentialProfileService, CodexProfileStatisticsError};
|
||||
pub use profile_statistics::{
|
||||
CodexCredentialProfileService, CodexProfileAvatarError, CodexProfileStatisticsError,
|
||||
};
|
||||
pub use quota::{
|
||||
CodexAccountQuotaSnapshot, CodexCredentialQuotaError, CodexCredentialQuotaService,
|
||||
CodexQuotaFact, CodexQuotaRefreshPolicy, CodexQuotaSyncSummary, CodexQuotaWindow,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
//! OpenAI 当前账号个人资料统计查询编排。
|
||||
//! OpenAI 当前账号个人资料统计与官方头像查询编排。
|
||||
|
||||
use std::time::SystemTime;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant, SystemTime},
|
||||
};
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use reqwest::Client;
|
||||
@@ -10,11 +14,14 @@ use uuid::Uuid;
|
||||
|
||||
use crate::transport::profile::CodexWireProfileState;
|
||||
use crate::transport::{
|
||||
CodexBackendClient, CodexClientError, CodexProfileStatistics, CodexRequestContext,
|
||||
CodexBackendClient, CodexClientError, CodexProfileAvatar, CodexProfileAvatarFetchError,
|
||||
CodexProfileStatistics, CodexRequestContext, fetch_profile_avatar,
|
||||
};
|
||||
|
||||
use super::CodexCredentialRepository;
|
||||
|
||||
const PROFILE_AVATAR_SOURCE_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
#[derive(Error)]
|
||||
pub enum CodexProfileStatisticsError {
|
||||
#[error("Codex profile-statistics credential data is invalid")]
|
||||
@@ -59,16 +66,55 @@ impl std::fmt::Debug for CodexProfileStatisticsError {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Error)]
|
||||
pub enum CodexProfileAvatarError {
|
||||
#[error(transparent)]
|
||||
ProfileStatistics(#[from] CodexProfileStatisticsError),
|
||||
#[error("Codex profile does not provide an avatar")]
|
||||
Missing,
|
||||
#[error("Codex profile avatar source is invalid")]
|
||||
InvalidSource,
|
||||
#[error("Codex profile avatar upstream returned HTTP {status}")]
|
||||
Upstream { status: u16 },
|
||||
#[error("Codex profile avatar transport is unavailable")]
|
||||
TransportUnavailable,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CodexProfileAvatarError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::ProfileStatistics(error) => formatter
|
||||
.debug_tuple("ProfileStatistics")
|
||||
.field(error)
|
||||
.finish(),
|
||||
Self::Missing => formatter.write_str("Missing"),
|
||||
Self::InvalidSource => formatter.write_str("InvalidSource"),
|
||||
Self::Upstream { status } => formatter
|
||||
.debug_struct("Upstream")
|
||||
.field("status", status)
|
||||
.finish(),
|
||||
Self::TransportUnavailable => formatter.write_str("TransportUnavailable"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedProfileAvatarSource {
|
||||
source: String,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
pub struct CodexCredentialProfileService {
|
||||
repository: CodexCredentialRepository,
|
||||
profile: CodexWireProfileState,
|
||||
http: Client,
|
||||
base_url: String,
|
||||
avatar_sources: Mutex<HashMap<ProviderAccountId, CachedProfileAvatarSource>>,
|
||||
}
|
||||
|
||||
impl CodexCredentialProfileService {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
pub fn new(
|
||||
repository: CodexCredentialRepository,
|
||||
profile: CodexWireProfileState,
|
||||
http: Client,
|
||||
@@ -79,6 +125,7 @@ impl CodexCredentialProfileService {
|
||||
profile,
|
||||
http,
|
||||
base_url,
|
||||
avatar_sources: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +161,7 @@ impl CodexCredentialProfileService {
|
||||
.authorization_header()
|
||||
.map_err(|_| CodexProfileStatisticsError::InvalidCredentialData)?;
|
||||
let request_id = format!("profile_statistics_{}", Uuid::now_v7().simple());
|
||||
CodexBackendClient::new(
|
||||
let statistics = CodexBackendClient::new(
|
||||
self.http.clone(),
|
||||
self.base_url.clone(),
|
||||
self.profile.clone(),
|
||||
@@ -126,7 +173,72 @@ impl CodexCredentialProfileService {
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.map_err(map_client_error)
|
||||
.map_err(map_client_error)?;
|
||||
self.remember_avatar_source(account_id, statistics.image_url.as_deref());
|
||||
Ok(statistics)
|
||||
}
|
||||
|
||||
pub async fn profile_avatar(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<CodexProfileAvatar, CodexProfileAvatarError> {
|
||||
let source = match self.cached_avatar_source(account_id) {
|
||||
Some(source) => source,
|
||||
None => self
|
||||
.profile_statistics(account_id)
|
||||
.await?
|
||||
.image_url
|
||||
.ok_or(CodexProfileAvatarError::Missing)?,
|
||||
};
|
||||
let desktop_user_agent = self.profile.snapshot().desktop_user_agent();
|
||||
fetch_profile_avatar(&self.http, &self.base_url, &desktop_user_agent, &source)
|
||||
.await
|
||||
.map_err(map_avatar_fetch_error)
|
||||
}
|
||||
|
||||
fn cached_avatar_source(&self, account_id: &ProviderAccountId) -> Option<String> {
|
||||
let now = Instant::now();
|
||||
let mut sources = self
|
||||
.avatar_sources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
sources.retain(|_, source| source.expires_at > now);
|
||||
sources.get(account_id).map(|source| source.source.clone())
|
||||
}
|
||||
|
||||
fn remember_avatar_source(&self, account_id: &ProviderAccountId, source: Option<&str>) {
|
||||
let now = Instant::now();
|
||||
let mut sources = self
|
||||
.avatar_sources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
sources.retain(|_, source| source.expires_at > now);
|
||||
match source {
|
||||
Some(source) => {
|
||||
sources.insert(
|
||||
account_id.clone(),
|
||||
CachedProfileAvatarSource {
|
||||
source: source.to_owned(),
|
||||
expires_at: now + PROFILE_AVATAR_SOURCE_TTL,
|
||||
},
|
||||
);
|
||||
}
|
||||
None => {
|
||||
sources.remove(account_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_avatar_fetch_error(error: CodexProfileAvatarFetchError) -> CodexProfileAvatarError {
|
||||
match error {
|
||||
CodexProfileAvatarFetchError::InvalidSource => CodexProfileAvatarError::InvalidSource,
|
||||
CodexProfileAvatarFetchError::Upstream { status } => {
|
||||
CodexProfileAvatarError::Upstream { status }
|
||||
}
|
||||
CodexProfileAvatarFetchError::TransportUnavailable => {
|
||||
CodexProfileAvatarError::TransportUnavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
|
||||
use super::affinity::CodexSessionAffinity;
|
||||
use super::affinity::{CODEX_ROOT_SESSION_TTL, CodexSessionAffinity};
|
||||
use super::catalog::CodexCredentialCatalogService;
|
||||
use super::cookie::CodexCookiePolicy;
|
||||
use super::quota::CodexCredentialQuotaService;
|
||||
@@ -41,7 +41,6 @@ const CLOUDFLARE_CHALLENGE_BACKOFF: [Duration; 4] = [
|
||||
Duration::from_secs(120),
|
||||
];
|
||||
const CLOUDFLARE_PATH_BLOCK_THRESHOLD: u32 = 3;
|
||||
const SESSION_AFFINITY_TTL: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
const SESSION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const CYBER_POLICY_SESSION_TTL: Duration = Duration::from_secs(60 * 60);
|
||||
|
||||
@@ -681,7 +680,7 @@ impl CodexCredentialSelector {
|
||||
&self.provider_kind,
|
||||
key,
|
||||
selected_account_id,
|
||||
SESSION_AFFINITY_TTL,
|
||||
CODEX_ROOT_SESSION_TTL,
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -955,7 +954,7 @@ impl CodexCredentialSelector {
|
||||
key,
|
||||
expected_affinity_account_id,
|
||||
account.id(),
|
||||
SESSION_AFFINITY_TTL,
|
||||
CODEX_ROOT_SESSION_TTL,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
mod admin;
|
||||
pub mod config;
|
||||
mod provider;
|
||||
mod session_transport;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -138,6 +139,7 @@ pub async fn initialize(
|
||||
profile.clone(),
|
||||
config.base_url().to_owned(),
|
||||
Arc::clone(&websocket_pool),
|
||||
config.stream_max_retries(),
|
||||
)
|
||||
.map_err(OpenAiInitializeError::Provider)?
|
||||
.with_session_identity(session_identity),
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
@@ -16,7 +17,7 @@ use gateway_core::engine::provider::{
|
||||
ProviderSelectionObservation, ProviderStream, UpstreamTransport,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, CancellationToken, ContinuationAttempt, UpstreamSendState,
|
||||
AttemptContext, AttemptTransport, CancellationToken, ContinuationAttempt, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
|
||||
@@ -47,6 +48,7 @@ use secrecy::{ExposeSecret, SecretString};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value, json};
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::credential::{
|
||||
CodexAccountFailure, CodexCredentialCatalogError, CodexCredentialCatalogService,
|
||||
@@ -56,6 +58,7 @@ use crate::credential::{
|
||||
SelectCodexCredential, SelectCodexProviderEndpointCredential,
|
||||
derive_codex_cyber_policy_session_key, derive_codex_session_affinity,
|
||||
};
|
||||
use crate::session_transport::CodexSessionTransportFallbacks;
|
||||
use crate::transport::canonical::{
|
||||
CodexCanonicalDecoder, CodexCanonicalError, CodexCanonicalOutcome,
|
||||
};
|
||||
@@ -69,11 +72,10 @@ use crate::transport::profile::{
|
||||
APPCAST_POLL_INTERVAL, CodexDesktopReleaseService, CodexWireProfileState,
|
||||
};
|
||||
use crate::transport::protocol::responses::{
|
||||
CodexResponsesRequest, PreviousResponseScope, ResponseEventSignals,
|
||||
};
|
||||
use crate::transport::protocol::websocket::{
|
||||
PREVIOUS_RESPONSE_NOT_FOUND_CODE, PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE,
|
||||
CodexResponsesRequest, PREVIOUS_RESPONSE_NOT_FOUND_CODE, PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE,
|
||||
PreviousResponseScope, ResponseEventSignals, TransportRequirement, transport_requirement,
|
||||
};
|
||||
use crate::transport::protocol::websocket::WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE;
|
||||
use crate::transport::request::{
|
||||
CodexRequestEncodeError, RequestAccountScope, encode_generate_request, scope_request_to_account,
|
||||
};
|
||||
@@ -127,6 +129,8 @@ pub struct CodexProvider {
|
||||
image_generations_url: Url,
|
||||
image_edits_url: Url,
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
stream_max_retries: u32,
|
||||
}
|
||||
|
||||
impl CodexProvider {
|
||||
@@ -141,6 +145,7 @@ impl CodexProvider {
|
||||
profile: CodexWireProfileState,
|
||||
base_url: String,
|
||||
websocket_pool: Arc<CodexWebSocketPool>,
|
||||
stream_max_retries: u32,
|
||||
) -> Result<Self, CodexProviderConfigError> {
|
||||
let responses_url = Url::parse(&endpoint_url(&base_url, CODEX_RESPONSES_PATH))
|
||||
.map_err(|_| CodexProviderConfigError::InvalidBaseUrl)?;
|
||||
@@ -161,6 +166,8 @@ impl CodexProvider {
|
||||
image_generations_url,
|
||||
image_edits_url,
|
||||
session_identity: None,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks::default(),
|
||||
stream_max_retries,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -289,8 +296,6 @@ impl Provider for CodexProvider {
|
||||
derive_codex_session_affinity(&upstream_request, context.client_api_key_ref());
|
||||
let cyber_policy_session_key =
|
||||
derive_codex_cyber_policy_session_key(&upstream_request, context.client_api_key_ref());
|
||||
let transport = selected_transport(&upstream_request);
|
||||
apply_transport(&mut upstream_request, transport);
|
||||
|
||||
let selection_started_at = Instant::now();
|
||||
let lease = self
|
||||
@@ -386,6 +391,39 @@ impl Provider for CodexProvider {
|
||||
lease.installation_id(),
|
||||
account_scope,
|
||||
);
|
||||
let requirement = transport_requirement(&upstream_request);
|
||||
let requested_transport = selected_transport(&upstream_request);
|
||||
let sticky_http_fallback = context.transport() == AttemptTransport::Default
|
||||
&& requested_transport == CodexProviderTransport::PreferWebSocket
|
||||
&& !requirement.requires_websocket()
|
||||
&& session_affinity.as_ref().is_some_and(|affinity| {
|
||||
self.session_transport_fallbacks
|
||||
.is_http_only(affinity.key())
|
||||
});
|
||||
let transport = if requirement.requires_websocket() {
|
||||
CodexProviderTransport::PreferWebSocket
|
||||
} else {
|
||||
match context.transport() {
|
||||
AttemptTransport::Default if sticky_http_fallback => {
|
||||
CodexProviderTransport::HttpOnly
|
||||
}
|
||||
AttemptTransport::Default => requested_transport,
|
||||
AttemptTransport::Retry(_) => requested_transport,
|
||||
AttemptTransport::Fallback => CodexProviderTransport::HttpOnly,
|
||||
}
|
||||
};
|
||||
if sticky_http_fallback {
|
||||
tracing::info!(
|
||||
request_id = %context.request_id(),
|
||||
attempt_index = context.attempt_index().get(),
|
||||
session_affinity_key_hash = session_affinity
|
||||
.as_ref()
|
||||
.map_or("", CodexSessionAffinity::key_hash),
|
||||
transport_requirement = requirement.as_str(),
|
||||
"Reusing session-sticky OpenAI upstream HTTP fallback"
|
||||
);
|
||||
}
|
||||
apply_transport(&mut upstream_request, transport);
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
provider_kind,
|
||||
upstream_model.clone(),
|
||||
@@ -409,7 +447,14 @@ impl Provider for CodexProvider {
|
||||
continuation_scope: None,
|
||||
});
|
||||
let allows_account_state_mutation = lease.allows_account_state_mutation();
|
||||
let session_affinity_key_hash = session_affinity
|
||||
.as_ref()
|
||||
.map(|affinity| affinity.key_hash().to_owned());
|
||||
let session_affinity_key = session_affinity.map(CodexSessionAffinity::into_key);
|
||||
let websocket_retry_count = match context.transport() {
|
||||
AttemptTransport::Retry(retry_index) => retry_index.get(),
|
||||
AttemptTransport::Default | AttemptTransport::Fallback => 0,
|
||||
};
|
||||
let events = cold_response_stream(ColdResponse {
|
||||
client: self.client.clone(),
|
||||
response_origin: self.responses_url.clone(),
|
||||
@@ -423,6 +468,10 @@ impl Provider for CodexProvider {
|
||||
lease: Arc::clone(&lease),
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks: self.session_transport_fallbacks.clone(),
|
||||
websocket_retry_count,
|
||||
stream_max_retries: self.stream_max_retries,
|
||||
session_capture,
|
||||
});
|
||||
let stream = ProviderStream::new(metadata, events, lease);
|
||||
@@ -520,6 +569,10 @@ struct ColdResponse {
|
||||
lease: Arc<CodexCredentialLease>,
|
||||
output_started_at: Instant,
|
||||
session_affinity_key: Option<ProviderSessionAffinityKey>,
|
||||
session_affinity_key_hash: Option<String>,
|
||||
session_transport_fallbacks: CodexSessionTransportFallbacks,
|
||||
websocket_retry_count: u32,
|
||||
stream_max_retries: u32,
|
||||
session_capture: Option<OpenAiSessionCapture>,
|
||||
}
|
||||
|
||||
@@ -661,7 +714,7 @@ async fn create_response_attempt(
|
||||
biased;
|
||||
_ = cancellation.cancelled() => Err(CodexHandshakeAttemptError::Cancelled),
|
||||
_ = tokio::time::sleep(handshake_deadline) => Err(CodexHandshakeAttemptError::Timeout),
|
||||
response = client.create_response_stream_with_pool_account(
|
||||
response = client.create_response_stream_with_deferred_websocket_recovery(
|
||||
request,
|
||||
request_context,
|
||||
Some(account_id),
|
||||
@@ -753,6 +806,12 @@ fn cold_json_response_stream(request: ColdJsonResponse) -> EventStream {
|
||||
account_selection,
|
||||
)
|
||||
.await;
|
||||
if let Err(CodexHandshakeAttemptError::Client(error)) = &response {
|
||||
log_client_upstream_error(
|
||||
UpstreamErrorLogContext::new(&request.context, &active_account, None),
|
||||
error,
|
||||
);
|
||||
}
|
||||
let response = match response.map_err(map_handshake_attempt_error) {
|
||||
Ok(response) => response,
|
||||
Err(mut failure) => {
|
||||
@@ -831,6 +890,10 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
lease,
|
||||
output_started_at,
|
||||
session_affinity_key,
|
||||
session_affinity_key_hash,
|
||||
session_transport_fallbacks,
|
||||
websocket_retry_count,
|
||||
stream_max_retries,
|
||||
mut session_capture,
|
||||
} = response;
|
||||
Box::pin(async_stream::try_stream! {
|
||||
@@ -862,6 +925,7 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
lease.escape_reason(),
|
||||
lease.account_switch(),
|
||||
);
|
||||
let request_transport_requirement = transport_requirement(&request);
|
||||
let response = create_response_attempt(
|
||||
&client,
|
||||
&request,
|
||||
@@ -879,10 +943,41 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
&cancellation,
|
||||
)
|
||||
.await;
|
||||
let websocket_failure_policy = match &response {
|
||||
Err(CodexHandshakeAttemptError::Client(error))
|
||||
if transport_policy == CodexProviderTransport::PreferWebSocket =>
|
||||
{
|
||||
websocket_client_failure_policy(error)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Err(CodexHandshakeAttemptError::Client(error)) = &response {
|
||||
log_client_upstream_error(
|
||||
UpstreamErrorLogContext::new(&context, &active_account, None),
|
||||
error,
|
||||
);
|
||||
}
|
||||
let response = response.map_err(map_handshake_attempt_error);
|
||||
let response = match response {
|
||||
Ok(response) => response,
|
||||
Err(mut failure) => {
|
||||
if let Some(policy) = websocket_failure_policy {
|
||||
apply_websocket_recovery_policy(
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy,
|
||||
requirement: request_transport_requirement,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
attempt_index: context.attempt_index().get(),
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(observation) = failure.observation.take() {
|
||||
yield ProviderEvent::observation(observation);
|
||||
}
|
||||
@@ -946,6 +1041,8 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
{
|
||||
active_account = current;
|
||||
}
|
||||
let response_transport = response.transport;
|
||||
let websocket_connection_id = response.websocket_connection_id;
|
||||
let mut body = response.body;
|
||||
let failure_diagnostics = response.diagnostics.clone();
|
||||
let failure_set_cookie_headers = response.set_cookie_headers.clone();
|
||||
@@ -990,7 +1087,17 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
_ = wait_for_replay_grace(replay_grace_deadline) => Ok(PreCommitPoll::GraceElapsed),
|
||||
chunk = body.next() => match chunk {
|
||||
Some(Ok(chunk)) => Ok(PreCommitPoll::Upstream(Some(chunk))),
|
||||
Some(Err(error)) => Err(map_stream_error(error)),
|
||||
Some(Err(error)) => {
|
||||
log_client_upstream_error(
|
||||
UpstreamErrorLogContext::new(
|
||||
&context,
|
||||
&active_account,
|
||||
websocket_connection_id,
|
||||
),
|
||||
&error,
|
||||
);
|
||||
Err(map_stream_error(error))
|
||||
}
|
||||
None => Ok(PreCommitPoll::Upstream(None)),
|
||||
},
|
||||
};
|
||||
@@ -1002,7 +1109,27 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(failure) => {
|
||||
Err(mut failure) => {
|
||||
if failure.websocket_transport_retryable
|
||||
&& response_transport == CodexBackendTransport::WebSocket
|
||||
&& !pre_commit_events.is_committed()
|
||||
{
|
||||
apply_websocket_recovery_policy(
|
||||
&mut failure,
|
||||
WebSocketRecoveryContext {
|
||||
policy: WebSocketFailurePolicy::Budgeted,
|
||||
requirement: request_transport_requirement,
|
||||
retry_count: websocket_retry_count,
|
||||
max_retries: stream_max_retries,
|
||||
request_id: context.request_id().as_str(),
|
||||
attempt_index: context.attempt_index().get(),
|
||||
account_id: active_account.id().as_str(),
|
||||
session_affinity_key: session_affinity_key.as_ref(),
|
||||
session_affinity_key_hash: session_affinity_key_hash.as_deref(),
|
||||
session_transport_fallbacks: &session_transport_fallbacks,
|
||||
},
|
||||
);
|
||||
}
|
||||
let updates = take_rate_limit_updates(rate_limit_updates.as_ref()).await;
|
||||
if !updates.is_empty() {
|
||||
passive_quota_observation.observe(&updates);
|
||||
@@ -1059,6 +1186,15 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
let service_tier_changed = observation_state
|
||||
.observe_upstream_service_tier(decoder.response_service_tier());
|
||||
let terminal_failure = canonical_failure.map(|(error, semantic_output_seen)| {
|
||||
log_canonical_upstream_error(
|
||||
UpstreamErrorLogContext::new(
|
||||
&context,
|
||||
&active_account,
|
||||
websocket_connection_id,
|
||||
),
|
||||
response_transport,
|
||||
&error,
|
||||
);
|
||||
let atomic_upstream_failure = matches!(&error, CodexCanonicalError::Upstream(_));
|
||||
(
|
||||
map_canonical_error(
|
||||
@@ -1150,6 +1286,15 @@ fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
};
|
||||
let terminal_failure = canonical_failure.map(|(error, semantic_output_seen)| {
|
||||
log_canonical_upstream_error(
|
||||
UpstreamErrorLogContext::new(
|
||||
&context,
|
||||
&active_account,
|
||||
websocket_connection_id,
|
||||
),
|
||||
response_transport,
|
||||
&error,
|
||||
);
|
||||
let atomic_upstream_failure = matches!(&error, CodexCanonicalError::Upstream(_));
|
||||
(
|
||||
map_canonical_error(
|
||||
@@ -1594,7 +1739,11 @@ fn codex_response_observation(
|
||||
WebSocketPoolKind::New
|
||||
});
|
||||
}
|
||||
if let Some(status_code) = diagnostics.status_code {
|
||||
// WebSocket opening 的 101 是成功升级事实,不是业务请求的失败 HTTP 状态。
|
||||
// opening 明确拒绝仍由 `codex_error_observation` 保存真实上游状态。
|
||||
if transport != CodexBackendTransport::WebSocket
|
||||
&& let Some(status_code) = diagnostics.status_code
|
||||
{
|
||||
observation = observation.with_status_code(status_code);
|
||||
}
|
||||
if let Some(request_id) = diagnostics.request_id.as_deref() {
|
||||
@@ -2033,6 +2182,7 @@ fn map_selection_error(error: CredentialSelectionError) -> ProviderError {
|
||||
|
||||
struct MappedProviderFailure {
|
||||
error: ProviderError,
|
||||
websocket_transport_retryable: bool,
|
||||
account_failure: Option<CodexAccountFailure>,
|
||||
/// 原始上游错误描述,仅在凭据错误状态下持久化。
|
||||
error_message: Option<String>,
|
||||
@@ -2047,6 +2197,7 @@ impl MappedProviderFailure {
|
||||
fn plain(error: ProviderError) -> Self {
|
||||
Self {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
account_failure: None,
|
||||
error_message: None,
|
||||
cyber_policy_failure: false,
|
||||
@@ -2196,6 +2347,144 @@ struct OpenAiFailureContext<'a> {
|
||||
allows_account_state_mutation: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct UpstreamErrorLogContext<'a> {
|
||||
request_id: &'a str,
|
||||
account_id: &'a str,
|
||||
attempt_index: u32,
|
||||
websocket_connection_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl<'a> UpstreamErrorLogContext<'a> {
|
||||
fn new(
|
||||
context: &'a AttemptContext,
|
||||
account: &'a ProviderAccount,
|
||||
websocket_connection_id: Option<Uuid>,
|
||||
) -> Self {
|
||||
Self {
|
||||
request_id: context.request_id().as_str(),
|
||||
account_id: account.id().as_str(),
|
||||
attempt_index: context.attempt_index().get(),
|
||||
websocket_connection_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_websocket_connection_id(mut self, connection_id: Option<Uuid>) -> Self {
|
||||
self.websocket_connection_id = self.websocket_connection_id.or(connection_id);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn log_client_upstream_error(context: UpstreamErrorLogContext<'_>, error: &CodexClientError) {
|
||||
match error {
|
||||
CodexClientError::Upstream {
|
||||
status,
|
||||
body,
|
||||
transport,
|
||||
..
|
||||
} => log_raw_upstream_body(
|
||||
context,
|
||||
*transport,
|
||||
"http_error_response",
|
||||
Some(status.as_u16()),
|
||||
None,
|
||||
None,
|
||||
body,
|
||||
),
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::Upstream(upstream)) => {
|
||||
log_raw_upstream_body(
|
||||
context,
|
||||
CodexBackendTransport::WebSocket,
|
||||
"websocket_opening_response",
|
||||
Some(upstream.status_code),
|
||||
None,
|
||||
None,
|
||||
&upstream.body,
|
||||
);
|
||||
}
|
||||
CodexClientError::WebSocket(error) => {
|
||||
let Some(close) = error.close_before_terminal() else {
|
||||
return;
|
||||
};
|
||||
let context = context.with_websocket_connection_id(close.connection_id());
|
||||
let websocket_connection_id = context
|
||||
.websocket_connection_id
|
||||
.map(|connection_id| connection_id.to_string())
|
||||
.unwrap_or_default();
|
||||
let close_reason_bytes = close.reason().map_or(0, str::len);
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = %context.account_id,
|
||||
attempt_index = context.attempt_index,
|
||||
websocket_connection_id,
|
||||
upstream_transport = WEBSOCKET_TRANSPORT,
|
||||
upstream_error_kind = "websocket_close",
|
||||
upstream_close_code = close.code().unwrap_or_default(),
|
||||
upstream_close_code_present = close.code().is_some(),
|
||||
upstream_last_event_type = close.last_event_type().unwrap_or_default(),
|
||||
upstream_last_event_type_present = close.last_event_type().is_some(),
|
||||
upstream_terminal_seen = false,
|
||||
upstream_close_reason_bytes = close_reason_bytes,
|
||||
upstream_close_reason_present = close.reason().is_some(),
|
||||
upstream_error_raw_present = close.reason().is_some(),
|
||||
"OpenAI upstream WebSocket closed before terminal event"
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn log_canonical_upstream_error(
|
||||
context: UpstreamErrorLogContext<'_>,
|
||||
transport: CodexBackendTransport,
|
||||
error: &CodexCanonicalError,
|
||||
) {
|
||||
let CodexCanonicalError::Upstream(failure) = error else {
|
||||
return;
|
||||
};
|
||||
log_raw_upstream_body(
|
||||
context,
|
||||
transport,
|
||||
"responses_error_event",
|
||||
failure.explicit_status_code,
|
||||
failure.upstream_code.as_deref(),
|
||||
failure.upstream_type.as_deref(),
|
||||
failure.raw_body(),
|
||||
);
|
||||
}
|
||||
|
||||
fn log_raw_upstream_body(
|
||||
context: UpstreamErrorLogContext<'_>,
|
||||
transport: CodexBackendTransport,
|
||||
error_kind: &'static str,
|
||||
status_code: Option<u16>,
|
||||
upstream_code: Option<&str>,
|
||||
upstream_type: Option<&str>,
|
||||
upstream_error_raw: &str,
|
||||
) {
|
||||
let websocket_connection_id = context
|
||||
.websocket_connection_id
|
||||
.map(|connection_id| connection_id.to_string())
|
||||
.unwrap_or_default();
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
account_id = %context.account_id,
|
||||
attempt_index = context.attempt_index,
|
||||
websocket_connection_id,
|
||||
upstream_transport = actual_transport_name(transport),
|
||||
upstream_error_kind = error_kind,
|
||||
upstream_status_code = status_code.unwrap_or_default(),
|
||||
upstream_status_code_present = status_code.is_some(),
|
||||
upstream_error_code = upstream_code.unwrap_or_default(),
|
||||
upstream_error_code_present = upstream_code.is_some(),
|
||||
upstream_error_type = upstream_type.unwrap_or_default(),
|
||||
upstream_error_type_present = upstream_type.is_some(),
|
||||
upstream_error_raw,
|
||||
upstream_error_raw_bytes = upstream_error_raw.len(),
|
||||
"OpenAI upstream returned an error payload"
|
||||
);
|
||||
}
|
||||
|
||||
async fn apply_failure(
|
||||
context: &OpenAiFailureContext<'_>,
|
||||
account: &ProviderAccount,
|
||||
@@ -2294,6 +2583,151 @@ fn map_handshake_error(error: CodexClientError) -> MappedProviderFailure {
|
||||
map_client_error(error, UpstreamSendState::Ambiguous, true)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum WebSocketFailurePolicy {
|
||||
Budgeted,
|
||||
ImmediateFallback,
|
||||
}
|
||||
|
||||
struct WebSocketRecoveryContext<'a> {
|
||||
policy: WebSocketFailurePolicy,
|
||||
requirement: TransportRequirement,
|
||||
retry_count: u32,
|
||||
max_retries: u32,
|
||||
request_id: &'a str,
|
||||
attempt_index: u32,
|
||||
account_id: &'a str,
|
||||
session_affinity_key: Option<&'a ProviderSessionAffinityKey>,
|
||||
session_affinity_key_hash: Option<&'a str>,
|
||||
session_transport_fallbacks: &'a CodexSessionTransportFallbacks,
|
||||
}
|
||||
|
||||
fn websocket_client_failure_policy(error: &CodexClientError) -> Option<WebSocketFailurePolicy> {
|
||||
match error {
|
||||
CodexClientError::Upstream {
|
||||
status,
|
||||
transport: CodexBackendTransport::WebSocket,
|
||||
..
|
||||
} if *status == reqwest::StatusCode::UPGRADE_REQUIRED => {
|
||||
Some(WebSocketFailurePolicy::ImmediateFallback)
|
||||
}
|
||||
CodexClientError::Upstream {
|
||||
status,
|
||||
transport: CodexBackendTransport::WebSocket,
|
||||
..
|
||||
} if status.is_server_error()
|
||||
|| matches!(
|
||||
*status,
|
||||
reqwest::StatusCode::REQUEST_TIMEOUT
|
||||
| reqwest::StatusCode::CONFLICT
|
||||
| reqwest::StatusCode::TOO_EARLY
|
||||
| reqwest::StatusCode::TOO_MANY_REQUESTS
|
||||
) =>
|
||||
{
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
CodexClientError::WebSocket(error)
|
||||
if !matches!(error, CodexWebSocketExchangeError::InvalidRequest(_)) =>
|
||||
{
|
||||
Some(WebSocketFailurePolicy::Budgeted)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_websocket_recovery_policy(
|
||||
failure: &mut MappedProviderFailure,
|
||||
context: WebSocketRecoveryContext<'_>,
|
||||
) {
|
||||
let recovery_is_safe = match failure.error.send_state() {
|
||||
UpstreamSendState::NotSent => context.requirement.allows_pre_send_http_fallback(),
|
||||
UpstreamSendState::Sent | UpstreamSendState::Ambiguous => {
|
||||
context.requirement.allows_pre_delivery_http_fallback()
|
||||
}
|
||||
};
|
||||
if !recovery_is_safe {
|
||||
return;
|
||||
}
|
||||
|
||||
// 传输恢复必须保持原账号可调度;最终 HTTP attempt 若仍失败,再按真实 HTTP
|
||||
// 结果更新账号健康度,避免中间 WS 错误把同账号钉选提前冷却掉。
|
||||
failure.account_failure = None;
|
||||
let fallback_now = context.policy == WebSocketFailurePolicy::ImmediateFallback
|
||||
|| context.retry_count >= context.max_retries;
|
||||
if !fallback_now {
|
||||
let Some(retry_index) = context.retry_count.checked_add(1).and_then(NonZeroU32::new) else {
|
||||
return;
|
||||
};
|
||||
let delay = failure
|
||||
.error
|
||||
.retry_after()
|
||||
.unwrap_or_else(|| websocket_retry_backoff(retry_index));
|
||||
failure
|
||||
.error
|
||||
.set_pre_delivery_transport_retry(retry_index, delay);
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
account_id = context.account_id,
|
||||
websocket_failure_kind = failure.error.kind().as_str(),
|
||||
websocket_failure_code = failure
|
||||
.error
|
||||
.upstream_code()
|
||||
.map_or("", OpaqueUpstreamValue::as_str),
|
||||
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
|
||||
upstream_status_code_present = failure.error.upstream_status().is_some(),
|
||||
websocket_retry_count = retry_index.get(),
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_retry_delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
|
||||
transport_requirement = context.requirement.as_str(),
|
||||
"OpenAI upstream WebSocket failed; retrying the same account"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
failure.error.set_pre_delivery_transport_fallback();
|
||||
let sticky_state_transition = context
|
||||
.session_affinity_key
|
||||
.map(|key| context.session_transport_fallbacks.disable_websocket(key));
|
||||
tracing::warn!(
|
||||
request_id = context.request_id,
|
||||
attempt_index = context.attempt_index,
|
||||
account_id = context.account_id,
|
||||
websocket_failure_kind = failure.error.kind().as_str(),
|
||||
websocket_failure_code = failure
|
||||
.error
|
||||
.upstream_code()
|
||||
.map_or("", OpaqueUpstreamValue::as_str),
|
||||
upstream_status_code = failure.error.upstream_status().unwrap_or_default(),
|
||||
upstream_status_code_present = failure.error.upstream_status().is_some(),
|
||||
websocket_retry_count = context.retry_count,
|
||||
websocket_max_retries = context.max_retries,
|
||||
websocket_fallback_reason = match context.policy {
|
||||
WebSocketFailurePolicy::Budgeted => "retry_budget_exhausted",
|
||||
WebSocketFailurePolicy::ImmediateFallback => "upgrade_required",
|
||||
},
|
||||
session_affinity_present = context.session_affinity_key.is_some(),
|
||||
session_affinity_key_hash = context.session_affinity_key_hash.unwrap_or(""),
|
||||
sticky_http_enabled = sticky_state_transition.is_some(),
|
||||
sticky_state_transition = sticky_state_transition.unwrap_or(false),
|
||||
"OpenAI upstream WebSocket disabled for the current session"
|
||||
);
|
||||
}
|
||||
|
||||
fn websocket_retry_backoff(retry_index: NonZeroU32) -> Duration {
|
||||
const INITIAL_DELAY_MS: u64 = 200;
|
||||
|
||||
let shift = retry_index.get().saturating_sub(1).min(63);
|
||||
let base_ms = INITIAL_DELAY_MS.saturating_mul(1_u64.checked_shl(shift).unwrap_or(u64::MAX));
|
||||
let mut random = [0_u8; 2];
|
||||
let jitter_per_mille = if getrandom::fill(&mut random).is_ok() {
|
||||
900_u64 + u64::from(u16::from_le_bytes(random) % 200)
|
||||
} else {
|
||||
1_000
|
||||
};
|
||||
Duration::from_millis(base_ms.saturating_mul(jitter_per_mille) / 1_000)
|
||||
}
|
||||
|
||||
fn continuation_replay_required_error() -> ProviderError {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
@@ -2309,8 +2743,10 @@ fn continuation_replay_required_error() -> ProviderError {
|
||||
|
||||
fn map_stream_error(error: CodexClientError) -> MappedProviderFailure {
|
||||
let allows_pre_delivery_retry = stream_transport_allows_pre_delivery_retry(&error);
|
||||
let websocket_failure = error.transport() == Some(CodexBackendTransport::WebSocket);
|
||||
let mut failure = map_client_error(error, UpstreamSendState::Sent, false);
|
||||
if allows_pre_delivery_retry {
|
||||
failure.websocket_transport_retryable = allows_pre_delivery_retry && websocket_failure;
|
||||
if allows_pre_delivery_retry && !websocket_failure {
|
||||
failure.error = failure.error.with_pre_delivery_retry();
|
||||
}
|
||||
failure
|
||||
@@ -2321,15 +2757,11 @@ fn stream_transport_allows_pre_delivery_retry(error: &CodexClientError) -> bool
|
||||
CodexClientError::Http(_)
|
||||
| CodexClientError::HttpJson(_)
|
||||
| CodexClientError::StreamIdleTimeout { .. } => true,
|
||||
CodexClientError::WebSocket(error) => matches!(
|
||||
CodexClientError::WebSocket(error) => !matches!(
|
||||
error,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
|
||||
| CodexWebSocketExchangeError::SendTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ClosedBeforeTerminal(_)
|
||||
| CodexWebSocketExchangeError::ReceiveIdleTimeout { .. }
|
||||
| CodexWebSocketExchangeError::ReusedConnectionDiedBeforeFirstEvent { .. }
|
||||
| CodexWebSocketExchangeError::InitialEventTimeout { .. }
|
||||
CodexWebSocketExchangeError::InvalidRequest(_)
|
||||
| CodexWebSocketExchangeError::Upstream(_)
|
||||
| CodexWebSocketExchangeError::ContinuationUnavailable { .. }
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
@@ -2417,11 +2849,25 @@ fn map_client_error(
|
||||
))
|
||||
}
|
||||
CodexClientError::WebSocket(error) => {
|
||||
let close_code = error.close_before_terminal().and_then(|close| close.code());
|
||||
let client_visible_error = websocket_client_visible_error(&error);
|
||||
let mut failure = MappedProviderFailure::plain(provider_error(
|
||||
websocket_error_kind(&error),
|
||||
websocket_send_state(&error),
|
||||
));
|
||||
if let Some(close_code) = close_code {
|
||||
failure.error =
|
||||
failure
|
||||
.error
|
||||
.with_upstream_code(OpaqueUpstreamValue::new(format!(
|
||||
"websocket_close_{close_code}"
|
||||
)));
|
||||
}
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
failure.error = failure.error.with_upstream_code(OpaqueUpstreamValue::new(
|
||||
WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned(),
|
||||
));
|
||||
}
|
||||
if let Some(client_visible_error) = client_visible_error {
|
||||
failure.error = failure
|
||||
.error
|
||||
@@ -2442,6 +2888,13 @@ fn map_client_error(
|
||||
fn websocket_client_visible_error(
|
||||
error: &CodexWebSocketExchangeError,
|
||||
) -> Option<ClientVisibleUpstreamError> {
|
||||
if matches!(error, CodexWebSocketExchangeError::ConnectionLimitReached) {
|
||||
return Some(ClientVisibleUpstreamError::new(
|
||||
"websocket connection limit reached",
|
||||
Some(WEBSOCKET_CONNECTION_LIMIT_REACHED_CODE.to_owned()),
|
||||
Some("websocket_error".to_owned()),
|
||||
));
|
||||
}
|
||||
let close = error.close_before_terminal()?;
|
||||
let message = close
|
||||
.reason()
|
||||
@@ -2516,6 +2969,7 @@ fn map_upstream_failure(
|
||||
}
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
account_failure: account_failure(
|
||||
category,
|
||||
failure.retry_after_seconds,
|
||||
@@ -2627,6 +3081,7 @@ const fn websocket_send_state(error: &CodexWebSocketExchangeError) -> UpstreamSe
|
||||
| CodexWebSocketExchangeError::SharedConnectFailed
|
||||
| CodexWebSocketExchangeError::ContinuationUnavailable { .. } => UpstreamSendState::NotSent,
|
||||
CodexWebSocketExchangeError::Upstream(_)
|
||||
| CodexWebSocketExchangeError::ConnectionLimitReached
|
||||
| CodexWebSocketExchangeError::InvalidSse(_)
|
||||
| CodexWebSocketExchangeError::UnexpectedBinaryEvent => UpstreamSendState::Sent,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
@@ -2656,6 +3111,7 @@ const fn websocket_error_kind(error: &CodexWebSocketExchangeError) -> ProviderEr
|
||||
ProviderErrorKind::Unavailable
|
||||
}
|
||||
CodexWebSocketExchangeError::Upstream(_) => ProviderErrorKind::Unavailable,
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached => ProviderErrorKind::RateLimited,
|
||||
CodexWebSocketExchangeError::Transport(_)
|
||||
| CodexWebSocketExchangeError::Connect(_)
|
||||
| CodexWebSocketExchangeError::PostSendAmbiguous { .. }
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
//! OpenAI 根会话级上游传输降级状态。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use gateway_core::provider_ports::ProviderSessionAffinityKey;
|
||||
|
||||
use crate::credential::CODEX_ROOT_SESSION_TTL;
|
||||
|
||||
/// Provider 是进程级单例;限制不可显式观测结束的客户端会话占用。
|
||||
const MAX_SESSION_HTTP_FALLBACKS: usize = 16_384;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CodexSessionTransportFallbacks {
|
||||
state: Arc<Mutex<SessionTransportFallbackState>>,
|
||||
ttl: Duration,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionTransportFallbackState {
|
||||
http_only: HashMap<ProviderSessionAffinityKey, Instant>,
|
||||
}
|
||||
|
||||
impl Default for CodexSessionTransportFallbacks {
|
||||
fn default() -> Self {
|
||||
Self::with_limits(CODEX_ROOT_SESSION_TTL, MAX_SESSION_HTTP_FALLBACKS)
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexSessionTransportFallbacks {
|
||||
fn with_limits(ttl: Duration, capacity: usize) -> Self {
|
||||
debug_assert!(!ttl.is_zero());
|
||||
debug_assert!(capacity > 0);
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(SessionTransportFallbackState::default())),
|
||||
ttl,
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回该根会话是否已粘性切到 HTTP,并为活跃会话续期。
|
||||
pub(crate) fn is_http_only(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.is_http_only_at(key, Instant::now())
|
||||
}
|
||||
|
||||
/// 在当前 WebSocket 重试预算耗尽后,为根会话启用粘性 HTTP。
|
||||
///
|
||||
/// 返回 `true` 表示本次发生了禁用状态转换;已禁用会话只续期。
|
||||
pub(crate) fn disable_websocket(&self, key: &ProviderSessionAffinityKey) -> bool {
|
||||
self.disable_websocket_at(key, Instant::now())
|
||||
}
|
||||
|
||||
fn is_http_only_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
let mut state = self.lock();
|
||||
let Some(last_seen) = state.http_only.get_mut(key) else {
|
||||
return false;
|
||||
};
|
||||
if now.saturating_duration_since(*last_seen) >= self.ttl {
|
||||
state.http_only.remove(key);
|
||||
return false;
|
||||
}
|
||||
*last_seen = now;
|
||||
true
|
||||
}
|
||||
|
||||
fn disable_websocket_at(&self, key: &ProviderSessionAffinityKey, now: Instant) -> bool {
|
||||
let mut state = self.lock();
|
||||
let already_disabled = state
|
||||
.http_only
|
||||
.get(key)
|
||||
.is_some_and(|last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if !already_disabled && !state.http_only.contains_key(key) {
|
||||
self.make_room(&mut state, now);
|
||||
}
|
||||
state.http_only.insert(key.clone(), now);
|
||||
!already_disabled
|
||||
}
|
||||
|
||||
fn make_room(&self, state: &mut SessionTransportFallbackState, now: Instant) {
|
||||
if state.http_only.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
state
|
||||
.http_only
|
||||
.retain(|_, last_seen| now.saturating_duration_since(*last_seen) < self.ttl);
|
||||
if state.http_only.len() < self.capacity {
|
||||
return;
|
||||
}
|
||||
let oldest = state
|
||||
.http_only
|
||||
.iter()
|
||||
.min_by_key(|(_, last_seen)| **last_seen)
|
||||
.map(|(key, _)| key.clone());
|
||||
if let Some(oldest) = oldest {
|
||||
state.http_only.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
fn lock(&self) -> MutexGuard<'_, SessionTransportFallbackState> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
@@ -59,7 +59,7 @@ pub enum CodexCanonicalError {
|
||||
Protocol(#[source] ProviderError),
|
||||
/// 上游在成功建立流后发送了明确的失败事件。
|
||||
#[error("Codex upstream reported a failed response")]
|
||||
Upstream(ResponsesSseFailure),
|
||||
Upstream(Box<ResponsesSseFailure>),
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexCanonicalError {
|
||||
@@ -291,7 +291,11 @@ impl CodexCanonicalDecoder {
|
||||
let signals = response_event_signals(event_type, &value);
|
||||
self.merge_timing_signals(signals);
|
||||
if matches!(event_type, Some("response.failed" | "error")) {
|
||||
let failure = ResponsesSseFailure::from_event(event_type.unwrap_or_default(), &value);
|
||||
let failure = ResponsesSseFailure::from_raw_event(
|
||||
event_type.unwrap_or_default(),
|
||||
&event.data,
|
||||
&value,
|
||||
);
|
||||
let mut canonical = Vec::new();
|
||||
if !self.started {
|
||||
// 失败首帧可能是唯一携带 response_id 的上游事实。身份投影只做
|
||||
@@ -307,7 +311,7 @@ impl CodexCanonicalDecoder {
|
||||
} else {
|
||||
output.extend(canonical.into_iter().map(ProviderEvent::canonical));
|
||||
}
|
||||
return Err(CodexCanonicalError::Upstream(failure));
|
||||
return Err(CodexCanonicalError::Upstream(Box::new(failure)));
|
||||
}
|
||||
let mut canonical = Vec::new();
|
||||
if let Some(event_type) = event_type {
|
||||
|
||||
@@ -22,6 +22,7 @@ use reqwest::{
|
||||
};
|
||||
use serde_json::{Value, map::Map};
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::transport::protocol::responses::{CodexResponsesRequest, TransportRequirement};
|
||||
|
||||
@@ -401,7 +402,7 @@ pub struct CodexRequestContext<'a> {
|
||||
pub beta_features: Option<&'a str>,
|
||||
/// x-responsesapi-include-timing-metrics 头值。
|
||||
pub include_timing_metrics: Option<&'a str>,
|
||||
/// 客户端版本号。
|
||||
/// 下游 `version` 扩展头;存在时上游值由 Desktop 版本画像统一生成。
|
||||
pub version: Option<&'a str>,
|
||||
/// 客户端 window ID。
|
||||
pub codex_window_id: Option<&'a str>,
|
||||
@@ -538,6 +539,8 @@ pub struct CodexBackendStreamingResponse {
|
||||
pub body: CodexBackendSseStream,
|
||||
/// 实际使用的上游传输。
|
||||
pub transport: CodexBackendTransport,
|
||||
/// WebSocket 响应所绑定的连接;HTTP transport 为 `None`。
|
||||
pub websocket_connection_id: Option<Uuid>,
|
||||
/// 响应头里的最新 turn state。
|
||||
pub turn_state: Option<String>,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
@@ -596,6 +599,7 @@ pub(crate) struct PreparedResponseTransport {
|
||||
pub(super) requirement: TransportRequirement,
|
||||
pub(super) route: PreparedResponseRoute,
|
||||
pub(super) metrics: CodexTransportMetrics,
|
||||
pub(super) defer_websocket_recovery: bool,
|
||||
}
|
||||
|
||||
pub(super) enum PreparedResponseRoute {
|
||||
|
||||
@@ -151,6 +151,7 @@ impl CodexBackendClient {
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
body: http_sse_stream(response, Arc::clone(&rate_limit_updates)),
|
||||
transport: CodexBackendTransport::HttpSse,
|
||||
websocket_connection_id: None,
|
||||
turn_state,
|
||||
set_cookie_headers,
|
||||
rate_limit_headers,
|
||||
@@ -181,6 +182,20 @@ impl CodexBackendClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Provider 生产路径:保留所有 WebSocket 失败,交由统一重试预算处理。
|
||||
pub(crate) async fn create_response_stream_with_deferred_websocket_recovery(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<CodexBackendStreamingResponse> {
|
||||
let prepared = self
|
||||
.prepare_response_transport(request, context, pool_account_id, false, true)
|
||||
.await?;
|
||||
self.create_response_stream_with_prepared(request, context, prepared)
|
||||
.await
|
||||
}
|
||||
|
||||
/// 在发送 payload 前完成 transport 选择和可取消的 WebSocket opening。
|
||||
#[doc(hidden)]
|
||||
pub(crate) async fn prepare_response_transport_with_pool_account(
|
||||
@@ -189,7 +204,7 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
self.prepare_response_transport(request, context, pool_account_id, false)
|
||||
self.prepare_response_transport(request, context, pool_account_id, false, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -201,12 +216,14 @@ impl CodexBackendClient {
|
||||
context: CodexRequestContext<'_>,
|
||||
pool_account_id: Option<&str>,
|
||||
force_fresh: bool,
|
||||
defer_websocket_recovery: bool,
|
||||
) -> CodexClientResult<PreparedResponseTransport> {
|
||||
let requirement = transport_requirement(request);
|
||||
if requirement == TransportRequirement::HttpRequired {
|
||||
return Ok(PreparedResponseTransport {
|
||||
requirement,
|
||||
route: PreparedResponseRoute::Http,
|
||||
defer_websocket_recovery,
|
||||
metrics: CodexTransportMetrics {
|
||||
decision: Some(CodexTransportDecision::HttpRequired),
|
||||
..CodexTransportMetrics::default()
|
||||
@@ -267,7 +284,8 @@ impl CodexBackendClient {
|
||||
let prepared = match prepared {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error)
|
||||
if requirement.allows_pre_send_http_fallback()
|
||||
if (!defer_websocket_recovery || error.requires_immediate_pool_fallback())
|
||||
&& requirement.allows_pre_send_http_fallback()
|
||||
&& error.allows_pre_send_http_fallback() =>
|
||||
{
|
||||
let decision = http_fallback_decision(&error);
|
||||
@@ -290,6 +308,7 @@ impl CodexBackendClient {
|
||||
transport_decision_wait_ms: Some(wait_ms),
|
||||
..CodexTransportMetrics::default()
|
||||
},
|
||||
defer_websocket_recovery,
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
|
||||
@@ -327,6 +346,7 @@ impl CodexBackendClient {
|
||||
prepared,
|
||||
})),
|
||||
metrics,
|
||||
defer_websocket_recovery,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -341,6 +361,7 @@ impl CodexBackendClient {
|
||||
requirement,
|
||||
route,
|
||||
metrics,
|
||||
defer_websocket_recovery,
|
||||
} = prepared;
|
||||
match route {
|
||||
PreparedResponseRoute::Http => self
|
||||
@@ -364,7 +385,8 @@ impl CodexBackendClient {
|
||||
{
|
||||
Ok(exchange) => exchange,
|
||||
Err(error)
|
||||
if requirement.allows_pre_delivery_http_fallback()
|
||||
if !defer_websocket_recovery
|
||||
&& requirement.allows_pre_delivery_http_fallback()
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
{
|
||||
return self
|
||||
@@ -380,16 +402,20 @@ impl CodexBackendClient {
|
||||
}
|
||||
Err(error) => return Err(websocket_exchange_error_to_client_error(error)),
|
||||
};
|
||||
// 仅普通新链请求允许交付前降级/重试;延续类请求保持旧语义:
|
||||
// 不进入交付边界,错误由 exchange 流原样上抛给下游。
|
||||
if requirement.allows_pre_delivery_http_fallback() {
|
||||
let mut fresh_retry_used = false;
|
||||
loop {
|
||||
match await_websocket_delivery_boundary(&mut exchange).await {
|
||||
Ok(DeliveryBoundary::Ready) => break,
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached { message }) => {
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached)
|
||||
if defer_websocket_recovery =>
|
||||
{
|
||||
return Err(CodexClientError::WebSocket(
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached,
|
||||
));
|
||||
}
|
||||
Ok(DeliveryBoundary::ConnectionLimitReached) => {
|
||||
if fresh_retry_used {
|
||||
// 新连接仍被限流:与官方重试预算耗尽一致,降级同账号 HTTP/SSE。
|
||||
return self
|
||||
.http_fallback_before_delivery(
|
||||
request,
|
||||
@@ -397,20 +423,18 @@ impl CodexBackendClient {
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
&message,
|
||||
"websocket connection limit reached",
|
||||
)
|
||||
.await
|
||||
;
|
||||
.await;
|
||||
}
|
||||
fresh_retry_used = true;
|
||||
tracing::warn!(
|
||||
request_id = %context.request_id,
|
||||
%message,
|
||||
"WebSocket connection limit reached; retrying on a fresh connection"
|
||||
);
|
||||
drop(exchange);
|
||||
match self
|
||||
.prepare_response_transport(request, context, None, true)
|
||||
.prepare_response_transport(request, context, None, true, false)
|
||||
.await
|
||||
{
|
||||
Ok(PreparedResponseTransport {
|
||||
@@ -479,19 +503,21 @@ impl CodexBackendClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) if !defer_websocket_recovery
|
||||
&& error.allows_pre_delivery_http_fallback() =>
|
||||
{
|
||||
return self
|
||||
.fallback_to_http_before_websocket_delivery(
|
||||
request,
|
||||
context,
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
if error.allows_pre_delivery_http_fallback() {
|
||||
return self
|
||||
.fallback_to_http_before_websocket_delivery(
|
||||
request,
|
||||
context,
|
||||
requirement,
|
||||
metrics,
|
||||
delivery_wait_started_at,
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
return Err(websocket_exchange_error_to_client_error(
|
||||
post_send_ambiguous(error),
|
||||
));
|
||||
@@ -513,6 +539,7 @@ impl CodexBackendClient {
|
||||
.map_err(websocket_exchange_error_to_client_error),
|
||||
),
|
||||
transport: CodexBackendTransport::WebSocket,
|
||||
websocket_connection_id: Some(exchange.websocket_connection_id),
|
||||
turn_state: exchange.turn_state,
|
||||
set_cookie_headers: exchange.set_cookie_headers,
|
||||
rate_limit_headers: exchange.rate_limit_headers,
|
||||
@@ -557,7 +584,6 @@ impl CodexBackendClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// 交付前降级同账号 HTTP/SSE 的公共路径:统一标记 decision 与等待耗时。
|
||||
async fn http_fallback_before_delivery(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
@@ -672,15 +698,16 @@ impl CodexBackendClient {
|
||||
X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
request.memgen_request.as_deref(),
|
||||
);
|
||||
// 客户端确实携带的普通协议头优先;没有携带时才使用上面的 Codex profile 默认值。
|
||||
// 客户端确实携带的普通协议头优先;画像身份头只能由 Codex profile 生成。
|
||||
for name in request.passthrough_headers.keys() {
|
||||
// 上游指纹必须由运行时画像统一生成:originator/User-Agent 即使绕过
|
||||
// API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
|
||||
// 上游指纹必须由运行时画像统一生成:originator/User-Agent/version 即使
|
||||
// 绕过 API 透传黑名单也不能覆盖画像,避免下游客户端暴露不一致指纹。
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"openai-beta"
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
| "version"
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
@@ -754,7 +781,17 @@ impl CodexBackendClient {
|
||||
"x-responsesapi-include-timing-metrics",
|
||||
context.include_timing_metrics,
|
||||
);
|
||||
insert_optional_protocol_header(&mut headers, "version", context.version);
|
||||
// 官方 Desktop 的版本画像来自 clientInfo.version。保留下游是否携带
|
||||
// `version` 扩展头的协议形状,但上游值统一使用当前 Desktop 制品版本。
|
||||
if context
|
||||
.version
|
||||
.is_some_and(|version| HeaderValue::from_str(version).is_ok())
|
||||
{
|
||||
headers.insert(
|
||||
HeaderName::from_static("version"),
|
||||
HeaderValue::from_str(&profile.desktop_version)?,
|
||||
);
|
||||
}
|
||||
insert_optional_protocol_header(
|
||||
&mut headers,
|
||||
"x-codex-parent-thread-id",
|
||||
@@ -802,12 +839,8 @@ impl CodexBackendClient {
|
||||
enum DeliveryBoundary {
|
||||
/// 已越过边界,可开始向下游投递。
|
||||
Ready,
|
||||
/// 首个可投递帧是上游连接寿命限制错误;该帧已放回流,
|
||||
/// 调用方可选择丢弃并换新连接重试,或原样投递。
|
||||
ConnectionLimitReached {
|
||||
/// 上游错误说明。
|
||||
message: String,
|
||||
},
|
||||
/// 首个可投递帧是上游连接寿命限制错误。
|
||||
ConnectionLimitReached,
|
||||
}
|
||||
|
||||
async fn await_websocket_delivery_boundary(
|
||||
@@ -818,21 +851,25 @@ async fn await_websocket_delivery_boundary(
|
||||
match exchange.body.next().await {
|
||||
Some(Ok(frame)) if is_websocket_lifecycle_prelude(&frame) => prelude.push(frame),
|
||||
Some(Ok(frame)) => {
|
||||
let limit_message = websocket_connection_limit_message(&frame);
|
||||
let connection_limit_reached = websocket_connection_limit_message(&frame).is_some();
|
||||
prelude.push(frame);
|
||||
let remaining =
|
||||
std::mem::replace(&mut exchange.body, Box::pin(futures::stream::empty()));
|
||||
exchange.body =
|
||||
Box::pin(futures::stream::iter(prelude.into_iter().map(Ok)).chain(remaining));
|
||||
return Ok(match limit_message {
|
||||
Some(message) => DeliveryBoundary::ConnectionLimitReached { message },
|
||||
None => DeliveryBoundary::Ready,
|
||||
return Ok(if connection_limit_reached {
|
||||
DeliveryBoundary::ConnectionLimitReached
|
||||
} else {
|
||||
DeliveryBoundary::Ready
|
||||
});
|
||||
}
|
||||
Some(Err(error)) => return Err(error),
|
||||
None => {
|
||||
return Err(CodexWebSocketExchangeError::closed_before_terminal(
|
||||
None, None,
|
||||
return Err(CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
exchange.websocket_connection_id,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
//! 在 transport 边界采集的上游响应诊断信息。
|
||||
|
||||
use super::client::CodexClientVisibleUpstreamResponse;
|
||||
use super::protocol::responses::ResponsesSseFailure;
|
||||
use super::protocol::responses::{
|
||||
PREVIOUS_RESPONSE_NOT_FOUND_CODE, ResponsesSseFailure,
|
||||
is_bare_invalid_previous_response_id_error,
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{StatusCode, header::HeaderMap};
|
||||
@@ -140,7 +143,7 @@ impl CodexUpstreamFailure {
|
||||
rate_limit_headers: &[(String, String)],
|
||||
send_phase: CodexUpstreamSendPhase,
|
||||
) -> Self {
|
||||
let fields = ParsedUpstreamError::from_body(body);
|
||||
let fields = ParsedUpstreamError::from_http_response(status, body);
|
||||
let category = classify_upstream_failure(
|
||||
UpstreamFailureSource::HttpResponse,
|
||||
Some(status),
|
||||
@@ -260,7 +263,7 @@ struct ParsedUpstreamError {
|
||||
}
|
||||
|
||||
impl ParsedUpstreamError {
|
||||
fn from_body(body: &str) -> Self {
|
||||
fn from_http_response(status: StatusCode, body: &str) -> Self {
|
||||
let Ok(value) = serde_json::from_str::<Value>(body) else {
|
||||
return Self {
|
||||
code: None,
|
||||
@@ -275,11 +278,7 @@ impl ParsedUpstreamError {
|
||||
.or_else(|| value.get("error"))
|
||||
.or_else(|| value.get("detail"))
|
||||
.unwrap_or(&value);
|
||||
let code = error
|
||||
.get("code")
|
||||
.or_else(|| value.get("code"))
|
||||
.and_then(Value::as_str)
|
||||
.and_then(non_empty_owned);
|
||||
let code_value = error.get("code").or_else(|| value.get("code"));
|
||||
let error_type = error
|
||||
.get("type")
|
||||
.or_else(|| value.get("type"))
|
||||
@@ -290,6 +289,16 @@ impl ParsedUpstreamError {
|
||||
.or_else(|| value.get("message"))
|
||||
.and_then(Value::as_str)
|
||||
.and_then(non_empty_owned);
|
||||
let code = if status == StatusCode::BAD_REQUEST
|
||||
&& is_bare_invalid_previous_response_id_error(
|
||||
code_value,
|
||||
error_type.as_deref(),
|
||||
client_message.as_deref(),
|
||||
) {
|
||||
Some(PREVIOUS_RESPONSE_NOT_FOUND_CODE.to_owned())
|
||||
} else {
|
||||
code_value.and_then(Value::as_str).and_then(non_empty_owned)
|
||||
};
|
||||
let message = client_message
|
||||
.clone()
|
||||
.or_else(|| error.as_str().and_then(non_empty_owned))
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod diagnostics;
|
||||
pub mod endpoints;
|
||||
pub mod headers;
|
||||
pub mod profile;
|
||||
pub mod profile_avatar;
|
||||
pub mod profile_statistics;
|
||||
pub mod protocol;
|
||||
pub mod request;
|
||||
@@ -41,6 +42,10 @@ pub use self::{
|
||||
WHAM_USAGE_PATH, endpoint_url, usage_endpoint_url,
|
||||
},
|
||||
headers::build_codex_base_headers,
|
||||
profile_avatar::{
|
||||
CodexProfileAvatar, CodexProfileAvatarFetchError, CodexProfileAvatarStreamError,
|
||||
fetch_profile_avatar,
|
||||
},
|
||||
profile_statistics::{
|
||||
CodexProfileActivityInsights, CodexProfileDailyUsage, CodexProfileInvocation,
|
||||
CodexProfileStatistics, CodexProfileStatisticsSummary,
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Codex 官方个人资料头像的固定来源流式 transport。
|
||||
|
||||
use std::{pin::Pin, time::Duration};
|
||||
|
||||
use async_stream::try_stream;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use reqwest::{
|
||||
Client, Url,
|
||||
header::{ACCEPT, CONTENT_TYPE, ETAG, USER_AGENT},
|
||||
};
|
||||
use tokio::time::timeout;
|
||||
|
||||
use super::endpoints::endpoint_url;
|
||||
|
||||
const OFFICIAL_AVATAR_ORIGIN: &str = "https://chatgpt.com";
|
||||
const OFFICIAL_AVATAR_PATH_PREFIX: &str = "/backend-api/estuary/public_content/enc/";
|
||||
const PROVIDER_AVATAR_PATH_PREFIX: &str = "/estuary/public_content/enc/";
|
||||
const PROFILE_AVATAR_ACCEPT: &str =
|
||||
"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
|
||||
const PROFILE_AVATAR_HEADERS_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const PROFILE_AVATAR_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// 头像正文流;不累积完整正文,也不设置总字节数上限。
|
||||
pub type CodexProfileAvatarStream =
|
||||
Pin<Box<dyn Stream<Item = Result<Bytes, CodexProfileAvatarStreamError>> + Send + 'static>>;
|
||||
|
||||
/// 响应开始后的头像流失败;不得携带带签名的上游 URL。
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("Codex profile avatar stream failed")]
|
||||
pub struct CodexProfileAvatarStreamError;
|
||||
|
||||
/// 已打开的官方头像响应;MIME 只透传,不做格式白名单判断。
|
||||
pub struct CodexProfileAvatar {
|
||||
pub content_type: Option<String>,
|
||||
pub content_length: Option<u64>,
|
||||
pub etag: Option<String>,
|
||||
pub body: CodexProfileAvatarStream,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CodexProfileAvatar {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CodexProfileAvatar")
|
||||
.field("content_type", &self.content_type)
|
||||
.field("content_length", &self.content_length)
|
||||
.field("etag", &self.etag)
|
||||
.field("body", &"<stream>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CodexProfileAvatarFetchError {
|
||||
#[error("Codex profile avatar source is invalid")]
|
||||
InvalidSource,
|
||||
#[error("Codex profile avatar upstream returned HTTP {status}")]
|
||||
Upstream { status: u16 },
|
||||
#[error("Codex profile avatar transport is unavailable")]
|
||||
TransportUnavailable,
|
||||
}
|
||||
|
||||
/// 校验官方来源后,通过当前 Provider base URL 打开头像字节流。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 来源不是固定的 ChatGPT Estuary 路径、请求失败或上游返回非成功状态时返回错误。
|
||||
pub async fn fetch_profile_avatar(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
desktop_user_agent: &str,
|
||||
source: &str,
|
||||
) -> Result<CodexProfileAvatar, CodexProfileAvatarFetchError> {
|
||||
let source_path = official_avatar_source_path(source)?;
|
||||
let target = Url::parse(&endpoint_url(base_url, &source_path))
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
|
||||
let response = timeout(
|
||||
PROFILE_AVATAR_HEADERS_TIMEOUT,
|
||||
client
|
||||
.get(target)
|
||||
.header(ACCEPT, PROFILE_AVATAR_ACCEPT)
|
||||
.header(USER_AGENT, desktop_user_agent)
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?
|
||||
.map_err(|_| CodexProfileAvatarFetchError::TransportUnavailable)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(CodexProfileAvatarFetchError::Upstream {
|
||||
status: status.as_u16(),
|
||||
});
|
||||
}
|
||||
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let etag = response
|
||||
.headers()
|
||||
.get(ETAG)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let content_length = response.content_length();
|
||||
let mut upstream = Box::pin(response.bytes_stream());
|
||||
let body = try_stream! {
|
||||
loop {
|
||||
match timeout(PROFILE_AVATAR_STREAM_IDLE_TIMEOUT, upstream.next()).await {
|
||||
Ok(Some(Ok(chunk))) => yield chunk,
|
||||
Ok(Some(Err(_))) | Err(_) => Err(CodexProfileAvatarStreamError)?,
|
||||
Ok(None) => break,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(CodexProfileAvatar {
|
||||
content_type,
|
||||
content_length,
|
||||
etag,
|
||||
body: Box::pin(body),
|
||||
})
|
||||
}
|
||||
|
||||
fn official_avatar_source_path(source: &str) -> Result<String, CodexProfileAvatarFetchError> {
|
||||
let source = Url::parse(source).map_err(|_| CodexProfileAvatarFetchError::InvalidSource)?;
|
||||
if source.origin().ascii_serialization() != OFFICIAL_AVATAR_ORIGIN
|
||||
|| !source.username().is_empty()
|
||||
|| source.password().is_some()
|
||||
|| source.query().is_some()
|
||||
|| source.fragment().is_some()
|
||||
{
|
||||
return Err(CodexProfileAvatarFetchError::InvalidSource);
|
||||
}
|
||||
let opaque_path = source
|
||||
.path()
|
||||
.strip_prefix(OFFICIAL_AVATAR_PATH_PREFIX)
|
||||
.filter(|path| !path.is_empty())
|
||||
.ok_or(CodexProfileAvatarFetchError::InvalidSource)?;
|
||||
Ok(format!("{PROVIDER_AVATAR_PATH_PREFIX}{opaque_path}"))
|
||||
}
|
||||
@@ -8,6 +8,24 @@ use reqwest::header::HeaderMap;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// 官方 Codex 客户端据此触发完整历史重放的稳定错误码。
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_CODE: &str = "previous_response_not_found";
|
||||
/// 本地生成 history unavailable 错误时使用的官方提示文本。
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE: &str =
|
||||
"Previous response was not found. Retrying the full request.";
|
||||
const INVALID_PREVIOUS_RESPONSE_ID_MESSAGE: &str = "Invalid `previous_response_id`.";
|
||||
|
||||
/// 精确匹配上游省略错误码的旧版 previous response 拒绝形状。
|
||||
pub(crate) fn is_bare_invalid_previous_response_id_error(
|
||||
code: Option<&Value>,
|
||||
error_type: Option<&str>,
|
||||
message: Option<&str>,
|
||||
) -> bool {
|
||||
code.is_none_or(Value::is_null)
|
||||
&& error_type == Some("invalid_request_error")
|
||||
&& message == Some(INVALID_PREVIOUS_RESPONSE_ID_MESSAGE)
|
||||
}
|
||||
|
||||
/// Codex Responses 上游请求体。
|
||||
///
|
||||
/// 发往上游的 Responses 请求。`body` 持有客户端原始 JSON object,逐字段(含顺序、
|
||||
@@ -54,7 +72,7 @@ pub struct CodexResponsesRequest {
|
||||
pub turn_metadata: Option<String>,
|
||||
/// beta features 透传头。
|
||||
pub beta_features: Option<String>,
|
||||
/// 客户端版本头。
|
||||
/// 下游 `version` 扩展头;存在时上游值由 Desktop 版本画像统一生成。
|
||||
pub version: Option<String>,
|
||||
/// timing metrics 透传头。
|
||||
pub include_timing_metrics: Option<String>,
|
||||
@@ -386,6 +404,8 @@ pub struct ResponsesSseFailure {
|
||||
pub explicit_status_code: Option<u16>,
|
||||
/// 上游显式重试间隔,或从官方限流消息中解析出的重试间隔。
|
||||
pub retry_after_seconds: Option<u64>,
|
||||
/// 上游错误事件的原始 JSON data;只应在明确的失败审计边界读取。
|
||||
raw_body: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ResponsesSseFailure {
|
||||
@@ -404,6 +424,10 @@ impl fmt::Debug for ResponsesSseFailure {
|
||||
|
||||
impl ResponsesSseFailure {
|
||||
pub fn from_event(event: &str, value: &Value) -> Self {
|
||||
Self::from_raw_event(event, &value.to_string(), value)
|
||||
}
|
||||
|
||||
pub(crate) fn from_raw_event(event: &str, raw_body: &str, value: &Value) -> Self {
|
||||
Self {
|
||||
event: event.to_string(),
|
||||
message: failure_message(value).unwrap_or_else(|| "Codex upstream SSE failed".into()),
|
||||
@@ -411,8 +435,15 @@ impl ResponsesSseFailure {
|
||||
upstream_type: failure_type(value),
|
||||
explicit_status_code: failure_explicit_status_code(value),
|
||||
retry_after_seconds: events::retry_after_seconds_from_value(value),
|
||||
raw_body: raw_body.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回上游错误事件未经重编码的 JSON data。
|
||||
#[must_use]
|
||||
pub fn raw_body(&self) -> &str {
|
||||
&self.raw_body
|
||||
}
|
||||
}
|
||||
|
||||
fn failure_message(value: &Value) -> Option<String> {
|
||||
|
||||
@@ -2,13 +2,12 @@ use gateway_protocol::openai::sse::encode_sse_event;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::transport::protocol::responses::{CodexResponsesRequest, transport_requirement};
|
||||
use crate::transport::protocol::responses::{
|
||||
CodexResponsesRequest, PREVIOUS_RESPONSE_NOT_FOUND_CODE,
|
||||
is_bare_invalid_previous_response_id_error, transport_requirement,
|
||||
};
|
||||
|
||||
const REDACTED_PAYLOAD_VALUE: &str = "<redacted>";
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_CODE: &str = "previous_response_not_found";
|
||||
pub(crate) const PREVIOUS_RESPONSE_NOT_FOUND_MESSAGE: &str =
|
||||
"Previous response was not found. Retrying the full request.";
|
||||
const INVALID_PREVIOUS_RESPONSE_ID_MESSAGE: &str = "Invalid `previous_response_id`.";
|
||||
|
||||
/// WebSocket 握手审计快照。
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -87,11 +86,11 @@ fn previous_response_not_found_event(value: &Value) -> Option<String> {
|
||||
return None;
|
||||
}
|
||||
let error = value.get("error")?.as_object()?;
|
||||
if error.get("code").is_some_and(|code| !code.is_null())
|
||||
|| error.get("type").and_then(Value::as_str) != Some("invalid_request_error")
|
||||
|| error.get("message").and_then(Value::as_str)
|
||||
!= Some(INVALID_PREVIOUS_RESPONSE_ID_MESSAGE)
|
||||
{
|
||||
if !is_bare_invalid_previous_response_id_error(
|
||||
error.get("code"),
|
||||
error.get("type").and_then(Value::as_str),
|
||||
error.get("message").and_then(Value::as_str),
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{fmt, time::Duration};
|
||||
|
||||
use gateway_protocol::openai::sse::SseError;
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::transport::client::CodexClientVisibleUpstreamResponse;
|
||||
use crate::transport::diagnostics::CodexUpstreamDiagnostics;
|
||||
@@ -65,6 +66,9 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// 上游 WebSocket 错误帧。
|
||||
#[error("{0}")]
|
||||
Upstream(Box<CodexWebSocketUpstreamError>),
|
||||
/// 上游要求结束当前 WebSocket 连接并在新连接上重试。
|
||||
#[error("websocket connection limit reached")]
|
||||
ConnectionLimitReached,
|
||||
/// 请求依赖的连接本地 previous response 无法在当前连接满足。
|
||||
#[error("websocket continuation unavailable: {reason}")]
|
||||
ContinuationUnavailable {
|
||||
@@ -104,13 +108,36 @@ pub enum CodexWebSocketExchangeError {
|
||||
/// close reason 只供原请求的客户端错误响应使用;`Debug` 与 `Display` 均不会输出它。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct CodexWebSocketCloseError {
|
||||
connection_id: Option<Uuid>,
|
||||
code: Option<u16>,
|
||||
reason: Option<String>,
|
||||
last_event_type: Option<String>,
|
||||
}
|
||||
|
||||
impl CodexWebSocketCloseError {
|
||||
pub(crate) fn new(code: Option<u16>, reason: Option<String>) -> Self {
|
||||
Self { code, reason }
|
||||
Self {
|
||||
connection_id: None,
|
||||
code,
|
||||
reason,
|
||||
last_event_type: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_connection_id(mut self, connection_id: Uuid) -> Self {
|
||||
self.connection_id = Some(connection_id);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_last_event_type(mut self, event_type: Option<String>) -> Self {
|
||||
self.last_event_type = event_type;
|
||||
self
|
||||
}
|
||||
|
||||
/// 返回承载该 close frame 的 WebSocket 连接标识。
|
||||
#[must_use]
|
||||
pub const fn connection_id(&self) -> Option<Uuid> {
|
||||
self.connection_id
|
||||
}
|
||||
|
||||
/// 返回上游 close code;没有 close frame 时为 `None`。
|
||||
@@ -124,14 +151,22 @@ impl CodexWebSocketCloseError {
|
||||
pub fn reason(&self) -> Option<&str> {
|
||||
self.reason.as_deref()
|
||||
}
|
||||
|
||||
/// 返回关闭前最后一条经过白名单校验的事件类型;不包含事件 payload。
|
||||
#[must_use]
|
||||
pub fn last_event_type(&self) -> Option<&str> {
|
||||
self.last_event_type.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for CodexWebSocketCloseError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("CodexWebSocketCloseError")
|
||||
.field("connection_id", &self.connection_id)
|
||||
.field("code", &self.code)
|
||||
.field("reason", &self.reason.as_ref().map(|_| "<redacted>"))
|
||||
.field("last_event_type", &self.last_event_type)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -192,8 +227,17 @@ impl fmt::Display for CodexWebSocketUpstreamError {
|
||||
}
|
||||
|
||||
impl CodexWebSocketExchangeError {
|
||||
pub(crate) fn closed_before_terminal(code: Option<u16>, reason: Option<String>) -> Self {
|
||||
Self::ClosedBeforeTerminal(CodexWebSocketCloseError::new(code, reason))
|
||||
pub(crate) fn closed_before_terminal_on(
|
||||
connection_id: Uuid,
|
||||
code: Option<u16>,
|
||||
reason: Option<String>,
|
||||
last_event_type: Option<String>,
|
||||
) -> Self {
|
||||
Self::ClosedBeforeTerminal(
|
||||
CodexWebSocketCloseError::new(code, reason)
|
||||
.with_connection_id(connection_id)
|
||||
.with_last_event_type(last_event_type),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn close_before_terminal(&self) -> Option<&CodexWebSocketCloseError> {
|
||||
@@ -245,6 +289,11 @@ impl CodexWebSocketExchangeError {
|
||||
)
|
||||
}
|
||||
|
||||
/// 精确连接状态不可用属于本地池路由事实,不消耗上游 WS 重试预算。
|
||||
pub(in crate::transport) fn requires_immediate_pool_fallback(&self) -> bool {
|
||||
matches!(self, Self::ContinuationUnavailable { .. })
|
||||
}
|
||||
|
||||
/// 首个业务事件交付前,普通 WebSocket 失败可切到同账号 HTTP。
|
||||
pub(in crate::transport) fn allows_pre_delivery_http_fallback(&self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -29,20 +29,32 @@ pub(super) enum ExchangeAction {
|
||||
Ignore,
|
||||
}
|
||||
|
||||
pub(super) struct ReducedWebSocketEvent {
|
||||
pub(super) action: ExchangeAction,
|
||||
pub(super) diagnostic_event_type: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn reduce_websocket_event(
|
||||
raw: &str,
|
||||
metadata: &mut CodexWebSocketConnectionMetadata,
|
||||
continuation: &mut WebSocketContinuationState,
|
||||
) -> Result<ExchangeAction, CodexWebSocketExchangeError> {
|
||||
) -> Result<ReducedWebSocketEvent, CodexWebSocketExchangeError> {
|
||||
// 每帧只解析一次 JSON,后续提取全部复用同一 Value;
|
||||
// 不可解析的帧不承载可路由的事件类型,忽略。
|
||||
let Ok(value) = serde_json::from_str::<Value>(raw) else {
|
||||
return Ok(ExchangeAction::Ignore);
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::Ignore,
|
||||
diagnostic_event_type: None,
|
||||
});
|
||||
};
|
||||
let diagnostic_event_type = diagnostic_event_type(websocket_event_type(&value));
|
||||
if let Some(parsed) = events::parse_rate_limits_event(&value) {
|
||||
let headers = events::rate_limits_to_header_pairs(&parsed);
|
||||
metadata.rate_limit_headers.extend(headers);
|
||||
return Ok(ExchangeAction::RateLimits(parsed));
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::RateLimits(parsed),
|
||||
diagnostic_event_type,
|
||||
});
|
||||
}
|
||||
|
||||
response_meta::merge_response_metadata(
|
||||
@@ -51,7 +63,10 @@ pub(super) fn reduce_websocket_event(
|
||||
);
|
||||
if let Some(turn_state) = websocket_metadata_turn_state(&value) {
|
||||
metadata.turn_state = Some(turn_state.clone());
|
||||
return Ok(ExchangeAction::TurnState(turn_state));
|
||||
return Ok(ReducedWebSocketEvent {
|
||||
action: ExchangeAction::TurnState(turn_state),
|
||||
diagnostic_event_type,
|
||||
});
|
||||
}
|
||||
|
||||
let event = websocket_event_type(&value);
|
||||
@@ -67,8 +82,24 @@ pub(super) fn reduce_websocket_event(
|
||||
Some("response.failed" | "error") => Some(WebSocketTerminalKind::Failed),
|
||||
_ => None,
|
||||
};
|
||||
Ok(match websocket_event_frame(&value, raw) {
|
||||
let action = match websocket_event_frame(&value, raw) {
|
||||
Some(frame) => ExchangeAction::Forward { frame, terminal },
|
||||
None => ExchangeAction::Ignore,
|
||||
};
|
||||
Ok(ReducedWebSocketEvent {
|
||||
action,
|
||||
diagnostic_event_type,
|
||||
})
|
||||
}
|
||||
|
||||
fn diagnostic_event_type(event_type: Option<&str>) -> Option<String> {
|
||||
const MAX_EVENT_TYPE_BYTES: usize = 128;
|
||||
|
||||
let event_type = event_type?;
|
||||
(!event_type.is_empty()
|
||||
&& event_type.len() <= MAX_EVENT_TYPE_BYTES
|
||||
&& event_type.bytes().all(|byte| {
|
||||
byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-')
|
||||
}))
|
||||
.then(|| event_type.to_owned())
|
||||
}
|
||||
|
||||
@@ -143,6 +143,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
.map(|pool_return| std::mem::take(&mut pool_return.continuation))
|
||||
.unwrap_or_default();
|
||||
let mut saw_upstream_activity = false;
|
||||
let mut last_event_type = None;
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
@@ -208,15 +209,18 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
tungstenite::Message::Close(frame) => {
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = CodexWebSocketExchangeError::closed_before_terminal(
|
||||
let error = CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
frame.as_ref().map(|frame| u16::from(frame.code)),
|
||||
frame.map(|frame| frame.reason.to_string()),
|
||||
last_event_type.clone(),
|
||||
);
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
@@ -228,18 +232,8 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let (frame, terminal) = match reduce_websocket_event(&raw, &mut metadata, &mut continuation)
|
||||
{
|
||||
Ok(ExchangeAction::RateLimits(rate_limits)) => {
|
||||
rate_limit_updates.lock().await.push(rate_limits);
|
||||
continue;
|
||||
}
|
||||
Ok(ExchangeAction::TurnState(turn_state)) => {
|
||||
*turn_state_update.lock().await = Some(turn_state);
|
||||
continue;
|
||||
}
|
||||
Ok(ExchangeAction::Forward { frame, terminal }) => (frame, terminal),
|
||||
Ok(ExchangeAction::Ignore) => continue,
|
||||
let reduced = match reduce_websocket_event(&raw, &mut metadata, &mut continuation) {
|
||||
Ok(reduced) => reduced,
|
||||
Err(error) => {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -251,6 +245,21 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Some(event_type) = reduced.diagnostic_event_type {
|
||||
last_event_type = Some(event_type);
|
||||
}
|
||||
let (frame, terminal) = match reduced.action {
|
||||
ExchangeAction::RateLimits(rate_limits) => {
|
||||
rate_limit_updates.lock().await.push(rate_limits);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::TurnState(turn_state) => {
|
||||
*turn_state_update.lock().await = Some(turn_state);
|
||||
continue;
|
||||
}
|
||||
ExchangeAction::Forward { frame, terminal } => (frame, terminal),
|
||||
ExchangeAction::Ignore => continue,
|
||||
};
|
||||
if tx.send(Ok(Bytes::from(frame))).await.is_err() {
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
@@ -287,19 +296,28 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
}
|
||||
}
|
||||
|
||||
let websocket_connection_id = websocket.connection_id();
|
||||
let upstream_close = websocket
|
||||
.exit_reason()
|
||||
.and_then(|reason| reason.upstream_close().cloned());
|
||||
.and_then(|reason| reason.upstream_close().cloned())
|
||||
.map(|close| close.with_connection_id(websocket_connection_id));
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
StreamWebSocketDiscardReason::UpstreamClosed,
|
||||
)
|
||||
.await;
|
||||
let error = upstream_close.map_or_else(
|
||||
|| CodexWebSocketExchangeError::closed_before_terminal(None, None),
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal,
|
||||
);
|
||||
let error = match upstream_close {
|
||||
Some(close) => CodexWebSocketExchangeError::ClosedBeforeTerminal(
|
||||
close.with_last_event_type(last_event_type),
|
||||
),
|
||||
None => CodexWebSocketExchangeError::closed_before_terminal_on(
|
||||
websocket_connection_id,
|
||||
None,
|
||||
None,
|
||||
last_event_type,
|
||||
),
|
||||
};
|
||||
let error = if reused_connection {
|
||||
reused_stream_receive_error(error)
|
||||
} else {
|
||||
|
||||
@@ -556,12 +556,19 @@ fn log_pump_exit(
|
||||
"Responses WebSocket pump stopped unexpectedly"
|
||||
);
|
||||
} else if matches!(reason, PumpExitReason::UpstreamCloseFrame { .. }) {
|
||||
let upstream_close = reason.upstream_close();
|
||||
let upstream_error_raw = upstream_close
|
||||
.and_then(CodexWebSocketCloseError::reason)
|
||||
.unwrap_or_default();
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
account_id = %account_id,
|
||||
conversation_id_hash = %conversation_id_hash,
|
||||
pump_exit_reason = reason.as_str(),
|
||||
pump_exit_detail = detail,
|
||||
upstream_close_code = ?upstream_close.and_then(CodexWebSocketCloseError::code),
|
||||
upstream_error_raw,
|
||||
upstream_error_raw_present = upstream_close.is_some_and(|close| close.reason().is_some()),
|
||||
backpressure_events,
|
||||
"Responses WebSocket pump received close frame"
|
||||
);
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use std::path::Path;
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use provider_openai::config::{CodexWireProfileConfig, OpenAiConfig};
|
||||
use provider_openai::config::{
|
||||
CodexWireProfileConfig, DEFAULT_STREAM_MAX_RETRIES, MAX_STREAM_MAX_RETRIES, OpenAiConfig,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn openai_config_builds_the_audited_wire_profile() {
|
||||
@@ -95,6 +97,7 @@ fn openai_config_restricts_upstream_base_url_to_https_or_loopback_http() {
|
||||
#[test]
|
||||
fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
let config = OpenAiConfig::default();
|
||||
assert_eq!(DEFAULT_STREAM_MAX_RETRIES, 3);
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -107,6 +110,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
config.auth.refresh_enabled,
|
||||
config.auth.oauth_client_id.as_str(),
|
||||
config.auth.oauth_token_endpoint.as_str(),
|
||||
config.stream_max_retries(),
|
||||
),
|
||||
(
|
||||
"https://chatgpt.com/backend-api",
|
||||
@@ -118,6 +122,7 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
true,
|
||||
"app_EMoamEEZ73f0CkXaXp7hrann",
|
||||
"https://auth.openai.com/oauth/token",
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -126,6 +131,17 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_stream_retry_budget_uses_the_official_hard_cap() {
|
||||
let mut config = OpenAiConfig::default();
|
||||
config.stream_max_retries = MAX_STREAM_MAX_RETRIES + 1;
|
||||
|
||||
assert_eq!(
|
||||
config.stream_max_retries(),
|
||||
u32::try_from(MAX_STREAM_MAX_RETRIES).expect("retry cap fits u32")
|
||||
);
|
||||
}
|
||||
|
||||
fn valid_config() -> OpenAiConfig {
|
||||
let mut config = OpenAiConfig::default();
|
||||
config.wire_profile = CodexWireProfileConfig {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use futures::TryStreamExt as _;
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use provider_openai::credential::{CodexCredentialProfileService, ImportCodexOAuthCredential};
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
@@ -155,3 +156,62 @@ async fn profile_statistics_keeps_profile_when_stats_are_unavailable() {
|
||||
assert_eq!(statistics.summary.total_text_tokens, None);
|
||||
assert_eq!(statistics.daily_usage, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_avatar_reuses_cached_source_and_sends_no_account_credentials() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/wham/profiles/me"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"profile": {
|
||||
"display_name": "Ada",
|
||||
"profile_picture_url": "https://chatgpt.com/backend-api/estuary/public_content/enc/avatar-token="
|
||||
},
|
||||
"stats": {},
|
||||
"metadata": {"stats_error": null}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/estuary/public_content/enc/avatar-token="))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "image/svg+xml")
|
||||
.set_body_bytes(b"<svg xmlns=\"http://www.w3.org/2000/svg\"/>"),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let (store, service) = service("acct_profile_avatar", &server).await;
|
||||
let account = store
|
||||
.account("acct_profile_avatar")
|
||||
.expect("profile account");
|
||||
|
||||
service
|
||||
.profile_statistics(account.id())
|
||||
.await
|
||||
.expect("profile statistics");
|
||||
let avatar = service
|
||||
.profile_avatar(account.id())
|
||||
.await
|
||||
.expect("profile avatar");
|
||||
let content_type = avatar.content_type.clone();
|
||||
let chunks = avatar
|
||||
.body
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("avatar body");
|
||||
let body = chunks.into_iter().flatten().collect::<Vec<_>>();
|
||||
let requests = server.received_requests().await.expect("recorded requests");
|
||||
let avatar_request = requests
|
||||
.iter()
|
||||
.find(|request| request.url.path() == "/estuary/public_content/enc/avatar-token=")
|
||||
.expect("avatar request");
|
||||
|
||||
assert_eq!(content_type.as_deref(), Some("image/svg+xml"));
|
||||
assert_eq!(body, b"<svg xmlns=\"http://www.w3.org/2000/svg\"/>");
|
||||
assert!(!avatar_request.headers.contains_key("authorization"));
|
||||
assert!(!avatar_request.headers.contains_key("cookie"));
|
||||
assert!(!avatar_request.headers.contains_key("chatgpt-account-id"));
|
||||
}
|
||||
|
||||
@@ -18,10 +18,11 @@ use gateway_core::engine::credential::{
|
||||
};
|
||||
use gateway_core::engine::provider::{Provider as _, ProviderRequest};
|
||||
use gateway_core::engine::{
|
||||
AccountAttemptContext, AttemptContext, CancellationToken, ContinuationAttempt, ModelRequestId,
|
||||
ProviderAccountStateOwner, RequestAttemptContext, UpstreamSendState,
|
||||
AccountAttemptContext, AttemptContext, AttemptTransport, CancellationToken,
|
||||
ContinuationAttempt, ModelRequestId, ProviderAccountStateOwner, RequestAttemptContext,
|
||||
UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{ContinuationFailure, ProviderErrorKind};
|
||||
use gateway_core::error::{ContinuationFailure, PreDeliveryRetry, ProviderErrorKind};
|
||||
use gateway_core::event::GatewayEvent;
|
||||
use gateway_core::operation::{
|
||||
CapabilityRequirements, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
|
||||
@@ -33,6 +34,7 @@ use gateway_core::routing::{
|
||||
ProviderModel, PublicModelId, RoutingContext, RuntimeAccount, RuntimeAccountDirectory,
|
||||
RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
use provider_openai::config::DEFAULT_STREAM_MAX_RETRIES;
|
||||
use provider_openai::credential::{
|
||||
CodexCookiePolicy, CodexCredentialCatalogService, CodexCredentialQuotaService,
|
||||
CodexCredentialSelector, ImportCodexOAuthCredential,
|
||||
@@ -147,7 +149,26 @@ fn provider_with_affinity(
|
||||
}
|
||||
|
||||
fn provider_with_base_url(store: &Arc<MemoryAccountStore>, base_url: String) -> CodexProvider {
|
||||
provider_with_affinity_and_base_url(store, Arc::new(MemorySessionAffinity::default()), base_url)
|
||||
provider_with_base_url_and_retry_budget(
|
||||
store,
|
||||
base_url,
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
}
|
||||
|
||||
fn provider_with_base_url_and_retry_budget(
|
||||
store: &Arc<MemoryAccountStore>,
|
||||
base_url: String,
|
||||
stream_max_retries: u32,
|
||||
) -> CodexProvider {
|
||||
provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
store,
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
base_url,
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
stream_max_retries,
|
||||
)
|
||||
.0
|
||||
}
|
||||
|
||||
fn provider_with_leases(
|
||||
@@ -186,6 +207,7 @@ fn provider_with_affinity_and_base_url_and_leases(
|
||||
session_affinity,
|
||||
base_url,
|
||||
leases,
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
.0
|
||||
}
|
||||
@@ -195,6 +217,7 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
session_affinity: Arc<MemorySessionAffinity>,
|
||||
base_url: String,
|
||||
leases: Arc<TestLeaseCoordinator>,
|
||||
stream_max_retries: u32,
|
||||
) -> (CodexProvider, Arc<CodexCredentialQuotaService>) {
|
||||
let profile = wire_profile();
|
||||
let http = reqwest::Client::builder()
|
||||
@@ -238,6 +261,7 @@ fn provider_and_quota_with_affinity_and_base_url_and_leases(
|
||||
profile,
|
||||
base_url,
|
||||
websocket_pool,
|
||||
stream_max_retries,
|
||||
)
|
||||
.expect("official OpenAI provider");
|
||||
(provider, quota)
|
||||
@@ -412,6 +436,7 @@ fn contract_account_scope() -> Arc<FrozenAccountScope> {
|
||||
"acct_truncated_stream",
|
||||
"acct_usage_limit_request_path",
|
||||
"acct_websocket_close",
|
||||
"acct_websocket_busy_replay",
|
||||
"acct_websocket_turn_state",
|
||||
]
|
||||
.into_iter()
|
||||
@@ -444,6 +469,10 @@ fn context(request_id: &str, cancellation: CancellationToken) -> AttemptContext
|
||||
)
|
||||
}
|
||||
|
||||
fn fallback_transport_context(request_id: &str) -> AttemptContext {
|
||||
context(request_id, CancellationToken::new()).with_transport(AttemptTransport::Fallback)
|
||||
}
|
||||
|
||||
fn diagnostic_context(request_id: &str, account_id: &str) -> AttemptContext {
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
@@ -510,6 +539,40 @@ fn replay_any_context(request_id: &str, owner_account_id: &str) -> AttemptContex
|
||||
.with_continuation_attempt(ContinuationAttempt::ReplayAny)
|
||||
}
|
||||
|
||||
fn pinned_continuation_context(
|
||||
request_id: &str,
|
||||
account_id: &str,
|
||||
client_previous_response_id: &str,
|
||||
upstream_previous_response_id: &str,
|
||||
attempt_index: u32,
|
||||
continuation_attempt: ContinuationAttempt,
|
||||
) -> AttemptContext {
|
||||
let account = ProviderAccountId::new(account_id).expect("account id");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let owner = ProviderAccountStateOwner::new(provider.clone(), account.clone());
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new(client_previous_response_id),
|
||||
PreviousResponseId::new(upstream_previous_response_id),
|
||||
ClientApiKeyId::new("key_openai_contract").expect("client key id"),
|
||||
provider,
|
||||
account,
|
||||
);
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new(request_id).expect("request id"),
|
||||
ClientApiKeyId::new("key_openai_contract").expect("client key id"),
|
||||
),
|
||||
NonZeroU32::new(attempt_index).expect("attempt index"),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(BTreeSet::new(), None, Some(owner))
|
||||
.with_account_scope(contract_account_scope()),
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.with_continuation_attempt(continuation_attempt)
|
||||
}
|
||||
|
||||
fn external_continuation_context(request_id: &str) -> AttemptContext {
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
@@ -729,21 +792,25 @@ async fn truncated_chunked_sse_server() -> (String, tokio::task::JoinHandle<()>)
|
||||
(base_url, server)
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) {
|
||||
async fn capture_http_request(stream: &mut TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 1024];
|
||||
loop {
|
||||
let read = stream.read(&mut buffer).await.expect("read HTTP request");
|
||||
if read == 0 {
|
||||
return;
|
||||
return request;
|
||||
}
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
if request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
return;
|
||||
return request;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) {
|
||||
drop(capture_http_request(stream).await);
|
||||
}
|
||||
|
||||
async fn write_http_chunk(stream: &mut TcpStream, body: &str) {
|
||||
stream
|
||||
.write_all(format!("{:X}\r\n", body.len()).as_bytes())
|
||||
@@ -1091,7 +1158,7 @@ async fn selection_infrastructure_errors_have_a_distinct_classification() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
async fn websocket_close_after_delivery_preserves_details_without_enabling_sticky_http() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
@@ -1130,11 +1197,49 @@ async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
}))
|
||||
.await
|
||||
.expect("close WebSocket");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept next-turn WebSocket connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("next-turn WebSocket request")
|
||||
.expect("valid next-turn WebSocket request");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_after_committed_close",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("complete next-turn WebSocket request");
|
||||
});
|
||||
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let first_operation = Operation::Generate(generate_with_session_context(
|
||||
"committed-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", generate_operation()),
|
||||
planned_request("openai", first_operation),
|
||||
context("req_websocket_close", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
@@ -1146,19 +1251,430 @@ async fn websocket_close_details_are_only_exposed_through_the_client_error() {
|
||||
None => panic!("WebSocket close must surface a provider error"),
|
||||
}
|
||||
};
|
||||
server.await.expect("WebSocket server");
|
||||
drop(stream);
|
||||
|
||||
assert!(!format!("{error:?}").contains("message too big"));
|
||||
assert!(!error.to_string().contains("message too big"));
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(error.allows_pre_delivery_retry());
|
||||
assert!(
|
||||
!error.allows_pre_delivery_retry(),
|
||||
"a close after the delivery boundary must not request a hidden replay"
|
||||
);
|
||||
let detail = error
|
||||
.client_visible_upstream_error()
|
||||
.expect("WebSocket close detail");
|
||||
assert_eq!(detail.message(), "message too big");
|
||||
assert_eq!(detail.code(), Some("1009"));
|
||||
assert_eq!(detail.error_type(), Some("websocket_close_error"));
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some("websocket_close_1009")
|
||||
);
|
||||
|
||||
let second_operation = Operation::Generate(generate_with_session_context(
|
||||
"committed-websocket-session",
|
||||
Some("thread-second"),
|
||||
None,
|
||||
));
|
||||
let mut next_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", second_operation),
|
||||
context("req_after_committed_close", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("post-delivery close must not enable sticky HTTP fallback");
|
||||
assert_eq!(next_stream.metadata().transport().as_str(), "websocket");
|
||||
while let Some(event) = next_stream.next().await {
|
||||
event.expect("next turn WebSocket response");
|
||||
}
|
||||
server.await.expect("WebSocket server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_retry_budget_exhaustion_is_sticky_only_for_the_current_session() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket listener");
|
||||
let base_url = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
for retry_count in 0..=2 {
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept WebSocket retry connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("WebSocket request")
|
||||
.expect("valid WebSocket request");
|
||||
for event in [
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": format!("resp_structural_{retry_count}"), "model": "gpt-5.4"}
|
||||
}),
|
||||
json!({
|
||||
"type": "response.in_progress",
|
||||
"response": {"id": format!("resp_structural_{retry_count}"), "model": "gpt-5.4"}
|
||||
}),
|
||||
] {
|
||||
websocket
|
||||
.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.expect("send structural WebSocket event");
|
||||
}
|
||||
if retry_count == 1 {
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "websocket_connection_limit_reached",
|
||||
"message": "create a new websocket connection"
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send connection-limit error");
|
||||
continue;
|
||||
}
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {
|
||||
"id": format!("msg_structural_{retry_count}"),
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "in_progress",
|
||||
"content": []
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send structural output event");
|
||||
let close_code = match retry_count {
|
||||
0 => CloseCode::Normal,
|
||||
_ => CloseCode::Error,
|
||||
};
|
||||
websocket
|
||||
.close(Some(CloseFrame {
|
||||
code: close_code,
|
||||
reason: "".into(),
|
||||
}))
|
||||
.await
|
||||
.expect("close WebSocket retry attempt");
|
||||
}
|
||||
|
||||
let (mut http, _) = listener.accept().await.expect("accept sticky HTTP request");
|
||||
let request = capture_http_request(&mut http).await;
|
||||
let request_head = String::from_utf8_lossy(&request).to_ascii_lowercase();
|
||||
assert!(!request_head.contains("upgrade: websocket"));
|
||||
http.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
|
||||
CAPTURE_COMPLETED_SSE.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write sticky HTTP response");
|
||||
|
||||
let (stream, _) = listener
|
||||
.accept()
|
||||
.await
|
||||
.expect("accept other-session WebSocket connection");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _request = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("other-session WebSocket request")
|
||||
.expect("valid other-session WebSocket request");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_other_session",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("complete other-session WebSocket request");
|
||||
});
|
||||
|
||||
let provider = provider_with_base_url_and_retry_budget(&store, base_url, 2);
|
||||
let mut saw_websocket_observation = false;
|
||||
for retry_count in 0..=2 {
|
||||
let operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let attempt_context = match NonZeroU32::new(retry_count) {
|
||||
Some(retry_index) => context(
|
||||
&format!("req_websocket_normal_close_{retry_count}"),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.with_transport(AttemptTransport::Retry(retry_index)),
|
||||
None => context("req_websocket_normal_close_0", CancellationToken::new()),
|
||||
};
|
||||
let mut stream = provider
|
||||
.execute(planned_request("openai", operation), attempt_context)
|
||||
.await
|
||||
.expect("prepare WebSocket provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(event)) => {
|
||||
if let Some(observation) = event.response_observation() {
|
||||
saw_websocket_observation = true;
|
||||
assert_eq!(observation.status_code(), None);
|
||||
if let Some(provider_metadata) = observation.provider_metadata() {
|
||||
let metadata: Value = serde_json::from_str(provider_metadata.as_json())
|
||||
.expect("OpenAI provider metadata JSON");
|
||||
assert_eq!(
|
||||
metadata.get("upstreamStatus").and_then(Value::as_u64),
|
||||
Some(101),
|
||||
"successful upgrade remains provider-owned diagnostics"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!event.has_client_event(),
|
||||
"structural events must remain behind the replay boundary"
|
||||
);
|
||||
}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("WebSocket close must surface a provider error"),
|
||||
}
|
||||
};
|
||||
drop(stream);
|
||||
|
||||
if retry_count < 2 {
|
||||
let Some(PreDeliveryRetry::SameAccountTransportRetry { retry_index, delay }) =
|
||||
error.pre_delivery_retry()
|
||||
else {
|
||||
panic!("expected a same-account WebSocket retry: {error:?}");
|
||||
};
|
||||
assert_eq!(retry_index.get(), retry_count + 1);
|
||||
assert!(!delay.is_zero());
|
||||
} else {
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback),
|
||||
"unexpected provider error at retry {retry_count}: {error:?}"
|
||||
);
|
||||
}
|
||||
let expected_close_code = match retry_count {
|
||||
0 => "websocket_close_1000",
|
||||
1 => "websocket_connection_limit_reached",
|
||||
_ => "websocket_close_1011",
|
||||
};
|
||||
assert_eq!(
|
||||
error.upstream_code().map(|code| code.as_str()),
|
||||
Some(expected_close_code)
|
||||
);
|
||||
}
|
||||
|
||||
assert!(saw_websocket_observation);
|
||||
|
||||
let second_operation = Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-second"),
|
||||
None,
|
||||
));
|
||||
let mut sticky_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", second_operation),
|
||||
context("req_websocket_sticky_http", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("same session should prepare an HTTP stream");
|
||||
assert_eq!(sticky_stream.metadata().transport().as_str(), "http_sse");
|
||||
while let Some(event) = sticky_stream.next().await {
|
||||
event.expect("sticky HTTP response");
|
||||
}
|
||||
drop(sticky_stream);
|
||||
|
||||
let other_operation = Operation::Generate(generate_with_session_context(
|
||||
"other-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
));
|
||||
let mut other_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", other_operation),
|
||||
context("req_other_session_websocket", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("another session should still prepare a WebSocket stream");
|
||||
assert_eq!(other_stream.metadata().transport().as_str(), "websocket");
|
||||
while let Some(event) = other_stream.next().await {
|
||||
event.expect("other-session WebSocket response");
|
||||
}
|
||||
drop(other_stream);
|
||||
|
||||
let warmup_operation = Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("warm up")),
|
||||
("session_id".to_owned(), json!("sticky-websocket-session")),
|
||||
("generate".to_owned(), json!(false)),
|
||||
("store".to_owned(), json!(false)),
|
||||
]),
|
||||
)
|
||||
.expect("warmup payload"),
|
||||
));
|
||||
let warmup_stream = provider
|
||||
.execute(
|
||||
planned_request("openai", warmup_operation),
|
||||
context(
|
||||
"req_sticky_session_required_warmup",
|
||||
CancellationToken::new(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("required warmup must bypass sticky HTTP fallback");
|
||||
assert_eq!(warmup_stream.metadata().transport().as_str(), "websocket");
|
||||
drop(warmup_stream);
|
||||
server.await.expect("WebSocket and HTTP server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_upgrade_required_immediately_enables_session_http_fallback() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_websocket_close").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind upstream listener");
|
||||
let base_url = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut opening, _) = listener.accept().await.expect("accept WS opening");
|
||||
let request = capture_http_request(&mut opening).await;
|
||||
assert!(String::from_utf8_lossy(&request).starts_with("GET /codex/responses"));
|
||||
let body = r#"{"error":{"code":"upgrade_required","message":"use HTTP"}}"#;
|
||||
opening
|
||||
.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 426 Upgrade Required\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write 426 response");
|
||||
|
||||
let (mut http, _) = listener.accept().await.expect("accept sticky HTTP request");
|
||||
let request = capture_http_request(&mut http).await;
|
||||
assert!(String::from_utf8_lossy(&request).starts_with("POST /codex/responses"));
|
||||
http.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{CAPTURE_COMPLETED_SSE}",
|
||||
CAPTURE_COMPLETED_SSE.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write HTTP fallback response");
|
||||
});
|
||||
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let operation = || {
|
||||
Operation::Generate(generate_with_session_context(
|
||||
"sticky-websocket-session",
|
||||
Some("thread-first"),
|
||||
None,
|
||||
))
|
||||
};
|
||||
let mut first = provider
|
||||
.execute(
|
||||
planned_request("openai", operation()),
|
||||
context("req_upgrade_required", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket stream");
|
||||
let error = loop {
|
||||
match first.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("426 opening must surface a fallback signal"),
|
||||
}
|
||||
};
|
||||
assert_eq!(error.upstream_status(), Some(426));
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback)
|
||||
);
|
||||
drop(first);
|
||||
|
||||
let mut second = provider
|
||||
.execute(
|
||||
planned_request("openai", operation()),
|
||||
context("req_upgrade_required_next_turn", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("same session should select HTTP");
|
||||
assert_eq!(second.metadata().transport().as_str(), "http_sse");
|
||||
while let Some(event) = second.next().await {
|
||||
event.expect("HTTP fallback response");
|
||||
}
|
||||
server.await.expect("upstream server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fallback_attempt_transport_forces_http_sse_for_a_websocket_request() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_http_sse_exhausted").await;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(CAPTURE_COMPLETED_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut stream = provider_with_base_url(&store, server.uri())
|
||||
.execute(
|
||||
planned_request("openai", generate_operation()),
|
||||
fallback_transport_context("req_fallback_transport_http"),
|
||||
)
|
||||
.await
|
||||
.expect("prepare fallback HTTP stream");
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("fallback HTTP response");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2071,6 +2587,7 @@ async fn affinity_quota_switch_should_clear_old_turn_state_without_a_provider_st
|
||||
Arc::clone(&affinity),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
);
|
||||
let session_id = "stable-affinity-switch-session";
|
||||
let first_payload = ProtocolPayload::json_object(
|
||||
@@ -2919,6 +3436,288 @@ async fn same_account_previous_response_not_found_should_remain_client_visible()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_websocket_busy_then_http_invalid_previous_response_should_request_official_replay() {
|
||||
const ACCOUNT_ID: &str = "acct_websocket_busy_replay";
|
||||
const CONVERSATION_ID: &str = "conversation-websocket-busy-replay";
|
||||
const CLIENT_PREVIOUS_RESPONSE_ID: &str = "client-resp-busy-seed";
|
||||
const UPSTREAM_PREVIOUS_RESPONSE_ID: &str = "resp_busy_seed";
|
||||
|
||||
fn operation(
|
||||
previous_response_id: Option<&str>,
|
||||
session_state: ProviderSessionState,
|
||||
) -> Operation {
|
||||
let mut body = Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
]);
|
||||
if let Some(previous_response_id) = previous_response_id {
|
||||
body.insert(
|
||||
"previous_response_id".to_owned(),
|
||||
json!(previous_response_id),
|
||||
);
|
||||
}
|
||||
Operation::Generate(
|
||||
GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", body).expect("OpenAI payload"),
|
||||
)
|
||||
.with_provider_session_state(session_state),
|
||||
)
|
||||
}
|
||||
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, ACCOUNT_ID).await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind upstream listener");
|
||||
let base_url = format!(
|
||||
"http://{}",
|
||||
listener.local_addr().expect("listener address")
|
||||
);
|
||||
let (release_busy_sender, release_busy_receiver) = oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.expect("accept WebSocket");
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
|
||||
let seed = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("seed request")
|
||||
.expect("valid seed request");
|
||||
let Message::Text(seed) = seed else {
|
||||
panic!("seed request must be text");
|
||||
};
|
||||
let seed: Value = serde_json::from_str(&seed).expect("seed request JSON");
|
||||
assert_eq!(seed.get("previous_response_id"), None);
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": UPSTREAM_PREVIOUS_RESPONSE_ID,
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("complete seed request");
|
||||
|
||||
let busy = websocket
|
||||
.next()
|
||||
.await
|
||||
.expect("busy continuation request")
|
||||
.expect("valid busy continuation request");
|
||||
let Message::Text(busy) = busy else {
|
||||
panic!("busy continuation request must be text");
|
||||
};
|
||||
let busy: Value = serde_json::from_str(&busy).expect("busy continuation JSON");
|
||||
assert_eq!(
|
||||
busy.get("previous_response_id").and_then(Value::as_str),
|
||||
Some(UPSTREAM_PREVIOUS_RESPONSE_ID)
|
||||
);
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "the exact WebSocket is busy"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send busy stream output");
|
||||
|
||||
let (mut http, _) = listener.accept().await.expect("accept HTTP fallback");
|
||||
read_http_request(&mut http).await;
|
||||
let body = json!({
|
||||
"error": {
|
||||
"code": null,
|
||||
"type": "invalid_request_error",
|
||||
"message": "Invalid `previous_response_id`."
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
http.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 400 Bad Request\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("write HTTP 400");
|
||||
|
||||
release_busy_receiver.await.expect("release busy stream");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_busy_finished",
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("complete busy stream");
|
||||
});
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let initial_session_state = ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("account_id".to_owned(), json!(ACCOUNT_ID)),
|
||||
("conversation_id".to_owned(), json!(CONVERSATION_ID)),
|
||||
("continuation_scope".to_owned(), json!("connection_local")),
|
||||
]),
|
||||
)
|
||||
.expect("initial provider session state");
|
||||
|
||||
let mut seed = provider
|
||||
.execute(
|
||||
planned_request("openai", operation(None, initial_session_state)),
|
||||
context("req_ws_busy_seed", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare seed stream");
|
||||
let mut session_state = None;
|
||||
while let Some(event) = seed.next().await {
|
||||
let event = event.expect("seed event");
|
||||
if let Some(update) = event.session_update() {
|
||||
session_state = Some(update.clone());
|
||||
}
|
||||
}
|
||||
let session_state = session_state.expect("seed session update");
|
||||
assert_eq!(
|
||||
session_state
|
||||
.payload()
|
||||
.get("continuation_scope")
|
||||
.and_then(Value::as_str),
|
||||
Some("connection_local")
|
||||
);
|
||||
|
||||
let continuation = operation(Some(CLIENT_PREVIOUS_RESPONSE_ID), session_state);
|
||||
let mut busy = provider
|
||||
.execute(
|
||||
planned_request("openai", continuation.clone()),
|
||||
pinned_continuation_context(
|
||||
"req_ws_busy_owner",
|
||||
ACCOUNT_ID,
|
||||
CLIENT_PREVIOUS_RESPONSE_ID,
|
||||
UPSTREAM_PREVIOUS_RESPONSE_ID,
|
||||
1,
|
||||
ContinuationAttempt::Native,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("prepare busy continuation stream");
|
||||
loop {
|
||||
let event = busy
|
||||
.next()
|
||||
.await
|
||||
.expect("busy stream event")
|
||||
.expect("valid busy stream event");
|
||||
if event.has_client_event() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut exact = provider
|
||||
.execute(
|
||||
planned_request("openai", continuation.clone()),
|
||||
pinned_continuation_context(
|
||||
"req_ws_busy_exact",
|
||||
ACCOUNT_ID,
|
||||
CLIENT_PREVIOUS_RESPONSE_ID,
|
||||
UPSTREAM_PREVIOUS_RESPONSE_ID,
|
||||
1,
|
||||
ContinuationAttempt::Native,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("prepare competing exact continuation");
|
||||
let exact_error = loop {
|
||||
match exact.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("competing exact continuation must fail while its socket is busy"),
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
exact_error.continuation_failure(),
|
||||
Some(ContinuationFailure::Busy)
|
||||
);
|
||||
|
||||
let mut replay = provider
|
||||
.execute(
|
||||
planned_request("openai", continuation),
|
||||
pinned_continuation_context(
|
||||
"req_ws_busy_replay",
|
||||
ACCOUNT_ID,
|
||||
CLIENT_PREVIOUS_RESPONSE_ID,
|
||||
UPSTREAM_PREVIOUS_RESPONSE_ID,
|
||||
2,
|
||||
ContinuationAttempt::ReplayAny,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("prepare replay-any continuation");
|
||||
let replay_error = loop {
|
||||
match replay.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("HTTP fallback 400 must produce a provider error"),
|
||||
}
|
||||
};
|
||||
let detail = replay_error
|
||||
.client_visible_upstream_error()
|
||||
.expect("client-visible replay error");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
replay_error.kind(),
|
||||
replay_error.continuation_failure(),
|
||||
detail.code(),
|
||||
detail.error_type(),
|
||||
detail.message(),
|
||||
),
|
||||
(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
Some(ContinuationFailure::HistoryUnavailable),
|
||||
Some("previous_response_not_found"),
|
||||
Some("invalid_request_error"),
|
||||
"Invalid `previous_response_id`.",
|
||||
)
|
||||
);
|
||||
|
||||
release_busy_sender.send(()).expect("release busy stream");
|
||||
while busy
|
||||
.next()
|
||||
.await
|
||||
.transpose()
|
||||
.expect("busy stream event")
|
||||
.is_some()
|
||||
{}
|
||||
server.await.expect("upstream server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn continuation_prefetch_over_64_kib_should_commit_wire_without_protocol_failure() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -3144,6 +3943,7 @@ async fn quota_limited_account_diagnostic_uses_upstream() {
|
||||
Arc::new(MemorySessionAffinity::default()),
|
||||
server.uri(),
|
||||
Arc::new(TestLeaseCoordinator::default()),
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
);
|
||||
quota
|
||||
.prepare_scheduling(std::slice::from_ref(&account))
|
||||
|
||||
@@ -854,9 +854,10 @@ fn decoder_should_preserve_changed_upstream_response_ids_as_wire() {
|
||||
}
|
||||
|
||||
fn assert_failed_event(code: &str, marker: &str) {
|
||||
let body = format!(
|
||||
"event: response.failed\ndata: {{\"response\":{{\"id\":\"resp_failed\",\"status\":\"failed\",\"error\":{{\"code\":\"{code}\",\"message\":\"{marker}\"}}}}}}\n\n"
|
||||
let raw_error = format!(
|
||||
"{{\"response\":{{\"id\":\"resp_failed\",\"status\":\"failed\",\"error\":{{\"code\":\"{code}\",\"message\":\"{marker}\"}}}}}}"
|
||||
);
|
||||
let body = format!("event: response.failed\ndata: {raw_error}\n\n");
|
||||
let failure = CodexCanonicalDecoder::new("fallback")
|
||||
.push(body.as_bytes())
|
||||
.expect_err("failed event must become a typed error");
|
||||
@@ -865,6 +866,7 @@ fn assert_failed_event(code: &str, marker: &str) {
|
||||
panic!("response.failed must preserve its typed upstream failure");
|
||||
};
|
||||
assert_eq!(upstream.upstream_code.as_deref(), Some(code));
|
||||
assert_eq!(upstream.raw_body(), raw_error);
|
||||
assert!(!failure.semantic_output_seen());
|
||||
assert!(!format!("{failure:?}").contains(marker));
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ fn request_with_opaque_headers(use_websocket: bool) -> CodexResponsesRequest {
|
||||
],
|
||||
["user-agent", STANDARD.encode(b"Codex future-client")],
|
||||
["originator", STANDARD.encode(b"future-originator")],
|
||||
["version", STANDARD.encode(b"26.999.10001")],
|
||||
["version", STANDARD.encode(b"26.999.10002")],
|
||||
["openai-beta", STANDARD.encode(b"future_responses=v2")],
|
||||
["openai-beta", STANDARD.encode(b"future_tools=v3")],
|
||||
[
|
||||
@@ -321,7 +323,7 @@ async fn backend_websocket_should_forward_context_headers_and_preserve_payload_f
|
||||
("x-codex-turn-metadata", "{\"thread_source\":\"subagent\"}"),
|
||||
("x-codex-beta-features", "feature-a"),
|
||||
("x-responsesapi-include-timing-metrics", "true"),
|
||||
("version", "26.318.11754"),
|
||||
("version", "1.2.3"),
|
||||
("x-codex-window-id", "cw_derived"),
|
||||
("x-codex-parent-thread-id", "parent-456"),
|
||||
("x-openai-subagent", "future_codex_mode"),
|
||||
@@ -356,14 +358,17 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
request
|
||||
});
|
||||
let request = request_with_opaque_headers(false);
|
||||
let expected_user_agent = test_wire_profile().snapshot().user_agent();
|
||||
let profile = test_wire_profile();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("HTTP client"),
|
||||
format!("http://{address}"),
|
||||
test_wire_profile(),
|
||||
profile,
|
||||
);
|
||||
|
||||
client
|
||||
@@ -374,6 +379,7 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
version: Some("26.825.51511"),
|
||||
..request_context("req_opaque_http", Some("lease-account"))
|
||||
},
|
||||
)
|
||||
@@ -413,6 +419,10 @@ async fn backend_http_should_restore_opaque_multivalue_header_bytes_and_lease_id
|
||||
raw_header_values(&raw, "originator"),
|
||||
vec![b"codex_cli_rs".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, "version"),
|
||||
vec![expected_desktop_version.into_bytes()]
|
||||
);
|
||||
for dropped in [
|
||||
"openai-beta",
|
||||
"x-oai-attestation",
|
||||
@@ -490,6 +500,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
account_id: Some("lease-account"),
|
||||
installation_id: Some("lease-installation"),
|
||||
turn_state: request.turn_state.as_deref(),
|
||||
version: Some("26.825.51511"),
|
||||
..request_context("req_opaque_ws", Some("lease-account"))
|
||||
},
|
||||
)
|
||||
@@ -526,6 +537,7 @@ async fn backend_websocket_should_drop_only_unrepresentable_opaque_header_values
|
||||
vec![test_wire_profile().snapshot().user_agent().into_bytes()]
|
||||
);
|
||||
assert_eq!(values("originator"), vec![b"codex_cli_rs".to_vec()]);
|
||||
assert_eq!(values("version"), vec![b"1.2.3".to_vec()]);
|
||||
assert_eq!(
|
||||
values("x-openai-internal-codex-residency"),
|
||||
vec![b"us".to_vec()]
|
||||
@@ -569,7 +581,9 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
request.codex_window_id = Some("cw_1".to_owned());
|
||||
request.parent_thread_id = Some("parent-1".to_owned());
|
||||
let profile = test_wire_profile();
|
||||
let expected_user_agent = profile.snapshot().user_agent();
|
||||
let profile_snapshot = profile.snapshot();
|
||||
let expected_user_agent = profile_snapshot.user_agent();
|
||||
let expected_desktop_version = profile_snapshot.desktop_version;
|
||||
let client = CodexBackendClient::new(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
@@ -629,6 +643,10 @@ async fn backend_http_should_send_codex_context_without_browser_headers() {
|
||||
Some("turn-state"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
read_header_value(&raw_request, "version"),
|
||||
Some(expected_desktop_version.as_str())
|
||||
);
|
||||
for required in [
|
||||
"authorization",
|
||||
"chatgpt-account-id",
|
||||
|
||||
@@ -1117,6 +1117,10 @@ async fn websocket_failure_after_first_delivery_should_not_open_http_fallback()
|
||||
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = *source else {
|
||||
panic!("expected upstream close frame source");
|
||||
};
|
||||
assert!(
|
||||
close.connection_id().is_some(),
|
||||
"close failures must retain their WebSocket connection correlation"
|
||||
);
|
||||
assert_eq!(close.code(), Some(1009));
|
||||
assert_eq!(close.reason(), Some("message too big after output"));
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ mod headers;
|
||||
mod http_client;
|
||||
mod latency;
|
||||
mod profile;
|
||||
mod profile_avatar;
|
||||
mod protocol;
|
||||
mod request;
|
||||
mod reset_credits;
|
||||
@@ -233,6 +234,7 @@ async fn collect_backend_response(
|
||||
let CodexBackendStreamingResponse {
|
||||
mut body,
|
||||
transport,
|
||||
websocket_connection_id: _,
|
||||
mut turn_state,
|
||||
set_cookie_headers,
|
||||
mut rate_limit_headers,
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
use futures::TryStreamExt as _;
|
||||
use provider_openai::transport::{CodexProfileAvatarFetchError, fetch_profile_avatar};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
const OFFICIAL_AVATAR_SOURCE: &str =
|
||||
"https://chatgpt.com/backend-api/estuary/public_content/enc/opaque-token=";
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_avatar_streams_unrestricted_content_type_and_body_size() {
|
||||
let server = MockServer::start().await;
|
||||
let body = vec![b'x'; 1024 * 1024 + 1];
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/estuary/public_content/enc/opaque-token="))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "application/x-avatar-fixture")
|
||||
.insert_header("etag", "\"avatar-v1\"")
|
||||
.set_body_bytes(body.clone()),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("client");
|
||||
|
||||
let avatar = fetch_profile_avatar(
|
||||
&client,
|
||||
&server.uri(),
|
||||
"codex_cli_rs/1.0.0 (linux; x86_64)",
|
||||
OFFICIAL_AVATAR_SOURCE,
|
||||
)
|
||||
.await
|
||||
.expect("profile avatar");
|
||||
let content_type = avatar.content_type.clone();
|
||||
let content_length = avatar.content_length;
|
||||
let etag = avatar.etag.clone();
|
||||
let chunks = avatar.body.try_collect::<Vec<_>>().await.expect("body");
|
||||
let streamed = chunks.into_iter().flatten().collect::<Vec<_>>();
|
||||
let requests = server.received_requests().await.expect("avatar request");
|
||||
let headers = &requests.first().expect("one avatar request").headers;
|
||||
|
||||
assert_eq!(
|
||||
content_type.as_deref(),
|
||||
Some("application/x-avatar-fixture")
|
||||
);
|
||||
assert_eq!(content_length, Some(body.len() as u64));
|
||||
assert_eq!(etag.as_deref(), Some("\"avatar-v1\""));
|
||||
assert_eq!(streamed, body);
|
||||
assert_eq!(
|
||||
headers.get("accept").and_then(|value| value.to_str().ok()),
|
||||
Some("image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("user-agent")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("codex_cli_rs/1.0.0 (linux; x86_64)")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn profile_avatar_rejects_non_official_sources_before_request() {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
for source in [
|
||||
"https://example.com/backend-api/estuary/public_content/enc/token",
|
||||
"https://chatgpt.com/other/token",
|
||||
"https://chatgpt.com/backend-api/estuary/public_content/enc/token?next=1",
|
||||
"https://chatgpt.com/backend-api/estuary/public_content/enc/",
|
||||
] {
|
||||
let error = fetch_profile_avatar(&client, "http://127.0.0.1:9", "test-agent", source)
|
||||
.await
|
||||
.expect_err("invalid source");
|
||||
assert!(matches!(error, CodexProfileAvatarFetchError::InvalidSource));
|
||||
}
|
||||
}
|
||||
@@ -1507,6 +1507,17 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "secret@example.com",
|
||||
"payload": "must not become a diagnostic event type"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.close(None).await.unwrap();
|
||||
});
|
||||
let mut request = codex_request("gpt-5.5", "be brief", Vec::new());
|
||||
@@ -1533,14 +1544,27 @@ async fn codex_backend_client_stream_should_error_when_websocket_closes_before_t
|
||||
.expect("stream should yield partial frame")
|
||||
.expect("partial frame should be valid");
|
||||
assert!(std::str::from_utf8(&first).unwrap().contains("partial"));
|
||||
let error = response
|
||||
.body
|
||||
.next()
|
||||
.await
|
||||
.expect("stream should yield close-before-terminal error")
|
||||
.expect_err("close before terminal should be an error");
|
||||
let error = loop {
|
||||
match response.body.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("stream should yield close-before-terminal error"),
|
||||
}
|
||||
};
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
source: Some(source),
|
||||
..
|
||||
}) = &error
|
||||
else {
|
||||
panic!("close-before-terminal should preserve its typed source: {error:?}");
|
||||
};
|
||||
let CodexWebSocketExchangeError::ClosedBeforeTerminal(close) = source.as_ref() else {
|
||||
panic!("post-send ambiguity should wrap a close-before-terminal error: {source:?}");
|
||||
};
|
||||
assert_eq!(close.last_event_type(), Some("response.output_text.delta"));
|
||||
|
||||
std::assert_matches!(
|
||||
error,
|
||||
CodexClientError::WebSocket(CodexWebSocketExchangeError::PostSendAmbiguous {
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ RUN npm install --global pnpm@11.7.0 \
|
||||
COPY frontend ./
|
||||
RUN pnpm build
|
||||
|
||||
FROM rust:1.98-bookworm@sha256:e70e2eec3d495fd5c8e0be74adda86507dfac7f51a724fbf9813ff59b2b247c7 AS backend-builder
|
||||
FROM rust:1.98-bookworm@sha256:82150a52ec202c1b14d7817e14516c392bb7f5cfebd88f1ed531cb37ebd39922 AS backend-builder
|
||||
|
||||
WORKDIR /app/backend
|
||||
ARG TARGETARCH
|
||||
@@ -40,7 +40,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
# 备份客户端来源:与 compose 服务端同一固定镜像,版本一致。
|
||||
FROM ${POSTGRES_IMAGE} AS pg-client
|
||||
|
||||
FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS runtime-base
|
||||
FROM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS runtime-base
|
||||
|
||||
ENV TZ=Asia/Shanghai
|
||||
|
||||
|
||||
@@ -63,25 +63,25 @@ openai:
|
||||
# 版本元组来自同一个官方 macOS Desktop ZIP;Provider 每 24 小时检查 appcast,
|
||||
# 仅在包内 Core 核验和 Redis 条件覆盖都成功后原子更新。
|
||||
originator: 'Codex Desktop'
|
||||
codex_version: '0.147.0-alpha.6.6'
|
||||
desktop_version: '26.803.81509'
|
||||
desktop_build: '6415'
|
||||
codex_version: '0.151.0-alpha.7.2'
|
||||
desktop_version: '26.825.51511'
|
||||
desktop_build: '7377'
|
||||
os_type: 'Mac OS'
|
||||
os_version: '15.7.1'
|
||||
arch: 'arm64'
|
||||
terminal: 'unknown'
|
||||
verified_at: '2026-08-03T00:00:00+08:00'
|
||||
verified_at: '2026-08-31T00:00:00+08:00'
|
||||
|
||||
xai:
|
||||
# OAuth 端点、额度与模型目录策略由 xAI Provider 固定管理;这里只声明请求画像启动基线。
|
||||
wire_profile:
|
||||
# Provider 每 24 小时读取官方 npm metadata,仅在进程内更新 client_version。
|
||||
client_identifier: 'grok-shell'
|
||||
client_version: '0.2.118'
|
||||
client_identifier: 'GrokShell'
|
||||
client_version: '1.0.13'
|
||||
client_mode: 'headless'
|
||||
target_os: 'linux'
|
||||
target_arch: 'x86_64'
|
||||
verified_at: '2026-07-21T00:00:00+08:00'
|
||||
verified_at: '2026-08-31T00:00:00+08:00'
|
||||
|
||||
# Compose 凭据桥接区;密码仍只维护在 store 中。
|
||||
services:
|
||||
|
||||
+53
-35
@@ -1,47 +1,65 @@
|
||||
import type { ESLint, Rule } from 'eslint'
|
||||
import antfu from '@antfu/eslint-config'
|
||||
|
||||
const templateStylePlugin = {
|
||||
rules: {
|
||||
'single-line-simple-interpolation': {
|
||||
meta: {
|
||||
type: 'layout',
|
||||
fixable: 'whitespace',
|
||||
schema: [],
|
||||
messages: {
|
||||
expectedSingleLine: 'Expected a simple mustache interpolation to stay on one line.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const { parserServices } = context.sourceCode
|
||||
if (!parserServices.defineTemplateBodyVisitor)
|
||||
return {}
|
||||
type LocatedRuleNode = Rule.Node & {
|
||||
loc: NonNullable<Rule.Node['loc']>
|
||||
}
|
||||
|
||||
return parserServices.defineTemplateBodyVisitor({
|
||||
'VExpressionContainer[expression!=null]': function (node) {
|
||||
if (node.parent.type !== 'VElement')
|
||||
return
|
||||
interface TemplateExpressionContainer {
|
||||
type: 'VExpressionContainer'
|
||||
parent: { type: string }
|
||||
expression: LocatedRuleNode
|
||||
loc: NonNullable<Rule.Node['loc']>
|
||||
range: NonNullable<Rule.Node['range']>
|
||||
}
|
||||
|
||||
const expression = node.expression
|
||||
if (
|
||||
expression.loc.start.line !== expression.loc.end.line
|
||||
|| node.loc.start.line === node.loc.end.line
|
||||
) {
|
||||
return
|
||||
}
|
||||
const singleLineSimpleInterpolationRule = {
|
||||
meta: {
|
||||
type: 'layout',
|
||||
fixable: 'whitespace',
|
||||
schema: [],
|
||||
messages: {
|
||||
expectedSingleLine: 'Expected a simple mustache interpolation to stay on one line.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const { parserServices } = context.sourceCode
|
||||
if (!parserServices.defineTemplateBodyVisitor)
|
||||
return {}
|
||||
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'expectedSingleLine',
|
||||
fix(fixer) {
|
||||
return fixer.replaceText(node, `{{ ${context.sourceCode.getText(expression)} }}`)
|
||||
},
|
||||
})
|
||||
return parserServices.defineTemplateBodyVisitor({
|
||||
'VExpressionContainer[expression!=null]': function (node: TemplateExpressionContainer) {
|
||||
if (node.parent.type !== 'VElement')
|
||||
return
|
||||
|
||||
const expression = node.expression
|
||||
if (
|
||||
expression.loc.start.line !== expression.loc.end.line
|
||||
|| node.loc.start.line === node.loc.end.line
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'expectedSingleLine',
|
||||
fix(fixer) {
|
||||
return fixer.replaceTextRange(
|
||||
node.range,
|
||||
`{{ ${context.sourceCode.getText(expression)} }}`,
|
||||
)
|
||||
},
|
||||
})
|
||||
},
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
} satisfies Rule.RuleModule
|
||||
|
||||
const templateStylePlugin = {
|
||||
rules: {
|
||||
'single-line-simple-interpolation': singleLineSimpleInterpolationRule,
|
||||
},
|
||||
} satisfies ESLint.Plugin
|
||||
|
||||
export default antfu(
|
||||
{
|
||||
|
||||
@@ -19,27 +19,27 @@
|
||||
"@boxicons/vue": "1.0.1",
|
||||
"@fontsource-variable/inter": "^5.3.0",
|
||||
"@fontsource-variable/jetbrains-mono": "^5.3.0",
|
||||
"@lucide/vue": "^1.33.0",
|
||||
"@lucide/vue": "^1.37.0",
|
||||
"@vueuse/core": "^14.4.0",
|
||||
"axios": "^1.19.0",
|
||||
"axios": "^1.20.0",
|
||||
"dayjs": "^1.11.23",
|
||||
"dompurify": "^3.4.14",
|
||||
"echarts": "^6.1.0",
|
||||
"es-toolkit": "^1.51.0",
|
||||
"es-toolkit": "^1.52.0",
|
||||
"gsap": "^3.15.0",
|
||||
"marked": "^18.0.10",
|
||||
"marked": "^18.0.11",
|
||||
"pinia": "^4.0.3",
|
||||
"pinia-plugin-persistedstate": "^4.7.1",
|
||||
"vue": "^3.5.41",
|
||||
"vue-router": "^5.2.0"
|
||||
"vue": "^3.5.42",
|
||||
"vue-router": "^5.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^9.3.0",
|
||||
"@tailwindcss/vite": "^4.3.3",
|
||||
"@types/node": "^26.2.0",
|
||||
"@types/node": "^26.4.0",
|
||||
"@vitejs/plugin-vue": "^6.0.8",
|
||||
"@vue/tsconfig": "^0.9.1",
|
||||
"eslint": "^10.9.0",
|
||||
"eslint": "^10.9.1",
|
||||
"eslint-plugin-format": "^2.0.1",
|
||||
"eslint-plugin-vuejs-accessibility": "^2.6.0",
|
||||
"tailwindcss": "^4.3.3",
|
||||
|
||||
Generated
+293
-369
File diff suppressed because it is too large
Load Diff
@@ -368,6 +368,21 @@ export function getAccountProfileStatistics(data: AccountIdParam) {
|
||||
})
|
||||
}
|
||||
|
||||
export function accountProfileAvatarUrl(accountId: string, sourceUrl: string) {
|
||||
const params = new URLSearchParams({
|
||||
accountId,
|
||||
version: stableAvatarVersion(sourceUrl),
|
||||
})
|
||||
return `/api/admin/accounts/profile-avatar?${params.toString()}`
|
||||
}
|
||||
|
||||
function stableAvatarVersion(value: string) {
|
||||
let hash = 0
|
||||
for (const character of value)
|
||||
hash = (hash * 33 + (character.codePointAt(0) ?? 0)) % 2_147_483_647
|
||||
return hash.toString(36)
|
||||
}
|
||||
|
||||
export function refreshAccountQuota(data: AccountIdParam) {
|
||||
return request<AccountQuotaResponse>({
|
||||
url: '/api/admin/accounts/quota/refresh',
|
||||
|
||||
@@ -211,19 +211,36 @@ export interface OpsError {
|
||||
requestId: string | null
|
||||
clientApiKeyId: string | null
|
||||
kind: string
|
||||
operation: string
|
||||
protocol: string | null
|
||||
clientTransport: string | null
|
||||
provider: string | null
|
||||
authenticationKind: string | null
|
||||
accountId: string | null
|
||||
accountName: string | null
|
||||
accountEmail: string | null
|
||||
route: string
|
||||
model: string | null
|
||||
requestedModel: string | null
|
||||
upstreamModel: string | null
|
||||
serviceTier: string | null
|
||||
clientStatusCode: number | null
|
||||
upstreamStatusCode: number | null
|
||||
transport: string | null
|
||||
attemptIndex: number | null
|
||||
failureClass: string
|
||||
providerErrorCode: string | null
|
||||
occurrenceCount: number
|
||||
responseId: string | null
|
||||
upstreamRequestId: string | null
|
||||
latencyMs: number | null
|
||||
clientIp: string | null
|
||||
userAgent: string | null
|
||||
reasoningEffort: string | null
|
||||
reasoningPreset: string | null
|
||||
requestKind: string | null
|
||||
subagentKind: string | null
|
||||
compact: boolean | null
|
||||
message: string
|
||||
metadata: OpsErrorMetadata
|
||||
createdAt: string
|
||||
@@ -406,98 +423,26 @@ type OpsErrorPageQuery = UsagePageQuery & {
|
||||
route?: string
|
||||
}
|
||||
|
||||
interface CursorPageResponse<Item> {
|
||||
items: Item[]
|
||||
total: number | null
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
type CursorPageQuery<Query> = Query & {
|
||||
pageSize: number
|
||||
cursor?: string
|
||||
}
|
||||
|
||||
interface UsageDetailQuery {
|
||||
id: string
|
||||
}
|
||||
|
||||
type UsageDiagnosticsQuery = UsageRangeQuery & { dimension: string }
|
||||
|
||||
export function createUsageRecordsPager() {
|
||||
return createPageLoader<UsageListRecord, UsageRangeQuery>(data =>
|
||||
request<CursorPageResponse<UsageListRecord>>({
|
||||
url: '/api/admin/usage/records',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
}))
|
||||
export function getUsageRecords(data: UsagePageQuery) {
|
||||
return request<UsageRecordsResponse>({
|
||||
url: '/api/admin/usage/records',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
})
|
||||
}
|
||||
|
||||
export function createOpsErrorsPager() {
|
||||
return createPageLoader<OpsError, Omit<OpsErrorPageQuery, keyof PageQuery>>(data =>
|
||||
request<CursorPageResponse<OpsError>>({
|
||||
url: '/api/admin/operations/errors',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
}))
|
||||
}
|
||||
|
||||
function createPageLoader<Item, Query extends object>(
|
||||
fetchPage: (query: CursorPageQuery<Query>) => Promise<CursorPageResponse<Item>>,
|
||||
) {
|
||||
const cursors = new Map<number, string | undefined>([[1, undefined]])
|
||||
let queryKey = ''
|
||||
let total = 0
|
||||
|
||||
function reset(nextQueryKey: string) {
|
||||
cursors.clear()
|
||||
cursors.set(1, undefined)
|
||||
queryKey = nextQueryKey
|
||||
total = 0
|
||||
}
|
||||
|
||||
async function load(data: Query & PageQuery) {
|
||||
const { currentPage, pageSize, ...query } = data
|
||||
const nextQueryKey = JSON.stringify([pageSize, query])
|
||||
if (queryKey !== nextQueryKey)
|
||||
reset(nextQueryKey)
|
||||
|
||||
const requestedPage = Math.max(1, Math.trunc(currentPage))
|
||||
let page = [...cursors.keys()].reduce(
|
||||
(known, candidate) => candidate <= requestedPage && candidate > known ? candidate : known,
|
||||
1,
|
||||
)
|
||||
let result: CursorPageResponse<Item>
|
||||
|
||||
while (page < requestedPage) {
|
||||
result = await fetchPage({
|
||||
...(query as Query),
|
||||
pageSize,
|
||||
cursor: cursors.get(page),
|
||||
})
|
||||
if (result.total != null)
|
||||
total = result.total
|
||||
if (!result.nextCursor)
|
||||
return { items: result.items, currentPage: page, pageSize, total }
|
||||
cursors.set(page + 1, result.nextCursor)
|
||||
page += 1
|
||||
}
|
||||
|
||||
result = await fetchPage({
|
||||
...(query as Query),
|
||||
pageSize,
|
||||
cursor: cursors.get(page),
|
||||
})
|
||||
if (result.total != null)
|
||||
total = result.total
|
||||
if (result.nextCursor)
|
||||
cursors.set(page + 1, result.nextCursor)
|
||||
else
|
||||
cursors.delete(page + 1)
|
||||
|
||||
return { items: result.items, currentPage: page, pageSize, total }
|
||||
}
|
||||
|
||||
return { load }
|
||||
export function getOpsErrors(data: OpsErrorPageQuery) {
|
||||
return request<OpsErrorsResponse>({
|
||||
url: '/api/admin/operations/errors',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
})
|
||||
}
|
||||
|
||||
export function getUsageRecordDetail(data: UsageDetailQuery) {
|
||||
|
||||
@@ -7,7 +7,7 @@ const props = withDefaults(
|
||||
defineProps<{
|
||||
provider?: string | null
|
||||
authenticationKind?: string | null
|
||||
size?: 'sm' | 'md'
|
||||
size?: 'xs' | 'sm' | 'md'
|
||||
}>(),
|
||||
{
|
||||
size: 'md',
|
||||
@@ -17,9 +17,21 @@ const props = withDefaults(
|
||||
const normalizedProvider = computed(() => (props.provider ?? '').trim().toLowerCase())
|
||||
const normalizedAuthenticationKind = computed(() => (props.authenticationKind ?? '').trim().toLowerCase())
|
||||
const showAuthenticationKind = computed(() => props.authenticationKind !== undefined)
|
||||
const groupGapClass = computed(() => (props.size === 'sm' ? 'gap-1' : 'gap-2'))
|
||||
const iconContainerClass = computed(() => (props.size === 'sm' ? 'size-5 rounded-md' : 'size-7 rounded-lg'))
|
||||
const iconClass = computed(() => (props.size === 'sm' ? 'size-3' : 'size-4'))
|
||||
const groupGapClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'gap-0.75'
|
||||
return props.size === 'sm' ? 'gap-1' : 'gap-2'
|
||||
})
|
||||
const iconContainerClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'size-4.5 rounded-md'
|
||||
return props.size === 'sm' ? 'size-5 rounded-md' : 'size-7 rounded-lg'
|
||||
})
|
||||
const iconClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'size-2.5'
|
||||
return props.size === 'sm' ? 'size-3' : 'size-4'
|
||||
})
|
||||
|
||||
const providerLabel = computed(() => formatProviderLabel(props.provider, '未知平台'))
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ const indicatorClasses = computed(() => [
|
||||
? 'bg-cp-bg-container-disabled text-cp-text-disabled'
|
||||
: checked.value
|
||||
? 'bg-cp-primary text-(--cp-button-primary-color)'
|
||||
: 'bg-cp-fill-tertiary text-transparent group-hover:bg-cp-bg-text-active',
|
||||
: 'bg-cp-fill-secondary text-transparent',
|
||||
])
|
||||
</script>
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import type { LineSeriesOption } from 'echarts'
|
||||
|
||||
type TemporalValue = number | null | undefined
|
||||
|
||||
interface GapRun {
|
||||
startIndex: number
|
||||
endIndex: number
|
||||
}
|
||||
|
||||
interface GapBridgeOptions {
|
||||
name: string
|
||||
color: string
|
||||
xAxisIndex?: number
|
||||
yAxisIndex?: number
|
||||
width?: number
|
||||
z?: number
|
||||
maxGapBuckets?: number
|
||||
}
|
||||
|
||||
const defaultShortGapBuckets = 2
|
||||
|
||||
export function sampleGapBridgeSeries(
|
||||
values: TemporalValue[],
|
||||
options: GapBridgeOptions,
|
||||
): LineSeriesOption[] {
|
||||
const data = values.map(normalizeValue)
|
||||
const maxGapBuckets = options.maxGapBuckets ?? defaultShortGapBuckets
|
||||
const series: LineSeriesOption[] = []
|
||||
|
||||
for (const [runIndex, run] of boundedGapRuns(data)
|
||||
.filter(run => gapLength(run) <= maxGapBuckets)
|
||||
.entries()) {
|
||||
const beforeIndex = run.startIndex - 1
|
||||
const afterIndex = run.endIndex + 1
|
||||
const beforeValue = data[beforeIndex]
|
||||
const afterValue = data[afterIndex]
|
||||
const bridge = Array.from<number | null>({ length: data.length }).fill(null)
|
||||
|
||||
if (beforeValue == null || afterValue == null)
|
||||
continue
|
||||
|
||||
const span = afterIndex - beforeIndex
|
||||
for (let index = beforeIndex; index <= afterIndex; index += 1) {
|
||||
const progress = (index - beforeIndex) / span
|
||||
bridge[index] = beforeValue + (afterValue - beforeValue) * progress
|
||||
}
|
||||
|
||||
series.push({
|
||||
name: `${options.name}-gap-${runIndex}`,
|
||||
type: 'line',
|
||||
data: bridge,
|
||||
connectNulls: false,
|
||||
smooth: false,
|
||||
showSymbol: false,
|
||||
silent: true,
|
||||
xAxisIndex: options.xAxisIndex ?? 0,
|
||||
yAxisIndex: options.yAxisIndex ?? 0,
|
||||
z: Math.max(0, (options.z ?? 2) - 1),
|
||||
lineStyle: {
|
||||
color: options.color,
|
||||
type: 'dashed',
|
||||
width: Math.max(1, (options.width ?? 2.2) - 0.7),
|
||||
opacity: 0.32,
|
||||
},
|
||||
itemStyle: { color: options.color, opacity: 0 },
|
||||
emphasis: { disabled: true },
|
||||
tooltip: { show: false },
|
||||
})
|
||||
}
|
||||
|
||||
return series
|
||||
}
|
||||
|
||||
export function requestActivityByBucket(
|
||||
buckets: string[],
|
||||
activity: Array<{ bucket: string, totalRequests: number }>,
|
||||
) {
|
||||
const requestsByBucket = new Map(
|
||||
activity.map(point => [point.bucket, Math.max(0, point.totalRequests)]),
|
||||
)
|
||||
return buckets.map(bucket => (requestsByBucket.get(bucket) ?? 0) > 0)
|
||||
}
|
||||
|
||||
export function zeroInactiveValues(
|
||||
values: TemporalValue[],
|
||||
active: boolean[],
|
||||
) {
|
||||
return values.map((value, index) => normalizeValue(value) ?? (active[index] === false ? 0 : null))
|
||||
}
|
||||
|
||||
function normalizeValue(value: TemporalValue) {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
||||
}
|
||||
|
||||
function boundedGapRuns(values: Array<number | null>) {
|
||||
return gapRuns(values.map(value => value == null)).filter(
|
||||
run => run.startIndex > 0 && run.endIndex < values.length - 1,
|
||||
)
|
||||
}
|
||||
|
||||
function gapRuns(isGap: boolean[]) {
|
||||
const runs: GapRun[] = []
|
||||
let startIndex = -1
|
||||
|
||||
for (let index = 0; index <= isGap.length; index += 1) {
|
||||
if (isGap[index] && startIndex < 0) {
|
||||
startIndex = index
|
||||
continue
|
||||
}
|
||||
if (isGap[index] || startIndex < 0)
|
||||
continue
|
||||
runs.push({ startIndex, endIndex: index - 1 })
|
||||
startIndex = -1
|
||||
}
|
||||
|
||||
return runs
|
||||
}
|
||||
|
||||
function gapLength(run: GapRun) {
|
||||
return run.endIndex - run.startIndex + 1
|
||||
}
|
||||
@@ -14,7 +14,7 @@ interface PageRequest {
|
||||
pageSize: number
|
||||
}
|
||||
|
||||
/** Element Plus 风格的页面分页状态;稳定深分页由 API pager 内部完成。 */
|
||||
/** Element Plus 风格的服务端页码分页;一次 execute 只发起一次列表请求。 */
|
||||
export function useStablePagedQuery<Result extends PageResult>(options: {
|
||||
initialPageSize: number
|
||||
load: (pagination: PageRequest) => Promise<Result>
|
||||
|
||||
@@ -16,12 +16,14 @@ const props = withDefaults(
|
||||
showPlan?: boolean
|
||||
titleMode?: 'local-part' | 'email'
|
||||
metaPosition?: 'title' | 'secondary'
|
||||
metaSize?: 'xs' | 'sm'
|
||||
}>(),
|
||||
{
|
||||
size: 'md',
|
||||
showPlan: false,
|
||||
titleMode: 'local-part',
|
||||
metaPosition: 'title',
|
||||
metaSize: 'sm',
|
||||
},
|
||||
)
|
||||
|
||||
@@ -54,6 +56,8 @@ const secondaryClass = computed(() =>
|
||||
: 'mt-0.5 font-mono text-cp-xs text-cp-text-quaternary',
|
||||
)
|
||||
|
||||
const metaGapClass = computed(() => props.metaSize === 'xs' ? 'gap-1' : 'gap-1.5')
|
||||
|
||||
const avatarToneClass = computed(() => {
|
||||
const identity = props.account.id || props.account.email || displayTitle.value
|
||||
return stablePresetVisualToneClass(identity)
|
||||
@@ -75,18 +79,20 @@ const avatarToneClass = computed(() => {
|
||||
</span>
|
||||
<span
|
||||
v-if="metaPosition === 'title' && (showPlan || $slots.meta)"
|
||||
class="inline-flex shrink-0 items-center justify-end gap-1.5"
|
||||
class="inline-flex shrink-0 items-center justify-end"
|
||||
:class="metaGapClass"
|
||||
>
|
||||
<slot name="meta" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" size="sm" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" :size="metaSize" />
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
v-if="metaPosition === 'secondary' && (showPlan || $slots.meta)"
|
||||
class="mt-0.5 inline-flex min-w-0 items-center gap-1.5"
|
||||
class="mt-0.5 inline-flex min-w-0 items-center"
|
||||
:class="metaGapClass"
|
||||
>
|
||||
<slot name="meta" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" size="sm" />
|
||||
<AccountPlanBadge v-if="showPlan" :plan-type="account.planType" :size="metaSize" />
|
||||
</div>
|
||||
<div v-else-if="secondaryText" class="truncate font-emphasis" :class="secondaryClass">
|
||||
{{ secondaryText }}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { stableVisualIndex } from '../utils/visualTone'
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
planType?: string | null
|
||||
size?: 'sm' | 'md'
|
||||
size?: 'xs' | 'sm' | 'md'
|
||||
}>(),
|
||||
{
|
||||
planType: null,
|
||||
@@ -29,11 +29,13 @@ const fallbackPalettes = [
|
||||
|
||||
const label = computed(() => props.planType?.trim() || 'Free')
|
||||
|
||||
const sizeClass = computed(() =>
|
||||
props.size === 'sm'
|
||||
const sizeClass = computed(() => {
|
||||
if (props.size === 'xs')
|
||||
return 'h-4.5 rounded-full px-1.5 text-[10px] font-bold'
|
||||
return props.size === 'sm'
|
||||
? 'h-5 rounded-full px-1.75 text-cp-xs font-bold'
|
||||
: 'h-5.5 rounded-full px-2 text-cp-xs font-heavy',
|
||||
)
|
||||
: 'h-5.5 rounded-full px-2 text-cp-xs font-heavy'
|
||||
})
|
||||
|
||||
const paletteClass = computed(() => {
|
||||
const key = label.value.toLowerCase()
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { AccountProfileStatisticsResponse } from '@/api'
|
||||
import { UserRound } from '@lucide/vue'
|
||||
import { computed, shallowRef, watch } from 'vue'
|
||||
|
||||
import { accountProfileAvatarUrl } from '@/api'
|
||||
import { formatCompactNumber, formatInteger } from '@/utils/number'
|
||||
import { stablePresetVisualToneClass } from '../../utils/visualTone'
|
||||
import AccountPlanBadge from '../AccountPlanBadge.vue'
|
||||
@@ -23,6 +24,10 @@ const initial = computed(() => Array.from(displayName.value.trim())[0]?.toUpperC
|
||||
const avatarToneClass = computed(() =>
|
||||
stablePresetVisualToneClass(props.account.id || accountIdentity.value || displayName.value),
|
||||
)
|
||||
const avatarUrl = computed(() => {
|
||||
const sourceUrl = props.profile.imageUrl?.trim()
|
||||
return sourceUrl ? accountProfileAvatarUrl(props.account.id, sourceUrl) : null
|
||||
})
|
||||
const metrics = computed(() => [
|
||||
{
|
||||
label: '累计 Token 数',
|
||||
@@ -49,7 +54,7 @@ const metrics = computed(() => [
|
||||
])
|
||||
|
||||
watch(
|
||||
() => props.profile.imageUrl,
|
||||
avatarUrl,
|
||||
() => {
|
||||
imageFailed.value = false
|
||||
},
|
||||
@@ -95,8 +100,8 @@ function formatDuration(value: number | null) {
|
||||
:class="avatarToneClass"
|
||||
>
|
||||
<img
|
||||
v-if="profile.imageUrl && !imageFailed"
|
||||
:src="profile.imageUrl"
|
||||
v-if="avatarUrl && !imageFailed"
|
||||
:src="avatarUrl"
|
||||
:alt="`${displayName} 的头像`"
|
||||
class="size-full object-cover"
|
||||
referrerpolicy="no-referrer"
|
||||
|
||||
@@ -157,7 +157,7 @@ const {
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex h-full min-h-0 w-full flex-col overflow-hidden">
|
||||
<div class="flex min-h-0 w-full flex-col xl:h-full xl:overflow-hidden">
|
||||
<BasePageHeader
|
||||
class="h-17"
|
||||
title="账号管理"
|
||||
@@ -167,7 +167,7 @@ const {
|
||||
<AccountOverviewCards :summary="accountSummary" />
|
||||
|
||||
<BaseCard
|
||||
class="mt-4 flex h-[calc(100dvh-250px)] min-h-125 flex-col"
|
||||
class="mt-4 flex flex-col xl:h-[calc(100dvh-250px)] xl:min-h-125"
|
||||
>
|
||||
<template #header>
|
||||
<AccountFilters
|
||||
@@ -188,9 +188,9 @@ const {
|
||||
</template>
|
||||
|
||||
<template #body>
|
||||
<div class="flex h-full min-h-0 flex-col">
|
||||
<div class="flex min-h-0 flex-col xl:h-full">
|
||||
<BaseTable
|
||||
class="min-h-0 flex-1 [--cp-table-row-height:72px]"
|
||||
class="h-100! min-h-100 flex-none [--cp-table-row-height:72px] xl:h-auto! xl:min-h-0 xl:flex-1"
|
||||
:columns="accountColumns"
|
||||
:rows="accounts"
|
||||
:loading="loading"
|
||||
|
||||
@@ -240,13 +240,14 @@ const statusBars = computed(() => {
|
||||
show-plan
|
||||
title-mode="email"
|
||||
meta-position="secondary"
|
||||
meta-size="xs"
|
||||
class="min-w-0"
|
||||
>
|
||||
<template #meta>
|
||||
<ProviderIconGroup
|
||||
:provider="account.provider"
|
||||
:authentication-kind="account.authenticationKind"
|
||||
size="sm"
|
||||
size="xs"
|
||||
/>
|
||||
</template>
|
||||
</AccountIdentityCell>
|
||||
|
||||
@@ -373,10 +373,10 @@ function aggregateUsageTrend(points: DashboardTrendPoint[]) {
|
||||
)
|
||||
const uncachedInputTokensValue = inputTokensValue - cachedTokensValue
|
||||
const effectiveTokensValue = uncachedInputTokensValue + outputTokensValue
|
||||
const cacheHitRateValue = inputTokensValue ? cachedTokensValue / inputTokensValue : null
|
||||
const cacheHitRateValue = inputTokensValue ? cachedTokensValue / inputTokensValue : 0
|
||||
const successRateValue = requestsValue
|
||||
? ((requestsValue - errorsValue) / requestsValue) * 100
|
||||
: null
|
||||
: 0
|
||||
|
||||
return {
|
||||
...first,
|
||||
@@ -392,12 +392,12 @@ function aggregateUsageTrend(points: DashboardTrendPoint[]) {
|
||||
uncachedInputTokensValue,
|
||||
effectiveTokens: formatCompactNumber(effectiveTokensValue),
|
||||
effectiveTokensValue: effectiveTokensValue > 0 ? effectiveTokensValue : null,
|
||||
cacheHitRate: cacheHitRateValue === null ? '—' : formatDashboardRate(cacheHitRateValue),
|
||||
cacheHitRate: formatDashboardRate(cacheHitRateValue),
|
||||
cacheHitRateValue,
|
||||
tokensValue: inputTokensValue + outputTokensValue,
|
||||
errors: formatCompactNumber(errorsValue),
|
||||
errorsValue,
|
||||
successRate: successRateValue === null ? '—' : `${successRateValue.toFixed(1)}%`,
|
||||
successRate: `${successRateValue.toFixed(1)}%`,
|
||||
successRateValue,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { dashboardTrendView, normalizeDashboardTrendKind } from './useDashb
|
||||
import { usePreferredReducedMotion } from '@vueuse/core'
|
||||
import { computed, shallowRef, watch } from 'vue'
|
||||
|
||||
import { sampleGapBridgeSeries } from '@/components/charts/timeSeriesGap'
|
||||
import { chartTooltipStyle, tooltipIndex } from '@/components/charts/tooltip'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
|
||||
@@ -197,7 +198,7 @@ export function useRequestTrendChart(options: {
|
||||
),
|
||||
tooltipItem(
|
||||
'缓存命中',
|
||||
trendPointText(point, 'cacheHitRate'),
|
||||
zeroAwareRateText(point, 'cacheHitRate', 'inputTokensValue'),
|
||||
trendColor('缓存', '--cp-color-text-tertiary', '#94A3B8'),
|
||||
),
|
||||
tooltipItem('请求', point?.requests, palette.value.textSecondary),
|
||||
@@ -235,19 +236,19 @@ export function useRequestTrendChart(options: {
|
||||
if (name === '错误数')
|
||||
return point?.errors ?? value
|
||||
if (name === '成功率')
|
||||
return point?.successRate ?? value
|
||||
return zeroAwareRateText(point, 'successRate', 'requestsValue') ?? value
|
||||
if (name === '总请求')
|
||||
return point?.requests ?? value
|
||||
return value
|
||||
}
|
||||
|
||||
function getSeries() {
|
||||
function getSeries(): Array<BarSeriesOption | LineSeriesOption> {
|
||||
if (activeKind.value === 'usage') {
|
||||
const cacheColor = trendColor('缓存', '--cp-color-text-tertiary', '#94A3B8')
|
||||
const inputColor = trendColor('输入', '--cp-color-blue-solid', '#5983F4')
|
||||
const outputColor = trendColor('输出', '--cp-color-green-solid', '#12B981')
|
||||
return [
|
||||
lineSeries('缓存', activeSeriesValues('cachedTokensValue'), cacheColor, {
|
||||
lineSeries('缓存', seriesValues('cachedTokensValue'), cacheColor, {
|
||||
area: true,
|
||||
areaStartAlpha: '30',
|
||||
areaEndAlpha: '08',
|
||||
@@ -257,7 +258,7 @@ export function useRequestTrendChart(options: {
|
||||
width: 1.25,
|
||||
z: 1,
|
||||
}),
|
||||
lineSeries('输入', activeSeriesValues('uncachedInputTokensValue'), inputColor, {
|
||||
lineSeries('输入', seriesValues('uncachedInputTokensValue'), inputColor, {
|
||||
area: true,
|
||||
areaStartAlpha: '2A',
|
||||
areaEndAlpha: '05',
|
||||
@@ -266,7 +267,7 @@ export function useRequestTrendChart(options: {
|
||||
width: 2.3,
|
||||
z: 3,
|
||||
}),
|
||||
lineSeries('输出', activeSeriesValues('outputTokensValue'), outputColor, {
|
||||
lineSeries('输出', seriesValues('outputTokensValue'), outputColor, {
|
||||
smooth: 0.24,
|
||||
width: 2.1,
|
||||
yAxisIndex: 1,
|
||||
@@ -277,19 +278,19 @@ export function useRequestTrendChart(options: {
|
||||
|
||||
if (activeKind.value === 'latency') {
|
||||
return [
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'总耗时 P95',
|
||||
activeSeriesValues('latencyP95Ms'),
|
||||
trendColor('总耗时', '--cp-color-orange-solid', '#F59E0B'),
|
||||
{ lineType: 'dashed', smooth: 0.2, width: 1.8, z: 2 },
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'首字 P95',
|
||||
activeSeriesValues('firstTokenP95Ms'),
|
||||
trendColor('首字', '--cp-color-cyan-solid', '#13C2C2'),
|
||||
{ area: true, smooth: 0.24, width: 2.5, z: 4 },
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'吞吐 P50',
|
||||
activeSeriesValues('outputThroughputP50'),
|
||||
trendColor('吞吐', '--cp-color-green-solid', '#12B981'),
|
||||
@@ -304,9 +305,9 @@ export function useRequestTrendChart(options: {
|
||||
seriesValues('errorsValue'),
|
||||
trendColor('错误', '--cp-color-red-solid', '#EF4444'),
|
||||
),
|
||||
lineSeries(
|
||||
...gapLineSeries(
|
||||
'成功率',
|
||||
seriesValues('successRateValue'),
|
||||
activeSeriesValues('successRateValue'),
|
||||
trendColor('成功', '--cp-color-green-solid', '#12B981'),
|
||||
{ width: 2.4, yAxisIndex: 1 },
|
||||
),
|
||||
@@ -325,7 +326,7 @@ export function useRequestTrendChart(options: {
|
||||
|
||||
function activeSeriesValues(key: string) {
|
||||
return points.value.map(point =>
|
||||
Number(point.requestsValue) > 0 ? trendPointNumber(point, key) : null,
|
||||
Number(point.requestsValue) > 0 ? trendPointNumber(point, key) : 0,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -377,6 +378,26 @@ export function useRequestTrendChart(options: {
|
||||
}
|
||||
}
|
||||
|
||||
function gapLineSeries(
|
||||
name: string,
|
||||
data: (number | null)[],
|
||||
color: string,
|
||||
lineOptions: LineSeriesOptions = {},
|
||||
) {
|
||||
const primary = lineSeries(name, data, color, lineOptions)
|
||||
return [
|
||||
primary,
|
||||
...sampleGapBridgeSeries(data, {
|
||||
name,
|
||||
color,
|
||||
width: lineOptions.width,
|
||||
xAxisIndex: lineOptions.xAxisIndex,
|
||||
yAxisIndex: lineOptions.yAxisIndex,
|
||||
z: lineOptions.z,
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
function barSeries(
|
||||
name: string,
|
||||
data: (number | null)[],
|
||||
@@ -490,7 +511,7 @@ function tooltipValue(source: unknown, key: string) {
|
||||
function trendPointProperty(point: TrendPoint | undefined, key: string) {
|
||||
if (!point || !(key in point))
|
||||
return undefined
|
||||
return (point as Record<string, unknown>)[key]
|
||||
return (point as unknown as Record<string, unknown>)[key]
|
||||
}
|
||||
|
||||
function trendPointText(point: TrendPoint | undefined, key: string) {
|
||||
@@ -498,6 +519,17 @@ function trendPointText(point: TrendPoint | undefined, key: string) {
|
||||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
function zeroAwareRateText(
|
||||
point: TrendPoint | undefined,
|
||||
valueKey: string,
|
||||
activityKey: string,
|
||||
) {
|
||||
const activity = trendPointProperty(point, activityKey)
|
||||
if (typeof activity === 'number' && activity <= 0)
|
||||
return '0.0%'
|
||||
return trendPointText(point, valueKey)
|
||||
}
|
||||
|
||||
function trendPointNumber(point: TrendPoint, key: string) {
|
||||
const value = trendPointProperty(point, key)
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
||||
|
||||
@@ -6,6 +6,7 @@ import BaseModal from '@/components/base/BaseModal/index.vue'
|
||||
import { displayValue } from '../utils/detail'
|
||||
import { presentOpsError } from '../utils/opsErrorPresentation'
|
||||
import UsageDetailCodePanel from './UsageDetailCodePanel.vue'
|
||||
import UsageDetailFieldGrid from './UsageDetailFieldGrid.vue'
|
||||
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -16,23 +17,52 @@ const open = defineModel<boolean>({ default: false })
|
||||
|
||||
const presentation = computed(() => props.record ? presentOpsError(props.record) : null)
|
||||
|
||||
const fields = computed(() => [
|
||||
{ label: '账号', value: props.record?.metadata.accountLabel },
|
||||
{ label: '时间', value: props.record?.createdAtDisplay },
|
||||
const panelClass = 'min-w-0 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5'
|
||||
const panelTitleClass = 'm-0 text-cp-sm leading-none font-heavy text-cp-text-secondary'
|
||||
|
||||
const accountLabel = computed(() => props.record?.accountEmail
|
||||
|| props.record?.accountName
|
||||
|| props.record?.metadata.accountLabel
|
||||
|| props.record?.accountId)
|
||||
|
||||
const requestFields = computed(() => [
|
||||
{ label: '客户端 IP', value: props.record?.clientIp, mono: true },
|
||||
{ label: 'User-Agent', value: props.record?.userAgent, mono: true, wrap: true },
|
||||
{ label: '协议', value: props.record?.protocol, mono: true },
|
||||
{ label: '客户端传输', value: props.record?.clientTransport, mono: true },
|
||||
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId, mono: true },
|
||||
{ label: '请求 ID', value: props.record?.requestId, mono: true },
|
||||
{ label: '响应 ID', value: props.record?.responseId, mono: true },
|
||||
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, mono: true },
|
||||
])
|
||||
|
||||
const routeFields = computed(() => [
|
||||
{ label: '端点', value: props.record?.route, mono: true },
|
||||
{ label: '请求模型', value: props.record?.requestedModel, mono: true },
|
||||
{ label: '上游模型', value: props.record?.upstreamModel, mono: true },
|
||||
{ label: '记录模型', value: props.record?.model, mono: true },
|
||||
{ label: '推理强度', value: props.record?.reasoningEffort, mono: true },
|
||||
{ label: '推理预设', value: props.record?.reasoningPreset, mono: true },
|
||||
{ label: '请求类型', value: props.record?.requestKind, mono: true },
|
||||
{ label: '子代理类型', value: props.record?.subagentKind, mono: true },
|
||||
{ label: '压缩请求', value: booleanDisplay(props.record?.compact) },
|
||||
{ label: '服务档位', value: props.record?.serviceTier, mono: true },
|
||||
{ label: '上游传输', value: props.record?.transport, mono: true },
|
||||
{ label: '尝试序号', value: props.record?.attemptIndex },
|
||||
])
|
||||
|
||||
const diagnosticFields = computed(() => [
|
||||
{ label: '账号', value: accountLabel.value, mono: true },
|
||||
{ label: '时间', value: props.record?.createdAtDisplay, mono: true },
|
||||
{ label: '记录来源', value: presentation.value?.sourceLabel },
|
||||
{ label: '组件', value: presentation.value?.componentLabel },
|
||||
{ label: '操作/触发', value: props.record?.operation, mono: true },
|
||||
{ label: '失败分类', value: presentation.value?.failureClassLabel },
|
||||
{ label: '平台/类型', value: providerKindLabel(props.record) },
|
||||
{ label: '端点', value: props.record?.route },
|
||||
{ label: '模型', value: props.record?.model },
|
||||
{ label: '账号 ID', value: props.record?.accountId },
|
||||
{ label: '客户端 Key ID', value: props.record?.clientApiKeyId },
|
||||
{ label: '传输方式', value: props.record?.transport },
|
||||
{ label: '尝试序号', value: props.record?.attemptIndex },
|
||||
{ label: '耗时', value: latencyDisplay(props.record?.latencyMs) },
|
||||
{ label: '请求 ID', value: props.record?.requestId, wide: true },
|
||||
{ label: '响应 ID', value: props.record?.responseId, wide: true },
|
||||
{ label: '上游请求 ID', value: props.record?.upstreamRequestId, wide: true },
|
||||
{ label: '账号 ID', value: props.record?.accountId, mono: true },
|
||||
{ label: 'Provider 错误码', value: props.record?.providerErrorCode, mono: true },
|
||||
{ label: '聚合次数', value: props.record?.occurrenceCount },
|
||||
{ label: '耗时', value: latencyDisplay(props.record?.latencyMs), mono: true },
|
||||
])
|
||||
|
||||
const metadataText = computed(() => {
|
||||
@@ -46,6 +76,14 @@ function latencyDisplay(value: unknown) {
|
||||
return typeof value === 'number' ? `${value} ms` : '—'
|
||||
}
|
||||
|
||||
function booleanDisplay(value: boolean | null | undefined) {
|
||||
if (value === true)
|
||||
return '是'
|
||||
if (value === false)
|
||||
return '否'
|
||||
return '—'
|
||||
}
|
||||
|
||||
function providerKindLabel(record: OpsError | null) {
|
||||
if (!record?.provider)
|
||||
return '—'
|
||||
@@ -59,7 +97,7 @@ function providerKindLabel(record: OpsError | null) {
|
||||
<BaseModal
|
||||
v-model="open"
|
||||
title="错误明细"
|
||||
description="状态码、失败分类与诊断信息"
|
||||
description="请求来源、链路上下文与原始诊断"
|
||||
tone="danger"
|
||||
size="xl"
|
||||
>
|
||||
@@ -79,34 +117,37 @@ function providerKindLabel(record: OpsError | null) {
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<dl
|
||||
class="mt-3 grid grid-cols-1 gap-3 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5 sm:grid-cols-2"
|
||||
>
|
||||
<div
|
||||
v-for="field in fields"
|
||||
:key="field.label"
|
||||
class="min-w-0"
|
||||
:class="field.wide ? 'sm:col-span-2' : undefined"
|
||||
>
|
||||
<dt class="text-cp-xs leading-none font-bold text-cp-text-quaternary">
|
||||
{{ field.label }}
|
||||
</dt>
|
||||
<dd
|
||||
class="mt-1.5 mb-0 truncate font-mono text-cp-sm leading-normal font-emphasis text-cp-text"
|
||||
:title="displayValue(field.value)"
|
||||
>
|
||||
{{ displayValue(field.value) }}
|
||||
</dd>
|
||||
</div>
|
||||
</dl>
|
||||
<div class="mt-3 grid min-w-0 gap-3 lg:grid-cols-2">
|
||||
<section :class="panelClass">
|
||||
<h3 :class="panelTitleClass">
|
||||
请求追踪
|
||||
</h3>
|
||||
<UsageDetailFieldGrid :items="requestFields" />
|
||||
</section>
|
||||
|
||||
<section class="mt-3 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5">
|
||||
<section :class="panelClass">
|
||||
<h3 :class="panelTitleClass">
|
||||
模型与链路
|
||||
</h3>
|
||||
<UsageDetailFieldGrid :items="routeFields" />
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="mt-3" :class="panelClass">
|
||||
<h3 :class="panelTitleClass">
|
||||
错误事件
|
||||
</h3>
|
||||
<UsageDetailFieldGrid :items="diagnosticFields" />
|
||||
</section>
|
||||
|
||||
<section class="mt-3" :class="panelClass">
|
||||
<UsageDetailCodePanel title="原始诊断" max-height="260px" :content="record.message" />
|
||||
</section>
|
||||
|
||||
<section
|
||||
v-if="metadataText"
|
||||
class="mt-3 rounded-cp-card bg-cp-fill-quaternary px-4 py-3.5"
|
||||
class="mt-3"
|
||||
:class="panelClass"
|
||||
>
|
||||
<UsageDetailCodePanel title="Metadata" max-height="260px" :content="metadataText" />
|
||||
</section>
|
||||
|
||||
@@ -8,11 +8,13 @@ import BaseIconButton from '@/components/base/BaseIconButton.vue'
|
||||
import BaseInput from '@/components/base/BaseInput.vue'
|
||||
import BaseTablePagination from '@/components/base/BaseTable/BaseTablePagination.vue'
|
||||
import BaseTable from '@/components/base/BaseTable/index.vue'
|
||||
import ProviderIconGroup from '@/components/ProviderIconGroup.vue'
|
||||
import { useOpsErrorsTable } from '../composables/useOpsErrorsTable'
|
||||
import { opsErrorColumns } from '../constants'
|
||||
import { presentOpsError } from '../utils/opsErrorPresentation'
|
||||
import OpsErrorDetailModal from './OpsErrorDetailModal.vue'
|
||||
import UsageStatusCodeBadge from './UsageStatusCodeBadge.vue'
|
||||
import UsageTransportBadge from './UsageTransportBadge.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
timeRangeParams: UsageTimeRangeParams
|
||||
@@ -46,6 +48,30 @@ function failureClassText(record: OpsError) {
|
||||
function errorSummary(record: OpsError) {
|
||||
return presentOpsError(record).summary
|
||||
}
|
||||
|
||||
function accountText(record: OpsError) {
|
||||
return record.accountEmail
|
||||
|| record.accountName
|
||||
|| record.metadata.accountLabel
|
||||
|| record.accountId
|
||||
|| '—'
|
||||
}
|
||||
|
||||
function modelText(record: OpsError) {
|
||||
return record.requestedModel || record.model || record.upstreamModel || '—'
|
||||
}
|
||||
|
||||
function modelTitle(record: OpsError) {
|
||||
const requested = record.requestedModel
|
||||
const upstream = record.upstreamModel
|
||||
return requested && upstream && requested !== upstream
|
||||
? `${requested} → ${upstream}`
|
||||
: modelText(record)
|
||||
}
|
||||
|
||||
function reasoningText(record: OpsError) {
|
||||
return record.reasoningPreset || record.reasoningEffort || '—'
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -111,6 +137,12 @@ function errorSummary(record: OpsError) {
|
||||
{{ failureClassText(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #provider="{ row }">
|
||||
<ProviderIconGroup
|
||||
:provider="String(row.provider || '')"
|
||||
:authentication-kind="row.authenticationKind"
|
||||
/>
|
||||
</template>
|
||||
<template #message="{ row }">
|
||||
<span class="block max-w-full truncate text-cp-sm font-emphasis text-cp-text" :title="errorSummary(row)">
|
||||
{{ errorSummary(row) }}
|
||||
@@ -119,9 +151,41 @@ function errorSummary(record: OpsError) {
|
||||
<template #accountId="{ row }">
|
||||
<span
|
||||
class="block max-w-full truncate font-mono text-cp-sm font-bold text-cp-text"
|
||||
:title="row.accountId || ''"
|
||||
:title="accountText(row)"
|
||||
>
|
||||
{{ row.metadata.accountLabel || '—' }}
|
||||
{{ accountText(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #model="{ row }">
|
||||
<span
|
||||
class="block max-w-full truncate font-mono text-cp-sm font-bold text-cp-text"
|
||||
:title="modelTitle(row)"
|
||||
>
|
||||
{{ modelText(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #reasoningEffort="{ row }">
|
||||
<span class="text-cp-sm font-bold text-cp-text-secondary">
|
||||
{{ reasoningText(row) }}
|
||||
</span>
|
||||
</template>
|
||||
<template #transport="{ row }">
|
||||
<UsageTransportBadge :transport="row.transport" />
|
||||
</template>
|
||||
<template #clientTransport="{ row }">
|
||||
<UsageTransportBadge :transport="row.clientTransport" />
|
||||
</template>
|
||||
<template #clientIp="{ row }">
|
||||
<span
|
||||
class="inline-flex h-6 max-w-full items-center rounded-full bg-cp-blue-bg px-2.5 font-mono text-cp-sm leading-none font-bold text-cp-blue-text-on-bg"
|
||||
:title="row.clientIp || '—'"
|
||||
>
|
||||
<span class="min-w-0 truncate">{{ row.clientIp || '—' }}</span>
|
||||
</span>
|
||||
</template>
|
||||
<template #userAgent="{ row }">
|
||||
<span class="block max-w-full wrap-break-word whitespace-normal font-mono text-cp-sm leading-[1.4] font-emphasis text-cp-text-secondary">
|
||||
{{ row.userAgent || '—' }}
|
||||
</span>
|
||||
</template>
|
||||
<template #actions="{ row }">
|
||||
|
||||
@@ -7,6 +7,10 @@ import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseEmpty from '@/components/base/BaseEmpty.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
import BaseChart from '@/components/charts/BaseChart.vue'
|
||||
import {
|
||||
requestActivityByBucket,
|
||||
zeroInactiveValues,
|
||||
} from '@/components/charts/timeSeriesGap'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { formatLocalizedCompactNumber as formatCompactNumber } from '@/utils/number'
|
||||
|
||||
@@ -14,10 +18,11 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageLineSeries,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import {
|
||||
@@ -28,11 +33,13 @@ import {
|
||||
} from '../utils/format'
|
||||
|
||||
type Cost = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['cost']
|
||||
type Activity = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['health']['points']
|
||||
type UsageChartPalette = ReturnType<typeof useChartPalette>['palette']['value']
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
cost: Cost
|
||||
activity: Activity
|
||||
loading?: boolean
|
||||
}>(),
|
||||
{
|
||||
@@ -43,6 +50,10 @@ const props = withDefaults(
|
||||
const activeView = shallowRef('cost')
|
||||
const { palette } = useChartPalette()
|
||||
const points = computed(() => props.cost.points)
|
||||
const requestActivity = computed(() => requestActivityByBucket(
|
||||
points.value.map(point => point.bucket),
|
||||
props.activity,
|
||||
))
|
||||
const hasNoCacheCost = computed(() =>
|
||||
points.value.some(point => point.noCacheCost != null),
|
||||
)
|
||||
@@ -74,6 +85,7 @@ const hasData = computed(() => {
|
||||
const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const legend = legendNames()
|
||||
const series = chartSeries(theme)
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -95,7 +107,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
min: activeView.value === 'cache' ? 0 : undefined,
|
||||
max: activeView.value === 'cache' ? 1 : undefined,
|
||||
}),
|
||||
series: chartSeries(theme),
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -119,9 +131,12 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
const chartPoints = points.value
|
||||
if (activeView.value === 'savings') {
|
||||
return [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'缓存节省',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.cacheSavings)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.cacheSavings)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.success,
|
||||
{ area: 'strong' },
|
||||
),
|
||||
@@ -130,12 +145,12 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
|
||||
if (activeView.value === 'cache') {
|
||||
return [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'缓存 Token 占比',
|
||||
chartPoints.map(point => point.cachedTokenRate),
|
||||
theme.normal,
|
||||
),
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'命中请求率',
|
||||
chartPoints.map(point => point.cacheHitRequestRate),
|
||||
theme.success,
|
||||
@@ -144,18 +159,24 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
}
|
||||
|
||||
const series = [
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'实际费用',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.estimatedCost)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.estimatedCost)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.success,
|
||||
{ area: 'strong' },
|
||||
),
|
||||
]
|
||||
if (hasNoCacheCost.value) {
|
||||
series.push(
|
||||
usageLineSeries(
|
||||
...usageGapAwareLineSeries(
|
||||
'无缓存费用',
|
||||
chartPoints.map(point => decimalDisplayNumber(point.noCacheCost)),
|
||||
zeroInactiveValues(
|
||||
chartPoints.map(point => decimalDisplayNumber(point.noCacheCost)),
|
||||
requestActivity.value,
|
||||
),
|
||||
theme.textMuted,
|
||||
),
|
||||
)
|
||||
@@ -165,29 +186,48 @@ function chartSeries(theme: UsageChartPalette): LineSeriesOption[] {
|
||||
|
||||
function formatTooltip(params: unknown) {
|
||||
const rows = tooltipRows(params)
|
||||
const point = points.value[tooltipIndex(rows[0])]
|
||||
const pointIndex = tooltipIndex(rows[0])
|
||||
const point = points.value[pointIndex]
|
||||
if (!point)
|
||||
return ''
|
||||
const theme = palette.value
|
||||
|
||||
if (activeView.value === 'savings') {
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`缓存节省: ${formatUsd(point.cacheSavings)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem(
|
||||
'缓存节省',
|
||||
formatUsd(costAmount(point.cacheSavings, pointIndex)),
|
||||
theme.success,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
if (activeView.value === 'cache') {
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`缓存 Token 占比: ${formatPercent(point.cachedTokenRate)}`,
|
||||
`命中请求率: ${formatPercent(point.cacheHitRequestRate)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem('缓存 Token 占比', formatPercent(point.cachedTokenRate), theme.normal),
|
||||
usageTooltipItem('命中请求率', formatPercent(point.cacheHitRequestRate), theme.success),
|
||||
])
|
||||
}
|
||||
|
||||
const lines = [
|
||||
`实际费用: ${formatUsd(point.estimatedCost)}`,
|
||||
usageTooltipItem(
|
||||
'实际费用',
|
||||
formatUsd(costAmount(point.estimatedCost, pointIndex)),
|
||||
theme.success,
|
||||
),
|
||||
]
|
||||
if (point.noCacheCost != null)
|
||||
lines.push(`无缓存费用: ${formatUsd(point.noCacheCost)}`)
|
||||
return usageTooltipContent(palette.value, point.label, lines)
|
||||
if (point.noCacheCost != null) {
|
||||
lines.push(usageTooltipItem(
|
||||
'无缓存费用',
|
||||
formatUsd(costAmount(point.noCacheCost, pointIndex)),
|
||||
theme.textMuted,
|
||||
))
|
||||
}
|
||||
return usageTooltipContent(theme, point.label, lines)
|
||||
}
|
||||
|
||||
function costAmount(value: string | null, pointIndex: number) {
|
||||
return value ?? (requestActivity.value[pointIndex] === false ? '0' : null)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -196,7 +236,7 @@ function formatTooltip(params: unknown) {
|
||||
as="article"
|
||||
title="成本效率"
|
||||
description="实际费用、缓存节省、服务层溢价与单位成本"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #actions>
|
||||
<BaseSegmented v-model="activeView" label="成本视图" :options="viewOptions" :disabled="loading" class="w-50" />
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import type { EChartsOption } from 'echarts'
|
||||
import type { BarSeriesOption, EChartsOption, LineSeriesOption } from 'echarts'
|
||||
import type { getUsageRecordInsightsOverview } from '@/api'
|
||||
import { BarChart } from 'echarts/charts'
|
||||
import { use } from 'echarts/core'
|
||||
@@ -15,9 +15,11 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import { formatPercent } from '../utils/format'
|
||||
@@ -60,6 +62,26 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const chartPoints = points.value
|
||||
const activePointCount = chartPoints.filter(point => requestCount(point) > 0).length
|
||||
const successRateSeries = usageGapAwareLineSeries(
|
||||
'服务成功率',
|
||||
chartPoints.map(serviceSuccessRate),
|
||||
theme.success,
|
||||
{ yAxisIndex: 1 },
|
||||
)
|
||||
const series: Array<BarSeriesOption | LineSeriesOption> = [
|
||||
outcomeSeries('成功', chartPoints.map(point => point.successRequests), theme.success),
|
||||
outcomeSeries('服务失败', chartPoints.map(point => point.failedRequests), theme.danger),
|
||||
outcomeSeries('取消', chartPoints.map(point => point.cancelledRequests), theme.info),
|
||||
outcomeSeries('未完成', chartPoints.map(point => point.incompleteRequests), theme.warning),
|
||||
outcomeSeries('调用方错误', chartPoints.map(point => point.callerErrorRequests), theme.normal),
|
||||
...successRateSeries.map((item, index) => index === 0
|
||||
? {
|
||||
...item,
|
||||
symbol: activePointCount <= 16 ? 'circle' : 'none',
|
||||
symbolSize: 4,
|
||||
}
|
||||
: item),
|
||||
]
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -85,25 +107,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
splitLine: false,
|
||||
}),
|
||||
],
|
||||
series: [
|
||||
outcomeSeries('成功', chartPoints.map(point => point.successRequests), theme.success),
|
||||
outcomeSeries('服务失败', chartPoints.map(point => point.failedRequests), theme.danger),
|
||||
outcomeSeries('取消', chartPoints.map(point => point.cancelledRequests), theme.info),
|
||||
outcomeSeries('未完成', chartPoints.map(point => point.incompleteRequests), theme.warning),
|
||||
outcomeSeries('调用方错误', chartPoints.map(point => point.callerErrorRequests), theme.normal),
|
||||
{
|
||||
name: '服务成功率',
|
||||
type: 'line',
|
||||
yAxisIndex: 1,
|
||||
data: chartPoints.map(serviceSuccessRate),
|
||||
connectNulls: true,
|
||||
smooth: 0.25,
|
||||
symbol: activePointCount <= 16 ? 'circle' : 'none',
|
||||
symbolSize: 4,
|
||||
lineStyle: { color: theme.success, width: 2.2 },
|
||||
itemStyle: { color: theme.success },
|
||||
},
|
||||
],
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -113,7 +117,9 @@ function requestCount(point: HealthPoint) {
|
||||
|
||||
function serviceSuccessRate(point: HealthPoint) {
|
||||
const total = Math.max(0, point.successRequests) + Math.max(0, point.failedRequests)
|
||||
return total > 0 ? Math.max(0, point.successRequests ?? 0) / total : null
|
||||
if (total > 0)
|
||||
return Math.max(0, point.successRequests ?? 0) / total
|
||||
return requestCount(point) === 0 ? 0 : null
|
||||
}
|
||||
|
||||
function outcomeSeries(name: string, data: number[], color: string) {
|
||||
@@ -134,15 +140,20 @@ function formatTooltip(params: unknown) {
|
||||
return ''
|
||||
|
||||
const successRate = serviceSuccessRate(point)
|
||||
const theme = palette.value
|
||||
|
||||
return usageTooltipContent(palette.value, point.label, [
|
||||
`请求量: ${formatCompactNumber(requestCount(point))}`,
|
||||
`成功: ${formatCompactNumber(point.successRequests)}`,
|
||||
`服务失败: ${formatCompactNumber(point.failedRequests)}`,
|
||||
`取消: ${formatCompactNumber(point.cancelledRequests)}`,
|
||||
`未完成: ${formatCompactNumber(point.incompleteRequests)}`,
|
||||
`调用方错误: ${formatCompactNumber(point.callerErrorRequests)}`,
|
||||
`服务成功率: ${successRate == null ? '无服务结果' : formatPercent(successRate)}`,
|
||||
return usageTooltipContent(theme, point.label, [
|
||||
usageTooltipItem('请求量', formatCompactNumber(requestCount(point)), theme.textSecondary),
|
||||
usageTooltipItem('成功', formatCompactNumber(point.successRequests), theme.success),
|
||||
usageTooltipItem('服务失败', formatCompactNumber(point.failedRequests), theme.danger),
|
||||
usageTooltipItem('取消', formatCompactNumber(point.cancelledRequests), theme.info),
|
||||
usageTooltipItem('未完成', formatCompactNumber(point.incompleteRequests), theme.warning),
|
||||
usageTooltipItem('调用方错误', formatCompactNumber(point.callerErrorRequests), theme.normal),
|
||||
usageTooltipItem(
|
||||
'服务成功率',
|
||||
successRate == null ? '无服务结果' : formatPercent(successRate),
|
||||
theme.success,
|
||||
),
|
||||
])
|
||||
}
|
||||
</script>
|
||||
@@ -152,7 +163,7 @@ function formatTooltip(params: unknown) {
|
||||
as="article"
|
||||
title="请求健康"
|
||||
:description="`按${granularityText}区分服务结果、取消、未完成与调用方错误`"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #body>
|
||||
<div class="grid min-h-66" :class="hasData ? 'gap-3' : 'h-full'">
|
||||
|
||||
@@ -28,7 +28,7 @@ const diagnosticDimension = defineModel('diagnosticDimension', {
|
||||
|
||||
<template>
|
||||
<section
|
||||
class="mt-5 grid grid-cols-1 gap-3 xl:auto-rows-[24.25rem] xl:grid-cols-2"
|
||||
class="mt-5 grid grid-cols-1 gap-3 xl:auto-rows-97 xl:grid-cols-2"
|
||||
aria-label="使用统计观测"
|
||||
>
|
||||
<UsageHealthCard
|
||||
@@ -43,8 +43,16 @@ const diagnosticDimension = defineModel('diagnosticDimension', {
|
||||
:loading="loading"
|
||||
/>
|
||||
|
||||
<UsagePerformanceCard :performance="overview.performance" :loading="loading" />
|
||||
<UsagePerformanceCard
|
||||
:performance="overview.performance"
|
||||
:activity="overview.health.points"
|
||||
:loading="loading"
|
||||
/>
|
||||
|
||||
<UsageCostCard :cost="overview.cost" :loading="loading" />
|
||||
<UsageCostCard
|
||||
:cost="overview.cost"
|
||||
:activity="overview.health.points"
|
||||
:loading="loading"
|
||||
/>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
@@ -7,6 +7,7 @@ import BaseCard from '@/components/base/BaseCard.vue'
|
||||
import BaseEmpty from '@/components/base/BaseEmpty.vue'
|
||||
import BaseSegmented from '@/components/base/BaseSegmented.vue'
|
||||
import BaseChart from '@/components/charts/BaseChart.vue'
|
||||
import { requestActivityByBucket, zeroInactiveValues } from '@/components/charts/timeSeriesGap'
|
||||
import { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { formatLocalizedCompactNumber as formatCompactNumber } from '@/utils/number'
|
||||
|
||||
@@ -14,20 +15,23 @@ import {
|
||||
tooltipIndex,
|
||||
tooltipRows,
|
||||
usageCategoryAxis,
|
||||
usageGapAwareLineSeries,
|
||||
usageLegend,
|
||||
usageLineSeries,
|
||||
usageTooltip,
|
||||
usageTooltipContent,
|
||||
usageTooltipItem,
|
||||
usageValueAxis,
|
||||
} from '../utils/chart'
|
||||
import { formatDuration, formatDurationAxis, formatPercent } from '../utils/format'
|
||||
|
||||
type Performance = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['performance']
|
||||
type PerformancePoint = Performance['points'][number]
|
||||
type Activity = Awaited<ReturnType<typeof getUsageRecordInsightsOverview>>['health']['points']
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
performance: Performance
|
||||
activity: Activity
|
||||
loading?: boolean
|
||||
}>(),
|
||||
{
|
||||
@@ -38,6 +42,10 @@ const props = withDefaults(
|
||||
const activeView = shallowRef('total')
|
||||
const { palette } = useChartPalette()
|
||||
const performancePoints = computed<PerformancePoint[]>(() => props.performance.points)
|
||||
const requestActivity = computed(() => requestActivityByBucket(
|
||||
performancePoints.value.map(point => point.bucket),
|
||||
props.activity,
|
||||
))
|
||||
|
||||
const viewOptions = [
|
||||
{ label: '总耗时', value: 'total' },
|
||||
@@ -54,14 +62,28 @@ const seriesLabels = computed(() => {
|
||||
return ['P50', 'P95', 'P99'] as const
|
||||
})
|
||||
|
||||
const selectedPoints = computed(() =>
|
||||
performancePoints.value.map(point => ({
|
||||
const selectedPoints = computed(() => {
|
||||
const points = performancePoints.value
|
||||
const active = requestActivity.value
|
||||
const firstValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'first')),
|
||||
active,
|
||||
)
|
||||
const secondValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'second')),
|
||||
active,
|
||||
)
|
||||
const thirdValues = zeroInactiveValues(
|
||||
points.map(point => seriesValue(point, 'third')),
|
||||
active,
|
||||
)
|
||||
return points.map((point, index) => ({
|
||||
point,
|
||||
first: seriesValue(point, 'first'),
|
||||
second: seriesValue(point, 'second'),
|
||||
third: seriesValue(point, 'third'),
|
||||
})),
|
||||
)
|
||||
first: firstValues[index] ?? null,
|
||||
second: secondValues[index] ?? null,
|
||||
third: thirdValues[index] ?? null,
|
||||
}))
|
||||
})
|
||||
|
||||
const summaryMetrics = computed(() => {
|
||||
const performance = props.performance
|
||||
@@ -98,6 +120,25 @@ const hasData = computed(
|
||||
const chartOption = computed<EChartsOption>(() => {
|
||||
const theme = palette.value
|
||||
const points = selectedPoints.value
|
||||
const series = [
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[0],
|
||||
points.map(point => point.first),
|
||||
theme.info,
|
||||
{ area: 'subtle' },
|
||||
),
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[1],
|
||||
points.map(point => point.second),
|
||||
theme.warning,
|
||||
),
|
||||
...usageGapAwareLineSeries(
|
||||
seriesLabels.value[2],
|
||||
points.map(point => point.third),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
{ yAxisIndex: activeView.value === 'scheduling' ? 1 : 0 },
|
||||
),
|
||||
]
|
||||
|
||||
return {
|
||||
animationDuration: 240,
|
||||
@@ -130,27 +171,7 @@ const chartOption = computed<EChartsOption>(() => {
|
||||
? value => `${formatCompactNumber(value)}/s`
|
||||
: formatDurationAxis,
|
||||
),
|
||||
series: [
|
||||
usageLineSeries(
|
||||
seriesLabels.value[0],
|
||||
points.map(point => point.first),
|
||||
theme.info,
|
||||
{ area: 'subtle' },
|
||||
),
|
||||
usageLineSeries(
|
||||
seriesLabels.value[1],
|
||||
points.map(point => point.second),
|
||||
theme.warning,
|
||||
),
|
||||
{
|
||||
...usageLineSeries(
|
||||
seriesLabels.value[2],
|
||||
points.map(point => point.third),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
),
|
||||
yAxisIndex: activeView.value === 'scheduling' ? 1 : 0,
|
||||
},
|
||||
],
|
||||
series,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -180,10 +201,23 @@ function formatTooltip(params: unknown) {
|
||||
if (!selected)
|
||||
return ''
|
||||
|
||||
return usageTooltipContent(palette.value, selected.point.label, [
|
||||
`${seriesLabels.value[0]}: ${formatSeriesValue(selected.first, 'first')}`,
|
||||
`${seriesLabels.value[1]}: ${formatSeriesValue(selected.second, 'second')}`,
|
||||
`${seriesLabels.value[2]}: ${formatSeriesValue(selected.third, 'third')}`,
|
||||
const theme = palette.value
|
||||
return usageTooltipContent(theme, selected.point.label, [
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[0],
|
||||
formatSeriesValue(selected.first, 'first'),
|
||||
theme.info,
|
||||
),
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[1],
|
||||
formatSeriesValue(selected.second, 'second'),
|
||||
theme.warning,
|
||||
),
|
||||
usageTooltipItem(
|
||||
seriesLabels.value[2],
|
||||
formatSeriesValue(selected.third, 'third'),
|
||||
activeView.value === 'scheduling' ? theme.success : theme.danger,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
@@ -205,7 +239,7 @@ function formatThroughput(value: number | null) {
|
||||
as="article"
|
||||
title="响应速度"
|
||||
description="延迟、吞吐与调度分位"
|
||||
class="h-full min-h-90"
|
||||
class="min-h-90 xl:h-full"
|
||||
>
|
||||
<template #actions>
|
||||
<BaseSegmented v-model="activeView" label="性能指标" :options="viewOptions" :disabled="loading" class="w-68" />
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { UsageTimeRangeParams } from './useUsageTimeRange'
|
||||
import { watchDebounced } from '@vueuse/core'
|
||||
|
||||
import { computed, onMounted, shallowRef, watch } from 'vue'
|
||||
import { createOpsErrorsPager } from '@/api'
|
||||
import { getOpsErrors } from '@/api'
|
||||
import { toast } from '@/components/base/BaseToast'
|
||||
import { useStablePagedQuery } from '@/composables/useStablePagedQuery'
|
||||
import { errorMessage, withMinimumDuration } from '@/utils/async'
|
||||
@@ -13,11 +13,9 @@ export function useOpsErrorsTable(timeRangeParams: Readonly<Ref<UsageTimeRangePa
|
||||
const searchQuery = shallowRef('')
|
||||
const failureClass = shallowRef('')
|
||||
const route = shallowRef('')
|
||||
const pager = createOpsErrorsPager()
|
||||
|
||||
const query = useStablePagedQuery({
|
||||
initialPageSize: 10,
|
||||
load: ({ currentPage, pageSize }) => pager.load({
|
||||
load: ({ currentPage, pageSize }) => getOpsErrors({
|
||||
currentPage,
|
||||
pageSize,
|
||||
search: searchQuery.value.trim() || undefined,
|
||||
|
||||
@@ -5,9 +5,9 @@ import { watchDebounced } from '@vueuse/core'
|
||||
|
||||
import { computed, onMounted, onScopeDispose, shallowRef, watch } from 'vue'
|
||||
import {
|
||||
createUsageRecordsPager,
|
||||
getUsageRecordInsightsDiagnostics,
|
||||
getUsageRecordInsightsOverview,
|
||||
getUsageRecords,
|
||||
getUsageRecordSummary,
|
||||
} from '@/api'
|
||||
import { toast } from '@/components/base/BaseToast'
|
||||
@@ -26,13 +26,6 @@ interface UsageLoadOptions {
|
||||
background?: boolean
|
||||
}
|
||||
|
||||
interface UsageTableParams {
|
||||
startTime: string
|
||||
endTime: string
|
||||
provider?: string
|
||||
search?: string
|
||||
}
|
||||
|
||||
export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
const loading = shallowRef(true)
|
||||
const analyticsLoading = shallowRef(true)
|
||||
@@ -49,7 +42,6 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
})
|
||||
const refreshingList = shallowRef(false)
|
||||
const diagnosticDimension = shallowRef('model')
|
||||
const pager = createUsageRecordsPager()
|
||||
let loadRequestId = 0
|
||||
let diagnosticRequestId = 0
|
||||
const scopedParams = () => ({
|
||||
@@ -70,21 +62,6 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
totalRecords.value = 0
|
||||
}
|
||||
|
||||
async function loadTablePage(
|
||||
params: UsageTableParams,
|
||||
targetPage: number,
|
||||
requestId: number,
|
||||
) {
|
||||
const result = await pager.load({
|
||||
currentPage: targetPage,
|
||||
pageSize: pageSize.value,
|
||||
...params,
|
||||
})
|
||||
if (requestId !== loadRequestId)
|
||||
return null
|
||||
return { result, page: result.currentPage }
|
||||
}
|
||||
|
||||
async function loadUsageRecords(loadOptions: UsageLoadOptions = {}) {
|
||||
const { scope = 'all', background = false } = loadOptions
|
||||
const requestId = ++loadRequestId
|
||||
@@ -114,8 +91,11 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
...tableTimeRangeParams.value,
|
||||
...filterParams(),
|
||||
}
|
||||
const requestedPage = currentPage.value
|
||||
const resultPromise = loadTablePage(tableParams, requestedPage, requestId)
|
||||
const resultPromise = getUsageRecords({
|
||||
currentPage: currentPage.value,
|
||||
pageSize: pageSize.value,
|
||||
...tableParams,
|
||||
})
|
||||
const analyticsPromise
|
||||
= scope === 'all'
|
||||
? loadUsageAnalytics(globalParams)
|
||||
@@ -123,10 +103,9 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
summary: summary.value,
|
||||
insights: insights.value,
|
||||
})
|
||||
const [tableResult, nextAnalytics] = await Promise.all([resultPromise, analyticsPromise])
|
||||
if (requestId !== loadRequestId || tableResult == null)
|
||||
const [result, nextAnalytics] = await Promise.all([resultPromise, analyticsPromise])
|
||||
if (requestId !== loadRequestId)
|
||||
return
|
||||
const { result, page: resultPage } = tableResult
|
||||
|
||||
records.value = result.items
|
||||
summary.value = nextAnalytics.summary
|
||||
@@ -137,8 +116,9 @@ export function useUsageRecordsTable(options: UseUsageRecordsTableOptions) {
|
||||
? nextAnalytics.insights.diagnostics
|
||||
: insights.value.diagnostics,
|
||||
}
|
||||
pageSize.value = result.pageSize
|
||||
totalRecords.value = result.total
|
||||
currentPage.value = resultPage
|
||||
currentPage.value = result.currentPage
|
||||
}
|
||||
catch (error: unknown) {
|
||||
if (requestId !== loadRequestId)
|
||||
|
||||
@@ -35,15 +35,21 @@ export const usageRecordColumns = defineTableColumns<UsageDisplayRecord>([
|
||||
|
||||
export const opsErrorColumns = defineTableColumns<OpsErrorRow>([
|
||||
{ key: 'accountId', label: '账号', kind: 'mono', size: '3xl' },
|
||||
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
|
||||
{ key: 'provider', label: '平台/类型', kind: 'custom', size: 'sm' },
|
||||
{ key: 'message', label: '错误摘要', kind: 'custom', size: '3xl' },
|
||||
{ key: 'failureClass', label: '失败分类', kind: 'mono' },
|
||||
{ key: 'upstreamStatusCode', label: '上游状态', kind: 'status', size: 'sm' },
|
||||
{ key: 'clientStatusCode', label: '客户端状态', kind: 'status', size: 'md' },
|
||||
{ key: 'failureClass', label: '失败分类', kind: 'mono' },
|
||||
{ key: 'kind', label: '事件', kind: 'mono' },
|
||||
{ key: 'model', label: '模型', kind: 'custom', size: 'xl' },
|
||||
{ key: 'reasoningEffort', label: '推理强度', kind: 'custom' },
|
||||
{ key: 'route', label: '端点', kind: 'mono' },
|
||||
{ key: 'model', label: '模型', kind: 'mono' },
|
||||
{ key: 'transport', label: '上游', kind: 'custom', size: 'md' },
|
||||
{ key: 'clientTransport', label: '接入', kind: 'custom', size: 'md' },
|
||||
{ key: 'createdAtDisplay', label: '时间', kind: 'datetime' },
|
||||
{ key: 'requestId', label: '请求 ID', kind: 'mono', size: '2xl' },
|
||||
{ key: 'message', label: '错误摘要', kind: 'custom', size: '3xl' },
|
||||
{ key: 'kind', label: '事件', kind: 'mono' },
|
||||
{ key: 'clientIp', label: 'IP', kind: 'custom', size: '3xl' },
|
||||
{ key: 'userAgent', label: 'User-Agent', kind: 'custom', size: '4xl' },
|
||||
{ key: 'actions', label: '操作', kind: 'actions', size: 'sm' },
|
||||
])
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { LineSeriesOption } from 'echarts'
|
||||
import type { useChartPalette } from '@/composables/useChartPalette'
|
||||
import { sampleGapBridgeSeries } from '@/components/charts/timeSeriesGap'
|
||||
import { chartTooltipStyle } from '@/components/charts/tooltip'
|
||||
import { escapeTooltip } from './format'
|
||||
|
||||
@@ -9,6 +10,8 @@ type UsageAreaStrength = 'strong' | 'subtle'
|
||||
interface UsageLineSeriesOptions {
|
||||
stack?: string
|
||||
area?: UsageAreaStrength | false
|
||||
xAxisIndex?: number
|
||||
yAxisIndex?: number
|
||||
}
|
||||
|
||||
const areaAlpha: Record<UsageAreaStrength, readonly [string, string]> = {
|
||||
@@ -36,6 +39,8 @@ export function usageLineSeries(
|
||||
data: data.map(value => value ?? null),
|
||||
connectNulls: false,
|
||||
stack: options.stack,
|
||||
xAxisIndex: options.xAxisIndex ?? 0,
|
||||
yAxisIndex: options.yAxisIndex ?? 0,
|
||||
smooth: true,
|
||||
showSymbol: showSymbols,
|
||||
showAllSymbol: showSymbols,
|
||||
@@ -61,6 +66,26 @@ export function usageLineSeries(
|
||||
}
|
||||
}
|
||||
|
||||
export function usageGapAwareLineSeries(
|
||||
name: string,
|
||||
data: Array<number | null | undefined>,
|
||||
color: string,
|
||||
options: UsageLineSeriesOptions = {},
|
||||
) {
|
||||
const primary = usageLineSeries(name, data, color, options)
|
||||
return [
|
||||
primary,
|
||||
...sampleGapBridgeSeries(data, {
|
||||
name,
|
||||
color,
|
||||
xAxisIndex: options.xAxisIndex,
|
||||
yAxisIndex: options.yAxisIndex,
|
||||
width: 2.2,
|
||||
z: 2,
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
export function usageTooltip(
|
||||
theme: UsageChartPalette,
|
||||
formatter: (params: unknown) => string,
|
||||
@@ -81,6 +106,10 @@ export function usageTooltipContent(
|
||||
return `<div style="margin:0 0 7px;padding:0 0 7px;border-bottom:1px solid ${theme.divider};color:${theme.textPrimary};font-family:'JetBrains Mono Variable','JetBrains Mono',monospace;font-size:11px;font-weight:750;line-height:1.2">${title}</div><div style="line-height:1.55">${lines.join('<br/>')}</div>`
|
||||
}
|
||||
|
||||
export function usageTooltipItem(label: string, value: string, color: string) {
|
||||
return `<span style="display:inline-block;width:7px;height:7px;margin-right:6px;border-radius:999px;background:${escapeTooltip(color)}"></span>${escapeTooltip(label)}: ${escapeTooltip(value)}`
|
||||
}
|
||||
|
||||
export function usageCategoryAxis(labels: string[], theme: UsageChartPalette) {
|
||||
return {
|
||||
type: 'category' as const,
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "3.3.10"
|
||||
version: "3.3.14"
|
||||
|
||||
Reference in New Issue
Block a user