mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
447869860a | ||
|
|
815c93a715 | ||
|
|
5843091969 | ||
|
|
1280c437b4 | ||
|
|
d965bc9050 | ||
|
|
bd9b017ed3 | ||
|
|
338cde1505 | ||
|
|
b25f96559a | ||
|
|
a8a96d8919 | ||
|
|
7b8339ede3 | ||
|
|
f439a30612 | ||
|
|
8572873c5f | ||
|
|
57c8ef358f | ||
|
|
70ab8ebc8d | ||
|
|
14ea5234ce | ||
|
|
f8480f3bdc | ||
|
|
52b8103bc9 | ||
|
|
dacdd3f0e2 | ||
|
|
ac36f5bc13 | ||
|
|
e0a022518c | ||
|
|
37f7a55427 | ||
|
|
f6085206eb | ||
|
|
9813c86055 | ||
|
|
260f775039 | ||
|
|
1424cea3a9 | ||
|
|
787ad28424 | ||
|
|
db23560b0e | ||
|
|
d700df8a4f | ||
|
|
c9adfd9a2b | ||
|
|
654b6baf89 |
@@ -42,8 +42,11 @@ else
|
||||
mkdir -p "$(dirname "$directory")"
|
||||
temporary="$(mktemp -d "${directory}.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
mkdir "$temporary/context"
|
||||
"${command[@]}" --output "type=local,dest=$temporary/snapshot" "$temporary/context"
|
||||
mkdir "$temporary"/{context,snapshot}
|
||||
# tar 流避开 local exporter 的逐文件同步阻塞,直接解包以免额外落盘整份归档。
|
||||
# https://github.com/moby/buildkit/issues/2950
|
||||
"${command[@]}" --output type=tar,dest=- "$temporary/context" \
|
||||
| tar -xf - -C "$temporary/snapshot"
|
||||
if usable "$temporary/snapshot"; then
|
||||
rm -rf "$directory"
|
||||
mv "$temporary/snapshot" "$directory"
|
||||
|
||||
Generated
+33
-2
@@ -145,6 +145,18 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "3.4.2"
|
||||
@@ -931,6 +943,23 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a"
|
||||
dependencies = [
|
||||
"compression-core",
|
||||
"flate2",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-core"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
|
||||
|
||||
[[package]]
|
||||
name = "config"
|
||||
version = "0.15.27"
|
||||
@@ -1717,6 +1746,7 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"regex",
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -3750,9 +3780,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -4743,6 +4773,7 @@ version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
|
||||
@@ -107,7 +107,7 @@ async fn execute(
|
||||
let settings = gateway_admin::initialize_settings(
|
||||
store.admin_ports().settings(),
|
||||
core.snapshot_control(),
|
||||
providers.admin,
|
||||
providers.admin.clone(),
|
||||
std::sync::Arc::new(gateway_host::pricing::ModelsDevPricing),
|
||||
);
|
||||
let mut services = gateway_admin::public_service::Registry::new(service_middleware(
|
||||
@@ -115,6 +115,12 @@ async fn execute(
|
||||
&plugin_runtime,
|
||||
));
|
||||
or_shutdown!(plugin_runtime, services.register_settings(&settings));
|
||||
let usage_data = gateway_admin::initialize_usage_data(
|
||||
store.admin_ports().usage_data(),
|
||||
store.admin_ports().settings(),
|
||||
providers.admin,
|
||||
);
|
||||
or_shutdown!(plugin_runtime, services.register_usage_data(&usage_data));
|
||||
let services = std::sync::Arc::new(services);
|
||||
or_shutdown!(plugin_runtime, plugin_runtime.bind_services(&services));
|
||||
let commands = or_shutdown!(plugin_runtime, plugin_runtime.prepare_command_line().await);
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::OnceLock,
|
||||
};
|
||||
|
||||
use syn::{Item, visit::Visit};
|
||||
@@ -26,22 +27,24 @@ pub(super) const WORKSPACE_MEMBERS: &[&str] = &[
|
||||
|
||||
#[test]
|
||||
fn workspace_member_list_matches_the_frozen_dag_scope() {
|
||||
let manifest =
|
||||
fs::read_to_string(backend_root().join("Cargo.toml")).expect("read workspace manifest");
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
assert!(
|
||||
manifest.contains(&format!("\"{member}\"")),
|
||||
"workspace must include {member}"
|
||||
);
|
||||
}
|
||||
let member_count = manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| line.starts_with("\"apps/") || line.starts_with("\"crates/"))
|
||||
.count();
|
||||
let metadata = cargo_metadata_json();
|
||||
let root = backend_root();
|
||||
let actual: BTreeSet<_> = metadata["packages"]
|
||||
.as_array()
|
||||
.expect("Cargo packages")
|
||||
.iter()
|
||||
.map(|package| {
|
||||
Path::new(package["manifest_path"].as_str().expect("package manifest"))
|
||||
.parent()
|
||||
.expect("package directory")
|
||||
.strip_prefix(&root)
|
||||
.expect("workspace member path")
|
||||
.to_path_buf()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
member_count,
|
||||
WORKSPACE_MEMBERS.len(),
|
||||
actual,
|
||||
WORKSPACE_MEMBERS.iter().map(PathBuf::from).collect(),
|
||||
"new workspace members must extend the dependency DAG rules"
|
||||
);
|
||||
}
|
||||
@@ -378,7 +381,7 @@ const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
#[test]
|
||||
fn workspace_internal_dependency_edges_match_frozen_dag() {
|
||||
let metadata = cargo_metadata_json();
|
||||
let actual = workspace_runtime_dependency_edges(&metadata);
|
||||
let actual = workspace_runtime_dependency_edges(metadata);
|
||||
let mut expected: Vec<(String, String)> = ALLOWED_INTERNAL_EDGES
|
||||
.iter()
|
||||
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
|
||||
@@ -474,24 +477,6 @@ fn adapter_public_module_surfaces_match_allowlist() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
let src = backend_root().join(member).join("src");
|
||||
let sources = super::rust_files(&src);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = src.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_modules_follow_conventional_file_layout() {
|
||||
let targets = test_source_roots();
|
||||
@@ -748,40 +733,46 @@ pub(super) fn backend_root() -> PathBuf {
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
fn cargo_metadata_json() -> serde_json::Value {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
fn cargo_metadata_json() -> &'static serde_json::Value {
|
||||
static METADATA: OnceLock<serde_json::Value> = OnceLock::new();
|
||||
METADATA.get_or_init(|| {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps", "--locked"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
})
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束
|
||||
/// 复用 Cargo 解析后的实际包名,覆盖重命名依赖与平台条件,排除测试和构建依赖
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
.expect("read member manifest");
|
||||
let mut in_dependencies = false;
|
||||
let mut names = Vec::new();
|
||||
for line in manifest.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
in_dependencies = line == "[dependencies]";
|
||||
continue;
|
||||
}
|
||||
if !in_dependencies || line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some((name, _)) = line.split_once('=') {
|
||||
names.push(name.trim().to_owned());
|
||||
}
|
||||
}
|
||||
names
|
||||
let manifest = backend_root().join(member).join("Cargo.toml");
|
||||
let package = cargo_metadata_json()["packages"]
|
||||
.as_array()
|
||||
.expect("Cargo packages")
|
||||
.iter()
|
||||
.find(|package| {
|
||||
Path::new(package["manifest_path"].as_str().expect("package manifest")) == manifest
|
||||
})
|
||||
.expect("workspace member");
|
||||
package["dependencies"]
|
||||
.as_array()
|
||||
.expect("package dependencies")
|
||||
.iter()
|
||||
.filter(|dependency| dependency["kind"].is_null())
|
||||
.map(|dependency| {
|
||||
dependency["name"]
|
||||
.as_str()
|
||||
.expect("dependency name")
|
||||
.to_owned()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assert_no_dependency(member: &str, forbidden: &[&str]) {
|
||||
|
||||
@@ -59,9 +59,16 @@ fn cargo_library_root_is_conventional_lib() {
|
||||
fn workspace_production_files_have_no_hidden_modules_or_test_hooks() {
|
||||
for member in architecture::WORKSPACE_MEMBERS {
|
||||
let root = architecture::backend_root().join(member).join("src");
|
||||
for relative in rust_files(&root) {
|
||||
let sources = rust_files(&root);
|
||||
assert!(!sources.is_empty(), "{member} has no production sources");
|
||||
for relative in sources {
|
||||
let path = root.join(relative);
|
||||
let source = fs::read_to_string(&path).expect("read production source");
|
||||
assert!(
|
||||
!source.contains("#[cfg(test)]"),
|
||||
"{} hosts tests in production src",
|
||||
path.display()
|
||||
);
|
||||
assert!(
|
||||
!source.contains("include!("),
|
||||
"{} uses include!",
|
||||
|
||||
@@ -39,6 +39,7 @@ pub use use_case::{
|
||||
proxies::ProxiesService,
|
||||
settings::SettingsService,
|
||||
system::SystemService,
|
||||
usage_data::UsageDataService,
|
||||
};
|
||||
|
||||
use model::AdminError;
|
||||
@@ -217,6 +218,7 @@ pub struct AdminServices {
|
||||
client_distribution: Arc<dyn ClientDistributionService>,
|
||||
observability: Arc<dyn ObservabilityService>,
|
||||
settings: Arc<dyn SettingsService>,
|
||||
usage_data: Arc<dyn UsageDataService>,
|
||||
system: Arc<dyn SystemService>,
|
||||
credentials: Arc<CredentialsService>,
|
||||
// Runtime 只持有 Weak,这个强引用保障账号端口与 Admin 服务同寿命
|
||||
@@ -285,6 +287,11 @@ impl AdminServices {
|
||||
self.client_distribution.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn usage_data(&self) -> &dyn UsageDataService {
|
||||
self.usage_data.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn observability(&self) -> &dyn ObservabilityService {
|
||||
self.observability.as_ref()
|
||||
@@ -467,8 +474,10 @@ async fn initialize_inner(
|
||||
registry.clone(),
|
||||
pricing_source,
|
||||
);
|
||||
let usage_data = initialize_usage_data(store.usage_data(), store.settings(), registry.clone());
|
||||
let mut public_services = public_service::Registry::new(service_middleware);
|
||||
public_services.register_settings(&settings)?;
|
||||
public_services.register_usage_data(&usage_data)?;
|
||||
let services = AdminServices {
|
||||
timezone,
|
||||
public_services: Arc::new(public_services),
|
||||
@@ -512,8 +521,10 @@ async fn initialize_inner(
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
timezone,
|
||||
system.clone(),
|
||||
)),
|
||||
settings,
|
||||
usage_data,
|
||||
system,
|
||||
credentials,
|
||||
_plugin_accounts: plugin_accounts,
|
||||
@@ -644,3 +655,15 @@ pub fn initialize_settings(
|
||||
pricing_source,
|
||||
))
|
||||
}
|
||||
|
||||
/// HTTP、插件及 CLI 共用的用量领域用例,宿主提供已验证的调用主体
|
||||
#[must_use]
|
||||
pub fn initialize_usage_data(
|
||||
store: Arc<dyn ports::usage_data::UsageDataStore>,
|
||||
settings: Arc<dyn ports::store::SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Arc<dyn UsageDataService> {
|
||||
Arc::new(use_case::usage_data::DefaultUsageDataService::new(
|
||||
store, settings, providers,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -100,6 +100,7 @@ pub struct AccountGroupListQuery {
|
||||
/// 完整账号分组摘要
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupRecord {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
@@ -139,6 +140,7 @@ pub struct AccountGroupOptionsPage {
|
||||
/// 创建账号分组
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateAccountGroup {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
@@ -148,6 +150,7 @@ pub struct CreateAccountGroup {
|
||||
/// 已生成稳定标识、可直接提交存储的创建命令
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NewAccountGroup {
|
||||
pub allow_image_generation: bool,
|
||||
pub fast_mode: FastMode,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
@@ -158,6 +161,7 @@ pub struct NewAccountGroup {
|
||||
/// 更新账号分组的描述字段
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateAccountGroup {
|
||||
pub allow_image_generation: Option<bool>,
|
||||
pub fast_mode: Option<FastMode>,
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
|
||||
@@ -124,6 +124,7 @@ pub struct AccountRecord {
|
||||
pub weight: AccountWeight,
|
||||
pub model_access: gateway_core::account::AccountModelAccess,
|
||||
pub outbound_proxy: Option<gateway_core::account::OutboundProxy>,
|
||||
pub outbound_proxy_enabled: bool,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota: QuotaState,
|
||||
@@ -133,6 +134,16 @@ pub struct AccountRecord {
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl AccountRecord {
|
||||
/// 配置保留绑定,执行请求时禁用代理按直连处理
|
||||
#[must_use]
|
||||
pub fn effective_outbound_proxy(&self) -> Option<&gateway_core::account::OutboundProxy> {
|
||||
self.outbound_proxy
|
||||
.as_ref()
|
||||
.filter(|_| self.outbound_proxy_enabled)
|
||||
}
|
||||
}
|
||||
|
||||
pub use super::observability::{
|
||||
AccountModelUsage, AccountPoolMetrics as AccountSummary, AccountRequestBucket,
|
||||
CurrencyCost as AccountCost,
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
//! 故障诊断导出的目标、已采集证据与缺口
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::{
|
||||
observability::{OpsError, UsageDetail},
|
||||
system::SystemVersion,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DiagnosticTarget {
|
||||
Request(String),
|
||||
Operation(String),
|
||||
}
|
||||
|
||||
impl DiagnosticTarget {
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
Self::Request(id) | Self::Operation(id) => id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DiagnosticGap {
|
||||
pub section: String,
|
||||
pub reason: &'static str,
|
||||
}
|
||||
|
||||
pub struct DiagnosticRequest {
|
||||
pub request_id: String,
|
||||
pub detail: Option<UsageDetail>,
|
||||
pub errors: Vec<OpsError>,
|
||||
}
|
||||
|
||||
/// 原始受控详情只能经导出投影脱敏后交付,不参与普通日志或 Debug
|
||||
pub struct DiagnosticBundle {
|
||||
pub target: DiagnosticTarget,
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub environment: Option<SystemVersion>,
|
||||
pub error: Option<OpsError>,
|
||||
pub requests: Vec<DiagnosticRequest>,
|
||||
pub gaps: Vec<DiagnosticGap>,
|
||||
}
|
||||
@@ -10,6 +10,7 @@ pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_distribution;
|
||||
pub mod client_keys;
|
||||
pub mod diagnostics;
|
||||
pub mod import_tasks;
|
||||
pub mod key_usage;
|
||||
pub mod observability;
|
||||
@@ -24,6 +25,7 @@ pub mod quota_forecast_sampling;
|
||||
pub mod retention;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
/// 管理用例对外返回的稳定错误分类
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -172,9 +172,19 @@ pub struct UsageFilter {
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
/// 请求列表的结果筛选,不改变汇总和计量口径
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum UsageRecordStatus {
|
||||
#[default]
|
||||
Success,
|
||||
Error,
|
||||
All,
|
||||
}
|
||||
|
||||
/// 用量记录分页查询
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageQuery {
|
||||
pub status: UsageRecordStatus,
|
||||
pub range: TimeRange,
|
||||
pub filter: UsageFilter,
|
||||
pub current_page: u32,
|
||||
@@ -653,6 +663,7 @@ pub struct DashboardObservation {
|
||||
/// 使用记录表格的窄读模型
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageListRecord {
|
||||
pub outcome: RequestOutcome,
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub id: String,
|
||||
pub endpoint: String,
|
||||
@@ -845,7 +856,7 @@ pub struct UsageAttempt {
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 一条请求及其全部尝试
|
||||
/// 一条请求及其保留的尝试与诊断轨迹
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageDetail {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
|
||||
@@ -69,6 +69,7 @@ pub struct ProxyTestResult {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyRecord {
|
||||
pub enabled: bool,
|
||||
pub auto_location: bool,
|
||||
pub detected_location: Option<DetectedProxyLocation>,
|
||||
pub location: Option<gateway_core::account::RequestLocation>,
|
||||
@@ -103,6 +104,7 @@ pub struct NewProxy {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UpdateProxy {
|
||||
pub enabled: Option<bool>,
|
||||
pub auto_location: Option<bool>,
|
||||
pub test: Option<ProxyTestResult>,
|
||||
/// 外层为空保留配置,内层为空恢复全局继承
|
||||
|
||||
@@ -24,7 +24,7 @@ impl RetentionPolicy {
|
||||
audit_days: u32,
|
||||
) -> Result<(), &'static str> {
|
||||
for (valid, field) in [
|
||||
(usage_days >= 31, "usage_retention_days"),
|
||||
((1..=31).contains(&usage_days), "usage_retention_days"),
|
||||
(ops_days > 0, "ops_event_retention_days"),
|
||||
(audit_days > 0, "audit_retention_days"),
|
||||
] {
|
||||
@@ -38,7 +38,7 @@ impl RetentionPolicy {
|
||||
pub fn try_new(usage_days: u32, ops_days: u32, audit_days: u32) -> Result<Self, AdminError> {
|
||||
if Self::validate_values(usage_days, ops_days, audit_days).is_err() {
|
||||
return Err(AdminError::invalid(
|
||||
"请求保留期至少 31 天,事件与审计保留期必须为正数",
|
||||
"请求保留期必须为 1~31 天,事件与审计保留期必须为正数",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
//! 由宿主传入的可信写入主体,不属于插件请求载荷
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageActor {
|
||||
Admin {
|
||||
subject: String,
|
||||
},
|
||||
Plugin {
|
||||
publisher: String,
|
||||
name: String,
|
||||
instance_id: String,
|
||||
},
|
||||
System,
|
||||
}
|
||||
|
||||
impl super::UsageRecordChange {
|
||||
#[must_use]
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
Self::Put { id, .. } | Self::Update { id, .. } | Self::Delete { id } => id,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! 按归属时间和维度维护统计贡献,业务来源与调整历史由调用方保存
|
||||
|
||||
use super::{UsageDelta, UsageRecordFilter};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageMetric {
|
||||
RequestCount,
|
||||
SuccessCount,
|
||||
FailureCount,
|
||||
CancelledCount,
|
||||
IncompleteCount,
|
||||
CompletedRequestCount,
|
||||
InputTokens,
|
||||
OutputTokens,
|
||||
CachedInputTokens,
|
||||
CacheWriteInputTokens,
|
||||
ReasoningOutputTokens,
|
||||
ImageInputTokens,
|
||||
ImageOutputTokens,
|
||||
TotalTokens,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageDimensions {
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub client_key_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
}
|
||||
|
||||
/// 整个键决定覆盖目标;调用方可将来源标识编码进 key,重试相同键不会累计
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregateKey {
|
||||
pub bucket_start_ms: i64,
|
||||
pub dimensions: UsageDimensions,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
/// 值按 bucket_start_ms 归属,不自动摊分;负值用于相对原生统计的修正
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregate {
|
||||
pub identity: UsageAggregateKey,
|
||||
pub counts: BTreeMap<UsageMetric, UsageDelta>,
|
||||
pub costs: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageAggregateChange {
|
||||
Put { aggregate: UsageAggregate },
|
||||
Delete { identity: UsageAggregateKey },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteUsageAggregates {
|
||||
pub changes: Vec<UsageAggregateChange>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregateWriteResult {
|
||||
pub changed: Vec<UsageAggregateKey>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListUsageAggregates {
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<String>,
|
||||
pub limit: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageAggregatePage {
|
||||
pub items: Vec<UsageAggregate>,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! 用量基础操作的错误分类,不保存失败或重试状态
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageErrorKind {
|
||||
InvalidInput,
|
||||
NotFound,
|
||||
Conflict,
|
||||
CapacityExceeded,
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageDataError {
|
||||
pub kind: UsageErrorKind,
|
||||
pub message: String,
|
||||
}
|
||||
impl UsageDataError {
|
||||
#[must_use]
|
||||
pub fn new(kind: UsageErrorKind, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
#[must_use]
|
||||
pub fn invalid(message: impl Into<String>) -> Self {
|
||||
Self::new(UsageErrorKind::InvalidInput, message)
|
||||
}
|
||||
#[must_use]
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::new(UsageErrorKind::Unavailable, message)
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for UsageDataError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for UsageDataError {}
|
||||
pub type UsageDataResult<T> = Result<T, UsageDataError>;
|
||||
@@ -0,0 +1,22 @@
|
||||
//! 用量明细、聚合维护与纯计价的公开合同
|
||||
|
||||
mod value;
|
||||
pub use value::*;
|
||||
mod record;
|
||||
pub use record::*;
|
||||
mod query;
|
||||
pub use query::*;
|
||||
mod pricing;
|
||||
pub use pricing::*;
|
||||
mod error;
|
||||
pub use error::*;
|
||||
|
||||
pub const USAGE_DATA_MAX_TARGETS: usize = 500;
|
||||
pub const USAGE_DATA_MAX_REQUEST_BYTES: usize = 2 * 1024 * 1024;
|
||||
pub const USAGE_DATA_MAX_PAGE_SIZE: u16 = 200;
|
||||
|
||||
mod actor;
|
||||
pub use actor::*;
|
||||
|
||||
mod aggregate;
|
||||
pub use aggregate::*;
|
||||
@@ -0,0 +1,62 @@
|
||||
//! 纯 Token 计价合同,费用是否采用与额外费用合并由调用方决定
|
||||
|
||||
use super::{UsageCost, UsageTokens};
|
||||
use gateway_core::metering::ModelPriceOverride;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsagePriceSelection {
|
||||
Current,
|
||||
Override { price: Box<ModelPriceOverride> },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingInput {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub tokens: UsageTokens,
|
||||
pub service_tier: Option<String>,
|
||||
pub price: UsagePriceSelection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EvaluateUsagePricing {
|
||||
pub items: Vec<UsagePricingInput>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingBreakdown {
|
||||
pub input_amount: String,
|
||||
pub output_amount: String,
|
||||
pub cache_read_amount: String,
|
||||
pub cache_write_amount: String,
|
||||
pub standard_amount: String,
|
||||
pub long_context_billing_applied: bool,
|
||||
pub service_tier: Option<String>,
|
||||
pub custom_multiplier_bps: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsagePricingResult {
|
||||
Estimated {
|
||||
id: String,
|
||||
cost: UsageCost,
|
||||
breakdown: UsagePricingBreakdown,
|
||||
},
|
||||
NotEstimable {
|
||||
id: String,
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsagePricingEvaluation {
|
||||
pub items: Vec<UsagePricingResult>,
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//! 请求明细的有界筛选与稳定顺序分页
|
||||
|
||||
use super::UsageRecord;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GetUsageRecord {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordFilter {
|
||||
pub start_ms: i64,
|
||||
pub end_ms: i64,
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub client_key_ref: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ListUsageRecords {
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<String>,
|
||||
pub limit: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordPage {
|
||||
pub items: Vec<UsageRecord>,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! 既有请求明细的用量读写合同,不承载插件任务或来源状态
|
||||
|
||||
use super::UsageCount;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Token 使用宿主口径:缓存包含在输入中,推理包含在输出中,未知值为 null
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageTokens {
|
||||
pub input_tokens: Option<UsageCount>,
|
||||
pub output_tokens: Option<UsageCount>,
|
||||
pub cached_input_tokens: Option<UsageCount>,
|
||||
pub cache_write_input_tokens: Option<UsageCount>,
|
||||
pub reasoning_output_tokens: Option<UsageCount>,
|
||||
pub image_input_tokens: Option<UsageCount>,
|
||||
pub image_output_tokens: Option<UsageCount>,
|
||||
pub total_tokens: Option<UsageCount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageCostSource {
|
||||
ProviderReported,
|
||||
Calculated,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageCost {
|
||||
#[default]
|
||||
Unavailable,
|
||||
Known {
|
||||
source: UsageCostSource,
|
||||
amount: String,
|
||||
currency: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum UsageOutcome {
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Cancelled,
|
||||
Incomplete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecordData {
|
||||
pub client_key_ref: String,
|
||||
pub provider: Option<String>,
|
||||
pub account_ref: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub protocol: String,
|
||||
pub operation: String,
|
||||
pub endpoint: String,
|
||||
pub client_transport: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub started_at_ms: i64,
|
||||
pub completed_at_ms: Option<i64>,
|
||||
pub outcome: UsageOutcome,
|
||||
pub tokens: UsageTokens,
|
||||
pub cost: UsageCost,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageRecord {
|
||||
pub id: String,
|
||||
pub data: UsageRecordData,
|
||||
pub billing_snapshot: Option<serde_json::Value>,
|
||||
pub provider_observation: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum UsageRecordChange {
|
||||
Put {
|
||||
id: String,
|
||||
data: Box<UsageRecordData>,
|
||||
},
|
||||
/// tokens 替换全部 Token 字段;cost 的 unavailable 明确清空费用
|
||||
Update {
|
||||
id: String,
|
||||
tokens: Option<UsageTokens>,
|
||||
cost: Option<UsageCost>,
|
||||
},
|
||||
Delete {
|
||||
id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WriteUsageRecords {
|
||||
pub changes: Vec<UsageRecordChange>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UsageWriteResult {
|
||||
pub changed_ids: Vec<String>,
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//! 用量计数使用十进制字符串,避免插件前端丢失整数精度
|
||||
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error};
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct UsageCount(pub u64);
|
||||
impl UsageCount {
|
||||
pub const ZERO: Self = Self(0);
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
impl From<u64> for UsageCount {
|
||||
fn from(value: u64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
impl fmt::Display for UsageCount {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
self.0.fmt(formatter)
|
||||
}
|
||||
}
|
||||
impl Serialize for UsageCount {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&self.0.to_string())
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for UsageCount {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
let value = String::deserialize(deserializer)?;
|
||||
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(D::Error::custom("count must be an unsigned decimal string"));
|
||||
}
|
||||
value.parse().map(Self).map_err(D::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
/// 有符号统计贡献受 PostgreSQL bigint 范围限制,JSON 使用十进制字符串
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct UsageDelta(pub i64);
|
||||
impl UsageDelta {
|
||||
#[must_use]
|
||||
pub const fn get(self) -> i64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
impl Serialize for UsageDelta {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&self.0.to_string())
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for UsageDelta {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
let value = String::deserialize(deserializer)?;
|
||||
let digits = value.strip_prefix('-').unwrap_or(&value);
|
||||
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(D::Error::custom("delta must be a signed decimal string"));
|
||||
}
|
||||
value.parse().map(Self).map_err(D::Error::custom)
|
||||
}
|
||||
}
|
||||
@@ -14,3 +14,4 @@ pub mod proxy;
|
||||
pub mod retention;
|
||||
pub mod store;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
@@ -26,6 +26,14 @@ use crate::model::{
|
||||
quota_forecast_sampling::QuotaForecastObservation,
|
||||
};
|
||||
|
||||
/// 按原生 Token 口径计算的报价输入,额外费用由调用方处理
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderUsagePricingInput {
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub usage: gateway_core::metering::Usage,
|
||||
}
|
||||
|
||||
/// Provider 管理失败的稳定分类
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProviderAdminErrorKind {
|
||||
@@ -161,6 +169,14 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
Default::default()
|
||||
}
|
||||
|
||||
fn evaluate_usage_pricing(
|
||||
&self,
|
||||
_input: &ProviderUsagePricingInput,
|
||||
_pricing: Option<&gateway_core::metering::ModelPriceOverride>,
|
||||
) -> Result<Option<gateway_core::metering::CalculatedCostBreakdown>, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
fn provider_kind(&self) -> &ProviderKind;
|
||||
|
||||
/// 只读取原生 Provider 能力及账号类型,不执行网络或数据库查询
|
||||
|
||||
@@ -485,6 +485,13 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
) -> AdminStoreResult<DiagnosticsObservation>;
|
||||
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage>;
|
||||
|
||||
/// 按精确目标读取受控错误,不受列表时间窗口与当前页面影响
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
target: &crate::model::diagnostics::DiagnosticTarget,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<Vec<crate::model::observability::OpsError>>;
|
||||
}
|
||||
|
||||
/// Runtime settings 与管理员 API Key 写入
|
||||
@@ -565,9 +572,15 @@ pub struct AdminStorePorts {
|
||||
plugins: Arc<dyn super::plugins::PluginStore>,
|
||||
plugin_state: Arc<dyn super::plugins::PluginStateStore>,
|
||||
plugin_resources: Arc<dyn super::plugin_resources::PluginResourceStore>,
|
||||
usage_data: Arc<dyn super::usage_data::UsageDataStore>,
|
||||
}
|
||||
|
||||
impl AdminStorePorts {
|
||||
#[must_use]
|
||||
pub fn usage_data(&self) -> Arc<dyn super::usage_data::UsageDataStore> {
|
||||
self.usage_data.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn plugin_resources(&self) -> Arc<dyn super::plugin_resources::PluginResourceStore> {
|
||||
self.plugin_resources.clone()
|
||||
@@ -598,6 +611,7 @@ impl AdminStorePorts {
|
||||
plugins: Arc<dyn super::plugins::PluginStore>,
|
||||
plugin_state: Arc<dyn super::plugins::PluginStateStore>,
|
||||
plugin_resources: Arc<dyn super::plugin_resources::PluginResourceStore>,
|
||||
usage_data: Arc<dyn super::usage_data::UsageDataStore>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
@@ -609,6 +623,7 @@ impl AdminStorePorts {
|
||||
plugins,
|
||||
plugin_state,
|
||||
plugin_resources,
|
||||
usage_data,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
//! 原生请求用量的有界读取和事务写入端口
|
||||
|
||||
use crate::model::{auth::AdminAuditEvent, usage_data::*};
|
||||
use async_trait::async_trait;
|
||||
|
||||
pub struct StoredUsageWrite {
|
||||
pub changes: Vec<(UsageRecordChange, AdminAuditEvent)>,
|
||||
}
|
||||
|
||||
pub struct StoredAggregateWrite {
|
||||
pub changes: Vec<(UsageAggregateChange, AdminAuditEvent)>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataStore: Send + Sync {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage>;
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
command: StoredAggregateWrite,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult>;
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord>;
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage>;
|
||||
async fn write_records(&self, command: StoredUsageWrite) -> UsageDataResult<UsageWriteResult>;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//! 公开服务写入主体由受信宿主边界设置,插件输入不能覆盖
|
||||
|
||||
use crate::model::usage_data::UsageActor;
|
||||
use std::future::Future;
|
||||
|
||||
tokio::task_local! {
|
||||
static ACTOR: UsageActor;
|
||||
}
|
||||
|
||||
/// 与 middleware Origin 独立传递,嵌套服务不会丢失真实调用主体
|
||||
pub async fn scope_actor<F: Future>(actor: UsageActor, future: F) -> F::Output {
|
||||
ACTOR.scope(actor, future).await
|
||||
}
|
||||
|
||||
pub(super) fn current_actor() -> Result<UsageActor, crate::model::usage_data::UsageDataError> {
|
||||
ACTOR.try_with(Clone::clone).map_err(|_| {
|
||||
crate::model::usage_data::UsageDataError::invalid(
|
||||
"usage mutation requires a trusted caller identity",
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -1,7 +1,10 @@
|
||||
//! 主动服务调用的统一组合入口;普通业务方法由所属 HTTP 或执行边界包裹
|
||||
|
||||
mod actor;
|
||||
mod registry;
|
||||
mod settings;
|
||||
mod usage_data;
|
||||
pub use actor::scope_actor;
|
||||
pub use registry::Registry;
|
||||
|
||||
use std::{future::Future, sync::Arc};
|
||||
|
||||
@@ -54,6 +54,57 @@ impl Registry {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn register_usage_data(
|
||||
&mut self,
|
||||
service: &Arc<dyn crate::UsageDataService>,
|
||||
) -> Result<(), AdminError> {
|
||||
super::usage_data::register(self, service)
|
||||
}
|
||||
|
||||
pub(crate) fn register_usage<I, O>(
|
||||
&mut self,
|
||||
name: &'static str,
|
||||
handler: impl Fn(I) -> BoxFuture<'static, crate::model::usage_data::UsageDataResult<O>>
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
) -> Result<(), AdminError>
|
||||
where
|
||||
I: DeserializeOwned + Send + 'static,
|
||||
O: Serialize + Send + 'static,
|
||||
{
|
||||
let std::collections::btree_map::Entry::Vacant(entry) = self.operations.entry(name) else {
|
||||
return Err(AdminError::internal(format!("公开服务重复登记:{name}")));
|
||||
};
|
||||
let handler = Arc::new(handler);
|
||||
entry.insert(Arc::new(move |input| {
|
||||
let handler = handler.clone();
|
||||
Box::pin(async move {
|
||||
// 在反序列化之前限制编码后的完整请求,避免单条记录绕过批量行数约束
|
||||
let bytes =
|
||||
serde_json::to_vec(&input).map_err(|_| Error::invalid("服务输入编码无效"))?;
|
||||
if bytes.len() > crate::model::usage_data::USAGE_DATA_MAX_REQUEST_BYTES {
|
||||
return Err(super::usage_data::encode_error(
|
||||
crate::model::usage_data::UsageDataError::new(
|
||||
crate::model::usage_data::UsageErrorKind::CapacityExceeded,
|
||||
"usage input exceeds 2 MiB",
|
||||
),
|
||||
));
|
||||
}
|
||||
let input = serde_json::from_value(input).map_err(|_| {
|
||||
super::usage_data::encode_error(
|
||||
crate::model::usage_data::UsageDataError::invalid("服务输入类型无效"),
|
||||
)
|
||||
})?;
|
||||
let output = handler(input)
|
||||
.await
|
||||
.map_err(super::usage_data::encode_error)?;
|
||||
serde_json::to_value(output).map_err(|_| Error::unavailable("服务结果编码失败"))
|
||||
})
|
||||
}));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn call(&self, operation: &str, input: Value) -> Result<Value, Error> {
|
||||
let (operation, handler) =
|
||||
self.operations
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
//! 原生用量能力的公开服务登记
|
||||
|
||||
use crate::{
|
||||
UsageDataService,
|
||||
model::{AdminError, usage_data::*},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) fn register(
|
||||
registry: &mut super::Registry,
|
||||
service: &Arc<dyn UsageDataService>,
|
||||
) -> Result<(), AdminError> {
|
||||
macro_rules! read {
|
||||
($name:literal, $method:ident, $input:ty) => {{
|
||||
let service = service.clone();
|
||||
registry.register_usage($name, move |request: $input| {
|
||||
let service = service.clone();
|
||||
Box::pin(async move { service.$method(request).await })
|
||||
})?;
|
||||
}};
|
||||
}
|
||||
read!("usage.records.get", get_record, GetUsageRecord);
|
||||
read!("usage.records.list", list_records, ListUsageRecords);
|
||||
read!(
|
||||
"usage.aggregates.list",
|
||||
list_aggregates,
|
||||
ListUsageAggregates
|
||||
);
|
||||
read!("pricing.evaluate", evaluate_pricing, EvaluateUsagePricing);
|
||||
let aggregates = service.clone();
|
||||
registry.register_usage(
|
||||
"usage.aggregates.write",
|
||||
move |request: WriteUsageAggregates| {
|
||||
let service = aggregates.clone();
|
||||
Box::pin(async move {
|
||||
let actor = super::actor::current_actor()?;
|
||||
service.write_aggregates(&actor, request).await
|
||||
})
|
||||
},
|
||||
)?;
|
||||
let service = service.clone();
|
||||
registry.register_usage("usage.records.write", move |request: WriteUsageRecords| {
|
||||
let service = service.clone();
|
||||
Box::pin(async move {
|
||||
let actor = super::actor::current_actor()?;
|
||||
service.write_records(&actor, request).await
|
||||
})
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn encode_error(
|
||||
error: UsageDataError,
|
||||
) -> gateway_core::engine::middleware::service::Error {
|
||||
gateway_core::engine::middleware::service::Error {
|
||||
kind: serde_json::to_value(error.kind)
|
||||
.ok()
|
||||
.and_then(|v| v.as_str().map(str::to_owned))
|
||||
.unwrap_or_else(|| "unavailable".into()),
|
||||
message: error.message.clone(),
|
||||
details: serde_json::to_value(error).ok(),
|
||||
}
|
||||
}
|
||||
@@ -160,6 +160,7 @@ impl AccountGroupService for DefaultAccountGroupService {
|
||||
self.store
|
||||
.create_account_group(
|
||||
NewAccountGroup {
|
||||
allow_image_generation: command.allow_image_generation,
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
|
||||
@@ -106,7 +106,8 @@ pub(super) async fn pending_authorization(
|
||||
proxy = required_credential(accounts, provider_kind, account_id, resource)
|
||||
.await?
|
||||
.credential
|
||||
.outbound_proxy;
|
||||
.effective_outbound_proxy()
|
||||
.cloned();
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
@@ -121,6 +122,9 @@ pub(super) async fn pending_authorization(
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
proxy_id = Some(record.id);
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("代理已禁用,请启用后再选择"));
|
||||
}
|
||||
Some(record.proxy)
|
||||
}
|
||||
Some(AccountProxySelection::Direct) | None => None,
|
||||
|
||||
@@ -225,6 +225,7 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
KeyUsageRecordKind::Success => KeyUsageRecords::Success(
|
||||
self.observations
|
||||
.list_usage_records(UsageQuery {
|
||||
status: Default::default(),
|
||||
range: query.usage.range,
|
||||
filter: usage_filter(&id, query.usage.model),
|
||||
current_page: query.current_page,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub mod plugins;
|
||||
pub mod proxies;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod usage_data;
|
||||
|
||||
mod credential_mutation;
|
||||
mod error;
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
//! 按故障目标收集诊断证据,补充读取失败不丢弃已取得的记录
|
||||
|
||||
use std::{
|
||||
collections::{HashSet, VecDeque},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use tokio::time::{Instant, timeout_at};
|
||||
|
||||
use super::{DefaultObservabilityService, map_store_error};
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind,
|
||||
diagnostics::{DiagnosticBundle, DiagnosticGap, DiagnosticRequest, DiagnosticTarget},
|
||||
observability::OpsError,
|
||||
};
|
||||
|
||||
const MAX_REQUESTS: usize = 8;
|
||||
const MAX_ERRORS: u16 = 128;
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
pub(super) async fn collect_diagnostics(
|
||||
&self,
|
||||
target: DiagnosticTarget,
|
||||
) -> Result<DiagnosticBundle, AdminError> {
|
||||
if target.id().trim().is_empty() || target.id().len() > 256 {
|
||||
return Err(AdminError::invalid("诊断目标 ID 不合法"));
|
||||
}
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
let (collected, environment) = tokio::join!(
|
||||
self.collect_records(target, deadline),
|
||||
timeout_at(deadline, self.system.version()),
|
||||
);
|
||||
let mut bundle = collected?;
|
||||
match environment {
|
||||
Ok(Ok(version)) => bundle.environment = Some(version),
|
||||
_ => gap(&mut bundle, "environment", "collection_failed"),
|
||||
}
|
||||
Ok(bundle)
|
||||
}
|
||||
|
||||
async fn collect_records(
|
||||
&self,
|
||||
target: DiagnosticTarget,
|
||||
deadline: Instant,
|
||||
) -> Result<DiagnosticBundle, AdminError> {
|
||||
let mut bundle = DiagnosticBundle {
|
||||
target,
|
||||
collected_at: chrono::Utc::now(),
|
||||
environment: None,
|
||||
error: None,
|
||||
requests: Vec::new(),
|
||||
gaps: Vec::new(),
|
||||
};
|
||||
let mut remaining_errors = MAX_ERRORS;
|
||||
let first_id = match &bundle.target {
|
||||
DiagnosticTarget::Request(id) => Some(id.clone()),
|
||||
DiagnosticTarget::Operation(_) => {
|
||||
let mut errors = self.export_errors(&bundle.target, 1, deadline).await?;
|
||||
let error = errors
|
||||
.pop()
|
||||
.ok_or_else(|| AdminError::not_found("错误记录不存在或已过期"))?;
|
||||
let id = error.request_id.clone();
|
||||
bundle.error = Some(error);
|
||||
remaining_errors -= 1;
|
||||
id
|
||||
}
|
||||
};
|
||||
let mut pending = VecDeque::from_iter(first_id);
|
||||
let mut seen = HashSet::new();
|
||||
while let Some(id) = pending.pop_front() {
|
||||
if !seen.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
if bundle.requests.len() >= MAX_REQUESTS {
|
||||
gap(&mut bundle, "requests", "related_request_limit");
|
||||
break;
|
||||
}
|
||||
let section = format!("requests.{id}");
|
||||
let target = DiagnosticTarget::Request(id.clone());
|
||||
let selected = bundle
|
||||
.error
|
||||
.as_ref()
|
||||
.filter(|error| error.request_id.as_deref() == Some(id.as_str()));
|
||||
// 事件导出已经持有选中记录,额外读取一个槽位用于去重后判断是否超限
|
||||
let read_limit = remaining_errors + 1 + u16::from(selected.is_some());
|
||||
let (detail, errors) = tokio::join!(
|
||||
timeout_at(deadline, self.store.usage_record_detail(&id)),
|
||||
self.export_errors(&target, read_limit, deadline),
|
||||
);
|
||||
let (detail, detail_error) = match detail {
|
||||
Ok(Ok(detail)) => (Some(detail), None),
|
||||
Ok(Err(error)) => (None, Some(map_store_error(error, "diagnostic request"))),
|
||||
Err(_) => (
|
||||
None,
|
||||
Some(AdminError::new(AdminErrorKind::Unavailable, "诊断采集超时")),
|
||||
),
|
||||
};
|
||||
let mut errors = match errors {
|
||||
Ok(mut errors) => {
|
||||
if let Some(selected) = selected {
|
||||
errors.retain(|error| {
|
||||
error.source != selected.source || error.event_id != selected.event_id
|
||||
});
|
||||
}
|
||||
errors
|
||||
}
|
||||
Err(error) => {
|
||||
if detail.is_none() && bundle.requests.is_empty() && bundle.error.is_none() {
|
||||
return Err(error);
|
||||
}
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.errors"),
|
||||
"collection_failed",
|
||||
);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
if detail.is_none()
|
||||
&& errors.is_empty()
|
||||
&& bundle.requests.is_empty()
|
||||
&& bundle.error.is_none()
|
||||
{
|
||||
return Err(
|
||||
detail_error.unwrap_or_else(|| AdminError::not_found("请求记录不存在或已过期"))
|
||||
);
|
||||
}
|
||||
if errors.len() > usize::from(remaining_errors) {
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.errors"),
|
||||
"error_record_limit",
|
||||
);
|
||||
errors.truncate(usize::from(remaining_errors));
|
||||
}
|
||||
remaining_errors =
|
||||
remaining_errors.saturating_sub(u16::try_from(errors.len()).unwrap_or(MAX_ERRORS));
|
||||
if let Some(record) = &detail {
|
||||
for related in &record.related_requests {
|
||||
if let Some(id) = related.get("requestId").and_then(serde_json::Value::as_str) {
|
||||
pending.push_back(id.to_owned());
|
||||
}
|
||||
}
|
||||
if record.request.outcome == crate::model::observability::RequestOutcome::Running {
|
||||
gap(&mut bundle, §ion, "request_not_finalized");
|
||||
}
|
||||
match &record.trace {
|
||||
None => gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"not_recorded_or_retained",
|
||||
),
|
||||
Some(trace) => {
|
||||
if trace["droppedEvents"].as_u64().unwrap_or(0) > 0 {
|
||||
gap(&mut bundle, &format!("{section}.trace"), "events_evicted");
|
||||
}
|
||||
let events = trace["events"].as_array();
|
||||
if !events.is_some_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
event["stage"] == "request.routed"
|
||||
&& event["data"]["accountPolicy"].is_object()
|
||||
})
|
||||
}) {
|
||||
gap(&mut bundle, §ion, "incident_policy_not_recorded");
|
||||
}
|
||||
for event in events.into_iter().flatten() {
|
||||
if event["stage"] == "capture.gap" || event["data"]["truncated"] == true
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"capture_gap_or_truncation",
|
||||
);
|
||||
}
|
||||
if event["stage"] == "retry.decided" && event["data"]["phase"].is_null()
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"legacy_retry_facts_not_recorded",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.attempts"),
|
||||
"best_effort_persistence",
|
||||
);
|
||||
} else {
|
||||
gap(&mut bundle, §ion, "request_detail_unavailable");
|
||||
}
|
||||
bundle.requests.push(DiagnosticRequest {
|
||||
request_id: id,
|
||||
detail,
|
||||
errors,
|
||||
});
|
||||
}
|
||||
Ok(bundle)
|
||||
}
|
||||
|
||||
async fn export_errors(
|
||||
&self,
|
||||
target: &DiagnosticTarget,
|
||||
limit: u16,
|
||||
deadline: Instant,
|
||||
) -> Result<Vec<OpsError>, AdminError> {
|
||||
timeout_at(deadline, self.store.diagnostic_errors(target, limit))
|
||||
.await
|
||||
.map_err(|_| AdminError::new(AdminErrorKind::Unavailable, "诊断采集超时"))?
|
||||
.map_err(|error| map_store_error(error, "diagnostic errors"))
|
||||
}
|
||||
}
|
||||
|
||||
fn gap(bundle: &mut DiagnosticBundle, section: &str, reason: &'static str) {
|
||||
if !bundle
|
||||
.gaps
|
||||
.iter()
|
||||
.any(|gap| gap.section == section && gap.reason == reason)
|
||||
{
|
||||
bundle.gaps.push(DiagnosticGap {
|
||||
section: section.to_owned(),
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,7 @@ const HEALTH_TIMELINE_SLOT_MINUTES: i64 = 15;
|
||||
const HEALTH_TIMELINE_MIN_SAMPLE_SIZE: u64 = 10;
|
||||
const HEALTH_TIMELINE_UNAVAILABLE_FAILURE_THRESHOLD: u64 = 3;
|
||||
const HEALTH_TIMELINE_STABLE_RELIABILITY: f64 = 95.0;
|
||||
mod export;
|
||||
mod store_cache;
|
||||
pub(crate) use store_cache::CachedObservabilityStore;
|
||||
|
||||
@@ -70,6 +71,10 @@ pub trait ObservabilityService: Send + Sync {
|
||||
dimension: DiagnosticDimension,
|
||||
) -> Result<DiagnosticsResult, AdminError>;
|
||||
async fn ops_errors(&self, query: OpsErrorQuery) -> Result<OpsErrorPage, AdminError>;
|
||||
async fn export_diagnostics(
|
||||
&self,
|
||||
target: crate::model::diagnostics::DiagnosticTarget,
|
||||
) -> Result<crate::model::diagnostics::DiagnosticBundle, AdminError>;
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultObservabilityService {
|
||||
@@ -78,6 +83,7 @@ pub(crate) struct DefaultObservabilityService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
system: Arc<dyn super::system::SystemService>,
|
||||
}
|
||||
|
||||
impl DefaultObservabilityService {
|
||||
@@ -88,6 +94,7 @@ impl DefaultObservabilityService {
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
system: Arc<dyn super::system::SystemService>,
|
||||
) -> Self {
|
||||
Self {
|
||||
timezone,
|
||||
@@ -95,6 +102,7 @@ impl DefaultObservabilityService {
|
||||
accounts,
|
||||
settings,
|
||||
providers,
|
||||
system,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,6 +200,13 @@ impl DefaultObservabilityService {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityService for DefaultObservabilityService {
|
||||
async fn export_diagnostics(
|
||||
&self,
|
||||
target: crate::model::diagnostics::DiagnosticTarget,
|
||||
) -> Result<crate::model::diagnostics::DiagnosticBundle, AdminError> {
|
||||
self.collect_diagnostics(target).await
|
||||
}
|
||||
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 有界统计缓存:身份和完整筛选参与键,删除版本阻止旧数据复活
|
||||
//! 有界统计缓存:身份和完整筛选参与键,统计发布代次使旧快照失效
|
||||
|
||||
use crate::model::observability::*;
|
||||
use crate::ports::store::{
|
||||
@@ -121,7 +121,7 @@ impl<K: Eq + Hash, T: AnalyticsValue> Cache<K, T> {
|
||||
};
|
||||
// 同键刷新只执行一次;失败不会缓存成成功,也不延长旧结果寿命
|
||||
let mut cached = slot.lock().await;
|
||||
// 等锁期间代次可能推进;取得同键锁后再核对,失败时也不能回用已删除事实
|
||||
// 等锁期间代次可能推进;取得同键锁后再核对,失败时也不能回用已被新统计替代的快照
|
||||
let generation = store.analytics_cache_generation().await?;
|
||||
if let Some(entry) = cached.as_mut()
|
||||
&& entry.value.metadata().generation == generation
|
||||
@@ -289,4 +289,12 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
self.inner.list_ops_errors(query).await
|
||||
}
|
||||
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
target: &crate::model::diagnostics::DiagnosticTarget,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<Vec<crate::model::observability::OpsError>> {
|
||||
self.inner.diagnostic_errors(target, limit).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ impl PluginResourceAccess for DefaultPluginResourceAccess {
|
||||
owner,
|
||||
resource_key,
|
||||
NewAccountGroup {
|
||||
allow_image_generation: command.allow_image_generation,
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
|
||||
@@ -305,6 +305,9 @@ impl PluginsService {
|
||||
.get(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "proxy"))?;
|
||||
if !record.enabled {
|
||||
return Err(AdminError::invalid("下载代理已禁用,请启用或改为直连"));
|
||||
}
|
||||
Ok(Some(PluginDistributionEgress::new(
|
||||
record.id,
|
||||
record.revision,
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
//! 公开原生用量读写与纯计价,插件拥有跨调用流程和恢复策略
|
||||
|
||||
mod pricing;
|
||||
mod validation;
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
auth::{AdminAuditEvent, AuditActorKind},
|
||||
usage_data::*,
|
||||
},
|
||||
ports::{provider::ProviderAdminRegistry, store::SettingsStore, usage_data::*},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[async_trait]
|
||||
pub trait UsageDataService: Send + Sync {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage>;
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult>;
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord>;
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage>;
|
||||
async fn write_records(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageRecords,
|
||||
) -> UsageDataResult<UsageWriteResult>;
|
||||
async fn evaluate_pricing(
|
||||
&self,
|
||||
request: EvaluateUsagePricing,
|
||||
) -> UsageDataResult<UsagePricingEvaluation>;
|
||||
}
|
||||
|
||||
pub struct DefaultUsageDataService {
|
||||
store: Arc<dyn UsageDataStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
}
|
||||
impl DefaultUsageDataService {
|
||||
pub fn new(
|
||||
store: Arc<dyn UsageDataStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
settings,
|
||||
providers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl UsageDataService for DefaultUsageDataService {
|
||||
async fn list_aggregates(
|
||||
&self,
|
||||
request: ListUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregatePage> {
|
||||
validation::query(&request.filter, request.cursor.as_deref(), request.limit)?;
|
||||
self.store.list_aggregates(request).await
|
||||
}
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageAggregates,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult> {
|
||||
validation::bounded(&request, request.changes.len())?;
|
||||
let mut changes = Vec::with_capacity(request.changes.len());
|
||||
for change in request.changes {
|
||||
validation::aggregate(&change)?;
|
||||
let identity = match &change {
|
||||
UsageAggregateChange::Put { aggregate } => &aggregate.identity,
|
||||
UsageAggregateChange::Delete { identity } => identity,
|
||||
};
|
||||
let action = match &change {
|
||||
UsageAggregateChange::Put { .. } => "usage.aggregate.put",
|
||||
UsageAggregateChange::Delete { .. } => "usage.aggregate.delete",
|
||||
};
|
||||
let audit = audit_event(
|
||||
actor,
|
||||
action,
|
||||
"usage_aggregate",
|
||||
&identity.key,
|
||||
vec!["aggregate".into()],
|
||||
);
|
||||
changes.push((change, audit));
|
||||
}
|
||||
self.store
|
||||
.write_aggregates(StoredAggregateWrite { changes })
|
||||
.await
|
||||
}
|
||||
async fn get_record(&self, request: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
validation::text(&request.id, 128)?;
|
||||
self.store.get_record(request).await
|
||||
}
|
||||
async fn list_records(&self, request: ListUsageRecords) -> UsageDataResult<UsageRecordPage> {
|
||||
validation::query(&request.filter, request.cursor.as_deref(), request.limit)?;
|
||||
self.store.list_records(request).await
|
||||
}
|
||||
async fn write_records(
|
||||
&self,
|
||||
actor: &UsageActor,
|
||||
request: WriteUsageRecords,
|
||||
) -> UsageDataResult<UsageWriteResult> {
|
||||
validation::bounded(&request, request.changes.len())?;
|
||||
let mut changes = Vec::with_capacity(request.changes.len());
|
||||
for change in request.changes {
|
||||
validation::change(&change)?;
|
||||
let action = match &change {
|
||||
UsageRecordChange::Put { .. } => "usage.put",
|
||||
UsageRecordChange::Update { .. } => "usage.update",
|
||||
UsageRecordChange::Delete { .. } => "usage.delete",
|
||||
};
|
||||
let fields = match &change {
|
||||
UsageRecordChange::Update { tokens, cost, .. } => tokens
|
||||
.as_ref()
|
||||
.map(|_| "tokens".to_owned())
|
||||
.into_iter()
|
||||
.chain(cost.as_ref().map(|_| "cost".to_owned()))
|
||||
.collect(),
|
||||
_ => vec!["record".into()],
|
||||
};
|
||||
let audit = audit_event(actor, action, "model_request", change.id(), fields);
|
||||
changes.push((change, audit));
|
||||
}
|
||||
self.store.write_records(StoredUsageWrite { changes }).await
|
||||
}
|
||||
async fn evaluate_pricing(
|
||||
&self,
|
||||
request: EvaluateUsagePricing,
|
||||
) -> UsageDataResult<UsagePricingEvaluation> {
|
||||
validation::bounded(&request, request.items.len())?;
|
||||
let stored = self
|
||||
.settings
|
||||
.load_pricing()
|
||||
.await
|
||||
.map_err(|_| UsageDataError::unavailable("读取价目失败"))?;
|
||||
let prices = gateway_core::metering::merge_pricing(stored.synced, &stored.overrides);
|
||||
let mut items = Vec::with_capacity(request.items.len());
|
||||
for input in request.items {
|
||||
validation::text(&input.id, 128)?;
|
||||
items.push(self.evaluate_one(input, &prices)?);
|
||||
}
|
||||
Ok(UsagePricingEvaluation { items })
|
||||
}
|
||||
}
|
||||
|
||||
fn audit_event(
|
||||
actor: &UsageActor,
|
||||
action: &str,
|
||||
entity_kind: &str,
|
||||
id: &str,
|
||||
changed_fields: Vec<String>,
|
||||
) -> AdminAuditEvent {
|
||||
let actor_ref = match actor {
|
||||
UsageActor::Plugin {
|
||||
instance_id,
|
||||
publisher,
|
||||
name,
|
||||
..
|
||||
} => format!("plugin:{publisher}/{name}:{instance_id}"),
|
||||
UsageActor::Admin { subject } => format!("admin:{subject}"),
|
||||
UsageActor::System => "system".into(),
|
||||
};
|
||||
AdminAuditEvent {
|
||||
id: format!("audit_{}", uuid::Uuid::now_v7().simple()),
|
||||
actor_kind: AuditActorKind::System,
|
||||
actor_admin_user_id: None,
|
||||
actor_ref,
|
||||
request_id: None,
|
||||
action: action.into(),
|
||||
entity_kind: entity_kind.into(),
|
||||
entity_ref: id.to_owned(),
|
||||
config_revision: None,
|
||||
changed_fields,
|
||||
occurred_at: chrono::Utc::now(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! 复用 Provider 的单请求 Token 计价规则,不写入历史或账本
|
||||
|
||||
use super::{DefaultUsageDataService, validation};
|
||||
use crate::{
|
||||
model::usage_data::*,
|
||||
ports::provider::{ProviderAdminErrorKind, ProviderUsagePricingInput},
|
||||
};
|
||||
use gateway_core::{
|
||||
metering::{PricingOverrides, Usage},
|
||||
routing::ProviderKind,
|
||||
};
|
||||
|
||||
impl DefaultUsageDataService {
|
||||
pub(super) fn evaluate_one(
|
||||
&self,
|
||||
input: UsagePricingInput,
|
||||
prices: &PricingOverrides,
|
||||
) -> UsageDataResult<UsagePricingResult> {
|
||||
validation::text(&input.provider, 128)?;
|
||||
validation::text(&input.model, 128)?;
|
||||
validation::tokens(&input.tokens)?;
|
||||
let unavailable = |reason: &str| UsagePricingResult::NotEstimable {
|
||||
id: input.id.clone(),
|
||||
reason: reason.into(),
|
||||
};
|
||||
let kind = ProviderKind::new(input.provider.clone())
|
||||
.map_err(|_| UsageDataError::invalid("Provider 无效"))?;
|
||||
let Ok(provider) = self.providers.require(&kind) else {
|
||||
return Ok(unavailable("Provider 没有可用的计价规则"));
|
||||
};
|
||||
let tokens = &input.tokens;
|
||||
let price = match &input.price {
|
||||
UsagePriceSelection::Current => prices
|
||||
.get(&input.provider)
|
||||
.and_then(|models| models.get(&input.model)),
|
||||
UsagePriceSelection::Override { price } => {
|
||||
price.validate().map_err(UsageDataError::invalid)?;
|
||||
Some(price.as_ref())
|
||||
}
|
||||
};
|
||||
let request = ProviderUsagePricingInput {
|
||||
upstream_model_id: input.model.clone(),
|
||||
service_tier: input.service_tier.clone(),
|
||||
usage: Usage {
|
||||
input_tokens: tokens.input_tokens.map(UsageCount::get),
|
||||
output_tokens: tokens.output_tokens.map(UsageCount::get),
|
||||
cached_input_tokens: tokens.cached_input_tokens.map(UsageCount::get),
|
||||
cache_write_input_tokens: tokens.cache_write_input_tokens.map(UsageCount::get),
|
||||
reasoning_output_tokens: tokens.reasoning_output_tokens.map(UsageCount::get),
|
||||
image_input_tokens: tokens.image_input_tokens.map(UsageCount::get),
|
||||
image_output_tokens: tokens.image_output_tokens.map(UsageCount::get),
|
||||
total_tokens: tokens.total_tokens.map(UsageCount::get),
|
||||
},
|
||||
};
|
||||
let calculated = match provider.evaluate_usage_pricing(&request, price) {
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => return Ok(unavailable("用量不完整、包含图像或价格不可用")),
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderAdminErrorKind::Unsupported | ProviderAdminErrorKind::Invalid
|
||||
) =>
|
||||
{
|
||||
return Ok(unavailable("Provider 不支持该计价输入"));
|
||||
}
|
||||
Err(_) => return Err(UsageDataError::unavailable("Provider 计价失败")),
|
||||
};
|
||||
Ok(UsagePricingResult::Estimated {
|
||||
id: input.id,
|
||||
cost: UsageCost::Known {
|
||||
source: UsageCostSource::Calculated,
|
||||
amount: calculated.total_amount().amount().canonical(),
|
||||
currency: calculated.total_amount().currency().as_str().into(),
|
||||
},
|
||||
breakdown: UsagePricingBreakdown {
|
||||
input_amount: calculated.input_amount().amount().canonical(),
|
||||
output_amount: calculated.output_amount().amount().canonical(),
|
||||
cache_read_amount: calculated.cache_read_amount().amount().canonical(),
|
||||
cache_write_amount: calculated.cache_write_amount().amount().canonical(),
|
||||
standard_amount: calculated.standard_amount().amount().canonical(),
|
||||
long_context_billing_applied: calculated.long_context_billing_applied(),
|
||||
service_tier: calculated.service_tier().map(str::to_owned),
|
||||
custom_multiplier_bps: calculated.custom_multiplier_bps(),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
//! 用量输入沿用原生记录的数值、时间和资源边界
|
||||
|
||||
use crate::model::usage_data::*;
|
||||
use gateway_core::metering::Decimal;
|
||||
use serde::Serialize;
|
||||
|
||||
pub(super) fn text(value: &str, max: usize) -> UsageDataResult<()> {
|
||||
if value.trim().is_empty() || value.len() > max || value.chars().any(char::is_control) {
|
||||
return Err(UsageDataError::invalid("文本为空、过长或包含控制字符"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn bounded(value: &impl Serialize, count: usize) -> UsageDataResult<()> {
|
||||
if count == 0
|
||||
|| count > USAGE_DATA_MAX_TARGETS
|
||||
|| serde_json::to_vec(value)
|
||||
.map_err(|_| UsageDataError::invalid("请求编码失败"))?
|
||||
.len()
|
||||
> USAGE_DATA_MAX_REQUEST_BYTES
|
||||
{
|
||||
return Err(UsageDataError::new(
|
||||
UsageErrorKind::CapacityExceeded,
|
||||
"每次操作接受 1~500 项且不超过 2 MiB",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn tokens(value: &UsageTokens) -> UsageDataResult<()> {
|
||||
if [
|
||||
value.input_tokens,
|
||||
value.output_tokens,
|
||||
value.cached_input_tokens,
|
||||
value.cache_write_input_tokens,
|
||||
value.reasoning_output_tokens,
|
||||
value.image_input_tokens,
|
||||
value.image_output_tokens,
|
||||
value.total_tokens,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.any(|v| v.get() > i64::MAX as u64)
|
||||
{
|
||||
return Err(UsageDataError::invalid("Token 超出原生存储范围"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn cost(value: &UsageCost) -> UsageDataResult<()> {
|
||||
if let UsageCost::Known {
|
||||
amount, currency, ..
|
||||
} = value
|
||||
{
|
||||
let _ = amount
|
||||
.parse::<Decimal>()
|
||||
.map_err(|_| UsageDataError::invalid("费用不是有效十进制数"))?;
|
||||
if currency.len() != 3 || !currency.bytes().all(|b| b.is_ascii_uppercase()) {
|
||||
return Err(UsageDataError::invalid("费用或币种不符合原生存储范围"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn change(value: &UsageRecordChange) -> UsageDataResult<()> {
|
||||
text(value.id(), 128)?;
|
||||
match value {
|
||||
UsageRecordChange::Put { data, .. } => {
|
||||
text(&data.client_key_ref, 128)?;
|
||||
for v in [
|
||||
&data.provider,
|
||||
&data.account_ref,
|
||||
&data.requested_model,
|
||||
&data.model,
|
||||
&data.service_tier,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(v, 128)?;
|
||||
}
|
||||
for v in [
|
||||
&data.protocol,
|
||||
&data.operation,
|
||||
&data.endpoint,
|
||||
&data.client_transport,
|
||||
] {
|
||||
text(v, 256)?;
|
||||
}
|
||||
if data.outcome == UsageOutcome::Running
|
||||
|| data
|
||||
.completed_at_ms
|
||||
.is_none_or(|end| end < data.started_at_ms)
|
||||
{
|
||||
return Err(UsageDataError::invalid("只允许写入已结束的请求明细"));
|
||||
}
|
||||
for at in [Some(data.started_at_ms), data.completed_at_ms]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if chrono::DateTime::from_timestamp_millis(at).is_none() {
|
||||
return Err(UsageDataError::invalid("请求时间无效"));
|
||||
}
|
||||
}
|
||||
tokens(&data.tokens)?;
|
||||
cost(&data.cost)?;
|
||||
}
|
||||
UsageRecordChange::Update {
|
||||
tokens: new_tokens,
|
||||
cost: new_cost,
|
||||
..
|
||||
} => {
|
||||
if new_tokens.is_none() && new_cost.is_none() {
|
||||
return Err(UsageDataError::invalid("修改必须指定 Token 或费用"));
|
||||
}
|
||||
if let Some(value) = new_tokens {
|
||||
tokens(value)?;
|
||||
}
|
||||
if let Some(value) = new_cost {
|
||||
cost(value)?;
|
||||
}
|
||||
}
|
||||
UsageRecordChange::Delete { .. } => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(super) fn query(
|
||||
filter: &UsageRecordFilter,
|
||||
cursor: Option<&str>,
|
||||
limit: Option<u16>,
|
||||
) -> UsageDataResult<()> {
|
||||
if limit.is_some_and(|v| v == 0 || v > USAGE_DATA_MAX_PAGE_SIZE)
|
||||
|| filter.start_ms >= filter.end_ms
|
||||
|| chrono::DateTime::from_timestamp_millis(filter.start_ms).is_none()
|
||||
|| chrono::DateTime::from_timestamp_millis(filter.end_ms).is_none()
|
||||
{
|
||||
return Err(UsageDataError::invalid("查询时间范围或分页大小无效"));
|
||||
}
|
||||
for v in [
|
||||
&filter.provider,
|
||||
&filter.model,
|
||||
&filter.account_ref,
|
||||
&filter.client_key_ref,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(v, 128)?;
|
||||
}
|
||||
if cursor.is_some_and(|v| v.len() > 4096) {
|
||||
return Err(UsageDataError::invalid("游标过长"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn aggregate(value: &UsageAggregateChange) -> UsageDataResult<()> {
|
||||
let identity = match value {
|
||||
UsageAggregateChange::Put { aggregate } => &aggregate.identity,
|
||||
UsageAggregateChange::Delete { identity } => identity,
|
||||
};
|
||||
text(&identity.key, 128)?;
|
||||
if chrono::DateTime::from_timestamp_millis(identity.bucket_start_ms).is_none() {
|
||||
return Err(UsageDataError::invalid("聚合归属时间无效"));
|
||||
}
|
||||
let d = &identity.dimensions;
|
||||
for value in [
|
||||
&d.provider,
|
||||
&d.model,
|
||||
&d.account_ref,
|
||||
&d.client_key_ref,
|
||||
&d.operation,
|
||||
&d.transport,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
text(value, 128)?;
|
||||
}
|
||||
if let UsageAggregateChange::Put { aggregate } = value {
|
||||
if aggregate.costs.len() > 32 {
|
||||
return Err(UsageDataError::invalid("费用币种数量超过限制"));
|
||||
}
|
||||
for (currency, amount) in &aggregate.costs {
|
||||
if currency.len() != 3
|
||||
|| !currency.bytes().all(|b| b.is_ascii_uppercase())
|
||||
|| amount.len() > 128
|
||||
{
|
||||
return Err(UsageDataError::invalid("聚合费用格式无效"));
|
||||
}
|
||||
amount
|
||||
.strip_prefix('-')
|
||||
.unwrap_or(amount)
|
||||
.parse::<Decimal>()
|
||||
.map_err(|_| UsageDataError::invalid("聚合费用不是有效十进制数"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,7 +4,13 @@ use gateway_admin::model::retention::{RetentionPolicy, RetentionTarget};
|
||||
|
||||
#[test]
|
||||
fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows() {
|
||||
for (usage, ops, audit) in [(30, 1, 1), (31, 0, 1), (31, 1, 0)] {
|
||||
for (usage, ops, audit) in [
|
||||
(0, 1, 1),
|
||||
(32, 1, 1),
|
||||
(u32::MAX, 1, 1),
|
||||
(1, 0, 1),
|
||||
(1, 1, 0),
|
||||
] {
|
||||
assert!(RetentionPolicy::try_new(usage, ops, audit).is_err());
|
||||
}
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
@@ -12,3 +18,13 @@ fn retention_policy_rejects_invalid_windows_and_preserves_independent_windows()
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 7);
|
||||
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_retention_accepts_short_windows_without_changing_other_targets() {
|
||||
for days in [1, 3, 7, 15, 30, 31] {
|
||||
let policy = RetentionPolicy::try_new(days, 30, 90).unwrap();
|
||||
assert_eq!(policy.days(RetentionTarget::ModelRequests), days);
|
||||
assert_eq!(policy.days(RetentionTarget::OpsEvents), 30);
|
||||
assert_eq!(policy.days(RetentionTarget::AdminAuditEvents), 90);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,6 +317,7 @@ impl AccountRuntimeStore for FakeRuntimeStore {
|
||||
fn group_record() -> AccountGroupRecord {
|
||||
let now = Utc::now();
|
||||
AccountGroupRecord {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
id: group_id(),
|
||||
name: "Primary".to_owned(),
|
||||
@@ -390,6 +391,7 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
.create(
|
||||
&context,
|
||||
CreateAccountGroup {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
name: " invalid".to_owned(),
|
||||
description: None,
|
||||
@@ -403,6 +405,7 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
.update(
|
||||
&context,
|
||||
UpdateAccountGroup {
|
||||
allow_image_generation: None,
|
||||
fast_mode: None,
|
||||
id: record.id,
|
||||
name: "valid".to_owned(),
|
||||
|
||||
@@ -3137,6 +3137,7 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
|
||||
pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
outbound_proxy_enabled: true,
|
||||
notes: None,
|
||||
model_access: Default::default(),
|
||||
outbound_proxy: None,
|
||||
|
||||
@@ -15,6 +15,7 @@ mod plugins;
|
||||
mod proxies;
|
||||
mod settings;
|
||||
mod system;
|
||||
mod usage_data;
|
||||
|
||||
use std::{
|
||||
str::FromStr,
|
||||
@@ -273,6 +274,7 @@ impl AdminHarness {
|
||||
self.plugin_store,
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
Arc::new(plugins::TestPluginPorts),
|
||||
),
|
||||
gateway_admin::AdminRuntimePorts {
|
||||
timezone: Default::default(),
|
||||
@@ -747,6 +749,14 @@ impl ClientKeyStore for UnavailableStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnavailableStore {
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
_target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
||||
_limit: u16,
|
||||
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
||||
Err(unavailable("diagnostic errors"))
|
||||
}
|
||||
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
|
||||
@@ -945,6 +945,14 @@ impl FixtureObservabilityStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for FixtureObservabilityStore {
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
_target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
||||
_limit: u16,
|
||||
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
||||
Err(super::unavailable("diagnostic errors"))
|
||||
}
|
||||
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
@@ -1194,6 +1202,7 @@ fn observation_range(end: DateTime<Utc>) -> TimeRange {
|
||||
|
||||
fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
UsageQuery {
|
||||
status: Default::default(),
|
||||
range: observation_range(now),
|
||||
filter: UsageFilter::default(),
|
||||
current_page: 1,
|
||||
@@ -1208,6 +1217,7 @@ fn total_record(
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
id: id.to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
|
||||
@@ -534,6 +534,7 @@ fn preview_fixture() -> Fixture {
|
||||
fn source_proxy() -> ProxyRecord {
|
||||
let now = chrono::Utc::now();
|
||||
ProxyRecord {
|
||||
enabled: true,
|
||||
auto_location: false,
|
||||
detected_location: None,
|
||||
location: None,
|
||||
@@ -608,6 +609,38 @@ async fn saved_source_proxy_is_resolved_for_queries_and_frozen_into_verified_sou
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disabled_source_proxy_rejects_queries_and_downloads_before_network_access() {
|
||||
let mut fixture = preview_fixture();
|
||||
fixture.binding.get_mut().unwrap().outbound_proxy_id = Some("source-proxy".into());
|
||||
let mut proxy = source_proxy();
|
||||
proxy.enabled = false;
|
||||
*fixture.proxy.get_mut().unwrap() = Some(proxy);
|
||||
let fixture = Arc::new(fixture);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.service()
|
||||
.check_update("test.example", &[])
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
let mut request = preview_request();
|
||||
request.outbound_proxy_id = Some("source-proxy".into());
|
||||
assert_eq!(
|
||||
fixture
|
||||
.service()
|
||||
.verify_remote(request)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.kind(),
|
||||
AdminErrorKind::Invalid
|
||||
);
|
||||
assert!(fixture.egresses.lock().unwrap().is_empty());
|
||||
assert_eq!(*fixture.downloads.lock().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_revision_change_during_download_rejects_the_verified_candidate() {
|
||||
let mut fixture = preview_fixture();
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 插件管理用例的测试入口与共用端口替身
|
||||
|
||||
mod usage_data;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_admin::model::plugins::distribution::{
|
||||
DownloadedPlugin, GithubReleaseQuery, PluginRelease, RemotePluginLocation, SourceCredential,
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
//! 通用用量端口的插件管理测试夹具,非本测试调用路径禁止访问
|
||||
use super::TestPluginPorts;
|
||||
use gateway_admin::{model::usage_data::*, ports::usage_data::*};
|
||||
#[async_trait::async_trait]
|
||||
impl UsageDataStore for TestPluginPorts {
|
||||
async fn get_record(&self, _: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_records(&self, _: ListUsageRecords) -> UsageDataResult<UsageRecordPage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_records(&self, _: StoredUsageWrite) -> UsageDataResult<UsageWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_aggregates(&self, _: ListUsageAggregates) -> UsageDataResult<UsageAggregatePage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
_: StoredAggregateWrite,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
}
|
||||
@@ -126,6 +126,7 @@ async fn authorization_uses_selected_proxy_regardless_of_probe_status() {
|
||||
.accounts(FakeAccountStore::new(kind, events.clone()))
|
||||
.proxies(Arc::new(TestProxies {
|
||||
record: Some(ProxyRecord {
|
||||
enabled: true,
|
||||
auto_location: false,
|
||||
detected_location: None,
|
||||
location: None,
|
||||
|
||||
@@ -82,6 +82,7 @@ fn contract_type(mut item: Item, name: &str) -> Item {
|
||||
| "RegeneratedAdminApiKey"
|
||||
| "CodexPrivacyPolicy"
|
||||
| "PrivacyRule"
|
||||
| "PrivacyStep"
|
||||
| "PrivacyPreviewRequest"
|
||||
| "PrivacyPreviewResult"
|
||||
);
|
||||
@@ -137,6 +138,7 @@ fn sdk_settings_contract_matches_host_declarations() {
|
||||
"PrivacyScope",
|
||||
"PrivacyAction",
|
||||
"PrivacyRule",
|
||||
"PrivacyStep",
|
||||
"PrivacyRuleOutcome",
|
||||
"PrivacyPreviewRequest",
|
||||
"PrivacyPreviewResult",
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//! 用量服务的 wire 精度、可信主体和独立 SDK 合同校验
|
||||
use gateway_admin::model::usage_data::*;
|
||||
use serde_json::json;
|
||||
#[test]
|
||||
fn counters_preserve_integer_precision_and_reject_json_numbers() {
|
||||
assert_eq!(
|
||||
serde_json::from_value::<UsageCount>(json!(u64::MAX.to_string()))
|
||||
.unwrap()
|
||||
.get(),
|
||||
u64::MAX
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<UsageDelta>(json!(i64::MIN.to_string()))
|
||||
.unwrap()
|
||||
.get(),
|
||||
i64::MIN
|
||||
);
|
||||
for value in [json!(1), json!("1.5"), json!("+1")] {
|
||||
assert!(serde_json::from_value::<UsageCount>(value.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageDelta>(value).is_err());
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_absent() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let denied = registry
|
||||
.call("usage.records.write", json!({"changes":[]}))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(denied.message.contains("trusted caller"));
|
||||
assert!(
|
||||
registry
|
||||
.call(
|
||||
"usage.records.write",
|
||||
json!({"changes":[],"actor":{"kind":"system"}})
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let ops = registry
|
||||
.operations()
|
||||
.filter(|op| op.starts_with("usage.") || *op == "pricing.evaluate")
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
ops,
|
||||
std::collections::BTreeSet::from([
|
||||
"usage.records.get",
|
||||
"usage.records.list",
|
||||
"usage.records.write",
|
||||
"usage.aggregates.list",
|
||||
"usage.aggregates.write",
|
||||
"pricing.evaluate"
|
||||
])
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn sdk_wire_types_match_host_contracts() {
|
||||
use quote::ToTokens as _;
|
||||
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let items = |path: std::path::PathBuf| {
|
||||
syn::parse_file(&std::fs::read_to_string(path).unwrap())
|
||||
.unwrap()
|
||||
.items
|
||||
.into_iter()
|
||||
.filter(|item| {
|
||||
matches!(
|
||||
item,
|
||||
syn::Item::Struct(_)
|
||||
| syn::Item::Enum(_)
|
||||
| syn::Item::Impl(_)
|
||||
| syn::Item::Type(_)
|
||||
)
|
||||
})
|
||||
.map(|item| item.into_token_stream().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for name in [
|
||||
"value.rs",
|
||||
"record.rs",
|
||||
"query.rs",
|
||||
"aggregate.rs",
|
||||
"pricing.rs",
|
||||
"error.rs",
|
||||
] {
|
||||
assert_eq!(
|
||||
items(root.join("src/model/usage_data").join(name)),
|
||||
items(
|
||||
root.join("../gateway-plugin/sdk/src/call/services/usage")
|
||||
.join(name)
|
||||
),
|
||||
"{name}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn batch_and_wire_limits_fail_before_reaching_storage() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let changes = (0..501)
|
||||
.map(|i| json!({"kind":"delete","id":format!("item_{i}")}))
|
||||
.collect::<Vec<_>>();
|
||||
let too_many = gateway_admin::public_service::scope_actor(
|
||||
UsageActor::System,
|
||||
registry.call("usage.records.write", json!({"changes":changes})),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(too_many.kind, "capacity_exceeded");
|
||||
let oversized = registry
|
||||
.call(
|
||||
"usage.records.get",
|
||||
json!({"id":"x".repeat(USAGE_DATA_MAX_REQUEST_BYTES)}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(oversized.kind, "capacity_exceeded");
|
||||
}
|
||||
@@ -25,6 +25,7 @@ gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
headers = "0.4.1"
|
||||
regex = "1.13.1"
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
@@ -32,7 +33,7 @@ sha2 = "0.11.0"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "sync", "time"] }
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
tower-http = { version = "0.7.0", features = ["compression-gzip", "cors", "fs", "request-id", "timeout", "trace"] }
|
||||
tracing = "0.1.44"
|
||||
url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
@@ -91,9 +91,15 @@ impl ListAccountGroupsQuery {
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_allow_image_generation() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct CreateAccountGroupRequest {
|
||||
#[serde(default = "default_allow_image_generation")]
|
||||
allow_image_generation: bool,
|
||||
#[serde(default)]
|
||||
fast_mode: FastMode,
|
||||
name: String,
|
||||
@@ -104,6 +110,7 @@ struct CreateAccountGroupRequest {
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateAccountGroupRequest {
|
||||
allow_image_generation: Option<bool>,
|
||||
fast_mode: Option<FastMode>,
|
||||
id: String,
|
||||
name: String,
|
||||
@@ -120,6 +127,7 @@ struct AccountGroupIdRequest {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupView {
|
||||
allow_image_generation: bool,
|
||||
fast_mode: FastMode,
|
||||
id: String,
|
||||
name: String,
|
||||
@@ -167,6 +175,7 @@ impl From<(AccountGroupRecord, crate::time::TimePresenter)> for AccountGroupView
|
||||
name: record.name,
|
||||
description: record.description,
|
||||
fast_mode: record.fast_mode,
|
||||
allow_image_generation: record.allow_image_generation,
|
||||
color: record.color.as_str().to_owned(),
|
||||
enabled: record.enabled,
|
||||
member_count: record.member_count,
|
||||
@@ -362,6 +371,7 @@ where
|
||||
.create(
|
||||
&auth.context().mutation_context(),
|
||||
CreateAccountGroup {
|
||||
allow_image_generation: request.allow_image_generation,
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
fast_mode: request.fast_mode,
|
||||
@@ -390,6 +400,7 @@ where
|
||||
.update(
|
||||
&auth.context().mutation_context(),
|
||||
UpdateAccountGroup {
|
||||
allow_image_generation: request.allow_image_generation,
|
||||
id: group_id(request.id)?,
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
//! 管理端普通 JSON 响应的压缩范围与大小门槛
|
||||
|
||||
use http::{Response, header};
|
||||
use http_body::Body;
|
||||
use tower_http::compression::predicate::Predicate;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct AdminJson;
|
||||
|
||||
impl Predicate for AdminJson {
|
||||
fn should_compress<B: Body>(&self, response: &Response<B>) -> bool {
|
||||
// 只处理长度已知的 JSON,不为判断门槛收集流式正文
|
||||
response
|
||||
.body()
|
||||
.size_hint()
|
||||
.exact()
|
||||
.is_some_and(|size| size >= 1024)
|
||||
&& response
|
||||
.headers()
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.split(';').next())
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("application/json"))
|
||||
}
|
||||
}
|
||||
@@ -9,12 +9,14 @@ use axum::{
|
||||
response::Response,
|
||||
routing::any,
|
||||
};
|
||||
use tower_http::compression::{CompressionLayer, CompressionLevel};
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backups;
|
||||
pub mod client_keys;
|
||||
mod compression;
|
||||
pub(crate) mod diagnostics;
|
||||
mod extract;
|
||||
pub mod observability;
|
||||
@@ -54,6 +56,12 @@ where
|
||||
.method_not_allowed_fallback(method_not_allowed)
|
||||
.route("/api/admin", any(admin_not_found))
|
||||
.route("/api/admin/{*path}", any(admin_not_found))
|
||||
.layer(
|
||||
CompressionLayer::new()
|
||||
.quality(CompressionLevel::Fastest)
|
||||
.compress_when(compression::AdminJson),
|
||||
)
|
||||
.merge(plugins::public_router().method_not_allowed_fallback(method_not_allowed))
|
||||
.layer(middleware::map_response(no_store))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
//! 诊断文件的 HTTP 下载与受控数据投影,普通详情响应不携带时间线
|
||||
|
||||
use axum::{extract::State, http::header, response::IntoResponse};
|
||||
use gateway_admin::model::{
|
||||
diagnostics::{DiagnosticBundle, DiagnosticTarget},
|
||||
observability::OpsError,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{
|
||||
AdminAuth, AdminError, AdminQuery, map_service_error, ops_error_view, usage_detail_view,
|
||||
};
|
||||
use crate::{auth::SessionState, time::TimePresenter};
|
||||
|
||||
mod privacy;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub(super) struct ExportQuery {
|
||||
request_id: Option<String>,
|
||||
error_id: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) async fn download<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<ExportQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let target = match (query.request_id, query.error_id) {
|
||||
(Some(id), None) => DiagnosticTarget::Request(id),
|
||||
(None, Some(id)) => DiagnosticTarget::Operation(id),
|
||||
_ => {
|
||||
return Err(map_service_error(
|
||||
gateway_admin::model::AdminError::invalid("请指定一个请求 ID 或错误记录 ID"),
|
||||
));
|
||||
}
|
||||
};
|
||||
let bundle = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.export_diagnostics(target)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let time = TimePresenter::new(state.admin_services().timezone());
|
||||
let document = privacy::redact(document(bundle, time));
|
||||
let body = serde_json::to_vec_pretty(&document).map_err(|_| {
|
||||
map_service_error(gateway_admin::model::AdminError::internal(
|
||||
"诊断文件生成失败",
|
||||
))
|
||||
})?;
|
||||
Ok((
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/json; charset=utf-8"),
|
||||
(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=diagnostics.json",
|
||||
),
|
||||
(header::CACHE_CONTROL, "private, no-store"),
|
||||
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
|
||||
],
|
||||
body,
|
||||
))
|
||||
}
|
||||
|
||||
fn document(bundle: DiagnosticBundle, time: TimePresenter) -> Value {
|
||||
let target = match &bundle.target {
|
||||
DiagnosticTarget::Request(id) => json!({"requestId": id}),
|
||||
DiagnosticTarget::Operation(id) => json!({"errorId": id}),
|
||||
};
|
||||
let requests = bundle.requests.into_iter().map(|record| {
|
||||
let evidence = record.detail.map(|mut detail| {
|
||||
let trace = detail.trace.take();
|
||||
let related = std::mem::take(&mut detail.related_requests);
|
||||
let configuration = json!({
|
||||
"revision": detail.request.config_revision,
|
||||
"routingScope": detail.request.routing_scope,
|
||||
"routingGroupIds": detail.request.routing_group_refs,
|
||||
"deadlineAt": detail.request.deadline_at,
|
||||
});
|
||||
// 页面 DTO 没有展示的尝试原因与等待事实仍属于导出证据
|
||||
let attempt_details = detail.attempts.iter().map(|attempt| json!({
|
||||
"id": attempt.id, "attemptIndex": attempt.attempt_index,
|
||||
"component": attempt.component, "operation": attempt.operation,
|
||||
"message": attempt.message, "retryAfterMs": attempt.retry_after_ms,
|
||||
})).collect::<Vec<_>>();
|
||||
json!({
|
||||
"detail": usage_detail_view(detail, time), "trace": trace,
|
||||
"relatedRequests": related, "incidentConfiguration": configuration,
|
||||
"attemptDetails": attempt_details,
|
||||
})
|
||||
});
|
||||
json!({
|
||||
"requestId": record.request_id,
|
||||
"evidence": evidence,
|
||||
"errors": record.errors.into_iter().map(|error| export_error(error, time)).collect::<Vec<_>>(),
|
||||
})
|
||||
}).collect::<Vec<_>>();
|
||||
json!({
|
||||
"schemaVersion": 5,
|
||||
"source": "gateway.diagnostics",
|
||||
"exportPolicy": "support_credentials_redacted",
|
||||
"target": target,
|
||||
"collectedAt": bundle.collected_at,
|
||||
"environment": {
|
||||
"scope": "export_time", "observedAt": bundle.collected_at,
|
||||
"gateway": bundle.environment.map(|version| json!({
|
||||
"version": version.version, "gitSha": version.git_sha,
|
||||
"buildTime": version.build_time, "deploymentMode": version.deployment_mode,
|
||||
})),
|
||||
},
|
||||
"error": bundle.error.map(|error| export_error(error, time)),
|
||||
"requests": requests,
|
||||
"availability": {"gaps": bundle.gaps.into_iter().map(|gap| json!({"section": gap.section, "reason": gap.reason})).collect::<Vec<_>>()},
|
||||
"omitted": ["credentials_and_authentication", "account_contact_details", "uncollected_wire_payloads", "unrelated_requests_and_system_secrets"],
|
||||
"notes": [
|
||||
"诊断包包含错误原因链、上游错误与业务上下文,用于故障分析",
|
||||
"认证字段、账号联系信息与可识别的内嵌凭据已脱敏,任意业务内容不属于自动识别范围",
|
||||
"environment 为导出时环境,incidentConfiguration 与 request.routed 为事发配置事实,当前版本不能代替事发版本",
|
||||
"缺失、截断、淘汰与读取失败见 availability.gaps,没有记录不表示没有发生",
|
||||
"attempts 为异步 best-effort 投影,firstTokenMs 为空不证明没有输出",
|
||||
"未采集的请求正文、过期记录、上游内部状态与退出前未落盘信息无法由导出补回"
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
// 复用错误字段投影,原始详情直接进入脱敏遍历,避免先为页面缩进再解析一遍
|
||||
fn export_error(mut error: OpsError, time: TimePresenter) -> Value {
|
||||
let details = error.error_details.take();
|
||||
let mut value = json!(ops_error_view(error, time));
|
||||
value["errorDetails"] = json!(details);
|
||||
value
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! 诊断文件按认证与观测合同移除敏感字段,保留错误事实并明确导出截断
|
||||
|
||||
use regex::Regex;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
const REDACTED: &str = "<redacted>";
|
||||
const MAX_TEXT: usize = 256 * 1024;
|
||||
static INLINE: LazyLock<Vec<(Regex, &str)>> = LazyLock::new(|| {
|
||||
[
|
||||
(r"(?i)\b((?:set-cookie|cookie2?|(?:proxy-)?authorization|x-api-key|x-oai-is(?:-update)?|x-oai-attestation)\s*:\s*)[^\r\n]+", "${1}<redacted>"),
|
||||
(r#"(?i)\b(Bearer|Basic)\s+[^\s"'<>;,]+"#, "${1} <redacted>"),
|
||||
// 官方 TokenData 的 access_token / id_token 为 JWT,不按猜测的令牌前缀过滤普通文本
|
||||
(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b", "<redacted>"),
|
||||
(r"(?i)([a-z][a-z0-9+.-]*://)[^\s/@]+@", "${1}<redacted>@"),
|
||||
(r#"(?i)\b((?:(?:personal[_-]?)?access[_-]?token|refresh[_-]?token|id[_-]?token|(?:(?:openai|x)[_-]?)?api[_-]?key|(?:current[_-]?|new[_-]?)?password|authorization|proxy[_-]?authorization|x[_-]?oai[_-]?(?:is(?:[_-]?update)?|attestation))["']?\s*[=:]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s&,;]+)"#, "${1}<redacted>"),
|
||||
].into_iter().map(|(pattern, replacement)| (Regex::new(pattern).expect("static credential pattern"), replacement)).collect()
|
||||
});
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct Report {
|
||||
credential_fields: usize,
|
||||
personal_fields: usize,
|
||||
inline_credentials: usize,
|
||||
truncated_values: usize,
|
||||
}
|
||||
|
||||
struct Redactor {
|
||||
report: Report,
|
||||
bytes: usize,
|
||||
nodes: usize,
|
||||
}
|
||||
|
||||
pub(super) fn redact(mut value: Value) -> Value {
|
||||
let mut redactor = Redactor {
|
||||
report: Report::default(),
|
||||
bytes: 8 * 1024 * 1024,
|
||||
nodes: 100_000,
|
||||
};
|
||||
// 先处理有界文件信息,正文耗尽预算也不能删除 schema、目标或既有缺口
|
||||
let mut error = value["error"].take();
|
||||
let mut requests = value["requests"].take();
|
||||
redactor.visit(&mut value, 0);
|
||||
redactor.visit(&mut error, 0);
|
||||
redactor.visit(&mut requests, 0);
|
||||
value["error"] = error;
|
||||
value["requests"] = requests;
|
||||
if redactor.report.truncated_values > 0 {
|
||||
// 缺口清单可能也触及预算,顶层统计始终保留以免截断变成静默丢失
|
||||
value["exportTruncated"] = json!(true);
|
||||
if !value["availability"]["gaps"].is_array() {
|
||||
value["availability"] = json!({"gaps": []});
|
||||
}
|
||||
value["availability"]["gaps"]
|
||||
.as_array_mut()
|
||||
.expect("gap array")
|
||||
.push(json!({
|
||||
"section": "export", "reason": "export_size_limit",
|
||||
}));
|
||||
}
|
||||
value["redaction"] = json!(redactor.report);
|
||||
value
|
||||
}
|
||||
|
||||
fn normalized(key: &str) -> String {
|
||||
key.chars()
|
||||
.filter(|character| !matches!(character, '_' | '-'))
|
||||
.flat_map(char::to_lowercase)
|
||||
.collect()
|
||||
}
|
||||
|
||||
// OAuth 字段来自 Provider credential schema 与官方 Codex TokenData;API Key 来自
|
||||
// CodexCprApiKeyExportAccount / AuthDotJson;密码来自管理登录合同,头部来自传输身份合同
|
||||
// 不用 token、secret、credential 等泛称推断敏感性,避免删除业务错误上下文
|
||||
fn secret(key: &str) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
"accesstoken"
|
||||
| "personalaccesstoken"
|
||||
| "refreshtoken"
|
||||
| "idtoken"
|
||||
| "apikey"
|
||||
| "openaiapikey"
|
||||
| "password"
|
||||
| "currentpassword"
|
||||
| "newpassword"
|
||||
| "authorization"
|
||||
| "proxyauthorization"
|
||||
| "xapikey"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "setcookie"
|
||||
| "xoaiis"
|
||||
| "xoaiisupdate"
|
||||
| "xoaiattestation"
|
||||
)
|
||||
}
|
||||
|
||||
impl Redactor {
|
||||
fn omitted(&mut self) -> Value {
|
||||
self.report.truncated_values += 1;
|
||||
json!({"omitted": "export_size_limit"})
|
||||
}
|
||||
|
||||
fn text(&mut self, text: &str) -> String {
|
||||
let mut result = text.to_owned();
|
||||
for (pattern, replacement) in INLINE.iter() {
|
||||
self.report.inline_credentials += pattern.find_iter(&result).count();
|
||||
result = pattern.replace_all(&result, *replacement).into_owned();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn visit(&mut self, value: &mut Value, depth: usize) {
|
||||
if self.nodes == 0 || self.bytes == 0 || depth > 32 {
|
||||
*value = self.omitted();
|
||||
return;
|
||||
}
|
||||
self.nodes -= 1;
|
||||
// 预算同时计入缩进和容器分隔符,避免深层 JSON 的下载体积显著超过正文预算
|
||||
self.bytes = self.bytes.saturating_sub(24 + depth * 4);
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if text.trim_start().starts_with('{') || text.trim_start().starts_with('[') {
|
||||
if text.len() > MAX_TEXT {
|
||||
*value = self.omitted();
|
||||
return;
|
||||
}
|
||||
if let Ok(parsed) = serde_json::from_str::<Value>(text) {
|
||||
*value = parsed;
|
||||
self.visit(value, depth + 1);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let cleaned = self.text(text);
|
||||
let limit = MAX_TEXT.min(self.bytes / 2);
|
||||
if cleaned.len() > limit {
|
||||
let boundary = cleaned.floor_char_boundary(limit);
|
||||
*value = json!({"text": &cleaned[..boundary], "omittedBytes": cleaned.len() - boundary});
|
||||
self.report.truncated_values += 1;
|
||||
} else {
|
||||
*text = cleaned;
|
||||
}
|
||||
self.bytes = self.bytes.saturating_sub(value.to_string().len());
|
||||
}
|
||||
Value::Array(items) => {
|
||||
// 头部也可能以二元组承载,不能只识别对象字段名
|
||||
if items.len() == 2
|
||||
&& items[0]
|
||||
.as_str()
|
||||
.is_some_and(|name| secret(&normalized(name)))
|
||||
{
|
||||
items[1] = json!(REDACTED);
|
||||
self.report.credential_fields += 1;
|
||||
}
|
||||
let original_len = items.len();
|
||||
let mut kept = 0;
|
||||
for item in items.iter_mut().take(4096) {
|
||||
if self.nodes == 0 || self.bytes == 0 {
|
||||
break;
|
||||
}
|
||||
self.visit(item, depth + 1);
|
||||
kept += 1;
|
||||
}
|
||||
if kept < original_len {
|
||||
items.truncate(kept);
|
||||
items.push(json!({"omittedItems": original_len - kept}));
|
||||
self.report.truncated_values += 1;
|
||||
}
|
||||
}
|
||||
Value::Object(object) => {
|
||||
if object
|
||||
.get("name")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|name| secret(&normalized(name)))
|
||||
&& object.contains_key("value")
|
||||
{
|
||||
object.insert("value".to_owned(), json!(REDACTED));
|
||||
self.report.credential_fields += 1;
|
||||
}
|
||||
let fields = std::mem::take(object);
|
||||
let total = fields.len();
|
||||
for (key, mut item) in fields {
|
||||
if object.len() >= 4096 || self.nodes == 0 || self.bytes == 0 {
|
||||
break;
|
||||
}
|
||||
let normalized = normalized(&key);
|
||||
if secret(&normalized) {
|
||||
item = json!(REDACTED);
|
||||
self.report.credential_fields += 1;
|
||||
} else if matches!(
|
||||
normalized.as_str(),
|
||||
"email"
|
||||
| "accountemail"
|
||||
| "clientip"
|
||||
| "accountname"
|
||||
| "accountnotes"
|
||||
| "credentialname"
|
||||
| "clientapikeyname"
|
||||
| "accountlabel"
|
||||
) {
|
||||
item = json!(REDACTED);
|
||||
self.report.personal_fields += 1;
|
||||
} else {
|
||||
self.visit(&mut item, depth + 1);
|
||||
}
|
||||
self.bytes = self.bytes.saturating_sub(key.len());
|
||||
object.insert(self.text(&key), item);
|
||||
}
|
||||
if object.len() < total {
|
||||
object.insert("_omittedFields".to_owned(), json!(total - object.len()));
|
||||
self.report.truncated_values += 1;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use super::{
|
||||
wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
mod export;
|
||||
mod presenter;
|
||||
mod query;
|
||||
mod routes;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 观测领域事实到管理页面的展示投影
|
||||
|
||||
use super::*;
|
||||
use gateway_core::metering::Usage;
|
||||
|
||||
pub(crate) fn display_duration(value: Option<u64>) -> String {
|
||||
let Some(value) = value.and_then(|value| i64::try_from(value).ok()) else {
|
||||
@@ -100,7 +101,7 @@ pub(crate) fn attempt_metrics_view(metrics: &domain::AttemptMetrics) -> AttemptM
|
||||
}
|
||||
|
||||
pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
TokenDetailsView {
|
||||
token_details_view(&Usage {
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
@@ -109,35 +110,11 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
total_tokens: record.total_tokens,
|
||||
input_tokens_display: record
|
||||
.input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
output_tokens_display: record
|
||||
.output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
cached_input_tokens_display: record
|
||||
.cached_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
cache_write_input_tokens_display: record
|
||||
.cache_write_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
reasoning_output_tokens_display: record
|
||||
.reasoning_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_input_tokens_display: record
|
||||
.image_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_output_tokens_display: record
|
||||
.image_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
total_tokens_display: record
|
||||
.total_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn usage_list_token_details(record: &domain::UsageListRecord) -> TokenDetailsView {
|
||||
TokenDetailsView {
|
||||
token_details_view(&Usage {
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_input_tokens: record.cached_input_tokens,
|
||||
@@ -146,28 +123,41 @@ pub(crate) fn usage_list_token_details(record: &domain::UsageListRecord) -> Toke
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
total_tokens: record.total_tokens,
|
||||
input_tokens_display: record
|
||||
})
|
||||
}
|
||||
|
||||
fn token_details_view(usage: &Usage) -> TokenDetailsView {
|
||||
TokenDetailsView {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_input_tokens: usage.cached_input_tokens,
|
||||
cache_write_input_tokens: usage.cache_write_input_tokens,
|
||||
reasoning_output_tokens: usage.reasoning_output_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
total_tokens: usage.total_tokens,
|
||||
input_tokens_display: usage
|
||||
.input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
output_tokens_display: record
|
||||
output_tokens_display: usage
|
||||
.output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
cached_input_tokens_display: record
|
||||
cached_input_tokens_display: usage
|
||||
.cached_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
cache_write_input_tokens_display: record
|
||||
cache_write_input_tokens_display: usage
|
||||
.cache_write_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_compact_number),
|
||||
reasoning_output_tokens_display: record
|
||||
reasoning_output_tokens_display: usage
|
||||
.reasoning_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_input_tokens_display: record
|
||||
image_input_tokens_display: usage
|
||||
.image_input_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
image_output_tokens_display: record
|
||||
image_output_tokens_display: usage
|
||||
.image_output_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
total_tokens_display: record
|
||||
total_tokens_display: usage
|
||||
.total_tokens
|
||||
.map_or_else(|| "-".to_owned(), format_number),
|
||||
}
|
||||
@@ -272,6 +262,7 @@ pub(crate) fn usage_list_record_view(
|
||||
.clone()
|
||||
.or_else(|| record.requested_model_id.clone());
|
||||
UsageListRecordView {
|
||||
logical_outcome: record.outcome.as_str().to_owned(),
|
||||
client_api_key_name: record.client_api_key_name,
|
||||
id: record.id,
|
||||
provider: record.provider_kind,
|
||||
@@ -545,8 +536,6 @@ pub(crate) fn usage_detail_view(
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageRecordDetailView {
|
||||
UsageRecordDetailView {
|
||||
trace: detail.trace,
|
||||
related_requests: detail.related_requests,
|
||||
request: usage_record_view(detail.request, time),
|
||||
attempts: detail
|
||||
.attempts
|
||||
@@ -626,7 +615,7 @@ pub(crate) fn ops_error_view(
|
||||
subagent_kind: error.subagent_kind,
|
||||
compact: error.compact,
|
||||
message: error.message,
|
||||
error_details: error.error_details,
|
||||
error_details: error.error_details.map(error_details_view),
|
||||
metadata: OpsErrorMetadataView {
|
||||
source: error.source,
|
||||
component: error.component,
|
||||
@@ -650,6 +639,20 @@ pub(crate) fn ops_error_view(
|
||||
}
|
||||
}
|
||||
|
||||
// 受控错误正文保持原文语义,嵌套 JSON 只在展示投影中展开缩进
|
||||
fn error_details_view(raw: String) -> String {
|
||||
let Ok(mut value) = serde_json::from_str::<Value>(&raw) else {
|
||||
return raw;
|
||||
};
|
||||
if let Some(upstream) = value.get_mut("upstream")
|
||||
&& let Some(text) = upstream.as_str()
|
||||
&& let Ok(parsed) = serde_json::from_str::<Value>(text)
|
||||
{
|
||||
*upstream = parsed;
|
||||
}
|
||||
serde_json::to_string_pretty(&value).unwrap_or(raw)
|
||||
}
|
||||
|
||||
pub(crate) fn ops_page_view(
|
||||
page: domain::OpsErrorPage,
|
||||
time: crate::time::TimePresenter,
|
||||
|
||||
@@ -29,6 +29,7 @@ impl DashboardQuery {
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub status: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
@@ -330,6 +331,12 @@ pub(crate) fn usage_command(
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
Ok(domain::UsageQuery {
|
||||
status: match query.status.as_deref() {
|
||||
None | Some("success") => domain::UsageRecordStatus::Success,
|
||||
Some("error") => domain::UsageRecordStatus::Error,
|
||||
Some("all") => domain::UsageRecordStatus::All,
|
||||
Some(_) => return Err(WireValidationError::new("status")),
|
||||
},
|
||||
range: usage_range(
|
||||
query.start_time.as_deref(),
|
||||
query.end_time.as_deref(),
|
||||
|
||||
@@ -29,6 +29,10 @@ where
|
||||
get(usage_insights_diagnostics::<S>),
|
||||
)
|
||||
.route("/api/admin/operations/errors", get(ops_errors::<S>))
|
||||
.route(
|
||||
"/api/admin/diagnostics/export",
|
||||
get(super::export::download::<S>),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_summary<S>(
|
||||
|
||||
@@ -86,6 +86,7 @@ pub struct ImageBillingView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageListRecordView {
|
||||
pub logical_outcome: String,
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub id: String,
|
||||
pub provider: Option<String>,
|
||||
@@ -275,8 +276,6 @@ pub struct UsageAttemptView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordDetailView {
|
||||
pub trace: Option<serde_json::Value>,
|
||||
pub related_requests: Vec<serde_json::Value>,
|
||||
#[serde(flatten)]
|
||||
pub request: UsageRecordView,
|
||||
pub attempts: Vec<UsageAttemptView>,
|
||||
|
||||
@@ -53,8 +53,19 @@ pub(super) fn router<S: SessionState + Clone + Send + Sync + 'static>() -> Route
|
||||
.route("/api/admin/plugins/extensions/{instance_id}/{artifact_sha256}/{revision}/callback-tickets", post(start_callback::<S>))
|
||||
.route("/api/admin/plugins/extensions/{instance_id}/{artifact_sha256}/{revision}/api/{*path}", any(handle::<S>))
|
||||
.route("/api/admin/plugins/extensions/{instance_id}/{artifact_sha256}/{revision}/resources/{*path}", get(resource::<S>))
|
||||
.route("/plugins/resources/{instance_id}/{artifact_sha256}/{revision}/{*path}", get(public_resource::<S>))
|
||||
.route("/plugins/callbacks/{instance_id}/{artifact_sha256}/{revision}/{*path}", get(callback::<S>))
|
||||
}
|
||||
|
||||
pub(in crate::admin) fn public_router<S: SessionState + Clone + Send + Sync + 'static>() -> Router<S>
|
||||
{
|
||||
Router::new()
|
||||
.route(
|
||||
"/plugins/resources/{instance_id}/{artifact_sha256}/{revision}/{*path}",
|
||||
get(public_resource::<S>),
|
||||
)
|
||||
.route(
|
||||
"/plugins/callbacks/{instance_id}/{artifact_sha256}/{revision}/{*path}",
|
||||
get(callback::<S>),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::admin) fn model_router() -> Router<crate::ApiState> {
|
||||
|
||||
@@ -5,7 +5,7 @@ mod distribution;
|
||||
mod instances;
|
||||
mod management;
|
||||
|
||||
pub(super) use management::model_router;
|
||||
pub(super) use management::{model_router, public_router};
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
||||
@@ -60,6 +60,7 @@ struct CreateRequest {
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateRequest {
|
||||
enabled: Option<bool>,
|
||||
auto_location: Option<bool>,
|
||||
#[serde(default, deserialize_with = "deserialize_location_update")]
|
||||
location: Option<Option<gateway_core::account::RequestLocation>>,
|
||||
@@ -145,6 +146,7 @@ struct ProxyAccountView {
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct ProxyView {
|
||||
enabled: bool,
|
||||
auto_location: bool,
|
||||
detected_location: Option<gateway_admin::model::proxies::DetectedProxyLocation>,
|
||||
location: Option<gateway_core::account::RequestLocation>,
|
||||
@@ -168,6 +170,7 @@ impl From<(ProxyRecord, crate::time::TimePresenter)> for ProxyView {
|
||||
let endpoint = record.proxy.endpoint();
|
||||
Self {
|
||||
auto_location: record.auto_location,
|
||||
enabled: record.enabled,
|
||||
detected_location: record.detected_location,
|
||||
location: record.location,
|
||||
id: record.id,
|
||||
@@ -419,6 +422,7 @@ where
|
||||
.proxies()
|
||||
.update(
|
||||
UpdateProxy {
|
||||
enabled: request.enabled,
|
||||
auto_location: request.auto_location,
|
||||
test: None,
|
||||
location: request.location,
|
||||
|
||||
@@ -17,9 +17,10 @@ use axum::{
|
||||
http::{HeaderName, HeaderValue, Request, StatusCode},
|
||||
response::{IntoResponse as _, Response},
|
||||
};
|
||||
use gateway_core::engine::middleware::FrozenMiddlewarePlan;
|
||||
use gateway_core::engine::middleware::http as contract;
|
||||
use gateway_core::engine::middleware::http::Settings;
|
||||
use gateway_core::engine::middleware::{FrozenMiddlewarePlan, MiddlewareError};
|
||||
use gateway_core::error::GatewayErrorKind;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::middleware::compose;
|
||||
use http_body::{Body as HttpBody, Frame, SizeHint};
|
||||
@@ -270,6 +271,11 @@ async fn handle(
|
||||
}
|
||||
response.map(|body| Body::new(ResponseBody::new(body, guard)))
|
||||
}
|
||||
Err(MiddlewareError::Gateway(error))
|
||||
if error.kind() == GatewayErrorKind::MiddlewareCapacityUnavailable =>
|
||||
{
|
||||
crate::openai::error::gateway_error_response(&error)
|
||||
}
|
||||
Err(error) => {
|
||||
let status = if error.is_rejected() {
|
||||
StatusCode::FORBIDDEN
|
||||
|
||||
@@ -13,6 +13,8 @@ use gateway_core::{
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub use gateway_core::engine::execution::gateway_error_from_engine;
|
||||
|
||||
use super::responses::ProtocolErrorBody;
|
||||
|
||||
/// OpenAI 风格错误响应
|
||||
@@ -148,39 +150,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
)
|
||||
}
|
||||
|
||||
/// 将 core Engine 错误收敛为稳定 Gateway 分类
|
||||
#[must_use]
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
match error {
|
||||
EngineError::Provider(error) => GatewayError::from_provider(error),
|
||||
EngineError::Cancelled => {
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
}
|
||||
EngineError::Deadline => {
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "gateway request timed out")
|
||||
}
|
||||
EngineError::ProviderNotRegistered { .. } | EngineError::EmptyRoutingPlan => {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
)
|
||||
}
|
||||
EngineError::Store(source) => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
.with_source(source.clone())
|
||||
}
|
||||
EngineError::ProviderMetadataMismatch
|
||||
| EngineError::ContinuationPinMismatch
|
||||
| EngineError::RequiredAccountMismatch
|
||||
| EngineError::AccountOutsideClientScope
|
||||
| EngineError::DownstreamCommitRequired
|
||||
| EngineError::NoActiveAttempt
|
||||
| EngineError::InvalidDeliveryState => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "gateway execution failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gateway 错误的 OpenAI HTTP 表达
|
||||
pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
let (status, default_type, default_code) = gateway_error_contract(error.kind());
|
||||
@@ -310,6 +279,11 @@ pub const fn gateway_error_contract(
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::MiddlewareCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ConcurrencyQueueFull => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
|
||||
@@ -5,7 +5,9 @@ use std::{borrow::Cow, fmt, net::IpAddr};
|
||||
use axum::http::HeaderMap;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload, ProviderSessionState};
|
||||
use gateway_core::operation::{
|
||||
ClientTurn, GenerateRequest, Operation, ProtocolPayload, ProviderSessionState,
|
||||
};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
is_transport_managed_request_header,
|
||||
@@ -497,12 +499,30 @@ pub(super) fn decode_request_object(
|
||||
};
|
||||
let protocol_context = request_headers.protocol_context(use_websocket, frame_turn_metadata);
|
||||
|
||||
let semantics =
|
||||
gateway_protocol::openai::codex_responses_request_semantics(&object, &protocol_context);
|
||||
let client_turn = if object.get("generate").and_then(Value::as_bool) != Some(false)
|
||||
&& semantics.request_kind.as_deref() == Some("turn")
|
||||
&& !semantics.compact
|
||||
{
|
||||
gateway_protocol::openai::codex_account_thread_id(&object, &protocol_context)
|
||||
.zip(gateway_protocol::openai::codex_turn_id(
|
||||
&object,
|
||||
&protocol_context,
|
||||
))
|
||||
.and_then(|(thread, turn)| ClientTurn::new(thread, turn))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let payload = ProtocolPayload::json_object(OPENAI_PROTOCOL, object)
|
||||
.map_err(|_| RequestDecodeError::CanonicalContract {
|
||||
field: "request".to_owned(),
|
||||
})?
|
||||
.with_context(protocol_context);
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(payload));
|
||||
let operation = Operation::Generate(
|
||||
GenerateRequest::from_protocol_payload(payload).with_client_turn(client_turn),
|
||||
);
|
||||
Ok(DecodedResponsesRequest {
|
||||
operation,
|
||||
metadata: ResponsesRequestMetadata {
|
||||
|
||||
@@ -106,6 +106,7 @@ async fn create_route_should_create_an_empty_group() {
|
||||
assert_eq!(value["data"]["record"]["memberCount"], 0);
|
||||
assert_eq!(value["data"]["record"]["color"], "#A855F780");
|
||||
assert_eq!(value["data"]["record"]["fastMode"], "default");
|
||||
assert_eq!(value["data"]["record"]["allowImageGeneration"], true);
|
||||
assert_eq!(value["data"]["configRevision"], 8);
|
||||
}
|
||||
|
||||
@@ -393,3 +394,40 @@ async fn fast_mode_rejects_invalid_values_and_creates_groups_in_each_mode() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_generation_flag_round_trips_and_omitted_updates_preserve_it() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let created = request(router(&fixture), Method::POST, "/api/admin/account-groups/create", Some(json!({
|
||||
"name":"No images", "description":null, "color":"#2563EBFF", "allowImageGeneration":false
|
||||
})), true).await;
|
||||
assert_eq!(created.status(), StatusCode::CREATED);
|
||||
let created = response_json(created).await;
|
||||
assert_eq!(created["data"]["record"]["allowImageGeneration"], false);
|
||||
let id = created["data"]["id"].as_str().unwrap();
|
||||
for (value, expected) in [
|
||||
(None, false),
|
||||
(Some(true), true),
|
||||
(Some(false), false),
|
||||
(None, false),
|
||||
] {
|
||||
let mut body =
|
||||
json!({"id":id,"name":"Image policy","description":null,"color":"#2563EBFF"});
|
||||
if let Some(value) = value {
|
||||
body["allowImageGeneration"] = json!(value);
|
||||
}
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/update",
|
||||
Some(body),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["record"]["allowImageGeneration"],
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,6 +112,7 @@ fn account_page_item(
|
||||
|
||||
AccountPageItem {
|
||||
account: AccountRecord {
|
||||
outbound_proxy_enabled: true,
|
||||
id: "acct_cooldown".to_owned(),
|
||||
provider_kind: gateway_core::routing::ProviderKind::new("openai").unwrap(),
|
||||
groups: Vec::new(),
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
//! 管理端 JSON 压缩的协商、响应合同与重复编码回归测试
|
||||
|
||||
use std::io::Read as _;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
response::Response,
|
||||
};
|
||||
use flate2::read::GzDecoder;
|
||||
use tower::ServiceExt as _;
|
||||
use tower_http::compression::CompressionLayer;
|
||||
|
||||
use super::AdminTestFixture;
|
||||
|
||||
async fn app() -> Router {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
crate::openai::api_router_with_admin(fixture.services)
|
||||
}
|
||||
|
||||
async fn get(router: Router, uri: &str, encoding: Option<&str>) -> Response {
|
||||
let mut request = Request::builder()
|
||||
.uri(uri)
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_admin_compression");
|
||||
if let Some(encoding) = encoding {
|
||||
request = request.header(header::ACCEPT_ENCODING, encoding);
|
||||
}
|
||||
router
|
||||
.oneshot(request.body(Body::empty()).expect("request"))
|
||||
.await
|
||||
.expect("response")
|
||||
}
|
||||
|
||||
async fn bytes(response: Response) -> Vec<u8> {
|
||||
to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("response body")
|
||||
.to_vec()
|
||||
}
|
||||
|
||||
fn decode_gzip(body: &[u8]) -> Vec<u8> {
|
||||
let mut decoded = Vec::new();
|
||||
GzDecoder::new(body)
|
||||
.read_to_end(&mut decoded)
|
||||
.expect("valid gzip body");
|
||||
decoded
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gzip_should_preserve_admin_json_and_response_headers() {
|
||||
let router = app().await;
|
||||
let expected = bytes(get(router.clone(), "/api/admin/settings", None).await).await;
|
||||
assert!(expected.len() >= 1024);
|
||||
|
||||
for encoding in ["gzip", "br, gzip;q=0.8, deflate;q=0.5", "*;q=0.5"] {
|
||||
let response = get(router.clone(), "/api/admin/settings", Some(encoding)).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[header::CONTENT_ENCODING], "gzip");
|
||||
assert_eq!(response.headers()[header::CONTENT_TYPE], "application/json");
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
assert_eq!(response.headers()["x-request-id"], "req_admin_compression");
|
||||
assert_eq!(response.headers()[header::VARY], "accept-encoding");
|
||||
assert!(!response.headers().contains_key(header::CONTENT_LENGTH));
|
||||
let compressed = bytes(response).await;
|
||||
assert!(compressed.len() < expected.len());
|
||||
assert_eq!(decode_gzip(&compressed), expected, "{encoding}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clients_without_gzip_support_should_receive_original_json() {
|
||||
let router = app().await;
|
||||
let expected = bytes(get(router.clone(), "/api/admin/settings", None).await).await;
|
||||
for encoding in [None, Some("identity"), Some("br"), Some("gzip;q=0")] {
|
||||
let response = get(router.clone(), "/api/admin/settings", encoding).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_ENCODING));
|
||||
assert_eq!(response.headers()[header::VARY], "accept-encoding");
|
||||
assert_eq!(bytes(response).await, expected, "{encoding:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn small_admin_json_should_remain_uncompressed() {
|
||||
let router = app().await;
|
||||
let uri = "/api/admin/settings/admin-api-key";
|
||||
let expected = bytes(get(router.clone(), uri, None).await).await;
|
||||
assert!(expected.len() < 1024);
|
||||
let response = get(router, uri, Some("gzip")).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_ENCODING));
|
||||
assert_eq!(bytes(response).await, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_second_compression_layer_should_not_encode_gzip_again() {
|
||||
let router = app().await;
|
||||
let expected = bytes(get(router.clone(), "/api/admin/settings", None).await).await;
|
||||
let response = get(
|
||||
router.layer(CompressionLayer::new()),
|
||||
"/api/admin/settings",
|
||||
Some("gzip"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.headers()[header::CONTENT_ENCODING], "gzip");
|
||||
assert_eq!(response.headers()[header::VARY], "accept-encoding");
|
||||
assert_eq!(decode_gzip(&bytes(response).await), expected);
|
||||
}
|
||||
@@ -511,6 +511,7 @@ mod provider {
|
||||
total_slots: None,
|
||||
},
|
||||
account: AccountRecord {
|
||||
outbound_proxy_enabled: true,
|
||||
notes: None,
|
||||
model_access: Default::default(),
|
||||
id: "acct_error_test".to_owned(),
|
||||
|
||||
@@ -88,6 +88,7 @@ mod account_groups;
|
||||
mod accounts;
|
||||
mod auth;
|
||||
mod client_keys;
|
||||
mod compression;
|
||||
mod error_diagnostics;
|
||||
mod errors;
|
||||
mod observability;
|
||||
@@ -184,6 +185,7 @@ impl AdminTestFixture {
|
||||
plugin_ports.clone(),
|
||||
plugin_ports.clone(),
|
||||
plugin_ports.clone(),
|
||||
plugin_ports.clone(),
|
||||
);
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new("openai", Arc::clone(&provider_error))),
|
||||
@@ -709,6 +711,7 @@ impl MemoryAccountGroupStore {
|
||||
(
|
||||
primary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
id: primary_id,
|
||||
name: "Alpha routing".to_owned(),
|
||||
@@ -731,6 +734,7 @@ impl MemoryAccountGroupStore {
|
||||
(
|
||||
secondary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
id: secondary_id,
|
||||
name: "Beta routing".to_owned(),
|
||||
@@ -889,6 +893,7 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
let now = Utc::now();
|
||||
let record = AccountGroupRecord {
|
||||
allow_image_generation: command.allow_image_generation,
|
||||
fast_mode: command.fast_mode,
|
||||
id: command.id.clone(),
|
||||
name: command.name,
|
||||
@@ -924,6 +929,9 @@ impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
if let Some(fast_mode) = command.fast_mode {
|
||||
record.fast_mode = fast_mode;
|
||||
}
|
||||
if let Some(allowed) = command.allow_image_generation {
|
||||
record.allow_image_generation = allowed;
|
||||
}
|
||||
record.updated_at = Utc::now();
|
||||
mutation(&mut state, command.id, true)
|
||||
}
|
||||
@@ -1379,6 +1387,27 @@ impl AccountRuntimeStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn diagnostic_errors(
|
||||
&self,
|
||||
target: &gateway_admin::model::diagnostics::DiagnosticTarget,
|
||||
limit: u16,
|
||||
) -> AdminStoreResult<Vec<gateway_admin::model::observability::OpsError>> {
|
||||
let errors = self.ops_errors.lock().expect("ops errors");
|
||||
Ok(errors
|
||||
.iter()
|
||||
.filter(|error| match target {
|
||||
gateway_admin::model::diagnostics::DiagnosticTarget::Request(id) => {
|
||||
error.request_id.as_ref() == Some(id)
|
||||
}
|
||||
gateway_admin::model::diagnostics::DiagnosticTarget::Operation(id) => {
|
||||
&error.event_id == id && error.source == "ops_event"
|
||||
}
|
||||
})
|
||||
.take(usize::from(limit))
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn query_timeout(&self) -> std::time::Duration {
|
||||
std::time::Duration::from_secs(10)
|
||||
}
|
||||
@@ -1468,11 +1497,12 @@ impl ObservabilityStore for UnusedStore {
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
async fn usage_record_detail(&self, request_id: &str) -> AdminStoreResult<UsageDetail> {
|
||||
self.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.clone()
|
||||
.filter(|detail| detail.request.id == request_id)
|
||||
.ok_or_else(|| unavailable("usage detail"))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
//! 诊断下载的权限、证据完整性、脱敏与部分失败合同
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::UsageDetail;
|
||||
use gateway_api::admin::observability;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::response::{ops_error_record, usage_record_with_account};
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
async fn export(
|
||||
fixture: &AdminTestFixture,
|
||||
query: &str,
|
||||
authenticated: bool,
|
||||
) -> (StatusCode, axum::http::HeaderMap, Value) {
|
||||
let mut request = Request::builder()
|
||||
.uri(format!("/api/admin/diagnostics/export{query}"))
|
||||
.header("x-request-id", "diagnostics-export-test");
|
||||
if authenticated {
|
||||
request = request.header(header::COOKIE, "cpr_session=valid-session");
|
||||
}
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(request.body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let body = to_bytes(response.into_body(), 20 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
(status, headers, serde_json::from_slice(&body).unwrap())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostic_export_requires_admin_and_exactly_one_target() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
assert_eq!(
|
||||
export(&fixture, "?errorId=missing", false).await.0,
|
||||
StatusCode::UNAUTHORIZED
|
||||
);
|
||||
for query in [
|
||||
"",
|
||||
"?requestId=a&errorId=b",
|
||||
"?requestId=",
|
||||
"?errorId=a&payload=untrusted",
|
||||
] {
|
||||
assert_eq!(
|
||||
export(&fixture, query, true).await.0,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"{query}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
export(&fixture, "?errorId=missing", true).await.0,
|
||||
StatusCode::NOT_FOUND
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostic_export_keeps_retry_evidence_and_original_errors_but_not_credentials() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
error.error_details = Some(json!({
|
||||
"causes": ["TLS certificate verification failed", "pool timeout after 5 seconds"],
|
||||
"upstream": json!({"error": {"code": "unknown_vendor_failure", "message": "quota exhausted"},
|
||||
"headers": {"Authorization": "fixture-authorization", "set-cookie": "fixture-cookie"},
|
||||
"access_token": "fixture-access", "refreshToken": "fixture-refresh",
|
||||
}).to_string(),
|
||||
"endpoint": "postgres://fixture-user:fixture-password@database:5432/main",
|
||||
"message": "connect failed: Bearer fixture-bearer, retry after 30 seconds",
|
||||
}).to_string());
|
||||
let mut unrelated = error.clone();
|
||||
unrelated.request_id = Some("another-request".into());
|
||||
unrelated.message = "unrelated evidence must not leak".into();
|
||||
fixture
|
||||
.ops_errors
|
||||
.lock()
|
||||
.unwrap()
|
||||
.extend([error, unrelated]);
|
||||
let trace = json!({"droppedEvents": 0, "events": [
|
||||
{"stage": "request.routed", "data": {"accountPolicy": {"maxAccountRotations": 3}}},
|
||||
{"stage": "account.selection", "data": {"candidates": [{"id": "acct_high", "weight": 2}, {"id": "acct_low", "weight": 1}]}},
|
||||
{"stage": "attempt.failed", "data": {"kind": "unavailable", "recovery": {"replaySafe": true}}},
|
||||
{"stage": "retry.decided", "data": {"phase": "execute", "retryable": false, "clientRetrySuppressed": true, "replayBarrierReached": false}},
|
||||
]});
|
||||
fixture.usage_detail.lock().unwrap().replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_err",
|
||||
"acct_err",
|
||||
"private name",
|
||||
"fixture@example.invalid",
|
||||
"api_key",
|
||||
Utc::now(),
|
||||
),
|
||||
attempts: Vec::new(),
|
||||
trace: Some(trace.clone()),
|
||||
related_requests: Vec::new(),
|
||||
});
|
||||
let (status, headers, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK, "{file}");
|
||||
assert_eq!(
|
||||
headers[header::CONTENT_DISPOSITION],
|
||||
"attachment; filename=diagnostics.json"
|
||||
);
|
||||
assert_eq!(headers[header::CACHE_CONTROL], "private, no-store");
|
||||
assert_eq!(file["schemaVersion"], 5);
|
||||
assert_eq!(file["requests"][0]["evidence"]["trace"], trace);
|
||||
assert_eq!(
|
||||
file["requests"][0]["evidence"]["incidentConfiguration"]["revision"],
|
||||
1
|
||||
);
|
||||
let details = &file["requests"][0]["errors"][0]["errorDetails"];
|
||||
assert_eq!(
|
||||
details["upstream"]["error"]["code"],
|
||||
"unknown_vendor_failure"
|
||||
);
|
||||
assert_eq!(details["causes"][0], "TLS certificate verification failed");
|
||||
assert_eq!(file["requests"][0]["errors"][0]["accountId"], "acct_err");
|
||||
assert!(
|
||||
file["availability"]["gaps"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|gap| gap["section"] == "environment")
|
||||
);
|
||||
let text = file.to_string();
|
||||
for marker in [
|
||||
"fixture-authorization",
|
||||
"fixture-cookie",
|
||||
"fixture-access",
|
||||
"fixture-refresh",
|
||||
"fixture-password",
|
||||
"fixture-user",
|
||||
"fixture-bearer",
|
||||
"fixture@example.invalid",
|
||||
"private name",
|
||||
"unrelated evidence must not leak",
|
||||
"203.0.113.8",
|
||||
] {
|
||||
assert!(!text.contains(marker), "leaked {marker}");
|
||||
}
|
||||
assert!(text.contains("retry after 30 seconds"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_error_exports_without_fabricating_a_model_request() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
error.source = "ops_event".into();
|
||||
error.request_id = None;
|
||||
error.component = "worker".into();
|
||||
error.error_details =
|
||||
Some(json!({"causes": ["database connection refused"], "upstream": null}).to_string());
|
||||
fixture.ops_errors.lock().unwrap().push(error);
|
||||
let (status, _, file) = export(&fixture, "?errorId=err_snapshot", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(file["requests"], json!([]));
|
||||
assert_eq!(file["error"]["requestId"], Value::Null);
|
||||
assert_eq!(
|
||||
file["error"]["errorDetails"]["causes"][0],
|
||||
"database connection refused"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_survives_missing_request_and_nested_credentials_are_redacted() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
error.error_details = Some(json!({
|
||||
"cookies": "Cookie: a=fixture-cookie-a; b=fixture-cookie-b\nupstream timeout",
|
||||
"uri": "https://provider.invalid/path?api_key=fixture-query&model=gpt-test",
|
||||
"headers": [["Authorization", "fixture-tuple"], {"name": "Cookie", "value": "fixture-header"}],
|
||||
}).to_string());
|
||||
fixture.ops_errors.lock().unwrap().push(error);
|
||||
let (status, _, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert!(file["requests"][0]["evidence"].is_null());
|
||||
assert!(
|
||||
file["availability"]["gaps"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|gap| gap["reason"] == "request_detail_unavailable")
|
||||
);
|
||||
let text = file.to_string();
|
||||
for marker in [
|
||||
"fixture-cookie-a",
|
||||
"fixture-cookie-b",
|
||||
"fixture-query",
|
||||
"fixture-tuple",
|
||||
"fixture-header",
|
||||
] {
|
||||
assert!(!text.contains(marker), "leaked {marker}");
|
||||
}
|
||||
assert!(text.contains("upstream timeout"));
|
||||
assert!(text.contains("model=gpt-test"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oversized_error_export_reports_truncation_instead_of_silent_loss() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
error.source = "ops_event".into();
|
||||
error.request_id = None;
|
||||
error.message = "诊断文本".repeat(100_000);
|
||||
fixture.ops_errors.lock().unwrap().push(error);
|
||||
let (status, _, file) = export(&fixture, "?errorId=err_snapshot", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(file["exportTruncated"], true);
|
||||
assert!(file["redaction"]["truncatedValues"].as_u64().unwrap() > 0);
|
||||
assert!(file["error"]["message"]["omittedBytes"].as_u64().unwrap() > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn related_read_failures_and_limits_keep_primary_evidence() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut related = vec![json!({"requestId": "req_err", "relation": "recovers"})];
|
||||
related.extend(
|
||||
(0..10).map(
|
||||
|index| json!({"requestId": format!("missing_{index}"), "relation": "recovered_by"}),
|
||||
),
|
||||
);
|
||||
fixture.usage_detail.lock().unwrap().replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_err",
|
||||
"acct_err",
|
||||
"fixture",
|
||||
"fixture@example.invalid",
|
||||
"api_key",
|
||||
Utc::now(),
|
||||
),
|
||||
attempts: Vec::new(),
|
||||
trace: None,
|
||||
related_requests: related,
|
||||
});
|
||||
fixture
|
||||
.ops_errors
|
||||
.lock()
|
||||
.unwrap()
|
||||
.extend((0..130).map(|index| {
|
||||
let mut error = ops_error_record();
|
||||
error.event_id = format!("error_{index}");
|
||||
error
|
||||
}));
|
||||
let (status, _, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(file["requests"].as_array().unwrap().len(), 8);
|
||||
assert_eq!(file["requests"][0]["errors"].as_array().unwrap().len(), 128);
|
||||
assert_eq!(
|
||||
file["requests"][0]["evidence"]["detail"]["requestId"],
|
||||
"req_err"
|
||||
);
|
||||
assert!(file["requests"][1]["evidence"].is_null());
|
||||
let gaps = file["availability"]["gaps"].as_array().unwrap();
|
||||
for reason in [
|
||||
"related_request_limit",
|
||||
"error_record_limit",
|
||||
"request_detail_unavailable",
|
||||
"not_recorded_or_retained",
|
||||
] {
|
||||
assert!(
|
||||
gaps.iter().any(|gap| gap["reason"] == reason),
|
||||
"missing {reason}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_schemas_are_redacted_without_guessing_business_field_names() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
let business = json!({
|
||||
"token": {"input": 128, "output": 64},
|
||||
"credentials": {"status": "expired", "provider": "openai"},
|
||||
"secret": "tool argument name", "clientSecret": "business field description",
|
||||
"accountId": "account-correlation", "userId": "user-correlation",
|
||||
"planType": "plus", "status": "active", "accessTokenExpiresAt": "2026-10-16T00:00:00Z",
|
||||
"error": {"code": "unknown_code", "message": "upstream capacity exceeded"},
|
||||
});
|
||||
error.error_details = Some(json!({
|
||||
"business": business,
|
||||
"account": {"accessToken": "fixture-access", "refreshToken": "fixture-refresh", "idToken": "fixture-id"},
|
||||
"auth": {"OPENAI_API_KEY": "fixture-key", "personal_access_token": "fixture-pat", "tokens": {
|
||||
"access_token": "fixture-official-access", "refresh_token": "fixture-official-refresh",
|
||||
"id_token": "fixture-official-id", "account_id": "account-correlation",
|
||||
}},
|
||||
}).to_string());
|
||||
fixture.ops_errors.lock().unwrap().push(error);
|
||||
let (status, _, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let details = &file["requests"][0]["errors"][0]["errorDetails"];
|
||||
assert_eq!(details["business"], business);
|
||||
assert_eq!(
|
||||
details["auth"]["tokens"]["account_id"],
|
||||
"account-correlation"
|
||||
);
|
||||
assert!(!file.to_string().contains("fixture-"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn export_budget_keeps_format_target_and_existing_gap_records() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture.usage_detail.lock().unwrap().replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_err",
|
||||
"acct_err",
|
||||
"QA",
|
||||
"qa@example.invalid",
|
||||
"oauth",
|
||||
Utc::now(),
|
||||
),
|
||||
attempts: Vec::new(),
|
||||
trace: Some(json!({"events": vec![vec![true; 32]; 3500]})),
|
||||
related_requests: Vec::new(),
|
||||
});
|
||||
let (status, _, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(file["schemaVersion"], 5);
|
||||
assert_eq!(file["source"], "gateway.diagnostics");
|
||||
assert_eq!(file["target"]["requestId"], "req_err");
|
||||
assert_eq!(file["exportTruncated"], true);
|
||||
let gaps = file["availability"]["gaps"].as_array().unwrap();
|
||||
assert!(gaps.iter().any(|gap| gap["reason"] == "export_size_limit"));
|
||||
assert!(
|
||||
gaps.iter()
|
||||
.any(|gap| gap["reason"] == "incident_policy_not_recorded")
|
||||
);
|
||||
assert!(serde_json::to_vec_pretty(&file).unwrap().len() < 16 * 1024 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selected_event_is_exported_once_and_shares_the_total_error_budget() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut selected = ops_error_record();
|
||||
selected.source = "ops_event".into();
|
||||
let mut errors = vec![selected];
|
||||
errors.extend((0..129).map(|index| {
|
||||
let mut error = ops_error_record();
|
||||
error.source = "ops_event".into();
|
||||
error.event_id = format!("related_{index}");
|
||||
error
|
||||
}));
|
||||
fixture.ops_errors.lock().unwrap().extend(errors);
|
||||
let (status, _, file) = export(&fixture, "?errorId=err_snapshot", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(file["error"]["id"], "err_snapshot");
|
||||
let errors = file["requests"][0]["errors"].as_array().unwrap();
|
||||
assert_eq!(errors.len(), 127);
|
||||
assert!(errors.iter().all(|error| error["id"] != "err_snapshot"));
|
||||
assert!(
|
||||
file["availability"]["gaps"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|gap| gap["reason"] == "error_record_limit")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_authentication_headers_are_redacted_without_losing_error_context() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut error = ops_error_record();
|
||||
error.error_details = Some(json!({
|
||||
"headers": "x-api-key: fixture-api-key\nx-oai-is: fixture-session\nx-oai-is-update: fixture-update\nx-oai-attestation: fixture-attestation fixture-attestation-tail\nauthorization: Custom fixture-auth\nproxy-authorization: Custom fixture-proxy\nupstream timeout",
|
||||
"message": "access-token=fixture-access; refresh-token=fixture-refresh; model=gpt-test",
|
||||
}).to_string());
|
||||
fixture.ops_errors.lock().unwrap().push(error);
|
||||
let (status, _, file) = export(&fixture, "?requestId=req_err", true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let text = file.to_string();
|
||||
for marker in [
|
||||
"fixture-api-key",
|
||||
"fixture-session",
|
||||
"fixture-update",
|
||||
"fixture-attestation",
|
||||
"fixture-attestation-tail",
|
||||
"fixture-auth",
|
||||
"fixture-proxy",
|
||||
"fixture-access",
|
||||
"fixture-refresh",
|
||||
] {
|
||||
assert!(!text.contains(marker), "leaked {marker}");
|
||||
}
|
||||
assert!(text.contains("upstream timeout"));
|
||||
assert!(text.contains("model=gpt-test"));
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Admin 观测 wire/路由的固定合同测试
|
||||
|
||||
mod export;
|
||||
mod query;
|
||||
mod response;
|
||||
|
||||
@@ -398,7 +398,7 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_attempt_failure_detail_keeps_missing_upstream_facts_and_preparation_trace() {
|
||||
async fn zero_attempt_failure_detail_omits_trace_and_keeps_missing_upstream_facts() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
@@ -470,7 +470,8 @@ async fn zero_attempt_failure_detail_keeps_missing_upstream_facts_and_preparatio
|
||||
assert_eq!(data["attempts"], json!([]));
|
||||
// 明细仍沿用全局 best-effort 合同,不从预备 trace 推断完整尝试列表
|
||||
assert_eq!(data["attemptsComplete"], false);
|
||||
assert_eq!(data["trace"], trace);
|
||||
assert!(data.get("trace").is_none());
|
||||
assert!(data.get("relatedRequests").is_none());
|
||||
assert_eq!(data["logicalOutcome"], "failed");
|
||||
assert_eq!(data["clientStatusCode"], 503);
|
||||
assert_eq!(data["message"], "no upstream account is available");
|
||||
@@ -495,8 +496,6 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::OpsError;
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
@@ -508,63 +507,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
.ops_errors
|
||||
.lock()
|
||||
.expect("ops errors")
|
||||
.push(OpsError {
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
source: "model_request".to_owned(),
|
||||
event_id: "err_snapshot".to_owned(),
|
||||
request_id: Some("req_err".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
protocol: Some("openai".to_owned()),
|
||||
client_transport: Some("http_sse".to_owned()),
|
||||
requested_model_id: Some("gpt-5.4".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
provider_account_name: None,
|
||||
provider_account_email: Some("err@example.invalid".to_owned()),
|
||||
provider_account_plan_type: Some("pro".to_owned()),
|
||||
provider_account_plan_type_display: Some("Pro".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-err".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
failure_kind: "upstream_error".to_owned(),
|
||||
upstream_send_state: Some("sent".to_owned()),
|
||||
client_status_code: Some(502),
|
||||
upstream_status_code: Some(502),
|
||||
provider_error_code: Some("upstream".to_owned()),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
error_details: Some(
|
||||
r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(),
|
||||
),
|
||||
client_ip: Some("203.0.113.8".to_owned()),
|
||||
user_agent: Some("codex-cli/0.144.0".to_owned()),
|
||||
reasoning_effort: Some("medium".to_owned()),
|
||||
reasoning_preset: None,
|
||||
request_kind: Some("root".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: Some(false),
|
||||
continuation_affinity_hash: Some("affinity-hash".to_owned()),
|
||||
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
|
||||
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
|
||||
upstream_connection_id: Some("connection-id".to_owned()),
|
||||
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
|
||||
upstream_connection_age_ms: Some(12_000),
|
||||
upstream_connection_idle_ms: Some(4_000),
|
||||
recovery_request_id: None,
|
||||
recovered_at: None,
|
||||
recovery_attempt_count: 0,
|
||||
recovery_retry_delay_ms: None,
|
||||
recovery_total_latency_ms: None,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
.push(ops_error_record());
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
@@ -621,7 +564,7 @@ async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
"userAgent": "codex-cli/0.144.0",
|
||||
"providerErrorCode": "upstream",
|
||||
"upstreamSendState": "sent",
|
||||
"errorDetails": "{\"error\":{\"code\":\"upstream\",\"message\":\"raw upstream marker\"}}",
|
||||
"errorDetails": serde_json::to_string_pretty(&json!({"error": {"code": "upstream", "message": "raw upstream marker"}})).unwrap(),
|
||||
"continuationUnavailableReason": "reused_connection_lost",
|
||||
"upstreamConnectionExitReason": "tcp_reset",
|
||||
"upstreamConnectionAgeMs": 12000,
|
||||
@@ -707,7 +650,7 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
);
|
||||
}
|
||||
|
||||
fn usage_record_with_account(
|
||||
pub(super) fn usage_record_with_account(
|
||||
id: &str,
|
||||
account_id: &str,
|
||||
name: &str,
|
||||
@@ -858,6 +801,7 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageListRecord {
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
id: "request_endpoint".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
@@ -1303,3 +1247,123 @@ async fn observability_queries_reject_group_filters_and_analytics_search() {
|
||||
assert!(observations.records.is_empty());
|
||||
assert!(observations.errors.is_empty());
|
||||
}
|
||||
|
||||
pub(super) fn ops_error_record() -> gateway_admin::model::observability::OpsError {
|
||||
use gateway_admin::model::observability::OpsError;
|
||||
OpsError {
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
source: "model_request".to_owned(),
|
||||
event_id: "err_snapshot".to_owned(),
|
||||
request_id: Some("req_err".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
protocol: Some("openai".to_owned()),
|
||||
client_transport: Some("http_sse".to_owned()),
|
||||
requested_model_id: Some("gpt-5.4".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
provider_account_name: None,
|
||||
provider_account_email: Some("err@example.invalid".to_owned()),
|
||||
provider_account_plan_type: Some("pro".to_owned()),
|
||||
provider_account_plan_type_display: Some("Pro".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-err".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
failure_kind: "upstream_error".to_owned(),
|
||||
upstream_send_state: Some("sent".to_owned()),
|
||||
client_status_code: Some(502),
|
||||
upstream_status_code: Some(502),
|
||||
provider_error_code: Some("upstream".to_owned()),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
error_details: Some(
|
||||
r#"{"error":{"code":"upstream","message":"raw upstream marker"}}"#.to_owned(),
|
||||
),
|
||||
client_ip: Some("203.0.113.8".to_owned()),
|
||||
user_agent: Some("codex-cli/0.144.0".to_owned()),
|
||||
reasoning_effort: Some("medium".to_owned()),
|
||||
reasoning_preset: None,
|
||||
request_kind: Some("root".to_owned()),
|
||||
subagent_kind: None,
|
||||
compact: Some(false),
|
||||
continuation_affinity_hash: Some("affinity-hash".to_owned()),
|
||||
continuation_previous_response_id_hash: Some("response-hash".to_owned()),
|
||||
continuation_unavailable_reason: Some("reused_connection_lost".to_owned()),
|
||||
upstream_connection_id: Some("connection-id".to_owned()),
|
||||
upstream_connection_exit_reason: Some("tcp_reset".to_owned()),
|
||||
upstream_connection_age_ms: Some(12_000),
|
||||
upstream_connection_idle_ms: Some(4_000),
|
||||
recovery_request_id: None,
|
||||
recovered_at: None,
|
||||
recovery_attempt_count: 0,
|
||||
recovery_retry_delay_ms: None,
|
||||
recovery_total_latency_ms: None,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_status_query_accepts_three_modes_and_rejects_unknown_values() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (status, expected) in [
|
||||
("success", StatusCode::OK),
|
||||
("error", StatusCode::OK),
|
||||
("all", StatusCode::OK),
|
||||
("failed", StatusCode::BAD_REQUEST),
|
||||
] {
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/usage/records?status={status}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "usage_status_fixture")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let actual = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
assert_eq!(
|
||||
actual,
|
||||
expected,
|
||||
"{status}: {}",
|
||||
String::from_utf8_lossy(&body)
|
||||
);
|
||||
if expected == StatusCode::OK {
|
||||
use gateway_admin::model::observability::UsageRecordStatus;
|
||||
let status_value = match status {
|
||||
"success" => UsageRecordStatus::Success,
|
||||
"error" => UsageRecordStatus::Error,
|
||||
_ => UsageRecordStatus::All,
|
||||
};
|
||||
assert_eq!(
|
||||
fixture
|
||||
.observations
|
||||
.lock()
|
||||
.unwrap()
|
||||
.records
|
||||
.last()
|
||||
.unwrap()
|
||||
.status,
|
||||
status_value
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -435,6 +435,35 @@ async fn resource_request(
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_html_resource_should_remain_uncompressed() {
|
||||
let fixture = published_fixture().await;
|
||||
let path = format!("ui/{}.html", "a".repeat(2048));
|
||||
let response = crate::openai::api_router_with_admin(fixture.services)
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!(
|
||||
"/api/admin/plugins/extensions/plugin-instance-diagnostic/{}/7/resources/{path}",
|
||||
"a".repeat(64)
|
||||
))
|
||||
.header("cookie", "cpr_session=management-session")
|
||||
.header("accept-encoding", "gzip")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()["content-type"], "text/html");
|
||||
assert!(!response.headers().contains_key("content-encoding"));
|
||||
assert_eq!(
|
||||
axum::body::to_bytes(response.into_body(), 4096)
|
||||
.await
|
||||
.unwrap(),
|
||||
path
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn static_resources_revalidate_privately_after_authorization() {
|
||||
use axum::body::to_bytes;
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! 插件管理 HTTP 测试入口,以及制品、实例与权限合同测试
|
||||
|
||||
mod usage_data;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_admin::model::plugins::distribution::{
|
||||
DownloadedPlugin, GithubReleaseQuery, PluginRelease, RemotePluginLocation, SourceCredential,
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
//! 通用用量端口的插件管理测试夹具,非本测试调用路径禁止访问
|
||||
use super::TestPluginPorts;
|
||||
use gateway_admin::{model::usage_data::*, ports::usage_data::*};
|
||||
#[async_trait::async_trait]
|
||||
impl UsageDataStore for TestPluginPorts {
|
||||
async fn get_record(&self, _: GetUsageRecord) -> UsageDataResult<UsageRecord> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_records(&self, _: ListUsageRecords) -> UsageDataResult<UsageRecordPage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_records(&self, _: StoredUsageWrite) -> UsageDataResult<UsageWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn list_aggregates(&self, _: ListUsageAggregates) -> UsageDataResult<UsageAggregatePage> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
async fn write_aggregates(
|
||||
&self,
|
||||
_: StoredAggregateWrite,
|
||||
) -> UsageDataResult<UsageAggregateWriteResult> {
|
||||
unreachable!("unexpected usage call")
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,43 @@ async fn proxy_location_rejects_invalid_and_incomplete_input() {
|
||||
#[derive(Default)]
|
||||
pub(super) struct MemoryProxies(Mutex<Option<ProxyRecord>>);
|
||||
|
||||
#[tokio::test]
|
||||
async fn proxy_enabled_updates_preserve_connection_and_omitted_state() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let (status, created) = request(
|
||||
&fixture,
|
||||
"/api/admin/proxies/create",
|
||||
Some(json!({
|
||||
"name":"Office", "proxyUrl":"http://proxy.example:8080"
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::CREATED);
|
||||
assert_eq!(created["data"]["record"]["enabled"], true);
|
||||
for (revision, enabled, expected) in [
|
||||
(1, Some(false), false),
|
||||
(2, None, false),
|
||||
(3, Some(true), true),
|
||||
] {
|
||||
let mut body = json!({"id":"proxy_test", "revision":revision, "name":"Office"});
|
||||
if let Some(enabled) = enabled {
|
||||
body["enabled"] = json!(enabled);
|
||||
}
|
||||
let (status, changed) =
|
||||
request(&fixture, "/api/admin/proxies/update", Some(body), true).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(changed["data"]["record"]["enabled"], expected);
|
||||
assert_eq!(
|
||||
changed["data"]["record"]["endpoint"],
|
||||
created["data"]["record"]["endpoint"]
|
||||
);
|
||||
let (_, listed) = request(&fixture, "/api/admin/proxies", None, true).await;
|
||||
assert_eq!(listed["data"]["items"][0]["enabled"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
fn missing() -> AdminStoreError {
|
||||
AdminStoreError::new(AdminStoreErrorKind::NotFound, "proxy", "missing proxy")
|
||||
}
|
||||
@@ -165,6 +202,7 @@ impl ProxyStore for MemoryProxies {
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<ProxyMutation> {
|
||||
let mut record = ProxyRecord {
|
||||
enabled: true,
|
||||
auto_location: command.auto_location,
|
||||
detected_location: None,
|
||||
location: command.location,
|
||||
@@ -197,6 +235,9 @@ impl ProxyStore for MemoryProxies {
|
||||
let mut stored = self.0.lock().unwrap();
|
||||
let record = stored.as_mut().ok_or_else(missing)?;
|
||||
record.name = command.name;
|
||||
if let Some(enabled) = command.enabled {
|
||||
record.enabled = enabled;
|
||||
}
|
||||
if let Some(auto_location) = command.auto_location {
|
||||
if auto_location != record.auto_location {
|
||||
record.detected_location = None;
|
||||
|
||||
@@ -62,7 +62,7 @@ fn update_body() -> Value {
|
||||
"smartScheduling": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"usageRetentionDays": 31,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
@@ -84,9 +84,10 @@ async fn privacy_save_requires_provider_validation_and_preview_requires_admin_au
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
let policy = json!({"enabled":true,"onError":"reject_request","rules":[{
|
||||
"id":"auth","name":"自定义认证头","enabled":true,"scope":"request_header",
|
||||
"selector":"authorization","action":"remove_field","pattern":null,
|
||||
"replacement":"","value":null,"replaceAll":true,"caseInsensitive":false,"multiLine":false
|
||||
"id":"auth","name":"自定义认证头","enabled":true,"steps":[{
|
||||
"scope":"request_header","selector":"authorization","action":"remove_field","pattern":null,
|
||||
"replacement":"","value":null,"replaceAll":true,"caseInsensitive":false,"multiLine":false
|
||||
}]
|
||||
}]});
|
||||
body["codexPrivacyPolicy"] = policy.clone();
|
||||
let response = app(fixture.state())
|
||||
@@ -140,7 +141,7 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
let custom = json!({"loadWeight": 2.0, "quotaWeight": 0.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3, "preferHigherWeight": true});
|
||||
let custom = json!({"loadWeight": 2.0, "quotaWeight": 0.0, "healthWeight": 1.0, "latencyWeight": 0.5, "resetWeight": 1.2, "queueWeight": 2.3});
|
||||
body["smartScheduling"] = custom.clone();
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
@@ -160,8 +161,8 @@ async fn smart_settings_round_trip_and_invalid_updates_leave_the_saved_value_int
|
||||
let mut invalid_values = vec![
|
||||
json!(null),
|
||||
json!({}),
|
||||
json!({"loadWeight":0,"quotaWeight":0,"healthWeight":0,"latencyWeight":0,"resetWeight":0,"queueWeight":0,"preferHigherWeight":true}),
|
||||
json!({"loadWeight":0.01,"quotaWeight":1,"healthWeight":1,"latencyWeight":1,"resetWeight":0,"queueWeight":0,"preferHigherWeight":false}),
|
||||
json!({"loadWeight":0,"quotaWeight":0,"healthWeight":0,"latencyWeight":0,"resetWeight":0,"queueWeight":0}),
|
||||
json!({"loadWeight":0.01,"quotaWeight":1,"healthWeight":1,"latencyWeight":1,"resetWeight":0,"queueWeight":0}),
|
||||
];
|
||||
for field in ["resetWeight", "queueWeight"] {
|
||||
for value in [json!(-0.1), json!(10.1), json!(0.01), json!(null)] {
|
||||
@@ -215,6 +216,56 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
assert_eq!(request.validate().unwrap_err().field(), "rotationStrategy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_request_retention_round_trips_and_invalid_days_do_not_replace_it() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
for days in [1, 3, 7, 15, 30, 31] {
|
||||
body["usageRetentionDays"] = json!(days);
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body.clone()),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let saved = response_json(response).await;
|
||||
assert_eq!(saved["data"]["usageRetentionDays"], days);
|
||||
body["configRevision"] = saved["data"]["configRevision"].clone();
|
||||
}
|
||||
for (days, expected_status) in [
|
||||
(json!(0), StatusCode::BAD_REQUEST),
|
||||
(json!(32), StatusCode::BAD_REQUEST),
|
||||
(json!(u32::MAX), StatusCode::BAD_REQUEST),
|
||||
(json!(-1), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!(1.5), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
(json!(null), StatusCode::UNPROCESSABLE_ENTITY),
|
||||
] {
|
||||
body["usageRetentionDays"] = days;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body.clone()),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected_status);
|
||||
}
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response_json(response).await["data"]["usageRetentionDays"],
|
||||
31
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_accepts_unlimited_default_account_concurrency() {
|
||||
let mut body = update_body();
|
||||
@@ -297,7 +348,7 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
smart_scheduling: gateway_core::account::SmartSchedulingConfig::default(),
|
||||
min_codex_desktop_version: Some("26.825.6671".to_owned()),
|
||||
min_codex_cli_version: Some("0.40.0".to_owned()),
|
||||
usage_retention_days: 32,
|
||||
usage_retention_days: 31,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
account_auto_freeze_enabled: true,
|
||||
@@ -348,7 +399,7 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
"smartSchedulingDefaults": gateway_core::account::SmartSchedulingConfig::default(),
|
||||
"minCodexDesktopVersion": "26.825.6671",
|
||||
"minCodexCliVersion": "0.40.0",
|
||||
"usageRetentionDays": 32,
|
||||
"usageRetentionDays": 31,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"accountAutoFreezeEnabled": true,
|
||||
|
||||
@@ -9,7 +9,7 @@ use axum::{
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use futures::stream;
|
||||
use futures::{StreamExt as _, stream};
|
||||
use gateway_admin::{
|
||||
model::system::{
|
||||
SystemOperationAccepted, SystemOperationKind, SystemOperationState, SystemOperationStatus,
|
||||
@@ -19,6 +19,7 @@ use gateway_admin::{
|
||||
ports::system::{SystemOperationError, SystemOperations, SystemUpdateEventStream},
|
||||
};
|
||||
use gateway_api::admin::system::{self, UpdateDetailQuery, UpdateRequest};
|
||||
use http_body_util::BodyExt as _;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState, unavailable_system};
|
||||
@@ -40,25 +41,39 @@ fn update_request_should_preserve_target_version_for_domain_validation() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn update_event_stream_should_preserve_event_id_and_download_progress_percent() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem)).await;
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = app(fixture.state())
|
||||
let response = gateway_api::admin::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/system/update/events")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::ACCEPT_ENCODING, "gzip")
|
||||
.header("x-request-id", "req_system_update_events")
|
||||
.body(Body::empty())
|
||||
.expect("update event request"),
|
||||
)
|
||||
);
|
||||
let response = tokio::time::timeout(std::time::Duration::from_secs(1), response)
|
||||
.await
|
||||
.expect("SSE response must not wait for stream completion")
|
||||
.expect("update event response");
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers()[header::CONTENT_TYPE],
|
||||
"text/event-stream"
|
||||
);
|
||||
assert!(!response.headers().contains_key(header::CONTENT_ENCODING));
|
||||
let mut stream = response.into_body();
|
||||
let frame = tokio::time::timeout(std::time::Duration::from_secs(1), stream.frame())
|
||||
.await
|
||||
.expect("update event body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("UTF-8 event stream");
|
||||
.expect("first SSE event must arrive before stream completion")
|
||||
.expect("first event")
|
||||
.expect("event body");
|
||||
let body = String::from_utf8(frame.into_data().expect("event data").to_vec())
|
||||
.expect("UTF-8 event stream");
|
||||
|
||||
assert!(
|
||||
body.contains(r#""id":"update-progress-1""#),
|
||||
@@ -72,7 +87,7 @@ async fn update_event_stream_should_preserve_event_id_and_download_progress_perc
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_return_accepted_and_report_restart_only_through_status() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem)).await;
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = app(fixture.state());
|
||||
let response = router
|
||||
@@ -119,7 +134,55 @@ fn app(state: AdminTestState) -> Router {
|
||||
system::router::<AdminTestState>().with_state(state)
|
||||
}
|
||||
|
||||
struct ProgressSystem;
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_compress_json_starting_at_one_kib() {
|
||||
async fn detail(notes: String, encoding: &str) -> axum::response::Response {
|
||||
let fixture =
|
||||
AdminTestFixture::with_system(Arc::new(ProgressSystem { notes: Some(notes) })).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
crate::openai::api_router_with_admin(fixture.services)
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/system/update/detail")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::ACCEPT_ENCODING, encoding)
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
let response = detail(String::new(), "identity").await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let base = to_bytes(response.into_body(), 4096).await.unwrap();
|
||||
assert!(base.len() < 1023);
|
||||
for size in [1023, 1024] {
|
||||
let response = detail("x".repeat(size - base.len()), "gzip").await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let compressed = response.headers().get(header::CONTENT_ENCODING).cloned();
|
||||
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
|
||||
if size == 1023 {
|
||||
assert!(compressed.is_none());
|
||||
assert_eq!(bytes.len(), size);
|
||||
} else {
|
||||
use std::io::Read as _;
|
||||
assert_eq!(compressed.unwrap(), "gzip");
|
||||
let mut decoded = Vec::new();
|
||||
flate2::read::GzDecoder::new(bytes.as_ref())
|
||||
.read_to_end(&mut decoded)
|
||||
.unwrap();
|
||||
assert_eq!(decoded.len(), size);
|
||||
let body: serde_json::Value = serde_json::from_slice(&decoded).unwrap();
|
||||
assert_eq!(body["data"]["notes"], "x".repeat(size - base.len()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ProgressSystem {
|
||||
notes: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SystemOperations for ProgressSystem {
|
||||
@@ -144,7 +207,7 @@ impl SystemOperations for ProgressSystem {
|
||||
deployment_mode: "docker".into(),
|
||||
build_type: "release".into(),
|
||||
release_url: None,
|
||||
notes: None,
|
||||
notes: self.notes.clone(),
|
||||
cached: false,
|
||||
update_supported: true,
|
||||
unsupported_reason: None,
|
||||
@@ -153,18 +216,21 @@ impl SystemOperations for ProgressSystem {
|
||||
}
|
||||
|
||||
fn update_events(&self) -> SystemUpdateEventStream {
|
||||
Box::pin(stream::once(async {
|
||||
SystemUpdateEvent {
|
||||
id: "update-progress-1".to_owned(),
|
||||
operation_id: Some("update-1".to_owned()),
|
||||
level: SystemUpdateEventLevel::Info,
|
||||
step: Some("download".to_owned()),
|
||||
message: "已下载 4.0 MiB / 10.0 MiB (40%)".to_owned(),
|
||||
terminal: true,
|
||||
progress_percent: Some(40),
|
||||
occurred_at: Utc::now(),
|
||||
}
|
||||
}))
|
||||
Box::pin(
|
||||
stream::once(async {
|
||||
SystemUpdateEvent {
|
||||
id: "update-progress-1".to_owned(),
|
||||
operation_id: Some("update-1".to_owned()),
|
||||
level: SystemUpdateEventLevel::Info,
|
||||
step: Some("download".to_owned()),
|
||||
message: "已下载 4.0 MiB / 10.0 MiB (40%)".to_owned(),
|
||||
terminal: false,
|
||||
progress_percent: Some(40),
|
||||
occurred_at: Utc::now(),
|
||||
}
|
||||
})
|
||||
.chain(stream::pending()),
|
||||
)
|
||||
}
|
||||
|
||||
async fn perform_update(
|
||||
@@ -216,7 +282,7 @@ impl SystemOperations for ProgressSystem {
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_detail_should_validate_and_forward_the_temporary_channel() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem)).await;
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = app(fixture.state());
|
||||
for (authenticated, query, expected) in [
|
||||
|
||||
@@ -22,13 +22,6 @@ fn manifest_should_depend_only_on_core_protocol_and_http_adapter_layers() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_should_include_gateway_api() {
|
||||
let workspace = include_str!("../../../Cargo.toml");
|
||||
|
||||
assert!(workspace.contains("\"crates/gateway-api\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_routes_should_use_only_get_post_and_static_paths() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/admin");
|
||||
|
||||
@@ -93,6 +93,7 @@ pub(super) async fn fixture() -> AdminTestFixture {
|
||||
|
||||
fn usage_record() -> UsageListRecord {
|
||||
UsageListRecord {
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Succeeded,
|
||||
client_api_key_name: Some("Production".to_owned()),
|
||||
id: "req-visible".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
|
||||
@@ -35,6 +35,7 @@ use tower::ServiceExt as _;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Mode {
|
||||
Overloaded,
|
||||
Rewrite,
|
||||
Upgrade {
|
||||
reject: bool,
|
||||
@@ -142,6 +143,12 @@ impl MiddlewarePlan for Plan {
|
||||
let mode = self.mode;
|
||||
Box::pin(async move {
|
||||
match mode {
|
||||
Mode::Overloaded => Err(MiddlewareError::Gateway(
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::MiddlewareCapacityUnavailable,
|
||||
"Request middleware is temporarily busy",
|
||||
),
|
||||
)),
|
||||
Mode::Service => unreachable!("service-only plan bypasses HTTP layers"),
|
||||
Mode::Upgrade { reject, idle, .. } => {
|
||||
let upgrade = request
|
||||
@@ -659,3 +666,25 @@ async fn plugin_http_dispatch_uses_host_identity_without_client_credentials() {
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_middleware_overload_is_an_explicit_503_before_native_routing() {
|
||||
let (router, plan) = router(Mode::Overloaded, None).await;
|
||||
for path in [
|
||||
"/healthz",
|
||||
"/assets/index.js",
|
||||
"/api/settings",
|
||||
"/v1/responses",
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(Request::builder().uri(path).body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(body["error"]["code"], "middleware_capacity_unavailable");
|
||||
}
|
||||
assert_eq!(plan.calls.load(Ordering::Acquire), 4);
|
||||
}
|
||||
|
||||
@@ -263,6 +263,18 @@ fn rate_limited_error_should_map_to_openai_retryable_status() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn middleware_capacity_is_service_unavailable() {
|
||||
assert_eq!(
|
||||
gateway_error_contract(GatewayErrorKind::MiddlewareCapacityUnavailable),
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"middleware_capacity_unavailable"
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_rejections_keep_distinct_retryable_http_codes() {
|
||||
for (kind, code) in [
|
||||
@@ -548,7 +560,7 @@ fn engine_store_error_should_collapse_to_safe_internal_error() {
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::Internal);
|
||||
assert_eq!(projected.safe_message(), "gateway execution failed");
|
||||
assert_eq!(projected.safe_message(), "request execution failed");
|
||||
}
|
||||
#[test]
|
||||
fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
//! 真实 HTTP / WebSocket 路由与 Core 执行入口共同拒绝生图,不触发准入或上游
|
||||
|
||||
use super::*;
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio_tungstenite::tungstenite::{Message, client::IntoClientRequest};
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn execution() -> Arc<DefaultExecutionService> {
|
||||
Arc::new(DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot_with_image_policy(
|
||||
"sk_image_policy",
|
||||
"openai",
|
||||
"key_image_policy",
|
||||
&["acct_api_test"],
|
||||
false,
|
||||
)),
|
||||
Arc::new(UnusedExecutionStore),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn denied_image_requests_return_403_before_admission_or_upstream() {
|
||||
let app = api_router(execution()).await;
|
||||
for (path, body) in [
|
||||
(
|
||||
"/v1/images/generations",
|
||||
json!({"model":"gpt-image-2","prompt":"test"}),
|
||||
),
|
||||
(
|
||||
"/v1/images/edits",
|
||||
json!({"model":"gpt-image-2","prompt":"test","images":[{"image_url":"data:image/png;base64,AA=="}]}),
|
||||
),
|
||||
(
|
||||
"/v1/responses",
|
||||
json!({"model":"model-a","input":"test","tools":[{"type":"image_generation"}]}),
|
||||
),
|
||||
(
|
||||
"/v1/responses",
|
||||
json!({"model":"model-a","input":"test","stream":true,"tool_choice":{"type":"image_generation"}}),
|
||||
),
|
||||
] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::post(path)
|
||||
.header(AUTHORIZATION, "Bearer sk_image_policy")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body: Value =
|
||||
serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{path}: {body}");
|
||||
assert_eq!(body["error"]["code"], "image_generation_disabled");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_websocket_create_enforces_image_policy() {
|
||||
let app = api_router(execution()).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.unwrap();
|
||||
request
|
||||
.headers_mut()
|
||||
.insert(AUTHORIZATION, "Bearer sk_image_policy".parse().unwrap());
|
||||
let (mut socket, _) = tokio_tungstenite::connect_async(request).await.unwrap();
|
||||
for image_fields in [
|
||||
json!({"tools":[{"type":"image_generation"}]}),
|
||||
json!({"tool_choice":{"type":"image_generation"}}),
|
||||
] {
|
||||
let mut body = json!({"type":"response.create","model":"model-a","input":"test"});
|
||||
body.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(image_fields.as_object().unwrap().clone());
|
||||
socket
|
||||
.send(Message::Text(body.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
let frame = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body: Value = serde_json::from_str(frame.to_text().unwrap()).unwrap();
|
||||
assert_eq!(body["type"], "error", "{body}");
|
||||
assert_eq!(body["status"], 403);
|
||||
assert_eq!(body["error"]["code"], "image_generation_disabled");
|
||||
}
|
||||
socket.close(None).await.unwrap();
|
||||
server.abort();
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
mod auth;
|
||||
mod endpoint;
|
||||
mod error;
|
||||
mod image_policy;
|
||||
mod images;
|
||||
mod live;
|
||||
mod middleware;
|
||||
@@ -292,6 +293,16 @@ fn snapshot_with_accounts(
|
||||
provider_name: &str,
|
||||
client_key_id: &str,
|
||||
accounts: &[&str],
|
||||
) -> RuntimeSnapshot {
|
||||
snapshot_with_image_policy(plaintext, provider_name, client_key_id, accounts, true)
|
||||
}
|
||||
|
||||
fn snapshot_with_image_policy(
|
||||
plaintext: &str,
|
||||
provider_name: &str,
|
||||
client_key_id: &str,
|
||||
accounts: &[&str],
|
||||
allow_image_generation: bool,
|
||||
) -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new(provider_name).expect("provider");
|
||||
let account_directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from_iter(
|
||||
@@ -323,10 +334,13 @@ fn snapshot_with_accounts(
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new(client_key_id).expect("key ID"),
|
||||
PlaintextClientApiKey::new(plaintext).expect("plaintext key"),
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)),
|
||||
Arc::new(
|
||||
FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)
|
||||
.with_image_generation(allow_image_generation),
|
||||
),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
|
||||
@@ -34,6 +34,49 @@ fn generate_request(body: Value) -> DecodedResponsesRequest {
|
||||
decode_request(body.to_string().as_bytes()).expect("test request should decode")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_attach_only_identified_codex_inference_turns() {
|
||||
let body = json!({"model":"smart-code", "input":"hello", "client_metadata": {
|
||||
"thread_id":"thread", "turn_id":"turn", "x-codex-turn-metadata": r#"{"request_kind":"turn"}"#
|
||||
}});
|
||||
let decoded = generate_request(body.clone());
|
||||
let identity = generate_operation(&decoded).client_turn().unwrap();
|
||||
assert_eq!(identity.thread_id(), "thread");
|
||||
assert_eq!(identity.turn_id(), "turn");
|
||||
assert_eq!(Value::Object(openai_wire_body(&decoded).clone()), body);
|
||||
for variant in [
|
||||
"warmup",
|
||||
"compaction",
|
||||
"missing_thread",
|
||||
"missing_turn",
|
||||
"unknown_kind",
|
||||
"oversized_id",
|
||||
] {
|
||||
let mut request = body.clone();
|
||||
match variant {
|
||||
"warmup" => request["generate"] = json!(false),
|
||||
"compaction" => {
|
||||
request["client_metadata"]["x-codex-turn-metadata"] =
|
||||
json!(r#"{"request_kind":"compaction"}"#)
|
||||
}
|
||||
"missing_thread" => request["client_metadata"]["thread_id"] = json!(""),
|
||||
"missing_turn" => request["client_metadata"]["turn_id"] = json!(""),
|
||||
"unknown_kind" => {
|
||||
request["client_metadata"]["x-codex-turn-metadata"] =
|
||||
json!(r#"{"request_kind":"other"}"#)
|
||||
}
|
||||
"oversized_id" => request["client_metadata"]["turn_id"] = json!("x".repeat(257)),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
generate_operation(&generate_request(request))
|
||||
.client_turn()
|
||||
.is_none(),
|
||||
"{variant}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_operation(
|
||||
decoded: &DecodedResponsesRequest,
|
||||
) -> &gateway_core::operation::GenerateRequest {
|
||||
|
||||
@@ -1,10 +1,192 @@
|
||||
//! 验证未知控制消息、迟到中断、结算期间收发和创建请求的串行准入
|
||||
|
||||
use super::*;
|
||||
use gateway_core::engine::client_retry::{ClientRetryKey, ClientRetryStore};
|
||||
use gateway_core::engine::response_control::{
|
||||
ResponseControlTransport, ResponseControlUnavailable,
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct RetryMarker(Mutex<Option<ModelRequestId>>);
|
||||
|
||||
impl ClientRetryStore for RetryMarker {
|
||||
fn load<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientRetryKey,
|
||||
) -> BoxFuture<'a, Result<Option<ModelRequestId>, StoreError>> {
|
||||
Box::pin(async { Ok(self.0.lock().unwrap().clone()) })
|
||||
}
|
||||
fn record_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientRetryKey,
|
||||
version: &'a ModelRequestId,
|
||||
_: Duration,
|
||||
) -> BoxFuture<'a, Result<(), StoreError>> {
|
||||
Box::pin(async move {
|
||||
*self.0.lock().unwrap() = Some(version.clone());
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
fn clear_failure<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientRetryKey,
|
||||
expected: &'a ModelRequestId,
|
||||
) -> BoxFuture<'a, Result<(), StoreError>> {
|
||||
Box::pin(async move {
|
||||
let mut marker = self.0.lock().unwrap();
|
||||
if marker.as_ref() == Some(expected) {
|
||||
*marker = None;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RetryBudgetProvider(AtomicUsize);
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for RetryBudgetProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn execute(
|
||||
self: Arc<Self>,
|
||||
request: ProviderRequest,
|
||||
context: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let number = self.0.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if number <= 7 || matches!(number, 9 | 10) {
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_status(503)
|
||||
.with_replay_safe()
|
||||
.with_client_visible_upstream_response(gateway_core::error::ClientVisibleUpstreamResponse::new(
|
||||
503, Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(br#"{"error":{"code":"server_error","message":"synthetic unavailable"}}"#),
|
||||
))
|
||||
.with_transient_retry(
|
||||
std::num::NonZeroU32::new(5).unwrap(),
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
Duration::ZERO,
|
||||
gateway_core::error::TransientRetryPolicy::RetryThenReturnError,
|
||||
);
|
||||
return Ok(ProviderStream::new(
|
||||
ProviderCallMetadata::new(
|
||||
request.candidate().provider().clone(),
|
||||
request.candidate().upstream_model().unwrap().clone(),
|
||||
ProviderAccountId::new("acct_api_test").unwrap(),
|
||||
UpstreamTransport::new("websocket").unwrap(),
|
||||
),
|
||||
futures::stream::iter([Err(error)]),
|
||||
(),
|
||||
));
|
||||
}
|
||||
Arc::new(ChargedWebSocketProvider)
|
||||
.execute(request, context)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_retry_budget_should_flow_through_http_and_live_websocket() {
|
||||
for websocket in [false, true] {
|
||||
let provider = Arc::new(RetryBudgetProvider::default());
|
||||
let marker = Arc::new(RetryMarker::default());
|
||||
let execution = Arc::new(
|
||||
DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(snapshot("sk_retry_probe", "openai")),
|
||||
Arc::new(SettlementPorts::default()),
|
||||
ProviderRegistry::new([provider.clone() as Arc<dyn Provider>]).unwrap(),
|
||||
Arc::new(InterruptAdmissions::default()),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(IgnoredClientApiKeyUsage),
|
||||
Arc::new(crate::support::RecordingDiagnostics::default()),
|
||||
)
|
||||
.with_client_retry_store(marker.clone()),
|
||||
);
|
||||
let app = api_router(execution).await;
|
||||
let mut body = json!({"model":"model-a", "input":"hello", "stream":true, "client_metadata": {
|
||||
"thread_id":"thread", "turn_id":"turn", "x-codex-turn-metadata":r#"{"request_kind":"turn"}"#
|
||||
}});
|
||||
let mut socket = None;
|
||||
let mut server = None;
|
||||
if websocket {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let app = app.clone();
|
||||
server = Some(Server(tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap()
|
||||
})));
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.unwrap();
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_static("Bearer sk_retry_probe"),
|
||||
);
|
||||
socket = Some(connect_async(request).await.unwrap().0);
|
||||
body["type"] = json!("response.create");
|
||||
}
|
||||
for (expected_attempts, success) in [(6, false), (7, false), (8, true), (11, true)] {
|
||||
if let Some(socket) = &mut socket {
|
||||
send(socket, body.clone()).await;
|
||||
loop {
|
||||
let event = next(socket).await;
|
||||
match event["type"].as_str() {
|
||||
Some("response.completed") => {
|
||||
assert!(success);
|
||||
break;
|
||||
}
|
||||
Some("response.failed" | "error") => {
|
||||
assert!(!success);
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header(AUTHORIZATION, "Bearer sk_retry_probe")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
let response = app.clone().oneshot(request).await.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
if success {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
);
|
||||
let payload = axum::body::to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
if success {
|
||||
assert!(String::from_utf8_lossy(&payload).contains("response.completed"));
|
||||
}
|
||||
}
|
||||
assert_eq!(provider.0.load(Ordering::SeqCst), expected_attempts);
|
||||
assert_eq!(marker.0.lock().unwrap().is_none(), success);
|
||||
}
|
||||
if let Some(mut socket) = socket {
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
drop(server);
|
||||
}
|
||||
}
|
||||
|
||||
struct ControlTransport {
|
||||
id: String,
|
||||
active: std::sync::atomic::AtomicBool,
|
||||
|
||||
@@ -8,6 +8,30 @@ use serde_json::json;
|
||||
|
||||
use super::decode_response_create;
|
||||
|
||||
#[test]
|
||||
fn client_retry_identity_should_follow_each_frame_instead_of_handshake_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-codex-turn-metadata",
|
||||
HeaderValue::from_static(
|
||||
r#"{"thread_id":"old-thread","turn_id":"old-turn","request_kind":"compaction"}"#,
|
||||
),
|
||||
);
|
||||
let headers = OpenAiRequestHeaders::from_headers(&headers);
|
||||
for turn in ["turn-one", "turn-two"] {
|
||||
let decoded = decode_response_create_with_context(&json!({
|
||||
"type":"response.create", "model":"smart-code", "input":"hello",
|
||||
"client_metadata": {"thread_id":"thread", "turn_id":turn, "x-codex-turn-metadata":r#"{"request_kind":"turn"}"#}
|
||||
}).to_string(), &headers).unwrap();
|
||||
let Operation::Generate(request) = decoded.operation() else {
|
||||
panic!("generate");
|
||||
};
|
||||
let identity = request.client_turn().unwrap();
|
||||
assert_eq!(identity.thread_id(), "thread");
|
||||
assert_eq!(identity.turn_id(), turn);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_create_should_carry_source_headers_to_provider_on_every_frame() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -247,6 +247,7 @@ impl ClientRoutingScope {
|
||||
/// 一次认证随 RuntimeSnapshot 冻结的账号目录与 Key 权限
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FrozenAccountScope {
|
||||
allow_image_generation: bool,
|
||||
fast_mode: FastMode,
|
||||
request_profiles: BTreeMap<ProviderKind, super::OpaqueProviderData>,
|
||||
directory: Arc<RuntimeAccountDirectory>,
|
||||
@@ -257,6 +258,18 @@ pub struct FrozenAccountScope {
|
||||
}
|
||||
|
||||
impl FrozenAccountScope {
|
||||
/// Key 绑定分组合并后的生图权限,与账号成员资格一同冻结
|
||||
#[must_use]
|
||||
pub const fn with_image_generation(mut self, allowed: bool) -> Self {
|
||||
self.allow_image_generation = allowed;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn allows_image_generation(&self) -> bool {
|
||||
self.allow_image_generation
|
||||
}
|
||||
|
||||
/// 与 Key 授权范围一同冻结;具体字段只由对应 Provider 解释
|
||||
#[must_use]
|
||||
pub fn with_request_profiles(
|
||||
@@ -296,6 +309,7 @@ impl FrozenAccountScope {
|
||||
ClientRoutingScope::Restricted { provider_kinds, .. } => Arc::clone(provider_kinds),
|
||||
};
|
||||
Self {
|
||||
allow_image_generation: true,
|
||||
fast_mode: FastMode::Default,
|
||||
request_profiles: BTreeMap::new(),
|
||||
directory,
|
||||
|
||||
@@ -510,6 +510,8 @@ pub struct AccountSelectionContext {
|
||||
pub policy: AccountSelectionPolicy,
|
||||
pub now: SystemTime,
|
||||
pub excluded_accounts: BTreeSet<ProviderAccountId>,
|
||||
/// 租约争用或队列让位只阻止立即选号,不排除等待资格
|
||||
pub busy_accounts: BTreeSet<ProviderAccountId>,
|
||||
pub preferred_account: Option<ProviderAccountId>,
|
||||
/// 优先账号可调度时,是否优先于其它账号的持久权重
|
||||
pub preferred_account_overrides_weight: bool,
|
||||
@@ -602,7 +604,7 @@ impl<'a> AccountSelection<'a> {
|
||||
pub struct AccountSelector;
|
||||
|
||||
impl AccountSelector {
|
||||
/// 返回调度策略可见的全部合格候选;权重层授权由调用策略的宿主适配器裁剪
|
||||
/// 策略同时约束立即选号与等待,候选保留仅因本地容量受阻的账号
|
||||
#[must_use]
|
||||
pub(crate) fn policy_candidates<'a>(
|
||||
&self,
|
||||
@@ -611,7 +613,12 @@ impl AccountSelector {
|
||||
) -> Vec<&'a AccountCandidate> {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|candidate| self.scheduling_blocker(candidate, context).is_none())
|
||||
.filter(|candidate| {
|
||||
matches!(
|
||||
self.scheduling_blocker(candidate, context),
|
||||
None | Some(AccountSchedulingBlocker::ConcurrencyLimit)
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -697,10 +704,7 @@ impl AccountSelector {
|
||||
.max()?;
|
||||
let (mut preferred, preferred_candidate) = self.preferred_decision(candidates, context);
|
||||
if let Some(candidate) = preferred_candidate {
|
||||
// 权重回切只裁决内置策略的软亲和,插件显式选号按实际选择记录结果
|
||||
let prefer_higher_weight = context.policy.strategy() == RotationStrategy::Smart
|
||||
&& context.policy.smart_scheduling().prefer_higher_weight();
|
||||
if (!context.preferred_account_overrides_weight || prefer_higher_weight)
|
||||
if !context.preferred_account_overrides_weight
|
||||
&& candidate.account.weight() < highest_weight
|
||||
{
|
||||
preferred =
|
||||
@@ -860,6 +864,9 @@ impl AccountSelector {
|
||||
if context.excluded_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::Excluded);
|
||||
}
|
||||
if context.busy_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::ConcurrencyLimit);
|
||||
}
|
||||
if context
|
||||
.concurrency_limit(&candidate.account)
|
||||
.limit()
|
||||
|
||||
@@ -6,7 +6,6 @@ use serde::{Deserialize, Serialize};
|
||||
#[serde(into = "SmartSchedulingValues")]
|
||||
pub struct SmartSchedulingConfig {
|
||||
weights: [u8; 6],
|
||||
prefer_higher_weight: bool,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SmartSchedulingConfig {
|
||||
@@ -24,7 +23,6 @@ impl Default for SmartSchedulingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
weights: [10, 8, 10, 5, 0, 0],
|
||||
prefer_higher_weight: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,10 +37,7 @@ pub enum SmartSchedulingConfigError {
|
||||
|
||||
impl SmartSchedulingConfig {
|
||||
/// 顺序为负载、剩余额度、健康、首输出延迟、额度重置和排队压力
|
||||
pub fn new(
|
||||
weights: [f64; 6],
|
||||
prefer_higher_weight: bool,
|
||||
) -> Result<Self, SmartSchedulingConfigError> {
|
||||
pub fn new(weights: [f64; 6]) -> Result<Self, SmartSchedulingConfigError> {
|
||||
let mut tenths = [0; 6];
|
||||
for (index, field) in [
|
||||
"loadWeight",
|
||||
@@ -67,10 +62,7 @@ impl SmartSchedulingConfig {
|
||||
if tenths == [0; 6] {
|
||||
return Err(SmartSchedulingConfigError::EmptyWeights);
|
||||
}
|
||||
Ok(Self {
|
||||
weights: tenths,
|
||||
prefer_higher_weight,
|
||||
})
|
||||
Ok(Self { weights: tenths })
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -78,11 +70,6 @@ impl SmartSchedulingConfig {
|
||||
self.weights.map(|weight| f64::from(weight) / 10.0)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn prefer_higher_weight(self) -> bool {
|
||||
self.prefer_higher_weight
|
||||
}
|
||||
|
||||
pub(crate) fn score_tolerance(self) -> f64 {
|
||||
// 与系数同步缩放,避免仅放大相同比例的系数就改变近似最优候选集合
|
||||
// 排队系数只参与入队选择,不能放大立即选号的候选容差
|
||||
@@ -105,24 +92,20 @@ struct SmartSchedulingValues {
|
||||
latency_weight: f64,
|
||||
reset_weight: f64,
|
||||
queue_weight: f64,
|
||||
prefer_higher_weight: bool,
|
||||
}
|
||||
|
||||
impl TryFrom<SmartSchedulingValues> for SmartSchedulingConfig {
|
||||
type Error = SmartSchedulingConfigError;
|
||||
|
||||
fn try_from(value: SmartSchedulingValues) -> Result<Self, Self::Error> {
|
||||
Self::new(
|
||||
[
|
||||
value.load_weight,
|
||||
value.quota_weight,
|
||||
value.health_weight,
|
||||
value.latency_weight,
|
||||
value.reset_weight,
|
||||
value.queue_weight,
|
||||
],
|
||||
value.prefer_higher_weight,
|
||||
)
|
||||
Self::new([
|
||||
value.load_weight,
|
||||
value.quota_weight,
|
||||
value.health_weight,
|
||||
value.latency_weight,
|
||||
value.reset_weight,
|
||||
value.queue_weight,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,7 +126,6 @@ impl From<SmartSchedulingConfig> for SmartSchedulingValues {
|
||||
latency_weight,
|
||||
reset_weight,
|
||||
queue_weight,
|
||||
prefer_higher_weight: value.prefer_higher_weight,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,6 +132,15 @@ impl TraceContext {
|
||||
}
|
||||
let data = json!({
|
||||
"kind": error.kind().as_str(), "sendState": error.send_state().as_str(),
|
||||
"recovery": {
|
||||
"replaySafe": error.replay_is_safe(),
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sameAccountCredentialRetry": error.retries_same_account(),
|
||||
"intent": error.pre_delivery_retry().map(|intent| format!("{intent:?}")),
|
||||
"continuationFailure": error.continuation_failure().map(|failure| format!("{failure:?}")),
|
||||
"continuationDisposition": error.continuation_recovery_disposition().map(|disposition| format!("{disposition:?}")),
|
||||
"retryAfterMs": error.retry_after().map(|delay| delay.as_millis()),
|
||||
},
|
||||
"diagnostic": error.diagnostic().map(|diagnostic| json!({
|
||||
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
|
||||
})),
|
||||
@@ -291,6 +300,27 @@ impl TraceContext {
|
||||
}
|
||||
summary["upstreamStatus"] =
|
||||
json!(data.get("upstreamStatus").and_then(Value::as_u64));
|
||||
if let Some(recovery) = data.get("recovery") {
|
||||
let mut kept = json!({});
|
||||
for key in [
|
||||
"replaySafe",
|
||||
"retryProhibited",
|
||||
"sameAccountCredentialRetry",
|
||||
] {
|
||||
kept[key] = json!(recovery.get(key).and_then(Value::as_bool));
|
||||
}
|
||||
for key in ["intent", "continuationFailure", "continuationDisposition"] {
|
||||
kept[key] = json!(
|
||||
recovery
|
||||
.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.map(|value| bounded(value, 256))
|
||||
);
|
||||
}
|
||||
kept["retryAfterMs"] =
|
||||
json!(recovery.get("retryAfterMs").and_then(Value::as_u64));
|
||||
summary["recovery"] = kept;
|
||||
}
|
||||
if let Some(diagnostic) = data.get("diagnostic").and_then(Value::as_object) {
|
||||
let mut kept = json!({});
|
||||
for (key, limit) in [("stage", 96), ("code", 96), ("message", 256)] {
|
||||
@@ -359,6 +389,7 @@ impl TraceContext {
|
||||
| "account.selected"
|
||||
| "attempt.failed"
|
||||
| "retry.decided"
|
||||
| "retry.policy"
|
||||
| "upstream.close"
|
||||
| "upstream.read.failed"
|
||||
| "capture.gap"
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! 客户端失败轮次的短期重试抑制与终态前状态更新
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use futures::{FutureExt, future::BoxFuture, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
|
||||
use super::ModelRequestId;
|
||||
use crate::{error::StoreError, operation::ClientTurn, policy::ClientApiKeyId};
|
||||
|
||||
const FAILURE_TTL: Duration = Duration::from_secs(300);
|
||||
const STORE_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
/// 不同客户端不能共享轮次失败状态
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ClientRetryKey {
|
||||
pub client_key_id: ClientApiKeyId,
|
||||
pub turn: ClientTurn,
|
||||
}
|
||||
|
||||
/// 可丢失的失败标记,版本使用创建该标记的模型请求 ID
|
||||
pub trait ClientRetryStore: Send + Sync {
|
||||
fn load<'a>(
|
||||
&'a self,
|
||||
key: &'a ClientRetryKey,
|
||||
) -> BoxFuture<'a, Result<Option<ModelRequestId>, StoreError>>;
|
||||
|
||||
fn record_failure<'a>(
|
||||
&'a self,
|
||||
key: &'a ClientRetryKey,
|
||||
version: &'a ModelRequestId,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'a, Result<(), StoreError>>;
|
||||
|
||||
/// 比较与删除必须原子执行,迟到成功不能删除较新的失败
|
||||
fn clear_failure<'a>(
|
||||
&'a self,
|
||||
key: &'a ClientRetryKey,
|
||||
expected: &'a ModelRequestId,
|
||||
) -> BoxFuture<'a, Result<(), StoreError>>;
|
||||
}
|
||||
|
||||
pub(super) struct ClientRetryGuard {
|
||||
store: Arc<dyn ClientRetryStore>,
|
||||
key: ClientRetryKey,
|
||||
request_id: ModelRequestId,
|
||||
observed: Option<ModelRequestId>,
|
||||
settled: bool,
|
||||
}
|
||||
|
||||
impl ClientRetryGuard {
|
||||
pub(super) async fn load(
|
||||
store: Arc<dyn ClientRetryStore>,
|
||||
key: ClientRetryKey,
|
||||
request_id: ModelRequestId,
|
||||
) -> Self {
|
||||
let observed = bounded("load", &request_id, store.load(&key))
|
||||
.await
|
||||
.flatten();
|
||||
Self {
|
||||
store,
|
||||
key,
|
||||
request_id,
|
||||
observed,
|
||||
settled: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn has_failure(&self) -> bool {
|
||||
self.observed.is_some()
|
||||
}
|
||||
|
||||
pub(super) async fn failure(&mut self) {
|
||||
if self.settled {
|
||||
return;
|
||||
}
|
||||
bounded(
|
||||
"record",
|
||||
&self.request_id,
|
||||
self.store
|
||||
.record_failure(&self.key, &self.request_id, FAILURE_TTL),
|
||||
)
|
||||
.await;
|
||||
self.settled = true;
|
||||
}
|
||||
|
||||
pub(super) async fn success(&mut self) {
|
||||
if self.settled {
|
||||
return;
|
||||
}
|
||||
if let Some(observed) = &self.observed {
|
||||
bounded(
|
||||
"clear",
|
||||
&self.request_id,
|
||||
self.store.clear_failure(&self.key, observed),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.settled = true;
|
||||
}
|
||||
}
|
||||
|
||||
async fn bounded<T>(
|
||||
action: &'static str,
|
||||
request_id: &ModelRequestId,
|
||||
operation: BoxFuture<'_, Result<T, StoreError>>,
|
||||
) -> Option<T> {
|
||||
let operation = operation.fuse();
|
||||
let timeout = Delay::new(STORE_TIMEOUT).fuse();
|
||||
pin_mut!(operation, timeout);
|
||||
select_biased! {
|
||||
result = operation => match result {
|
||||
Ok(value) => Some(value),
|
||||
Err(error) => {
|
||||
tracing::warn!(action, request_id = request_id.as_str(), error_kind = ?error.kind(), "客户端重试标记不可用,保留原请求结果与重试策略");
|
||||
None
|
||||
}
|
||||
},
|
||||
() = timeout => {
|
||||
tracing::warn!(action, request_id = request_id.as_str(), "客户端重试标记操作超时,保留原请求结果与重试策略");
|
||||
None
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use super::client_retry::{ClientRetryGuard, ClientRetryKey};
|
||||
use super::nested::ExecutionEffects;
|
||||
use super::observation::{
|
||||
RequestObservationDispatch, ResponseObservation, WebSocketResponseAttempt,
|
||||
@@ -293,6 +294,20 @@ where
|
||||
"admissionMs": request.admission_decision_ms,
|
||||
"continuation": format!("{continuation_attempt:?}"),
|
||||
"candidateCount": plan.candidates().len(),
|
||||
"accountPolicy": {
|
||||
"strategy": plan.account_selection_policy().strategy().as_str(),
|
||||
"affinity": plan.account_selection_policy().openai_account_affinity().as_str(),
|
||||
"maxAccountRotations": plan.account_selection_policy().max_account_rotations(),
|
||||
"maxRoutingAttempts": plan.max_attempts().get(),
|
||||
"maxConcurrentPerAccount": plan.account_selection_policy().max_concurrent_per_account().get(),
|
||||
"queueMaxWaiting": plan.account_selection_policy().queue_policy().max_waiting,
|
||||
"queueTimeoutMs": plan.account_selection_policy().queue_policy().timeout.as_millis(),
|
||||
"sessionAffinityTtlMs": plan.account_selection_policy().openai_session_affinity_ttl().as_millis(),
|
||||
"smartWeights": plan.account_selection_policy().smart_scheduling().weights(),
|
||||
},
|
||||
"requiredAccount": account_selection.required_account().map(|account| account.as_str()),
|
||||
"hasMiddleware": middleware.is_some(),
|
||||
"hasRetryPolicy": request_policy.is_some(),
|
||||
}),
|
||||
);
|
||||
let image_generation_requested = operation.image_generation_requested();
|
||||
@@ -358,6 +373,7 @@ where
|
||||
last_retryable_failure_events: Vec::new(),
|
||||
last_observed_provider: None,
|
||||
pending_terminal_failure: None,
|
||||
client_retry: None,
|
||||
};
|
||||
|
||||
if session.cancellation.is_cancelled() {
|
||||
@@ -368,6 +384,26 @@ where
|
||||
session.finish_interruption(&EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
if !session.account_selection.is_diagnostic()
|
||||
&& let Some(store) = &self.engine.client_retry
|
||||
&& let Operation::Generate(generate) = &session.operation
|
||||
&& let Some(turn) = generate.client_turn()
|
||||
{
|
||||
let guard = ClientRetryGuard::load(
|
||||
Arc::clone(store),
|
||||
ClientRetryKey {
|
||||
client_key_id: session.client_api_key_ref.clone(),
|
||||
turn: turn.clone(),
|
||||
},
|
||||
session.request_id.clone(),
|
||||
)
|
||||
.await;
|
||||
session.trace.record(
|
||||
"client_retry.loaded",
|
||||
json!({"previousFailure": guard.has_failure()}),
|
||||
);
|
||||
session.client_retry = Some(guard);
|
||||
}
|
||||
Ok(session)
|
||||
}
|
||||
}
|
||||
@@ -502,6 +538,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
last_observed_provider: Option<crate::identity::ProviderKind>,
|
||||
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误
|
||||
pending_terminal_failure: Option<PendingTerminalFailure>,
|
||||
client_retry: Option<ClientRetryGuard>,
|
||||
}
|
||||
|
||||
struct PendingTerminalFailure {
|
||||
@@ -575,6 +612,7 @@ where
|
||||
error,
|
||||
send_state,
|
||||
} => {
|
||||
self.record_client_retry_failure(&error).await;
|
||||
let requirement = if self.delivery_committed {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
@@ -912,7 +950,13 @@ where
|
||||
.translate_native_response(event, terminal);
|
||||
match translated {
|
||||
Ok(events) if events.is_empty() => continue,
|
||||
Ok(events) => return Ok(PullOutcome::Events(events)),
|
||||
Ok(events) => {
|
||||
// 成功终态交付后客户端即可续发工具结果,不能等流 EOF 或异步观测再清理
|
||||
if terminal && let Some(guard) = &mut self.client_retry {
|
||||
guard.success().await;
|
||||
}
|
||||
return Ok(PullOutcome::Events(events));
|
||||
}
|
||||
Err(error) => {
|
||||
self.finish_provider_error(&error).await?;
|
||||
return Err(provider_engine_error(error));
|
||||
@@ -1170,6 +1214,17 @@ where
|
||||
retry_allowed: continuation_retry || candidate_retry,
|
||||
})
|
||||
.await?;
|
||||
attempt_trace.record("retry.decided", json!({
|
||||
"phase": "prepare", "retryable": retryable,
|
||||
"continuationRetry": continuation_retry, "candidateRetry": candidate_retry,
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sendState": error.send_state().as_str(),
|
||||
"requestSendState": self.current_send_state().as_str(),
|
||||
"continuation": format!("{:?}", self.continuation_attempt),
|
||||
"accountRotations": self.account_rotations,
|
||||
"maxAccountRotations": self.plan.account_selection_policy().max_account_rotations(),
|
||||
"remainingRoutingAttempts": self.plan.max_attempts().get().saturating_sub(self.routing_attempts),
|
||||
}));
|
||||
if retryable {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
@@ -1582,22 +1637,32 @@ where
|
||||
&& attempt_send_state != UpstreamSendState::Ambiguous
|
||||
&& provider_proved_replay_safe
|
||||
&& self.routing_attempts < self.plan.max_attempts().get();
|
||||
let suppress_transient_retry = is_return_error_retry(&error)
|
||||
&& self
|
||||
.client_retry
|
||||
.as_ref()
|
||||
.is_some_and(ClientRetryGuard::has_failure);
|
||||
let transient_retry = match error.pre_delivery_retry() {
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransientRetry {
|
||||
max_retries,
|
||||
initial_delay,
|
||||
max_delay,
|
||||
}) if ordinary_retry => {
|
||||
jitter,
|
||||
..
|
||||
}) if ordinary_retry && !suppress_transient_retry => {
|
||||
let retries = self
|
||||
.transient_retry_counts
|
||||
.entry(current.metadata.provider_account_id().clone())
|
||||
.or_default();
|
||||
if *retries < max_retries.get() {
|
||||
let multiplier = 1_u32.checked_shl(*retries).unwrap_or(u32::MAX);
|
||||
// 服务器建议优先于本地退避,不能被本地上限缩短或再次指数放大
|
||||
let delay = error
|
||||
.retry_after()
|
||||
.unwrap_or_else(|| initial_delay.saturating_mul(multiplier).min(max_delay));
|
||||
// 服务器建议优先于本地退避,不截断、不放大,也不叠加随机等待
|
||||
let delay = error.retry_after().unwrap_or_else(|| {
|
||||
initial_delay
|
||||
.saturating_mul(multiplier)
|
||||
.min(max_delay)
|
||||
.saturating_add(jitter)
|
||||
});
|
||||
*retries = retries.saturating_add(1);
|
||||
Some(delay)
|
||||
} else {
|
||||
@@ -1620,9 +1685,11 @@ where
|
||||
// 同一排除分支。换号预算只在这里消耗;同账号分支(瞬态退避、传输恢复、
|
||||
// 凭据恢复重放、continuation 精确重连)不消耗。不能直接把预算门写进
|
||||
// ordinary_retry:它是同账号瞬态退避的前置条件,会被连带误伤
|
||||
// RetryThenReturnError 耗尽后直接交付错误,不能落入 ordinary_retry 的换号兜底
|
||||
let rotation_retry = !continuation_retry
|
||||
&& !same_account_retry
|
||||
&& transient_retry.is_none()
|
||||
&& !is_return_error_retry(&error)
|
||||
&& transport_recovery.is_none()
|
||||
&& (ordinary_retry || account_rotation_retry)
|
||||
&& self.account_rotations
|
||||
@@ -1664,10 +1731,12 @@ where
|
||||
};
|
||||
|
||||
self.trace.attempt(current.index.get()).record("retry.decided", json!({
|
||||
"phase": "execute",
|
||||
"retryable": retryable, "continuationRetry": continuation_retry,
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
"connectionRetry": connection_retry_requested,
|
||||
"connectionRetries": self.connection_retries,
|
||||
@@ -1676,6 +1745,17 @@ where
|
||||
"delayMs": transient_retry.or(transport_recovery.map(|(_, delay)| delay)).map(duration_ms),
|
||||
"downstreamCommitted": self.response_commit.committed_at().is_some(),
|
||||
"replayBarrierReached": self.replay_barrier_reached,
|
||||
"deliveryPending": self.delivery_pending,
|
||||
"providerProvedReplaySafe": provider_proved_replay_safe,
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"requiredAccount": self.account_selection.required_account().map(|account| account.as_str()),
|
||||
"continuation": format!("{:?}", self.continuation_attempt),
|
||||
"connectionBudgetExhausted": self.connection_budget.exhausted(),
|
||||
"remainingRoutingAttempts": self.plan.max_attempts().get().saturating_sub(self.routing_attempts),
|
||||
"maxAccountRotations": self.plan.account_selection_policy().max_account_rotations(),
|
||||
"transientRetriesUsed": self.transient_retry_counts.get(current.metadata.provider_account_id()).copied().unwrap_or_default(),
|
||||
"credentialRecoveryAttempted": self.credential_recovery_attempted_accounts.contains(current.metadata.provider_account_id()),
|
||||
"hasRetryPolicy": self.request_policy.is_some(),
|
||||
"sendState": format!("{attempt_send_state:?}"),
|
||||
}));
|
||||
if retryable {
|
||||
@@ -1775,6 +1855,7 @@ where
|
||||
&& !self.cancellation.is_cancelled()
|
||||
&& !self.execution_effect_observed();
|
||||
let allowed = facts.retry_allowed;
|
||||
let attempt_index = facts.attempt_index;
|
||||
let decision = {
|
||||
let cancellation = self.cancellation.clone();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
@@ -1787,6 +1868,16 @@ where
|
||||
decision = decision => Ok(decision),
|
||||
}
|
||||
}?;
|
||||
self.trace.attempt(attempt_index.get()).record(
|
||||
"retry.policy",
|
||||
json!({
|
||||
"hostAllowed": allowed,
|
||||
"policyStopped": decision == super::policy::RetryDecision::Stop,
|
||||
"executionEffectObserved": self.execution_effect_observed(),
|
||||
"cancelled": self.cancellation.is_cancelled(),
|
||||
"deadlineElapsed": self.deadline.is_elapsed(),
|
||||
}),
|
||||
);
|
||||
// 策略只收窄宿主已允许的恢复,返回后仍复核期限和外部副作用
|
||||
Ok(allowed
|
||||
&& decision != super::policy::RetryDecision::Stop
|
||||
@@ -1948,6 +2039,9 @@ where
|
||||
if self.is_finalized() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(guard) = &mut self.client_retry {
|
||||
guard.success().await;
|
||||
}
|
||||
let completed_at = SystemTime::now();
|
||||
self.observation.finish();
|
||||
let upstream_request_id = self
|
||||
@@ -2022,6 +2116,7 @@ where
|
||||
error: &ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
) -> Result<(), EngineError> {
|
||||
self.record_client_retry_failure(error).await;
|
||||
let send_state = self.raise_send_watermark(send_state);
|
||||
let outcome = if error.kind() == ProviderErrorKind::Cancelled {
|
||||
ExecutionOutcome::Cancelled
|
||||
@@ -2049,6 +2144,16 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn record_client_retry_failure(&mut self, error: &ProviderError) {
|
||||
// 只记录本策略的终态失败,不让其他错误改变客户端轮次的退避预算
|
||||
if self.attempts > 0
|
||||
&& is_return_error_retry(error)
|
||||
&& let Some(guard) = &mut self.client_retry
|
||||
{
|
||||
guard.failure().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_interruption(&mut self, error: &EngineError) -> Result<(), EngineError> {
|
||||
// 结算 I/O 可能继续等待,先冻结交付事实以保持终态与实际 HTTP 状态一致
|
||||
self.response_commit.finish();
|
||||
@@ -2343,6 +2448,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
fn is_return_error_retry(error: &ProviderError) -> bool {
|
||||
matches!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(crate::error::PreDeliveryRetry::SameAccountTransientRetry {
|
||||
policy: crate::error::TransientRetryPolicy::RetryThenReturnError,
|
||||
..
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
async fn best_effort_store_write<T>(
|
||||
operation: &'static str,
|
||||
request_id: &ModelRequestId,
|
||||
|
||||
@@ -92,6 +92,7 @@ pub struct DefaultExecutionService {
|
||||
execution_extensions: Option<ExecutionExtensionIndex>,
|
||||
frontend_authentication: Option<FrontendAuthenticationExtensionIndex>,
|
||||
session_affinity: Option<Arc<dyn ProviderSessionAffinityPort>>,
|
||||
client_retry: Option<Arc<dyn crate::engine::client_retry::ClientRetryStore>>,
|
||||
snapshot_refresh: Option<Arc<RuntimeSnapshotPublisher>>,
|
||||
active_requests: ActiveRequestRegistry,
|
||||
}
|
||||
@@ -122,6 +123,7 @@ impl DefaultExecutionService {
|
||||
execution_extensions: None,
|
||||
frontend_authentication: None,
|
||||
session_affinity: None,
|
||||
client_retry: None,
|
||||
snapshot_refresh: None,
|
||||
active_requests: Arc::default(),
|
||||
}
|
||||
@@ -133,6 +135,15 @@ impl DefaultExecutionService {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_client_retry_store(
|
||||
mut self,
|
||||
store: Arc<dyn crate::engine::client_retry::ClientRetryStore>,
|
||||
) -> Self {
|
||||
self.client_retry = Some(store);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_observers(mut self, observers: RequestObserverExtensionIndex) -> Self {
|
||||
self.request_observers = Some(observers);
|
||||
@@ -389,6 +400,10 @@ impl DefaultExecutionService {
|
||||
started_at: SystemTime,
|
||||
request_id: Option<ModelRequestId>,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
crate::engine::image_generation::check_image_generation(
|
||||
authorization.account_scope.allows_image_generation(),
|
||||
&request.operation,
|
||||
)?;
|
||||
let request_id = request_id.map_or_else(new_request_id, Ok)?;
|
||||
if authorization.cancellation.is_cancelled() {
|
||||
return Err(GatewayError::new(
|
||||
@@ -829,8 +844,15 @@ impl DefaultExecutionService {
|
||||
started_at,
|
||||
deadline_at: authorization.deadline_at,
|
||||
};
|
||||
let coordinator =
|
||||
AttemptCoordinator::new(GatewayEngine::new(self.observations.clone(), providers));
|
||||
let mut engine = GatewayEngine::new(self.observations.clone(), providers);
|
||||
// 插件内部调用不共享外部客户端的失败轮次,探测另走 transient engine
|
||||
if authorization.nested.is_none()
|
||||
&& authorization.bound.is_none()
|
||||
&& let Some(store) = &self.client_retry
|
||||
{
|
||||
engine = engine.with_client_retry_store(Arc::clone(store));
|
||||
}
|
||||
let coordinator = AttemptCoordinator::new(engine);
|
||||
let core = match coordinator
|
||||
.start_observed(
|
||||
new_request,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
//! 冻结分组生图权限的请求与发送前门禁
|
||||
|
||||
use crate::{
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
operation::Operation,
|
||||
};
|
||||
|
||||
pub(super) fn check_image_generation(
|
||||
allowed: bool,
|
||||
operation: &Operation,
|
||||
) -> Result<(), GatewayError> {
|
||||
if !allowed && operation.image_generation_requested() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"Image generation is not allowed by the API key's account groups",
|
||||
)
|
||||
.with_client_code("image_generation_disabled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user