mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-09 00:05:47 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ae38dc0c5 | ||
|
|
503e47fac2 | ||
|
|
b4aea0ea09 | ||
|
|
63036a3694 | ||
|
|
a3e24198cd | ||
|
|
fdd124e72f | ||
|
|
3a2ccbc5e6 | ||
|
|
aa0ce52034 | ||
|
|
7e7acad6a3 | ||
|
|
b3dd9b5337 | ||
|
|
0e1c609cd9 | ||
|
|
12771aca87 | ||
|
|
23b85172b5 | ||
|
|
d89db84a77 | ||
|
|
6504761ce4 | ||
|
|
5877b0cb2f | ||
|
|
958e13c370 |
@@ -17,6 +17,7 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
|
||||
| 开发、排查或审查插件接入、宿主 SDK、Runtime、扩展接口或独立网关插件;目标在 `backend/crates/gateway-plugin/`、`modules/plugins/` | 完整读取[插件职责边界](references/plugin-boundaries.md)及所用 [SDK](../../../backend/crates/gateway-plugin/sdk/README.md) 类型与能力合同;宿主维护自身合同、资源与稳定性,插件业务错误在插件侧修复,不在宿主、SDK 或 Runtime 中补偿;独立插件开发、排查、审查另用 [cpr-plugin-dev](../cpr-plugin-dev/SKILL.md) |
|
||||
| 开发、排查或审查服务端接口、路由、调度、账号与用量规则、持久化、后台任务或协议行为;目标是 `backend/` 中的实现、配置、迁移,或 Rust 源码 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)的模块与依赖规则、所属模块及[后端自审](../../../docs/architecture.md#后端自审);Rust 使用 `$rust-best-practices` 和下方后端注释要求,宿主 Rust 同时读取 [Rust 模块组织约定](../../../docs/architecture.md#32-rust-模块组织约定) |
|
||||
| 修改页面接口调用,或开发、排查、审查面向管理端或模型客户端的 HTTP/WS API 合同 | [API](../../../docs/api.md)对应章节;优先复用已有接口,核对调用方 → 服务端解析 → 实际查询或执行,不能只凭前端类型认定参数生效;按实际改动叠加前后端规范 |
|
||||
| 新增、修改或审查客户端可见错误、失败提示、后台任务状态文案 | 下方[错误文案](#错误文案),并按实际出口核对[错误与诊断三层边界](../../../docs/architecture.md#错误与诊断三层边界) |
|
||||
| 模块或调用链变更 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)及所属模块章节,不通读整份架构 |
|
||||
| 新增、调整、清理或审查测试 | 下方[测试维护](#测试维护),按当前行为、职责归属与已有覆盖判断保留、合并或删除 |
|
||||
| 修改部署行为、`deploy/` 配置或数据库迁移 | 按范围读取[部署](../../../deploy/README.md)、[迁移](../../../backend/migrations/README.md)对应章节 |
|
||||
@@ -55,3 +56,23 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
|
||||
- 每个 Rust 文件顶层使用简短的中文 `//!` 注释说明文件职责,覆盖生产源码、测试与构建脚本;测试文件说明测试范围或辅助用途,生成文件同步维护生成器
|
||||
- 注释不使用中文或英文句号,多句说明按语义分行;保留 URL、版本号、标识符和代码示例中有语义的点号
|
||||
- 文件说明聚焦当前功能与职责边界,局部注释解释原因和约束,避免复述代码或记录修改经过
|
||||
|
||||
## 错误文案
|
||||
|
||||
适用于项目自定义的客户端提示,包括 HTTP、WS、SSE 和任务失败状态;原始上游错误与内部诊断遵守[三层边界](../../../docs/architecture.md#错误与诊断三层边界),不为统一外观翻译、截短或改写
|
||||
|
||||
- 管理端自定义提示统一中文,模型 API 自定义提示统一英文;英文句首大写、句末使用句号,API、Codex 等缩写及专有名词保持正确大小写,中文短提示不加句末句号
|
||||
- 默认一句说明具体问题,确有必要才补第二句处理办法,不设机械字数限制;明确字段、对象或限制,避免只说「请求不合法」或堆叠内部术语
|
||||
- 处理办法必须明确且适用;不把「请稍后重试」「联系管理员」作为统一后缀,不罗列尚未确认的网络、权限、磁盘等可能原因,不对执行结果未知的操作笼统建议重试
|
||||
- 机器码、状态与结构化字段负责程序判断,文案供人阅读;只改措辞时保留分类、重试和账号处置语义,先核对现有调用方是否依赖消息文本
|
||||
- 在拥有错误事实的模块提供安全提示,保留独立的内部原因链;不在响应出口批量首字母大写、翻译或裁剪任意消息,以免改写上游原文
|
||||
|
||||
| 场景 | 文案示例 |
|
||||
| --- | --- |
|
||||
| 日费用额度耗尽 | `Daily spending limit reached.` |
|
||||
| 请求过大 | `Request too large. Reduce the input size.` |
|
||||
| 客户端过旧 | `Codex CLI is outdated. Upgrade to 0.152.0 or later.` |
|
||||
| 名称为空 | `请输入 API Key 名称` |
|
||||
| 更新包校验失败 | `更新包校验失败,请重新下载` |
|
||||
|
||||
审查时沿实际出口检查语言、大小写、标点、长度和处理建议;复用现有行为测试核对错误码及敏感信息边界,不为逐条文案建立重复的快照或字符串清单测试
|
||||
|
||||
Generated
+2
@@ -1743,6 +1743,8 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"regex",
|
||||
"secrecy",
|
||||
"serde",
|
||||
|
||||
@@ -126,30 +126,32 @@ pub struct RetentionDecision {
|
||||
/// 自动保留规则:
|
||||
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理
|
||||
/// - 自动计划备份满足任一启用阈值即可进入删除
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(全局倒序下标 0)
|
||||
/// - 手动备份不进入本规则;手工/计划记录各自的 `expires_at` 由到期扫描处理
|
||||
/// - `offset` 是当前页在全部 completed 计划备份中的起始位置
|
||||
#[must_use]
|
||||
pub fn decide_retention(
|
||||
retention_days: u32,
|
||||
retention_count: u32,
|
||||
now: DateTime<Utc>,
|
||||
offset: u64,
|
||||
scheduled_completed_desc: &[BackupRecord],
|
||||
) -> Vec<RetentionDecision> {
|
||||
let mut decisions = Vec::new();
|
||||
for (index, record) in scheduled_completed_desc.iter().enumerate() {
|
||||
let position = offset.saturating_add(index as u64);
|
||||
if record.status != BackupStatus::Completed {
|
||||
continue;
|
||||
}
|
||||
// 最近一个成功的计划备份永远保留
|
||||
if index == 0 {
|
||||
if position == 0 {
|
||||
continue;
|
||||
}
|
||||
let expired_by_days = retention_days > 0
|
||||
&& record.completed_at.is_some_and(|completed_at| {
|
||||
completed_at <= now - Duration::days(i64::from(retention_days))
|
||||
});
|
||||
let exceeded_count =
|
||||
retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX);
|
||||
let exceeded_count = retention_count > 0 && position >= u64::from(retention_count);
|
||||
if expired_by_days || exceeded_count {
|
||||
decisions.push(RetentionDecision {
|
||||
record_id: record.id.clone(),
|
||||
|
||||
@@ -38,8 +38,8 @@ use super::policy::{BackupSchedule, decide_retention};
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量
|
||||
const RETENTION_BATCH_SIZE: usize = 10;
|
||||
/// 保留扫描一次读取的 completed 计划备份上限
|
||||
const RETENTION_SCAN_LIMIT: u32 = 1000;
|
||||
/// 保留扫描每页读取的 completed 计划备份数量
|
||||
const RETENTION_PAGE_SIZE: u32 = 1000;
|
||||
/// 每个循环最多完成的待删除记录数量
|
||||
const PENDING_DELETION_BATCH: u32 = 20;
|
||||
|
||||
@@ -112,7 +112,7 @@ impl BackupTask {
|
||||
self.recover_intermediate(cancellation).await?;
|
||||
self.finish_pending_deletions().await?;
|
||||
self.process_next_task(cancellation).await?;
|
||||
self.run_retention_batch(Utc::now()).await?;
|
||||
self.run_retention_batch(Utc::now(), cancellation).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -637,7 +637,11 @@ impl BackupTask {
|
||||
}
|
||||
|
||||
/// 执行一小批到期保留清理:先处理 `expires_at` 已到期的记录,再按天数/份数清理计划备份
|
||||
async fn run_retention_batch(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
async fn run_retention_batch(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
|
||||
// 1. expires_at 已到期的记录(手动或计划)无条件进入删除
|
||||
@@ -659,18 +663,41 @@ impl BackupTask {
|
||||
}
|
||||
|
||||
// 2. 计划备份按 retentionDays / retentionCount 清理
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(RETENTION_SCAN_LIMIT)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let decisions = decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
&records,
|
||||
);
|
||||
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
|
||||
if settings.retention_days == 0 && settings.retention_count == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut offset = 0;
|
||||
let mut decisions = Vec::new();
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(offset, RETENTION_PAGE_SIZE)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let remaining = RETENTION_BATCH_SIZE - decisions.len();
|
||||
decisions.extend(
|
||||
decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
offset,
|
||||
&records,
|
||||
)
|
||||
.into_iter()
|
||||
.take(remaining),
|
||||
);
|
||||
if decisions.len() == RETENTION_BATCH_SIZE
|
||||
|| records.len() < RETENTION_PAGE_SIZE as usize
|
||||
{
|
||||
break;
|
||||
}
|
||||
offset += records.len() as u64;
|
||||
}
|
||||
// 收齐本批候选后再删除,避免删除前页记录导致后续分页跳过数据
|
||||
for decision in decisions {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&decision.record_id, now, None)
|
||||
|
||||
@@ -68,6 +68,7 @@ pub struct AdminError {
|
||||
kind: AdminErrorKind,
|
||||
message: String,
|
||||
source: Option<gateway_core::error::ErrorSource>,
|
||||
sensitive_context_redacted: bool,
|
||||
}
|
||||
|
||||
impl AdminError {
|
||||
@@ -77,6 +78,7 @@ impl AdminError {
|
||||
kind,
|
||||
message: message.into(),
|
||||
source: None,
|
||||
sensitive_context_redacted: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,9 +88,19 @@ impl AdminError {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn redact_sensitive_context(mut self) -> Self {
|
||||
self.sensitive_context_redacted = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<gateway_core::error::ErrorDetails> {
|
||||
gateway_core::error::ErrorDetails::capture(self.source.as_ref(), None, false)
|
||||
gateway_core::error::ErrorDetails::capture(
|
||||
self.source.as_ref(),
|
||||
None,
|
||||
self.sensitive_context_redacted,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -1352,6 +1352,84 @@ pub struct UsageInsights {
|
||||
pub providers: Vec<ProviderObservation>,
|
||||
}
|
||||
|
||||
/// 单次洞察查询所需的指标,持久化层据此选择计数、分布与费用状态
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum UsageInsightMetric {
|
||||
Health,
|
||||
Latency,
|
||||
FirstToken,
|
||||
Throughput,
|
||||
Load,
|
||||
Cost,
|
||||
Savings,
|
||||
Cache,
|
||||
}
|
||||
|
||||
/// 单个指标的时间点,活动计数与指标共用读取快照
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageMetricPoint<T> {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
pub value: T,
|
||||
}
|
||||
|
||||
/// 各指标复用时间序列形状,汇总与时间点保留各自的类型
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageMetricSeries<S, P = S> {
|
||||
pub summary: S,
|
||||
pub points: Vec<UsageMetricPoint<P>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageThroughputPercentiles {
|
||||
pub p10: Option<u64>,
|
||||
pub p50: Option<u64>,
|
||||
pub p90: Option<u64>,
|
||||
}
|
||||
|
||||
/// 成本卡片各视图共同显示的汇总
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCostSummary {
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub cache_savings: Option<DecimalAmount>,
|
||||
pub tier_premium: Option<DecimalAmount>,
|
||||
pub cost_per_successful_request: Option<DecimalAmount>,
|
||||
pub coverage: CostCoverage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCostComparison {
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub no_cache_cost: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCacheEfficiency {
|
||||
pub input_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 只发布本次请求的指标,未查询的值不伪装成零
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum UsageInsightData {
|
||||
Health(UsageInsightsHealth),
|
||||
Latency(UsageMetricSeries<LatencyPercentiles>),
|
||||
FirstToken(UsageMetricSeries<LatencyPercentiles>),
|
||||
Throughput(UsageMetricSeries<UsageThroughputPercentiles>),
|
||||
Load(UsageMetricSeries<UsageInsightsLoad>),
|
||||
Cost(UsageMetricSeries<UsageCostSummary, UsageCostComparison>),
|
||||
Savings(UsageMetricSeries<UsageCostSummary, Option<DecimalAmount>>),
|
||||
Cache(UsageMetricSeries<UsageCostSummary, UsageCacheEfficiency>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageInsight {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub granularity: Granularity,
|
||||
pub data: UsageInsightData,
|
||||
}
|
||||
|
||||
/// 已完成分母计算的诊断项
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsItem {
|
||||
|
||||
@@ -72,7 +72,7 @@ pub fn account_quota_forecasts(
|
||||
},
|
||||
extrapolated: false,
|
||||
source: None,
|
||||
unavailable_reason: Some("没有可统计的周/月额度窗口,请先刷新账号额度。"),
|
||||
unavailable_reason: Some("缺少周/月额度窗口,请刷新账号额度"),
|
||||
low_sample: false,
|
||||
incomplete_cost: false,
|
||||
incomplete_tokens: false,
|
||||
@@ -138,30 +138,27 @@ impl AccountQuotaForecast {
|
||||
self.incomplete_tokens = usage.is_some_and(|usage| usage.missing_token_count > 0);
|
||||
let method = sample.map_or(QuotaForecastMethod::Cumulative, |sample| sample.method);
|
||||
let Some(start) = start.filter(|start| *start <= now && now < reset_at) else {
|
||||
self.unavailable_reason = Some("额度窗口已过期或边界无效,请刷新账号额度后重试。");
|
||||
self.unavailable_reason = Some("额度窗口已过期或无效,请刷新账号额度");
|
||||
return;
|
||||
};
|
||||
if !observed_at.is_some_and(|observed| start <= observed && observed <= now) {
|
||||
self.unavailable_reason = Some("缺少本周期的额度快照,请先刷新账号额度。");
|
||||
self.unavailable_reason = Some("缺少本周期的额度快照,请刷新账号额度");
|
||||
return;
|
||||
}
|
||||
if sample.is_some_and(|sample| sample.discontinuous) {
|
||||
self.unavailable_reason =
|
||||
Some("额度观测出现回落或累计记录不连续,正在重新积累配对样本。");
|
||||
self.unavailable_reason = Some("额度回落或记录不连续,正在重新积累配对样本");
|
||||
return;
|
||||
}
|
||||
if account_added_at > start && method == QuotaForecastMethod::Cumulative {
|
||||
self.unavailable_reason = Some(
|
||||
"本周期开始时的记录不完整,正在积累至少 5 个百分点的配对观测,无需等待下次重置。",
|
||||
);
|
||||
self.unavailable_reason = Some("周期初记录不完整,正在积累至少 5 个百分点的配对观测");
|
||||
return;
|
||||
}
|
||||
let Some(percent) = percent else {
|
||||
self.unavailable_reason = Some("已用比例未知,请刷新额度后查看预测。");
|
||||
self.unavailable_reason = Some("已用比例未知,请刷新账号额度");
|
||||
return;
|
||||
};
|
||||
let Some(usage) = usage.filter(|usage| usage.request_count > 0) else {
|
||||
self.unavailable_reason = Some("本周期没有网关用量记录,暂时无法预测额度。");
|
||||
self.unavailable_reason = Some("本周期没有网关用量记录,无法预测额度");
|
||||
return;
|
||||
};
|
||||
let Some(sample) = sample.filter(|sample| {
|
||||
@@ -172,8 +169,7 @@ impl AccountQuotaForecast {
|
||||
&& sample.sampled_percent >= MIN_USED_PERCENT
|
||||
&& sample.sampled_percent <= percent
|
||||
}) else {
|
||||
self.unavailable_reason =
|
||||
Some("有效额度进度不足 5 个百分点或采样边界无效,请继续积累用量。");
|
||||
self.unavailable_reason = Some("有效额度进度不足 5 个百分点或采样边界无效");
|
||||
return;
|
||||
};
|
||||
self.low_sample = sample.sampled_percent < LOW_SAMPLE_PERCENT
|
||||
@@ -196,7 +192,7 @@ impl AccountQuotaForecast {
|
||||
.and_then(|(used, remaining)| estimate(used + remaining, factor));
|
||||
self.unavailable_reason = if self.estimated_tokens.is_none() && self.estimated_usd.is_none()
|
||||
{
|
||||
Some("本周期暂无可用于估算的 Token 或费用数据,请积累用量后重试。")
|
||||
Some("本周期缺少可用于估算的 Token 或费用数据")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
@@ -136,9 +136,10 @@ pub trait BackupRepository: Send + Sync {
|
||||
timezone: &str,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 按完成时间倒序读取 completed 计划备份,供保留策略扫描(最多 `limit` 条)
|
||||
/// 按完成时间及 ID 倒序分页读取 completed 计划备份,offset 相对完整集合
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
}
|
||||
@@ -388,6 +389,7 @@ impl BackupRepository for UnavailableBackupRepository {
|
||||
}
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_offset: u64,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
|
||||
@@ -148,7 +148,7 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
> {
|
||||
Err(gateway_core::settings::privacy::PrivacyError {
|
||||
rule_index: 0,
|
||||
reason: "Provider 不支持隐私策略",
|
||||
reason: "该服务商不支持隐私策略",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
> {
|
||||
Err(gateway_core::settings::privacy::PrivacyError {
|
||||
rule_index: 0,
|
||||
reason: "Provider 不支持隐私策略",
|
||||
reason: "该服务商不支持隐私策略",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,8 @@ use crate::model::{
|
||||
DashboardObservation, DashboardQuery, DiagnosticDimension, DiagnosticsObservation,
|
||||
Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange,
|
||||
UsageBillingProjection, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsightsObservation, UsagePage, UsageQuery, UsageSummaryObservation,
|
||||
UsageInsightMetric, UsageInsightsObservation, UsagePage, UsageQuery,
|
||||
UsageSummaryObservation,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult,
|
||||
@@ -456,11 +457,22 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>>;
|
||||
|
||||
/// 同一读取快照中的汇总、趋势和已投影情景费用
|
||||
/// 不需要并发时关闭 include_concurrency,返回空并发序列并跳过明细计算
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
/// 只读取所选指标及其可见汇总依赖,未选择的字段不供业务层消费
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
metric: UsageInsightMetric,
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage>;
|
||||
|
||||
@@ -38,11 +38,13 @@ pub enum SystemOperationErrorKind {
|
||||
}
|
||||
|
||||
/// 不泄漏路径、命令行或发布凭据的系统操作错误
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("system operation failed: {message}")]
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct SystemOperationError {
|
||||
kind: SystemOperationErrorKind,
|
||||
message: String,
|
||||
source: Option<gateway_core::error::ErrorSource>,
|
||||
sensitive_context_redacted: bool,
|
||||
}
|
||||
|
||||
impl SystemOperationError {
|
||||
@@ -51,9 +53,37 @@ impl SystemOperationError {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
source: None,
|
||||
sensitive_context_redacted: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
|
||||
self.source = Some(source.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn redact_sensitive_context(mut self) -> Self {
|
||||
self.sensitive_context_redacted = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn sensitive_context_redacted(&self) -> bool {
|
||||
self.sensitive_context_redacted
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn error_details(&self) -> Option<gateway_core::error::ErrorDetails> {
|
||||
gateway_core::error::ErrorDetails::capture(
|
||||
self.source.as_ref(),
|
||||
None,
|
||||
self.sensitive_context_redacted,
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> SystemOperationErrorKind {
|
||||
self.kind
|
||||
|
||||
@@ -116,14 +116,14 @@ pub trait AccountsService: Send + Sync {
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<AccountPersonalInfo, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持个人信息"))
|
||||
Err(AdminError::invalid("该服务商不支持个人信息查询"))
|
||||
}
|
||||
|
||||
async fn profile_avatar(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderProfileAvatar, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持账号头像"))
|
||||
Err(AdminError::invalid("该服务商不支持账号头像"))
|
||||
}
|
||||
|
||||
async fn reset_credit_automation_status(
|
||||
@@ -154,7 +154,7 @@ pub trait AccountsService: Send + Sync {
|
||||
_context: &MutationContext,
|
||||
_account_id: ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持重置额度"))
|
||||
Err(AdminError::invalid("该服务商不支持额度重置"))
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
@@ -162,7 +162,7 @@ pub trait AccountsService: Send + Sync {
|
||||
_context: &MutationContext,
|
||||
_command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持重置额度"))
|
||||
Err(AdminError::invalid("该服务商不支持额度重置"))
|
||||
}
|
||||
|
||||
async fn models(
|
||||
@@ -315,7 +315,7 @@ impl DefaultAccountsService {
|
||||
.load_account(account_id.as_str(), runtime)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account"))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 账号不存在"))
|
||||
.ok_or_else(|| AdminError::not_found("服务商账号不存在"))
|
||||
}
|
||||
|
||||
async fn provider_for_account(
|
||||
@@ -564,7 +564,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.zip(quotas)
|
||||
.map(|(mut item, quota)| {
|
||||
let id = ProviderAccountId::new(item.account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Provider 账号 ID 不合法"))?;
|
||||
.map_err(|_| AdminError::invalid("服务商账号 ID 不合法"))?;
|
||||
let capabilities = providers
|
||||
.require(&item.account.provider_kind)
|
||||
.map(|provider| {
|
||||
@@ -743,7 +743,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError> {
|
||||
let account_id = ProviderAccountId::new(command.account_id.clone())
|
||||
.map_err(|_| AdminError::invalid("Provider 账号 ID 不合法"))?;
|
||||
.map_err(|_| AdminError::invalid("服务商账号 ID 不合法"))?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let enabled = command.enabled;
|
||||
let result = self
|
||||
@@ -771,7 +771,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.iter()
|
||||
.map(|id| {
|
||||
ProviderAccountId::new(id.clone())
|
||||
.map_err(|_| AdminError::invalid("Provider 账号 ID 不合法"))
|
||||
.map_err(|_| AdminError::invalid("服务商账号 ID 不合法"))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut providers = BTreeMap::<
|
||||
|
||||
@@ -326,17 +326,15 @@ impl DefaultBackupService {
|
||||
/// 校验存储配置;空串视为非法,Secret 缺省表示保留旧值
|
||||
fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminError> {
|
||||
validate_endpoint(&command.endpoint)?;
|
||||
validate_nonempty("region", &command.region)?;
|
||||
validate_nonempty("bucket", &command.bucket)?;
|
||||
validate_nonempty("accessKeyId", &command.access_key_id)?;
|
||||
validate_nonempty("存储区域", &command.region)?;
|
||||
validate_nonempty("存储桶名称", &command.bucket)?;
|
||||
validate_nonempty("访问密钥 ID", &command.access_key_id)?;
|
||||
if let Some(secret) = command.secret_access_key.as_ref() {
|
||||
let value = secret.expose_secret();
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request(
|
||||
"secretAccessKey 不能为空字符串;缺省表示保留旧值",
|
||||
));
|
||||
return Err(AdminError::bad_request("访问密钥不能为空"));
|
||||
}
|
||||
validate_length("secretAccessKey", value, 512)?;
|
||||
validate_length("访问密钥", value, 512)?;
|
||||
}
|
||||
crate::model::backup::normalize_prefix(&command.prefix).map_err(map_backup_error)?;
|
||||
Ok(())
|
||||
@@ -344,17 +342,15 @@ fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminErr
|
||||
|
||||
fn validate_endpoint(value: &str) -> Result<(), AdminError> {
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request("endpoint 不能为空"));
|
||||
return Err(AdminError::bad_request("备份存储地址不能为空"));
|
||||
}
|
||||
if !value.starts_with("https://") {
|
||||
return Err(AdminError::bad_request(
|
||||
"endpoint 必须使用 HTTPS;仅测试或显式开发配置允许 HTTP",
|
||||
));
|
||||
return Err(AdminError::bad_request("备份存储地址必须使用 HTTPS"));
|
||||
}
|
||||
if value.contains('@') || value.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(AdminError::bad_request("endpoint 格式非法"));
|
||||
return Err(AdminError::bad_request("备份存储地址格式不合法"));
|
||||
}
|
||||
validate_length("endpoint", value, 2048)
|
||||
validate_length("备份存储地址", value, 2048)
|
||||
}
|
||||
|
||||
fn validate_nonempty(field: &str, value: &str) -> Result<(), AdminError> {
|
||||
|
||||
@@ -94,7 +94,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
.get_client_key(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "client API key"))?
|
||||
.ok_or_else(|| AdminError::not_found("Client API Key 不存在"))
|
||||
.ok_or_else(|| AdminError::not_found("API Key 不存在"))
|
||||
}
|
||||
|
||||
async fn budget(&self, id: &ClientApiKeyId) -> Result<ClientBudgetStatus, AdminError> {
|
||||
@@ -149,7 +149,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
.reveal_client_key(id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "client API key"))?
|
||||
.ok_or_else(|| AdminError::not_found("Client API Key 不存在"))
|
||||
.ok_or_else(|| AdminError::not_found("API Key 不存在"))
|
||||
}
|
||||
|
||||
async fn create(
|
||||
@@ -158,7 +158,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
command: CreateClientKey,
|
||||
) -> Result<CreatedClientKey, AdminError> {
|
||||
crate::model::client_keys::validate_group_ids(&command.group_ids)
|
||||
.map_err(|_| AdminError::invalid("Client API Key 分组不合法"))?;
|
||||
.map_err(|_| AdminError::invalid("API Key 分组不合法"))?;
|
||||
for (provider, profile) in &command.request_profile_overrides {
|
||||
self.providers
|
||||
.require(provider)
|
||||
@@ -202,7 +202,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
command: UpdateClientKey,
|
||||
) -> Result<ClientKeyMutation, AdminError> {
|
||||
crate::model::client_keys::validate_group_ids(&command.group_ids)
|
||||
.map_err(|_| AdminError::invalid("Client API Key 分组不合法"))?;
|
||||
.map_err(|_| AdminError::invalid("API Key 分组不合法"))?;
|
||||
for (provider, profile) in &command.request_profile_override_updates {
|
||||
if let Some(profile) = profile {
|
||||
self.providers
|
||||
@@ -277,9 +277,7 @@ fn validate_cursor(query: &ClientKeyListQuery) -> Result<(), AdminError> {
|
||||
return Ok(());
|
||||
};
|
||||
if cursor.sort != query.sort {
|
||||
return Err(AdminError::invalid(
|
||||
"Client API Key 游标排序与查询条件不一致",
|
||||
));
|
||||
return Err(AdminError::invalid("分页游标与排序条件不一致"));
|
||||
}
|
||||
let matches = matches!(
|
||||
(cursor.sort.field, &cursor.value),
|
||||
@@ -300,7 +298,7 @@ fn validate_cursor(query: &ClientKeyListQuery) -> Result<(), AdminError> {
|
||||
if matches {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(AdminError::invalid("Client API Key 游标不合法"))
|
||||
Err(AdminError::invalid("API Key 分页游标不合法"))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ pub(super) async fn required_credential(
|
||||
.credential_details(provider_kind, account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
.ok_or_else(|| AdminError::not_found("服务商凭据不存在"))
|
||||
}
|
||||
|
||||
pub(super) async fn required_plugin_credential(
|
||||
@@ -77,7 +77,7 @@ pub(super) async fn required_plugin_credential(
|
||||
.credential_details_by_id(account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))
|
||||
.ok_or_else(|| AdminError::not_found("服务商凭据不存在"))
|
||||
}
|
||||
|
||||
pub(super) async fn import_proxy_binding(
|
||||
@@ -156,7 +156,7 @@ pub(super) fn validate_prepared_import(
|
||||
.iter()
|
||||
.any(|credential| credential.provider_kind != *provider_kind)
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与请求范围不一致"));
|
||||
return Err(AdminError::conflict("服务商返回的凭据与请求范围不符"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -176,7 +176,7 @@ pub(super) fn validate_prepared_rotation_facts(
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
{
|
||||
return Err(AdminError::conflict("Provider 准备结果与当前凭据不一致"));
|
||||
return Err(AdminError::conflict("凭据处理结果与当前凭据不符"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -42,15 +42,17 @@ pub(super) fn map_provider_error(
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "Provider 请求不合法",
|
||||
AdminErrorKind::NotFound => "Provider 资源不存在",
|
||||
AdminErrorKind::Conflict => "Provider 资源状态冲突,请刷新后重试",
|
||||
AdminErrorKind::BadGateway => "上游服务请求失败",
|
||||
AdminErrorKind::UpstreamResultUnknown => "上游执行结果未知,请刷新状态后再决定是否重试",
|
||||
AdminErrorKind::Unavailable => "Provider 服务暂不可用",
|
||||
AdminErrorKind::Internal => "服务内部错误",
|
||||
_ => "Provider 操作失败",
|
||||
let message = match error.kind() {
|
||||
ProviderAdminErrorKind::Invalid => "服务商请求参数不合法",
|
||||
ProviderAdminErrorKind::Unsupported => "该服务商不支持此操作",
|
||||
ProviderAdminErrorKind::NotFound => "服务商资源不存在",
|
||||
ProviderAdminErrorKind::Conflict => "服务商资源状态冲突,请刷新后重试",
|
||||
ProviderAdminErrorKind::BadGateway => "上游服务请求失败",
|
||||
ProviderAdminErrorKind::Ambiguous => "上游执行结果未知,请刷新状态后再决定是否重试",
|
||||
ProviderAdminErrorKind::Unavailable | ProviderAdminErrorKind::CredentialRefreshRequired => {
|
||||
"服务商暂不可用"
|
||||
}
|
||||
ProviderAdminErrorKind::Internal => "服务内部错误",
|
||||
};
|
||||
AdminError::new(kind, error.public_message().unwrap_or(message)).with_source(OperationFailure {
|
||||
operation: resource,
|
||||
|
||||
@@ -194,7 +194,8 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
self.observations.usage_insights_observation(
|
||||
query.range,
|
||||
filter,
|
||||
Granularity::for_range(query.range)
|
||||
Granularity::for_range(query.range),
|
||||
false,
|
||||
),
|
||||
self.observations.usage_trend(
|
||||
today,
|
||||
|
||||
@@ -19,11 +19,14 @@ use crate::{
|
||||
DiagnosticDimension, DiagnosticsItem, DiagnosticsResult, Granularity, HealthStatus,
|
||||
HealthTimeline, HealthTimelinePoint, OpsErrorPage, OpsErrorQuery, ProviderBillingInput,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind, TrendPoint,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCalculatedBillingFact,
|
||||
UsageConcurrencyPoint, UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost,
|
||||
UsageInsightsCostPoint, UsageInsightsHealth, UsageInsightsHealthPoint,
|
||||
UsageInsightsLoad, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCacheEfficiency,
|
||||
UsageCalculatedBillingFact, UsageConcurrencyPoint, UsageCostComparison,
|
||||
UsageCostSummary, UsageDetail, UsageFilter, UsageInsight, UsageInsightData,
|
||||
UsageInsightMetric, UsageInsights, UsageInsightsCost, UsageInsightsCostPoint,
|
||||
UsageInsightsHealth, UsageInsightsHealthPoint, UsageInsightsLoad,
|
||||
UsageInsightsObservation, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageMetricPoint, UsageMetricSeries, UsageOverview, UsagePage, UsageQuery,
|
||||
UsageSummary, UsageThroughputPercentiles,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
@@ -65,6 +68,12 @@ pub trait ObservabilityService: Send + Sync {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> Result<UsageInsights, AdminError>;
|
||||
async fn usage_metric(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
metric: UsageInsightMetric,
|
||||
) -> Result<UsageInsight, AdminError>;
|
||||
async fn diagnostics(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
@@ -234,6 +243,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range,
|
||||
UsageFilter::default(),
|
||||
Granularity::for_range(range),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "dashboard trend"))?;
|
||||
@@ -291,21 +301,25 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let granularity = Granularity::for_range(range);
|
||||
let observation = self
|
||||
.store
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
.usage_insights_observation(range, filter, granularity, true)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage insights"))?;
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs, observation.concurrency)
|
||||
insights_from_observation(observation)
|
||||
}
|
||||
|
||||
async fn usage_metric(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
metric: UsageInsightMetric,
|
||||
) -> Result<UsageInsight, AdminError> {
|
||||
let observation = self
|
||||
.store
|
||||
.usage_metric_observation(range, filter, Granularity::for_range(range), metric)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage metric"))?;
|
||||
let insights = insights_from_observation(observation)?;
|
||||
Ok(select_usage_metric(insights, metric))
|
||||
}
|
||||
|
||||
async fn diagnostics(
|
||||
@@ -382,6 +396,148 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
fn insights_from_observation(
|
||||
observation: UsageInsightsObservation,
|
||||
) -> Result<UsageInsights, AdminError> {
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs, observation.concurrency)
|
||||
}
|
||||
|
||||
fn select_usage_metric(insights: UsageInsights, metric: UsageInsightMetric) -> UsageInsight {
|
||||
let activity = insights
|
||||
.health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| (point.bucket_start, point.total_requests))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let data = match metric {
|
||||
UsageInsightMetric::Health => UsageInsightData::Health(insights.health),
|
||||
UsageInsightMetric::Latency => UsageInsightData::Latency(metric_series(
|
||||
insights.performance.latency_percentiles,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.latency_percentiles)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::FirstToken => UsageInsightData::FirstToken(metric_series(
|
||||
insights.performance.first_token_latency_percentiles,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.first_token_latency_percentiles)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Throughput => UsageInsightData::Throughput(metric_series(
|
||||
UsageThroughputPercentiles {
|
||||
p10: insights.performance.output_throughput_p10,
|
||||
p50: insights.performance.output_throughput_p50,
|
||||
p90: insights.performance.output_throughput_p90,
|
||||
},
|
||||
insights.performance.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageThroughputPercentiles {
|
||||
p10: point.output_throughput_p10,
|
||||
p50: point.output_throughput_p50,
|
||||
p90: point.output_throughput_p90,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Load => UsageInsightData::Load(metric_series(
|
||||
insights.performance.load,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.load)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Cost => UsageInsightData::Cost(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights.cost.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageCostComparison {
|
||||
estimated_cost: point.estimated_cost,
|
||||
no_cache_cost: point.no_cache_cost,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Savings => UsageInsightData::Savings(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights
|
||||
.cost
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.cache_savings)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Cache => UsageInsightData::Cache(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights.cost.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageCacheEfficiency {
|
||||
input_tokens: point.input_tokens,
|
||||
cached_token_rate: point.cached_token_rate,
|
||||
cache_hit_request_rate: point.cache_hit_request_rate,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
};
|
||||
UsageInsight {
|
||||
analytics: insights.analytics,
|
||||
granularity: insights.granularity,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
fn cost_summary(cost: &UsageInsightsCost) -> UsageCostSummary {
|
||||
UsageCostSummary {
|
||||
estimated_cost: cost.estimated_cost.clone(),
|
||||
cache_savings: cost.cache_savings.clone(),
|
||||
tier_premium: cost.tier_premium.clone(),
|
||||
cost_per_successful_request: cost.cost_per_successful_request.clone(),
|
||||
coverage: cost.coverage.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_series<S, P>(
|
||||
summary: S,
|
||||
points: impl Iterator<Item = (DateTime<Utc>, P)>,
|
||||
activity: &BTreeMap<DateTime<Utc>, u64>,
|
||||
) -> UsageMetricSeries<S, P> {
|
||||
UsageMetricSeries {
|
||||
summary,
|
||||
points: points
|
||||
.map(|(bucket_start, value)| UsageMetricPoint {
|
||||
bucket_start,
|
||||
request_count: activity.get(&bucket_start).copied().unwrap_or_default(),
|
||||
value,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_usage_insights(
|
||||
overview: UsageOverview,
|
||||
trend: Vec<RequestMetricPoint>,
|
||||
|
||||
@@ -178,7 +178,8 @@ pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
dashboard: Cache<DashboardQuery, DashboardObservation>,
|
||||
summaries: Cache<UsageKey, UsageSummaryObservation>,
|
||||
insights: Cache<(UsageKey, Granularity), UsageInsightsObservation>,
|
||||
insights: Cache<(UsageKey, Granularity, bool), UsageInsightsObservation>,
|
||||
metrics: Cache<(UsageKey, Granularity, UsageInsightMetric), UsageInsightsObservation>,
|
||||
diagnostics: Cache<(UsageKey, DiagnosticDimension, u16), DiagnosticsObservation>,
|
||||
}
|
||||
impl CachedObservabilityStore {
|
||||
@@ -188,6 +189,7 @@ impl CachedObservabilityStore {
|
||||
dashboard: Cache::default(),
|
||||
summaries: Cache::default(),
|
||||
insights: Cache::default(),
|
||||
metrics: Cache::default(),
|
||||
diagnostics: Cache::default(),
|
||||
}
|
||||
}
|
||||
@@ -237,19 +239,44 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.insights
|
||||
.get(
|
||||
((range, filter.clone()), granularity),
|
||||
((range, filter.clone()), granularity, include_concurrency),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
self.inner.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
include_concurrency,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
metric: UsageInsightMetric,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.metrics
|
||||
.get(
|
||||
((range, filter.clone()), granularity, metric),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_metric_observation(range, filter, granularity, metric)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
|
||||
@@ -104,7 +104,7 @@ impl PluginAccountAccess for DefaultPluginAccountAccess {
|
||||
.load_credential_for_plugin(account_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "plugin credential"))?
|
||||
.ok_or_else(|| AdminError::not_found("Provider 凭据不存在"))?;
|
||||
.ok_or_else(|| AdminError::not_found("服务商凭据不存在"))?;
|
||||
Ok(PluginAccountCredential {
|
||||
account: credential.account,
|
||||
provider_material: credential.provider_material,
|
||||
|
||||
@@ -71,7 +71,7 @@ impl PluginsService {
|
||||
validate_update_policy(&binding)?;
|
||||
let PluginUpdateSource::Github { repository } = &binding.source else {
|
||||
return Err(AdminError::invalid(
|
||||
"该来源不提供 Release 更新查询,请显式选择并安装插件包",
|
||||
"该来源不支持更新查询,请手动安装插件包",
|
||||
));
|
||||
};
|
||||
let (tag, allow_prerelease) = match &binding.policy {
|
||||
@@ -356,9 +356,7 @@ fn validate_update_policy(binding: &PluginSourceBinding) -> Result<(), AdminErro
|
||||
if !matches!(binding.source, PluginUpdateSource::Github { .. })
|
||||
&& !matches!(binding.policy, PluginUpdatePolicy::Manual {})
|
||||
{
|
||||
return Err(AdminError::invalid(
|
||||
"只有 GitHub 来源支持 Release 更新检查策略",
|
||||
));
|
||||
return Err(AdminError::invalid("仅 GitHub 来源支持发行版本更新检查"));
|
||||
}
|
||||
if let PluginUpdatePolicy::Pinned { tag, .. } = &binding.policy
|
||||
&& (tag.is_empty()
|
||||
@@ -366,7 +364,7 @@ fn validate_update_policy(binding: &PluginSourceBinding) -> Result<(), AdminErro
|
||||
|| tag.chars().any(char::is_control)
|
||||
|| tag.trim() != tag)
|
||||
{
|
||||
return Err(AdminError::invalid("固定 Release tag 不合法"));
|
||||
return Err(AdminError::invalid("固定版本标签不合法"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -47,14 +47,14 @@ pub trait SettingsService: Send + Sync {
|
||||
&self,
|
||||
_provider: &str,
|
||||
) -> Result<gateway_core::account::OpaqueProviderData, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持客户端身份配置"))
|
||||
Err(AdminError::invalid("该服务商不支持客户端身份配置"))
|
||||
}
|
||||
async fn preview_client_profile(
|
||||
&self,
|
||||
_provider: &str,
|
||||
_configuration: Option<&gateway_core::account::OpaqueProviderData>,
|
||||
) -> Result<gateway_core::account::OpaqueProviderData, AdminError> {
|
||||
Err(AdminError::invalid("当前 Provider 不支持客户端身份配置"))
|
||||
Err(AdminError::invalid("该服务商不支持客户端身份配置"))
|
||||
}
|
||||
async fn load(&self) -> Result<RuntimeSettings, AdminError>;
|
||||
async fn replace(
|
||||
@@ -86,7 +86,7 @@ impl DefaultSettingsService {
|
||||
provider: &str,
|
||||
) -> Result<std::sync::Arc<dyn crate::ports::provider::ProviderAdmin>, AdminError> {
|
||||
let kind = gateway_core::routing::ProviderKind::new(provider)
|
||||
.map_err(|_| AdminError::invalid("Provider 不合法"))?;
|
||||
.map_err(|_| AdminError::invalid("服务商无效"))?;
|
||||
self.providers
|
||||
.require(&kind)
|
||||
.map_err(|error| super::map_provider_error(error, "client profile"))
|
||||
@@ -170,7 +170,7 @@ impl SettingsService for DefaultSettingsService {
|
||||
.keys()
|
||||
.any(|provider| !providers.contains_key(provider))
|
||||
{
|
||||
return Err(AdminError::invalid("Provider 不支持价目管理"));
|
||||
return Err(AdminError::invalid("该服务商不支持价目管理"));
|
||||
}
|
||||
let mut current = self.pricing_source.fetch().await?;
|
||||
if current != command.preview {
|
||||
@@ -242,7 +242,7 @@ impl SettingsService for DefaultSettingsService {
|
||||
|| model.chars().any(char::is_control)
|
||||
})
|
||||
{
|
||||
return Err(AdminError::invalid("Provider、模型 ID 或批量数量不合法"));
|
||||
return Err(AdminError::invalid("服务商、模型 ID 或批量数量不合法"));
|
||||
}
|
||||
if command.change == PricingChange::Delete
|
||||
&& command.models.iter().any(|model| {
|
||||
|
||||
@@ -146,12 +146,11 @@ fn map_system_error(error: SystemOperationError) -> AdminError {
|
||||
SystemOperationErrorKind::Upstream => AdminErrorKind::BadGateway,
|
||||
SystemOperationErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
let message = match kind {
|
||||
AdminErrorKind::Invalid => "系统操作请求不合法",
|
||||
AdminErrorKind::Conflict => error.message(),
|
||||
AdminErrorKind::BadGateway => "系统更新服务请求失败",
|
||||
AdminErrorKind::Internal => "系统操作失败",
|
||||
_ => "系统操作失败",
|
||||
let failure = AdminError::new(kind, error.message());
|
||||
let failure = if error.sensitive_context_redacted() {
|
||||
failure.redact_sensitive_context()
|
||||
} else {
|
||||
failure
|
||||
};
|
||||
AdminError::new(kind, message)
|
||||
failure.with_source(error)
|
||||
}
|
||||
|
||||
@@ -24,9 +24,9 @@ impl DefaultUsageDataService {
|
||||
reason: reason.into(),
|
||||
};
|
||||
let kind = ProviderKind::new(input.provider.clone())
|
||||
.map_err(|_| UsageDataError::invalid("Provider 无效"))?;
|
||||
.map_err(|_| UsageDataError::invalid("服务商无效"))?;
|
||||
let Ok(provider) = self.providers.require(&kind) else {
|
||||
return Ok(unavailable("Provider 没有可用的计价规则"));
|
||||
return Ok(unavailable("服务商没有可用的计价规则"));
|
||||
};
|
||||
let tokens = &input.tokens;
|
||||
let price = match &input.price {
|
||||
@@ -61,9 +61,9 @@ impl DefaultUsageDataService {
|
||||
ProviderAdminErrorKind::Unsupported | ProviderAdminErrorKind::Invalid
|
||||
) =>
|
||||
{
|
||||
return Ok(unavailable("Provider 不支持该计价输入"));
|
||||
return Ok(unavailable("服务商不支持该计价输入"));
|
||||
}
|
||||
Err(_) => return Err(UsageDataError::unavailable("Provider 计价失败")),
|
||||
Err(_) => return Err(UsageDataError::unavailable("服务商计价失败")),
|
||||
};
|
||||
Ok(UsagePricingResult::Estimated {
|
||||
id: input.id,
|
||||
|
||||
@@ -106,7 +106,7 @@ fn retention_count_keeps_newest_n() {
|
||||
Some(base),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(0, 2, now, &records);
|
||||
let decisions = decide_retention(0, 2, now, 0, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["oldest"]);
|
||||
assert_eq!(decisions[0].reason, RetentionReason::Count);
|
||||
@@ -131,7 +131,7 @@ fn retention_days_deletes_old_scheduled_but_keeps_newest() {
|
||||
Some(now - Duration::days(40)),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(30, 0, now, &records);
|
||||
let decisions = decide_retention(30, 0, now, 0, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["ancient"]);
|
||||
}
|
||||
@@ -146,7 +146,7 @@ fn newest_scheduled_never_auto_deleted_even_when_ancient() {
|
||||
now - Duration::days(400),
|
||||
Some(now - Duration::days(400)),
|
||||
)];
|
||||
assert!(decide_retention(30, 1, now, &records).is_empty());
|
||||
assert!(decide_retention(30, 1, now, 0, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -168,7 +168,7 @@ fn disabled_thresholds_delete_nothing() {
|
||||
Some(now - Duration::days(90)),
|
||||
),
|
||||
];
|
||||
assert!(decide_retention(0, 0, now, &records).is_empty());
|
||||
assert!(decide_retention(0, 0, now, 0, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -470,7 +470,8 @@ impl BackupRepository for FakeBackupRepository {
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let mut records = self
|
||||
.records
|
||||
@@ -488,7 +489,11 @@ impl BackupRepository for FakeBackupRepository {
|
||||
.cmp(&a.completed_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
Ok(records)
|
||||
Ok(records
|
||||
.into_iter()
|
||||
.skip(usize::try_from(offset).unwrap_or(usize::MAX))
|
||||
.take(limit as usize)
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -239,6 +239,75 @@ async fn retention_cleans_expired_scheduled_backups() {
|
||||
assert!(object_store.object(&new_seed.object_key).is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_scans_all_pages_and_deletes_only_one_batch_per_cycle() {
|
||||
for (retention_days, retention_count, retained) in [
|
||||
(0, 999, 999),
|
||||
(0, 1000, 1000),
|
||||
(0, 1001, 1001),
|
||||
(30, 0, 1000),
|
||||
] {
|
||||
let mut settings = configured_settings();
|
||||
settings.retention_days = retention_days;
|
||||
settings.retention_count = retention_count;
|
||||
let repository = Arc::new(FakeBackupRepository::new(settings));
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let now = Utc::now();
|
||||
let mut seeds = Vec::new();
|
||||
for index in 0..retained + 15 {
|
||||
let completed_at = if retention_days > 0 && index >= retained {
|
||||
now - Duration::days(40) - Duration::minutes(index)
|
||||
} else {
|
||||
now - Duration::minutes(index)
|
||||
};
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id(&format!("retention-{index}")),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(completed_at),
|
||||
object_key: format!("codex/retention-{index}.dump"),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert scheduled backup");
|
||||
repository.set_completed(&seed.id, completed_at);
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
seeds.push(seed);
|
||||
}
|
||||
let task = BackupTask::new(
|
||||
repository.clone(),
|
||||
Arc::new(FakeDumpPort::new()),
|
||||
object_store.clone(),
|
||||
);
|
||||
|
||||
for deleted in [10, 15] {
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run retention cycle");
|
||||
let records = repository.all_records();
|
||||
assert_eq!(records.len(), seeds.len() - deleted);
|
||||
for (index, seed) in seeds.iter().enumerate() {
|
||||
let retained = usize::try_from(retained).unwrap();
|
||||
let should_remain = index < retained || index >= retained + deleted;
|
||||
assert_eq!(
|
||||
records.iter().any(|record| record.id == seed.id),
|
||||
should_remain,
|
||||
"record {index}, retention days {retention_days}, count {retention_count}"
|
||||
);
|
||||
assert_eq!(
|
||||
object_store.object(&seed.object_key).is_some(),
|
||||
should_remain
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timezone_change_rebases_future_cursor_without_running_old_due_backup() {
|
||||
let mut settings = configured_settings();
|
||||
|
||||
@@ -3088,7 +3088,7 @@ async fn accounts_refresh_should_not_expose_the_provider_failure_message() {
|
||||
.expect_err("Provider preparation must fail");
|
||||
|
||||
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
|
||||
assert_eq!(error.to_string(), "Provider 资源状态冲突,请刷新后重试");
|
||||
assert_eq!(error.to_string(), "服务商资源状态冲突,请刷新后重试");
|
||||
assert!(!error.to_string().contains(upstream_message));
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
|
||||
@@ -805,10 +805,27 @@ impl ObservabilityStore for UnavailableStore {
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
_: Granularity,
|
||||
_include_concurrency: bool,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
Err(unavailable("usage insights"))
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
Err(unavailable("usage records"))
|
||||
}
|
||||
|
||||
@@ -1046,6 +1046,50 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_and_usage_insights_keep_concurrency_cache_entries_separate() {
|
||||
for dashboard_first in [true, false] {
|
||||
let now = Utc::now();
|
||||
let range = observation_range(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store
|
||||
.concurrency
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(UsageConcurrencyPoint {
|
||||
bucket_start: range.start,
|
||||
average_concurrency: 2.0,
|
||||
peak_concurrency: 3,
|
||||
});
|
||||
let services = observability_services(store.clone()).await;
|
||||
for dashboard in [
|
||||
dashboard_first,
|
||||
!dashboard_first,
|
||||
dashboard_first,
|
||||
!dashboard_first,
|
||||
] {
|
||||
if dashboard {
|
||||
services
|
||||
.observability()
|
||||
.dashboard_trend(range, TrendKind::Usage)
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
let result = services
|
||||
.observability()
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.performance.load.peak_concurrency, 3);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
*store.concurrency_selections.lock().unwrap(),
|
||||
vec![!dashboard_first, dashboard_first]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
yesterday: Mutex<RequestMetrics>,
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
@@ -1053,6 +1097,8 @@ struct FixtureObservabilityStore {
|
||||
projected_costs: Mutex<Vec<UsageProjectedCostPoint>>,
|
||||
concurrency: Mutex<Vec<UsageConcurrencyPoint>>,
|
||||
insights_fail: AtomicBool,
|
||||
concurrency_selections: Mutex<Vec<bool>>,
|
||||
metric_selections: Mutex<Vec<gateway_admin::model::observability::UsageInsightMetric>>,
|
||||
summary_fail: AtomicBool,
|
||||
diagnostics: Mutex<DiagnosticsObservation>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
@@ -1081,6 +1127,8 @@ impl FixtureObservabilityStore {
|
||||
projected_costs: Mutex::new(Vec::new()),
|
||||
concurrency: Mutex::new(Vec::new()),
|
||||
insights_fail: AtomicBool::new(false),
|
||||
concurrency_selections: Mutex::new(Vec::new()),
|
||||
metric_selections: Mutex::new(Vec::new()),
|
||||
summary_fail: AtomicBool::new(false),
|
||||
diagnostics: Mutex::new(DiagnosticsObservation::default()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
@@ -1197,8 +1245,13 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.usage_granularities.lock().unwrap().push(granularity);
|
||||
self.concurrency_selections
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(include_concurrency);
|
||||
if self.insights_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection read failed"));
|
||||
}
|
||||
@@ -1210,10 +1263,31 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
.lock()
|
||||
.expect("projected costs")
|
||||
.clone(),
|
||||
concurrency: self.concurrency.lock().expect("concurrency").clone(),
|
||||
concurrency: if include_concurrency {
|
||||
self.concurrency.lock().expect("concurrency").clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.metric_selections.lock().unwrap().push(metric);
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
||||
@@ -1741,3 +1815,48 @@ async fn insights_cache_should_reject_deleted_building_and_old_generation_times(
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_metric_cache_isolates_selections_and_filters() {
|
||||
use gateway_admin::model::observability::UsageInsightMetric as Metric;
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let metrics = [
|
||||
Metric::Health,
|
||||
Metric::Latency,
|
||||
Metric::FirstToken,
|
||||
Metric::Throughput,
|
||||
Metric::Load,
|
||||
Metric::Cost,
|
||||
Metric::Savings,
|
||||
Metric::Cache,
|
||||
];
|
||||
for _ in 0..2 {
|
||||
for metric in metrics {
|
||||
services
|
||||
.observability()
|
||||
.usage_metric(range, UsageFilter::default(), metric)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
assert_eq!(*store.metric_selections.lock().unwrap(), metrics);
|
||||
assert_eq!(
|
||||
*store.concurrency_selections.lock().unwrap(),
|
||||
vec![false, false, false, false, true, false, false, false]
|
||||
);
|
||||
services
|
||||
.observability()
|
||||
.usage_metric(
|
||||
range,
|
||||
UsageFilter {
|
||||
provider_kind: Some("openai".into()),
|
||||
..Default::default()
|
||||
},
|
||||
Metric::Latency,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(store.metric_selections.lock().unwrap().len(), 9);
|
||||
}
|
||||
|
||||
@@ -16,11 +16,15 @@ use gateway_admin::{
|
||||
struct RecordingSystemOperations {
|
||||
target: Mutex<Option<Option<String>>>,
|
||||
channel: Mutex<Option<gateway_admin::model::system::SystemUpdateChannel>>,
|
||||
failure: Option<SystemOperationError>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl SystemOperations for RecordingSystemOperations {
|
||||
async fn version(&self) -> Result<SystemVersion, SystemOperationError> {
|
||||
if let Some(error) = &self.failure {
|
||||
return Err(error.clone());
|
||||
}
|
||||
Ok(SystemVersion {
|
||||
version: "1.0.0".to_owned(),
|
||||
git_sha: "unknown".to_owned(),
|
||||
@@ -119,6 +123,33 @@ impl SystemOperations for RecordingSystemOperations {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn system_failure_should_preserve_safe_message_source_and_redaction_marker() {
|
||||
let operations = std::sync::Arc::new(RecordingSystemOperations {
|
||||
failure: Some(
|
||||
SystemOperationError::new(
|
||||
gateway_admin::ports::system::SystemOperationErrorKind::Upstream,
|
||||
"更新包下载失败",
|
||||
)
|
||||
.with_source(std::io::Error::other("native transport diagnostic"))
|
||||
.redact_sensitive_context(),
|
||||
),
|
||||
..RecordingSystemOperations::default()
|
||||
});
|
||||
let services = super::AdminHarness::new().system(operations).build().await;
|
||||
let error = services.system().version().await.expect_err("host failure");
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
gateway_admin::model::AdminErrorKind::BadGateway
|
||||
);
|
||||
assert_eq!(error.message(), "更新包下载失败");
|
||||
assert!(!format!("{error} {error:?}").contains("native transport diagnostic"));
|
||||
let details = error.error_details().expect("host error details");
|
||||
assert!(details.as_str().contains("native transport diagnostic"));
|
||||
let details: serde_json::Value = serde_json::from_str(details.as_str()).expect("details JSON");
|
||||
assert_eq!(details["redacted"], true);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn system_update_should_normalize_blank_target_to_latest() {
|
||||
let operations = std::sync::Arc::new(RecordingSystemOperations::default());
|
||||
|
||||
@@ -25,13 +25,16 @@ gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
headers = "0.4.1"
|
||||
hyper = { version = "=1.11.1", default-features = false }
|
||||
hyper-util = { version = "=0.1.21", default-features = false, features = ["tokio"] }
|
||||
regex = "1.13.1"
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt", "sync", "time"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["io-util", "macros", "rt", "sync", "time"] }
|
||||
tokio-tungstenite = { version = "0.29.0", default-features = false, features = ["handshake"] }
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
tower-http = { version = "0.7.0", features = ["compression-gzip", "cors", "fs", "request-id", "timeout", "trace"] }
|
||||
tracing = "0.1.44"
|
||||
|
||||
@@ -569,7 +569,8 @@ where
|
||||
.models(&account_id, false)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_models_data(result);
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let data = account_models_data(result, time);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -607,7 +608,8 @@ where
|
||||
.models(&account_id, true)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_models_data(result);
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let data = account_models_data(result, time);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,11 @@ pub(super) fn account_refresh_data(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
pub(super) fn account_models_data(
|
||||
result: ProviderModels,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> AccountModelsData {
|
||||
let observed_at = result.observed_at;
|
||||
AccountModelsData {
|
||||
models: result
|
||||
.models
|
||||
@@ -56,6 +60,9 @@ pub(super) fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
observed_at: observed_at.map(|value| time.rfc3339(&value)),
|
||||
observed_at_display: observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| time.datetime(&value)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -722,8 +722,11 @@ pub struct AccountModelView {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountModelsData {
|
||||
pub models: Vec<AccountModelView>,
|
||||
pub observed_at: Option<String>,
|
||||
pub observed_at_display: String,
|
||||
}
|
||||
|
||||
/// 账号模型目录文件;`catalog` 直接作为 Codex `model_catalog_json` 的文件正文落盘
|
||||
|
||||
@@ -1066,13 +1066,28 @@ fn mutation_response(
|
||||
}
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
match error.field() {
|
||||
"cursor" => AdminError::bad_request("Client API Key 游标不合法"),
|
||||
let message = match error.field() {
|
||||
"cursor" => "API Key 分页游标不合法",
|
||||
"clientKeyRevealNotFound" | "clientKeyMutationNotFound" => {
|
||||
AdminError::not_found("Client API Key 不存在")
|
||||
return AdminError::not_found("API Key 不存在");
|
||||
}
|
||||
_ => AdminError::bad_request("Client API Key 请求不合法"),
|
||||
}
|
||||
"id" => "API Key ID 不能为空",
|
||||
"name" => "API Key 名称不能为空",
|
||||
"label" => "API Key 标签不能为空",
|
||||
"customKey" => "自定义 API Key 仅支持英文字母、数字和英文符号",
|
||||
"groupIds" => "账号分组无效、重复或数量超限",
|
||||
"maxConcurrency" => "并发请求上限超出支持范围",
|
||||
"requestsPerMinute" => "每分钟请求上限超出支持范围",
|
||||
"dailyLimitUsd" => "每日美元限额须非负,整数和小数各最多 10 位",
|
||||
"weeklyLimitUsd" => "每周美元限额须非负,整数和小数各最多 10 位",
|
||||
"providerRequestProfileOverrides" => "API Key 客户端身份配置无效或字段冲突",
|
||||
"limit" => "每页数量须为 1 至 65535",
|
||||
"search" => "搜索内容过长或包含控制字符",
|
||||
"sortBy" => "排序字段不受支持",
|
||||
"sortDirection" => "排序方向必须为 asc(升序)或 desc(降序)",
|
||||
_ => "API Key 请求参数不合法",
|
||||
};
|
||||
AdminError::bad_request(message)
|
||||
}
|
||||
|
||||
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
|
||||
|
||||
@@ -1062,22 +1062,6 @@ pub(crate) fn usage_insights_view(
|
||||
insights: domain::UsageInsights,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageInsightsOverviewView {
|
||||
let health_points = insights
|
||||
.health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| OverviewHealthPointView {
|
||||
bucket: point.bucket_start,
|
||||
label: time.label(point.bucket_start, "%m-%d %H:%M"),
|
||||
total_requests: point.total_requests,
|
||||
success_requests: point.success_requests,
|
||||
failed_requests: point.failed_requests,
|
||||
cancelled_requests: point.cancelled_requests,
|
||||
incomplete_requests: point.incomplete_requests,
|
||||
caller_error_requests: point.caller_error_requests,
|
||||
error_rate: point.error_rate,
|
||||
})
|
||||
.collect();
|
||||
let performance_points = insights
|
||||
.performance
|
||||
.points
|
||||
@@ -1128,19 +1112,7 @@ pub(crate) fn usage_insights_view(
|
||||
UsageInsightsOverviewView {
|
||||
analytics: analytics_view(insights.analytics, time),
|
||||
granularity: granularity_name(insights.granularity).to_owned(),
|
||||
health: OverviewHealthView {
|
||||
total_requests: insights.health.total_requests,
|
||||
success_requests: insights.health.success_requests,
|
||||
failed_requests: insights.health.failed_requests,
|
||||
cancelled_requests: insights.health.cancelled_requests,
|
||||
incomplete_requests: insights.health.incomplete_requests,
|
||||
caller_error_requests: insights.health.caller_error_requests,
|
||||
success_rate: insights.health.success_rate,
|
||||
completion_rate: insights.health.completion_rate,
|
||||
request_change_rate: None,
|
||||
success_rate_change: None,
|
||||
points: health_points,
|
||||
},
|
||||
health: overview_health_view(&insights.health, time),
|
||||
performance: OverviewPerformanceView {
|
||||
latency_p50_ms: insights
|
||||
.performance
|
||||
@@ -1236,6 +1208,116 @@ pub(crate) fn usage_insights_view(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_metric_view(
|
||||
insight: domain::UsageInsight,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageMetricView {
|
||||
use domain::UsageInsightData;
|
||||
let data = match insight.data {
|
||||
UsageInsightData::Health(health) => {
|
||||
UsageMetricDataView::Health(overview_health_view(&health, time))
|
||||
}
|
||||
UsageInsightData::Latency(series) => UsageMetricDataView::Latency(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_latency_view,
|
||||
metric_latency_view,
|
||||
)),
|
||||
UsageInsightData::FirstToken(series) => UsageMetricDataView::FirstToken(
|
||||
metric_series_view(series, time, metric_latency_view, metric_latency_view),
|
||||
),
|
||||
UsageInsightData::Throughput(series) => UsageMetricDataView::Throughput(
|
||||
metric_series_view(series, time, metric_throughput_view, metric_throughput_view),
|
||||
),
|
||||
UsageInsightData::Load(series) => UsageMetricDataView::Load(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
|value| overview_load_view(&value),
|
||||
|value| overview_load_view(&value),
|
||||
)),
|
||||
UsageInsightData::Cost(series) => UsageMetricDataView::Cost(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| UsageCostComparisonView {
|
||||
estimated_cost: value.estimated_cost.as_ref().map(ToString::to_string),
|
||||
no_cache_cost: value.no_cache_cost.as_ref().map(ToString::to_string),
|
||||
},
|
||||
)),
|
||||
UsageInsightData::Savings(series) => UsageMetricDataView::Savings(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| value.as_ref().map(ToString::to_string),
|
||||
)),
|
||||
UsageInsightData::Cache(series) => UsageMetricDataView::Cache(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| UsageCacheView {
|
||||
input_tokens: value.input_tokens,
|
||||
cached_token_rate: value.cached_token_rate,
|
||||
cache_hit_request_rate: value.cache_hit_request_rate,
|
||||
},
|
||||
)),
|
||||
};
|
||||
UsageMetricView {
|
||||
analytics: analytics_view(insight.analytics, time),
|
||||
granularity: granularity_name(insight.granularity).to_owned(),
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_series_view<S, P, SV, PV>(
|
||||
series: domain::UsageMetricSeries<S, P>,
|
||||
time: crate::time::TimePresenter,
|
||||
summary: impl FnOnce(S) -> SV,
|
||||
point: impl Fn(P) -> PV,
|
||||
) -> UsageMetricSeriesView<SV, PV> {
|
||||
UsageMetricSeriesView {
|
||||
summary: summary(series.summary),
|
||||
points: series
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|value| UsageMetricPointView {
|
||||
bucket: value.bucket_start,
|
||||
label: time.label(value.bucket_start, "%m-%d %H:%M"),
|
||||
request_count: value.request_count,
|
||||
value: point(value.value),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_latency_view(value: domain::LatencyPercentiles) -> UsageLatencyView {
|
||||
UsageLatencyView {
|
||||
p50_ms: value.p50_ms.map(|value| value.as_f64()),
|
||||
p95_ms: value.p95_ms.map(|value| value.as_f64()),
|
||||
p99_ms: value.p99_ms.map(|value| value.as_f64()),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_throughput_view(value: domain::UsageThroughputPercentiles) -> UsageThroughputView {
|
||||
UsageThroughputView {
|
||||
p10: value.p10,
|
||||
p50: value.p50,
|
||||
p90: value.p90,
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_cost_summary_view(value: domain::UsageCostSummary) -> UsageCostSummaryView {
|
||||
UsageCostSummaryView {
|
||||
estimated_cost: value.estimated_cost.as_ref().map(ToString::to_string),
|
||||
cache_savings: value.cache_savings.as_ref().map(ToString::to_string),
|
||||
tier_premium: value.tier_premium.as_ref().map(ToString::to_string),
|
||||
cost_per_successful_request: value
|
||||
.cost_per_successful_request
|
||||
.as_ref()
|
||||
.map(ToString::to_string),
|
||||
coverage: cost_coverage_view(&value.coverage),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn granularity_name(granularity: domain::Granularity) -> &'static str {
|
||||
match granularity {
|
||||
domain::Granularity::FifteenMinutes => "15m",
|
||||
@@ -1322,3 +1404,37 @@ pub(crate) fn analytics_view(
|
||||
histogram_version: metadata.histogram_version,
|
||||
}
|
||||
}
|
||||
|
||||
fn overview_health_view(
|
||||
health: &domain::UsageInsightsHealth,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> OverviewHealthView {
|
||||
let health_points = health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| OverviewHealthPointView {
|
||||
bucket: point.bucket_start,
|
||||
label: time.label(point.bucket_start, "%m-%d %H:%M"),
|
||||
total_requests: point.total_requests,
|
||||
success_requests: point.success_requests,
|
||||
failed_requests: point.failed_requests,
|
||||
cancelled_requests: point.cancelled_requests,
|
||||
incomplete_requests: point.incomplete_requests,
|
||||
caller_error_requests: point.caller_error_requests,
|
||||
error_rate: point.error_rate,
|
||||
})
|
||||
.collect();
|
||||
OverviewHealthView {
|
||||
total_requests: health.total_requests,
|
||||
success_requests: health.success_requests,
|
||||
failed_requests: health.failed_requests,
|
||||
cancelled_requests: health.cancelled_requests,
|
||||
incomplete_requests: health.incomplete_requests,
|
||||
caller_error_requests: health.caller_error_requests,
|
||||
success_rate: health.success_rate,
|
||||
completion_rate: health.completion_rate,
|
||||
request_change_rate: None,
|
||||
success_rate_change: None,
|
||||
points: health_points,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,71 @@ pub struct UsageAnalyticsQuery {
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
/// 卡片内的指标选择与时间、Provider 范围
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageMetricQuery<T> {
|
||||
pub metric: T,
|
||||
pub provider: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl<T> UsageMetricQuery<T> {
|
||||
pub fn into_parts(self) -> (UsageAnalyticsQuery, T) {
|
||||
(
|
||||
UsageAnalyticsQuery {
|
||||
provider: self.provider,
|
||||
period: self.period,
|
||||
as_of: self.as_of,
|
||||
start_time: self.start_time,
|
||||
end_time: self.end_time,
|
||||
},
|
||||
self.metric,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum UsagePerformanceMetricKind {
|
||||
Latency,
|
||||
FirstToken,
|
||||
Throughput,
|
||||
Load,
|
||||
}
|
||||
|
||||
impl From<UsagePerformanceMetricKind> for domain::UsageInsightMetric {
|
||||
fn from(value: UsagePerformanceMetricKind) -> Self {
|
||||
match value {
|
||||
UsagePerformanceMetricKind::Latency => Self::Latency,
|
||||
UsagePerformanceMetricKind::FirstToken => Self::FirstToken,
|
||||
UsagePerformanceMetricKind::Throughput => Self::Throughput,
|
||||
UsagePerformanceMetricKind::Load => Self::Load,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum UsageCostMetricKind {
|
||||
Cost,
|
||||
Savings,
|
||||
Cache,
|
||||
}
|
||||
|
||||
impl From<UsageCostMetricKind> for domain::UsageInsightMetric {
|
||||
fn from(value: UsageCostMetricKind) -> Self {
|
||||
match value {
|
||||
UsageCostMetricKind::Cost => Self::Cost,
|
||||
UsageCostMetricKind::Savings => Self::Savings,
|
||||
UsageCostMetricKind::Cache => Self::Cache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageQuery {
|
||||
/// 校验 Element Plus 风格的页码分页字段
|
||||
pub fn validate_pagination(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
|
||||
@@ -24,6 +24,12 @@ where
|
||||
"/api/admin/usage/insights/overview",
|
||||
get(usage_insights_overview::<S>),
|
||||
)
|
||||
.route("/api/admin/usage/insights/health", get(usage_health::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/insights/performance",
|
||||
get(usage_performance::<S>),
|
||||
)
|
||||
.route("/api/admin/usage/insights/cost", get(usage_cost::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
get(usage_insights_diagnostics::<S>),
|
||||
@@ -207,6 +213,74 @@ where
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_health<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageAnalyticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
usage_metric_response(state, query, domain::UsageInsightMetric::Health).await
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_performance<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageMetricQuery<UsagePerformanceMetricKind>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let (range, metric) = query.into_parts();
|
||||
usage_metric_response(state, range, metric.into()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_cost<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageMetricQuery<UsageCostMetricKind>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let (range, metric) = query.into_parts();
|
||||
usage_metric_response(state, range, metric.into()).await
|
||||
}
|
||||
|
||||
async fn usage_metric_response<S>(
|
||||
state: S,
|
||||
query: UsageAnalyticsQuery,
|
||||
metric: domain::UsageInsightMetric,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let range = usage_range(
|
||||
query.start_time.as_deref(),
|
||||
query.end_time.as_deref(),
|
||||
query.period.as_deref(),
|
||||
query.as_of,
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
..Default::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_metric(range, filter, metric)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_metric_view(result, time)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_diagnostics<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
|
||||
@@ -759,6 +759,86 @@ pub struct UsageInsightsOverviewView {
|
||||
pub providers: Vec<ProviderOverviewView>,
|
||||
}
|
||||
|
||||
/// 单项洞察响应
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub granularity: String,
|
||||
#[serde(flatten)]
|
||||
pub data: UsageMetricDataView,
|
||||
}
|
||||
|
||||
/// 指标标签与响应字段一一对应,未查询的数据不会出现在响应中
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "metric", content = "data", rename_all = "camelCase")]
|
||||
pub enum UsageMetricDataView {
|
||||
Health(OverviewHealthView),
|
||||
Latency(UsageMetricSeriesView<UsageLatencyView>),
|
||||
FirstToken(UsageMetricSeriesView<UsageLatencyView>),
|
||||
Throughput(UsageMetricSeriesView<UsageThroughputView>),
|
||||
Load(UsageMetricSeriesView<OverviewLoadView>),
|
||||
Cost(UsageMetricSeriesView<UsageCostSummaryView, UsageCostComparisonView>),
|
||||
Savings(UsageMetricSeriesView<UsageCostSummaryView, Option<String>>),
|
||||
Cache(UsageMetricSeriesView<UsageCostSummaryView, UsageCacheView>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricSeriesView<S, P = S> {
|
||||
pub summary: S,
|
||||
pub points: Vec<UsageMetricPointView<P>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricPointView<P> {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub request_count: u64,
|
||||
pub value: P,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageLatencyView {
|
||||
pub p50_ms: Option<f64>,
|
||||
pub p95_ms: Option<f64>,
|
||||
pub p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct UsageThroughputView {
|
||||
pub p10: Option<u64>,
|
||||
pub p50: Option<u64>,
|
||||
pub p90: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCostSummaryView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub cache_savings: Option<String>,
|
||||
pub tier_premium: Option<String>,
|
||||
pub cost_per_successful_request: Option<String>,
|
||||
pub coverage: CostCoverageView,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCostComparisonView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub no_cache_cost: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCacheView {
|
||||
pub input_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 诊断聚合项目
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
||||
@@ -581,13 +581,13 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
|
||||
|
||||
fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
let message = match error.field() {
|
||||
"settingsRefreshConcurrencyOverflow" => "refreshConcurrency 不合法".to_owned(),
|
||||
"settingsMaxConcurrencyOverflow" => "maxConcurrentPerAccount 不合法".to_owned(),
|
||||
"settingsUsageRetentionOverflow" => "usageRetentionDays 不合法".to_owned(),
|
||||
"settingsOpsRetentionOverflow" => "opsEventRetentionDays 不合法".to_owned(),
|
||||
"settingsAuditRetentionOverflow" => "auditRetentionDays 不合法".to_owned(),
|
||||
"requestLocation" => "请求位置不合法,请检查国家代码、地区和城市".to_owned(),
|
||||
"providerRequestProfiles" => "Provider 请求画像格式不合法或字段冲突".to_owned(),
|
||||
"settingsRefreshConcurrencyOverflow" => "刷新并发数不合法".to_owned(),
|
||||
"settingsMaxConcurrencyOverflow" => "单账号并发上限不合法".to_owned(),
|
||||
"settingsUsageRetentionOverflow" => "用量记录保留天数不合法".to_owned(),
|
||||
"settingsOpsRetentionOverflow" => "运维事件保留天数不合法".to_owned(),
|
||||
"settingsAuditRetentionOverflow" => "审计记录保留天数不合法".to_owned(),
|
||||
"requestLocation" => "请求位置的国家代码、地区或城市不合法".to_owned(),
|
||||
"providerRequestProfiles" => "服务商客户端身份配置无效或字段冲突".to_owned(),
|
||||
"minCodexDesktopVersion" => "Codex Desktop 最低版本格式不合法".to_owned(),
|
||||
"minCodexCliVersion" => "Codex CLI 最低版本格式不合法".to_owned(),
|
||||
field => format!("{field} 字段不合法"),
|
||||
|
||||
@@ -33,7 +33,8 @@ pub fn openai_error_response(
|
||||
}
|
||||
}
|
||||
|
||||
/// Codex 将这两个容量码视为不可重试;仅在交付边界转换,内部仍保留真实上游事实
|
||||
/// 无重试提示的容量拒绝及旧版 Codex 限流错误需要可重试投影
|
||||
/// 仅在交付边界转换,内部仍保留真实上游事实
|
||||
pub(super) fn client_error_code(code: &str) -> &str {
|
||||
match code {
|
||||
"server_is_overloaded" | "slow_down" => "server_error",
|
||||
@@ -83,7 +84,7 @@ pub fn protocol_error_response(status: StatusCode, body: ProtocolErrorBody) -> R
|
||||
pub fn missing_client_api_key_response() -> (StatusCode, Json<Value>) {
|
||||
openai_error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"Missing or invalid API key",
|
||||
"Missing or invalid API key.",
|
||||
"invalid_request_error",
|
||||
"invalid_api_key",
|
||||
)
|
||||
@@ -93,7 +94,7 @@ pub fn missing_client_api_key_response() -> (StatusCode, Json<Value>) {
|
||||
pub fn runtime_unavailable_response() -> (StatusCode, Json<Value>) {
|
||||
openai_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Gateway runtime configuration is temporarily unavailable",
|
||||
"Gateway configuration is temporarily unavailable.",
|
||||
"server_error",
|
||||
"runtime_configuration_unavailable",
|
||||
)
|
||||
@@ -110,7 +111,7 @@ pub fn client_version_rejection_response(
|
||||
let (message, code, current_version) = match rejection.current() {
|
||||
Some(current) => (
|
||||
format!(
|
||||
"{client_label} {current} is below the minimum required version {}. Upgrade {client_label} and retry.",
|
||||
"{client_label} {current} is outdated. Upgrade to {} or later.",
|
||||
rejection.min()
|
||||
),
|
||||
"client_version_too_old",
|
||||
@@ -118,15 +119,15 @@ pub fn client_version_rejection_response(
|
||||
),
|
||||
None => (
|
||||
format!(
|
||||
"A valid {client_label} version is required. Upgrade to version {} or newer and retry.",
|
||||
rejection.min()
|
||||
"Cannot read the {client_label} version. Check the User-Agent and version headers."
|
||||
),
|
||||
"client_version_unavailable",
|
||||
None,
|
||||
),
|
||||
};
|
||||
// Codex 将 400 视为不可重试的请求错误,426 则会触发 WS 回退和 HTTP 重试
|
||||
(
|
||||
StatusCode::UPGRADE_REQUIRED,
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": message,
|
||||
@@ -144,7 +145,7 @@ pub fn client_version_rejection_response(
|
||||
pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
openai_error_response(
|
||||
StatusCode::NOT_FOUND,
|
||||
"The requested model does not exist or is not available",
|
||||
"The requested model does not exist or is not available.",
|
||||
"invalid_request_error",
|
||||
"model_not_found",
|
||||
)
|
||||
|
||||
@@ -124,7 +124,7 @@ async fn image_operation(
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI protocol identifier is invalid",
|
||||
"OpenAI protocol identifier is invalid.",
|
||||
)
|
||||
})?
|
||||
.with_context(context);
|
||||
|
||||
@@ -26,7 +26,7 @@ pub(super) async fn normalize_edit_body(
|
||||
return Ok(body);
|
||||
}
|
||||
if headers.get_all(CONTENT_TYPE).iter().count() != 1 {
|
||||
return Err(invalid("Image edit requires a single Content-Type header"));
|
||||
return Err(invalid("Image edit requires a single Content-Type header."));
|
||||
}
|
||||
|
||||
let mut request = Request::new(Body::from(body));
|
||||
@@ -35,7 +35,7 @@ pub(super) async fn normalize_edit_body(
|
||||
DefaultBodyLimit::disable().apply(&mut request);
|
||||
let mut multipart = Multipart::from_request(request, &())
|
||||
.await
|
||||
.map_err(|_| invalid("Image edit multipart boundary is invalid"))?;
|
||||
.map_err(|_| invalid("Image edit multipart boundary is invalid."))?;
|
||||
let mut body = Vec::new();
|
||||
let mut serializer = serde_json::Serializer::new(&mut body);
|
||||
// 直接序列化字段,避免经过 Map 后覆盖同名参数
|
||||
@@ -44,17 +44,17 @@ pub(super) async fn normalize_edit_body(
|
||||
while let Some(field) = multipart
|
||||
.next_field()
|
||||
.await
|
||||
.map_err(|_| invalid("Image edit multipart body is invalid"))?
|
||||
.map_err(|_| invalid("Image edit multipart body is invalid."))?
|
||||
{
|
||||
let name = field
|
||||
.name()
|
||||
.ok_or_else(|| invalid("Image edit multipart fields must have names"))?
|
||||
.ok_or_else(|| invalid("Image edit multipart fields must have names."))?
|
||||
.to_owned();
|
||||
let is_file = field.file_name().is_some();
|
||||
let bytes = field
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|_| invalid("Image edit multipart field is invalid"))?;
|
||||
.map_err(|_| invalid("Image edit multipart field is invalid."))?;
|
||||
if matches!(name.as_str(), "image" | "image[]") {
|
||||
// 按表单中的出现顺序追加,多图顺序参与编辑语义
|
||||
images.push(image_reference(&bytes)?);
|
||||
@@ -65,7 +65,7 @@ pub(super) async fn normalize_edit_body(
|
||||
} else {
|
||||
if is_file {
|
||||
return Err(invalid(
|
||||
"Only image, image[] and mask may contain image files",
|
||||
"Only image, image[] and mask may contain image files.",
|
||||
));
|
||||
}
|
||||
scalar_field(&name, &bytes)?
|
||||
@@ -76,7 +76,7 @@ pub(super) async fn normalize_edit_body(
|
||||
}
|
||||
if images.is_empty() {
|
||||
return Err(invalid(
|
||||
"Image edit multipart requires an image or image[] upload",
|
||||
"Image edit multipart requires an image or image[] upload.",
|
||||
));
|
||||
}
|
||||
fields
|
||||
@@ -92,14 +92,14 @@ fn scalar_field(name: &str, bytes: &[u8]) -> Result<Value, GatewayError> {
|
||||
match name {
|
||||
"n" | "output_compression" | "partial_images" => serde_json::from_slice::<Number>(bytes)
|
||||
.map(Value::Number)
|
||||
.map_err(|_| invalid("Image edit numeric fields must contain JSON numbers")),
|
||||
.map_err(|_| invalid("Image edit numeric fields must contain JSON numbers.")),
|
||||
"stream" => serde_json::from_slice::<bool>(bytes)
|
||||
.map(Value::Bool)
|
||||
.map_err(|_| invalid("Image edit stream field must contain a JSON boolean")),
|
||||
.map_err(|_| invalid("Image edit stream field must contain a JSON boolean.")),
|
||||
// 其余表单字段保留名称与文本值,不因字段尚未识别而丢弃或拒绝
|
||||
_ => {
|
||||
let text = std::str::from_utf8(bytes)
|
||||
.map_err(|_| invalid("Image edit text fields must be valid UTF-8"))?;
|
||||
.map_err(|_| invalid("Image edit text fields must be valid UTF-8."))?;
|
||||
Ok(Value::String(text.to_owned()))
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ fn image_reference(bytes: &[u8]) -> Result<Value, GatewayError> {
|
||||
} else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP") {
|
||||
"image/webp"
|
||||
} else {
|
||||
return Err(invalid("Image edit uploads must be PNG, JPEG or WebP"));
|
||||
return Err(invalid("Image edit uploads must be PNG, JPEG or WebP."));
|
||||
};
|
||||
Ok(json!({ "image_url": format!("data:{mime};base64,{}", STANDARD.encode(bytes)) }))
|
||||
}
|
||||
@@ -126,6 +126,6 @@ fn invalid(message: &'static str) -> GatewayError {
|
||||
fn encode_error(_: serde_json::Error) -> GatewayError {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"Could not encode image edit JSON",
|
||||
"Could not encode image edit JSON.",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ pub(crate) async fn live_call(
|
||||
return live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live call request has an invalid model",
|
||||
"Codex live call request has an invalid model.",
|
||||
"invalid_request",
|
||||
);
|
||||
}
|
||||
@@ -184,7 +184,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::PAYLOAD_TOO_LARGE,
|
||||
"Codex live request body too large",
|
||||
"Codex live request body too large.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
@@ -194,7 +194,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live multipart body is invalid",
|
||||
"Codex live multipart body is invalid.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
@@ -205,7 +205,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live multipart body is invalid",
|
||||
"Codex live multipart body is invalid.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?
|
||||
@@ -218,7 +218,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live multipart body is invalid",
|
||||
"Codex live multipart body is invalid.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
@@ -229,7 +229,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live multipart body is invalid",
|
||||
"Codex live multipart body is invalid.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
@@ -237,7 +237,7 @@ async fn read_call_body(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live session field must contain valid JSON",
|
||||
"Codex live session field must contain valid JSON.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
@@ -251,7 +251,7 @@ async fn read_call_body(
|
||||
None => Err(Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live multipart body requires an sdp field",
|
||||
"Codex live multipart body requires an sdp field.",
|
||||
"invalid_request",
|
||||
))),
|
||||
}
|
||||
|
||||
@@ -121,7 +121,7 @@ async fn sip_unsupported() -> Response {
|
||||
pub(crate) fn realtime_unsupported_response(capability: &str) -> Response {
|
||||
openai_error_response(
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
&format!("{capability} are not supported by the Codex OAuth upstream"),
|
||||
&format!("Unsupported by Codex OAuth: {capability}."),
|
||||
"not_supported_error",
|
||||
"realtime_capability_not_supported",
|
||||
)
|
||||
@@ -168,7 +168,7 @@ pub(crate) fn live_call_operation(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live call request is invalid",
|
||||
"Codex live call request is invalid.",
|
||||
"invalid_request",
|
||||
))
|
||||
};
|
||||
@@ -251,7 +251,7 @@ pub(crate) fn encode_call_request(
|
||||
))
|
||||
};
|
||||
if sdp.trim().is_empty() {
|
||||
return Err(invalid("Codex live call request requires an SDP offer"));
|
||||
return Err(invalid("Codex live call request requires an SDP offer."));
|
||||
}
|
||||
let mut payload = serde_json::Map::new();
|
||||
payload.insert("sdp".to_owned(), Value::String(sdp.to_owned()));
|
||||
@@ -260,7 +260,7 @@ pub(crate) fn encode_call_request(
|
||||
}
|
||||
serde_json::to_vec(&payload)
|
||||
.map(Bytes::from)
|
||||
.map_err(|_| invalid("Codex live call request is invalid"))
|
||||
.map_err(|_| invalid("Codex live call request is invalid."))
|
||||
}
|
||||
|
||||
/// 共用响应身份与传输边界;Live 原始正文另保留上游 Content-Type
|
||||
@@ -310,7 +310,7 @@ pub(crate) fn normalize_call_request(
|
||||
Box::new(live_error_response(
|
||||
LiveErrorShape::Live,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Codex live call request requires a UTF-8 SDP offer",
|
||||
"Codex live call request requires a UTF-8 SDP offer.",
|
||||
"invalid_request",
|
||||
))
|
||||
})?;
|
||||
|
||||
@@ -149,7 +149,7 @@ fn upgrade_required_response() -> Response {
|
||||
let mut response = live_error_response(
|
||||
LiveErrorShape::Realtime,
|
||||
StatusCode::UPGRADE_REQUIRED,
|
||||
"WebSocket upgrade required",
|
||||
"WebSocket upgrade required.",
|
||||
"websocket_upgrade_required",
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
|
||||
@@ -246,7 +246,7 @@ pub(crate) fn error_response(error: MiddlewareError) -> Response {
|
||||
MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => {
|
||||
openai_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"Request rejected by middleware",
|
||||
"Request rejected by middleware.",
|
||||
"invalid_request_error",
|
||||
"middleware_rejected",
|
||||
)
|
||||
@@ -255,7 +255,7 @@ pub(crate) fn error_response(error: MiddlewareError) -> Response {
|
||||
MiddlewareError::Fault | MiddlewareError::Remote { .. } | MiddlewareError::InvalidState => {
|
||||
openai_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"Request middleware failed",
|
||||
"Request middleware failed.",
|
||||
"server_error",
|
||||
"middleware_failed",
|
||||
)
|
||||
|
||||
@@ -72,7 +72,7 @@ async fn models_response(
|
||||
{
|
||||
return openai_error_response(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Invalid client_version",
|
||||
"Invalid client_version.",
|
||||
"invalid_request_error",
|
||||
"invalid_client_version",
|
||||
)
|
||||
@@ -134,7 +134,7 @@ async fn models_response(
|
||||
fn catalog_unavailable_response() -> Response {
|
||||
openai_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Model catalog is temporarily unavailable",
|
||||
"Model catalog is temporarily unavailable.",
|
||||
"server_error",
|
||||
"model_catalog_unavailable",
|
||||
)
|
||||
|
||||
@@ -984,7 +984,7 @@ impl MiddlewareBody for StreamingExecutionBody {
|
||||
MiddlewareError::Engine(error) => gateway_error_from_engine(&error),
|
||||
_ => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response stream could not be delivered",
|
||||
"Gateway response stream could not be delivered.",
|
||||
),
|
||||
};
|
||||
self.push_stream_error(error);
|
||||
@@ -1068,6 +1068,6 @@ fn done_frame() -> MiddlewareFrame {
|
||||
fn invalid_execution_stream() -> MiddlewareError {
|
||||
MiddlewareError::Gateway(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response stream violated its execution lifecycle",
|
||||
"Gateway response stream violated its execution lifecycle.",
|
||||
))
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ async fn stream_failure(response: Response, error_encoder: Option<fn(&Value) ->
|
||||
raw_message.unwrap_or_else(|| {
|
||||
status
|
||||
.canonical_reason()
|
||||
.unwrap_or("Response stream failed")
|
||||
.unwrap_or("Response stream failed.")
|
||||
}),
|
||||
)
|
||||
});
|
||||
|
||||
@@ -4,8 +4,9 @@ use super::{
|
||||
frame::*,
|
||||
pump::run_pump,
|
||||
state::{ConnectionCommand, ConnectionStats},
|
||||
transport::Socket,
|
||||
};
|
||||
use axum::extract::ws::{CloseFrame, Message, Utf8Bytes, WebSocket, close_code};
|
||||
use axum::extract::ws::{CloseFrame, Message, Utf8Bytes, close_code};
|
||||
use futures::{Sink, Stream};
|
||||
use gateway_core::{
|
||||
engine::middleware::{FrozenMiddlewarePlan, MiddlewareHeader},
|
||||
@@ -21,7 +22,7 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
sync::{mpsc, oneshot},
|
||||
sync::{mpsc, oneshot, watch},
|
||||
task::JoinHandle,
|
||||
time::{Instant, timeout},
|
||||
};
|
||||
@@ -38,19 +39,21 @@ pub struct ResponsesWebSocketConnection {
|
||||
exited: oneshot::Receiver<PumpExitReason>,
|
||||
pump_task: Option<JoinHandle<()>>,
|
||||
stats: Arc<ConnectionStats>,
|
||||
activity: watch::Sender<Instant>,
|
||||
config: ConnectionConfig,
|
||||
exit_reason: Option<PumpExitReason>,
|
||||
}
|
||||
|
||||
impl ResponsesWebSocketConnection {
|
||||
pub(in super::super) fn new(
|
||||
socket: WebSocket,
|
||||
socket: Socket,
|
||||
connection_id: String,
|
||||
cancellation: CancellationToken,
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
headers: Arc<[MiddlewareHeader]>,
|
||||
) -> Self {
|
||||
spawn_connection(
|
||||
socket.activity.clone(),
|
||||
socket,
|
||||
Arc::<str>::from(connection_id),
|
||||
cancellation,
|
||||
@@ -278,8 +281,7 @@ impl ResponsesWebSocketConnection {
|
||||
|
||||
pub(in super::super) fn log_summary(&self, request_count: u64) {
|
||||
let connection_age_ms = self.age().as_millis();
|
||||
let read_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_read_ms.load(Ordering::Relaxed)));
|
||||
let read_idle_ms = self.activity.borrow().elapsed().as_millis();
|
||||
let write_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_write_ms.load(Ordering::Relaxed)));
|
||||
tracing::info!(
|
||||
@@ -319,7 +321,9 @@ impl Drop for ResponsesWebSocketConnection {
|
||||
|
||||
/// 为 WebSocket transport 启动包含冻结中间件计划的单 owner pump
|
||||
/// cancellation 必须属于本连接,连接退出或句柄释放时会取消它
|
||||
/// activity 仅由 transport 的实际入站字节或 pump 的完整消息更新,不能由写入刷新
|
||||
pub fn spawn_connection<S, E>(
|
||||
activity: watch::Sender<Instant>,
|
||||
socket: S,
|
||||
connection_id: Arc<str>,
|
||||
cancellation: CancellationToken,
|
||||
@@ -342,6 +346,7 @@ where
|
||||
.unwrap_or_default();
|
||||
let pump_task = tokio::spawn(run_pump(
|
||||
socket,
|
||||
activity.clone(),
|
||||
command_rx,
|
||||
incoming_tx,
|
||||
exit_tx,
|
||||
@@ -366,6 +371,7 @@ where
|
||||
exited: exit_rx,
|
||||
pump_task: Some(pump_task),
|
||||
stats,
|
||||
activity,
|
||||
config,
|
||||
exit_reason: None,
|
||||
}
|
||||
|
||||
@@ -5,9 +5,12 @@ mod handle;
|
||||
mod middleware;
|
||||
mod pump;
|
||||
mod state;
|
||||
mod transport;
|
||||
|
||||
pub use frame::{
|
||||
ConnectionConfig, ConnectionEvent, ConnectionWriteError, FramePhase, PumpExitReason,
|
||||
WriteContext, WriteOutcome,
|
||||
};
|
||||
pub use handle::{ResponsesWebSocketConnection, spawn_connection};
|
||||
pub(super) use transport::Socket;
|
||||
pub(crate) use transport::Upgrade;
|
||||
|
||||
@@ -26,6 +26,7 @@ use tokio::{
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_pump<S, E>(
|
||||
socket: S,
|
||||
activity: watch::Sender<Instant>,
|
||||
commands: mpsc::Receiver<ConnectionCommand>,
|
||||
incoming: mpsc::Sender<PendingConnectionEvent>,
|
||||
exited: oneshot::Sender<PumpExitReason>,
|
||||
@@ -58,7 +59,7 @@ pub(super) async fn run_pump<S, E>(
|
||||
let (writer, mut reader) = socket.split();
|
||||
let writer = Arc::new(middleware::Writer::new(writer, context.clone()));
|
||||
let sender: Arc<dyn contract::Sender> = writer.clone();
|
||||
let (activity, received_activity) = watch::channel(());
|
||||
let received_activity = activity.subscribe();
|
||||
// 心跳独立于业务写入等待,但共享唯一 writer,避免长写入或排队阻止失联检测
|
||||
let reason = tokio::select! {
|
||||
biased;
|
||||
@@ -81,7 +82,7 @@ async fn read_connection<S, E>(
|
||||
socket: &mut S,
|
||||
incoming: &mpsc::Sender<PendingConnectionEvent>,
|
||||
sender: Arc<dyn contract::Sender>,
|
||||
activity: &watch::Sender<()>,
|
||||
activity: &watch::Sender<Instant>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
@@ -106,10 +107,9 @@ where
|
||||
}
|
||||
}
|
||||
message = socket.next() => {
|
||||
if matches!(&message, Some(Ok(_))) { context.stats.record_read(context.opened_at); }
|
||||
if matches!(&message, Some(Ok(message)) if !matches!(message, Message::Close(_))) {
|
||||
if matches!(&message, Some(Ok(_))) {
|
||||
// 链路活动以真实入站为准,插件丢弃消息不应抹掉对端已经响应的事实
|
||||
activity.send_replace(());
|
||||
activity.send_replace(Instant::now());
|
||||
}
|
||||
let message = match message {
|
||||
Some(Ok(message)) => match middleware::transform(message, contract::Direction::Incoming, sender.clone(), context).await {
|
||||
@@ -201,7 +201,7 @@ where
|
||||
|
||||
async fn heartbeat_connection<S, E>(
|
||||
writer: Arc<middleware::Writer<S>>,
|
||||
mut activity: watch::Receiver<()>,
|
||||
mut activity: watch::Receiver<Instant>,
|
||||
context: &PumpContext,
|
||||
) -> PumpExitReason
|
||||
where
|
||||
|
||||
@@ -30,18 +30,10 @@ pub(super) struct ConnectionStats {
|
||||
pub(super) ping_received_count: AtomicU64,
|
||||
pub(super) ping_written_count: AtomicU64,
|
||||
pub(super) pong_received_count: AtomicU64,
|
||||
pub(super) last_read_ms: AtomicU64,
|
||||
pub(super) last_write_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl ConnectionStats {
|
||||
pub(super) fn record_read(&self, opened_at: Instant) {
|
||||
self.last_read_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn record_write(&self, opened_at: Instant) {
|
||||
self.last_write_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
//! 保留 Axum 握手校验,在 WS 消息解码前记录下游真实接收进度
|
||||
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
FromRequestParts,
|
||||
ws::{self, Message},
|
||||
},
|
||||
http::{HeaderValue, StatusCode, Version, header, request::Parts},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::{Sink, Stream, ready};
|
||||
use hyper::upgrade::{OnUpgrade, Upgraded};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::{
|
||||
io::{AsyncRead, AsyncWrite, ReadBuf},
|
||||
sync::watch,
|
||||
time::Instant,
|
||||
};
|
||||
use tokio_tungstenite::{
|
||||
WebSocketStream,
|
||||
tungstenite::{
|
||||
self,
|
||||
protocol::{Role, WebSocketConfig},
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) struct Upgrade {
|
||||
pending: OnUpgrade,
|
||||
key: Option<HeaderValue>,
|
||||
}
|
||||
|
||||
impl<S: Send + Sync> FromRequestParts<S> for Upgrade {
|
||||
type Rejection = ws::rejection::WebSocketUpgradeRejection;
|
||||
|
||||
async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
|
||||
let pending = parts.extensions.get::<OnUpgrade>().cloned();
|
||||
// Axum 的 WebSocket 不暴露底层 I/O;只复用其握手校验,由本层接管唯一升级流
|
||||
let _validated = ws::WebSocketUpgrade::from_request_parts(parts, state).await?;
|
||||
let pending = pending.expect("Axum validated the presence of OnUpgrade");
|
||||
let key = (parts.version <= Version::HTTP_11)
|
||||
.then(|| parts.headers.get(header::SEC_WEBSOCKET_KEY).cloned())
|
||||
.flatten();
|
||||
Ok(Self { pending, key })
|
||||
}
|
||||
}
|
||||
|
||||
impl Upgrade {
|
||||
pub(in super::super) fn on_upgrade<F, Fut>(self, callback: F) -> Response
|
||||
where
|
||||
F: FnOnce(Socket) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
tokio::spawn(async move {
|
||||
let Ok(upgraded) = self.pending.await else {
|
||||
return;
|
||||
};
|
||||
let (activity, _) = watch::channel(Instant::now());
|
||||
let io = Receiving {
|
||||
inner: TokioIo::new(upgraded),
|
||||
activity: activity.clone(),
|
||||
};
|
||||
// 与原生 Responses 升级保持一致,不引入消息或帧长度上限
|
||||
let config = WebSocketConfig::default()
|
||||
.max_message_size(None)
|
||||
.max_frame_size(None);
|
||||
let inner = WebSocketStream::from_raw_socket(io, Role::Server, Some(config)).await;
|
||||
callback(Socket { inner, activity }).await;
|
||||
});
|
||||
if let Some(key) = self.key {
|
||||
(
|
||||
StatusCode::SWITCHING_PROTOCOLS,
|
||||
[
|
||||
(header::CONNECTION, "upgrade".to_owned()),
|
||||
(header::UPGRADE, "websocket".to_owned()),
|
||||
(
|
||||
header::SEC_WEBSOCKET_ACCEPT,
|
||||
tungstenite::handshake::derive_accept_key(key.as_bytes()),
|
||||
),
|
||||
],
|
||||
)
|
||||
.into_response()
|
||||
} else {
|
||||
StatusCode::OK.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Receiving {
|
||||
inner: TokioIo<Upgraded>,
|
||||
activity: watch::Sender<Instant>,
|
||||
}
|
||||
|
||||
impl AsyncRead for Receiving {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buffer: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let before = buffer.filled().len();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buffer);
|
||||
if buffer.filled().len() > before {
|
||||
// 完整消息尚未收齐时,客户端可能仍在发送、暂时无法读取 Ping
|
||||
self.activity.send_replace(Instant::now());
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for Receiving {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buffer: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(&mut self.inner).poll_write(cx, buffer)
|
||||
}
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
pub(in super::super) struct Socket {
|
||||
inner: WebSocketStream<Receiving>,
|
||||
pub(super) activity: watch::Sender<Instant>,
|
||||
}
|
||||
|
||||
impl Stream for Socket {
|
||||
type Item = Result<Message, axum::Error>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
loop {
|
||||
let message = match ready!(Pin::new(&mut self.inner).poll_next(cx)) {
|
||||
Some(Ok(message)) => message,
|
||||
Some(Err(error)) => return Poll::Ready(Some(Err(axum::Error::new(error)))),
|
||||
None => return Poll::Ready(None),
|
||||
};
|
||||
let message = match message {
|
||||
tungstenite::Message::Text(text) => Message::Text(
|
||||
ws::Utf8Bytes::try_from(bytes::Bytes::from(text)).map_err(axum::Error::new)?,
|
||||
),
|
||||
tungstenite::Message::Binary(data) => Message::Binary(data),
|
||||
tungstenite::Message::Ping(data) => Message::Ping(data),
|
||||
tungstenite::Message::Pong(data) => Message::Pong(data),
|
||||
tungstenite::Message::Close(frame) => {
|
||||
Message::Close(frame.map(|frame| ws::CloseFrame {
|
||||
code: frame.code.into(),
|
||||
reason: frame.reason.to_string().into(),
|
||||
}))
|
||||
}
|
||||
tungstenite::Message::Frame(_) => continue,
|
||||
};
|
||||
return Poll::Ready(Some(Ok(message)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<Message> for Socket {
|
||||
type Error = axum::Error;
|
||||
|
||||
fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Pin::new(&mut self.inner)
|
||||
.poll_ready(cx)
|
||||
.map_err(axum::Error::new)
|
||||
}
|
||||
fn start_send(mut self: Pin<&mut Self>, message: Message) -> Result<(), Self::Error> {
|
||||
let message = match message {
|
||||
Message::Text(text) => tungstenite::Message::Text(
|
||||
tungstenite::Utf8Bytes::try_from(bytes::Bytes::from(text))
|
||||
.map_err(axum::Error::new)?,
|
||||
),
|
||||
Message::Binary(data) => tungstenite::Message::Binary(data),
|
||||
Message::Ping(data) => tungstenite::Message::Ping(data),
|
||||
Message::Pong(data) => tungstenite::Message::Pong(data),
|
||||
Message::Close(frame) => {
|
||||
tungstenite::Message::Close(frame.map(|frame| tungstenite::protocol::CloseFrame {
|
||||
code: frame.code.into(),
|
||||
reason: frame.reason.to_string().into(),
|
||||
}))
|
||||
}
|
||||
};
|
||||
Pin::new(&mut self.inner)
|
||||
.start_send(message)
|
||||
.map_err(axum::Error::new)
|
||||
}
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Pin::new(&mut self.inner)
|
||||
.poll_flush(cx)
|
||||
.map_err(axum::Error::new)
|
||||
}
|
||||
fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Pin::new(&mut self.inner)
|
||||
.poll_close(cx)
|
||||
.map_err(axum::Error::new)
|
||||
}
|
||||
}
|
||||
@@ -400,13 +400,13 @@ fn middleware_gateway_error(error: MiddlewareError) -> GatewayError {
|
||||
MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"request middleware rejected the request",
|
||||
"Request middleware rejected the request.",
|
||||
)
|
||||
}
|
||||
MiddlewareError::Fault | MiddlewareError::Remote { .. } | MiddlewareError::InvalidState => {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"request middleware returned an invalid response",
|
||||
"Request middleware returned an invalid response.",
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -770,6 +770,21 @@ pub(super) async fn send_control(
|
||||
payload: &str,
|
||||
request_id: &Arc<str>,
|
||||
) -> ForwardOutcome {
|
||||
// HTTP 桥接没有原连接可中断;普通 error 会使 Codex 丢弃仍可正常完成的响应流
|
||||
// 仅在发送前确认无绑定时拒绝中断,不把发送结果不明的失败伪装成明确拒绝
|
||||
if control.binding().is_none()
|
||||
&& let Some(message) = super::interrupt::ResponseInterrupt::unavailable_event(payload)
|
||||
{
|
||||
return if connection
|
||||
.send_text(message, WriteContext::request(request_id, FramePhase::Data))
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
ForwardOutcome::Continue
|
||||
} else {
|
||||
ForwardOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
let result = tokio::select! {
|
||||
_ = connection.wait_for_exit() => return ForwardOutcome::Disconnect,
|
||||
result = control.send(payload) => result,
|
||||
@@ -779,7 +794,7 @@ pub(super) async fn send_control(
|
||||
connection,
|
||||
&GatewayError::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"No upstream WebSocket connection is available for control messages",
|
||||
"No upstream WebSocket connection is available for control messages.",
|
||||
),
|
||||
request_id,
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 当前逻辑响应的中断意图,跨重试连接保留并随新响应 ID 转发
|
||||
//! 当前逻辑响应的中断意图、无控制通道时的拒绝与跨重试转发
|
||||
|
||||
use std::{borrow::Cow, collections::BTreeSet};
|
||||
|
||||
@@ -53,14 +53,12 @@ impl ResponseInterrupt {
|
||||
|
||||
/// 未知控制消息、模式和其他逻辑响应的 ID 仍走原始透明通道
|
||||
pub(super) fn retain(&mut self, payload: &str) -> bool {
|
||||
let Ok(event) = serde_json::from_str::<Value>(payload) else {
|
||||
let Some(event) = interrupt_request(payload) else {
|
||||
return false;
|
||||
};
|
||||
if event["type"] != "response.interrupt"
|
||||
|| event["mode"] != "discard_partial_items"
|
||||
|| !event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| self.delivered_ids.contains(id))
|
||||
if !event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| self.delivered_ids.contains(id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -69,6 +67,18 @@ impl ResponseInterrupt {
|
||||
true
|
||||
}
|
||||
|
||||
/// 中断失败不代表模型响应失败,客户端仍可收完真实终态后续接
|
||||
pub(super) fn unavailable_event(payload: &str) -> Option<String> {
|
||||
let event = interrupt_request(payload)?;
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"type": "response.interrupt.failed",
|
||||
"response_id": event["response_id"],
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn forward_pending(&mut self, control: &ResponseControl) {
|
||||
if self.sent {
|
||||
return;
|
||||
@@ -89,3 +99,13 @@ impl ResponseInterrupt {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn interrupt_request(payload: &str) -> Option<Value> {
|
||||
let event: Value = serde_json::from_str(payload).ok()?;
|
||||
(event["type"] == "response.interrupt"
|
||||
&& event["mode"] == "discard_partial_items"
|
||||
&& event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| !id.is_empty()))
|
||||
.then_some(event)
|
||||
}
|
||||
|
||||
@@ -11,11 +11,7 @@ use std::{
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
Extension, State,
|
||||
connect_info::ConnectInfo,
|
||||
ws::{WebSocket, WebSocketUpgrade},
|
||||
},
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{HeaderMap, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
@@ -45,7 +41,9 @@ use crate::{
|
||||
use super::{
|
||||
request::OpenAiRequestHeaders, request_client_context, validation::ResponseValidationFacts,
|
||||
};
|
||||
use connection::{ConnectionEvent, FramePhase, ResponsesWebSocketConnection, WriteContext};
|
||||
use connection::{
|
||||
ConnectionEvent, FramePhase, ResponsesWebSocketConnection, Socket, Upgrade, WriteContext,
|
||||
};
|
||||
use forward::{
|
||||
ConnectionReplaySnapshot, ForwardOutcome, execution_response, forward_control_event,
|
||||
forward_response, new_replay_capture, send_control, send_gateway_error, send_middleware_error,
|
||||
@@ -54,7 +52,7 @@ use forward::{
|
||||
pub use protocol::{ResponseCreateFrameError, decode_response_create_with_context};
|
||||
use protocol::{connection_limit_event, is_response_create};
|
||||
|
||||
const TEXT_FRAMES_ONLY: &str = "Responses WebSocket accepts text frames only";
|
||||
const TEXT_FRAMES_ONLY: &str = "Responses WebSocket accepts text frames only.";
|
||||
const CONNECTION_LIMIT_CLOSE_REASON: &str = "Responses websocket connection limit reached";
|
||||
|
||||
/// 将已认证的 `GET /v1/responses` 升级为 Responses WebSocket
|
||||
@@ -64,7 +62,7 @@ pub(crate) async fn responses_websocket(
|
||||
middleware: Option<Extension<FrozenMiddlewarePlan>>,
|
||||
lifetime: Option<Extension<Arc<crate::middleware::CallLifetime>>>,
|
||||
headers: HeaderMap,
|
||||
websocket: WebSocketUpgrade,
|
||||
websocket: Upgrade,
|
||||
) -> Response {
|
||||
let service = state.openai().clone();
|
||||
let client = match authenticate_client(&service, &headers).await {
|
||||
@@ -104,7 +102,7 @@ impl ResponsesWebSocketAdapter {
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn upgrade_with_client_context(
|
||||
&self,
|
||||
websocket: WebSocketUpgrade,
|
||||
websocket: Upgrade,
|
||||
client: AuthenticatedClient,
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
@@ -138,15 +136,10 @@ impl ResponsesWebSocketAdapter {
|
||||
middleware,
|
||||
};
|
||||
let origin = crate::middleware::current();
|
||||
websocket
|
||||
// 覆盖 axum/tungstenite 的私有 64 MiB message 与 16 MiB frame 默认值
|
||||
// Responses JSON 的协议可接受性由上游决定,代理不另设 wire 长度上限
|
||||
.max_message_size(usize::MAX)
|
||||
.max_frame_size(usize::MAX)
|
||||
.on_upgrade(move |socket| async move {
|
||||
let _lifetime = lifetime;
|
||||
crate::middleware::scope(origin, serve_responses_websocket(socket, session)).await;
|
||||
})
|
||||
websocket.on_upgrade(move |socket| async move {
|
||||
let _lifetime = lifetime;
|
||||
crate::middleware::scope(origin, serve_responses_websocket(socket, session)).await;
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,7 +156,7 @@ struct ResponsesWebSocketSession {
|
||||
middleware: Option<FrozenMiddlewarePlan>,
|
||||
}
|
||||
|
||||
async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocketSession) {
|
||||
async fn serve_responses_websocket(socket: Socket, session: ResponsesWebSocketSession) {
|
||||
let ResponsesWebSocketSession {
|
||||
service,
|
||||
client,
|
||||
@@ -288,13 +281,13 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
let current = execution.request_settings().ok_or_else(|| {
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::Internal,
|
||||
"runtime snapshot is unavailable",
|
||||
"Gateway configuration is unavailable.",
|
||||
)
|
||||
})?;
|
||||
let rebased = settings.rebase(current.snapshot()).map_err(|_| {
|
||||
gateway_core::error::GatewayError::new(
|
||||
gateway_core::error::GatewayErrorKind::InvalidRequest,
|
||||
"request settings are invalid",
|
||||
"Request settings are invalid.",
|
||||
)
|
||||
})?;
|
||||
client = client.with_request_settings(rebased);
|
||||
|
||||
@@ -65,7 +65,7 @@ fn search_operation(body: Bytes, headers: &HeaderMap) -> Result<Operation, Gatew
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI protocol identifier is invalid",
|
||||
"OpenAI protocol identifier is invalid.",
|
||||
)
|
||||
})?
|
||||
.with_context(context);
|
||||
|
||||
@@ -101,7 +101,7 @@ impl OpenAiService {
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
"Requested model was not found.",
|
||||
)
|
||||
})?;
|
||||
let previous_response_id = match metadata.continuation() {
|
||||
@@ -142,7 +142,7 @@ impl OpenAiService {
|
||||
let provider = ProviderKind::new("openai").map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI provider identifier is invalid",
|
||||
"OpenAI provider identifier is invalid.",
|
||||
)
|
||||
})?;
|
||||
self.execution
|
||||
|
||||
@@ -63,7 +63,7 @@ async fn usage_response(
|
||||
if Query::<UsageQuery>::try_from_uri(&uri).is_err() {
|
||||
return openai_error_response(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"Usage query does not accept query parameters",
|
||||
"Usage query does not accept query parameters.",
|
||||
"invalid_request_error",
|
||||
"invalid_usage_query",
|
||||
)
|
||||
@@ -80,7 +80,7 @@ async fn usage_response(
|
||||
Err(_) => {
|
||||
return openai_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Key usage is temporarily unavailable",
|
||||
"Key usage is temporarily unavailable.",
|
||||
"server_error",
|
||||
"usage_unavailable",
|
||||
)
|
||||
|
||||
@@ -73,18 +73,18 @@ async fn count_tokens(
|
||||
};
|
||||
let provider = match ProviderKind::new(provider) {
|
||||
Ok(provider) => provider,
|
||||
Err(_) => return invalid_request("provider identifier is invalid"),
|
||||
Err(_) => return invalid_request("Provider identifier is invalid."),
|
||||
};
|
||||
let model = match UpstreamModelId::new(model) {
|
||||
Ok(model) => model,
|
||||
Err(_) => return invalid_request("provider model identifier is invalid"),
|
||||
Err(_) => return invalid_request("Provider model identifier is invalid."),
|
||||
};
|
||||
if serde_json::from_slice::<serde::de::IgnoredAny>(&body).is_err() {
|
||||
return invalid_request("token count request must be valid JSON");
|
||||
return invalid_request("Token count request must be valid JSON.");
|
||||
}
|
||||
let payload = match RawJsonPayload::new(TOKEN_COUNT_PROTOCOL, body) {
|
||||
Ok(payload) => payload,
|
||||
Err(_) => return invalid_request("token count request is invalid"),
|
||||
Err(_) => return invalid_request("Token count request is invalid."),
|
||||
};
|
||||
let (client_ip, user_agent) = request_client_context(
|
||||
&headers,
|
||||
@@ -124,12 +124,12 @@ async fn provider_http(
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
if method == Method::GET && !body.is_empty() {
|
||||
return invalid_request("GET provider HTTP operations cannot carry a request body");
|
||||
return invalid_request("GET provider HTTP operations cannot carry a request body.");
|
||||
}
|
||||
let method = match method {
|
||||
Method::GET => ProviderHttpMethod::Get,
|
||||
Method::POST => ProviderHttpMethod::Post,
|
||||
_ => return invalid_request("provider HTTP method is unsupported"),
|
||||
_ => return invalid_request("Provider HTTP method is unsupported."),
|
||||
};
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers).await {
|
||||
@@ -138,7 +138,7 @@ async fn provider_http(
|
||||
};
|
||||
let provider = match ProviderKind::new(provider) {
|
||||
Ok(provider) => provider,
|
||||
Err(_) => return invalid_request("provider identifier is invalid"),
|
||||
Err(_) => return invalid_request("Provider identifier is invalid."),
|
||||
};
|
||||
let connection_options = headers
|
||||
.get_all(header::CONNECTION)
|
||||
@@ -158,7 +158,7 @@ async fn provider_http(
|
||||
.collect();
|
||||
let payload = match RawHttpPayload::new(PROVIDER_HTTP_PROTOCOL, body) {
|
||||
Ok(payload) => payload,
|
||||
Err(_) => return invalid_request("provider HTTP request is invalid"),
|
||||
Err(_) => return invalid_request("Provider HTTP request is invalid."),
|
||||
};
|
||||
let operation = match ProviderHttpRequest::new(
|
||||
endpoint,
|
||||
@@ -168,7 +168,7 @@ async fn provider_http(
|
||||
payload,
|
||||
) {
|
||||
Ok(request) => Operation::ProviderHttp(request),
|
||||
Err(_) => return invalid_request("provider HTTP request is invalid"),
|
||||
Err(_) => return invalid_request("Provider HTTP request is invalid."),
|
||||
};
|
||||
let (client_ip, user_agent) = request_client_context(
|
||||
&headers,
|
||||
|
||||
@@ -9,6 +9,86 @@ use tower::ServiceExt as _;
|
||||
|
||||
use super::super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_model_reads_and_refreshes_preserve_provider_observation_time() {
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::provider_credentials::{ProviderModel, ProviderModels};
|
||||
use gateway_core::{
|
||||
account::{AccountStatusFacts, CredentialState, QuotaState},
|
||||
routing::UpstreamModelId,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let observed_at: DateTime<Utc> = "2026-09-12T08:00:00Z".parse().unwrap();
|
||||
*fixture.account.lock().unwrap() = Some(account_page_item(
|
||||
AccountStatusFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
observed_at,
|
||||
"api_key",
|
||||
));
|
||||
let router = admin::accounts::router::<AdminTestState>().with_state(fixture.state());
|
||||
for observed_at in [Some(observed_at), None] {
|
||||
*fixture.provider_models.lock().unwrap() = Some(ProviderModels {
|
||||
models: vec![ProviderModel {
|
||||
id: UpstreamModelId::new("model-a").unwrap(),
|
||||
name: "Model A".to_owned(),
|
||||
}],
|
||||
observed_at,
|
||||
});
|
||||
for (method, uri, body) in [
|
||||
(
|
||||
"GET",
|
||||
"/api/admin/accounts/models?accountId=acct_cooldown",
|
||||
"",
|
||||
),
|
||||
(
|
||||
"POST",
|
||||
"/api/admin/accounts/models/refresh",
|
||||
r#"{"accountId":"acct_cooldown"}"#,
|
||||
),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("x-request-id", "req_model_catalog")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK, "{method} {uri}");
|
||||
let body = to_bytes(response.into_body(), 8192).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(
|
||||
value["data"]["models"],
|
||||
json!([{ "id": "model-a", "label": "model-a" }])
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["observedAt"],
|
||||
json!(observed_at.map(|_| "2026-09-12T16:00:00+08:00"))
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["observedAtDisplay"],
|
||||
observed_at.map_or("—", |_| "2026-09-12 16:00:00")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_proxy_binding_is_distinct_from_the_redacted_endpoint() {
|
||||
use chrono::Utc;
|
||||
|
||||
@@ -69,7 +69,7 @@ mod personal_info {
|
||||
fn personal_info_response_preserves_unknown_subscription_and_profile_error() {
|
||||
let response = AccountPersonalInfoData::from((
|
||||
AccountPersonalInfo {
|
||||
profile: Err(AdminError::unavailable("Provider 服务暂不可用")),
|
||||
profile: Err(AdminError::unavailable("服务商暂不可用")),
|
||||
subscription: None,
|
||||
},
|
||||
gateway_api::TimePresenter::new(Default::default()),
|
||||
@@ -78,7 +78,7 @@ mod personal_info {
|
||||
serde_json::to_value(response).unwrap(),
|
||||
json!({
|
||||
"profile": null,
|
||||
"profileError": "Provider 服务暂不可用",
|
||||
"profileError": "服务商暂不可用",
|
||||
"subscription": null,
|
||||
})
|
||||
);
|
||||
|
||||
@@ -171,6 +171,70 @@ async fn malformed_admin_query_should_use_a_safe_bad_request_envelope() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_key_validation_should_identify_the_invalid_field_without_echoing_values() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let secret = "private-key-must-not-leak\n";
|
||||
for (field, value, message) in [
|
||||
("name", json!(" "), "API Key 名称不能为空"),
|
||||
(
|
||||
"customKey",
|
||||
json!(secret),
|
||||
"自定义 API Key 仅支持英文字母、数字和英文符号",
|
||||
),
|
||||
(
|
||||
"dailyLimitUsd",
|
||||
json!("-1"),
|
||||
"每日美元限额须非负,整数和小数各最多 10 位",
|
||||
),
|
||||
(
|
||||
"weeklyLimitUsd",
|
||||
json!("0.00000000001"),
|
||||
"每周美元限额须非负,整数和小数各最多 10 位",
|
||||
),
|
||||
(
|
||||
"maxConcurrency",
|
||||
json!(u64::MAX),
|
||||
"并发请求上限超出支持范围",
|
||||
),
|
||||
] {
|
||||
for path in [
|
||||
"/api/admin/client-keys/create",
|
||||
"/api/admin/client-keys/update",
|
||||
] {
|
||||
if path.ends_with("/update") && field == "customKey" {
|
||||
continue;
|
||||
}
|
||||
let mut payload = json!({
|
||||
"name": "key",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0,
|
||||
});
|
||||
if path.ends_with("/update") {
|
||||
payload["id"] = json!("key_existing");
|
||||
}
|
||||
payload[field] = value.clone();
|
||||
let mut request = request(Method::POST, path, Body::from(payload.to_string()));
|
||||
request
|
||||
.headers_mut()
|
||||
.insert(header::COOKIE, SESSION_COOKIE.parse().unwrap());
|
||||
request
|
||||
.headers_mut()
|
||||
.insert(header::CONTENT_TYPE, "application/json".parse().unwrap());
|
||||
let response = app(fixture.state()).oneshot(request).await.unwrap();
|
||||
let (status, _, body) = response_json(response).await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({"code": 40001, "message": message, "data": null})
|
||||
);
|
||||
assert!(!body.to_string().contains(secret.trim()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_time_range_should_use_its_published_business_code() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
@@ -445,8 +509,10 @@ mod provider {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (kind, status, code, message) in [
|
||||
(Kind::Invalid, 400, 40001, "服务商请求参数不合法"),
|
||||
(Kind::Unsupported, 400, 40001, "该服务商不支持此操作"),
|
||||
(Kind::BadGateway, 502, 50201, "上游服务请求失败"),
|
||||
(Kind::Unavailable, 503, 50301, "Provider 服务暂不可用"),
|
||||
(Kind::Unavailable, 503, 50301, "服务商暂不可用"),
|
||||
(
|
||||
Kind::Ambiguous,
|
||||
502,
|
||||
|
||||
@@ -113,6 +113,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
pub provider_error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
pub provider_quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
pub provider_models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
pub account: Arc<Mutex<Option<AccountPageItem>>>,
|
||||
pub account_usage: Arc<Mutex<Vec<AccountUsage>>>,
|
||||
}
|
||||
@@ -189,16 +190,19 @@ impl AdminTestFixture {
|
||||
plugin_ports.clone(),
|
||||
);
|
||||
let provider_quota = Arc::new(Mutex::new(None));
|
||||
let provider_models = Arc::new(Mutex::new(None));
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new(
|
||||
"openai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
Arc::clone(&provider_models),
|
||||
)),
|
||||
Arc::new(UnusedProvider::new(
|
||||
"xai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
Arc::clone(&provider_models),
|
||||
)),
|
||||
];
|
||||
let bundle = gateway_admin::initialize(
|
||||
@@ -247,6 +251,7 @@ impl AdminTestFixture {
|
||||
dashboard_summary_range,
|
||||
provider_error,
|
||||
provider_quota,
|
||||
provider_models,
|
||||
account,
|
||||
account_usage,
|
||||
}
|
||||
@@ -693,6 +698,7 @@ pub(super) struct MemoryObservations {
|
||||
pub trend: Option<Vec<RequestMetricPoint>>,
|
||||
pub concurrency: Vec<gateway_admin::model::observability::UsageConcurrencyPoint>,
|
||||
pub summaries: Vec<(TimeRange, UsageFilter)>,
|
||||
pub concurrency_selections: Vec<bool>,
|
||||
pub trends: Vec<(TimeRange, UsageFilter)>,
|
||||
pub records: Vec<UsageQuery>,
|
||||
pub errors: Vec<OpsErrorQuery>,
|
||||
@@ -1491,8 +1497,10 @@ impl ObservabilityStore for UnusedStore {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
_: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
let mut data = self.observations.lock().expect("observations");
|
||||
data.concurrency_selections.push(include_concurrency);
|
||||
data.summaries.push((range, filter.clone()));
|
||||
data.trends.push((range, filter));
|
||||
Ok(
|
||||
@@ -1507,11 +1515,31 @@ impl ObservabilityStore for UnusedStore {
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage trend"))?,
|
||||
costs: Vec::new(),
|
||||
concurrency: data.concurrency.clone(),
|
||||
concurrency: if include_concurrency {
|
||||
data.concurrency.clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
self.observations
|
||||
.lock()
|
||||
@@ -1584,6 +1612,7 @@ struct UnusedProvider {
|
||||
kind: ProviderKind,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
}
|
||||
|
||||
impl UnusedProvider {
|
||||
@@ -1591,11 +1620,13 @@ impl UnusedProvider {
|
||||
kind: &str,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
error,
|
||||
quota,
|
||||
models,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1706,7 +1737,11 @@ impl ProviderAdmin for UnusedProvider {
|
||||
_: &ProviderAccountId,
|
||||
_: bool,
|
||||
) -> Result<ProviderModels, ProviderAdminError> {
|
||||
Err(unsupported_provider())
|
||||
self.models
|
||||
.lock()
|
||||
.expect("provider models")
|
||||
.clone()
|
||||
.ok_or_else(unsupported_provider)
|
||||
}
|
||||
|
||||
async fn export_credentials(
|
||||
|
||||
@@ -1382,6 +1382,157 @@ async fn usage_insights_maps_load_summary_and_points_to_camel_case() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_card_routes_return_only_the_selected_metric_and_validate_options() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Duration;
|
||||
use gateway_admin::model::observability::{
|
||||
AttemptMetrics, CurrencyCost, Granularity, LatencyPercentiles, PercentileMilliseconds,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, UsageConcurrencyPoint, UsageOverview,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let range = TimeRange::new(now - Duration::minutes(15), now).unwrap();
|
||||
let metrics = RequestMetrics {
|
||||
request_count: 10,
|
||||
success_count: 8,
|
||||
failure_count: 2,
|
||||
latency_percentiles: LatencyPercentiles {
|
||||
p95_ms: Some(PercentileMilliseconds::new(43.5).unwrap()),
|
||||
..Default::default()
|
||||
},
|
||||
first_token_latency_percentiles: LatencyPercentiles {
|
||||
p95_ms: Some(PercentileMilliseconds::new(17.25).unwrap()),
|
||||
..Default::default()
|
||||
},
|
||||
output_throughput_p50: Some(22),
|
||||
..Default::default()
|
||||
};
|
||||
{
|
||||
let mut observations = fixture.observations.lock().unwrap();
|
||||
observations.summary = Some(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: metrics.clone(),
|
||||
attempts: AttemptMetrics {
|
||||
costs: vec![CurrencyCost {
|
||||
currency: "USD".into(),
|
||||
amount: "12.5".parse().unwrap(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
providers: Vec::new(),
|
||||
});
|
||||
observations.trend = Some(vec![RequestMetricPoint {
|
||||
bucket_start: range.start,
|
||||
granularity: Granularity::FifteenMinutes,
|
||||
metrics,
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
}]);
|
||||
observations.concurrency = vec![UsageConcurrencyPoint {
|
||||
bucket_start: range.start,
|
||||
average_concurrency: 1.25,
|
||||
peak_concurrency: 3,
|
||||
}];
|
||||
}
|
||||
let router = observability::router::<AdminTestState>().with_state(fixture.state());
|
||||
for (path, metric, field, expected) in [
|
||||
("health", "health", "totalRequests", json!(10)),
|
||||
(
|
||||
"performance?metric=latency",
|
||||
"latency",
|
||||
"p95Ms",
|
||||
json!(43.5),
|
||||
),
|
||||
(
|
||||
"performance?metric=firstToken",
|
||||
"firstToken",
|
||||
"p95Ms",
|
||||
json!(17.25),
|
||||
),
|
||||
(
|
||||
"performance?metric=throughput",
|
||||
"throughput",
|
||||
"p50",
|
||||
json!(22),
|
||||
),
|
||||
(
|
||||
"performance?metric=load",
|
||||
"load",
|
||||
"peakConcurrency",
|
||||
json!(3),
|
||||
),
|
||||
("cost?metric=cost", "cost", "estimatedCost", json!("12.5")),
|
||||
(
|
||||
"cost?metric=savings",
|
||||
"savings",
|
||||
"estimatedCost",
|
||||
json!("12.5"),
|
||||
),
|
||||
("cost?metric=cache", "cache", "estimatedCost", json!("12.5")),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/usage/insights/{path}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_card_metric")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{path}: {value}");
|
||||
let result = &value["data"];
|
||||
assert_eq!(result["metric"], metric);
|
||||
assert_eq!(result.as_object().unwrap().len(), 4);
|
||||
let data = &result["data"];
|
||||
if metric == "health" {
|
||||
assert_eq!(data[field], expected);
|
||||
} else {
|
||||
assert_eq!(data["summary"][field], expected, "{path}");
|
||||
assert_eq!(data["points"][0]["requestCount"], 10);
|
||||
assert_eq!(data.as_object().unwrap().len(), 2);
|
||||
}
|
||||
}
|
||||
for path in [
|
||||
"performance",
|
||||
"cost",
|
||||
"performance?metric=cost",
|
||||
"cost?metric=load",
|
||||
"health?metric=load",
|
||||
"performance?metric=unknown",
|
||||
"cost?metric=cache&status=error",
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/usage/insights/{path}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_card_metric")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observability_queries_reject_group_filters_and_analytics_search() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
@@ -218,6 +218,7 @@ async fn overview_scopes_every_query_and_projects_only_key_visible_fields() {
|
||||
assert!(!data.to_string().contains("private-sentinel"));
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
assert_eq!(observations.summaries.len(), 1);
|
||||
assert_eq!(observations.concurrency_selections, vec![false]);
|
||||
assert_eq!(observations.trends.len(), 2);
|
||||
for (_, filter) in observations
|
||||
.summaries
|
||||
|
||||
@@ -562,9 +562,12 @@ async fn http_settings_freeze_before_plan_resolution_and_apply_before_admission(
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::UPGRADE_REQUIRED,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"request changes must not alter the published baseline"
|
||||
);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap();
|
||||
assert_eq!(body["error"]["code"], "client_version_too_old");
|
||||
assert_eq!(
|
||||
snapshots
|
||||
.acquire()
|
||||
|
||||
@@ -357,7 +357,7 @@ fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
);
|
||||
assert_eq!(
|
||||
projected.safe_message(),
|
||||
"all eligible upstream accounts are temporarily busy"
|
||||
"All eligible upstream accounts are temporarily busy."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -389,7 +389,7 @@ fn no_eligible_provider_account_should_preserve_safe_provider_detail() {
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"no upstream provider is currently available for this request"
|
||||
"No upstream provider is currently available for this request."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -407,7 +407,7 @@ fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
|
||||
);
|
||||
assert_eq!(
|
||||
projected.safe_message(),
|
||||
"provider account infrastructure is temporarily unavailable"
|
||||
"Upstream accounts are temporarily inaccessible."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -422,7 +422,7 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::NoAvailableProvider);
|
||||
assert_eq!(
|
||||
projected.safe_message(),
|
||||
"no upstream provider is currently available for this request"
|
||||
"No upstream provider is currently available for this request."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -479,7 +479,7 @@ fn engine_provider_credential_failures_should_not_impersonate_client_auth_failur
|
||||
assert_eq!(mapped.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert_eq!(
|
||||
mapped.safe_message(),
|
||||
"upstream authentication resource is unavailable"
|
||||
"The upstream account cannot access the service."
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -550,7 +550,7 @@ fn engine_provider_runtime_failures_should_collapse_to_safe_unavailability() {
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::UpstreamUnavailable);
|
||||
assert_eq!(projected.safe_message(), "upstream service is unavailable");
|
||||
assert_eq!(projected.safe_message(), "Upstream service is unavailable.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -560,13 +560,13 @@ fn engine_store_error_should_collapse_to_safe_internal_error() {
|
||||
|
||||
let projected = gateway_error_from_engine(&error);
|
||||
assert_eq!(projected.kind(), GatewayErrorKind::Internal);
|
||||
assert_eq!(projected.safe_message(), "request execution failed");
|
||||
assert_eq!(projected.safe_message(), "Request execution failed.");
|
||||
}
|
||||
#[test]
|
||||
fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
let (status, body) = openai_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
"upstream service is unavailable",
|
||||
"Upstream service is unavailable.",
|
||||
"server_error",
|
||||
"upstream_unavailable",
|
||||
);
|
||||
@@ -577,7 +577,7 @@ fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
StatusCode::BAD_GATEWAY,
|
||||
serde_json::json!({
|
||||
"error": {
|
||||
"message": "upstream service is unavailable",
|
||||
"message": "Upstream service is unavailable.",
|
||||
"type": "server_error",
|
||||
"code": "upstream_unavailable"
|
||||
}
|
||||
@@ -767,12 +767,12 @@ mod model_routing {
|
||||
(
|
||||
"missing-model",
|
||||
BTreeMap::new(),
|
||||
"the requested model was not found in the provider catalogs available to this API key; check the model name",
|
||||
"Model unavailable for this API key. Check the model name.",
|
||||
),
|
||||
(
|
||||
"model-a",
|
||||
BTreeMap::from([("model-a".to_owned(), "missing-target".to_owned())]),
|
||||
"the requested model maps to an upstream model that was not found in the provider catalogs available to this API key; check the configured model mapping",
|
||||
"Mapped model unavailable for this API key. Check the model mapping.",
|
||||
),
|
||||
] {
|
||||
for stream in [false, true] {
|
||||
|
||||
@@ -446,7 +446,7 @@ async fn models_should_reject_recognized_cli_below_configured_min() {
|
||||
.await
|
||||
.expect("list models response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read version rejection body");
|
||||
@@ -470,7 +470,7 @@ async fn models_should_reject_recognized_cli_with_invalid_version() {
|
||||
.await
|
||||
.expect("list models response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read version rejection body");
|
||||
@@ -495,7 +495,7 @@ async fn models_should_reject_desktop_below_its_configured_min() {
|
||||
.await
|
||||
.expect("list models response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read version rejection body");
|
||||
|
||||
@@ -2406,7 +2406,7 @@ async fn atomic_uncommitted_upstream_failure_batch_should_be_forwarded_once() {
|
||||
assert_eq!(body.matches("event: response.failed").count(), 1);
|
||||
assert!(body.contains(std::str::from_utf8(&raw_created).expect("created frame")));
|
||||
assert!(body.contains(std::str::from_utf8(&raw_failure).expect("failure frame")));
|
||||
assert!(!body.contains("upstream capacity is temporarily unavailable"));
|
||||
assert!(!body.contains("Upstream capacity is temporarily unavailable."));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_error"]);
|
||||
}
|
||||
@@ -3000,7 +3000,7 @@ async fn official_core_clients_should_be_version_checked_before_starting_respons
|
||||
assert!(String::from_utf8_lossy(&body).contains("event: response.completed\n"));
|
||||
assert!(trace.snapshot().contains(&"commit"));
|
||||
} else {
|
||||
assert_eq!(status, StatusCode::UPGRADE_REQUIRED, "{product}/{version}");
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST, "{product}/{version}");
|
||||
let value: Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(value["error"]["code"], "client_version_too_old");
|
||||
assert_eq!(value["error"]["current_version"], version);
|
||||
@@ -3071,7 +3071,7 @@ async fn chatgpt_remote_responses_should_only_skip_missing_desktop_versions() {
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
if let Some(code) = expected_code {
|
||||
let value: Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::UPGRADE_REQUIRED, "{name} {version:?}");
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST, "{name} {version:?}");
|
||||
assert_eq!(value["error"]["code"], code);
|
||||
assert!(execution.session.lock().unwrap().is_some());
|
||||
} else {
|
||||
|
||||
@@ -66,6 +66,7 @@ async fn client_close_does_not_flush_a_retained_sender_with_an_inflight_write()
|
||||
dropped: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
let mut connection = spawn_connection(
|
||||
tokio::sync::watch::channel(tokio::time::Instant::now()).0,
|
||||
socket,
|
||||
Arc::from("retained-writer"),
|
||||
CancellationToken::new(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Responses WebSocket 连接生命周期与多轮请求边界测试
|
||||
|
||||
mod middleware;
|
||||
mod transport;
|
||||
|
||||
use std::{
|
||||
fmt,
|
||||
@@ -311,6 +312,7 @@ fn test_connection(stall_writes: bool) -> PumpHarness {
|
||||
dropped: Arc::clone(&dropped),
|
||||
};
|
||||
let connection = spawn_connection(
|
||||
tokio::sync::watch::channel(tokio::time::Instant::now()).0,
|
||||
socket,
|
||||
Arc::from("ws_test"),
|
||||
cancellation.clone(),
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
//! 实际 HTTP 升级及 WS 编解码下,慢速上传与半帧停传的心跳边界
|
||||
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
sync::{Arc, atomic::Ordering},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use futures::{SinkExt, StreamExt, ready};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{
|
||||
io::{AsyncRead, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf},
|
||||
time::{Instant, Sleep},
|
||||
};
|
||||
use tokio_tungstenite::{
|
||||
WebSocketStream, client_async,
|
||||
tungstenite::{Message, client::IntoClientRequest},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::{CrossingLimitExecution, CrossingLimitTrace, api_router, authenticated_client};
|
||||
|
||||
struct SlowUpload {
|
||||
inner: DuplexStream,
|
||||
slow: bool,
|
||||
next_write: Option<Pin<Box<Sleep>>>,
|
||||
}
|
||||
|
||||
impl AsyncRead for SlowUpload {
|
||||
fn poll_read(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buffer: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_read(cx, buffer)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for SlowUpload {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buffer: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
if self.slow {
|
||||
if let Some(delay) = self.next_write.as_mut() {
|
||||
ready!(delay.as_mut().poll(cx));
|
||||
}
|
||||
let size = ready!(
|
||||
Pin::new(&mut self.inner).poll_write(cx, &buffer[..buffer.len().min(4096)])
|
||||
)?;
|
||||
self.next_write = Some(Box::pin(tokio::time::sleep(Duration::from_secs(1))));
|
||||
Poll::Ready(Ok(size))
|
||||
} else {
|
||||
Pin::new(&mut self.inner).poll_write(cx, buffer)
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect() -> (WebSocketStream<SlowUpload>, Arc<CrossingLimitTrace>) {
|
||||
let trace = Arc::new(CrossingLimitTrace::default());
|
||||
let app = api_router(Arc::new(CrossingLimitExecution {
|
||||
client: authenticated_client("sk_ws_crossing_limit"),
|
||||
trace: trace.clone(),
|
||||
}))
|
||||
.await;
|
||||
let (client, server) = tokio::io::duplex(65536);
|
||||
tokio::spawn(async move {
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(
|
||||
TokioIo::new(server),
|
||||
hyper::service::service_fn(move |request| app.clone().oneshot(request)),
|
||||
)
|
||||
.with_upgrades()
|
||||
.await
|
||||
.expect("serve Responses upgrade");
|
||||
});
|
||||
let mut request = "ws://localhost/v1/responses".into_client_request().unwrap();
|
||||
request.headers_mut().insert(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer sk_ws_crossing_limit".parse().unwrap(),
|
||||
);
|
||||
let (socket, response) = client_async(
|
||||
request,
|
||||
SlowUpload {
|
||||
inner: client,
|
||||
slow: false,
|
||||
next_write: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), http::StatusCode::SWITCHING_PROTOCOLS);
|
||||
(socket, trace)
|
||||
}
|
||||
|
||||
async fn receive_event(socket: &mut WebSocketStream<SlowUpload>, expected: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
let message = socket
|
||||
.next()
|
||||
.await
|
||||
.expect("connection stays open")
|
||||
.expect("valid frame");
|
||||
if let Message::Text(text) = message {
|
||||
let event: Value = serde_json::from_str(&text).unwrap();
|
||||
if event["type"] == expected {
|
||||
break;
|
||||
}
|
||||
assert_eq!(event["type"], "response.metadata");
|
||||
} else {
|
||||
assert!(matches!(message, Message::Ping(_)));
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("receive response after upload");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slow_unfragmented_uploads_survive_heartbeats_on_first_and_later_requests() {
|
||||
let (mut socket, trace) = connect().await;
|
||||
let input = "x".repeat(320 * 1024);
|
||||
for round in 1..=2 {
|
||||
let started = Instant::now();
|
||||
socket.get_mut().slow = true;
|
||||
// 对齐 Codex 的发送语义:整个 send 完成前不读 Ping,也不主动发送 Pong
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(100),
|
||||
socket.send(Message::Text(
|
||||
json!({"type": "response.create", "model": "model-a", "input": input})
|
||||
.to_string()
|
||||
.into(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("upload completes")
|
||||
.expect("heartbeat must not interrupt an active upload");
|
||||
assert!(started.elapsed() >= Duration::from_secs(80));
|
||||
socket.get_mut().slow = false;
|
||||
receive_event(&mut socket, "response.created").await;
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), round);
|
||||
trace.release_terminal.notify_one();
|
||||
receive_event(&mut socket, "response.completed").await;
|
||||
}
|
||||
assert_eq!(
|
||||
*trace.inputs.lock().unwrap(),
|
||||
vec![json!(input), json!(input)]
|
||||
);
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stalled_partial_frame_without_pongs_still_releases_the_connection() {
|
||||
let (mut socket, trace) = connect().await;
|
||||
// 已掩码的文本帧声明 4096 字节,只交付一个字节,之后不再收发
|
||||
socket
|
||||
.get_mut()
|
||||
.write_all(&[0x81, 0xfe, 0x10, 0x00, 0, 0, 0, 0, b'x'])
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(56)).await;
|
||||
let error = socket
|
||||
.get_mut()
|
||||
.write_all(b"x")
|
||||
.await
|
||||
.expect_err("idle partial frame must be closed");
|
||||
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), 0);
|
||||
}
|
||||
@@ -266,7 +266,7 @@ async fn non_structured_bodies_use_safe_fallback_without_losing_status_or_ids()
|
||||
assert_eq!(
|
||||
error["error"],
|
||||
json!({
|
||||
"message": "upstream service is unavailable",
|
||||
"message": "Upstream service is unavailable.",
|
||||
"code": "upstream_unavailable",
|
||||
"type": "server_error",
|
||||
})
|
||||
@@ -365,7 +365,10 @@ async fn provider_failure_facts_survive_without_a_complete_http_response() {
|
||||
"observed-turn-state"
|
||||
);
|
||||
assert!(error["headers"].get("x-oai-request-id").is_none());
|
||||
assert_eq!(error["error"]["message"], "upstream service is unavailable");
|
||||
assert_eq!(
|
||||
error["error"]["message"],
|
||||
"Upstream service is unavailable."
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证未知控制消息、迟到中断、结算期间收发和创建请求的串行准入
|
||||
//! 验证控制消息透传、HTTP 中断拒绝、结算期间收发和创建请求的串行准入
|
||||
|
||||
use super::*;
|
||||
use gateway_core::engine::client_retry::{ClientRetryKey, ClientRetryStore};
|
||||
@@ -244,7 +244,9 @@ impl ResponseControlTransport for ControlTransport {
|
||||
#[derive(Default)]
|
||||
struct InterruptProvider {
|
||||
calls: AtomicUsize,
|
||||
store: Arc<SettlementPorts>,
|
||||
fail_first: bool,
|
||||
http_only: bool,
|
||||
retry_started: tokio::sync::Notify,
|
||||
retry_release: tokio::sync::Notify,
|
||||
transport: Mutex<Option<Arc<ControlTransport>>>,
|
||||
@@ -284,7 +286,9 @@ impl Provider for InterruptProvider {
|
||||
let id = format!("resp_interrupt_{number}");
|
||||
let transport = Arc::new(ControlTransport::new(id));
|
||||
let owner: Arc<dyn ResponseControlTransport> = transport.clone();
|
||||
context.response_control().unwrap().bind(&owner);
|
||||
if !self.http_only {
|
||||
context.response_control().unwrap().bind(&owner);
|
||||
}
|
||||
*self.transport.lock().unwrap() = Some(transport.clone());
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
@@ -296,7 +300,12 @@ impl Provider for InterruptProvider {
|
||||
"acct_api_test"
|
||||
})
|
||||
.unwrap(),
|
||||
UpstreamTransport::new("websocket").unwrap(),
|
||||
UpstreamTransport::new(if self.http_only {
|
||||
"http_sse"
|
||||
} else {
|
||||
"websocket"
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let body = futures::stream::unfold(
|
||||
(0, transport, self),
|
||||
@@ -457,7 +466,7 @@ async fn connect(
|
||||
} else {
|
||||
snapshot("sk_ws_interrupt", "openai")
|
||||
}),
|
||||
Arc::new(SettlementPorts::default()),
|
||||
provider.store.clone(),
|
||||
ProviderRegistry::new([provider as Arc<dyn Provider>]).unwrap(),
|
||||
admissions,
|
||||
Arc::new(UnusedContinuation),
|
||||
@@ -489,6 +498,140 @@ fn interrupt(id: &str, mode: &str) -> Value {
|
||||
json!({"type":"response.interrupt","response_id":id,"mode":mode,"future_extension":true})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_response_rejects_interrupt_without_aborting_or_admitting_queued_requests() {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
http_only: true,
|
||||
..Default::default()
|
||||
});
|
||||
let admissions = Arc::new(InterruptAdmissions::default());
|
||||
let (mut socket, _server) = connect(provider.clone(), admissions.clone()).await;
|
||||
let create = json!({"type":"response.create","model":"model-a","input":"hello"});
|
||||
send(&mut socket, create.clone()).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "response.created");
|
||||
send(&mut socket, create).await;
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_interrupt_0", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await,
|
||||
json!({
|
||||
"type":"response.interrupt.failed", "response_id":"resp_interrupt_0"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(admissions.active.load(Ordering::SeqCst), 1);
|
||||
assert!(
|
||||
provider
|
||||
.transport
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.requests
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
provider.complete.notify_one();
|
||||
let completed = next(&mut socket).await;
|
||||
assert_eq!(completed["type"], "response.completed");
|
||||
assert_eq!(completed["response"]["id"], "resp_interrupt_0");
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_1"
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
provider.store.finalizations.lock().unwrap()[0].outcome,
|
||||
gateway_core::engine::ExecutionOutcome::Succeeded
|
||||
);
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_response_rejects_late_interrupt_during_settlement_and_when_idle() {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
http_only: true,
|
||||
..Default::default()
|
||||
});
|
||||
let (release, wait_for_release) = tokio::sync::oneshot::channel();
|
||||
let admissions = Arc::new(InterruptAdmissions {
|
||||
release_gate: Mutex::new(Some(wait_for_release)),
|
||||
..Default::default()
|
||||
});
|
||||
let (mut socket, _server) = connect(provider.clone(), admissions.clone()).await;
|
||||
let create = json!({"type":"response.create","model":"model-a","input":"hello"});
|
||||
send(&mut socket, create.clone()).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "response.created");
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
admissions.release_started.notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let control = interrupt("resp_interrupt_0", "discard_partial_items");
|
||||
let rejection = json!({"type":"response.interrupt.failed","response_id":"resp_interrupt_0"});
|
||||
send(&mut socket, control.clone()).await;
|
||||
assert_eq!(next(&mut socket).await, rejection);
|
||||
assert_eq!(admissions.active.load(Ordering::SeqCst), 1);
|
||||
release.send(()).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while admissions.active.load(Ordering::SeqCst) != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
send(&mut socket, control).await;
|
||||
assert_eq!(next(&mut socket).await, rejection);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
|
||||
send(&mut socket, create).await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_1"
|
||||
);
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_control_channel_keeps_unknown_and_invalid_controls_as_errors() {
|
||||
let provider = Arc::new(InterruptProvider::default());
|
||||
let (mut socket, _server) =
|
||||
connect(provider.clone(), Arc::new(InterruptAdmissions::default())).await;
|
||||
for control in [
|
||||
interrupt("resp_unknown", "future_mode"),
|
||||
interrupt("", "discard_partial_items"),
|
||||
json!({"type":"response.interrupt", "mode":"discard_partial_items"}),
|
||||
json!({"type":"future.control"}),
|
||||
] {
|
||||
send(&mut socket, control).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "error");
|
||||
}
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_unknown", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await,
|
||||
json!({
|
||||
"type":"response.interrupt.failed", "response_id":"resp_unknown"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 0);
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_interrupt_after_terminal_is_sent_upstream_and_keeps_the_connection_reusable() {
|
||||
let provider = Arc::new(InterruptProvider::default());
|
||||
|
||||
@@ -1241,7 +1241,7 @@ async fn official_core_version_gate_should_run_before_websocket_upgrade() {
|
||||
else {
|
||||
panic!("expected an HTTP version rejection");
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::UPGRADE_REQUIRED);
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let value: Value = serde_json::from_slice(response.body().as_deref().unwrap()).unwrap();
|
||||
assert_eq!(value["error"]["code"], "client_version_too_old");
|
||||
assert_eq!(value["error"]["current_version"], "0.158.0");
|
||||
|
||||
@@ -385,11 +385,11 @@ impl QueueRejection {
|
||||
match self {
|
||||
Self::Full => crate::error::GatewayError::new(
|
||||
crate::error::GatewayErrorKind::ConcurrencyQueueFull,
|
||||
"concurrency wait queue is full",
|
||||
"Request queue is full.",
|
||||
),
|
||||
Self::Timeout => crate::error::GatewayError::new(
|
||||
crate::error::GatewayErrorKind::ConcurrencyQueueTimeout,
|
||||
"concurrency wait deadline elapsed",
|
||||
"Request queue wait timed out.",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1059,7 +1059,7 @@ where
|
||||
let Some(candidate) = self.plan.candidates().get(self.candidate_index).cloned() else {
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream Provider is available",
|
||||
"No upstream service is available.",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
@@ -1365,7 +1365,7 @@ where
|
||||
self.record_provider_failure(candidate.provider().clone());
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"provider metadata did not match the frozen candidate",
|
||||
"Provider metadata did not match the frozen candidate.",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
@@ -1399,7 +1399,7 @@ where
|
||||
drop(stream);
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"provider selected an account outside the frozen client scope",
|
||||
"Provider selected an account outside the frozen client scope.",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
@@ -1422,7 +1422,7 @@ where
|
||||
drop(stream);
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"provider did not use the required account",
|
||||
"Provider did not use the required account.",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
@@ -1448,7 +1448,7 @@ where
|
||||
drop(stream);
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"native continuation binding did not match selected account",
|
||||
"Native continuation binding did not match selected account.",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
@@ -2401,11 +2401,11 @@ where
|
||||
} else {
|
||||
ExecutionOutcome::Failed
|
||||
},
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed"),
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "Request deadline elapsed."),
|
||||
),
|
||||
_ => (
|
||||
ExecutionOutcome::Failed,
|
||||
GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
GatewayError::new(GatewayErrorKind::Internal, "Request execution failed."),
|
||||
),
|
||||
};
|
||||
let send_state = if self.attempts == 0 {
|
||||
|
||||
@@ -196,7 +196,7 @@ impl PreparedRootExecution {
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"execution settings are invalid",
|
||||
"Execution settings are invalid.",
|
||||
)
|
||||
})?;
|
||||
self.client.policy = settings.apply_policy(self.client.policy.clone());
|
||||
@@ -224,7 +224,7 @@ impl PreparedRootExecution {
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = Deadline::from_timeout(started_at, client.execution_timeout())
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "system clock is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Internal, "System clock is invalid.")
|
||||
})?;
|
||||
let execution_effects = Arc::new(ExecutionEffects::default());
|
||||
let execution_effects_baseline = execution_effects.epoch();
|
||||
@@ -411,7 +411,7 @@ pub trait ExecutionService: Send + Sync {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Unauthorized,
|
||||
"plugin model entry is unavailable",
|
||||
"Plugin model entry is unavailable.",
|
||||
))
|
||||
})
|
||||
}
|
||||
@@ -505,6 +505,7 @@ pub(super) fn duration_ms(duration: Duration) -> u64 {
|
||||
}
|
||||
|
||||
pub(super) fn new_request_id() -> Result<ModelRequestId, GatewayError> {
|
||||
ModelRequestId::new(format!("req_{}", Uuid::now_v7().simple()))
|
||||
.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "failed to allocate request ID"))
|
||||
ModelRequestId::new(format!("req_{}", Uuid::now_v7().simple())).map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "Failed to allocate request ID.")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,15 +9,15 @@ use crate::{
|
||||
pub(super) fn authentication_gateway_error(error: ClientAuthenticationError) -> GatewayError {
|
||||
match error {
|
||||
ClientAuthenticationError::InvalidKey => {
|
||||
GatewayError::new(GatewayErrorKind::Unauthorized, "client API key is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Unauthorized, "Client API key is invalid.")
|
||||
}
|
||||
ClientAuthenticationError::SnapshotUnavailable => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"runtime snapshot is unavailable",
|
||||
"Gateway configuration is unavailable.",
|
||||
),
|
||||
ClientAuthenticationError::ProviderUnavailable => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"frontend authentication provider is unavailable",
|
||||
"Client authentication service is unavailable.",
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -30,24 +30,24 @@ pub(super) fn map_routing_error(error: crate::validation::RoutingError) -> Gatew
|
||||
} => GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
if model == mapped_model {
|
||||
"the requested model was not found in the provider catalogs available to this API key; check the model name"
|
||||
"Model unavailable for this API key. Check the model name."
|
||||
} else {
|
||||
"the requested model maps to an upstream model that was not found in the provider catalogs available to this API key; check the configured model mapping"
|
||||
"Mapped model unavailable for this API key. Check the model mapping."
|
||||
},
|
||||
),
|
||||
crate::validation::RoutingError::NoCapableProvider { .. }
|
||||
| crate::validation::RoutingError::NoCapableProviderEndpoint { .. }
|
||||
| crate::validation::RoutingError::EmptyAccountScope => GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no provider can execute this request",
|
||||
"No provider can execute this request.",
|
||||
),
|
||||
crate::validation::RoutingError::UnsupportedProviderEndpoint { .. } => GatewayError::new(
|
||||
GatewayErrorKind::Unsupported,
|
||||
"the selected provider does not support this operation",
|
||||
"The selected provider does not support this operation.",
|
||||
),
|
||||
_ => GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"runtime routing configuration is invalid",
|
||||
"Runtime routing configuration is invalid.",
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -55,23 +55,23 @@ pub(super) fn map_routing_error(error: crate::validation::RoutingError) -> Gatew
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
match error {
|
||||
EngineError::Cancelled => {
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "Request was cancelled.")
|
||||
}
|
||||
EngineError::Deadline => {
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")
|
||||
GatewayError::new(GatewayErrorKind::Timeout, "Request deadline elapsed.")
|
||||
}
|
||||
EngineError::Provider(provider) => GatewayError::from_provider(provider),
|
||||
EngineError::EmptyRoutingPlan | EngineError::ProviderNotRegistered { .. } => {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no provider is available",
|
||||
"No provider is available.",
|
||||
)
|
||||
}
|
||||
EngineError::Store(source) => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "request execution failed")
|
||||
GatewayError::new(GatewayErrorKind::Internal, "Request execution failed.")
|
||||
.with_source(source.clone())
|
||||
}
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "request execution failed"),
|
||||
_ => GatewayError::new(GatewayErrorKind::Internal, "Request execution failed."),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ impl NestedExecutionGraph {
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ConcurrencyQueueFull,
|
||||
"nested model execution concurrency is exhausted",
|
||||
"Nested model execution concurrency is exhausted.",
|
||||
));
|
||||
}
|
||||
if self
|
||||
@@ -135,7 +135,7 @@ impl NestedExecutionGraph {
|
||||
self.active.fetch_sub(1, Ordering::AcqRel);
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"nested model execution budget is exhausted",
|
||||
"Nested model execution budget is exhausted.",
|
||||
));
|
||||
}
|
||||
Ok(NestedExecutionPermit {
|
||||
|
||||
@@ -333,7 +333,7 @@ impl DefaultExecutionService {
|
||||
let authentication = client.authentication.clone().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"bound execution identity cannot enter the public start path",
|
||||
"Bound execution identity cannot enter the public start path.",
|
||||
)
|
||||
})?;
|
||||
let authentication = match client.settings.clone() {
|
||||
@@ -418,13 +418,13 @@ impl DefaultExecutionService {
|
||||
if authorization.cancellation.is_cancelled() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Cancelled,
|
||||
"parent request was cancelled",
|
||||
"Parent request was cancelled.",
|
||||
));
|
||||
}
|
||||
if authorization.deadline_at.is_elapsed() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Timeout,
|
||||
"request deadline elapsed",
|
||||
"Request deadline elapsed.",
|
||||
));
|
||||
}
|
||||
let account_group_ids = authorization
|
||||
@@ -501,7 +501,7 @@ impl DefaultExecutionService {
|
||||
if !authorization.account_scope.provider_kinds().contains(&provider) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"nested provider is outside the frozen account scope",
|
||||
"Nested provider is outside the frozen account scope.",
|
||||
));
|
||||
}
|
||||
routing_context.required_provider = Some(provider);
|
||||
@@ -532,21 +532,21 @@ impl DefaultExecutionService {
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"model routing policy failed",
|
||||
"Model routing policy failed.",
|
||||
)
|
||||
})? {
|
||||
ModelRouteDecision::Unhandled => {}
|
||||
ModelRouteDecision::Reject => {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model routing policy rejected the request",
|
||||
"Model routing policy rejected the request.",
|
||||
));
|
||||
}
|
||||
ModelRouteDecision::Route { provider, model } => {
|
||||
if provider.is_none() && model.is_none() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"model routing policy returned an empty target",
|
||||
"Model routing policy returned an empty target.",
|
||||
));
|
||||
}
|
||||
if let Some(provider) = provider {
|
||||
@@ -561,7 +561,7 @@ impl DefaultExecutionService {
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model routing policy exceeded the frozen provider scope",
|
||||
"Model routing policy exceeded the frozen provider scope.",
|
||||
));
|
||||
}
|
||||
target_context.required_provider = Some(provider);
|
||||
@@ -601,26 +601,26 @@ impl DefaultExecutionService {
|
||||
Err(error)
|
||||
if error.kind() == NativeContinuationStoreErrorKind::OwnershipMismatch =>
|
||||
{
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "continuation does not belong to this client API key"));
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "Continuation does not belong to this client API key."));
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(request_id = request_id.as_str(), %error, "Continuation affinity 记录无效,已拒绝续接");
|
||||
return Err(GatewayError::new(GatewayErrorKind::Internal, "continuation state is invalid"));
|
||||
return Err(GatewayError::new(GatewayErrorKind::Internal, "Continuation state is invalid."));
|
||||
}
|
||||
},
|
||||
_ = timeout => None,
|
||||
};
|
||||
Some(match pin {
|
||||
Some(pin) if !pin.matches_client(request.client.policy.key_id()) => {
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "continuation does not belong to this client API key"));
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "Continuation does not belong to this client API key."));
|
||||
}
|
||||
Some(pin) if !authorization.account_scope.allows(pin.account()) => {
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "continuation account is outside the client account scope"));
|
||||
return Err(GatewayError::new(GatewayErrorKind::PolicyDenied, "Continuation account is outside the client account scope."));
|
||||
}
|
||||
Some(pin)
|
||||
if !plan.candidates().iter().any(|candidate| candidate.provider() == pin.provider()) =>
|
||||
{
|
||||
return Err(GatewayError::new(GatewayErrorKind::NoAvailableProvider, "continuation provider is not available"));
|
||||
return Err(GatewayError::new(GatewayErrorKind::NoAvailableProvider, "Continuation provider is not available."));
|
||||
}
|
||||
Some(pin) => {
|
||||
attach_continuation_session_state(&mut request.operation, &pin);
|
||||
@@ -728,7 +728,7 @@ impl DefaultExecutionService {
|
||||
let permit = authorization.nested_permit.take().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"nested execution admission is unavailable",
|
||||
"Nested execution admission is unavailable.",
|
||||
)
|
||||
})?;
|
||||
(ExecutionAdmission::Nested(permit), None)
|
||||
@@ -953,19 +953,19 @@ impl DefaultExecutionService {
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"parent model request is not active",
|
||||
"Parent model request is not active.",
|
||||
)
|
||||
})?;
|
||||
if authority.cancellation.is_cancelled() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Cancelled,
|
||||
"parent request was cancelled",
|
||||
"Parent request was cancelled.",
|
||||
));
|
||||
}
|
||||
if authority.deadline_at.is_elapsed() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Timeout,
|
||||
"parent request deadline elapsed",
|
||||
"Parent request deadline elapsed.",
|
||||
));
|
||||
}
|
||||
Ok(authority)
|
||||
@@ -989,7 +989,7 @@ impl DefaultExecutionService {
|
||||
if parent.extension_scope.len() >= ExtensionCallScope::MAXIMUM_DEPTH {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"nested model execution depth is exhausted",
|
||||
"Nested model execution depth is exhausted.",
|
||||
));
|
||||
}
|
||||
let extension_scope = parent
|
||||
@@ -998,7 +998,7 @@ impl DefaultExecutionService {
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"recursive plugin model execution is not allowed",
|
||||
"Recursive plugin model execution is not allowed.",
|
||||
)
|
||||
})?;
|
||||
let (account_scope, required_provider) = restrict_model_execution_scope(
|
||||
@@ -1057,7 +1057,7 @@ impl DefaultExecutionService {
|
||||
refresh.refresh().await.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"current runtime snapshot is unavailable",
|
||||
"Gateway configuration is unavailable.",
|
||||
)
|
||||
})?;
|
||||
}
|
||||
@@ -1068,7 +1068,7 @@ impl DefaultExecutionService {
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"current runtime snapshot is unavailable",
|
||||
"Gateway configuration is unavailable.",
|
||||
)
|
||||
})?;
|
||||
let policy = snapshot
|
||||
@@ -1077,11 +1077,14 @@ impl DefaultExecutionService {
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Unauthorized,
|
||||
"client API key no longer exists",
|
||||
"Client API key no longer exists.",
|
||||
)
|
||||
})?;
|
||||
policy.authorize().map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"Client API key is disabled.",
|
||||
)
|
||||
})?;
|
||||
Ok(AuthenticatedClient {
|
||||
policy: match &settings {
|
||||
@@ -1112,13 +1115,13 @@ impl DefaultExecutionService {
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Cancelled,
|
||||
"bound plugin invocation was cancelled",
|
||||
"Bound plugin invocation was cancelled.",
|
||||
));
|
||||
}
|
||||
if extension_scope.len() >= ExtensionCallScope::MAXIMUM_DEPTH {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"plugin model execution depth exceeded",
|
||||
"Plugin model execution depth exceeded.",
|
||||
));
|
||||
}
|
||||
let extension_scope = extension_scope
|
||||
@@ -1126,13 +1129,13 @@ impl DefaultExecutionService {
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"plugin model execution scope is recursive",
|
||||
"Plugin model execution scope is recursive.",
|
||||
)
|
||||
})?;
|
||||
let now = SystemTime::now();
|
||||
let deadline_at =
|
||||
Deadline::from_timeout(now, client.execution_timeout()).ok_or_else(|| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "system clock is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Internal, "System clock is invalid.")
|
||||
})?;
|
||||
let (account_scope, required_provider) = restrict_model_execution_scope(
|
||||
client.policy.account_scope().as_ref(),
|
||||
@@ -1194,13 +1197,13 @@ impl DefaultExecutionService {
|
||||
if !parent.account_scope.provider_kinds().contains(&provider) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"affinity provider is outside the frozen authorization scope",
|
||||
"Affinity provider is outside the frozen authorization scope.",
|
||||
));
|
||||
}
|
||||
let affinity = self.session_affinity.as_ref().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"provider session affinity is unavailable",
|
||||
"Provider session affinity is unavailable.",
|
||||
)
|
||||
})?;
|
||||
let account = {
|
||||
@@ -1211,15 +1214,15 @@ impl DefaultExecutionService {
|
||||
result = load => result.map_err(|error| match error.kind() {
|
||||
ProviderStoreErrorKind::Unavailable => GatewayError::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider session affinity is temporarily unavailable",
|
||||
"Provider session affinity is temporarily unavailable.",
|
||||
),
|
||||
ProviderStoreErrorKind::InvalidData | ProviderStoreErrorKind::Conflict => {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "provider session affinity is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Internal, "Provider session affinity is invalid.")
|
||||
}
|
||||
})?,
|
||||
_ = timeout => return Err(GatewayError::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider session affinity lookup timed out",
|
||||
"Provider session affinity lookup timed out.",
|
||||
)),
|
||||
}
|
||||
};
|
||||
@@ -1261,7 +1264,7 @@ impl DefaultExecutionService {
|
||||
if remaining.is_zero() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Timeout,
|
||||
"request deadline elapsed",
|
||||
"Request deadline elapsed.",
|
||||
));
|
||||
}
|
||||
// 在发送原子准入之前接管取消清理,覆盖 Redis 已取得租约但返回尚未被观察的窗口
|
||||
@@ -1280,9 +1283,9 @@ impl DefaultExecutionService {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
pin_mut!(acquire, timeout, cancelled);
|
||||
let decision = select_biased! {
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "request admission was cancelled")),
|
||||
result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "request admission is temporarily unavailable").with_source(source))?,
|
||||
_ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "request deadline elapsed")),
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "Request admission was cancelled.")),
|
||||
result = acquire => result.map_err(|source| GatewayError::new(GatewayErrorKind::NoAvailableProvider, "Request admission is temporarily unavailable.").with_source(source))?,
|
||||
_ = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "Request deadline elapsed.")),
|
||||
};
|
||||
match decision {
|
||||
ClientAdmissionDecision::Granted => {
|
||||
@@ -1307,7 +1310,7 @@ impl DefaultExecutionService {
|
||||
if reason == ClientAdmissionRejection::RateLimited || policy.max_waiting == 0 {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"request exceeds client API key limits",
|
||||
"Request exceeds client API key limits.",
|
||||
));
|
||||
}
|
||||
if waiting.elapsed().is_zero()
|
||||
@@ -1345,12 +1348,12 @@ impl DefaultExecutionService {
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"runtime snapshot is unavailable",
|
||||
"Gateway configuration is unavailable.",
|
||||
)
|
||||
})?;
|
||||
let public_model =
|
||||
PublicModelId::new(upstream_model.as_str().to_owned()).map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::Unsupported, "requested model is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Unsupported, "Requested model is invalid.")
|
||||
})?;
|
||||
let routing_context = RoutingContext {
|
||||
required_provider: Some(provider_kind),
|
||||
@@ -1361,11 +1364,11 @@ impl DefaultExecutionService {
|
||||
.map_err(map_routing_error)?;
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = started_at.checked_add(DIAGNOSTIC_TIMEOUT).ok_or_else(|| {
|
||||
GatewayError::new(GatewayErrorKind::Internal, "system clock is invalid")
|
||||
GatewayError::new(GatewayErrorKind::Internal, "System clock is invalid.")
|
||||
})?;
|
||||
let request_id = new_request_id()?;
|
||||
let actor = ClientApiKeyId::new("admin_connection_test")
|
||||
.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "invalid admin actor"))?;
|
||||
.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "Invalid admin actor."))?;
|
||||
let new_request = NewModelRequest {
|
||||
id: request_id,
|
||||
client_api_key_name_snapshot: None,
|
||||
@@ -1742,7 +1745,7 @@ impl ClientIdentityPort for DefaultExecutionService {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
pin_mut!(cancelled);
|
||||
select_biased! {
|
||||
() = cancelled => Err(GatewayError::new(GatewayErrorKind::Cancelled, "client verification was cancelled")),
|
||||
() = cancelled => Err(GatewayError::new(GatewayErrorKind::Cancelled, "Client verification was cancelled.")),
|
||||
result = verification => result.map_err(authentication_gateway_error),
|
||||
}
|
||||
}
|
||||
@@ -1768,9 +1771,9 @@ impl NestedModelExecutionPort for DefaultExecutionService {
|
||||
.fuse();
|
||||
pin_mut!(cancelled, timeout, catalog);
|
||||
let models = select_biased! {
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "model catalog call was cancelled")),
|
||||
() = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "model catalog deadline elapsed")),
|
||||
result = catalog => result.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "model catalog is unavailable"))?,
|
||||
() = cancelled => return Err(GatewayError::new(GatewayErrorKind::Cancelled, "Model catalog call was cancelled.")),
|
||||
() = timeout => return Err(GatewayError::new(GatewayErrorKind::Timeout, "Model catalog deadline elapsed.")),
|
||||
result = catalog => result.map_err(|_| GatewayError::new(GatewayErrorKind::Internal, "Model catalog is unavailable."))?,
|
||||
};
|
||||
Ok(models
|
||||
.into_iter()
|
||||
@@ -1850,13 +1853,13 @@ fn restrict_model_execution_scope(
|
||||
if !parent_scope.allows(parent_account) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"parent attempt account is outside the frozen account scope",
|
||||
"Parent attempt account is outside the frozen account scope.",
|
||||
));
|
||||
}
|
||||
if account == Some(parent_account) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"nested execution cannot wait for the parent attempt account",
|
||||
"Nested execution cannot wait for the parent attempt account.",
|
||||
));
|
||||
}
|
||||
account_scope = account_scope.excluding_account(parent_account.clone());
|
||||
@@ -1866,7 +1869,7 @@ fn restrict_model_execution_scope(
|
||||
if !account_scope.allows(account) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model account is outside the frozen authorization scope",
|
||||
"Model account is outside the frozen authorization scope.",
|
||||
));
|
||||
}
|
||||
let account_provider = account_scope
|
||||
@@ -1875,7 +1878,7 @@ fn restrict_model_execution_scope(
|
||||
.ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model account is unavailable in the frozen snapshot",
|
||||
"Model account is unavailable in the frozen snapshot.",
|
||||
)
|
||||
})?;
|
||||
if provider
|
||||
@@ -1884,7 +1887,7 @@ fn restrict_model_execution_scope(
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model provider and account do not match",
|
||||
"Model provider and account do not match.",
|
||||
));
|
||||
}
|
||||
Some(account_provider)
|
||||
@@ -1898,7 +1901,7 @@ fn restrict_model_execution_scope(
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"model provider is outside the frozen authorization scope",
|
||||
"Model provider is outside the frozen authorization scope.",
|
||||
));
|
||||
}
|
||||
Ok((Arc::new(account_scope), required_provider))
|
||||
|
||||
@@ -12,7 +12,7 @@ pub(super) fn check_image_generation(
|
||||
if !allowed && operation.image_generation_requested() {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"Image generation is not allowed by the API key's account groups",
|
||||
"Image generation is not allowed by the API key's account groups.",
|
||||
)
|
||||
.with_client_code("image_generation_disabled"));
|
||||
}
|
||||
|
||||
@@ -1203,62 +1203,63 @@ impl GatewayError {
|
||||
#[must_use]
|
||||
pub fn from_provider(error: &ProviderError) -> Self {
|
||||
let gateway = match error.kind() {
|
||||
ProviderErrorKind::InvalidRequest => {
|
||||
Self::new(GatewayErrorKind::InvalidRequest, "invalid upstream request")
|
||||
}
|
||||
ProviderErrorKind::InvalidRequest => Self::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"Invalid upstream request.",
|
||||
),
|
||||
ProviderErrorKind::SessionPolicyRestricted => Self::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"session policy excludes all eligible accounts",
|
||||
"No accounts are available under this conversation's restrictions.",
|
||||
),
|
||||
ProviderErrorKind::ContinuationRecoveryRequired => Self::new(
|
||||
GatewayErrorKind::InvalidRequest,
|
||||
"conversation continuation must be rebuilt",
|
||||
"Previous response unavailable. Resend the full conversation history.",
|
||||
),
|
||||
ProviderErrorKind::Unsupported => Self::new(
|
||||
GatewayErrorKind::Unsupported,
|
||||
"requested capability is unsupported",
|
||||
"Requested capability is unsupported.",
|
||||
),
|
||||
ProviderErrorKind::Unauthorized | ProviderErrorKind::PermissionDenied => Self::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"upstream authentication resource is unavailable",
|
||||
"The upstream account cannot access the service.",
|
||||
),
|
||||
ProviderErrorKind::RequestPolicyDenied => Self::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"request policy rejected the request",
|
||||
"Request policy rejected the request.",
|
||||
),
|
||||
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted => Self::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
"Upstream capacity is temporarily unavailable.",
|
||||
),
|
||||
ProviderErrorKind::ConcurrencyQueueFull => Self::new(
|
||||
GatewayErrorKind::ConcurrencyQueueFull,
|
||||
"concurrency wait queue is full",
|
||||
"Request queue is full.",
|
||||
),
|
||||
ProviderErrorKind::ConcurrencyQueueTimeout => Self::new(
|
||||
GatewayErrorKind::ConcurrencyQueueTimeout,
|
||||
"concurrency wait deadline elapsed",
|
||||
"Request queue wait timed out.",
|
||||
),
|
||||
ProviderErrorKind::AccountCapacityUnavailable => Self::new(
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy",
|
||||
"All eligible upstream accounts are temporarily busy.",
|
||||
),
|
||||
ProviderErrorKind::NoEligibleAccount => Self::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
"No upstream provider is currently available for this request.",
|
||||
),
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable => Self::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable",
|
||||
"Upstream accounts are temporarily inaccessible.",
|
||||
),
|
||||
ProviderErrorKind::Timeout => {
|
||||
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
|
||||
Self::new(GatewayErrorKind::Timeout, "Upstream request timed out.")
|
||||
}
|
||||
ProviderErrorKind::MessageTooBig => Self::new(
|
||||
GatewayErrorKind::MessageTooBig,
|
||||
"upstream rejected the request because the message is too large",
|
||||
"Request too large. Reduce the input size.",
|
||||
),
|
||||
ProviderErrorKind::Cancelled => {
|
||||
Self::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
Self::new(GatewayErrorKind::Cancelled, "Request was cancelled.")
|
||||
}
|
||||
ProviderErrorKind::Transport
|
||||
| ProviderErrorKind::Protocol
|
||||
@@ -1266,7 +1267,7 @@ impl GatewayError {
|
||||
| ProviderErrorKind::UpstreamCapacityUnavailable
|
||||
| ProviderErrorKind::ProcessTerminated => Self::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"upstream service is unavailable",
|
||||
"Upstream service is unavailable.",
|
||||
),
|
||||
};
|
||||
let gateway = match error.diagnostic() {
|
||||
|
||||
@@ -37,26 +37,26 @@ impl CancellationReason {
|
||||
#[must_use]
|
||||
pub const fn message(self) -> &'static str {
|
||||
match self {
|
||||
Self::Requested => "request was cancelled",
|
||||
Self::Requested => "Request was cancelled.",
|
||||
Self::Lease {
|
||||
kind: RequestLeaseKind::ClientAdmission,
|
||||
failure,
|
||||
} => match failure {
|
||||
LeaseFailure::Lost => "request was cancelled: client admission lease was lost",
|
||||
LeaseFailure::Lost => "Request was cancelled: client admission lease was lost.",
|
||||
LeaseFailure::RenewalTimedOut => {
|
||||
"request was cancelled: client admission lease renewal timed out"
|
||||
"Request was cancelled: client admission lease renewal timed out."
|
||||
}
|
||||
LeaseFailure::Expired => "request was cancelled: client admission lease expired",
|
||||
LeaseFailure::Expired => "Request was cancelled: client admission lease expired.",
|
||||
},
|
||||
Self::Lease {
|
||||
kind: RequestLeaseKind::Credential,
|
||||
failure,
|
||||
} => match failure {
|
||||
LeaseFailure::Lost => "request was cancelled: credential lease was lost",
|
||||
LeaseFailure::Lost => "Request was cancelled: credential lease was lost.",
|
||||
LeaseFailure::RenewalTimedOut => {
|
||||
"request was cancelled: credential lease renewal timed out"
|
||||
"Request was cancelled: credential lease renewal timed out."
|
||||
}
|
||||
LeaseFailure::Expired => "request was cancelled: credential lease expired",
|
||||
LeaseFailure::Expired => "Request was cancelled: credential lease expired.",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3776,7 +3776,7 @@ fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() {
|
||||
(error.kind(), error.client_message()),
|
||||
(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"the requested model was not found in the provider catalogs available to this API key; check the model name",
|
||||
"Model unavailable for this API key. Check the model name.",
|
||||
)
|
||||
);
|
||||
let rejections = store.entry_rejections.lock().unwrap();
|
||||
|
||||
@@ -280,7 +280,7 @@ fn gateway_error_should_keep_client_visible_upstream_fields_out_of_safe_diagnost
|
||||
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"upstream capacity is temporarily unavailable"
|
||||
"Upstream capacity is temporarily unavailable."
|
||||
);
|
||||
assert_eq!(gateway.client_message(), message);
|
||||
assert_eq!(gateway.client_error_code(), Some("quota_exhausted"));
|
||||
@@ -304,7 +304,7 @@ fn continuation_recovery_should_keep_its_internal_classification_and_client_cont
|
||||
assert_eq!(error.kind().as_str(), "continuation_recovery_required");
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"conversation continuation must be rebuilt"
|
||||
"Previous response unavailable. Resend the full conversation history."
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.client_error_code(),
|
||||
@@ -329,7 +329,7 @@ fn message_too_big_should_map_to_a_request_scoped_client_contract() {
|
||||
assert_eq!(gateway.kind().as_str(), "message_too_big");
|
||||
assert_eq!(
|
||||
gateway.safe_message(),
|
||||
"upstream rejected the request because the message is too large"
|
||||
"Request too large. Reduce the input size."
|
||||
);
|
||||
assert_eq!(gateway.client_error_code(), Some("message_too_big"));
|
||||
assert_eq!(gateway.client_error_type(), Some("invalid_request_error"));
|
||||
|
||||
@@ -160,6 +160,7 @@ impl HostBundle {
|
||||
lease: Arc<dyn WorkerLeaderLeasePort>,
|
||||
diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
|
||||
) -> Result<(), HostError> {
|
||||
self.system.set_diagnostics(Arc::clone(&diagnostics));
|
||||
self.workers.start(plan, lease, diagnostics)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) fn extract_release(
|
||||
temp_dir: &Path,
|
||||
) -> Result<ExtractedRelease, OperationError> {
|
||||
let file = fs::File::open(archive_path)
|
||||
.map_err(|error| internal(format!("failed to open release archive: {error}")))?;
|
||||
.map_err(|error| internal("无法打开更新包").with_source(error))?;
|
||||
let decoder = flate2::read::GzDecoder::new(file);
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
let binary_path = temp_dir.join(APP_BINARY_NAME);
|
||||
@@ -37,61 +37,55 @@ pub(crate) fn extract_release(
|
||||
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|error| internal(format!("failed to read release archive: {error}")))?
|
||||
.map_err(|error| internal("无法读取更新包").with_source(error))?
|
||||
{
|
||||
let mut entry =
|
||||
entry.map_err(|error| internal(format!("invalid archive entry: {error}")))?;
|
||||
entry.map_err(|error| internal("更新包内容损坏,请重新下载").with_source(error))?;
|
||||
let path = entry
|
||||
.path()
|
||||
.map_err(|error| internal(format!("invalid archive path: {error}")))?
|
||||
.map_err(|error| internal("更新包中的文件路径无效").with_source(error))?
|
||||
.into_owned();
|
||||
if unsafe_archive_path(&path) {
|
||||
return Err(invalid("release archive contains an unsafe path"));
|
||||
return Err(invalid("更新包包含不安全的文件路径,已停止更新"));
|
||||
}
|
||||
let official_relative = official_plugin_relative_path(&path)?;
|
||||
if !entry.header().entry_type().is_file() {
|
||||
if official_relative.is_some() {
|
||||
return Err(invalid(
|
||||
"release archive contains a non-file official plugin asset",
|
||||
));
|
||||
return Err(invalid("更新包中的官方插件资源不是普通文件"));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
file_count = file_count.saturating_add(1);
|
||||
extracted_size = extracted_size.saturating_add(entry.header().size().unwrap_or(u64::MAX));
|
||||
if file_count > MAX_ARCHIVE_FILES || extracted_size > MAX_EXTRACTED_SIZE {
|
||||
return Err(invalid("release archive expands beyond safety limits"));
|
||||
return Err(invalid("更新包解压后的大小或文件数量超出限制"));
|
||||
}
|
||||
|
||||
if path.file_name().is_some_and(|name| name == APP_BINARY_NAME) {
|
||||
if found_binary {
|
||||
return Err(invalid("release archive contains duplicate binaries"));
|
||||
return Err(invalid("更新包包含重复的程序文件"));
|
||||
}
|
||||
entry
|
||||
.unpack(&binary_path)
|
||||
.map_err(|error| internal(format!("failed to extract binary: {error}")))?;
|
||||
.map_err(|error| internal("无法解压程序文件").with_source(error))?;
|
||||
found_binary = true;
|
||||
continue;
|
||||
}
|
||||
if let Some(relative) = official_relative {
|
||||
if !official_files.insert(relative.clone()) {
|
||||
return Err(invalid(
|
||||
"release archive contains a duplicate official plugin asset",
|
||||
));
|
||||
return Err(invalid("更新包包含重复的官方插件文件"));
|
||||
}
|
||||
fs::create_dir_all(&official_plugins_dir).map_err(|error| {
|
||||
internal(format!("failed to create official plugin dir: {error}"))
|
||||
})?;
|
||||
fs::create_dir_all(&official_plugins_dir)
|
||||
.map_err(|error| internal("无法创建官方插件目录").with_source(error))?;
|
||||
let target = official_plugins_dir.join(&relative);
|
||||
entry.unpack(&target).map_err(|error| {
|
||||
internal(format!("failed to extract official plugin asset: {error}"))
|
||||
})?;
|
||||
entry
|
||||
.unpack(&target)
|
||||
.map_err(|error| internal("无法解压官方插件文件").with_source(error))?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
fs::set_permissions(&target, fs::Permissions::from_mode(0o444)).map_err(
|
||||
|error| internal(format!("failed to protect official plugin asset: {error}")),
|
||||
)?;
|
||||
fs::set_permissions(&target, fs::Permissions::from_mode(0o444))
|
||||
.map_err(|error| internal("无法设置官方插件文件权限").with_source(error))?;
|
||||
}
|
||||
found_official_manifest |= relative == Path::new("plugin-release-manifest.json");
|
||||
continue;
|
||||
@@ -102,29 +96,26 @@ pub(crate) fn extract_release(
|
||||
}
|
||||
let target = web_dist_dir.join(relative);
|
||||
if let Some(parent) = target.parent() {
|
||||
fs::create_dir_all(parent).map_err(|error| {
|
||||
internal(format!("failed to create web asset dir: {error}"))
|
||||
})?;
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|error| internal("无法创建管理页面目录").with_source(error))?;
|
||||
}
|
||||
entry
|
||||
.unpack(&target)
|
||||
.map_err(|error| internal(format!("failed to extract web asset: {error}")))?;
|
||||
.map_err(|error| internal("无法解压管理页面文件").with_source(error))?;
|
||||
found_web = true;
|
||||
}
|
||||
}
|
||||
if !found_binary {
|
||||
return Err(invalid("release archive does not contain codex-proxy-rs"));
|
||||
return Err(invalid("更新包缺少网关程序文件"));
|
||||
}
|
||||
if !found_official_manifest {
|
||||
return Err(invalid(
|
||||
"release archive does not contain the official plugin manifest",
|
||||
));
|
||||
return Err(invalid("更新包缺少官方插件发行清单"));
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
fs::set_permissions(&binary_path, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| internal(format!("failed to chmod binary: {error}")))?;
|
||||
.map_err(|error| internal("无法设置程序执行权限").with_source(error))?;
|
||||
}
|
||||
Ok(ExtractedRelease {
|
||||
binary_path,
|
||||
@@ -187,9 +178,7 @@ fn official_plugin_relative_path(path: &Path) -> Result<Option<PathBuf>, Operati
|
||||
return Ok(None);
|
||||
}
|
||||
if relative.len() != 1 {
|
||||
return Err(invalid(
|
||||
"release archive contains a nested official plugin asset",
|
||||
));
|
||||
return Err(invalid("更新包中的官方插件目录结构不符合要求"));
|
||||
}
|
||||
let file_name = relative[0];
|
||||
let is_manifest = file_name == "plugin-release-manifest.json";
|
||||
@@ -197,9 +186,7 @@ fn official_plugin_relative_path(path: &Path) -> Result<Option<PathBuf>, Operati
|
||||
.to_str()
|
||||
.is_some_and(|value| value.ends_with(".tar.gz"));
|
||||
if !is_manifest && !is_archive {
|
||||
return Err(invalid(
|
||||
"release archive contains an unexpected official plugin asset",
|
||||
));
|
||||
return Err(invalid("更新包包含不支持的官方插件文件类型"));
|
||||
}
|
||||
return Ok(Some(PathBuf::from(file_name)));
|
||||
}
|
||||
|
||||
@@ -130,7 +130,7 @@ impl SystemUpdateConfig {
|
||||
}
|
||||
env::current_exe()
|
||||
.and_then(fs::canonicalize)
|
||||
.map_err(|error| internal(format!("failed to resolve executable: {error}")))
|
||||
.map_err(|error| internal("无法确定当前程序位置").with_source(error))
|
||||
}
|
||||
|
||||
pub(crate) fn official_plugins_dir(&self) -> Result<PathBuf, OperationError> {
|
||||
@@ -138,7 +138,7 @@ impl SystemUpdateConfig {
|
||||
let parent = executable
|
||||
.parent()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.ok_or_else(|| internal("failed to resolve official plugin release directory"))?;
|
||||
.ok_or_else(|| internal("无法确定官方插件安装目录"))?;
|
||||
Ok(parent.join("plugins").join("official"))
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ impl SystemUpdateConfig {
|
||||
self.web_dist_dir
|
||||
.as_deref()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.ok_or_else(|| conflict("web assets directory is not configured"))
|
||||
.ok_or_else(|| conflict("尚未配置管理页面文件目录,无法在线更新"))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
//! 更新 owner 消费失败时记录受控详情,初始化和关闭阶段保持尽力写入
|
||||
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use gateway_core::diagnostics::{OperationalDiagnostics, OperationalFailure};
|
||||
|
||||
use super::OperationError;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct Diagnostics(Arc<OnceLock<Arc<dyn OperationalDiagnostics>>>);
|
||||
|
||||
impl Diagnostics {
|
||||
pub(super) fn bind(&self, diagnostics: Arc<dyn OperationalDiagnostics>) {
|
||||
let _ = self.0.set(diagnostics);
|
||||
}
|
||||
|
||||
pub(super) async fn record(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
correlation_id: Option<&str>,
|
||||
error: &OperationError,
|
||||
) {
|
||||
let Some(diagnostics) = self.0.get() else {
|
||||
return;
|
||||
};
|
||||
let mut failure = OperationalFailure::new(
|
||||
"system_update",
|
||||
operation,
|
||||
"system_operation_failed",
|
||||
error.message(),
|
||||
);
|
||||
failure.correlation_id = correlation_id.map(str::to_owned);
|
||||
failure.details = error.error_details();
|
||||
if diagnostics.record_failure(failure).await.is_err() {
|
||||
tracing::warn!(operation, "系统操作诊断记录失败");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_on_drop(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
correlation_id: Option<String>,
|
||||
error: OperationError,
|
||||
) {
|
||||
// 析构不能等待异步存储,运行时已关闭时沿用诊断尽力写入的合同
|
||||
if self.0.get().is_some()
|
||||
&& let Ok(runtime) = tokio::runtime::Handle::try_current()
|
||||
{
|
||||
let diagnostics = self.clone();
|
||||
drop(runtime.spawn(async move {
|
||||
diagnostics
|
||||
.record(operation, correlation_id.as_deref(), &error)
|
||||
.await;
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ pub(crate) async fn download_file(
|
||||
) -> Result<(), OperationError> {
|
||||
validate_download_url(url, api_base)?;
|
||||
if expected_size == 0 || expected_size > MAX_DOWNLOAD_SIZE {
|
||||
return Err(invalid("release archive size is invalid"));
|
||||
return Err(invalid("更新包大小无效或超出限制"));
|
||||
}
|
||||
let client = download_client(api_base, Duration::from_secs(120))?;
|
||||
let response = client
|
||||
@@ -33,32 +33,32 @@ pub(crate) async fn download_file(
|
||||
.header(reqwest::header::ACCEPT_ENCODING, "identity")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| upstream(format!("release download failed: {error}")))?;
|
||||
.map_err(|error| super::error::http("更新包下载失败", error))?;
|
||||
if !response.status().is_success() {
|
||||
return Err(upstream(format!(
|
||||
"release download failed with {}",
|
||||
response.status()
|
||||
"更新包下载失败(HTTP {})",
|
||||
response.status().as_u16()
|
||||
)));
|
||||
}
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|size| size != expected_size)
|
||||
{
|
||||
return Err(upstream("downloaded size does not match release metadata"));
|
||||
return Err(upstream("下载文件大小与发行信息不一致"));
|
||||
}
|
||||
let mut file = fs::File::create(destination)
|
||||
.map_err(|error| super::internal(format!("failed to create download file: {error}")))?;
|
||||
.map_err(|error| super::internal("无法创建下载文件").with_source(error))?;
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut downloaded = 0_u64;
|
||||
let mut next_progress = 10_u8;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|error| upstream(format!("download stream failed: {error}")))?;
|
||||
let chunk = chunk.map_err(|error| super::error::http("更新包下载中断", error))?;
|
||||
downloaded = downloaded.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
|
||||
if downloaded > expected_size {
|
||||
return Err(invalid("download exceeds declared release size"));
|
||||
return Err(invalid("下载文件超过发行信息声明的大小,已停止更新"));
|
||||
}
|
||||
file.write_all(&chunk)
|
||||
.map_err(|error| super::internal(format!("failed to write download: {error}")))?;
|
||||
.map_err(|error| super::internal("无法保存下载内容").with_source(error))?;
|
||||
let percent = download_percent(downloaded, expected_size);
|
||||
if percent >= next_progress || downloaded == expected_size {
|
||||
events.emit_progress(
|
||||
@@ -74,9 +74,9 @@ pub(crate) async fn download_file(
|
||||
}
|
||||
}
|
||||
file.sync_all()
|
||||
.map_err(|error| super::internal(format!("failed to sync download: {error}")))?;
|
||||
.map_err(|error| super::internal("无法完成下载文件写入").with_source(error))?;
|
||||
if downloaded != expected_size {
|
||||
return Err(upstream("downloaded size does not match release metadata"));
|
||||
return Err(upstream("下载文件大小与发行信息不一致"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -90,7 +90,7 @@ pub(crate) async fn verify_checksum(
|
||||
) -> Result<(), OperationError> {
|
||||
validate_download_url(checksum_url, api_base)?;
|
||||
if checksum_size == 0 || checksum_size > MAX_CHECKSUM_SIZE {
|
||||
return Err(invalid("release checksum size is invalid"));
|
||||
return Err(invalid("校验文件大小无效或超出限制"));
|
||||
}
|
||||
let client = download_client(api_base, Duration::from_secs(30))?;
|
||||
let response = client
|
||||
@@ -98,27 +98,27 @@ pub(crate) async fn verify_checksum(
|
||||
.header(reqwest::header::ACCEPT_ENCODING, "identity")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| upstream(format!("checksum download failed: {error}")))?;
|
||||
.map_err(|error| super::error::http("校验文件下载失败", error))?;
|
||||
if !response.status().is_success() {
|
||||
return Err(upstream(format!(
|
||||
"checksum download failed with {}",
|
||||
response.status()
|
||||
"校验文件下载失败(HTTP {})",
|
||||
response.status().as_u16()
|
||||
)));
|
||||
}
|
||||
let bytes = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|error| upstream(format!("checksum read failed: {error}")))?;
|
||||
.map_err(|error| super::error::http("校验文件读取失败", error))?;
|
||||
if bytes.len() > MAX_CHECKSUM_SIZE as usize || bytes.len() as u64 != checksum_size {
|
||||
return Err(invalid("checksum document size is invalid"));
|
||||
return Err(invalid("校验文件大小与发行信息不一致"));
|
||||
}
|
||||
let body =
|
||||
std::str::from_utf8(&bytes).map_err(|_| invalid("checksum document is not UTF-8"))?;
|
||||
std::str::from_utf8(&bytes).map_err(|_| invalid("校验文件编码无效,无法校验更新包"))?;
|
||||
let expected = checksum_for(body, file_name)
|
||||
.ok_or_else(|| upstream("release checksum entry is missing"))?;
|
||||
.ok_or_else(|| upstream("校验文件中缺少当前更新包的校验值"))?;
|
||||
let actual = sha256_file(file_path)?;
|
||||
if !expected.eq_ignore_ascii_case(&actual) {
|
||||
return Err(upstream("release checksum mismatch"));
|
||||
return Err(upstream("更新包校验失败,请重新下载"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -137,13 +137,13 @@ fn checksum_for(body: &str, file_name: &str) -> Option<String> {
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, OperationError> {
|
||||
let mut file = fs::File::open(path)
|
||||
.map_err(|error| super::internal(format!("failed to open checksum file: {error}")))?;
|
||||
.map_err(|error| super::internal("无法打开待校验文件").with_source(error))?;
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file
|
||||
.read(&mut buffer)
|
||||
.map_err(|error| super::internal(format!("failed to read checksum file: {error}")))?;
|
||||
.map_err(|error| super::internal("无法读取待校验文件").with_source(error))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -16,6 +16,16 @@ pub(super) fn upstream(message: impl Into<String>) -> OperationError {
|
||||
SystemOperationError::new(SystemOperationErrorKind::Upstream, message)
|
||||
}
|
||||
|
||||
pub(super) fn http(message: &'static str, source: reqwest::Error) -> OperationError {
|
||||
let failure = upstream(message);
|
||||
let failure = if source.url().is_some() {
|
||||
failure.redact_sensitive_context()
|
||||
} else {
|
||||
failure
|
||||
};
|
||||
failure.with_source(source.without_url())
|
||||
}
|
||||
|
||||
pub(super) fn internal(message: impl Into<String>) -> OperationError {
|
||||
SystemOperationError::new(SystemOperationErrorKind::Internal, message)
|
||||
}
|
||||
|
||||
@@ -52,8 +52,7 @@ impl ReleaseFiles {
|
||||
}
|
||||
|
||||
fn read_error(error: io::Error) -> OperationError {
|
||||
tracing::warn!(error = %error, "核对系统安装文件失败");
|
||||
internal("无法核对安装文件,请检查文件完整性与读取权限")
|
||||
internal("无法核对安装文件").with_source(error)
|
||||
}
|
||||
|
||||
fn hash_directory(path: &Path) -> io::Result<String> {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
mod archive;
|
||||
mod config;
|
||||
mod diagnostics;
|
||||
mod download;
|
||||
mod error;
|
||||
mod events;
|
||||
|
||||
@@ -44,6 +44,7 @@ pub struct ProcessSystemOperations {
|
||||
operation_lock: Arc<AsyncMutex<Option<OperationFileLock>>>,
|
||||
release_cache: Arc<ReleaseCache>,
|
||||
running_files: Arc<Result<ReleaseFiles, OperationError>>,
|
||||
diagnostics: super::diagnostics::Diagnostics,
|
||||
}
|
||||
|
||||
impl ProcessSystemOperations {
|
||||
@@ -62,13 +63,22 @@ impl ProcessSystemOperations {
|
||||
operation_lock: Arc::new(AsyncMutex::new(None)),
|
||||
release_cache: Arc::new(ReleaseCache::default()),
|
||||
running_files,
|
||||
diagnostics: super::diagnostics::Diagnostics::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Store 就绪后绑定受控诊断端口,不改变 Host 的初始化顺序
|
||||
pub fn set_diagnostics(
|
||||
&self,
|
||||
diagnostics: Arc<dyn gateway_core::diagnostics::OperationalDiagnostics>,
|
||||
) {
|
||||
self.diagnostics.bind(diagnostics);
|
||||
}
|
||||
|
||||
fn lock_operation(&self) -> Result<OwnedMutexGuard<Option<OperationFileLock>>, OperationError> {
|
||||
let operation = Arc::clone(&self.operation_lock)
|
||||
.try_lock_owned()
|
||||
.map_err(|_| conflict("system operation is already running"))?;
|
||||
.map_err(|_| conflict("已有系统操作正在执行"))?;
|
||||
if operation.is_some() {
|
||||
return Err(conflict("服务正在重启"));
|
||||
}
|
||||
@@ -95,7 +105,7 @@ impl ProcessSystemOperations {
|
||||
let selected = self.resolve_channel(channel)?;
|
||||
if let Err(error) = validate_update_target(&self.config.version, &target, selected) {
|
||||
self.events
|
||||
.error_terminal(None, Some("preflight"), error.to_string());
|
||||
.error_terminal(None, Some("preflight"), error.message().to_owned());
|
||||
return Err(error);
|
||||
}
|
||||
let status = self.reconcile_installation()?;
|
||||
@@ -109,6 +119,7 @@ impl ProcessSystemOperations {
|
||||
&target,
|
||||
&self.config.version,
|
||||
file_lock,
|
||||
self.diagnostics.clone(),
|
||||
)?;
|
||||
let accepted = SystemOperationAccepted::Update {
|
||||
operation_id: operation_id.clone(),
|
||||
@@ -125,7 +136,18 @@ impl ProcessSystemOperations {
|
||||
() = service.cancellation.cancelled() => Err(conflict("服务关闭,更新已中断")),
|
||||
result = service.perform_update_inner(&target, selected, &operation_id, preflight.as_ref()) => result,
|
||||
};
|
||||
if let Err(error) = operation.complete(&result) {
|
||||
let completion = operation.complete(&result);
|
||||
if let Err(error) = &result {
|
||||
service
|
||||
.diagnostics
|
||||
.record("install", Some(&operation_id), error)
|
||||
.await;
|
||||
}
|
||||
if let Err(error) = completion {
|
||||
service
|
||||
.diagnostics
|
||||
.record("persist_update_status", Some(&operation_id), &error)
|
||||
.await;
|
||||
tracing::warn!(error = %error, "系统更新终态落盘失败");
|
||||
return;
|
||||
}
|
||||
@@ -138,7 +160,7 @@ impl ProcessSystemOperations {
|
||||
Err(error) => service.events.error_terminal(
|
||||
Some(&operation_id),
|
||||
Some("failed"),
|
||||
error.to_string(),
|
||||
error.message().to_owned(),
|
||||
),
|
||||
}
|
||||
}));
|
||||
@@ -156,7 +178,7 @@ impl ProcessSystemOperations {
|
||||
.config
|
||||
.update_repository
|
||||
.as_deref()
|
||||
.ok_or_else(|| conflict("update repository is not configured"))?;
|
||||
.ok_or_else(|| conflict("未配置更新仓库 CPR_UPDATE_REPOSITORY"))?;
|
||||
self.events.info(
|
||||
Some(operation_id),
|
||||
Some("release"),
|
||||
@@ -211,22 +233,22 @@ impl ProcessSystemOperations {
|
||||
self.events
|
||||
.info(Some(operation_id), Some("verify"), "正在校验更新资源");
|
||||
if archive.size == 0 || archive.size > MAX_DOWNLOAD_SIZE {
|
||||
return Err(invalid("release archive size is invalid"));
|
||||
return Err(invalid("更新包大小无效或超出限制"));
|
||||
}
|
||||
let checksum = release
|
||||
.assets
|
||||
.iter()
|
||||
.find(|asset| asset.name == "checksums.txt")
|
||||
.ok_or_else(|| upstream("release checksums.txt is required"))?;
|
||||
.ok_or_else(|| upstream("发行版本缺少 checksums.txt 校验文件,无法安全更新"))?;
|
||||
if checksum.size == 0 || checksum.size > MAX_CHECKSUM_SIZE {
|
||||
return Err(invalid("release checksum size is invalid"));
|
||||
return Err(invalid("校验文件大小无效或超出限制"));
|
||||
}
|
||||
self.events
|
||||
.info(Some(operation_id), Some("prepare"), "正在创建临时更新目录");
|
||||
fs::create_dir_all(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to prepare update temp dir: {error}")))?;
|
||||
.map_err(|error| internal("无法创建更新临时目录").with_source(error))?;
|
||||
let temp_root = fs::canonicalize(&self.config.update_temp_dir)
|
||||
.map_err(|error| internal(format!("failed to resolve update temp dir: {error}")))?;
|
||||
.map_err(|error| internal("无法确定更新临时目录位置").with_source(error))?;
|
||||
let temp = UpdateTempDir::create(&temp_root)?;
|
||||
let archive_path = temp.path().join(&archive.name);
|
||||
self.events
|
||||
@@ -283,7 +305,7 @@ impl ProcessSystemOperations {
|
||||
self.config.web_dist_dir()?,
|
||||
extracted,
|
||||
)?;
|
||||
let mut applied = AppliedReleaseGuard::new(&self.config);
|
||||
let mut applied = AppliedReleaseGuard::new(&self.config, &self.diagnostics);
|
||||
let applied_manifest = read_release_manifest(&self.config.official_plugins_dir()?)?;
|
||||
if applied_manifest.as_ref() != release_manifest.as_ref() {
|
||||
return Err(applied.rollback(conflict("已应用的更新候选与预检候选不一致")));
|
||||
@@ -368,12 +390,15 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
.await
|
||||
{
|
||||
Ok(detail) => Ok(detail),
|
||||
Err(error) => Ok(base_update_detail(
|
||||
&self.config,
|
||||
channel,
|
||||
self.config.update_support_error(),
|
||||
Some(error.to_string()),
|
||||
)),
|
||||
Err(error) => {
|
||||
self.diagnostics.record("check_release", None, &error).await;
|
||||
Ok(base_update_detail(
|
||||
&self.config,
|
||||
channel,
|
||||
self.config.update_support_error(),
|
||||
Some(error.message().to_owned()),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,7 +479,7 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
)?;
|
||||
let result = match rollback_release(&self.config) {
|
||||
Ok(()) => {
|
||||
let mut applied = AppliedReleaseGuard::new(&self.config);
|
||||
let mut applied = AppliedReleaseGuard::new(&self.config, &self.diagnostics);
|
||||
let result = match read_release_manifest(&self.config.official_plugins_dir()?) {
|
||||
Ok(manifest) if manifest.as_ref() == release_manifest.as_ref() => {
|
||||
preflight.confirm_revision(plugin_revision).await
|
||||
@@ -478,24 +503,28 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
SystemOperationKind::Rollback,
|
||||
None,
|
||||
None,
|
||||
result.as_ref().err().map(ToString::to_string),
|
||||
result
|
||||
.as_ref()
|
||||
.err()
|
||||
.map(|error| error.message().to_owned()),
|
||||
)?;
|
||||
match &result {
|
||||
Ok(()) => self.events.success_terminal(
|
||||
Some(&operation_id),
|
||||
Some("done"),
|
||||
"previous release restored",
|
||||
"上一版本已恢复,请重启服务",
|
||||
),
|
||||
Err(error) => self.events.error_terminal(
|
||||
Some(&operation_id),
|
||||
Some("failed"),
|
||||
error.message().to_owned(),
|
||||
),
|
||||
Err(error) => {
|
||||
self.events
|
||||
.error_terminal(Some(&operation_id), Some("failed"), error.to_string())
|
||||
}
|
||||
}
|
||||
drop(file_lock);
|
||||
result?;
|
||||
Ok(SystemOperationAccepted::Rollback {
|
||||
operation_id,
|
||||
message: "回滚完成,请重启服务。".to_owned(),
|
||||
message: "回滚完成,请重启服务".to_owned(),
|
||||
need_restart: true,
|
||||
})
|
||||
}
|
||||
@@ -513,7 +542,7 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
// 载入半成品产物
|
||||
let mut operation = self.lock_operation()?;
|
||||
if !self.config.self_restart_enabled {
|
||||
return Err(conflict("self restart is disabled"));
|
||||
return Err(conflict("当前部署未启用在线重启"));
|
||||
}
|
||||
let file_lock = OperationFileLock::acquire(&self.config.update_lock_file)?;
|
||||
let candidate = self.installed_restart_candidate()?;
|
||||
@@ -555,36 +584,40 @@ impl SystemOperations for ProcessSystemOperations {
|
||||
fn read_release_manifest(directory: &Path) -> Result<Arc<[u8]>, OperationError> {
|
||||
let path = directory.join(OFFICIAL_PLUGIN_MANIFEST);
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| invalid(format!("official plugin manifest is unavailable: {error}")))?;
|
||||
.map_err(|error| invalid("无法访问官方插件发行清单").with_source(error))?;
|
||||
if !metadata.file_type().is_file() || metadata.file_type().is_symlink() {
|
||||
return Err(invalid("official plugin manifest is not a regular file"));
|
||||
return Err(invalid("官方插件发行清单必须是普通文件"));
|
||||
}
|
||||
if metadata.len() == 0 || metadata.len() > MAX_OFFICIAL_PLUGIN_MANIFEST_SIZE {
|
||||
return Err(invalid("official plugin manifest size is invalid"));
|
||||
return Err(invalid("官方插件发行清单大小无效或超出限制"));
|
||||
}
|
||||
fs::read(path)
|
||||
.map(Arc::<[u8]>::from)
|
||||
.map_err(|error| invalid(format!("official plugin manifest cannot be read: {error}")))
|
||||
.map_err(|error| invalid("无法读取官方插件发行清单").with_source(error))
|
||||
}
|
||||
|
||||
fn restore_applied_release(config: &SystemUpdateConfig, cause: OperationError) -> OperationError {
|
||||
match rollback_release(config) {
|
||||
Ok(()) => cause,
|
||||
Err(error) => internal(format!(
|
||||
"release verification failed and restoring the previous files failed: {error}"
|
||||
)),
|
||||
Err(error) => internal("更新校验失败,且无法恢复原文件")
|
||||
.with_source(gateway_core::error::ErrorSource::new(cause).with_cleanup(error)),
|
||||
}
|
||||
}
|
||||
|
||||
struct AppliedReleaseGuard<'a> {
|
||||
config: &'a SystemUpdateConfig,
|
||||
diagnostics: &'a super::diagnostics::Diagnostics,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl<'a> AppliedReleaseGuard<'a> {
|
||||
const fn new(config: &'a SystemUpdateConfig) -> Self {
|
||||
const fn new(
|
||||
config: &'a SystemUpdateConfig,
|
||||
diagnostics: &'a super::diagnostics::Diagnostics,
|
||||
) -> Self {
|
||||
Self {
|
||||
config,
|
||||
diagnostics,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
@@ -605,6 +638,8 @@ impl Drop for AppliedReleaseGuard<'_> {
|
||||
&& let Err(error) = rollback_release(self.config)
|
||||
{
|
||||
tracing::error!(error = %error, "系统发行物临界区取消后的恢复失败");
|
||||
self.diagnostics
|
||||
.record_on_drop("restore_cancelled_installation", None, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,19 +13,15 @@ const DEFAULT_RESTART_DELAY_MS: u64 = 1_200;
|
||||
pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), OperationError> {
|
||||
let executable = config.executable_path()?;
|
||||
let metadata = fs::metadata(&executable)
|
||||
.map_err(|error| internal(format!("failed to schedule replacement process: {error}")))?;
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))?;
|
||||
if !metadata.is_file() {
|
||||
return Err(internal(
|
||||
"failed to schedule replacement process: executable is not a file",
|
||||
));
|
||||
return Err(internal("当前程序不是普通文件,无法在线重启"));
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
if metadata.permissions().mode() & 0o111 == 0 {
|
||||
return Err(internal(
|
||||
"failed to schedule replacement process: executable is not executable",
|
||||
));
|
||||
return Err(internal("当前程序没有执行权限,无法在线重启"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,5 +54,5 @@ pub(crate) fn spawn_replacement(config: &SystemUpdateConfig) -> Result<(), Opera
|
||||
command
|
||||
.spawn()
|
||||
.map(|_| ())
|
||||
.map_err(|error| internal(format!("failed to schedule replacement process: {error}")))
|
||||
.map_err(|error| internal("无法启动替换进程").with_source(error))
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user