mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-10 08:06:03 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ae38dc0c5 | ||
|
|
503e47fac2 | ||
|
|
b4aea0ea09 | ||
|
|
63036a3694 | ||
|
|
a3e24198cd | ||
|
|
fdd124e72f | ||
|
|
3a2ccbc5e6 | ||
|
|
aa0ce52034 | ||
|
|
7e7acad6a3 | ||
|
|
b3dd9b5337 | ||
|
|
0e1c609cd9 | ||
|
|
12771aca87 | ||
|
|
23b85172b5 |
@@ -126,30 +126,32 @@ pub struct RetentionDecision {
|
||||
/// 自动保留规则:
|
||||
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理
|
||||
/// - 自动计划备份满足任一启用阈值即可进入删除
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(全局倒序下标 0)
|
||||
/// - 手动备份不进入本规则;手工/计划记录各自的 `expires_at` 由到期扫描处理
|
||||
/// - `offset` 是当前页在全部 completed 计划备份中的起始位置
|
||||
#[must_use]
|
||||
pub fn decide_retention(
|
||||
retention_days: u32,
|
||||
retention_count: u32,
|
||||
now: DateTime<Utc>,
|
||||
offset: u64,
|
||||
scheduled_completed_desc: &[BackupRecord],
|
||||
) -> Vec<RetentionDecision> {
|
||||
let mut decisions = Vec::new();
|
||||
for (index, record) in scheduled_completed_desc.iter().enumerate() {
|
||||
let position = offset.saturating_add(index as u64);
|
||||
if record.status != BackupStatus::Completed {
|
||||
continue;
|
||||
}
|
||||
// 最近一个成功的计划备份永远保留
|
||||
if index == 0 {
|
||||
if position == 0 {
|
||||
continue;
|
||||
}
|
||||
let expired_by_days = retention_days > 0
|
||||
&& record.completed_at.is_some_and(|completed_at| {
|
||||
completed_at <= now - Duration::days(i64::from(retention_days))
|
||||
});
|
||||
let exceeded_count =
|
||||
retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX);
|
||||
let exceeded_count = retention_count > 0 && position >= u64::from(retention_count);
|
||||
if expired_by_days || exceeded_count {
|
||||
decisions.push(RetentionDecision {
|
||||
record_id: record.id.clone(),
|
||||
|
||||
@@ -38,8 +38,8 @@ use super::policy::{BackupSchedule, decide_retention};
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量
|
||||
const RETENTION_BATCH_SIZE: usize = 10;
|
||||
/// 保留扫描一次读取的 completed 计划备份上限
|
||||
const RETENTION_SCAN_LIMIT: u32 = 1000;
|
||||
/// 保留扫描每页读取的 completed 计划备份数量
|
||||
const RETENTION_PAGE_SIZE: u32 = 1000;
|
||||
/// 每个循环最多完成的待删除记录数量
|
||||
const PENDING_DELETION_BATCH: u32 = 20;
|
||||
|
||||
@@ -112,7 +112,7 @@ impl BackupTask {
|
||||
self.recover_intermediate(cancellation).await?;
|
||||
self.finish_pending_deletions().await?;
|
||||
self.process_next_task(cancellation).await?;
|
||||
self.run_retention_batch(Utc::now()).await?;
|
||||
self.run_retention_batch(Utc::now(), cancellation).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -637,7 +637,11 @@ impl BackupTask {
|
||||
}
|
||||
|
||||
/// 执行一小批到期保留清理:先处理 `expires_at` 已到期的记录,再按天数/份数清理计划备份
|
||||
async fn run_retention_batch(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
async fn run_retention_batch(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
|
||||
// 1. expires_at 已到期的记录(手动或计划)无条件进入删除
|
||||
@@ -659,18 +663,41 @@ impl BackupTask {
|
||||
}
|
||||
|
||||
// 2. 计划备份按 retentionDays / retentionCount 清理
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(RETENTION_SCAN_LIMIT)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let decisions = decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
&records,
|
||||
);
|
||||
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
|
||||
if settings.retention_days == 0 && settings.retention_count == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut offset = 0;
|
||||
let mut decisions = Vec::new();
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(offset, RETENTION_PAGE_SIZE)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let remaining = RETENTION_BATCH_SIZE - decisions.len();
|
||||
decisions.extend(
|
||||
decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
offset,
|
||||
&records,
|
||||
)
|
||||
.into_iter()
|
||||
.take(remaining),
|
||||
);
|
||||
if decisions.len() == RETENTION_BATCH_SIZE
|
||||
|| records.len() < RETENTION_PAGE_SIZE as usize
|
||||
{
|
||||
break;
|
||||
}
|
||||
offset += records.len() as u64;
|
||||
}
|
||||
// 收齐本批候选后再删除,避免删除前页记录导致后续分页跳过数据
|
||||
for decision in decisions {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&decision.record_id, now, None)
|
||||
|
||||
@@ -1352,6 +1352,84 @@ pub struct UsageInsights {
|
||||
pub providers: Vec<ProviderObservation>,
|
||||
}
|
||||
|
||||
/// 单次洞察查询所需的指标,持久化层据此选择计数、分布与费用状态
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum UsageInsightMetric {
|
||||
Health,
|
||||
Latency,
|
||||
FirstToken,
|
||||
Throughput,
|
||||
Load,
|
||||
Cost,
|
||||
Savings,
|
||||
Cache,
|
||||
}
|
||||
|
||||
/// 单个指标的时间点,活动计数与指标共用读取快照
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageMetricPoint<T> {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
pub value: T,
|
||||
}
|
||||
|
||||
/// 各指标复用时间序列形状,汇总与时间点保留各自的类型
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageMetricSeries<S, P = S> {
|
||||
pub summary: S,
|
||||
pub points: Vec<UsageMetricPoint<P>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageThroughputPercentiles {
|
||||
pub p10: Option<u64>,
|
||||
pub p50: Option<u64>,
|
||||
pub p90: Option<u64>,
|
||||
}
|
||||
|
||||
/// 成本卡片各视图共同显示的汇总
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCostSummary {
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub cache_savings: Option<DecimalAmount>,
|
||||
pub tier_premium: Option<DecimalAmount>,
|
||||
pub cost_per_successful_request: Option<DecimalAmount>,
|
||||
pub coverage: CostCoverage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCostComparison {
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub no_cache_cost: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageCacheEfficiency {
|
||||
pub input_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 只发布本次请求的指标,未查询的值不伪装成零
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum UsageInsightData {
|
||||
Health(UsageInsightsHealth),
|
||||
Latency(UsageMetricSeries<LatencyPercentiles>),
|
||||
FirstToken(UsageMetricSeries<LatencyPercentiles>),
|
||||
Throughput(UsageMetricSeries<UsageThroughputPercentiles>),
|
||||
Load(UsageMetricSeries<UsageInsightsLoad>),
|
||||
Cost(UsageMetricSeries<UsageCostSummary, UsageCostComparison>),
|
||||
Savings(UsageMetricSeries<UsageCostSummary, Option<DecimalAmount>>),
|
||||
Cache(UsageMetricSeries<UsageCostSummary, UsageCacheEfficiency>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UsageInsight {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub granularity: Granularity,
|
||||
pub data: UsageInsightData,
|
||||
}
|
||||
|
||||
/// 已完成分母计算的诊断项
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsItem {
|
||||
|
||||
@@ -136,9 +136,10 @@ pub trait BackupRepository: Send + Sync {
|
||||
timezone: &str,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 按完成时间倒序读取 completed 计划备份,供保留策略扫描(最多 `limit` 条)
|
||||
/// 按完成时间及 ID 倒序分页读取 completed 计划备份,offset 相对完整集合
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
}
|
||||
@@ -388,6 +389,7 @@ impl BackupRepository for UnavailableBackupRepository {
|
||||
}
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_offset: u64,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
|
||||
@@ -30,7 +30,8 @@ use crate::model::{
|
||||
DashboardObservation, DashboardQuery, DiagnosticDimension, DiagnosticsObservation,
|
||||
Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange,
|
||||
UsageBillingProjection, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsightsObservation, UsagePage, UsageQuery, UsageSummaryObservation,
|
||||
UsageInsightMetric, UsageInsightsObservation, UsagePage, UsageQuery,
|
||||
UsageSummaryObservation,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult,
|
||||
@@ -456,11 +457,22 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
) -> AdminStoreResult<Vec<RequestMetricPoint>>;
|
||||
|
||||
/// 同一读取快照中的汇总、趋势和已投影情景费用
|
||||
/// 不需要并发时关闭 include_concurrency,返回空并发序列并跳过明细计算
|
||||
async fn usage_insights_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
/// 只读取所选指标及其可见汇总依赖,未选择的字段不供业务层消费
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
metric: UsageInsightMetric,
|
||||
) -> AdminStoreResult<UsageInsightsObservation>;
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage>;
|
||||
|
||||
@@ -194,7 +194,8 @@ impl KeyUsageService for DefaultKeyUsageService {
|
||||
self.observations.usage_insights_observation(
|
||||
query.range,
|
||||
filter,
|
||||
Granularity::for_range(query.range)
|
||||
Granularity::for_range(query.range),
|
||||
false,
|
||||
),
|
||||
self.observations.usage_trend(
|
||||
today,
|
||||
|
||||
@@ -19,11 +19,14 @@ use crate::{
|
||||
DiagnosticDimension, DiagnosticsItem, DiagnosticsResult, Granularity, HealthStatus,
|
||||
HealthTimeline, HealthTimelinePoint, OpsErrorPage, OpsErrorQuery, ProviderBillingInput,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, Trend, TrendKind, TrendPoint,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCalculatedBillingFact,
|
||||
UsageConcurrencyPoint, UsageDetail, UsageFilter, UsageInsights, UsageInsightsCost,
|
||||
UsageInsightsCostPoint, UsageInsightsHealth, UsageInsightsHealthPoint,
|
||||
UsageInsightsLoad, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageSummary,
|
||||
TrendSummary, UsageBilling, UsageBillingProjection, UsageCacheEfficiency,
|
||||
UsageCalculatedBillingFact, UsageConcurrencyPoint, UsageCostComparison,
|
||||
UsageCostSummary, UsageDetail, UsageFilter, UsageInsight, UsageInsightData,
|
||||
UsageInsightMetric, UsageInsights, UsageInsightsCost, UsageInsightsCostPoint,
|
||||
UsageInsightsHealth, UsageInsightsHealthPoint, UsageInsightsLoad,
|
||||
UsageInsightsObservation, UsageInsightsPerformance, UsageInsightsPerformancePoint,
|
||||
UsageMetricPoint, UsageMetricSeries, UsageOverview, UsagePage, UsageQuery,
|
||||
UsageSummary, UsageThroughputPercentiles,
|
||||
},
|
||||
provider_credentials::ProviderQuotaRequest,
|
||||
},
|
||||
@@ -65,6 +68,12 @@ pub trait ObservabilityService: Send + Sync {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> Result<UsageInsights, AdminError>;
|
||||
async fn usage_metric(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
metric: UsageInsightMetric,
|
||||
) -> Result<UsageInsight, AdminError>;
|
||||
async fn diagnostics(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
@@ -234,6 +243,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range,
|
||||
UsageFilter::default(),
|
||||
Granularity::for_range(range),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "dashboard trend"))?;
|
||||
@@ -291,21 +301,25 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let granularity = Granularity::for_range(range);
|
||||
let observation = self
|
||||
.store
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
.usage_insights_observation(range, filter, granularity, true)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage insights"))?;
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs, observation.concurrency)
|
||||
insights_from_observation(observation)
|
||||
}
|
||||
|
||||
async fn usage_metric(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
metric: UsageInsightMetric,
|
||||
) -> Result<UsageInsight, AdminError> {
|
||||
let observation = self
|
||||
.store
|
||||
.usage_metric_observation(range, filter, Granularity::for_range(range), metric)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage metric"))?;
|
||||
let insights = insights_from_observation(observation)?;
|
||||
Ok(select_usage_metric(insights, metric))
|
||||
}
|
||||
|
||||
async fn diagnostics(
|
||||
@@ -382,6 +396,148 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
fn insights_from_observation(
|
||||
observation: UsageInsightsObservation,
|
||||
) -> Result<UsageInsights, AdminError> {
|
||||
let mut standard_costs = UsageCostScenarios::default();
|
||||
for point in observation.costs {
|
||||
if let Some(amount) = point.standard_amount {
|
||||
standard_costs.standard.add(point.bucket_start, amount)?;
|
||||
}
|
||||
if let Some(amount) = point.no_cache_amount {
|
||||
standard_costs.no_cache.add(point.bucket_start, amount)?;
|
||||
}
|
||||
}
|
||||
let overview = observation.overview;
|
||||
let trend = observation.trend;
|
||||
build_usage_insights(overview, trend, standard_costs, observation.concurrency)
|
||||
}
|
||||
|
||||
fn select_usage_metric(insights: UsageInsights, metric: UsageInsightMetric) -> UsageInsight {
|
||||
let activity = insights
|
||||
.health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| (point.bucket_start, point.total_requests))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let data = match metric {
|
||||
UsageInsightMetric::Health => UsageInsightData::Health(insights.health),
|
||||
UsageInsightMetric::Latency => UsageInsightData::Latency(metric_series(
|
||||
insights.performance.latency_percentiles,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.latency_percentiles)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::FirstToken => UsageInsightData::FirstToken(metric_series(
|
||||
insights.performance.first_token_latency_percentiles,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.first_token_latency_percentiles)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Throughput => UsageInsightData::Throughput(metric_series(
|
||||
UsageThroughputPercentiles {
|
||||
p10: insights.performance.output_throughput_p10,
|
||||
p50: insights.performance.output_throughput_p50,
|
||||
p90: insights.performance.output_throughput_p90,
|
||||
},
|
||||
insights.performance.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageThroughputPercentiles {
|
||||
p10: point.output_throughput_p10,
|
||||
p50: point.output_throughput_p50,
|
||||
p90: point.output_throughput_p90,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Load => UsageInsightData::Load(metric_series(
|
||||
insights.performance.load,
|
||||
insights
|
||||
.performance
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.load)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Cost => UsageInsightData::Cost(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights.cost.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageCostComparison {
|
||||
estimated_cost: point.estimated_cost,
|
||||
no_cache_cost: point.no_cache_cost,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Savings => UsageInsightData::Savings(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights
|
||||
.cost
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|point| (point.bucket_start, point.cache_savings)),
|
||||
&activity,
|
||||
)),
|
||||
UsageInsightMetric::Cache => UsageInsightData::Cache(metric_series(
|
||||
cost_summary(&insights.cost),
|
||||
insights.cost.points.into_iter().map(|point| {
|
||||
(
|
||||
point.bucket_start,
|
||||
UsageCacheEfficiency {
|
||||
input_tokens: point.input_tokens,
|
||||
cached_token_rate: point.cached_token_rate,
|
||||
cache_hit_request_rate: point.cache_hit_request_rate,
|
||||
},
|
||||
)
|
||||
}),
|
||||
&activity,
|
||||
)),
|
||||
};
|
||||
UsageInsight {
|
||||
analytics: insights.analytics,
|
||||
granularity: insights.granularity,
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
fn cost_summary(cost: &UsageInsightsCost) -> UsageCostSummary {
|
||||
UsageCostSummary {
|
||||
estimated_cost: cost.estimated_cost.clone(),
|
||||
cache_savings: cost.cache_savings.clone(),
|
||||
tier_premium: cost.tier_premium.clone(),
|
||||
cost_per_successful_request: cost.cost_per_successful_request.clone(),
|
||||
coverage: cost.coverage.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_series<S, P>(
|
||||
summary: S,
|
||||
points: impl Iterator<Item = (DateTime<Utc>, P)>,
|
||||
activity: &BTreeMap<DateTime<Utc>, u64>,
|
||||
) -> UsageMetricSeries<S, P> {
|
||||
UsageMetricSeries {
|
||||
summary,
|
||||
points: points
|
||||
.map(|(bucket_start, value)| UsageMetricPoint {
|
||||
bucket_start,
|
||||
request_count: activity.get(&bucket_start).copied().unwrap_or_default(),
|
||||
value,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_usage_insights(
|
||||
overview: UsageOverview,
|
||||
trend: Vec<RequestMetricPoint>,
|
||||
|
||||
@@ -178,7 +178,8 @@ pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
dashboard: Cache<DashboardQuery, DashboardObservation>,
|
||||
summaries: Cache<UsageKey, UsageSummaryObservation>,
|
||||
insights: Cache<(UsageKey, Granularity), UsageInsightsObservation>,
|
||||
insights: Cache<(UsageKey, Granularity, bool), UsageInsightsObservation>,
|
||||
metrics: Cache<(UsageKey, Granularity, UsageInsightMetric), UsageInsightsObservation>,
|
||||
diagnostics: Cache<(UsageKey, DiagnosticDimension, u16), DiagnosticsObservation>,
|
||||
}
|
||||
impl CachedObservabilityStore {
|
||||
@@ -188,6 +189,7 @@ impl CachedObservabilityStore {
|
||||
dashboard: Cache::default(),
|
||||
summaries: Cache::default(),
|
||||
insights: Cache::default(),
|
||||
metrics: Cache::default(),
|
||||
diagnostics: Cache::default(),
|
||||
}
|
||||
}
|
||||
@@ -237,19 +239,44 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.insights
|
||||
.get(
|
||||
((range, filter.clone()), granularity),
|
||||
((range, filter.clone()), granularity, include_concurrency),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_insights_observation(range, filter, granularity)
|
||||
self.inner.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
include_concurrency,
|
||||
)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
granularity: Granularity,
|
||||
metric: UsageInsightMetric,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.metrics
|
||||
.get(
|
||||
((range, filter.clone()), granularity, metric),
|
||||
self.inner.as_ref(),
|
||||
TTL,
|
||||
|| {
|
||||
self.inner
|
||||
.usage_metric_observation(range, filter, granularity, metric)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
|
||||
@@ -106,7 +106,7 @@ fn retention_count_keeps_newest_n() {
|
||||
Some(base),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(0, 2, now, &records);
|
||||
let decisions = decide_retention(0, 2, now, 0, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["oldest"]);
|
||||
assert_eq!(decisions[0].reason, RetentionReason::Count);
|
||||
@@ -131,7 +131,7 @@ fn retention_days_deletes_old_scheduled_but_keeps_newest() {
|
||||
Some(now - Duration::days(40)),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(30, 0, now, &records);
|
||||
let decisions = decide_retention(30, 0, now, 0, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["ancient"]);
|
||||
}
|
||||
@@ -146,7 +146,7 @@ fn newest_scheduled_never_auto_deleted_even_when_ancient() {
|
||||
now - Duration::days(400),
|
||||
Some(now - Duration::days(400)),
|
||||
)];
|
||||
assert!(decide_retention(30, 1, now, &records).is_empty());
|
||||
assert!(decide_retention(30, 1, now, 0, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -168,7 +168,7 @@ fn disabled_thresholds_delete_nothing() {
|
||||
Some(now - Duration::days(90)),
|
||||
),
|
||||
];
|
||||
assert!(decide_retention(0, 0, now, &records).is_empty());
|
||||
assert!(decide_retention(0, 0, now, 0, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -470,7 +470,8 @@ impl BackupRepository for FakeBackupRepository {
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let mut records = self
|
||||
.records
|
||||
@@ -488,7 +489,11 @@ impl BackupRepository for FakeBackupRepository {
|
||||
.cmp(&a.completed_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
Ok(records)
|
||||
Ok(records
|
||||
.into_iter()
|
||||
.skip(usize::try_from(offset).unwrap_or(usize::MAX))
|
||||
.take(limit as usize)
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -239,6 +239,75 @@ async fn retention_cleans_expired_scheduled_backups() {
|
||||
assert!(object_store.object(&new_seed.object_key).is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_scans_all_pages_and_deletes_only_one_batch_per_cycle() {
|
||||
for (retention_days, retention_count, retained) in [
|
||||
(0, 999, 999),
|
||||
(0, 1000, 1000),
|
||||
(0, 1001, 1001),
|
||||
(30, 0, 1000),
|
||||
] {
|
||||
let mut settings = configured_settings();
|
||||
settings.retention_days = retention_days;
|
||||
settings.retention_count = retention_count;
|
||||
let repository = Arc::new(FakeBackupRepository::new(settings));
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let now = Utc::now();
|
||||
let mut seeds = Vec::new();
|
||||
for index in 0..retained + 15 {
|
||||
let completed_at = if retention_days > 0 && index >= retained {
|
||||
now - Duration::days(40) - Duration::minutes(index)
|
||||
} else {
|
||||
now - Duration::minutes(index)
|
||||
};
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id(&format!("retention-{index}")),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(completed_at),
|
||||
object_key: format!("codex/retention-{index}.dump"),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert scheduled backup");
|
||||
repository.set_completed(&seed.id, completed_at);
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
seeds.push(seed);
|
||||
}
|
||||
let task = BackupTask::new(
|
||||
repository.clone(),
|
||||
Arc::new(FakeDumpPort::new()),
|
||||
object_store.clone(),
|
||||
);
|
||||
|
||||
for deleted in [10, 15] {
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run retention cycle");
|
||||
let records = repository.all_records();
|
||||
assert_eq!(records.len(), seeds.len() - deleted);
|
||||
for (index, seed) in seeds.iter().enumerate() {
|
||||
let retained = usize::try_from(retained).unwrap();
|
||||
let should_remain = index < retained || index >= retained + deleted;
|
||||
assert_eq!(
|
||||
records.iter().any(|record| record.id == seed.id),
|
||||
should_remain,
|
||||
"record {index}, retention days {retention_days}, count {retention_count}"
|
||||
);
|
||||
assert_eq!(
|
||||
object_store.object(&seed.object_key).is_some(),
|
||||
should_remain
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timezone_change_rebases_future_cursor_without_running_old_due_backup() {
|
||||
let mut settings = configured_settings();
|
||||
|
||||
@@ -805,10 +805,27 @@ impl ObservabilityStore for UnavailableStore {
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
_: Granularity,
|
||||
_include_concurrency: bool,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
Err(unavailable("usage insights"))
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, _: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
Err(unavailable("usage records"))
|
||||
}
|
||||
|
||||
@@ -1046,6 +1046,50 @@ async fn dashboard_should_preserve_provider_plan_names() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_and_usage_insights_keep_concurrency_cache_entries_separate() {
|
||||
for dashboard_first in [true, false] {
|
||||
let now = Utc::now();
|
||||
let range = observation_range(now);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
store
|
||||
.concurrency
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(UsageConcurrencyPoint {
|
||||
bucket_start: range.start,
|
||||
average_concurrency: 2.0,
|
||||
peak_concurrency: 3,
|
||||
});
|
||||
let services = observability_services(store.clone()).await;
|
||||
for dashboard in [
|
||||
dashboard_first,
|
||||
!dashboard_first,
|
||||
dashboard_first,
|
||||
!dashboard_first,
|
||||
] {
|
||||
if dashboard {
|
||||
services
|
||||
.observability()
|
||||
.dashboard_trend(range, TrendKind::Usage)
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
let result = services
|
||||
.observability()
|
||||
.usage_insights(range, UsageFilter::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.performance.load.peak_concurrency, 3);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
*store.concurrency_selections.lock().unwrap(),
|
||||
vec![!dashboard_first, dashboard_first]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
yesterday: Mutex<RequestMetrics>,
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
@@ -1053,6 +1097,8 @@ struct FixtureObservabilityStore {
|
||||
projected_costs: Mutex<Vec<UsageProjectedCostPoint>>,
|
||||
concurrency: Mutex<Vec<UsageConcurrencyPoint>>,
|
||||
insights_fail: AtomicBool,
|
||||
concurrency_selections: Mutex<Vec<bool>>,
|
||||
metric_selections: Mutex<Vec<gateway_admin::model::observability::UsageInsightMetric>>,
|
||||
summary_fail: AtomicBool,
|
||||
diagnostics: Mutex<DiagnosticsObservation>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
@@ -1081,6 +1127,8 @@ impl FixtureObservabilityStore {
|
||||
projected_costs: Mutex::new(Vec::new()),
|
||||
concurrency: Mutex::new(Vec::new()),
|
||||
insights_fail: AtomicBool::new(false),
|
||||
concurrency_selections: Mutex::new(Vec::new()),
|
||||
metric_selections: Mutex::new(Vec::new()),
|
||||
summary_fail: AtomicBool::new(false),
|
||||
diagnostics: Mutex::new(DiagnosticsObservation::default()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
@@ -1197,8 +1245,13 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
granularity: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<UsageInsightsObservation> {
|
||||
self.usage_granularities.lock().unwrap().push(granularity);
|
||||
self.concurrency_selections
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(include_concurrency);
|
||||
if self.insights_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection read failed"));
|
||||
}
|
||||
@@ -1210,10 +1263,31 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
.lock()
|
||||
.expect("projected costs")
|
||||
.clone(),
|
||||
concurrency: self.concurrency.lock().expect("concurrency").clone(),
|
||||
concurrency: if include_concurrency {
|
||||
self.concurrency.lock().expect("concurrency").clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.metric_selections.lock().unwrap().push(metric);
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
let items = self.usage_records.lock().expect("usage records").clone();
|
||||
let total = u64::try_from(items.len()).unwrap_or(u64::MAX);
|
||||
@@ -1741,3 +1815,48 @@ async fn insights_cache_should_reject_deleted_building_and_old_generation_times(
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_metric_cache_isolates_selections_and_filters() {
|
||||
use gateway_admin::model::observability::UsageInsightMetric as Metric;
|
||||
let range = observation_range(Utc::now());
|
||||
let store = Arc::new(FixtureObservabilityStore::new(range));
|
||||
let services = observability_services(store.clone()).await;
|
||||
let metrics = [
|
||||
Metric::Health,
|
||||
Metric::Latency,
|
||||
Metric::FirstToken,
|
||||
Metric::Throughput,
|
||||
Metric::Load,
|
||||
Metric::Cost,
|
||||
Metric::Savings,
|
||||
Metric::Cache,
|
||||
];
|
||||
for _ in 0..2 {
|
||||
for metric in metrics {
|
||||
services
|
||||
.observability()
|
||||
.usage_metric(range, UsageFilter::default(), metric)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
assert_eq!(*store.metric_selections.lock().unwrap(), metrics);
|
||||
assert_eq!(
|
||||
*store.concurrency_selections.lock().unwrap(),
|
||||
vec![false, false, false, false, true, false, false, false]
|
||||
);
|
||||
services
|
||||
.observability()
|
||||
.usage_metric(
|
||||
range,
|
||||
UsageFilter {
|
||||
provider_kind: Some("openai".into()),
|
||||
..Default::default()
|
||||
},
|
||||
Metric::Latency,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(store.metric_selections.lock().unwrap().len(), 9);
|
||||
}
|
||||
|
||||
@@ -569,7 +569,8 @@ where
|
||||
.models(&account_id, false)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_models_data(result);
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let data = account_models_data(result, time);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
@@ -607,7 +608,8 @@ where
|
||||
.models(&account_id, true)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_models_data(result);
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let data = account_models_data(result, time);
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,11 @@ pub(super) fn account_refresh_data(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
pub(super) fn account_models_data(
|
||||
result: ProviderModels,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> AccountModelsData {
|
||||
let observed_at = result.observed_at;
|
||||
AccountModelsData {
|
||||
models: result
|
||||
.models
|
||||
@@ -56,6 +60,9 @@ pub(super) fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
observed_at: observed_at.map(|value| time.rfc3339(&value)),
|
||||
observed_at_display: observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| time.datetime(&value)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -722,8 +722,11 @@ pub struct AccountModelView {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountModelsData {
|
||||
pub models: Vec<AccountModelView>,
|
||||
pub observed_at: Option<String>,
|
||||
pub observed_at_display: String,
|
||||
}
|
||||
|
||||
/// 账号模型目录文件;`catalog` 直接作为 Codex `model_catalog_json` 的文件正文落盘
|
||||
|
||||
@@ -1062,22 +1062,6 @@ pub(crate) fn usage_insights_view(
|
||||
insights: domain::UsageInsights,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageInsightsOverviewView {
|
||||
let health_points = insights
|
||||
.health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| OverviewHealthPointView {
|
||||
bucket: point.bucket_start,
|
||||
label: time.label(point.bucket_start, "%m-%d %H:%M"),
|
||||
total_requests: point.total_requests,
|
||||
success_requests: point.success_requests,
|
||||
failed_requests: point.failed_requests,
|
||||
cancelled_requests: point.cancelled_requests,
|
||||
incomplete_requests: point.incomplete_requests,
|
||||
caller_error_requests: point.caller_error_requests,
|
||||
error_rate: point.error_rate,
|
||||
})
|
||||
.collect();
|
||||
let performance_points = insights
|
||||
.performance
|
||||
.points
|
||||
@@ -1128,19 +1112,7 @@ pub(crate) fn usage_insights_view(
|
||||
UsageInsightsOverviewView {
|
||||
analytics: analytics_view(insights.analytics, time),
|
||||
granularity: granularity_name(insights.granularity).to_owned(),
|
||||
health: OverviewHealthView {
|
||||
total_requests: insights.health.total_requests,
|
||||
success_requests: insights.health.success_requests,
|
||||
failed_requests: insights.health.failed_requests,
|
||||
cancelled_requests: insights.health.cancelled_requests,
|
||||
incomplete_requests: insights.health.incomplete_requests,
|
||||
caller_error_requests: insights.health.caller_error_requests,
|
||||
success_rate: insights.health.success_rate,
|
||||
completion_rate: insights.health.completion_rate,
|
||||
request_change_rate: None,
|
||||
success_rate_change: None,
|
||||
points: health_points,
|
||||
},
|
||||
health: overview_health_view(&insights.health, time),
|
||||
performance: OverviewPerformanceView {
|
||||
latency_p50_ms: insights
|
||||
.performance
|
||||
@@ -1236,6 +1208,116 @@ pub(crate) fn usage_insights_view(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_metric_view(
|
||||
insight: domain::UsageInsight,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageMetricView {
|
||||
use domain::UsageInsightData;
|
||||
let data = match insight.data {
|
||||
UsageInsightData::Health(health) => {
|
||||
UsageMetricDataView::Health(overview_health_view(&health, time))
|
||||
}
|
||||
UsageInsightData::Latency(series) => UsageMetricDataView::Latency(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_latency_view,
|
||||
metric_latency_view,
|
||||
)),
|
||||
UsageInsightData::FirstToken(series) => UsageMetricDataView::FirstToken(
|
||||
metric_series_view(series, time, metric_latency_view, metric_latency_view),
|
||||
),
|
||||
UsageInsightData::Throughput(series) => UsageMetricDataView::Throughput(
|
||||
metric_series_view(series, time, metric_throughput_view, metric_throughput_view),
|
||||
),
|
||||
UsageInsightData::Load(series) => UsageMetricDataView::Load(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
|value| overview_load_view(&value),
|
||||
|value| overview_load_view(&value),
|
||||
)),
|
||||
UsageInsightData::Cost(series) => UsageMetricDataView::Cost(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| UsageCostComparisonView {
|
||||
estimated_cost: value.estimated_cost.as_ref().map(ToString::to_string),
|
||||
no_cache_cost: value.no_cache_cost.as_ref().map(ToString::to_string),
|
||||
},
|
||||
)),
|
||||
UsageInsightData::Savings(series) => UsageMetricDataView::Savings(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| value.as_ref().map(ToString::to_string),
|
||||
)),
|
||||
UsageInsightData::Cache(series) => UsageMetricDataView::Cache(metric_series_view(
|
||||
series,
|
||||
time,
|
||||
metric_cost_summary_view,
|
||||
|value| UsageCacheView {
|
||||
input_tokens: value.input_tokens,
|
||||
cached_token_rate: value.cached_token_rate,
|
||||
cache_hit_request_rate: value.cache_hit_request_rate,
|
||||
},
|
||||
)),
|
||||
};
|
||||
UsageMetricView {
|
||||
analytics: analytics_view(insight.analytics, time),
|
||||
granularity: granularity_name(insight.granularity).to_owned(),
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_series_view<S, P, SV, PV>(
|
||||
series: domain::UsageMetricSeries<S, P>,
|
||||
time: crate::time::TimePresenter,
|
||||
summary: impl FnOnce(S) -> SV,
|
||||
point: impl Fn(P) -> PV,
|
||||
) -> UsageMetricSeriesView<SV, PV> {
|
||||
UsageMetricSeriesView {
|
||||
summary: summary(series.summary),
|
||||
points: series
|
||||
.points
|
||||
.into_iter()
|
||||
.map(|value| UsageMetricPointView {
|
||||
bucket: value.bucket_start,
|
||||
label: time.label(value.bucket_start, "%m-%d %H:%M"),
|
||||
request_count: value.request_count,
|
||||
value: point(value.value),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_latency_view(value: domain::LatencyPercentiles) -> UsageLatencyView {
|
||||
UsageLatencyView {
|
||||
p50_ms: value.p50_ms.map(|value| value.as_f64()),
|
||||
p95_ms: value.p95_ms.map(|value| value.as_f64()),
|
||||
p99_ms: value.p99_ms.map(|value| value.as_f64()),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_throughput_view(value: domain::UsageThroughputPercentiles) -> UsageThroughputView {
|
||||
UsageThroughputView {
|
||||
p10: value.p10,
|
||||
p50: value.p50,
|
||||
p90: value.p90,
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_cost_summary_view(value: domain::UsageCostSummary) -> UsageCostSummaryView {
|
||||
UsageCostSummaryView {
|
||||
estimated_cost: value.estimated_cost.as_ref().map(ToString::to_string),
|
||||
cache_savings: value.cache_savings.as_ref().map(ToString::to_string),
|
||||
tier_premium: value.tier_premium.as_ref().map(ToString::to_string),
|
||||
cost_per_successful_request: value
|
||||
.cost_per_successful_request
|
||||
.as_ref()
|
||||
.map(ToString::to_string),
|
||||
coverage: cost_coverage_view(&value.coverage),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn granularity_name(granularity: domain::Granularity) -> &'static str {
|
||||
match granularity {
|
||||
domain::Granularity::FifteenMinutes => "15m",
|
||||
@@ -1322,3 +1404,37 @@ pub(crate) fn analytics_view(
|
||||
histogram_version: metadata.histogram_version,
|
||||
}
|
||||
}
|
||||
|
||||
fn overview_health_view(
|
||||
health: &domain::UsageInsightsHealth,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> OverviewHealthView {
|
||||
let health_points = health
|
||||
.points
|
||||
.iter()
|
||||
.map(|point| OverviewHealthPointView {
|
||||
bucket: point.bucket_start,
|
||||
label: time.label(point.bucket_start, "%m-%d %H:%M"),
|
||||
total_requests: point.total_requests,
|
||||
success_requests: point.success_requests,
|
||||
failed_requests: point.failed_requests,
|
||||
cancelled_requests: point.cancelled_requests,
|
||||
incomplete_requests: point.incomplete_requests,
|
||||
caller_error_requests: point.caller_error_requests,
|
||||
error_rate: point.error_rate,
|
||||
})
|
||||
.collect();
|
||||
OverviewHealthView {
|
||||
total_requests: health.total_requests,
|
||||
success_requests: health.success_requests,
|
||||
failed_requests: health.failed_requests,
|
||||
cancelled_requests: health.cancelled_requests,
|
||||
incomplete_requests: health.incomplete_requests,
|
||||
caller_error_requests: health.caller_error_requests,
|
||||
success_rate: health.success_rate,
|
||||
completion_rate: health.completion_rate,
|
||||
request_change_rate: None,
|
||||
success_rate_change: None,
|
||||
points: health_points,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,71 @@ pub struct UsageAnalyticsQuery {
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
/// 卡片内的指标选择与时间、Provider 范围
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageMetricQuery<T> {
|
||||
pub metric: T,
|
||||
pub provider: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl<T> UsageMetricQuery<T> {
|
||||
pub fn into_parts(self) -> (UsageAnalyticsQuery, T) {
|
||||
(
|
||||
UsageAnalyticsQuery {
|
||||
provider: self.provider,
|
||||
period: self.period,
|
||||
as_of: self.as_of,
|
||||
start_time: self.start_time,
|
||||
end_time: self.end_time,
|
||||
},
|
||||
self.metric,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum UsagePerformanceMetricKind {
|
||||
Latency,
|
||||
FirstToken,
|
||||
Throughput,
|
||||
Load,
|
||||
}
|
||||
|
||||
impl From<UsagePerformanceMetricKind> for domain::UsageInsightMetric {
|
||||
fn from(value: UsagePerformanceMetricKind) -> Self {
|
||||
match value {
|
||||
UsagePerformanceMetricKind::Latency => Self::Latency,
|
||||
UsagePerformanceMetricKind::FirstToken => Self::FirstToken,
|
||||
UsagePerformanceMetricKind::Throughput => Self::Throughput,
|
||||
UsagePerformanceMetricKind::Load => Self::Load,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum UsageCostMetricKind {
|
||||
Cost,
|
||||
Savings,
|
||||
Cache,
|
||||
}
|
||||
|
||||
impl From<UsageCostMetricKind> for domain::UsageInsightMetric {
|
||||
fn from(value: UsageCostMetricKind) -> Self {
|
||||
match value {
|
||||
UsageCostMetricKind::Cost => Self::Cost,
|
||||
UsageCostMetricKind::Savings => Self::Savings,
|
||||
UsageCostMetricKind::Cache => Self::Cache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageQuery {
|
||||
/// 校验 Element Plus 风格的页码分页字段
|
||||
pub fn validate_pagination(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
|
||||
@@ -24,6 +24,12 @@ where
|
||||
"/api/admin/usage/insights/overview",
|
||||
get(usage_insights_overview::<S>),
|
||||
)
|
||||
.route("/api/admin/usage/insights/health", get(usage_health::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/insights/performance",
|
||||
get(usage_performance::<S>),
|
||||
)
|
||||
.route("/api/admin/usage/insights/cost", get(usage_cost::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
get(usage_insights_diagnostics::<S>),
|
||||
@@ -207,6 +213,74 @@ where
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_health<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageAnalyticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
usage_metric_response(state, query, domain::UsageInsightMetric::Health).await
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_performance<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageMetricQuery<UsagePerformanceMetricKind>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let (range, metric) = query.into_parts();
|
||||
usage_metric_response(state, range, metric.into()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_cost<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageMetricQuery<UsageCostMetricKind>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let (range, metric) = query.into_parts();
|
||||
usage_metric_response(state, range, metric.into()).await
|
||||
}
|
||||
|
||||
async fn usage_metric_response<S>(
|
||||
state: S,
|
||||
query: UsageAnalyticsQuery,
|
||||
metric: domain::UsageInsightMetric,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
{
|
||||
let time = crate::time::TimePresenter::new(state.admin_services().timezone());
|
||||
let range = usage_range(
|
||||
query.start_time.as_deref(),
|
||||
query.end_time.as_deref(),
|
||||
query.period.as_deref(),
|
||||
query.as_of,
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
..Default::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_metric(range, filter, metric)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_metric_view(result, time)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_diagnostics<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
|
||||
@@ -759,6 +759,86 @@ pub struct UsageInsightsOverviewView {
|
||||
pub providers: Vec<ProviderOverviewView>,
|
||||
}
|
||||
|
||||
/// 单项洞察响应
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricView {
|
||||
pub analytics: AnalyticsView,
|
||||
pub granularity: String,
|
||||
#[serde(flatten)]
|
||||
pub data: UsageMetricDataView,
|
||||
}
|
||||
|
||||
/// 指标标签与响应字段一一对应,未查询的数据不会出现在响应中
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "metric", content = "data", rename_all = "camelCase")]
|
||||
pub enum UsageMetricDataView {
|
||||
Health(OverviewHealthView),
|
||||
Latency(UsageMetricSeriesView<UsageLatencyView>),
|
||||
FirstToken(UsageMetricSeriesView<UsageLatencyView>),
|
||||
Throughput(UsageMetricSeriesView<UsageThroughputView>),
|
||||
Load(UsageMetricSeriesView<OverviewLoadView>),
|
||||
Cost(UsageMetricSeriesView<UsageCostSummaryView, UsageCostComparisonView>),
|
||||
Savings(UsageMetricSeriesView<UsageCostSummaryView, Option<String>>),
|
||||
Cache(UsageMetricSeriesView<UsageCostSummaryView, UsageCacheView>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricSeriesView<S, P = S> {
|
||||
pub summary: S,
|
||||
pub points: Vec<UsageMetricPointView<P>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageMetricPointView<P> {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub request_count: u64,
|
||||
pub value: P,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageLatencyView {
|
||||
pub p50_ms: Option<f64>,
|
||||
pub p95_ms: Option<f64>,
|
||||
pub p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct UsageThroughputView {
|
||||
pub p10: Option<u64>,
|
||||
pub p50: Option<u64>,
|
||||
pub p90: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCostSummaryView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub cache_savings: Option<String>,
|
||||
pub tier_premium: Option<String>,
|
||||
pub cost_per_successful_request: Option<String>,
|
||||
pub coverage: CostCoverageView,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCostComparisonView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub no_cache_cost: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageCacheView {
|
||||
pub input_tokens: u64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 诊断聚合项目
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
||||
@@ -770,6 +770,21 @@ pub(super) async fn send_control(
|
||||
payload: &str,
|
||||
request_id: &Arc<str>,
|
||||
) -> ForwardOutcome {
|
||||
// HTTP 桥接没有原连接可中断;普通 error 会使 Codex 丢弃仍可正常完成的响应流
|
||||
// 仅在发送前确认无绑定时拒绝中断,不把发送结果不明的失败伪装成明确拒绝
|
||||
if control.binding().is_none()
|
||||
&& let Some(message) = super::interrupt::ResponseInterrupt::unavailable_event(payload)
|
||||
{
|
||||
return if connection
|
||||
.send_text(message, WriteContext::request(request_id, FramePhase::Data))
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
ForwardOutcome::Continue
|
||||
} else {
|
||||
ForwardOutcome::Disconnect
|
||||
};
|
||||
}
|
||||
let result = tokio::select! {
|
||||
_ = connection.wait_for_exit() => return ForwardOutcome::Disconnect,
|
||||
result = control.send(payload) => result,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 当前逻辑响应的中断意图,跨重试连接保留并随新响应 ID 转发
|
||||
//! 当前逻辑响应的中断意图、无控制通道时的拒绝与跨重试转发
|
||||
|
||||
use std::{borrow::Cow, collections::BTreeSet};
|
||||
|
||||
@@ -53,14 +53,12 @@ impl ResponseInterrupt {
|
||||
|
||||
/// 未知控制消息、模式和其他逻辑响应的 ID 仍走原始透明通道
|
||||
pub(super) fn retain(&mut self, payload: &str) -> bool {
|
||||
let Ok(event) = serde_json::from_str::<Value>(payload) else {
|
||||
let Some(event) = interrupt_request(payload) else {
|
||||
return false;
|
||||
};
|
||||
if event["type"] != "response.interrupt"
|
||||
|| event["mode"] != "discard_partial_items"
|
||||
|| !event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| self.delivered_ids.contains(id))
|
||||
if !event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| self.delivered_ids.contains(id))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -69,6 +67,18 @@ impl ResponseInterrupt {
|
||||
true
|
||||
}
|
||||
|
||||
/// 中断失败不代表模型响应失败,客户端仍可收完真实终态后续接
|
||||
pub(super) fn unavailable_event(payload: &str) -> Option<String> {
|
||||
let event = interrupt_request(payload)?;
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"type": "response.interrupt.failed",
|
||||
"response_id": event["response_id"],
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn forward_pending(&mut self, control: &ResponseControl) {
|
||||
if self.sent {
|
||||
return;
|
||||
@@ -89,3 +99,13 @@ impl ResponseInterrupt {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn interrupt_request(payload: &str) -> Option<Value> {
|
||||
let event: Value = serde_json::from_str(payload).ok()?;
|
||||
(event["type"] == "response.interrupt"
|
||||
&& event["mode"] == "discard_partial_items"
|
||||
&& event["response_id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| !id.is_empty()))
|
||||
.then_some(event)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,86 @@ use tower::ServiceExt as _;
|
||||
|
||||
use super::super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_model_reads_and_refreshes_preserve_provider_observation_time() {
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::provider_credentials::{ProviderModel, ProviderModels};
|
||||
use gateway_core::{
|
||||
account::{AccountStatusFacts, CredentialState, QuotaState},
|
||||
routing::UpstreamModelId,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let observed_at: DateTime<Utc> = "2026-09-12T08:00:00Z".parse().unwrap();
|
||||
*fixture.account.lock().unwrap() = Some(account_page_item(
|
||||
AccountStatusFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
observed_at,
|
||||
"api_key",
|
||||
));
|
||||
let router = admin::accounts::router::<AdminTestState>().with_state(fixture.state());
|
||||
for observed_at in [Some(observed_at), None] {
|
||||
*fixture.provider_models.lock().unwrap() = Some(ProviderModels {
|
||||
models: vec![ProviderModel {
|
||||
id: UpstreamModelId::new("model-a").unwrap(),
|
||||
name: "Model A".to_owned(),
|
||||
}],
|
||||
observed_at,
|
||||
});
|
||||
for (method, uri, body) in [
|
||||
(
|
||||
"GET",
|
||||
"/api/admin/accounts/models?accountId=acct_cooldown",
|
||||
"",
|
||||
),
|
||||
(
|
||||
"POST",
|
||||
"/api/admin/accounts/models/refresh",
|
||||
r#"{"accountId":"acct_cooldown"}"#,
|
||||
),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("x-request-id", "req_model_catalog")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK, "{method} {uri}");
|
||||
let body = to_bytes(response.into_body(), 8192).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(
|
||||
value["data"]["models"],
|
||||
json!([{ "id": "model-a", "label": "model-a" }])
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["observedAt"],
|
||||
json!(observed_at.map(|_| "2026-09-12T16:00:00+08:00"))
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["observedAtDisplay"],
|
||||
observed_at.map_or("—", |_| "2026-09-12 16:00:00")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_proxy_binding_is_distinct_from_the_redacted_endpoint() {
|
||||
use chrono::Utc;
|
||||
|
||||
@@ -113,6 +113,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
pub provider_error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
pub provider_quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
pub provider_models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
pub account: Arc<Mutex<Option<AccountPageItem>>>,
|
||||
pub account_usage: Arc<Mutex<Vec<AccountUsage>>>,
|
||||
}
|
||||
@@ -189,16 +190,19 @@ impl AdminTestFixture {
|
||||
plugin_ports.clone(),
|
||||
);
|
||||
let provider_quota = Arc::new(Mutex::new(None));
|
||||
let provider_models = Arc::new(Mutex::new(None));
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new(
|
||||
"openai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
Arc::clone(&provider_models),
|
||||
)),
|
||||
Arc::new(UnusedProvider::new(
|
||||
"xai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
Arc::clone(&provider_models),
|
||||
)),
|
||||
];
|
||||
let bundle = gateway_admin::initialize(
|
||||
@@ -247,6 +251,7 @@ impl AdminTestFixture {
|
||||
dashboard_summary_range,
|
||||
provider_error,
|
||||
provider_quota,
|
||||
provider_models,
|
||||
account,
|
||||
account_usage,
|
||||
}
|
||||
@@ -693,6 +698,7 @@ pub(super) struct MemoryObservations {
|
||||
pub trend: Option<Vec<RequestMetricPoint>>,
|
||||
pub concurrency: Vec<gateway_admin::model::observability::UsageConcurrencyPoint>,
|
||||
pub summaries: Vec<(TimeRange, UsageFilter)>,
|
||||
pub concurrency_selections: Vec<bool>,
|
||||
pub trends: Vec<(TimeRange, UsageFilter)>,
|
||||
pub records: Vec<UsageQuery>,
|
||||
pub errors: Vec<OpsErrorQuery>,
|
||||
@@ -1491,8 +1497,10 @@ impl ObservabilityStore for UnusedStore {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
_: Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
let mut data = self.observations.lock().expect("observations");
|
||||
data.concurrency_selections.push(include_concurrency);
|
||||
data.summaries.push((range, filter.clone()));
|
||||
data.trends.push((range, filter));
|
||||
Ok(
|
||||
@@ -1507,11 +1515,31 @@ impl ObservabilityStore for UnusedStore {
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage trend"))?,
|
||||
costs: Vec::new(),
|
||||
concurrency: data.concurrency.clone(),
|
||||
concurrency: if include_concurrency {
|
||||
data.concurrency.clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: gateway_admin::model::observability::TimeRange,
|
||||
filter: gateway_admin::model::observability::UsageFilter,
|
||||
granularity: gateway_admin::model::observability::Granularity,
|
||||
metric: gateway_admin::model::observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<gateway_admin::model::observability::UsageInsightsObservation> {
|
||||
self.usage_insights_observation(
|
||||
range,
|
||||
filter,
|
||||
granularity,
|
||||
metric == gateway_admin::model::observability::UsageInsightMetric::Load,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageQuery) -> AdminStoreResult<UsagePage> {
|
||||
self.observations
|
||||
.lock()
|
||||
@@ -1584,6 +1612,7 @@ struct UnusedProvider {
|
||||
kind: ProviderKind,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
}
|
||||
|
||||
impl UnusedProvider {
|
||||
@@ -1591,11 +1620,13 @@ impl UnusedProvider {
|
||||
kind: &str,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
models: Arc<Mutex<Option<ProviderModels>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
error,
|
||||
quota,
|
||||
models,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1706,7 +1737,11 @@ impl ProviderAdmin for UnusedProvider {
|
||||
_: &ProviderAccountId,
|
||||
_: bool,
|
||||
) -> Result<ProviderModels, ProviderAdminError> {
|
||||
Err(unsupported_provider())
|
||||
self.models
|
||||
.lock()
|
||||
.expect("provider models")
|
||||
.clone()
|
||||
.ok_or_else(unsupported_provider)
|
||||
}
|
||||
|
||||
async fn export_credentials(
|
||||
|
||||
@@ -1382,6 +1382,157 @@ async fn usage_insights_maps_load_summary_and_points_to_camel_case() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_card_routes_return_only_the_selected_metric_and_validate_options() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Duration;
|
||||
use gateway_admin::model::observability::{
|
||||
AttemptMetrics, CurrencyCost, Granularity, LatencyPercentiles, PercentileMilliseconds,
|
||||
RequestMetricPoint, RequestMetrics, TimeRange, UsageConcurrencyPoint, UsageOverview,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let range = TimeRange::new(now - Duration::minutes(15), now).unwrap();
|
||||
let metrics = RequestMetrics {
|
||||
request_count: 10,
|
||||
success_count: 8,
|
||||
failure_count: 2,
|
||||
latency_percentiles: LatencyPercentiles {
|
||||
p95_ms: Some(PercentileMilliseconds::new(43.5).unwrap()),
|
||||
..Default::default()
|
||||
},
|
||||
first_token_latency_percentiles: LatencyPercentiles {
|
||||
p95_ms: Some(PercentileMilliseconds::new(17.25).unwrap()),
|
||||
..Default::default()
|
||||
},
|
||||
output_throughput_p50: Some(22),
|
||||
..Default::default()
|
||||
};
|
||||
{
|
||||
let mut observations = fixture.observations.lock().unwrap();
|
||||
observations.summary = Some(UsageOverview {
|
||||
analytics: Default::default(),
|
||||
range,
|
||||
requests: metrics.clone(),
|
||||
attempts: AttemptMetrics {
|
||||
costs: vec![CurrencyCost {
|
||||
currency: "USD".into(),
|
||||
amount: "12.5".parse().unwrap(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
providers: Vec::new(),
|
||||
});
|
||||
observations.trend = Some(vec![RequestMetricPoint {
|
||||
bucket_start: range.start,
|
||||
granularity: Granularity::FifteenMinutes,
|
||||
metrics,
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
}]);
|
||||
observations.concurrency = vec![UsageConcurrencyPoint {
|
||||
bucket_start: range.start,
|
||||
average_concurrency: 1.25,
|
||||
peak_concurrency: 3,
|
||||
}];
|
||||
}
|
||||
let router = observability::router::<AdminTestState>().with_state(fixture.state());
|
||||
for (path, metric, field, expected) in [
|
||||
("health", "health", "totalRequests", json!(10)),
|
||||
(
|
||||
"performance?metric=latency",
|
||||
"latency",
|
||||
"p95Ms",
|
||||
json!(43.5),
|
||||
),
|
||||
(
|
||||
"performance?metric=firstToken",
|
||||
"firstToken",
|
||||
"p95Ms",
|
||||
json!(17.25),
|
||||
),
|
||||
(
|
||||
"performance?metric=throughput",
|
||||
"throughput",
|
||||
"p50",
|
||||
json!(22),
|
||||
),
|
||||
(
|
||||
"performance?metric=load",
|
||||
"load",
|
||||
"peakConcurrency",
|
||||
json!(3),
|
||||
),
|
||||
("cost?metric=cost", "cost", "estimatedCost", json!("12.5")),
|
||||
(
|
||||
"cost?metric=savings",
|
||||
"savings",
|
||||
"estimatedCost",
|
||||
json!("12.5"),
|
||||
),
|
||||
("cost?metric=cache", "cache", "estimatedCost", json!("12.5")),
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/usage/insights/{path}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_card_metric")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{path}: {value}");
|
||||
let result = &value["data"];
|
||||
assert_eq!(result["metric"], metric);
|
||||
assert_eq!(result.as_object().unwrap().len(), 4);
|
||||
let data = &result["data"];
|
||||
if metric == "health" {
|
||||
assert_eq!(data[field], expected);
|
||||
} else {
|
||||
assert_eq!(data["summary"][field], expected, "{path}");
|
||||
assert_eq!(data["points"][0]["requestCount"], 10);
|
||||
assert_eq!(data.as_object().unwrap().len(), 2);
|
||||
}
|
||||
}
|
||||
for path in [
|
||||
"performance",
|
||||
"cost",
|
||||
"performance?metric=cost",
|
||||
"cost?metric=load",
|
||||
"health?metric=load",
|
||||
"performance?metric=unknown",
|
||||
"cost?metric=cache&status=error",
|
||||
] {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/usage/insights/{path}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_card_metric")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observability_queries_reject_group_filters_and_analytics_search() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
@@ -218,6 +218,7 @@ async fn overview_scopes_every_query_and_projects_only_key_visible_fields() {
|
||||
assert!(!data.to_string().contains("private-sentinel"));
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
assert_eq!(observations.summaries.len(), 1);
|
||||
assert_eq!(observations.concurrency_selections, vec![false]);
|
||||
assert_eq!(observations.trends.len(), 2);
|
||||
for (_, filter) in observations
|
||||
.summaries
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证未知控制消息、迟到中断、结算期间收发和创建请求的串行准入
|
||||
//! 验证控制消息透传、HTTP 中断拒绝、结算期间收发和创建请求的串行准入
|
||||
|
||||
use super::*;
|
||||
use gateway_core::engine::client_retry::{ClientRetryKey, ClientRetryStore};
|
||||
@@ -244,7 +244,9 @@ impl ResponseControlTransport for ControlTransport {
|
||||
#[derive(Default)]
|
||||
struct InterruptProvider {
|
||||
calls: AtomicUsize,
|
||||
store: Arc<SettlementPorts>,
|
||||
fail_first: bool,
|
||||
http_only: bool,
|
||||
retry_started: tokio::sync::Notify,
|
||||
retry_release: tokio::sync::Notify,
|
||||
transport: Mutex<Option<Arc<ControlTransport>>>,
|
||||
@@ -284,7 +286,9 @@ impl Provider for InterruptProvider {
|
||||
let id = format!("resp_interrupt_{number}");
|
||||
let transport = Arc::new(ControlTransport::new(id));
|
||||
let owner: Arc<dyn ResponseControlTransport> = transport.clone();
|
||||
context.response_control().unwrap().bind(&owner);
|
||||
if !self.http_only {
|
||||
context.response_control().unwrap().bind(&owner);
|
||||
}
|
||||
*self.transport.lock().unwrap() = Some(transport.clone());
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
@@ -296,7 +300,12 @@ impl Provider for InterruptProvider {
|
||||
"acct_api_test"
|
||||
})
|
||||
.unwrap(),
|
||||
UpstreamTransport::new("websocket").unwrap(),
|
||||
UpstreamTransport::new(if self.http_only {
|
||||
"http_sse"
|
||||
} else {
|
||||
"websocket"
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
let body = futures::stream::unfold(
|
||||
(0, transport, self),
|
||||
@@ -457,7 +466,7 @@ async fn connect(
|
||||
} else {
|
||||
snapshot("sk_ws_interrupt", "openai")
|
||||
}),
|
||||
Arc::new(SettlementPorts::default()),
|
||||
provider.store.clone(),
|
||||
ProviderRegistry::new([provider as Arc<dyn Provider>]).unwrap(),
|
||||
admissions,
|
||||
Arc::new(UnusedContinuation),
|
||||
@@ -489,6 +498,140 @@ fn interrupt(id: &str, mode: &str) -> Value {
|
||||
json!({"type":"response.interrupt","response_id":id,"mode":mode,"future_extension":true})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_response_rejects_interrupt_without_aborting_or_admitting_queued_requests() {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
http_only: true,
|
||||
..Default::default()
|
||||
});
|
||||
let admissions = Arc::new(InterruptAdmissions::default());
|
||||
let (mut socket, _server) = connect(provider.clone(), admissions.clone()).await;
|
||||
let create = json!({"type":"response.create","model":"model-a","input":"hello"});
|
||||
send(&mut socket, create.clone()).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "response.created");
|
||||
send(&mut socket, create).await;
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_interrupt_0", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await,
|
||||
json!({
|
||||
"type":"response.interrupt.failed", "response_id":"resp_interrupt_0"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(admissions.active.load(Ordering::SeqCst), 1);
|
||||
assert!(
|
||||
provider
|
||||
.transport
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.requests
|
||||
.lock()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
provider.complete.notify_one();
|
||||
let completed = next(&mut socket).await;
|
||||
assert_eq!(completed["type"], "response.completed");
|
||||
assert_eq!(completed["response"]["id"], "resp_interrupt_0");
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_1"
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
provider.store.finalizations.lock().unwrap()[0].outcome,
|
||||
gateway_core::engine::ExecutionOutcome::Succeeded
|
||||
);
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_response_rejects_late_interrupt_during_settlement_and_when_idle() {
|
||||
let provider = Arc::new(InterruptProvider {
|
||||
http_only: true,
|
||||
..Default::default()
|
||||
});
|
||||
let (release, wait_for_release) = tokio::sync::oneshot::channel();
|
||||
let admissions = Arc::new(InterruptAdmissions {
|
||||
release_gate: Mutex::new(Some(wait_for_release)),
|
||||
..Default::default()
|
||||
});
|
||||
let (mut socket, _server) = connect(provider.clone(), admissions.clone()).await;
|
||||
let create = json!({"type":"response.create","model":"model-a","input":"hello"});
|
||||
send(&mut socket, create.clone()).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "response.created");
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
admissions.release_started.notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let control = interrupt("resp_interrupt_0", "discard_partial_items");
|
||||
let rejection = json!({"type":"response.interrupt.failed","response_id":"resp_interrupt_0"});
|
||||
send(&mut socket, control.clone()).await;
|
||||
assert_eq!(next(&mut socket).await, rejection);
|
||||
assert_eq!(admissions.active.load(Ordering::SeqCst), 1);
|
||||
release.send(()).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while admissions.active.load(Ordering::SeqCst) != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
send(&mut socket, control).await;
|
||||
assert_eq!(next(&mut socket).await, rejection);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
|
||||
send(&mut socket, create).await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await["response"]["id"],
|
||||
"resp_interrupt_1"
|
||||
);
|
||||
provider.complete.notify_one();
|
||||
assert_eq!(next(&mut socket).await["type"], "response.completed");
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_control_channel_keeps_unknown_and_invalid_controls_as_errors() {
|
||||
let provider = Arc::new(InterruptProvider::default());
|
||||
let (mut socket, _server) =
|
||||
connect(provider.clone(), Arc::new(InterruptAdmissions::default())).await;
|
||||
for control in [
|
||||
interrupt("resp_unknown", "future_mode"),
|
||||
interrupt("", "discard_partial_items"),
|
||||
json!({"type":"response.interrupt", "mode":"discard_partial_items"}),
|
||||
json!({"type":"future.control"}),
|
||||
] {
|
||||
send(&mut socket, control).await;
|
||||
assert_eq!(next(&mut socket).await["type"], "error");
|
||||
}
|
||||
send(
|
||||
&mut socket,
|
||||
interrupt("resp_unknown", "discard_partial_items"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
next(&mut socket).await,
|
||||
json!({
|
||||
"type":"response.interrupt.failed", "response_id":"resp_unknown"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider.calls.load(Ordering::SeqCst), 0);
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_interrupt_after_terminal_is_sent_upstream_and_keeps_the_connection_reusable() {
|
||||
let provider = Arc::new(InterruptProvider::default());
|
||||
|
||||
@@ -168,14 +168,18 @@ impl BackupObjectStorePort for S3ObjectStoreAdapter {
|
||||
.await
|
||||
{
|
||||
Ok(get) => {
|
||||
let received = get
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|_| {
|
||||
BackupError::new(code::UPSTREAM_FAILURE, "读取探针对象响应失败".to_owned())
|
||||
})?
|
||||
.into_bytes();
|
||||
let received = match get.body.collect().await {
|
||||
Ok(body) => body.into_bytes(),
|
||||
Err(_) => {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: ConnectionTestStage::GetObject.as_str(),
|
||||
code: Some(code::UPSTREAM_FAILURE),
|
||||
message: "读取探针对象响应失败".to_owned(),
|
||||
});
|
||||
}
|
||||
};
|
||||
if received.as_ref() != body.as_slice() {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(ConnectionTestResult {
|
||||
|
||||
@@ -657,6 +657,7 @@ impl BackupRepository for PgBackupRepository {
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
offset: u64,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let limit = limit.clamp(1, 10000);
|
||||
@@ -667,9 +668,13 @@ impl BackupRepository for PgBackupRepository {
|
||||
from backup_records
|
||||
where trigger_kind = 'scheduled' and status = 'completed'
|
||||
order by completed_at desc, id desc
|
||||
limit $1",
|
||||
limit $1 offset $2",
|
||||
)
|
||||
.bind(i64::from(limit))
|
||||
.bind(
|
||||
i64::try_from(offset)
|
||||
.map_err(|source| store_invalid("page offset").with_source(source))?,
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|source| {
|
||||
|
||||
@@ -287,6 +287,7 @@ impl PgObservabilityRepository {
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> StoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.query_budget
|
||||
.run(
|
||||
@@ -295,7 +296,13 @@ impl PgObservabilityRepository {
|
||||
|mut connection| async move {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
|
||||
let (mut overview, trend, costs, concurrency) = snapshot
|
||||
.insights(range, &filter, granularity, self.timezone)
|
||||
.insights(
|
||||
range,
|
||||
&filter,
|
||||
granularity,
|
||||
self.timezone,
|
||||
include_concurrency,
|
||||
)
|
||||
.await?;
|
||||
overview.analytics = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
@@ -310,6 +317,30 @@ impl PgObservabilityRepository {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn usage_metric_observation(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
metric: admin_observability::UsageInsightMetric,
|
||||
) -> StoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.query_budget
|
||||
.run(
|
||||
&self.pool,
|
||||
"load usage metric snapshot",
|
||||
|mut connection| async move {
|
||||
let mut snapshot = analytics::AnalyticsSnapshot::new(&mut connection).await?;
|
||||
let mut observation = snapshot
|
||||
.metric(range, &filter, granularity, self.timezone, metric)
|
||||
.await?;
|
||||
observation.overview.analytics = snapshot.metadata();
|
||||
snapshot.finish().await?;
|
||||
Ok(observation)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn provider_account_usage(
|
||||
&self,
|
||||
query: ProviderAccountUsageQuery,
|
||||
@@ -498,12 +529,32 @@ impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
include_concurrency: bool,
|
||||
) -> AdminStoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.repository
|
||||
.usage_insights_observation(
|
||||
store_range(range).map_err(observability_error)?,
|
||||
store_usage_filter(filter),
|
||||
granularity,
|
||||
include_concurrency,
|
||||
)
|
||||
.await
|
||||
.map_err(observability_error)
|
||||
}
|
||||
|
||||
async fn usage_metric_observation(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
granularity: admin_observability::Granularity,
|
||||
metric: admin_observability::UsageInsightMetric,
|
||||
) -> AdminStoreResult<admin_observability::UsageInsightsObservation> {
|
||||
self.repository
|
||||
.usage_metric_observation(
|
||||
store_range(range).map_err(observability_error)?,
|
||||
store_usage_filter(filter),
|
||||
granularity,
|
||||
metric,
|
||||
)
|
||||
.await
|
||||
.map_err(observability_error)
|
||||
|
||||
@@ -335,6 +335,7 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
include_concurrency: bool,
|
||||
) -> StoreResult<(
|
||||
UsageOverview,
|
||||
Vec<RequestMetricPoint>,
|
||||
@@ -345,19 +346,13 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
.load(range, filter, None, Some(granularity), timezone, true)
|
||||
.await?;
|
||||
let trend = metric_points(range, granularity, timezone, &states, true)?;
|
||||
let costs = states
|
||||
.iter()
|
||||
.map(|((bucket_start, _), state)| {
|
||||
Ok(admin_observability::UsageProjectedCostPoint {
|
||||
bucket_start: *bucket_start,
|
||||
standard_amount: state.projected_cost("standard_usd")?,
|
||||
no_cache_amount: state.projected_cost("no_cache_usd")?,
|
||||
})
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let concurrency = self
|
||||
.concurrency(range, filter, granularity, timezone)
|
||||
.await?;
|
||||
let costs = projected_cost_points(&states)?;
|
||||
let concurrency = if include_concurrency {
|
||||
self.concurrency(range, filter, granularity, timezone)
|
||||
.await?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
// 趋势已包含可合并的完整指标和情景费用,概览直接复用同一份状态
|
||||
// 精确分位数不可跨桶合并,明细回退时仍按完整区间单独计算概览
|
||||
let overview = if states.values().any(MetricState::has_exact_percentiles) {
|
||||
@@ -369,6 +364,107 @@ impl<'a> AnalyticsSnapshot<'a> {
|
||||
Ok((overview, trend, costs, concurrency))
|
||||
}
|
||||
|
||||
pub async fn metric(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
timezone: gateway_core::time::DeploymentTimeZone,
|
||||
metric: admin_observability::UsageInsightMetric,
|
||||
) -> StoreResult<admin_observability::UsageInsightsObservation> {
|
||||
use admin_observability::UsageInsightMetric;
|
||||
let distribution_names: &[&str] = match metric {
|
||||
UsageInsightMetric::Latency => &["latency_ms"],
|
||||
UsageInsightMetric::FirstToken => &["first_token_ms"],
|
||||
UsageInsightMetric::Throughput => &["throughput"],
|
||||
UsageInsightMetric::Load => &["admission_decision_ms", "account_selection_wait_ms"],
|
||||
_ => &[],
|
||||
};
|
||||
let costs = matches!(
|
||||
metric,
|
||||
UsageInsightMetric::Cost | UsageInsightMetric::Savings | UsageInsightMetric::Cache
|
||||
);
|
||||
let counters: &[&str] = match metric {
|
||||
UsageInsightMetric::Health => &[
|
||||
"request_count",
|
||||
"success_count",
|
||||
"failure_count",
|
||||
"cancelled_count",
|
||||
"incomplete_count",
|
||||
"caller_error_count",
|
||||
],
|
||||
// 成本视图均显示费用、节省、溢价、单次成功费用与覆盖率汇总
|
||||
UsageInsightMetric::Cost | UsageInsightMetric::Savings => &[
|
||||
"request_count",
|
||||
"success_count",
|
||||
"provider_reported_count",
|
||||
"calculated_count",
|
||||
"unavailable_count",
|
||||
],
|
||||
UsageInsightMetric::Cache => &[
|
||||
"request_count",
|
||||
"success_count",
|
||||
"input_tokens",
|
||||
"cached_input_tokens",
|
||||
"cache_hit_request_count",
|
||||
"cache_eligible_request_count",
|
||||
"provider_reported_count",
|
||||
"calculated_count",
|
||||
"unavailable_count",
|
||||
],
|
||||
_ => &["request_count"],
|
||||
};
|
||||
let selection = ReadSelection {
|
||||
distributions: if distribution_names.is_empty() {
|
||||
DistributionRead::None
|
||||
} else {
|
||||
DistributionRead::Full
|
||||
},
|
||||
costs,
|
||||
keys: None,
|
||||
counters: Some(counters),
|
||||
distribution_names: Some(distribution_names),
|
||||
};
|
||||
let states = self
|
||||
.load_selected(range, filter, None, Some(granularity), timezone, selection)
|
||||
.await?;
|
||||
let trend = metric_points(range, granularity, timezone, &states, costs)?;
|
||||
let projected_costs = if costs {
|
||||
projected_cost_points(&states)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
// 明细筛选的精确分位数仍需按完整范围读取,不能合并各桶分位数
|
||||
let total = if states.values().any(MetricState::has_exact_percentiles) {
|
||||
merge_states(
|
||||
self.load_selected(range, filter, None, None, timezone, selection)
|
||||
.await?
|
||||
.into_values(),
|
||||
)?
|
||||
} else {
|
||||
merge_states(states.into_values())?
|
||||
};
|
||||
let concurrency = if metric == UsageInsightMetric::Load {
|
||||
self.concurrency(range, filter, granularity, timezone)
|
||||
.await?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let overview = UsageOverview {
|
||||
range,
|
||||
requests: total.request_metrics()?,
|
||||
attempts: total.attempts()?,
|
||||
providers: Vec::new(),
|
||||
analytics: self.metadata(),
|
||||
};
|
||||
Ok(admin_observability::UsageInsightsObservation {
|
||||
overview: admin_usage_overview(overview),
|
||||
trend: trend.into_iter().map(admin_request_metric_point).collect(),
|
||||
costs: projected_costs,
|
||||
concurrency,
|
||||
})
|
||||
}
|
||||
|
||||
async fn concurrency(
|
||||
&mut self,
|
||||
range: ObservabilityRange,
|
||||
@@ -1072,6 +1168,21 @@ pub(super) fn push_summary_buckets(
|
||||
}
|
||||
}
|
||||
|
||||
fn projected_cost_points(
|
||||
states: &BTreeMap<(DateTime<Utc>, String), MetricState>,
|
||||
) -> StoreResult<Vec<admin_observability::UsageProjectedCostPoint>> {
|
||||
states
|
||||
.iter()
|
||||
.map(|((bucket_start, _), state)| {
|
||||
Ok(admin_observability::UsageProjectedCostPoint {
|
||||
bucket_start: *bucket_start,
|
||||
standard_amount: state.projected_cost("standard_usd")?,
|
||||
no_cache_amount: state.projected_cost("no_cache_usd")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn metric_points(
|
||||
range: ObservabilityRange,
|
||||
granularity: ObservationGranularity,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证 S3 分片完整读取、并发上限与取消和失败时的请求回收
|
||||
//! 验证 S3 连接探针清理、分片完整读取、并发上限与取消和失败时的请求回收
|
||||
|
||||
use std::{io, time::Duration};
|
||||
|
||||
@@ -456,3 +456,83 @@ async fn failed_multipart_retains_abort_failure_without_changing_primary_code()
|
||||
assert!(!error.message().contains("probe error"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_probe_attempts_cleanup_after_an_interrupted_get_body() {
|
||||
for delete_allowed in [true, false] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let storage = BackupStorageConfig {
|
||||
storage_revision: 1,
|
||||
endpoint: format!("http://{}", listener.local_addr().unwrap()),
|
||||
region: "auto".into(),
|
||||
bucket: "backup-bucket".into(),
|
||||
access_key_id: "test-access-key".into(),
|
||||
secret_access_key: SecretString::from("test-secret-key"),
|
||||
prefix: "backups".into(),
|
||||
force_path_style: true,
|
||||
};
|
||||
let mut probe =
|
||||
tokio::spawn(
|
||||
async move { S3ObjectStoreAdapter::new().test_connection(&storage).await },
|
||||
);
|
||||
let (mut put, request, body) = accept_request(&listener).await;
|
||||
assert!(request.starts_with("PUT "));
|
||||
let object_path = request
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap()
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.split('?')
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
respond(&mut put, &xml_response("")).await;
|
||||
let (mut head, request, _) = accept_request(&listener).await;
|
||||
assert!(request.starts_with("HEAD "));
|
||||
respond(&mut head, "HTTP/1.1 200 OK\r\ncontent-length: 64\r\nx-amz-meta-probe: backup-connection-test\r\nconnection: close\r\n\r\n").await;
|
||||
let (mut get, request, _) = accept_request(&listener).await;
|
||||
assert!(request.starts_with("GET "));
|
||||
get.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 64\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
get.write_all(&body[..32]).await.unwrap();
|
||||
get.shutdown().await.unwrap();
|
||||
drop(get);
|
||||
let (mut delete, request, _) = tokio::select! {
|
||||
result = &mut probe => panic!("probe returned before cleaning up the uploaded object: {result:?}"),
|
||||
request = accept_request(&listener) => request,
|
||||
};
|
||||
assert!(request.starts_with("DELETE "));
|
||||
let delete_path = request
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap()
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.split('?')
|
||||
.next()
|
||||
.unwrap();
|
||||
assert_eq!(delete_path, object_path);
|
||||
respond(
|
||||
&mut delete,
|
||||
&if delete_allowed {
|
||||
xml_response("")
|
||||
} else {
|
||||
error_response("403 Forbidden", "AccessDenied")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = tokio::time::timeout(Duration::from_secs(3), probe)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.expect("body failure is a staged connection test result");
|
||||
assert!(!result.ok);
|
||||
assert_eq!(result.stage, "getObject");
|
||||
assert_eq!(result.code, Some(code::UPSTREAM_FAILURE));
|
||||
assert_eq!(result.message, "读取探针对象响应失败");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +144,7 @@ async fn usage_summary_preserves_counts_latency_samples_and_costs_across_read_pa
|
||||
range,
|
||||
filter,
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -233,6 +234,7 @@ async fn usage_summary_preserves_counts_latency_samples_and_costs_across_read_pa
|
||||
full,
|
||||
exact_filter,
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -302,6 +304,7 @@ async fn provider_counts_match_for_rollups_boundaries_and_filtered_facts() {
|
||||
range,
|
||||
filter.clone(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -411,6 +414,7 @@ async fn insights_share_bucket_states_without_merging_exact_percentiles() {
|
||||
range,
|
||||
filter.clone(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -431,7 +435,7 @@ async fn insights_share_bucket_states_without_merging_exact_percentiles() {
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
] {
|
||||
let observation = repo
|
||||
.usage_insights_observation(range, filter.clone(), granularity)
|
||||
.usage_insights_observation(range, filter.clone(), granularity, true)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(observation.overview.requests, expected.requests);
|
||||
@@ -1110,6 +1114,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
|
||||
range,
|
||||
Default::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1122,6 +1127,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
|
||||
..Default::default()
|
||||
},
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1156,6 +1162,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
|
||||
ObservabilityRange::new(start, start + TimeDelta::minutes(10)).unwrap(),
|
||||
Default::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1170,6 +1177,7 @@ async fn persisted_histograms_merge_long_tails_and_distinguish_empty_from_zero()
|
||||
.unwrap(),
|
||||
Default::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1775,9 +1783,37 @@ async fn insights_concurrency_counts_logical_request_overlap_once_per_bucket() {
|
||||
..Default::default()
|
||||
},
|
||||
granularity,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let without_concurrency = repo
|
||||
.usage_insights_observation(
|
||||
ObservabilityRange::new(
|
||||
start + TimeDelta::minutes(from),
|
||||
start + TimeDelta::minutes(to),
|
||||
)
|
||||
.unwrap(),
|
||||
UsageRecordFilter {
|
||||
provider_kind: provider.map(str::to_owned),
|
||||
..Default::default()
|
||||
},
|
||||
granularity,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(without_concurrency.concurrency.is_empty());
|
||||
assert_eq!(
|
||||
without_concurrency.overview.requests,
|
||||
insights.overview.requests
|
||||
);
|
||||
assert_eq!(
|
||||
without_concurrency.overview.attempts,
|
||||
insights.overview.attempts
|
||||
);
|
||||
assert_eq!(without_concurrency.trend, insights.trend);
|
||||
assert_eq!(without_concurrency.costs, insights.costs);
|
||||
assert_eq!(insights.concurrency.len(), expected.len());
|
||||
for (point, (average, peak)) in insights.concurrency.iter().zip(expected) {
|
||||
assert!(
|
||||
@@ -1823,6 +1859,7 @@ async fn insights_concurrency_weights_calendar_days_across_dst() {
|
||||
..Default::default()
|
||||
},
|
||||
gateway_admin::model::observability::Granularity::Day,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1874,6 +1911,7 @@ async fn sql_edge_histograms_preserve_repeated_samples_and_power_of_two_boundari
|
||||
range,
|
||||
Default::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -1885,6 +1923,7 @@ async fn sql_edge_histograms_preserve_repeated_samples_and_power_of_two_boundari
|
||||
range,
|
||||
Default::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
|
||||
@@ -78,6 +78,63 @@ async fn enable_verified_schedule(repository: &PgBackupRepository) -> BackupSett
|
||||
.expect("enable schedule")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduled_completed_pages_keep_stable_order_and_exclude_other_records() {
|
||||
let Some(db) = TestDatabase::create("backup_retention_pages").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgBackupRepository::new(db.pool.clone());
|
||||
let now = Utc::now();
|
||||
sqlx::query(
|
||||
"insert into backup_records (
|
||||
id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
error_code, error_message, started_at, completed_at, created_at, updated_at
|
||||
)
|
||||
select 'backup_' || lpad(to_hex(n), 32, '0'),
|
||||
case when n = 1006 then 'manual' else 'scheduled' end,
|
||||
case when n = 1007 then 'failed' else 'completed' end,
|
||||
case when n = 1006 then null else $1 - n * interval '1 minute' end,
|
||||
'codex/' || n || '.dump', 1, repeat('a', 64),
|
||||
case when n = 1007 then 'backup.pg_dump_failed' end,
|
||||
case when n = 1007 then '导出失败' end,
|
||||
$1 - interval '2 days', $1, $1 - interval '2 days', $1
|
||||
from generate_series(1, 1007) as n",
|
||||
)
|
||||
.bind(now)
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.expect("seed completed backups with tied completion times");
|
||||
|
||||
let first = repository
|
||||
.list_scheduled_completed_desc(0, 1000)
|
||||
.await
|
||||
.expect("first page");
|
||||
let second = repository
|
||||
.list_scheduled_completed_desc(1000, 1000)
|
||||
.await
|
||||
.expect("second page");
|
||||
assert_eq!(first.len(), 1000);
|
||||
assert_eq!(second.len(), 5);
|
||||
let ids: Vec<_> = first
|
||||
.iter()
|
||||
.chain(&second)
|
||||
.map(|record| record.id.clone())
|
||||
.collect();
|
||||
let expected: Vec<_> = (1..=1005)
|
||||
.rev()
|
||||
.map(|index| format!("backup_{index:032x}"))
|
||||
.collect();
|
||||
assert_eq!(ids, expected);
|
||||
assert!(
|
||||
repository
|
||||
.list_scheduled_completed_desc(1005, 1000)
|
||||
.await
|
||||
.expect("past final page")
|
||||
.is_empty()
|
||||
);
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn storage_rotation_pauses_schedule_and_rejects_stale_verification() {
|
||||
let Some(db) = TestDatabase::create("backup_rotation").await else {
|
||||
|
||||
@@ -2036,6 +2036,7 @@ async fn zero_attempt_failure_does_not_enter_successful_usage_or_cost_aggregates
|
||||
zero_attempt_range(&request),
|
||||
UsageRecordFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2055,6 +2056,7 @@ async fn zero_attempt_failure_does_not_enter_successful_usage_or_cost_aggregates
|
||||
zero_attempt_range(&request),
|
||||
UsageRecordFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2083,6 +2085,7 @@ async fn zero_attempt_failure_does_not_enter_successful_usage_or_cost_aggregates
|
||||
..Default::default()
|
||||
},
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2112,6 +2115,7 @@ async fn zero_attempt_failure_does_not_enter_successful_usage_or_cost_aggregates
|
||||
zero_attempt_range(&request),
|
||||
filter.clone(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2142,6 +2146,7 @@ async fn zero_attempt_failure_does_not_enter_successful_usage_or_cost_aggregates
|
||||
zero_attempt_range(&request),
|
||||
filter,
|
||||
admin_observability::Granularity::FifteenMinutes,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("billing projection");
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
//! 单项洞察保留统计口径,未选择的分布和并发不进入结果
|
||||
|
||||
use super::*;
|
||||
use admin_observability::{UsageInsightMetric as Metric, UsageInsightsObservation};
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_metrics_preserve_selected_values_for_aggregate_and_exact_reads() {
|
||||
let Some(database) = TestDatabase::create("usage_metrics").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now).await.unwrap();
|
||||
seed_calculated_billing_facts(&database.pool, now)
|
||||
.await
|
||||
.unwrap();
|
||||
super::super::drain_analytics(&database.pool).await;
|
||||
let store = admin_observability_store(&database.pool);
|
||||
let range =
|
||||
admin_observability::TimeRange::new(now - TimeDelta::hours(2), now + TimeDelta::hours(1))
|
||||
.unwrap();
|
||||
let granularity = admin_observability::Granularity::FifteenMinutes;
|
||||
for filter in [
|
||||
admin_observability::UsageFilter::default(),
|
||||
admin_observability::UsageFilter {
|
||||
model: Some("public-model".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let full = store
|
||||
.usage_insights_observation(range, filter.clone(), granularity, true)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(full.overview.requests.request_count > 0);
|
||||
for metric in [
|
||||
Metric::Health,
|
||||
Metric::Latency,
|
||||
Metric::FirstToken,
|
||||
Metric::Throughput,
|
||||
Metric::Load,
|
||||
Metric::Cost,
|
||||
Metric::Savings,
|
||||
Metric::Cache,
|
||||
] {
|
||||
let selected = store
|
||||
.usage_metric_observation(range, filter.clone(), granularity, metric)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
selected.overview.requests.request_count, full.overview.requests.request_count,
|
||||
"{metric:?}"
|
||||
);
|
||||
assert_eq!(selected.trend.len(), full.trend.len());
|
||||
assert_selected_metrics(&selected.overview.requests, &full.overview.requests, metric);
|
||||
for (actual, expected) in selected.trend.iter().zip(&full.trend) {
|
||||
assert_eq!(actual.bucket_start, expected.bucket_start);
|
||||
assert_eq!(actual.metrics.request_count, expected.metrics.request_count);
|
||||
assert_selected_metrics(&actual.metrics, &expected.metrics, metric);
|
||||
}
|
||||
assert_costs(&selected, &full, metric);
|
||||
if metric == Metric::Load {
|
||||
assert_eq!(selected.concurrency, full.concurrency);
|
||||
} else {
|
||||
assert!(selected.concurrency.is_empty(), "{metric:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
fn assert_selected_metrics(
|
||||
actual: &admin_observability::RequestMetrics,
|
||||
expected: &admin_observability::RequestMetrics,
|
||||
metric: Metric,
|
||||
) {
|
||||
match metric {
|
||||
Metric::Health => {
|
||||
assert_eq!(actual.success_count, expected.success_count);
|
||||
assert_eq!(actual.failure_count, expected.failure_count);
|
||||
assert_eq!(actual.cancelled_count, expected.cancelled_count);
|
||||
assert_eq!(actual.incomplete_count, expected.incomplete_count);
|
||||
assert_eq!(actual.caller_error_count, expected.caller_error_count);
|
||||
}
|
||||
Metric::Latency => assert_eq!(actual.latency_percentiles, expected.latency_percentiles),
|
||||
Metric::FirstToken => assert_eq!(
|
||||
actual.first_token_latency_percentiles,
|
||||
expected.first_token_latency_percentiles
|
||||
),
|
||||
Metric::Throughput => assert_eq!(
|
||||
(
|
||||
actual.output_throughput_p10,
|
||||
actual.output_throughput_p50,
|
||||
actual.output_throughput_p90
|
||||
),
|
||||
(
|
||||
expected.output_throughput_p10,
|
||||
expected.output_throughput_p50,
|
||||
expected.output_throughput_p90
|
||||
)
|
||||
),
|
||||
Metric::Load => {
|
||||
assert_eq!(
|
||||
actual.admission_decision_percentiles,
|
||||
expected.admission_decision_percentiles
|
||||
);
|
||||
assert_eq!(
|
||||
actual.account_selection_wait_percentiles,
|
||||
expected.account_selection_wait_percentiles
|
||||
);
|
||||
}
|
||||
Metric::Cost | Metric::Savings => {
|
||||
assert_eq!(actual.success_count, expected.success_count);
|
||||
}
|
||||
Metric::Cache => {
|
||||
assert_eq!(actual.success_count, expected.success_count);
|
||||
assert_eq!(actual.input_tokens, expected.input_tokens);
|
||||
assert_eq!(actual.cached_input_tokens, expected.cached_input_tokens);
|
||||
assert_eq!(
|
||||
actual.cache_hit_request_count,
|
||||
expected.cache_hit_request_count
|
||||
);
|
||||
assert_eq!(
|
||||
actual.cache_eligible_request_count,
|
||||
expected.cache_eligible_request_count
|
||||
);
|
||||
}
|
||||
}
|
||||
if metric != Metric::Latency {
|
||||
assert_eq!(actual.latency_percentiles, Default::default());
|
||||
}
|
||||
if metric != Metric::FirstToken {
|
||||
assert_eq!(actual.first_token_latency_percentiles, Default::default());
|
||||
}
|
||||
if metric != Metric::Throughput {
|
||||
assert_eq!(actual.output_throughput_p50, None);
|
||||
}
|
||||
if metric != Metric::Load {
|
||||
assert_eq!(actual.admission_decision_percentiles, Default::default());
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_costs(
|
||||
actual: &UsageInsightsObservation,
|
||||
expected: &UsageInsightsObservation,
|
||||
metric: Metric,
|
||||
) {
|
||||
if matches!(metric, Metric::Cost | Metric::Savings | Metric::Cache) {
|
||||
assert_eq!(
|
||||
actual.overview.attempts.costs,
|
||||
expected.overview.attempts.costs
|
||||
);
|
||||
assert_eq!(
|
||||
actual.overview.attempts.cost_coverage,
|
||||
expected.overview.attempts.cost_coverage
|
||||
);
|
||||
assert_eq!(actual.costs, expected.costs);
|
||||
for (actual, expected) in actual.trend.iter().zip(&expected.trend) {
|
||||
assert_eq!(actual.costs, expected.costs);
|
||||
assert_eq!(actual.cost_coverage, expected.cost_coverage);
|
||||
}
|
||||
} else {
|
||||
assert!(actual.overview.attempts.costs.is_empty());
|
||||
assert!(actual.costs.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 观测存储测试入口,以及查询范围、过滤与端口合同测试
|
||||
|
||||
mod metric;
|
||||
mod projection;
|
||||
|
||||
use std::{
|
||||
@@ -157,6 +158,7 @@ async fn output_throughput_uses_official_response_duration_without_local_fallbac
|
||||
range,
|
||||
UsageRecordFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1172,6 +1174,7 @@ async fn recovered_continuation_failure_should_be_visible_in_ops_but_hidden_from
|
||||
range,
|
||||
UsageRecordFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -1538,6 +1541,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
range,
|
||||
UsageRecordFilter::default(),
|
||||
admin_observability::Granularity::FifteenMinutes,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("projected costs");
|
||||
@@ -1556,6 +1560,7 @@ async fn calculated_usage_billing_facts_keep_only_completed_calculated_costs() {
|
||||
admin_observability::TimeRange::new(range.start, range.end).unwrap(),
|
||||
admin_observability::UsageFilter::default(),
|
||||
admin_observability::Granularity::FifteenMinutes,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("admin projected costs");
|
||||
@@ -1844,6 +1849,7 @@ async fn admin_observability_adapter_preserves_utc_queries_metrics_costs_and_det
|
||||
range,
|
||||
admin_observability::UsageFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2332,6 +2338,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
range,
|
||||
UsageRecordFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2359,6 +2366,7 @@ async fn observability_queries_preserve_request_account_cost_and_diagnostic_fact
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
@@ -2694,6 +2702,7 @@ async fn calendar_trends_share_exact_day_boundaries_for_requests_costs_and_empty
|
||||
range,
|
||||
Default::default(),
|
||||
admin_observability::Granularity::Day,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("calendar cost buckets");
|
||||
@@ -3004,7 +3013,7 @@ async fn observability_store_executes_caller_selection_without_replacing_policy(
|
||||
.unwrap();
|
||||
super::drain_analytics(&database.pool).await;
|
||||
let facts = store
|
||||
.usage_insights_observation(range, UsageFilter::default(), granularity)
|
||||
.usage_insights_observation(range, UsageFilter::default(), granularity, true)
|
||||
.await
|
||||
.unwrap();
|
||||
let facts = facts
|
||||
|
||||
@@ -534,6 +534,7 @@ async fn unassigned_correction_changes_totals_without_inventing_dimension_member
|
||||
range,
|
||||
NativeFilter::default(),
|
||||
gateway_admin::model::observability::Granularity::Hour,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map(|snapshot| snapshot.overview)
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@
|
||||
/// 名单中的字段可能是合法 HTTP 字段,
|
||||
/// 过滤表示网关不继承下游环境,不代表官方服务端必然拒绝该字段
|
||||
/// 按已知来源命名空间覆盖扩展;其他未知业务头不因源码中未出现而被过滤
|
||||
pub(super) fn is_non_codex_request_header(name: &str) -> bool {
|
||||
pub(crate) fn is_non_codex_request_header(name: &str) -> bool {
|
||||
// Cloudflare 链路信息及 Access 认证不跨到上游;cf-* 不只包含诊断字段
|
||||
name.starts_with("cf-")
|
||||
// 反代记录的是客户端到网关这一段地址、协议和路由
|
||||
@@ -0,0 +1,23 @@
|
||||
//! OpenAI 上下游协议差异与外部缺陷补偿的统一入口
|
||||
//! 规则只处理请求事实;身份保护、会话状态和连接生命周期由对应职责模块维护
|
||||
|
||||
mod headers;
|
||||
mod request;
|
||||
mod search_history;
|
||||
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
pub(crate) use headers::is_non_codex_request_header;
|
||||
pub(crate) use request::normalize_codex_request_body;
|
||||
|
||||
/// 选定 Codex/OAuth 上游后处理客户端正文兼容
|
||||
pub(crate) fn normalize_selected_codex_downstream_body(
|
||||
body: &mut Map<String, Value>,
|
||||
responses_lite: bool,
|
||||
middleware_headers: &[MiddlewareHeader],
|
||||
request_id: &str,
|
||||
) {
|
||||
request::normalize_non_codex_request_body(body);
|
||||
search_history::normalize_search_history(body, responses_lite, middleware_headers, request_id);
|
||||
}
|
||||
+2
-2
@@ -4,7 +4,7 @@ use serde_json::{Map, Value, json};
|
||||
|
||||
/// 非官方客户端及跨客户端历史回填的兼容入口;调用方必须已选定 Codex/OAuth 上游
|
||||
/// 只处理已验证被拒绝的字段,不根据客户端品牌推断整份历史是否合法
|
||||
pub(crate) fn normalize_non_codex_request_body(body: &mut Map<String, Value>) {
|
||||
pub(super) fn normalize_non_codex_request_body(body: &mut Map<String, Value>) {
|
||||
let Some(input) = body.get_mut("input").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
@@ -37,7 +37,7 @@ pub(crate) fn normalize_non_codex_request_body(body: &mut Map<String, Value>) {
|
||||
///
|
||||
/// 兼容基准是 Codex Core/Desktop 的模型请求,不是公开 OpenAI Responses API
|
||||
/// 未知字段继续透传,不能因官方请求结构中没有某个字段就将其列入过滤规则
|
||||
pub(in crate::transport) fn normalize_codex_request_body(body: &mut Map<String, Value>) {
|
||||
pub(crate) fn normalize_codex_request_body(body: &mut Map<String, Value>) {
|
||||
// 官方 Core/Desktop 显式发送 store=false;仅为缺字段的下游请求补齐,保留显式值
|
||||
body.entry("store").or_insert(Value::Bool(false));
|
||||
|
||||
+4
-8
@@ -4,13 +4,12 @@ use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_protocol::openai::X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
// TODO(https://github.com/openai/codex/issues/51962): 官方修复历史搜索项与工具声明不匹配后,
|
||||
// 用含 web_search_call 的普通请求和 Lite 压缩请求验证直连均可完成,再移除此补偿及对应合同测试
|
||||
// Codex 0.163.0 本地压缩可保留搜索历史却使用空工具;真实上游对照中补声明可消除相应 500
|
||||
pub(super) fn normalize_search_history(
|
||||
request: &mut CodexResponsesRequest,
|
||||
body: &mut Map<String, Value>,
|
||||
responses_lite: bool,
|
||||
middleware_headers: &[MiddlewareHeader],
|
||||
request_id: &str,
|
||||
) {
|
||||
@@ -22,11 +21,8 @@ pub(super) fn normalize_search_history(
|
||||
.name()
|
||||
.eq_ignore_ascii_case(X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER)
|
||||
})
|
||||
.map_or(
|
||||
request.responses_lite.as_deref() == Some("true"),
|
||||
|header| header.value().as_ref() == b"true",
|
||||
);
|
||||
if normalize_body(request.body_mut(), responses_lite) {
|
||||
.map_or(responses_lite, |header| header.value().as_ref() == b"true");
|
||||
if normalize_body(body, responses_lite) {
|
||||
tracing::debug!(
|
||||
request_id,
|
||||
compatibility = "web_search_history",
|
||||
@@ -32,6 +32,8 @@ const PLAN_CATALOG_CACHE_TTL: Duration = Duration::from_secs(5 * 60);
|
||||
const MAX_CATALOG_FETCH_ATTEMPTS: usize = 3;
|
||||
const MAX_PLAN_CATALOG_MODELS: usize = 2_048;
|
||||
const MAX_CLIENT_CATALOGS: usize = 32;
|
||||
// 保持原先 32 份、每份 1 MiB 的原生正文缓存预算,不把缓存容量变成上游目录合同
|
||||
const MAX_CLIENT_CATALOG_CACHE_BYTES: usize = 32 * 1024 * 1024;
|
||||
const MODEL_CATALOG_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const MAX_CONCURRENT_API_CATALOGS: usize = 4;
|
||||
const CLIENT_CATALOG_ERROR_TTL: Duration = Duration::from_secs(5);
|
||||
@@ -218,6 +220,20 @@ struct CachedClientCatalog {
|
||||
result: ClientCatalogResult,
|
||||
}
|
||||
|
||||
impl CachedClientCatalog {
|
||||
fn native_bytes(&self) -> usize {
|
||||
self.result.as_ref().map_or(0, |models| {
|
||||
models
|
||||
.iter()
|
||||
.map(|model| match &model.content {
|
||||
ProviderModelContent::Native(payload) => payload.body().len(),
|
||||
ProviderModelContent::Adapted(_) => 0,
|
||||
})
|
||||
.sum()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ClientCatalogEntry {
|
||||
key: ClientCatalogKey,
|
||||
created_at: Instant,
|
||||
@@ -460,6 +476,30 @@ impl CodexCredentialCatalogService {
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// 完成中的 OnceCell 仍由并发读者持有;移出缓存只影响后续查询,不丢弃本次成功结果
|
||||
let mut cache = self
|
||||
.client_catalogs
|
||||
.lock()
|
||||
.map_err(|_| CodexCredentialCatalogError::Cache)?;
|
||||
if result.native_bytes() > MAX_CLIENT_CATALOG_CACHE_BYTES {
|
||||
cache.retain(|entry| !Arc::ptr_eq(&entry.value, &value));
|
||||
} else {
|
||||
let mut bytes: usize = cache
|
||||
.iter()
|
||||
.filter_map(|entry| entry.value.get())
|
||||
.map(CachedClientCatalog::native_bytes)
|
||||
.sum();
|
||||
while bytes > MAX_CLIENT_CATALOG_CACHE_BYTES {
|
||||
let Some(index) = cache.iter().position(|entry| entry.value.initialized()) else {
|
||||
break;
|
||||
};
|
||||
let entry = cache.remove(index);
|
||||
bytes -= entry
|
||||
.value
|
||||
.get()
|
||||
.map_or(0, CachedClientCatalog::native_bytes);
|
||||
}
|
||||
}
|
||||
result.result.clone()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! OpenAI Provider 专属能力
|
||||
|
||||
mod admin;
|
||||
mod compat;
|
||||
pub mod config;
|
||||
mod jitter;
|
||||
mod provider;
|
||||
|
||||
@@ -396,6 +396,7 @@ pub(super) struct ColdResponse {
|
||||
pub(super) session_transport_key: Option<ProviderSessionAffinityKey>,
|
||||
pub(super) session_transport_key_hash: Option<String>,
|
||||
pub(super) session_transport_recovery: CodexSessionTransportRecovery,
|
||||
pub(super) guardian_routing: guardian::GuardianRouting,
|
||||
pub(super) websocket_retry_count: u32,
|
||||
pub(super) stream_max_retries: u32,
|
||||
pub(super) session_capture: Option<OpenAiSessionCapture>,
|
||||
@@ -781,6 +782,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
session_transport_key,
|
||||
session_transport_key_hash,
|
||||
session_transport_recovery,
|
||||
guardian_routing,
|
||||
websocket_retry_count,
|
||||
stream_max_retries,
|
||||
mut session_capture,
|
||||
@@ -951,6 +953,7 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
}
|
||||
}
|
||||
let response_transport = response.transport;
|
||||
let routing_affinity = response.routing_affinity;
|
||||
let websocket_connection_id = response.websocket_connection_id;
|
||||
let mut body = response.body;
|
||||
let mut failure_diagnostics = response.diagnostics.clone();
|
||||
@@ -1132,6 +1135,13 @@ pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
)
|
||||
});
|
||||
let timing_signals = decoder.take_timing_signals();
|
||||
if let Some(affinity) = &routing_affinity {
|
||||
for fact in events.iter().flat_map(ProviderEvent::canonical_facts) {
|
||||
if let GatewayEvent::Started(meta) = fact {
|
||||
guardian_routing.remember(context.client_api_key_ref(), active_account.id(), meta.response_id(), affinity);
|
||||
}
|
||||
}
|
||||
}
|
||||
let output_timing_changed = observation_state
|
||||
.observe_timing_signals(timing_signals, context.timing_started_at());
|
||||
let upstream_timing_changed = observation_state
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
//! 按 Client Key 和账号关联父响应的连接路由快照,不持久化 Cookie 或改变选号
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use gateway_core::{account::ProviderAccountId, policy::ClientApiKeyId};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::transport::routing_affinity::ChatGptRoutingAffinity;
|
||||
|
||||
// 与连接续接 tombstone 使用相同保留窗口;最多 4096 份、每份两个 8 KiB Cookie
|
||||
const RETENTION: Duration = Duration::from_mins(30);
|
||||
const MAX_PARENT_RESPONSES: usize = 4096;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct GuardianRouting {
|
||||
parents: Arc<Mutex<HashMap<[u8; 32], ParentRouting>>>,
|
||||
}
|
||||
|
||||
struct ParentRouting {
|
||||
captured_at: Instant,
|
||||
affinity: ChatGptRoutingAffinity,
|
||||
}
|
||||
|
||||
impl GuardianRouting {
|
||||
pub(super) fn get(
|
||||
&self,
|
||||
client_key: &ClientApiKeyId,
|
||||
account: &ProviderAccountId,
|
||||
response_id: &str,
|
||||
) -> Option<ChatGptRoutingAffinity> {
|
||||
let key = parent_key(client_key, account, response_id);
|
||||
let mut parents = self
|
||||
.parents
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let parent = parents.get(&key)?;
|
||||
if parent.captured_at.elapsed() >= RETENTION {
|
||||
parents.remove(&key);
|
||||
return None;
|
||||
}
|
||||
Some(parent.affinity.clone())
|
||||
}
|
||||
|
||||
pub(super) fn remember(
|
||||
&self,
|
||||
client_key: &ClientApiKeyId,
|
||||
account: &ProviderAccountId,
|
||||
response_id: &str,
|
||||
affinity: &ChatGptRoutingAffinity,
|
||||
) {
|
||||
let key = parent_key(client_key, account, response_id);
|
||||
let mut parents = self
|
||||
.parents
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if parents.contains_key(&key) {
|
||||
return;
|
||||
}
|
||||
if parents.len() >= MAX_PARENT_RESPONSES {
|
||||
parents.retain(|_, parent| parent.captured_at.elapsed() < RETENTION);
|
||||
if parents.len() >= MAX_PARENT_RESPONSES
|
||||
&& let Some(oldest) = parents
|
||||
.iter()
|
||||
.min_by_key(|(_, parent)| parent.captured_at)
|
||||
.map(|(key, _)| *key)
|
||||
{
|
||||
parents.remove(&oldest);
|
||||
}
|
||||
}
|
||||
parents.insert(
|
||||
key,
|
||||
ParentRouting {
|
||||
captured_at: Instant::now(),
|
||||
affinity: affinity.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_key(
|
||||
client_key: &ClientApiKeyId,
|
||||
account: &ProviderAccountId,
|
||||
response_id: &str,
|
||||
) -> [u8; 32] {
|
||||
let mut hash = Sha256::new();
|
||||
for part in [client_key.as_str(), account.as_str(), response_id] {
|
||||
hash.update(part.len().to_le_bytes());
|
||||
hash.update(part.as_bytes());
|
||||
}
|
||||
hash.finalize().into()
|
||||
}
|
||||
@@ -100,11 +100,11 @@ use crate::transport::{
|
||||
CodexBackendTransport, CodexClientError, CodexRateLimitUpdates, CodexRequestContext,
|
||||
CodexResponseMetadata, CodexResponseMetadataUpdates, CodexTransportMetrics,
|
||||
CodexUpstreamDiagnostics, CodexWebSocketPool, endpoint_url,
|
||||
normalize_selected_codex_downstream_body,
|
||||
};
|
||||
|
||||
mod execution;
|
||||
mod failure;
|
||||
mod guardian;
|
||||
mod live;
|
||||
mod observation;
|
||||
mod upstream_adapter;
|
||||
@@ -157,6 +157,7 @@ pub struct CodexProvider {
|
||||
live_calls_url: Url,
|
||||
session_identity: Option<CodexSessionIdentity>,
|
||||
session_transport_recovery: CodexSessionTransportRecovery,
|
||||
guardian_routing: guardian::GuardianRouting,
|
||||
stream_max_retries: u32,
|
||||
live_registry: Arc<CodexLiveRegistry>,
|
||||
live_gateway: Option<Arc<CodexLiveGateway>>,
|
||||
@@ -361,6 +362,7 @@ impl CodexProvider {
|
||||
live_calls_url,
|
||||
session_identity: None,
|
||||
session_transport_recovery: CodexSessionTransportRecovery::default(),
|
||||
guardian_routing: guardian::GuardianRouting::default(),
|
||||
stream_max_retries,
|
||||
live_registry: Arc::new(CodexLiveRegistry::default()),
|
||||
live_gateway: None,
|
||||
@@ -937,11 +939,43 @@ impl CodexProvider {
|
||||
let state_before = trace
|
||||
.is_enabled()
|
||||
.then(|| crate::transport::diagnostics::evidence::request_state(&upstream_request));
|
||||
let guardian_parent = (!state_owner_cross_account
|
||||
&& upstream_request.is_guardian()
|
||||
&& matches!(
|
||||
lease.authentication(),
|
||||
crate::credential::CodexRuntimeAuthentication::OAuth(_)
|
||||
))
|
||||
.then(|| {
|
||||
upstream_request
|
||||
.client_metadata()?
|
||||
.get("parent_response_id")?
|
||||
.as_str()
|
||||
})
|
||||
.flatten()
|
||||
.and_then(|response_id| {
|
||||
self.guardian_routing
|
||||
.get(
|
||||
context.client_api_key_ref(),
|
||||
lease.account_id(),
|
||||
response_id,
|
||||
)
|
||||
.map(|affinity| (response_id.to_owned(), affinity))
|
||||
});
|
||||
scope_request_to_account(
|
||||
&mut upstream_request,
|
||||
lease.installation_id(),
|
||||
account_scope,
|
||||
);
|
||||
// 父响应的可信关联只恢复此字段,不把首次 Guardian 请求的其他账号状态视为可信
|
||||
if let Some((response_id, _)) = &guardian_parent
|
||||
&& let Some(Value::Object(metadata)) =
|
||||
upstream_request.body_mut().get_mut("client_metadata")
|
||||
{
|
||||
metadata.insert(
|
||||
"parent_response_id".to_owned(),
|
||||
Value::String(response_id.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(before) = state_before {
|
||||
trace.record_evidence("provider.request.account_scope", serde_json::json!({
|
||||
"scope": match account_scope {
|
||||
@@ -962,9 +996,10 @@ impl CodexProvider {
|
||||
crate::credential::CodexRuntimeAuthentication::OAuth(_)
|
||||
);
|
||||
if codex_oauth {
|
||||
normalize_selected_codex_downstream_body(
|
||||
&mut upstream_request,
|
||||
generate.protocol_payload().context(),
|
||||
let responses_lite = upstream_request.responses_lite.as_deref() == Some("true");
|
||||
crate::compat::normalize_selected_codex_downstream_body(
|
||||
upstream_request.body_mut(),
|
||||
responses_lite,
|
||||
&middleware_headers,
|
||||
context.request_id().as_str(),
|
||||
);
|
||||
@@ -1107,6 +1142,7 @@ impl CodexProvider {
|
||||
.for_account(lease.account())
|
||||
.map_err(|error| map_client_error(error, UpstreamSendState::NotSent, false).error)?
|
||||
.with_responses_api_base_url(lease.authentication().responses_api_base_url())
|
||||
.with_routing_affinity(guardian_parent.map(|(_, affinity)| affinity))
|
||||
.with_connection_budget(context.connection_budget().clone())
|
||||
.with_middleware_headers(middleware_headers)
|
||||
.with_outbound_middleware(outbound_middleware)
|
||||
@@ -1123,6 +1159,7 @@ impl CodexProvider {
|
||||
session_transport_key,
|
||||
session_transport_key_hash,
|
||||
session_transport_recovery: self.session_transport_recovery.clone(),
|
||||
guardian_routing: self.guardian_routing.clone(),
|
||||
websocket_retry_count,
|
||||
stream_max_retries: self.stream_max_retries,
|
||||
session_capture,
|
||||
|
||||
@@ -308,14 +308,11 @@ impl DaemonTask for OpenAiCatalogEtagTask {
|
||||
() = cancellation.cancelled() => return Ok(()),
|
||||
() = self.catalog.wait_for_etag_refresh() => {},
|
||||
};
|
||||
if let Err(error) = self.catalog.refresh().await {
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
"OpenAI model catalog ETag refresh failed"
|
||||
);
|
||||
} else {
|
||||
context.succeeded();
|
||||
}
|
||||
// 失败的 ETag 已重新入队,交回 Host 退避后再领取,避免立即重试
|
||||
self.catalog.refresh().await.map_err(|error| {
|
||||
WorkerTaskError::safe("OpenAI 模型目录 ETag 刷新失败").with_source(error)
|
||||
})?;
|
||||
context.succeeded();
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -11,9 +11,6 @@ use serde::Deserialize;
|
||||
|
||||
use super::client::OpenAiUpstreamProtocol;
|
||||
|
||||
/// 单次 Codex 模型目录响应允许的最大字节数
|
||||
pub const MAX_CODEX_MODEL_CATALOG_BYTES: usize = 1024 * 1024;
|
||||
|
||||
const MAX_CATALOG_MODELS: usize = 2_048;
|
||||
const MAX_DISPLAY_NAME_BYTES: usize = 256;
|
||||
const MAX_DESCRIPTION_BYTES: usize = 4 * 1024;
|
||||
@@ -256,9 +253,6 @@ impl CodexModelCatalogSnapshot {
|
||||
/// Codex 模型目录不满足完整快照约束
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum CodexModelCatalogError {
|
||||
/// 响应超过硬上限
|
||||
#[error("Codex model catalog response exceeds the byte limit")]
|
||||
ResponseTooLarge,
|
||||
/// 顶层不是唯一受支持的 `{models:[...]}` wire
|
||||
#[error("Codex model catalog response violates the official wire contract")]
|
||||
InvalidWire,
|
||||
@@ -351,9 +345,6 @@ fn parse_native_model_catalog(
|
||||
etag: Option<&str>,
|
||||
protocol: OpenAiUpstreamProtocol,
|
||||
) -> Result<CodexModelCatalogSnapshot, CodexModelCatalogError> {
|
||||
if body.len() > MAX_CODEX_MODEL_CATALOG_BYTES {
|
||||
return Err(CodexModelCatalogError::ResponseTooLarge);
|
||||
}
|
||||
let wire: CodexModelsWire =
|
||||
serde_json::from_slice(body).map_err(|_| CodexModelCatalogError::InvalidWire)?;
|
||||
if wire.models.is_empty() {
|
||||
@@ -615,9 +606,6 @@ pub(crate) fn parse_api_model_catalog(
|
||||
id: String,
|
||||
display_name: Option<String>,
|
||||
}
|
||||
if body.len() > MAX_CODEX_MODEL_CATALOG_BYTES {
|
||||
return Err(CodexModelCatalogError::ResponseTooLarge);
|
||||
}
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_slice(body).map_err(|_| CodexModelCatalogError::InvalidWire)?;
|
||||
// 部分 Responses 上游按客户端版本返回完整 Codex 目录;不能降格为只有 ID 的画像
|
||||
|
||||
@@ -630,6 +630,7 @@ pub type CodexResponseMetadataUpdates = CodexWebSocketResponseMetadataUpdates;
|
||||
|
||||
/// Codex Responses 上游流式响应
|
||||
pub struct CodexBackendStreamingResponse {
|
||||
pub(crate) routing_affinity: Option<super::routing_affinity::ChatGptRoutingAffinity>,
|
||||
/// HTTP 为 SSE 字节块,WebSocket 为完整 UTF-8 文本消息
|
||||
pub body: CodexBackendSseStream,
|
||||
/// 实际使用的上游传输
|
||||
@@ -696,6 +697,7 @@ impl OpenAiUpstreamProtocol {
|
||||
/// Codex HTTP/SSE 上游客户端
|
||||
#[derive(Clone)]
|
||||
pub struct CodexBackendClient {
|
||||
pub(super) routing_affinity: Option<super::routing_affinity::ChatGptRoutingAffinity>,
|
||||
pub(super) privacy: Option<(
|
||||
Arc<dyn gateway_core::settings::privacy::CompiledPrivacyPolicy>,
|
||||
gateway_core::lifecycle::CancellationToken,
|
||||
@@ -718,6 +720,14 @@ pub struct CodexBackendClient {
|
||||
}
|
||||
|
||||
impl CodexBackendClient {
|
||||
pub(crate) fn with_routing_affinity(
|
||||
mut self,
|
||||
affinity: Option<super::routing_affinity::ChatGptRoutingAffinity>,
|
||||
) -> Self {
|
||||
self.routing_affinity = affinity;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn with_timezone(
|
||||
mut self,
|
||||
|
||||
@@ -10,13 +10,14 @@ use gateway_core::operation::ProviderHttpHeader;
|
||||
use gateway_protocol::openai::is_transport_managed_request_header;
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
|
||||
|
||||
use crate::compat::is_non_codex_request_header;
|
||||
|
||||
use super::{
|
||||
client::{
|
||||
CodexBackendClient, CodexClientError, CodexClientResult, CodexRequestContext,
|
||||
CodexTransportMetrics, elapsed_duration_millis, http_version_name, retry_after_seconds,
|
||||
},
|
||||
diagnostics::CodexUpstreamDiagnostics,
|
||||
downstream::is_non_codex_request_header,
|
||||
endpoints::endpoint_url,
|
||||
headers::is_managed_identity_header,
|
||||
response_meta,
|
||||
|
||||
@@ -17,10 +17,7 @@ use reqwest::{
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
|
||||
use crate::transport::{
|
||||
catalog::{
|
||||
CodexModelCatalogError, CodexModelCatalogSnapshot, MAX_CODEX_MODEL_CATALOG_BYTES,
|
||||
catalog_etag, parse_codex_model_catalog,
|
||||
},
|
||||
catalog::{CodexModelCatalogSnapshot, catalog_etag, parse_codex_model_catalog},
|
||||
diagnostics::CodexUpstreamSendPhase,
|
||||
endpoints::endpoint_url,
|
||||
headers::websocket_header_pairs,
|
||||
@@ -66,6 +63,7 @@ impl CodexBackendClient {
|
||||
) -> Self {
|
||||
let base_url = base_url.into().trim_end_matches('/').to_string();
|
||||
Self {
|
||||
routing_affinity: None,
|
||||
privacy: None,
|
||||
timezone: Default::default(),
|
||||
connection_budget: None,
|
||||
@@ -125,6 +123,10 @@ impl CodexBackendClient {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers =
|
||||
self.request_headers_for_http_response(&profile, upstream_request, context)?;
|
||||
let endpoint = endpoint_url(&self.base_url, self.protocol.responses_path());
|
||||
if let Some(affinity) = &self.routing_affinity {
|
||||
affinity.apply(&endpoint, &mut headers);
|
||||
}
|
||||
// OAuth 请求遵循 Codex 压缩合同;API Key 上游使用普通 JSON
|
||||
// Codex 上游只交付 SSE;即使下游请求 `stream: false`,也要上游流式执行,
|
||||
// 再由 API 层收集 canonical events 并返回完整 JSON
|
||||
@@ -145,7 +147,6 @@ impl CodexBackendClient {
|
||||
self.apply_privacy(&mut upstream_body, &mut headers)?;
|
||||
let body =
|
||||
serde_json::to_vec(&upstream_body).map_err(CodexClientError::RequestBodyEncode)?;
|
||||
let endpoint = endpoint_url(&self.base_url, self.protocol.responses_path());
|
||||
let trace = context
|
||||
.trace
|
||||
.cloned()
|
||||
@@ -261,6 +262,7 @@ impl CodexBackendClient {
|
||||
|
||||
let rate_limit_updates = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
routing_affinity: None,
|
||||
body: http_sse_stream(response, Arc::clone(&rate_limit_updates), trace),
|
||||
transport: CodexBackendTransport::HttpSse,
|
||||
websocket_connection_id: None,
|
||||
@@ -324,6 +326,12 @@ impl CodexBackendClient {
|
||||
let profile = self.profile.snapshot();
|
||||
let mut headers =
|
||||
self.request_headers_for_websocket_response(&profile, &websocket_request, context)?;
|
||||
if let Some(affinity) = &self.routing_affinity {
|
||||
affinity.apply(
|
||||
&endpoint_url(&self.base_url, self.protocol.responses_path()),
|
||||
&mut headers,
|
||||
);
|
||||
}
|
||||
let mut body = serde_json::Value::Object(std::mem::take(websocket_request.body_mut()));
|
||||
profile.apply_context_versions(&mut body);
|
||||
super::correlation::scope_request(&mut body, &mut headers, context.installation_id);
|
||||
@@ -600,6 +608,9 @@ impl CodexBackendClient {
|
||||
}));
|
||||
}
|
||||
Ok(CodexBackendStreamingResponse {
|
||||
routing_affinity: (self.protocol == OpenAiUpstreamProtocol::Codex)
|
||||
.then_some(exchange.routing_affinity)
|
||||
.flatten(),
|
||||
body: Box::pin(
|
||||
exchange
|
||||
.body
|
||||
@@ -644,6 +655,7 @@ impl CodexBackendClient {
|
||||
.with_egress_key(&self.egress_key)
|
||||
.with_outbound_proxy(self.outbound_proxy.as_ref())
|
||||
.with_connection_profile(connection_profile);
|
||||
key = key.with_routing_affinity(self.routing_affinity.as_ref().map(|pin| pin.id()));
|
||||
if let Some(connection_id) = request.downstream_websocket_connection_id.as_deref() {
|
||||
key = key.with_downstream_connection_id(connection_id);
|
||||
}
|
||||
@@ -684,9 +696,9 @@ impl CodexBackendClient {
|
||||
.then(|| catalog_etag(response.headers()))
|
||||
.transpose()?
|
||||
.flatten();
|
||||
let body = read_model_catalog_body(response).await?;
|
||||
let body = response.bytes().await?;
|
||||
if !status.is_success() {
|
||||
let body = String::from_utf8_lossy_owned(body);
|
||||
let body = String::from_utf8_lossy(&body).into_owned();
|
||||
return Err(CodexClientError::Upstream {
|
||||
status,
|
||||
retry_after_seconds: retry_after_seconds
|
||||
@@ -721,28 +733,6 @@ fn is_websocket_replayable_prelude(frame: &[u8]) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
async fn read_model_catalog_body(response: ReqwestResponse) -> CodexClientResult<Vec<u8>> {
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > MAX_CODEX_MODEL_CATALOG_BYTES as u64)
|
||||
{
|
||||
return Err(CodexModelCatalogError::ResponseTooLarge.into());
|
||||
}
|
||||
let mut body = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
let Some(next_len) = body.len().checked_add(chunk.len()) else {
|
||||
return Err(CodexModelCatalogError::ResponseTooLarge.into());
|
||||
};
|
||||
if next_len > MAX_CODEX_MODEL_CATALOG_BYTES {
|
||||
return Err(CodexModelCatalogError::ResponseTooLarge.into());
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn websocket_connection_profile(
|
||||
headers: &HeaderMap,
|
||||
middleware_headers: &[gateway_core::engine::middleware::MiddlewareHeader],
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
//! Grok 客户端经 OpenAI Responses 入口访问 Codex 上游时的隔离兼容规则
|
||||
|
||||
use reqwest::header::HeaderName;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
const GROK_INTRODUCTION: &str = "You are Grok released by xAI.";
|
||||
|
||||
pub(super) fn is_client_header(name: &str) -> bool {
|
||||
name.starts_with("x-grok-") || name.starts_with("x-xai-")
|
||||
}
|
||||
|
||||
fn is_request_marker(name: &str) -> bool {
|
||||
matches!(
|
||||
name,
|
||||
"x-grok-model-override" | "x-grok-turn-idx" | "x-grok-session-id"
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn normalize_request_body(body: &mut Map<String, Value>, context: &Map<String, Value>) {
|
||||
if !has_client_marker(context) {
|
||||
return;
|
||||
}
|
||||
let Some(input) = body.get_mut("input").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
for item in input {
|
||||
let Some(item) = item.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
if item.get("type").and_then(Value::as_str) == Some("message")
|
||||
&& item.get("role").and_then(Value::as_str) == Some("developer")
|
||||
&& let Some(content) = item.get_mut("content")
|
||||
{
|
||||
normalize_instruction_identity(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_client_marker(context: &Map<String, Value>) -> bool {
|
||||
context
|
||||
.get(PASSTHROUGH_HEADERS_CONTEXT_KEY)
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|entries| {
|
||||
entries
|
||||
.iter()
|
||||
.filter_map(Value::as_array)
|
||||
.filter(|entry| entry.len() == 2)
|
||||
.filter_map(|entry| entry.first().and_then(Value::as_str))
|
||||
.filter_map(|name| HeaderName::from_bytes(name.as_bytes()).ok())
|
||||
// 未知的扩展头只做出站过滤,不据此修改请求正文
|
||||
.any(|name| is_request_marker(name.as_str()))
|
||||
})
|
||||
}
|
||||
|
||||
// 只删除已观察到的指令开场白,保留其余文字、空白与内容块边界
|
||||
fn normalize_instruction_identity(content: &mut Value) {
|
||||
let text = match content {
|
||||
Value::String(text) => Some(text),
|
||||
Value::Array(parts) => parts.first_mut().and_then(|part| {
|
||||
if part.get("type").and_then(Value::as_str) != Some("input_text") {
|
||||
return None;
|
||||
}
|
||||
match part.get_mut("text") {
|
||||
Some(Value::String(text)) => Some(text),
|
||||
_ => None,
|
||||
}
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(text) = text
|
||||
&& text.starts_with(GROK_INTRODUCTION)
|
||||
{
|
||||
text.replace_range(..GROK_INTRODUCTION.len(), "");
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
//! 下游客户端对 Codex Core/Desktop 请求协议的兼容处理
|
||||
//! 账号身份保护、会话规范化和 HTTP 传输规则由对应职责模块维护
|
||||
|
||||
mod body;
|
||||
mod grok;
|
||||
mod headers;
|
||||
mod search_history;
|
||||
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
pub(super) use body::normalize_codex_request_body;
|
||||
|
||||
pub(crate) fn normalize_selected_codex_downstream_body(
|
||||
request: &mut CodexResponsesRequest,
|
||||
context: &Map<String, Value>,
|
||||
middleware_headers: &[MiddlewareHeader],
|
||||
request_id: &str,
|
||||
) {
|
||||
let body = request.body_mut();
|
||||
body::normalize_non_codex_request_body(body);
|
||||
grok::normalize_request_body(body, context);
|
||||
search_history::normalize_search_history(request, middleware_headers, request_id);
|
||||
}
|
||||
|
||||
pub(super) fn is_non_codex_request_header(name: &str) -> bool {
|
||||
headers::is_non_codex_request_header(name) || grok::is_client_header(name)
|
||||
}
|
||||
@@ -10,7 +10,6 @@ mod client_sse;
|
||||
pub(crate) mod connection;
|
||||
mod correlation;
|
||||
pub mod diagnostics;
|
||||
mod downstream;
|
||||
pub mod endpoints;
|
||||
pub mod headers;
|
||||
mod outbound;
|
||||
@@ -21,9 +20,9 @@ pub mod protocol;
|
||||
pub mod request;
|
||||
pub mod reset_credits;
|
||||
mod response_meta;
|
||||
pub(crate) mod routing_affinity;
|
||||
pub(crate) mod session;
|
||||
pub mod subscription;
|
||||
pub(crate) use downstream::normalize_selected_codex_downstream_body;
|
||||
pub(crate) use endpoints::parse_upstream_base_url;
|
||||
pub mod tls;
|
||||
pub mod usage;
|
||||
@@ -34,7 +33,7 @@ pub use self::{
|
||||
catalog::{
|
||||
CodexCatalogCapabilities, CodexCatalogCapabilityEvidence, CodexCatalogLimits,
|
||||
CodexCatalogMetadata, CodexCatalogModel, CodexCatalogVisibility, CodexModelCatalogError,
|
||||
CodexModelCatalogSnapshot, MAX_CODEX_MODEL_CATALOG_BYTES, parse_codex_model_catalog,
|
||||
CodexModelCatalogSnapshot, parse_codex_model_catalog,
|
||||
},
|
||||
client::{
|
||||
CodexAccountSelectionTelemetry, CodexBackendClient, CodexBackendJsonResponse,
|
||||
|
||||
@@ -13,7 +13,7 @@ use serde::Serialize as _;
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::transport::downstream::{is_non_codex_request_header, normalize_codex_request_body};
|
||||
use crate::compat::{is_non_codex_request_header, normalize_codex_request_body};
|
||||
use crate::transport::headers::is_managed_identity_header;
|
||||
use crate::transport::profile::CodexRequestLocation;
|
||||
use crate::transport::protocol::responses::{
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
//! Responses 连接握手的路由 Cookie 快照,仅用于同一上游端点的 Guardian 请求
|
||||
|
||||
use std::{fmt, sync::Arc};
|
||||
|
||||
use reqwest::{
|
||||
cookie::{CookieStore, Jar},
|
||||
header::{COOKIE, HeaderMap, HeaderValue, SET_COOKIE},
|
||||
};
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const ROUTING_COOKIE_NAMES: [&str; 2] = ["__oailb", "__cflb"];
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ChatGptRoutingAffinity {
|
||||
id: Uuid,
|
||||
url: Url,
|
||||
jar: Arc<Jar>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ChatGptRoutingAffinity {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ChatGptRoutingAffinity")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatGptRoutingAffinity {
|
||||
pub(crate) fn from_response(endpoint: &str, headers: &HeaderMap) -> Option<Self> {
|
||||
let url = cookie_url(endpoint)?;
|
||||
let mut cookies = Vec::with_capacity(2);
|
||||
for name in ROUTING_COOKIE_NAMES {
|
||||
let mut matching = headers.get_all(SET_COOKIE).iter().filter(|header| {
|
||||
header
|
||||
.to_str()
|
||||
.ok()
|
||||
.and_then(|value| value.split_once('='))
|
||||
.is_some_and(|(candidate, _)| candidate.trim() == name)
|
||||
});
|
||||
if let Some(cookie) = matching.next() {
|
||||
// 与 Codex 0.166 的连接快照一致,歧义或超长的 pin 不用于亲和
|
||||
if cookie.as_bytes().len() > 8192 || matching.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
cookies.push(cookie);
|
||||
}
|
||||
}
|
||||
let jar = Jar::default();
|
||||
jar.set_cookies(&mut cookies.into_iter(), &url);
|
||||
jar.cookies(&url)?;
|
||||
Some(Self {
|
||||
id: Uuid::new_v4(),
|
||||
url,
|
||||
jar: Arc::new(jar),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn id(&self) -> Uuid {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub(crate) fn apply(&self, endpoint: &str, headers: &mut HeaderMap) {
|
||||
let Some(url) = cookie_url(endpoint) else {
|
||||
return;
|
||||
};
|
||||
// 网关可配置 OAuth backend;快照只信任实际握手的 origin 与路径
|
||||
if url.origin() != self.url.origin() || url.path() != self.url.path() {
|
||||
return;
|
||||
}
|
||||
let Some(pin) = self.jar.cookies(&url) else {
|
||||
return;
|
||||
};
|
||||
let Ok(pin) = pin.to_str() else { return };
|
||||
let mut parts: Vec<_> = headers
|
||||
.get_all(COOKIE)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(';'))
|
||||
.map(str::trim)
|
||||
.filter(|part| {
|
||||
!part.is_empty()
|
||||
&& !part
|
||||
.split_once('=')
|
||||
.is_some_and(|(name, _)| ROUTING_COOKIE_NAMES.contains(&name.trim()))
|
||||
})
|
||||
.collect();
|
||||
parts.push(pin);
|
||||
if let Ok(mut value) = HeaderValue::from_str(&parts.join("; ")) {
|
||||
value.set_sensitive(true);
|
||||
headers.insert(COOKIE, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cookie_url(endpoint: &str) -> Option<Url> {
|
||||
let mut url = Url::parse(endpoint).ok()?;
|
||||
let scheme = match url.scheme() {
|
||||
"https" | "wss" => "https",
|
||||
"http" | "ws" => "http",
|
||||
_ => return None,
|
||||
};
|
||||
url.set_scheme(scheme).ok()?;
|
||||
Some(url)
|
||||
}
|
||||
@@ -25,6 +25,7 @@ pub(super) use self::stream::{WebSocketStreamPoolReturn, stream_websocket_respon
|
||||
|
||||
/// Responses WebSocket 文本流交互结果
|
||||
pub struct CodexWebSocketStreamingExchange {
|
||||
pub(crate) routing_affinity: Option<crate::transport::routing_affinity::ChatGptRoutingAffinity>,
|
||||
/// 关联请求日志与底层 pump 生命周期日志的连接标识
|
||||
pub(crate) websocket_connection_id: Uuid,
|
||||
/// 上游完整 WebSocket 文本消息流
|
||||
|
||||
@@ -116,6 +116,7 @@ pub(in crate::transport::websocket) fn stream_websocket_response(
|
||||
});
|
||||
|
||||
CodexWebSocketStreamingExchange {
|
||||
routing_affinity: response_metadata.routing_affinity,
|
||||
websocket_connection_id,
|
||||
body: Box::pin(body),
|
||||
turn_state: response_metadata.turn_state,
|
||||
@@ -403,7 +404,10 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
reason => CodexWebSocketExchangeError::StreamEndedBeforeTerminal {
|
||||
reason: reason.map_or("stream_eof", PumpExitReason::as_str),
|
||||
timeout: match reason {
|
||||
Some(PumpExitReason::PongWriteTimeout { timeout }) => Some(*timeout),
|
||||
Some(
|
||||
PumpExitReason::PingWriteTimeout { timeout }
|
||||
| PumpExitReason::PongWriteTimeout { timeout },
|
||||
) => Some(*timeout),
|
||||
_ => None,
|
||||
},
|
||||
last_event_type,
|
||||
|
||||
@@ -181,6 +181,10 @@ pub(super) fn websocket_connection_metadata(
|
||||
connection: &CodexWebSocketConnection,
|
||||
) -> CodexWebSocketConnectionMetadata {
|
||||
CodexWebSocketConnectionMetadata {
|
||||
routing_affinity: crate::transport::routing_affinity::ChatGptRoutingAffinity::from_response(
|
||||
&connection.endpoint,
|
||||
response.headers(),
|
||||
),
|
||||
turn_state: response_meta::turn_state(response.headers()),
|
||||
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
|
||||
rate_limit_headers: response_meta::rate_limit_headers(response.headers()),
|
||||
|
||||
@@ -24,6 +24,7 @@ pub struct CodexWebSocketPoolKey {
|
||||
account_id: String,
|
||||
conversation_id: String,
|
||||
connection_profile: String,
|
||||
routing_affinity_id: Option<Uuid>,
|
||||
pub(super) downstream_connection_id: String,
|
||||
egress_key: String,
|
||||
pub(super) proxy_capacity_key: String,
|
||||
@@ -41,6 +42,7 @@ impl CodexWebSocketPoolKey {
|
||||
account_id: account_id.into(),
|
||||
conversation_id: conversation_id.into(),
|
||||
connection_profile: String::new(),
|
||||
routing_affinity_id: None,
|
||||
downstream_connection_id: String::new(),
|
||||
egress_key: String::new(),
|
||||
proxy_capacity_key: String::new(),
|
||||
@@ -66,6 +68,11 @@ impl CodexWebSocketPoolKey {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
pub(crate) fn with_routing_affinity(mut self, id: Option<Uuid>) -> Self {
|
||||
self.routing_affinity_id = id;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_egress_key(mut self, key: &str) -> Self {
|
||||
self.egress_key = key.to_owned();
|
||||
self
|
||||
@@ -84,6 +91,10 @@ impl CodexWebSocketPoolKey {
|
||||
}
|
||||
|
||||
pub(crate) fn stable_hash(&self) -> String {
|
||||
let routing_affinity = self
|
||||
.routing_affinity_id
|
||||
.map(|id| id.to_string())
|
||||
.unwrap_or_default();
|
||||
short_sha256([
|
||||
self.base_url.as_str(),
|
||||
self.account_id.as_str(),
|
||||
@@ -91,6 +102,7 @@ impl CodexWebSocketPoolKey {
|
||||
self.connection_profile.as_str(),
|
||||
self.downstream_connection_id.as_str(),
|
||||
self.egress_key.as_str(),
|
||||
routing_affinity.as_str(),
|
||||
])
|
||||
}
|
||||
|
||||
@@ -99,6 +111,7 @@ impl CodexWebSocketPoolKey {
|
||||
&& self.account_id == other.account_id
|
||||
&& self.conversation_id == other.conversation_id
|
||||
&& self.egress_key == other.egress_key
|
||||
&& self.routing_affinity_id == other.routing_affinity_id
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +184,7 @@ impl WebSocketPoolState {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CodexWebSocketConnectionMetadata {
|
||||
pub(crate) routing_affinity: Option<crate::transport::routing_affinity::ChatGptRoutingAffinity>,
|
||||
pub(crate) turn_state: Option<String>,
|
||||
pub(crate) set_cookie_headers: Vec<String>,
|
||||
pub(crate) rate_limit_headers: Vec<(String, String)>,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 单 owner 驱动上游 WebSocket,被动回复心跳并在终止后有界释放 socket
|
||||
//! 单 owner 驱动上游 WebSocket,定时保活、回复心跳并在终止后有界释放 socket
|
||||
//!
|
||||
//! 业务消息通过有界队列交付,Ping/Pong 由 pump 消费,不刷新生成期间的消息等待超时
|
||||
//! 入站背压暂停读取,仍可接收关闭命令;已观察到的终止先释放 socket,再等待终止消息交付
|
||||
@@ -42,6 +42,8 @@ pub(crate) type RawWsStream = Box<dyn WebSocketIo>;
|
||||
|
||||
const PUMP_COMMAND_BUFFER: usize = 32;
|
||||
const CLOSE_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
const PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
const PING_WRITE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const PONG_WRITE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// pump 日志的账号级关联上下文;空闲连接被上游重置等场景用于回溯归属
|
||||
@@ -78,6 +80,9 @@ pub(crate) enum PumpExitReason {
|
||||
message: String,
|
||||
metric_reason: &'static str,
|
||||
},
|
||||
PingWriteTimeout {
|
||||
timeout: Duration,
|
||||
},
|
||||
PongWriteTimeout {
|
||||
timeout: Duration,
|
||||
},
|
||||
@@ -97,6 +102,7 @@ impl PumpExitReason {
|
||||
Self::UpstreamCloseFrame { .. } => "upstream_close_frame",
|
||||
Self::UpstreamEof => "upstream_eof",
|
||||
Self::InboundTransportError { .. } => "inbound_transport_error",
|
||||
Self::PingWriteTimeout { .. } => "ping_write_timeout",
|
||||
Self::PongWriteTimeout { .. } => "pong_write_timeout",
|
||||
Self::KeepaliveTransportError { .. } => "keepalive_transport_error",
|
||||
Self::MessageReceiverClosed => "message_receiver_closed",
|
||||
@@ -109,7 +115,9 @@ impl PumpExitReason {
|
||||
| Self::InboundTransportError { message, .. }
|
||||
| Self::KeepaliveTransportError { message, .. } => Some(message.clone()),
|
||||
Self::UpstreamCloseFrame { close } => close.as_ref().map(ToString::to_string),
|
||||
Self::PongWriteTimeout { timeout } => Some(format!("{timeout:?}")),
|
||||
Self::PingWriteTimeout { timeout } | Self::PongWriteTimeout { timeout } => {
|
||||
Some(format!("{timeout:?}"))
|
||||
}
|
||||
Self::CommandChannelClosed
|
||||
| Self::LocalClose
|
||||
| Self::UpstreamEof
|
||||
@@ -132,6 +140,7 @@ impl PumpExitReason {
|
||||
}
|
||||
}
|
||||
Self::UpstreamEof => "eof",
|
||||
Self::PingWriteTimeout { .. } => "ping_write_timeout",
|
||||
Self::PongWriteTimeout { .. } => "pong_write_timeout",
|
||||
Self::MessageReceiverClosed => "message_receiver_closed",
|
||||
}
|
||||
@@ -150,6 +159,7 @@ impl PumpExitReason {
|
||||
Self::OutboundTransportError { .. }
|
||||
| Self::UpstreamEof
|
||||
| Self::InboundTransportError { .. }
|
||||
| Self::PingWriteTimeout { .. }
|
||||
| Self::PongWriteTimeout { .. }
|
||||
| Self::KeepaliveTransportError { .. }
|
||||
)
|
||||
@@ -400,6 +410,9 @@ async fn pump_loop(
|
||||
) {
|
||||
let mut pending_inbound = None;
|
||||
let mut backpressure_events = 0_u64;
|
||||
// 首次保活留出完整间隔,背压恢复后跳过错过的 tick,避免集中补发
|
||||
let mut heartbeat = tokio::time::interval_at(Instant::now() + PING_INTERVAL, PING_INTERVAL);
|
||||
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
let reason = 'pump: loop {
|
||||
if pending_inbound.is_some() {
|
||||
@@ -434,6 +447,22 @@ async fn pump_loop(
|
||||
break 'pump reason;
|
||||
}
|
||||
}
|
||||
_ = heartbeat.tick() => {
|
||||
// Ping 只保活链路,不设 Pong deadline,也不刷新业务空闲或连接池回收时间
|
||||
match tokio::time::timeout(
|
||||
PING_WRITE_TIMEOUT,
|
||||
inner.stream.send(Message::Ping(Default::default())),
|
||||
).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(error)) => break 'pump PumpExitReason::KeepaliveTransportError {
|
||||
message: error.to_string(),
|
||||
metric_reason: transport_metric_reason(&error),
|
||||
},
|
||||
Err(_) => break 'pump PumpExitReason::PingWriteTimeout {
|
||||
timeout: PING_WRITE_TIMEOUT,
|
||||
},
|
||||
}
|
||||
}
|
||||
message = inner.stream.next() => {
|
||||
match message {
|
||||
None => break 'pump PumpExitReason::UpstreamEof,
|
||||
|
||||
@@ -1826,7 +1826,10 @@ fn reset_credit_command(account_id: ProviderAccountId) -> ConsumeProviderResetCr
|
||||
}
|
||||
}
|
||||
|
||||
fn initialized_provider_request(operation: Operation, account_id: &str) -> ProviderRequest {
|
||||
pub(crate) fn initialized_provider_request(
|
||||
operation: Operation,
|
||||
account_id: &str,
|
||||
) -> ProviderRequest {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let upstream_model = UpstreamModelId::new("gpt-5.4").expect("upstream model");
|
||||
let public_model = PublicModelId::new(upstream_model.as_str()).expect("public model");
|
||||
@@ -1856,7 +1859,7 @@ fn initialized_provider_request(operation: Operation, account_id: &str) -> Provi
|
||||
ProviderRequest::new(operation, plan.candidates()[0].clone())
|
||||
}
|
||||
|
||||
fn initialized_attempt_context(request_id: &str, account_id: &str) -> AttemptContext {
|
||||
pub(crate) fn initialized_attempt_context(request_id: &str, account_id: &str) -> AttemptContext {
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new(request_id).expect("request id"),
|
||||
@@ -1887,6 +1890,15 @@ pub(crate) async fn initialized_test_provider(
|
||||
accounts: Arc<MemoryAccountStore>,
|
||||
base_url: String,
|
||||
) -> Arc<dyn gateway_core::engine::provider::Provider> {
|
||||
initialized_test_bundle(accounts, base_url)
|
||||
.await
|
||||
.core_provider()
|
||||
}
|
||||
|
||||
pub(crate) async fn initialized_test_bundle(
|
||||
accounts: Arc<MemoryAccountStore>,
|
||||
base_url: String,
|
||||
) -> provider_openai::ProviderBundle {
|
||||
let mut config = valid_config();
|
||||
config.config.api.base_url = base_url;
|
||||
provider_openai::initialize(
|
||||
@@ -1895,7 +1907,6 @@ pub(crate) async fn initialized_test_provider(
|
||||
)
|
||||
.await
|
||||
.expect("initialized provider")
|
||||
.core_provider()
|
||||
}
|
||||
|
||||
fn provider_ports() -> ProviderStorePorts {
|
||||
|
||||
+15
-1
@@ -5,7 +5,7 @@ use gateway_core::operation::{GenerateRequest, ProtocolPayload};
|
||||
use provider_openai::encode_generate_request;
|
||||
use tokio_tungstenite::tungstenite::http::HeaderMap as WsHeaderMap;
|
||||
|
||||
use super::super::*;
|
||||
use super::*;
|
||||
|
||||
const DOWNSTREAM_TRANSPORT_HEADERS: &[(&str, &str)] = &[
|
||||
("CF-Visitor", r#"{"scheme":"https"}"#),
|
||||
@@ -42,6 +42,9 @@ const DOWNSTREAM_CLIENT_HEADERS: &[(&str, &str)] = &[
|
||||
("session_id", "synthetic-alias"),
|
||||
("sec-fetch-future", "future"),
|
||||
("sec-ch-ua-future", "future"),
|
||||
];
|
||||
|
||||
const BUSINESS_EXTENSION_HEADERS: &[(&str, &str)] = &[
|
||||
("x-grok-turn-idx", "7"),
|
||||
("X-Grok-Model-Override", "synthetic-model"),
|
||||
("x-grok-session-id", "synthetic-grok-session"),
|
||||
@@ -124,6 +127,7 @@ fn request_with_opaque_headers(use_websocket: bool) -> CodexResponsesRequest {
|
||||
DOWNSTREAM_TRANSPORT_HEADERS
|
||||
.iter()
|
||||
.chain(DOWNSTREAM_CLIENT_HEADERS)
|
||||
.chain(BUSINESS_EXTENSION_HEADERS)
|
||||
.map(|(name, value)| json!([name, STANDARD.encode(value.as_bytes())])),
|
||||
);
|
||||
let payload = ProtocolPayload::json_object(
|
||||
@@ -230,6 +234,13 @@ async fn backend_http_should_preserve_business_headers_without_downstream_transp
|
||||
.expect("opaque HTTP response");
|
||||
let raw = server.await.expect("opaque HTTP server task");
|
||||
|
||||
for &(name, value) in BUSINESS_EXTENSION_HEADERS {
|
||||
assert_eq!(
|
||||
raw_header_values(&raw, name),
|
||||
vec![value.as_bytes().to_vec()],
|
||||
"lost {name}"
|
||||
);
|
||||
}
|
||||
for &(name, _) in DOWNSTREAM_CLIENT_HEADERS {
|
||||
assert!(raw_header_values(&raw, name).is_empty(), "leaked {name}");
|
||||
}
|
||||
@@ -384,6 +395,9 @@ async fn backend_websocket_should_preserve_business_headers_without_downstream_t
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for &(name, value) in BUSINESS_EXTENSION_HEADERS {
|
||||
assert_eq!(values(name), vec![value.as_bytes().to_vec()], "lost {name}");
|
||||
}
|
||||
for (name, _) in DOWNSTREAM_TRANSPORT_HEADERS
|
||||
.iter()
|
||||
.chain(DOWNSTREAM_CLIENT_HEADERS)
|
||||
@@ -0,0 +1,25 @@
|
||||
//! 验证上下游兼容规则经实际请求编码和 HTTP/WS 传输生效
|
||||
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use provider_openai::transport::protocol::responses::CodexResponsesRequest;
|
||||
use provider_openai::transport::{CodexBackendClient, CodexRequestContext, CodexWebSocketPool};
|
||||
use serde_json::{Map, Value, json};
|
||||
use tokio::{
|
||||
io::AsyncReadExt,
|
||||
net::{TcpListener, TcpStream},
|
||||
};
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
use crate::transport::{
|
||||
CodexBackendClientTestExt, accept_codex_test_websocket, accept_codex_test_websocket_with,
|
||||
completed_websocket_response, read_http_request_with_body, request_context, test_wire_profile,
|
||||
write_completed_sse_response,
|
||||
};
|
||||
|
||||
mod headers;
|
||||
mod request;
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
use gateway_core::operation::{GenerateRequest, ProtocolPayload};
|
||||
use provider_openai::encode_generate_request;
|
||||
|
||||
use super::super::*;
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encoder_should_preserve_lite_input_without_top_level_instructions_or_tools() {
|
||||
@@ -1573,3 +1573,41 @@ async fn catalog_account_membership_changes_advance_generation_without_model_cha
|
||||
BTreeSet::from([first.id().clone(), second.id().clone()])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_catalog_byte_budget_evicts_completed_entries_without_rejecting_large_catalogs() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account = seed_account(&store, "acct_catalog_bytes").await;
|
||||
let scope = client_scope(&[account]);
|
||||
let server = MockServer::start().await;
|
||||
for (version, bytes, calls) in [
|
||||
("small", 1024, 1),
|
||||
("large", 17 * 1024 * 1024, 2),
|
||||
("other", 17 * 1024 * 1024, 1),
|
||||
("oversized", 33 * 1024 * 1024, 2),
|
||||
] {
|
||||
Mock::given(method("GET")).and(path("/codex/models"))
|
||||
.and(query_param("client_version", version))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({"models":[{
|
||||
"slug":"gpt-large", "display_name":"Large", "base_instructions":"x".repeat(bytes)
|
||||
}]}))).expect(calls).mount(&server).await;
|
||||
}
|
||||
let service = service_with_catalog_cache(&store, server.uri(), catalog_cache());
|
||||
for version in [
|
||||
"large",
|
||||
"other",
|
||||
"other",
|
||||
"large",
|
||||
"small",
|
||||
"oversized",
|
||||
"oversized",
|
||||
"small",
|
||||
] {
|
||||
let models = service
|
||||
.client_model_catalog(&scope, version)
|
||||
.await
|
||||
.expect("large catalog delivered");
|
||||
assert_eq!(models.len(), 1);
|
||||
}
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! OpenAI Provider、凭据与上游传输的测试入口
|
||||
|
||||
mod admin;
|
||||
mod compat;
|
||||
mod config;
|
||||
mod credential;
|
||||
mod jitter;
|
||||
|
||||
@@ -281,3 +281,293 @@ async fn middleware_can_override_account_headers_on_http_and_websocket() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RoutingUpstream {
|
||||
url: String,
|
||||
requests: Arc<Mutex<Vec<(usize, reqwest::header::HeaderMap, Value)>>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Drop for RoutingUpstream {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl RoutingUpstream {
|
||||
async fn start(cookies: Option<Vec<String>>) -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let capture = Arc::clone(&requests);
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = tokio::task::JoinSet::new();
|
||||
let mut index = 0;
|
||||
loop {
|
||||
let (socket, _) = listener.accept().await.unwrap();
|
||||
let capture = Arc::clone(&capture);
|
||||
let cookies = cookies.clone().unwrap_or_else(|| {
|
||||
vec![
|
||||
format!("__oailb=pin-{index}; Path=/codex/responses; Max-Age=300"),
|
||||
format!("__cflb=edge-{index}; Path=/; Max-Age=300"),
|
||||
]
|
||||
});
|
||||
connections.spawn(async move {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
let mut socket = crate::transport::accept_codex_test_websocket_with(socket, |request, response| {
|
||||
headers = request.headers().clone();
|
||||
for cookie in cookies {
|
||||
response.headers_mut().append("set-cookie", cookie.parse().unwrap());
|
||||
}
|
||||
}).await;
|
||||
while let Some(Ok(message)) = socket.next().await {
|
||||
let Message::Text(text) = message else { continue };
|
||||
let body: Value = serde_json::from_str(&text).unwrap();
|
||||
let id = body["client_metadata"]["fixture_response_id"].as_str().unwrap().to_owned();
|
||||
capture.lock().unwrap().push((index, headers.clone(), body));
|
||||
for event in ["response.created", "response.completed"] {
|
||||
socket.send(Message::Text(json!({
|
||||
"type":event, "response":{"id":id,"model":"gpt-5.4","status":"completed","output":[]}
|
||||
}).to_string().into())).await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
index += 1;
|
||||
}
|
||||
});
|
||||
Self {
|
||||
url,
|
||||
requests,
|
||||
task,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn routing_operation(id: &str, parent: Option<&str>, conversation: Option<&str>) -> Operation {
|
||||
let mut metadata = json!({"fixture_response_id":id});
|
||||
if let Some(parent) = parent {
|
||||
metadata["parent_response_id"] = json!(parent);
|
||||
metadata["x-openai-subagent"] = json!("guardian");
|
||||
}
|
||||
let mut protocol_context = Map::from_iter([("use_websocket".to_owned(), json!(true))]);
|
||||
if parent.is_some() {
|
||||
protocol_context.insert(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([["x-codex-guardian", "cmV2aWV3ZXI="]]),
|
||||
);
|
||||
}
|
||||
if let Some(conversation) = conversation {
|
||||
protocol_context.insert(
|
||||
"downstream_websocket_connection_id".to_owned(),
|
||||
json!(conversation),
|
||||
);
|
||||
}
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
json!({
|
||||
"model":"gpt-5.4", "input":"hello", "client_metadata":metadata
|
||||
})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(protocol_context);
|
||||
let mut generate = GenerateRequest::from_protocol_payload(payload);
|
||||
if let Some(conversation) = conversation {
|
||||
generate = generate.with_provider_session_state(ProviderSessionState::new("openai", json!({
|
||||
"account_id":SELECTED_ACCOUNT, "conversation_id":conversation, "continuation_scope":"persisted"
|
||||
}).as_object().unwrap().clone()).unwrap());
|
||||
}
|
||||
Operation::Generate(generate)
|
||||
}
|
||||
|
||||
fn routing_context(id: &str, key: &str, account: &str) -> AttemptContext {
|
||||
AttemptContext::new(
|
||||
RequestAttemptContext::new(
|
||||
ModelRequestId::new(format!("req_{id}")).unwrap(),
|
||||
ClientApiKeyId::new(key).unwrap(),
|
||||
),
|
||||
NonZeroU32::new(1).unwrap(),
|
||||
SystemTime::now() + Duration::from_secs(30),
|
||||
account_policy(),
|
||||
AccountAttemptContext::new(
|
||||
BTreeSet::new(),
|
||||
Some(ProviderAccountId::new(account).unwrap()),
|
||||
None,
|
||||
)
|
||||
.with_account_scope(contract_account_scope()),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn execute_routing(
|
||||
provider: &Arc<CodexProvider>,
|
||||
operation: Operation,
|
||||
context: AttemptContext,
|
||||
) {
|
||||
let mut stream = Arc::clone(provider)
|
||||
.execute(planned_request("openai", operation), context)
|
||||
.await
|
||||
.unwrap();
|
||||
while let Some(event) = stream.next().await {
|
||||
event.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guardian_pins_follow_parent_connection_and_separate_reviewer_pool_reuse() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
let server = RoutingUpstream::start(None).await;
|
||||
let provider = provider_with_base_url(&store, server.url.clone());
|
||||
for (id, parent, conversation) in [
|
||||
("parent_a", None, "root_a"),
|
||||
("parent_b", None, "root_b"),
|
||||
("review_a", Some("parent_a"), "reviewer"),
|
||||
("review_a_again", Some("parent_a"), "reviewer"),
|
||||
("review_b", Some("parent_b"), "reviewer"),
|
||||
] {
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation(id, parent, Some(conversation)),
|
||||
routing_context(id, "key_routing", SELECTED_ACCOUNT),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let requests = server.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 5);
|
||||
assert_eq!(
|
||||
requests[2].0, requests[3].0,
|
||||
"same parent keeps the reviewer socket"
|
||||
);
|
||||
assert_ne!(
|
||||
requests[3].0, requests[4].0,
|
||||
"another parent needs its own routed socket"
|
||||
);
|
||||
for (review, parent) in [(2, 0), (3, 0), (4, 1)] {
|
||||
let cookie = requests[review].1["cookie"].to_str().unwrap();
|
||||
assert!(cookie.contains(&format!("__oailb=pin-{parent}")));
|
||||
assert!(cookie.contains(&format!("__cflb=edge-{parent}")));
|
||||
}
|
||||
assert!(
|
||||
!requests[1].1.contains_key("cookie"),
|
||||
"ordinary traffic cannot inherit another connection's pins"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guardian_parent_pins_require_matching_client_key_and_selected_account() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
create_account(&store, "acct_scope_old").await;
|
||||
let server = RoutingUpstream::start(None).await;
|
||||
let provider = provider_with_base_url(&store, server.url.clone());
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation("parent", None, None),
|
||||
routing_context("parent", "key_routing", SELECTED_ACCOUNT),
|
||||
)
|
||||
.await;
|
||||
for (id, key, account, parent) in [
|
||||
("same", "key_routing", SELECTED_ACCOUNT, "parent"),
|
||||
("other_key", "key_other", SELECTED_ACCOUNT, "parent"),
|
||||
("other_account", "key_routing", "acct_scope_old", "parent"),
|
||||
("unknown", "key_routing", SELECTED_ACCOUNT, "unknown_parent"),
|
||||
] {
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation(id, Some(parent), None),
|
||||
routing_context(id, key, account),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let requests = server.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 5);
|
||||
for (index, (_, headers, body)) in requests.iter().enumerate().skip(1) {
|
||||
assert_eq!(headers.contains_key("cookie"), index == 1);
|
||||
assert_eq!(
|
||||
body.pointer("/client_metadata/parent_response_id")
|
||||
.and_then(Value::as_str),
|
||||
(index == 1).then_some("parent")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guardian_cookie_expiry_keeps_existing_socket_but_does_not_refresh_parent_pin() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
let server = RoutingUpstream::start(Some(vec![
|
||||
"__oailb=short_lived; Path=/; Max-Age=1".to_owned(),
|
||||
]))
|
||||
.await;
|
||||
let provider = provider_with_base_url(&store, server.url.clone());
|
||||
for (id, parent, conversation) in [
|
||||
("parent_a", None, "root"),
|
||||
("review_a", Some("parent_a"), "reviewer"),
|
||||
] {
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation(id, parent, Some(conversation)),
|
||||
routing_context(id, "key_routing", SELECTED_ACCOUNT),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(1100)).await;
|
||||
for (id, parent, conversation) in [
|
||||
("parent_b", None, "root"),
|
||||
("review_b", Some("parent_b"), "reviewer"),
|
||||
("review_new", Some("parent_b"), "another_reviewer"),
|
||||
] {
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation(id, parent, Some(conversation)),
|
||||
routing_context(id, "key_routing", SELECTED_ACCOUNT),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let requests = server.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 5);
|
||||
assert_eq!(requests[0].0, requests[2].0);
|
||||
assert_eq!(requests[1].0, requests[3].0);
|
||||
assert_ne!(requests[3].0, requests[4].0);
|
||||
assert!(requests[1].1.contains_key("cookie"));
|
||||
assert!(!requests[4].1.contains_key("cookie"));
|
||||
assert_eq!(
|
||||
requests[4].2["client_metadata"]["parent_response_id"],
|
||||
"parent_b"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guardian_ignores_expired_ambiguous_and_wrong_path_routing_cookies() {
|
||||
for cookies in [
|
||||
vec!["__oailb=expired; Path=/; Max-Age=0"],
|
||||
vec!["__oailb=first; Path=/", "__oailb=second; Path=/"],
|
||||
vec!["__oailb=other; Path=/other"],
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, SELECTED_ACCOUNT).await;
|
||||
let server =
|
||||
RoutingUpstream::start(Some(cookies.into_iter().map(str::to_owned).collect())).await;
|
||||
let provider = provider_with_base_url(&store, server.url.clone());
|
||||
for (id, parent) in [("parent", None), ("review", Some("parent"))] {
|
||||
execute_routing(
|
||||
&provider,
|
||||
routing_operation(id, parent, None),
|
||||
routing_context(id, "key_routing", SELECTED_ACCOUNT),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let requests = server.requests.lock().unwrap();
|
||||
assert!(!requests[1].1.contains_key("cookie"));
|
||||
assert!(
|
||||
requests[1]
|
||||
.2
|
||||
.pointer("/client_metadata/parent_response_id")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8823,6 +8823,12 @@ async fn stream_failures_preserve_error_fields_independent_of_retry_classificati
|
||||
ProviderErrorKind::RateLimited,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"unsupported_parameter",
|
||||
"Hosted tool 'web_search' requires authorization and metering that are not supported by rustponsesapi.",
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
false,
|
||||
),
|
||||
(
|
||||
"invalid_prompt",
|
||||
"Invalid prompt: we've limited access to this content for safety reasons.",
|
||||
@@ -8856,9 +8862,21 @@ async fn stream_failures_preserve_error_fields_independent_of_retry_classificati
|
||||
events.push(json!({"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello"}));
|
||||
}
|
||||
let mut original = json!({"type": "response.failed", "response": {"id": "resp_capacity", "error": {"code": code, "message": message}}});
|
||||
if code == "unsupported_parameter" {
|
||||
original = json!({"type":"error","status":400,"error":{
|
||||
"type":"invalid_request_error","code":code,"message":message,"param":"tools"
|
||||
}});
|
||||
}
|
||||
if code == "turn_rate_limit_exceeded" {
|
||||
original["response"]["error"]["headers"] = json!({"retry-after": "7"});
|
||||
}
|
||||
let expected_status = original["status"]
|
||||
.as_u64()
|
||||
.map(|status| u16::try_from(status).expect("valid upstream status"));
|
||||
let expected_error_type = original.get("error").unwrap_or(&original["response"]["error"])
|
||||
["type"]
|
||||
.as_str()
|
||||
.map(str::to_owned);
|
||||
events.push(original.clone());
|
||||
let (base_url, _http_server, websocket_server) = if use_websocket {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener");
|
||||
@@ -8922,6 +8940,7 @@ async fn stream_failures_preserve_error_fields_independent_of_retry_classificati
|
||||
None => panic!("expected upstream failure"),
|
||||
}
|
||||
};
|
||||
assert!(error.continuation_recovery_disposition().is_none());
|
||||
let capacity = expected_kind == ProviderErrorKind::UpstreamCapacityUnavailable;
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
@@ -8929,7 +8948,7 @@ async fn stream_failures_preserve_error_fields_independent_of_retry_classificati
|
||||
"unexpected error for {code}, websocket={use_websocket}, semantic_output={semantic_output}: {error:?}"
|
||||
);
|
||||
assert_eq!(error.replay_is_safe(), capacity && !semantic_output);
|
||||
assert_eq!(error.upstream_status(), None);
|
||||
assert_eq!(error.upstream_status(), expected_status);
|
||||
if code == "turn_rate_limit_exceeded" {
|
||||
assert!(error.retry_is_prohibited());
|
||||
assert_eq!(error.retry_after(), Some(Duration::from_secs(7)));
|
||||
@@ -8945,9 +8964,9 @@ async fn stream_failures_preserve_error_fields_independent_of_retry_classificati
|
||||
assert_eq!(
|
||||
error.upstream_failure_fields(),
|
||||
Some(&gateway_core::upstream::UpstreamFailureFields {
|
||||
status: None,
|
||||
status: expected_status,
|
||||
code: Some(code.to_owned()),
|
||||
error_type: None,
|
||||
error_type: expected_error_type,
|
||||
message: Some(message.to_owned()),
|
||||
})
|
||||
);
|
||||
@@ -11116,8 +11135,6 @@ const API_KEY_DOWNSTREAM_HEADERS: &[&str] = &[
|
||||
"referer",
|
||||
"sec-ch-ua",
|
||||
"sec-fetch-site",
|
||||
"x-grok-turn-idx",
|
||||
"x-xai-future-field",
|
||||
"session_id",
|
||||
"x-openai-actor-authorization",
|
||||
"authorization",
|
||||
@@ -11126,6 +11143,8 @@ const API_KEY_DOWNSTREAM_HEADERS: &[&str] = &[
|
||||
];
|
||||
|
||||
const API_KEY_BUSINESS_HEADERS: &[&str] = &[
|
||||
"x-grok-turn-idx",
|
||||
"x-xai-future-field",
|
||||
"session-id",
|
||||
"thread-id",
|
||||
"x-codex-future",
|
||||
@@ -12553,130 +12572,40 @@ pub(crate) async fn assert_local_connection_capacity_is_not_an_upstream_failure(
|
||||
assert!(failure.pre_delivery_retry().is_none());
|
||||
}
|
||||
|
||||
const GROK_PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
fn grok_marked_context() -> Map<String, Value> {
|
||||
Map::from_iter([(
|
||||
GROK_PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
json!([["X-Grok-Model-Override", "dGVzdA=="]]),
|
||||
)])
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grok_marked_request_removes_only_exact_leading_instruction_identity() {
|
||||
let original = "You are Grok released by xAI.\nKeep Grok tool names and xAI examples.";
|
||||
for (content, expected) in [
|
||||
(json!("You are Grok released by xAI."), json!("")),
|
||||
(
|
||||
async fn downstream_client_markers_preserve_instruction_content() {
|
||||
let original = "You are Grok released by xAI.\nKeep tool names and examples.";
|
||||
for api_key in [false, true] {
|
||||
for content in [
|
||||
json!(original),
|
||||
json!("\nKeep Grok tool names and xAI examples."),
|
||||
),
|
||||
(
|
||||
json!([
|
||||
{"type": "input_text", "text": original, "future": "Grok"},
|
||||
{"type": "input_text", "text": original}
|
||||
{"type": "input_text", "text": original, "future": "keep"},
|
||||
{"type": "input_text", "text": "Keep the second block."}
|
||||
]),
|
||||
json!([
|
||||
{"type": "input_text", "text": "\nKeep Grok tool names and xAI examples.", "future": "Grok"},
|
||||
{"type": "input_text", "text": original}
|
||||
]),
|
||||
),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"instructions": "Keep top-level instructions.",
|
||||
"input": [{"type": "message", "role": "developer", "content": content}],
|
||||
"tools": [{"type": "function", "name": "grok", "description": original}]
|
||||
})
|
||||
.as_object()
|
||||
.expect("request object")
|
||||
.clone();
|
||||
body = capture_grok_request(body, grok_marked_context(), false).await;
|
||||
assert_eq!(body["input"][0]["content"], expected);
|
||||
assert_eq!(body["instructions"], "Keep top-level instructions.");
|
||||
assert_eq!(body["tools"][0]["description"], original);
|
||||
let once = body.clone();
|
||||
body = capture_grok_request(body, grok_marked_context(), false).await;
|
||||
assert_eq!(body, once);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grok_unrelated_text_and_unmarked_requests_are_preserved() {
|
||||
let original = "You are Grok released by xAI. Be concise.";
|
||||
for (role, content, context) in [
|
||||
("user", original, grok_marked_context()),
|
||||
(
|
||||
"developer",
|
||||
"Quoted example: You are Grok released by xAI.",
|
||||
grok_marked_context(),
|
||||
),
|
||||
("developer", original, Map::new()),
|
||||
(
|
||||
"developer",
|
||||
original,
|
||||
Map::from_iter([(
|
||||
GROK_PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
json!([["X-Grok-Future-Field", "dGVzdA=="]]),
|
||||
)]),
|
||||
),
|
||||
(
|
||||
"developer",
|
||||
original,
|
||||
Map::from_iter([(
|
||||
GROK_PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
json!([["X-AuthenticateResponse", "dGVzdA=="]]),
|
||||
)]),
|
||||
),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "gpt-5.4", "store": false, "stream": true,
|
||||
"input": [{"type": "message", "role": role, "content": content}]
|
||||
})
|
||||
.as_object()
|
||||
.expect("request object")
|
||||
.clone();
|
||||
let original_body = body.clone();
|
||||
body = capture_grok_request(body, context, false).await;
|
||||
assert_eq!(body, original_body);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grok_downstream_identity_should_only_change_selected_codex_oauth_requests() {
|
||||
let original = "You are Grok released by xAI. Be concise.";
|
||||
let original_body = json!({
|
||||
"model": "gpt-5.4",
|
||||
"store": false,
|
||||
"stream": true,
|
||||
"instructions": "Preserve original instructions.",
|
||||
"input": [{"type": "message", "role": "developer", "content": original}]
|
||||
});
|
||||
for (api_key, marked, expected) in [
|
||||
(false, true, " Be concise."),
|
||||
(false, false, original),
|
||||
(true, true, original),
|
||||
] {
|
||||
let sent = capture_grok_request(
|
||||
original_body.as_object().expect("request object").clone(),
|
||||
if marked {
|
||||
grok_marked_context()
|
||||
} else {
|
||||
Map::new()
|
||||
},
|
||||
api_key,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(sent["input"][0]["content"], expected);
|
||||
if !marked || api_key {
|
||||
for (key, expected) in original_body.as_object().expect("request object") {
|
||||
assert_eq!(sent.get(key), Some(expected), "client field: {key}");
|
||||
}
|
||||
] {
|
||||
let body = json!({
|
||||
"model": "gpt-5.4", "store": false, "stream": true,
|
||||
"instructions": original,
|
||||
"input": [
|
||||
{"type": "message", "role": "developer", "content": content},
|
||||
{"type": "message", "role": "user", "content": original}
|
||||
],
|
||||
"tools": [{"type": "function", "name": "echo", "description": original}]
|
||||
})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
let context = Map::from_iter([(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([["X-Grok-Model-Override", STANDARD.encode(b"test")]]),
|
||||
)]);
|
||||
let sent = capture_client_request(body.clone(), context, api_key).await;
|
||||
assert_eq!(sent, body);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn capture_grok_request(
|
||||
async fn capture_client_request(
|
||||
body: Map<String, Value>,
|
||||
mut protocol_context: Map<String, Value>,
|
||||
api_key: bool,
|
||||
@@ -12717,7 +12646,7 @@ async fn capture_grok_request(
|
||||
"openai",
|
||||
Operation::Generate(GenerateRequest::from_protocol_payload(payload)),
|
||||
),
|
||||
context("req_grok_boundary", CancellationToken::new()),
|
||||
context("req_client_body", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("request preparation");
|
||||
|
||||
@@ -3,5 +3,6 @@
|
||||
mod contract;
|
||||
mod failure;
|
||||
mod live;
|
||||
mod workers;
|
||||
|
||||
pub(crate) use contract::assert_local_connection_capacity_is_not_an_upstream_failure;
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
//! 验证 OpenAI 目录 ETag worker 的失败交接、重启恢复与取消
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::StreamExt;
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload};
|
||||
use gateway_core::task::{WorkerContribution, WorkerDaemonContext, WorkerRunnable};
|
||||
use provider_openai::credential::ImportCodexOAuthCredential;
|
||||
use serde_json::{Map, json};
|
||||
use tokio::time::timeout;
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
use crate::admin::{
|
||||
initialized_attempt_context, initialized_provider_request, initialized_test_bundle,
|
||||
};
|
||||
use crate::support::{MemoryAccountStore, profile, secret};
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_etag_failure_returns_to_supervisor_and_recovers_pending_refresh() {
|
||||
let account_id = "acct_catalog_etag_worker";
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
store
|
||||
.seed_oauth_credential(ImportCodexOAuthCredential {
|
||||
account_id: account_id.to_owned(),
|
||||
name: account_id.to_owned(),
|
||||
secret: secret("at-catalog-etag-worker"),
|
||||
verified_account: profile("chatgpt-catalog-etag-worker"),
|
||||
next_refresh_at: Some(Utc::now() + chrono::Duration::minutes(30)),
|
||||
enabled: true,
|
||||
})
|
||||
.await;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.insert_header("x-models-etag", "models-worker-v2")
|
||||
.set_body_string(concat!(
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_etag\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"output\":[]}}\n\n"
|
||||
)),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/codex/models"))
|
||||
.respond_with(ResponseTemplate::new(503))
|
||||
.with_priority(1)
|
||||
.up_to_n_times(2)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/codex/models"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(
|
||||
include_bytes!("../transport/fixtures/official_models_snapshot.json"),
|
||||
"application/json",
|
||||
))
|
||||
.with_priority(2)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let mut bundle = initialized_test_bundle(store, server.uri()).await;
|
||||
let provider = bundle.core_provider();
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object(
|
||||
"openai",
|
||||
json!({"model": "gpt-5.4", "input": "catalog refresh"})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone(),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([("use_websocket".to_owned(), json!(false))])),
|
||||
));
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
initialized_provider_request(operation, account_id),
|
||||
initialized_attempt_context("req_catalog_etag_worker", account_id),
|
||||
)
|
||||
.await
|
||||
.expect("provider stream");
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("response carrying catalog ETag");
|
||||
}
|
||||
let worker = bundle
|
||||
.take_worker_contributions()
|
||||
.into_iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.owner() == "openai-model-etag" =>
|
||||
{
|
||||
Some(registration)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("catalog ETag worker");
|
||||
let WorkerRunnable::Daemon { task, .. } = worker.runnable else {
|
||||
panic!("catalog ETag worker must be a daemon");
|
||||
};
|
||||
|
||||
for expected_requests in 1..=2 {
|
||||
let context = WorkerDaemonContext::new(CancellationToken::new());
|
||||
let result = timeout(Duration::from_secs(2), task.run(context.clone())).await;
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let paths: Vec<_> = requests.iter().map(|request| request.url.path()).collect();
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"failed refresh must return control to the supervisor: {paths:?}, completed={}",
|
||||
context.completed_cycles()
|
||||
);
|
||||
let error = result.unwrap().expect_err("catalog unavailable");
|
||||
assert!(
|
||||
error.error_details().is_some(),
|
||||
"preserve the failure cause"
|
||||
);
|
||||
assert_eq!(context.completed_cycles(), 0);
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/codex/models")
|
||||
.count(),
|
||||
expected_requests,
|
||||
"each run must perform only one failed refresh"
|
||||
);
|
||||
}
|
||||
|
||||
let context = WorkerDaemonContext::new(CancellationToken::new());
|
||||
let run = task.run(context.clone());
|
||||
tokio::pin!(run);
|
||||
tokio::select! {
|
||||
result = &mut run => panic!("daemon exited before recovery: {result:?}"),
|
||||
result = timeout(Duration::from_secs(2), context.progress_since(0)) => {
|
||||
result.expect("pending ETag must survive failed runs");
|
||||
}
|
||||
}
|
||||
assert_eq!(context.completed_cycles(), 1);
|
||||
context.cancellation().cancel();
|
||||
timeout(Duration::from_secs(2), run)
|
||||
.await
|
||||
.expect("idle worker must stop on cancellation")
|
||||
.expect("cancelled worker");
|
||||
}
|
||||
@@ -4,9 +4,8 @@ use chrono::{TimeZone, Utc};
|
||||
use gateway_core::routing::ModelServiceTier;
|
||||
use provider_openai::transport::profile::{CodexWireProfile, CodexWireProfileState};
|
||||
use provider_openai::transport::{
|
||||
CodexBackendClient, CodexCatalogCapabilityEvidence, CodexCatalogVisibility, CodexClientError,
|
||||
CodexModelCatalogError, CodexRequestContext, MAX_CODEX_MODEL_CATALOG_BYTES,
|
||||
build_reqwest_client, parse_codex_model_catalog,
|
||||
CodexBackendClient, CodexCatalogCapabilityEvidence, CodexCatalogVisibility,
|
||||
CodexModelCatalogError, CodexRequestContext, build_reqwest_client, parse_codex_model_catalog,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use wiremock::matchers::{header, method, path, query_param};
|
||||
@@ -318,17 +317,6 @@ fn invalid_request_slug_should_fail_the_entire_snapshot() {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_over_hard_limit_should_fail_before_json_parsing() {
|
||||
let body = vec![b' '; MAX_CODEX_MODEL_CATALOG_BYTES + 1];
|
||||
let result = parse_codex_model_catalog(&body, None);
|
||||
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(CodexModelCatalogError::ResponseTooLarge)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_should_send_official_catalog_headers_and_query() {
|
||||
let server = MockServer::start().await;
|
||||
@@ -386,15 +374,16 @@ async fn fetch_should_send_official_catalog_headers_and_query() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_should_reject_streamed_body_over_hard_limit() {
|
||||
async fn fetch_preserves_large_model_instructions() {
|
||||
let instructions = "model instructions\n".repeat(100_000);
|
||||
let body = json!({"models":[{
|
||||
"slug":"gpt-native", "display_name":"Native", "base_instructions":instructions,
|
||||
"model_messages":{"future":{"template":"preserved"}}
|
||||
}]});
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/codex/models"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_bytes(vec![
|
||||
b' ';
|
||||
MAX_CODEX_MODEL_CATALOG_BYTES
|
||||
+ 1
|
||||
]))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(&body))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
@@ -404,14 +393,12 @@ async fn fetch_should_reject_streamed_body_over_hard_limit() {
|
||||
profile(),
|
||||
);
|
||||
|
||||
let result = client.fetch_models_with_context(context(), None).await;
|
||||
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(CodexClientError::ModelCatalog(
|
||||
CodexModelCatalogError::ResponseTooLarge
|
||||
))
|
||||
));
|
||||
let result = client
|
||||
.fetch_models_with_context(context(), Some("0.166.0"))
|
||||
.await
|
||||
.unwrap();
|
||||
let preserved: Value = serde_json::from_slice(result.models()[0].document().body()).unwrap();
|
||||
assert!(preserved == body["models"][0]);
|
||||
}
|
||||
|
||||
fn profile() -> CodexWireProfileState {
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
//! 下游客户端正文与头部兼容处理的测试入口
|
||||
|
||||
mod body;
|
||||
mod headers;
|
||||
@@ -53,7 +53,6 @@ mod catalog;
|
||||
mod client;
|
||||
mod connection;
|
||||
mod diagnostics;
|
||||
mod downstream;
|
||||
mod endpoints;
|
||||
mod headers;
|
||||
mod http_client;
|
||||
@@ -127,7 +126,7 @@ where
|
||||
.expect("accept test WebSocket")
|
||||
}
|
||||
|
||||
fn test_wire_profile() -> CodexWireProfileState {
|
||||
pub(crate) fn test_wire_profile() -> CodexWireProfileState {
|
||||
CodexWireProfileState::new(CodexWireProfile {
|
||||
client_kind: provider_openai::transport::profile::selection::ClientKind::Desktop,
|
||||
originator: "codex_cli_rs".to_owned(),
|
||||
@@ -145,7 +144,7 @@ fn test_wire_profile() -> CodexWireProfileState {
|
||||
})
|
||||
}
|
||||
|
||||
fn request_context<'a>(
|
||||
pub(crate) fn request_context<'a>(
|
||||
request_id: &'a str,
|
||||
account_id: Option<&'a str>,
|
||||
) -> CodexRequestContext<'a> {
|
||||
@@ -173,7 +172,7 @@ fn request_context<'a>(
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CollectedBackendResponse {
|
||||
pub(crate) struct CollectedBackendResponse {
|
||||
body: String,
|
||||
transport: CodexBackendTransport,
|
||||
usage: Option<TokenUsage>,
|
||||
@@ -186,7 +185,7 @@ struct CollectedBackendResponse {
|
||||
connection_local_continuation: bool,
|
||||
}
|
||||
|
||||
trait CodexBackendClientTestExt {
|
||||
pub(crate) trait CodexBackendClientTestExt {
|
||||
async fn create_response(
|
||||
&self,
|
||||
request: &CodexResponsesRequest,
|
||||
@@ -262,6 +261,7 @@ async fn collect_backend_response(
|
||||
response_metadata,
|
||||
mut transport_metrics,
|
||||
connection_local_continuation,
|
||||
..
|
||||
} = response;
|
||||
let mut body_bytes = Vec::new();
|
||||
while let Some(chunk) = body.next().await {
|
||||
@@ -391,7 +391,7 @@ fn pooled_websocket_request(conversation_id: &str) -> CodexResponsesRequest {
|
||||
request
|
||||
}
|
||||
|
||||
fn completed_websocket_response(
|
||||
pub(crate) fn completed_websocket_response(
|
||||
response_id: &str,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
@@ -457,7 +457,7 @@ async fn read_http_request(stream: &mut TcpStream) -> String {
|
||||
String::from_utf8(request).expect("HTTP request is UTF-8")
|
||||
}
|
||||
|
||||
async fn read_http_request_with_body(stream: &mut TcpStream) -> Vec<u8> {
|
||||
pub(crate) async fn read_http_request_with_body(stream: &mut TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 4096];
|
||||
loop {
|
||||
@@ -492,7 +492,7 @@ async fn read_http_request_with_body(stream: &mut TcpStream) -> Vec<u8> {
|
||||
request
|
||||
}
|
||||
|
||||
async fn write_completed_sse_response(stream: &mut TcpStream) {
|
||||
pub(crate) async fn write_completed_sse_response(stream: &mut TcpStream) {
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_http_fallback\",\"model\":\"gpt-test\"}}\n\n",
|
||||
|
||||
@@ -1493,10 +1493,12 @@ async fn codex_backend_client_should_timeout_when_upstream_is_silent() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_stream_should_allow_silence_below_idle_timeout() {
|
||||
async fn websocket_stream_should_keep_alive_without_requiring_pong_below_idle_timeout() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (created_tx, created_rx) = tokio::sync::oneshot::channel();
|
||||
let (ping_tx, ping_rx) = tokio::sync::oneshot::channel();
|
||||
let (finish_tx, finish_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
@@ -1513,7 +1515,10 @@ async fn websocket_stream_should_allow_silence_below_idle_timeout() {
|
||||
.await
|
||||
.unwrap();
|
||||
created_tx.send(()).unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
std::assert_matches!(websocket.next().await.unwrap().unwrap(), Message::Ping(_));
|
||||
ping_tx.send(()).unwrap();
|
||||
// 暂停读取和 flush,避免 tungstenite 自动回复 Pong
|
||||
finish_rx.await.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_structural", 3, 1).into(),
|
||||
@@ -1529,7 +1534,21 @@ async fn websocket_stream_should_allow_silence_below_idle_timeout() {
|
||||
created_rx.await.unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
tokio::time::advance(Duration::from_secs(26)).await;
|
||||
tokio::time::resume();
|
||||
timeout(Duration::from_secs(2), ping_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
tokio::time::resume();
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!response_task.is_finished(),
|
||||
"missing Pong must not end the response"
|
||||
);
|
||||
finish_tx.send(()).unwrap();
|
||||
let response = response_task
|
||||
.await
|
||||
.expect("websocket task should finish")
|
||||
@@ -1725,6 +1744,12 @@ async fn codex_backend_client_stream_should_keep_socket_after_structural_activit
|
||||
.await
|
||||
.unwrap();
|
||||
first_websocket.close(None).await.unwrap();
|
||||
// 完成关闭握手,避免未读的保活 Ping 导致 TCP reset 丢失已发送的响应
|
||||
while let Some(Ok(message)) = first_websocket.next().await {
|
||||
if matches!(message, Message::Close(_)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
@@ -1744,7 +1769,9 @@ async fn codex_backend_client_stream_should_keep_socket_after_structural_activit
|
||||
let mut stream = response.body;
|
||||
let mut body = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.expect("delayed websocket stream chunk should be valid");
|
||||
let chunk = chunk.unwrap_or_else(|error| {
|
||||
panic!("delayed websocket stream chunk should be valid: {error}")
|
||||
});
|
||||
body.push_str(std::str::from_utf8(&chunk).unwrap());
|
||||
}
|
||||
server.await.unwrap();
|
||||
@@ -2087,10 +2114,11 @@ async fn codex_backend_client_stream_should_wait_for_terminal_after_active_webso
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls() {
|
||||
#[tokio::test]
|
||||
async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls_despite_pong() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (pong_tx, mut pong_rx) = tokio::sync::mpsc::channel(1);
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
@@ -2106,7 +2134,16 @@ async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
futures::future::pending::<()>().await;
|
||||
while let Some(Ok(message)) = websocket.next().await {
|
||||
match message {
|
||||
Message::Ping(_) => {
|
||||
websocket.flush().await.unwrap();
|
||||
pong_tx.send(()).await.unwrap();
|
||||
}
|
||||
Message::Close(_) => break,
|
||||
other => panic!("unexpected frame during stalled response: {other:?}"),
|
||||
}
|
||||
}
|
||||
});
|
||||
let mut request = codex_request("gpt-5.5", "be brief", Vec::new());
|
||||
request.set_previous_response_id(Some("resp_previous".to_string()));
|
||||
@@ -2132,7 +2169,18 @@ async fn codex_backend_client_stream_should_timeout_when_active_websocket_stalls
|
||||
.expect("stream should yield partial frame")
|
||||
.expect("partial frame should be valid");
|
||||
assert!(std::str::from_utf8(&first).unwrap().contains("partial"));
|
||||
tokio::time::advance(Duration::from_secs(5 * 60)).await;
|
||||
for _ in 0..11 {
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::time::resume();
|
||||
timeout(Duration::from_secs(2), pong_rx.recv())
|
||||
.await
|
||||
.expect("gateway should send keepalive Ping")
|
||||
.expect("upstream should reply with Pong");
|
||||
}
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::time::resume();
|
||||
tokio::task::yield_now().await;
|
||||
let error = response
|
||||
.body
|
||||
|
||||
@@ -365,7 +365,7 @@ async fn codex_backend_client_should_open_fresh_socket_when_idle_pooled_websocke
|
||||
.unwrap();
|
||||
second_websocket.close(None).await.unwrap();
|
||||
});
|
||||
// 无 maintenance、无主动 ping:完全依赖 pump 后台读取感知连接死亡
|
||||
// 不启动维护,验证 pump 读取到上游关闭后即可阻止复用
|
||||
let pool = Arc::new(CodexWebSocketPool::with_config(
|
||||
websocket_pool_config_for_tests(None),
|
||||
));
|
||||
@@ -1210,11 +1210,10 @@ async fn concurrent_pool_transport_counts(max_connecting: usize) -> (usize, usiz
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_pool_should_reject_expired_idle_continuation_despite_upstream_ping() {
|
||||
async fn websocket_pool_should_reject_expired_idle_continuation_despite_keepalive() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (send_ping_tx, send_ping_rx) = tokio::sync::oneshot::channel();
|
||||
let (pong_seen_tx, pong_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let (ping_seen_tx, ping_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut first = accept_codex_test_websocket(stream).await;
|
||||
@@ -1225,16 +1224,16 @@ async fn websocket_pool_should_reject_expired_idle_continuation_despite_upstream
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
send_ping_rx.await.unwrap();
|
||||
let payload = b"upstream-idle-probe".to_vec();
|
||||
first
|
||||
.send(Message::Ping(payload.clone().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
let pong = first.next().await.unwrap().unwrap();
|
||||
assert_eq!(pong, Message::Pong(payload.into()));
|
||||
pong_seen_tx.send(()).unwrap();
|
||||
std::assert_matches!(first.next().await.unwrap().unwrap(), Message::Close(_));
|
||||
std::assert_matches!(first.next().await.unwrap().unwrap(), Message::Ping(_));
|
||||
first.flush().await.unwrap();
|
||||
ping_seen_tx.send(()).unwrap();
|
||||
loop {
|
||||
match first.next().await.unwrap().unwrap() {
|
||||
Message::Ping(_) => first.flush().await.unwrap(),
|
||||
Message::Close(_) => break,
|
||||
other => panic!("unexpected idle frame: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut second = accept_codex_test_websocket(stream).await;
|
||||
@@ -1269,8 +1268,7 @@ async fn websocket_pool_should_reject_expired_idle_continuation_despite_upstream
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(299)).await;
|
||||
tokio::time::resume();
|
||||
send_ping_tx.send(()).unwrap();
|
||||
timeout(Duration::from_secs(2), pong_seen_rx)
|
||||
timeout(Duration::from_secs(2), ping_seen_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
@@ -1332,12 +1330,17 @@ async fn websocket_pool_should_gc_expired_idle_connections() {
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let close = timeout(Duration::from_secs(60), first_websocket.next())
|
||||
.await
|
||||
.expect("gc sweep should close the expired idle websocket")
|
||||
.expect("gc sweep should send a close frame")
|
||||
.expect("close frame should be valid");
|
||||
std::assert_matches!(close, Message::Close(_));
|
||||
timeout(Duration::from_secs(60), async {
|
||||
loop {
|
||||
match first_websocket.next().await.unwrap().unwrap() {
|
||||
Message::Ping(_) => first_websocket.flush().await.unwrap(),
|
||||
Message::Close(_) => break,
|
||||
other => panic!("unexpected idle frame: {other:?}"),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("gc sweep should close the expired idle websocket");
|
||||
|
||||
let (second_stream, _) = listener.accept().await.unwrap();
|
||||
accepted_connections_for_server.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -1791,8 +1794,13 @@ async fn websocket_pool_should_reclaim_idle_connections_on_maintenance_ticks() {
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let message = websocket.next().await.unwrap().unwrap();
|
||||
std::assert_matches!(message, Message::Close(_));
|
||||
loop {
|
||||
match websocket.next().await.unwrap().unwrap() {
|
||||
Message::Ping(_) => websocket.flush().await.unwrap(),
|
||||
Message::Close(_) => break,
|
||||
other => panic!("unexpected idle frame: {other:?}"),
|
||||
}
|
||||
}
|
||||
closed_tx.send(()).unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
|
||||
+2
-1
@@ -305,7 +305,8 @@ Responses 流式请求完成准入后,每 15 秒无输出会发送一次 SSE
|
||||
`api.request_timeout_seconds` 默认 `null`,只控制 HTTP 路由返回响应前的等待,不是流式正文或 WebSocket 每轮执行的总时限。
|
||||
排队、插件显式执行期限与客户端断开的边界见 [请求期限](../docs/api.md#请求期限)
|
||||
|
||||
OpenAI 上游 WebSocket 被动回应 Ping,不主动发送心跳;Pong 写入最多等待 5 秒。
|
||||
OpenAI Responses 上游 WebSocket 每 25 秒发送 Ping 保活,并回应上游 Ping;Ping/Pong 写入最多等待 5 秒,
|
||||
不因等待 Pong 超时主动断开。入站背压期间暂停主动 Ping,恢复后不集中补发。
|
||||
下游 Responses WebSocket 每 25 秒发送 Ping,第一条未响应的 Ping 发出后 30 秒没有入站活动即断开;
|
||||
接收中的消息字节、Pong 和其他有效入站消息均算活动,慢速上传不必等整帧收齐才刷新探活状态。
|
||||
后续 Ping 不延长这次截止,心跳入队及写入最多等待 5 秒。两段心跳各自处理,不跨 WS 转发或转换为 SSE 事件。
|
||||
|
||||
+38
-13
@@ -437,11 +437,6 @@ OpenAI Provider 的 Responses 上游编码会移除 Codex 不接受的顶层 `te
|
||||
不删除工具参数 schema、输入内容或 `client_metadata` 内的同名业务字段;其他未知字段继续透传。
|
||||
`prompt_cache_key` 不用于补造会话或线程请求头;客户端未提供缓存键、会话或线程身份时保持缺省
|
||||
|
||||
Grok 客户端经 Codex/OAuth 上游执行时,仅在带有 `x-grok-model-override`、`x-grok-turn-idx` 或
|
||||
`x-grok-session-id` 标记的请求中,移除 `developer` 消息首个文本块开头的
|
||||
`You are Grok released by xAI.`。其余内容、用户消息和顶层 `instructions` 保留;
|
||||
普通 Codex 请求与 API Key 上游不应用这条兼容规则
|
||||
|
||||
Codex/OAuth 上游的历史回填按字段形状兼容,不以 User-Agent 品牌区分:显式 `type: "reasoning"`
|
||||
的 `input` 项移除顶层 `status`;该项具有非空字符串 `encrypted_content` 时,还会移除非空数组
|
||||
`content`。其他字段及顺序保持不变,缺少非空加密载荷的明文历史由上游判定。
|
||||
@@ -457,6 +452,10 @@ turn metadata 的 `workspaces` 也会保留仓库绝对路径、Git 远端地址
|
||||
切号或归属未知时移除。`x-codex-guardian`、`guardian_credits_requested` 和序列化
|
||||
`x-codex-turn-metadata` 内普通扩展的同名 `parent_response_id` 保持原样
|
||||
|
||||
同一 Client Key、同一 OAuth 账号的 Guardian 请求可利用网关记录的父响应 WS 握手信息,
|
||||
尽力复用父请求的上游路由。路由 Cookie 仅在网关与上游之间使用,不传给客户端;
|
||||
网关重启、记录淘汰或 Cookie 到期后仍可按普通路由请求,不保证上游路由亲和
|
||||
|
||||
#### WebSocket 与上游传输
|
||||
|
||||
公开 HTTP 请求在路由匹配前经过已绑定的 `http` 中间件;插件可以改写握手路径和请求参数。
|
||||
@@ -467,7 +466,7 @@ Responses WebSocket 接受文本消息,`response.create` 在同一连接串行
|
||||
留在有界接收队列中,待当前响应完成终结和写出后再逐条校验、准入与执行,不因请求提前到达而断开。
|
||||
接收队列容量为 32 个事件,超载仍关闭连接;Ping/Pong、客户端关闭和服务关闭不等待队列中的请求执行
|
||||
|
||||
网关分别维护两段链路的心跳:上游 WS 被动回复 Pong,下游 WS 独立探活;Ping/Pong 不跨段转发,也不转换为 SSE 事件。
|
||||
网关分别维护两段链路的心跳:上游 WS 定时发送 Ping 保活并回复 Pong,下游 WS 独立探活;Ping/Pong 不跨段转发,也不转换为 SSE 事件。
|
||||
下游探活包含消息接收中的字节进度,完整消息收齐后才进入中间件与请求处理。
|
||||
客户端断开时取消当前执行,并释放仍归该客户端所有的上游连接。`response.completed` 只结束本轮响应,保留客户端 WS 等待下一轮请求
|
||||
|
||||
@@ -487,8 +486,12 @@ Responses WebSocket 接受文本消息,`response.create` 在同一连接串行
|
||||
跨账号缺少原连接上下文时,仍通过恢复错误要求客户端完整重放
|
||||
|
||||
控制帧不重新选号、不创建推理 attempt。响应结束后原连接仍可接收控制帧并返回上游事件;
|
||||
下一轮执行使用新绑定的控制入口。除上述重试期间保留中断的情况外,尚未建立原连接、原连接已失效、
|
||||
实际走 HTTP 或 Provider 未提供控制通道时返回 `400`,不能通过控制帧新建上游连接
|
||||
下一轮执行使用新绑定的控制入口。没有绑定上游控制通道时(包括实际走 HTTP),有效的
|
||||
`response.interrupt` / `discard_partial_items` 请求返回带原 `response_id` 的
|
||||
`response.interrupt.failed`,当前模型响应继续交付真实事件与终态,客户端可收完后再续接。
|
||||
该事件只表示网关无法转发中断,不表示上游已停止生成或丢弃部分输出。
|
||||
未知控制消息、未知模式或无效响应 ID 字段在无控制通道时仍返回 `400`。
|
||||
除上述重试期间保留中断的情况外,已绑定的原连接失效或发送失败仍返回 `400`,不能通过控制帧新建上游连接
|
||||
|
||||
中断后继续转发上游事件与终态;只有 `response.incomplete` 的 `incomplete_details.reason` 为
|
||||
`interrupted` 时,才按官方中断语义保留原连接续接能力。部分输出是否被丢弃以上游终态为准。
|
||||
@@ -877,9 +880,9 @@ config 返回 `{ name, plaintextKey }`,仅读取服务端会话绑定的当前
|
||||
| `POST` | `/api/admin/accounts/reset-credits` | `{ accountId, creditId?, redeemRequestId }` | 使用 UUIDv4 幂等键消费一张 Provider 重置卡 |
|
||||
| `GET` | `/api/admin/accounts/reset-credits/automation` | `accountId` | 读取自动使用开关与本地计划,不查询上游 |
|
||||
| `POST` | `/api/admin/accounts/reset-credits/automation` | `{ accountId, enabled }` | 设置账号自动使用开关,默认关闭 |
|
||||
| `GET` | `/api/admin/accounts/models` | `accountId` | 优先读取该 Provider + 套餐的模型 cache,缺失时有限实时拉取 |
|
||||
| `GET` | `/api/admin/accounts/models` | `accountId` | 优先读取 Provider 模型目录缓存,缺失时有限实时拉取;返回 `{ models, observedAt, observedAtDisplay }` |
|
||||
| `GET` | `/api/admin/accounts/models/catalog` | `accountId` | 读取指定账号的完整 Codex 原生模型目录,返回 `{ modelCount, observedAt, catalog }` |
|
||||
| `POST` | `/api/admin/accounts/models/refresh` | `{ accountId }` | 强制拉取最新模型并覆盖 cache |
|
||||
| `POST` | `/api/admin/accounts/models/refresh` | `{ accountId }` | 强制拉取最新模型并覆盖缓存,响应结构与模型目录查询相同 |
|
||||
| `GET` | `/api/admin/accounts/connection-test` | `accountId`、`modelId` | 通过 SSE 返回实时连接测试事件,不作为业务 Responses 用量记录 |
|
||||
| `POST` | `/api/admin/accounts/oauth/start` | `{ provider, name, accountId?, outboundProxyId?, outboundProxyUrl? }` | 为支持登录的 Provider 创建 flow;`accountId` 表示重新授权 |
|
||||
| `POST` | `/api/admin/accounts/oauth/complete` | `{ provider, flowId, callbackUrl, settings? }` | 消费 OAuth callback;首次授权可附带账号设置,重新授权保留原设置 |
|
||||
@@ -908,6 +911,11 @@ config 返回 `{ name, plaintextKey }`,仅读取服务端会话绑定的当前
|
||||
响应中的 `catalog` 可直接保存为 JSON 文件,用于 Codex 的 `model_catalog_json` 配置。
|
||||
账号设置保存不等待上游模型目录刷新完成
|
||||
|
||||
模型目录查询和刷新响应中的 `observedAt` 表示 Provider 最近一次成功获取目录的时间,`observedAtDisplay` 按部署时区格式化;Provider 未提供观察时间时分别返回 `null` 和 `—`
|
||||
|
||||
`models` 只表示上游目录返回了这些模型 ID,不表示 Responses、Chat Completions、Embeddings 等所有端点均已验证可用。
|
||||
连接测试仅验证所选模型在 Provider 测试路径上的请求,其他端点需通过对应请求确认;目录观察时间不改变路由或账号能力判断
|
||||
|
||||
账号限流详情在 `quota` 中返回:`rateLimitReason` 为 `upstream_rate_limit`(上游临时限流)、
|
||||
`error_protection`(自定义错误规则触发保护)或 `null`。`recoveryProbeRequired` 表示解除冻结是否需要成功探测;
|
||||
此时 `rateLimitedUntil` 是最早探测时间,到期后仍保持 `rate_limited`,直到探测成功或手动恢复。
|
||||
@@ -2161,7 +2169,10 @@ errorCode, errorMessage, startedAt, completedAt, expiresAt, createdAt, updatedAt
|
||||
| `GET` | `/api/admin/usage/records` | 请求记录分页列表 |
|
||||
| `GET` | `/api/admin/usage/records/detail` | 按 `id` 查询请求详情 |
|
||||
| `GET` | `/api/admin/usage/records/summary` | 当前筛选条件的请求汇总 |
|
||||
| `GET` | `/api/admin/usage/insights/overview` | 用量、成本与成功率洞察 |
|
||||
| `GET` | `/api/admin/usage/insights/overview` | 完整用量洞察 |
|
||||
| `GET` | `/api/admin/usage/insights/health` | 请求健康 |
|
||||
| `GET` | `/api/admin/usage/insights/performance` | 响应性能;`metric=latency\|firstToken\|throughput\|load` |
|
||||
| `GET` | `/api/admin/usage/insights/cost` | 成本效率;`metric=cost\|savings\|cache` |
|
||||
| `GET` | `/api/admin/usage/insights/diagnostics` | 按维度聚合诊断 |
|
||||
| `GET` | `/api/admin/operations/errors` | 运维错误分页列表 |
|
||||
| `GET` | `/api/admin/diagnostics/export` | 按 `requestId` 或 `errorId` 下载诊断 ZIP 附件 |
|
||||
@@ -2183,7 +2194,7 @@ errorCode, errorMessage, startedAt, completedAt, expiresAt, createdAt, updatedAt
|
||||
|
||||
`points[]` 提供趋势图与首页卡片使用的请求数、Tokens、缓存比例、错误数、成功率、
|
||||
`latencyValue`、首 Token P95、总延迟 P95 和输出吞吐 P50,以及时间标签与展示值
|
||||
请求并发与内部调度等待通过用量洞察的 `performance.load` 查询
|
||||
请求并发与内部调度等待通过 `/api/admin/usage/insights/performance?metric=load` 查询
|
||||
|
||||
### Dashboard 客户端身份
|
||||
|
||||
@@ -2220,7 +2231,7 @@ Dashboard 默认 `period=today`,用量、诊断和错误查询默认 `period=7
|
||||
Dashboard 返回原始 `asOf` 及 `asOfDisplay`,作为当前数据的查询锚点
|
||||
|
||||
请求明细列表接受 `currentPage`、`pageSize`、时间范围、`provider`、`clientApiKeyName`、`accountName`、`model` 和 `status`。
|
||||
汇总和洞察只接受时间范围与 `provider`;诊断另接受 `dimension`、`model` 和 `statusCode`。
|
||||
汇总和洞察接受时间范围与 `provider`;性能和成本另要求 `metric`,诊断另接受 `dimension`、`model` 和 `statusCode`。
|
||||
各接口拒绝未知参数,明细筛选不改变顶部统计范围
|
||||
|
||||
请求列表的 `status` 可取 `success`、`error`、`all`,默认 `success`;未知值返回 `400`。
|
||||
@@ -2265,6 +2276,20 @@ Dashboard 返回原始 `asOf` 及 `asOfDisplay`,作为当前数据的查询锚
|
||||
运维错误列表的 `search` 使用字面量包含匹配,`%`、`_` 和 `\` 按普通字符处理,不支持密钥值或可见前缀搜索。
|
||||
请求详情通过 `/api/admin/usage/records/detail?id=...` 查询
|
||||
|
||||
### 卡片洞察查询
|
||||
|
||||
请求健康、响应性能、成本效率与热点诊断分别查询自己的路由;性能和成本按选中的 `metric` 查询,诊断按 `dimension` 查询。
|
||||
各卡片独立加载和显示错误,切换选项取消该卡片的旧请求;时间范围与平台筛选共同作用于所有卡片。
|
||||
`overview` 返回完整洞察,管理端卡片使用分项接口
|
||||
|
||||
健康、性能与成本接口返回 `analytics`、`granularity`、`metric` 和 `data`。
|
||||
健康的 `metric` 固定为 `health`,`data` 含健康摘要和趋势;性能与成本的 `data` 含 `summary` 和 `points`,
|
||||
每个点包含 `bucket`、`label`、`requestCount` 和所选指标的 `value`。缺失样本保留 `null`,不填充未查询指标
|
||||
|
||||
性能摘要与趋势分别返回总耗时分位数、首 Token 分位数、吞吐分位数或负载指标;只有 `metric=load` 读取请求并发。
|
||||
成本的三个选项共用费用、节省、溢价、每次成功请求费用和覆盖率摘要,趋势分别返回实际与无缓存估算费用、缓存节省或缓存比例。
|
||||
缺少 `metric`、选择其他卡片的指标或传入未知指标均返回 `400`
|
||||
|
||||
### 记录范围与统计口径
|
||||
|
||||
OpenAI 推理强度优先取 `input` 中最后一条 `type: "configuration_update"` 项的 `reasoning.effort`,
|
||||
|
||||
+16
-7
@@ -330,7 +330,8 @@ Provider 可为模型提供来源账号集合;Core 在公开目录查询时结
|
||||
不解析上游字段;API 负责协议输出,不直接依赖具体 Provider。原生目录失败不会退回通用画像,只有不提供
|
||||
该协议原生目录的 Provider(如 xAI 的 Codex 适配)才走画像转换。目录读取不创建计量请求或推理 attempt。
|
||||
OpenAI 的账号/凭据 revision/客户端版本隔离、并发合并、TTL、容量与失效均归属 credential catalog
|
||||
service;客户端原生对象保存在有界进程缓存中,与套餐 Redis 模型 ID 缓存分开管理
|
||||
service;客户端原生对象保存在有界进程缓存中,与套餐 Redis 模型 ID 缓存分开管理。
|
||||
该缓存最多保留 32 项,原生正文总预算为 32 MiB;单次结果超过预算时仍正常交付,但不驻留缓存
|
||||
|
||||
全局请求位置及开关由运行设置持久化;快照仅在开关开启时生成全局覆盖,并随
|
||||
`RuntimeSnapshot → RoutingPlan → AttemptContext` 冻结传递,关闭时保留客户端原有字段;
|
||||
@@ -493,6 +494,11 @@ OpenAI 的连接池监督该取消信号,按当前归属保护绑定连接;
|
||||
回收先移出可复用集合,再在锁外有界关闭并等待 pump/socket 释放,物理容量随 socket 归还;批量关闭并行执行。
|
||||
协议合同与运行参数分别见 [Responses 传输](api.md#websocket-与上游传输)和[部署与运维](../deploy/README.md#公网访问)
|
||||
|
||||
OpenAI Provider 按 Client Key、所选 OAuth 账号和父响应 ID 关联 WS 握手的路由 Cookie 快照,
|
||||
只供 Guardian 使用,不参与选号或写入账号凭据。进程内最多保存 4096 个父响应,保留 30 分钟;
|
||||
缺失或失效时按普通上游路由执行。transport 负责 Cookie 有效期和同 origin、同端点路径约束,
|
||||
连接池按快照身份隔离审查连接;复用父连接不会重新计算 Cookie 有效期,Cookie 到期不强制关闭已建立的审查连接
|
||||
|
||||
OpenAI 的 OAuth 与 API Key 共用现有账号和事务。API Key 的 Base URL、密钥和传输策略属于 Provider 凭据 JSON,
|
||||
随 credential revision 更新;普通详情只投影非敏感连接设置。API Key 客户端目录按账号、凭据版本和客户端版本隔离,
|
||||
标准 `data` 模型列表通过通用画像输出,完整 Codex `models` 目录保留原生对象。通用账号层按 Provider 提交的
|
||||
@@ -515,16 +521,17 @@ OpenAI 模型目录用于发现,不因目录缺项拒绝请求;管理员配
|
||||
不凭 canonical 增量补造内容,也不让 SSE/WS 转发额外保存整份输出
|
||||
- Responses、Images 与 standalone Search 的业务扩展头保留原始多值字节。API 负责剥离鉴权、账号身份和 HTTP 传输字段,
|
||||
并提取会话语义;`gateway-protocol` 共享 HTTP 传输与网关链路字段分类。客户端兼容规则集中在
|
||||
`providers/openai/src/transport/downstream/`:`headers.rs` 管理下游环境头和已提取语义的头部别名,
|
||||
`body.rs` 管理已知顶层参数的过滤、缺省值补齐和已确认不兼容的 `input` 形状适配;兼容基准为 Codex Core/Desktop 请求协议,
|
||||
不持有账号身份保护或会话规范化逻辑。
|
||||
`providers/openai/src/compat/`,与上游缺陷补偿统一管理:`headers.rs` 管理下游环境头和已提取语义的头部别名,
|
||||
`request.rs` 管理参数过滤、缺省值及请求形状适配;兼容基准为 Codex Core/Desktop 请求协议。
|
||||
`search_history.rs` 处理搜索历史缺少工具声明的临时补偿。
|
||||
`compat` 只消费请求正文和协议头,不依赖 Provider 或 transport 的执行实现,不持有身份、会话或连接状态。
|
||||
Provider 在 `transport/request.rs` 解码不透明头时组合兼容、身份与 HTTP 规则,
|
||||
同时调用不依赖账号的正文兼容规则。`body.rs` 中非官方客户端及跨客户端历史回填兼容的入口,
|
||||
同时调用不依赖账号的正文兼容规则。非官方客户端及跨客户端历史回填兼容的入口,
|
||||
由 Provider 在选定 Codex/OAuth 账号后、HTTP/WS 分流前调用;API Key 上游跳过该入口。
|
||||
长期协议适配与临时缺陷补偿按文件区分;临时补偿注明外部依据、适用范围和移除条件,直连验证修复后清理对应规则与合同场景。
|
||||
`transport/headers.rs` 负责上游身份保护和官方头组装。
|
||||
会话别名只规范化请求头,不清除正文身份字段;未知业务扩展与响应诊断头不受影响,字段见
|
||||
[Responses 合同](api.md#3-openai-数据面与模型目录)。Grok 专属请求标记和已知指令开场白的兼容只在
|
||||
`downstream/grok.rs` 内判断,且仅用于 Codex/OAuth 上游;普通请求的提示词、工具及业务正文不做品牌清洗
|
||||
[Responses 合同](api.md#3-openai-数据面与模型目录)
|
||||
- Live 共用 OpenAI 的头部身份、下游环境和传输边界,业务扩展不使用固定白名单。
|
||||
引导、sideband 与 hangup 由 Provider 在选定或加载钉住账号后统一处理出站关联标识;
|
||||
API 保留原始入站事实,Core 只传递头值字节和通话归属,具体合同见 [Codex Live 语音通话](api.md#codex-live-语音通话)
|
||||
@@ -1003,6 +1010,8 @@ Store 从请求源事实维护窄事实及十五分钟、小时统计状态,
|
||||
维护锁竞争造成的空轮不重置批量,未提交事务与未确认队列项由后续周期重试。
|
||||
|
||||
用量洞察的请求并发由 Store 在同一读取快照中按请求开始、完成时间聚合,运行中请求计至查询范围末尾。
|
||||
Admin 按卡片与所选指标向 Store 请求计数、金额或分布,缓存按范围、筛选和指标隔离结果。
|
||||
性能卡片只有负载选项读取请求并发;Key 用量与 Dashboard 趋势不请求并发计算,完整洞察的缓存仍按并发选择隔离。
|
||||
每个时间桶返回按持续时间加权的平均并发和峰值,Admin 按各桶的有效时长合并全区间平均值,并取全区间峰值。
|
||||
该趋势使用请求记录,不读取 Redis 准入租约,也不参与容量裁决;筛选与统计口径见[用量统计](api.md#记录范围与统计口径)
|
||||
|
||||
|
||||
@@ -418,6 +418,7 @@ utility / arbitrary variant;Vue Transition、跨浏览器 Range、动态富文
|
||||
- 表格斑马纹使用 `table-row-stripe-bg`;选中行悬停使用 `table-row-selected-hover-bg`,不被普通 Hover 覆盖
|
||||
- 结构化浮层头部使用 `popover-header-bg`,明暗模式分别检查它与表格背景的区分
|
||||
- Card 和选项默认不增加装饰性边框;键盘焦点必须保留可见反馈
|
||||
- 帮助与说明图标常态、Hover 和 Active 均保持透明背景,按钮使用 `variant="link"`,保留键盘焦点反馈;已有说明浮层时不重复显示原生 `title` 提示
|
||||
- 阴影保持中性,`shadowStrength` 只调节层级强弱,不给阴影染品牌色
|
||||
|
||||
## 扩展指南
|
||||
|
||||
@@ -336,6 +336,8 @@ export interface AccountResetCreditResultResponse {
|
||||
|
||||
export interface AccountModelsResponse {
|
||||
models: Array<{ id: string, label: string }>
|
||||
observedAt?: string | null
|
||||
observedAtDisplay?: string
|
||||
}
|
||||
|
||||
export interface AccountModelCatalogResponse {
|
||||
|
||||
@@ -327,6 +327,37 @@ export interface UsageInsightsOverviewResponse {
|
||||
cost: UsageOverviewCost
|
||||
}
|
||||
|
||||
export type UsagePerformanceMetric = 'latency' | 'firstToken' | 'throughput' | 'load'
|
||||
export type UsageCostMetric = 'cost' | 'savings' | 'cache'
|
||||
|
||||
export interface UsageMetricSeries<S, P = S> {
|
||||
summary: S
|
||||
points: { bucket: string, label: string, requestCount: number, value: P }[]
|
||||
}
|
||||
|
||||
interface UsageLatencyPercentiles {
|
||||
p50Ms: number | null
|
||||
p95Ms: number | null
|
||||
p99Ms: number | null
|
||||
}
|
||||
|
||||
type UsageCostSummary = Pick<UsageOverviewCost, 'estimatedCost' | 'cacheSavings' | 'tierPremium' | 'costPerSuccessfulRequest' | 'coverage'>
|
||||
|
||||
interface UsageMetricData {
|
||||
health: UsageOverviewHealth
|
||||
latency: UsageMetricSeries<UsageLatencyPercentiles>
|
||||
firstToken: UsageMetricSeries<UsageLatencyPercentiles>
|
||||
throughput: UsageMetricSeries<{ p10: number | null, p50: number | null, p90: number | null }>
|
||||
load: UsageMetricSeries<UsageOverviewLoad>
|
||||
cost: UsageMetricSeries<UsageCostSummary, { estimatedCost: string | null, noCacheCost: string | null }>
|
||||
savings: UsageMetricSeries<UsageCostSummary, string | null>
|
||||
cache: UsageMetricSeries<UsageCostSummary, { inputTokens: number, cachedTokenRate: number, cacheHitRequestRate: number | null }>
|
||||
}
|
||||
|
||||
export type UsageMetricResponse<M extends keyof UsageMetricData> = {
|
||||
[K in M]: { analytics: AnalyticsMetadata, granularity: string, metric: K, data: UsageMetricData[K] }
|
||||
}[M]
|
||||
|
||||
export interface UsageDiagnosticItem {
|
||||
key: string
|
||||
name: string
|
||||
@@ -427,6 +458,33 @@ export function getUsageRecordInsightsDiagnostics(data: UsageDiagnosticsQuery, o
|
||||
})
|
||||
}
|
||||
|
||||
export function getUsageHealth(data: UsageRangeQuery, options: RequestOptions = {}) {
|
||||
return request<UsageMetricResponse<'health'>>({
|
||||
url: '/api/admin/usage/insights/health',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
...options,
|
||||
})
|
||||
}
|
||||
|
||||
export function getUsagePerformance<M extends UsagePerformanceMetric>(data: UsageRangeQuery & { metric: M }, options: RequestOptions = {}) {
|
||||
return request<UsageMetricResponse<M>>({
|
||||
url: '/api/admin/usage/insights/performance',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
...options,
|
||||
})
|
||||
}
|
||||
|
||||
export function getUsageCost<M extends UsageCostMetric>(data: UsageRangeQuery & { metric: M }, options: RequestOptions = {}) {
|
||||
return request<UsageMetricResponse<M>>({
|
||||
url: '/api/admin/usage/insights/cost',
|
||||
method: 'GET',
|
||||
params: data,
|
||||
...options,
|
||||
})
|
||||
}
|
||||
|
||||
export function getDiagnosticsDownloadUrl(data: DiagnosticsDownloadParams) {
|
||||
return `${API_BASE_URL}/api/admin/diagnostics/export?${new URLSearchParams(data)}`
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ defineExpose({ focusField })
|
||||
<template v-if="field.description" #label-extra>
|
||||
<ZPopover trigger="hover-click" placement="top-start">
|
||||
<template #reference>
|
||||
<ZIconButton :aria-label="t('common.form.fieldHelp', { label: field.label })" class="size-3.5!" :title="undefined">
|
||||
<ZIconButton variant="link" :aria-label="t('common.form.fieldHelp', { label: field.label })" class="size-3.5!" :title="undefined">
|
||||
<Info class="size-3.5" />
|
||||
</ZIconButton>
|
||||
</template>
|
||||
|
||||
@@ -19,10 +19,10 @@ defineProps<{
|
||||
<template #reference>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex size-4 items-center justify-center rounded-full outline-none focus-visible:ring-2 focus-visible:ring-cp-control-outline"
|
||||
class="inline-flex size-4 items-center justify-center rounded-full bg-transparent outline-none focus-visible:ring-2 focus-visible:ring-cp-control-outline"
|
||||
:class="tone === 'warning'
|
||||
? 'bg-cp-warning-container text-cp-warning-on-container hover:bg-cp-warning-container-hover'
|
||||
: 'bg-cp-primary-container text-cp-primary-on-container hover:bg-cp-primary-container-hover'"
|
||||
? 'text-cp-warning-text'
|
||||
: 'text-cp-primary-text'"
|
||||
:aria-label="triggerLabel"
|
||||
>
|
||||
<Info class="size-3" />
|
||||
|
||||
@@ -118,7 +118,6 @@
|
||||
"noModelsAvailable": "No models available for testing"
|
||||
},
|
||||
"credentials": {
|
||||
"wsFirst": "WS first",
|
||||
"accountName": "Account name",
|
||||
"enterUpstreamName": "Enter the upstream name",
|
||||
"baseUrlHelp": "Enter the API base URL, endpoint paths are appended automatically",
|
||||
@@ -126,7 +125,7 @@
|
||||
"apiKeyPlaceholder": "Leave blank to keep the current key",
|
||||
"enterUpstreamKey": "Enter the upstream key",
|
||||
"transportLabel": "Transport",
|
||||
"webSocketHelp": "WS first requires upstream support for Responses WebSocket",
|
||||
"webSocketHelp": "WS requires upstream support for Responses WebSocket",
|
||||
"nameRequired": "Enter an account name",
|
||||
"validation": {
|
||||
"urlLength": "Upstream API URL must not exceed 2,048 characters",
|
||||
@@ -163,7 +162,6 @@
|
||||
"editor": {
|
||||
"title": "Edit account",
|
||||
"transportLabel": "Upstream transport",
|
||||
"upstreamConnection": "Upstream connection",
|
||||
"loadError": "Could not load upstream settings",
|
||||
"notesHelp": "Up to 500 characters, leave blank to clear notes",
|
||||
"oauthUnavailable": "This account has no OAuth upstream settings",
|
||||
@@ -188,9 +186,12 @@
|
||||
"reserve": "Reserve",
|
||||
"allowlist": "Allowlist",
|
||||
"blocklist": "Blocklist",
|
||||
"modelLimitations": "Model limitations",
|
||||
"modelCatalog": "Model catalog",
|
||||
"refreshModel": "Refresh model",
|
||||
"modelRestrictionMode": "Model restriction mode",
|
||||
"catalogHelpTitle": "Model catalog details",
|
||||
"catalogObservedAtHelp": "The time shows when the upstream catalog was last successfully fetched",
|
||||
"catalogDescription": "The catalog lists model IDs, verify endpoint support with an actual request — a connection test only checks the endpoint it uses",
|
||||
"searchOrAddModels": "Search or add models",
|
||||
"searchOrEnterIdAndPressEnter": "Search or enter ID and press Enter to add",
|
||||
"selectModel": "Select model",
|
||||
@@ -229,6 +230,7 @@
|
||||
"outboundTunnel": "Outbound tunnel"
|
||||
},
|
||||
"scheduling": {
|
||||
"preserve": "Keep unchanged",
|
||||
"scheduling": "Scheduling",
|
||||
"switchAccountSchedule": "Switch account schedule",
|
||||
"concurrencyLimit": "Concurrency limit",
|
||||
|
||||
@@ -118,7 +118,6 @@
|
||||
"noModelsAvailable": "没有可测试模型"
|
||||
},
|
||||
"credentials": {
|
||||
"wsFirst": "WS 优先",
|
||||
"accountName": "账号名称",
|
||||
"enterUpstreamName": "输入上游名称",
|
||||
"baseUrlHelp": "填写 API 前缀,接口路径会自动追加",
|
||||
@@ -126,7 +125,7 @@
|
||||
"apiKeyPlaceholder": "留空保留当前密钥",
|
||||
"enterUpstreamKey": "输入上游密钥",
|
||||
"transportLabel": "传输方式",
|
||||
"webSocketHelp": "WS 优先需要上游支持 Responses WebSocket",
|
||||
"webSocketHelp": "WS 需要上游支持 Responses WebSocket",
|
||||
"nameRequired": "请输入账号名称",
|
||||
"validation": {
|
||||
"urlLength": "上游 API 地址不能超过 2048 个字符",
|
||||
@@ -163,7 +162,6 @@
|
||||
"editor": {
|
||||
"title": "编辑账号",
|
||||
"transportLabel": "上游传输方式",
|
||||
"upstreamConnection": "上游连接",
|
||||
"loadError": "上游设置读取失败",
|
||||
"notesHelp": "最多 500 字,留空可清除备注",
|
||||
"oauthUnavailable": "该账号没有 OAuth 上游设置",
|
||||
@@ -188,9 +186,12 @@
|
||||
"reserve": "保留",
|
||||
"allowlist": "白名单",
|
||||
"blocklist": "黑名单",
|
||||
"modelLimitations": "模型限制",
|
||||
"modelCatalog": "模型目录",
|
||||
"refreshModel": "刷新模型",
|
||||
"modelRestrictionMode": "模型限制模式",
|
||||
"catalogHelpTitle": "模型目录说明",
|
||||
"catalogObservedAtHelp": "时间表示最近一次成功获取上游目录的时间",
|
||||
"catalogDescription": "目录仅提供模型 ID,端点能力请通过实际请求确认,连接测试仅验证测试所用端点",
|
||||
"searchOrAddModels": "搜索或添加模型",
|
||||
"searchOrEnterIdAndPressEnter": "搜索或输入 ID,回车添加",
|
||||
"selectModel": "选择模型",
|
||||
@@ -230,6 +231,7 @@
|
||||
},
|
||||
"scheduling": {
|
||||
"scheduling": "调度",
|
||||
"preserve": "保留",
|
||||
"switchAccountSchedule": "切换账号调度",
|
||||
"concurrencyLimit": "并发限制",
|
||||
"accountConcurrencyLimit": "账号并发限制",
|
||||
|
||||
@@ -118,7 +118,6 @@
|
||||
"noModelsAvailable": "沒有可測試模型"
|
||||
},
|
||||
"credentials": {
|
||||
"wsFirst": "WS 優先",
|
||||
"accountName": "賬號名稱",
|
||||
"enterUpstreamName": "輸入上游名稱",
|
||||
"baseUrlHelp": "填寫 API 前綴,接口路徑會自動追加",
|
||||
@@ -126,7 +125,7 @@
|
||||
"apiKeyPlaceholder": "留空保留當前密鑰",
|
||||
"enterUpstreamKey": "輸入上游密鑰",
|
||||
"transportLabel": "傳輸方式",
|
||||
"webSocketHelp": "WS 優先需要上游支持 Responses WebSocket",
|
||||
"webSocketHelp": "WS 需要上游支持 Responses WebSocket",
|
||||
"nameRequired": "請輸入賬號名稱",
|
||||
"validation": {
|
||||
"urlLength": "上游 API 地址不能超過 2048 個字符",
|
||||
@@ -163,7 +162,6 @@
|
||||
"editor": {
|
||||
"title": "編輯賬號",
|
||||
"transportLabel": "上游傳輸方式",
|
||||
"upstreamConnection": "上游連接",
|
||||
"loadError": "上游設置讀取失敗",
|
||||
"notesHelp": "最多 500 字,留空可清除備註",
|
||||
"oauthUnavailable": "該賬號沒有 OAuth 上游設置",
|
||||
@@ -188,9 +186,12 @@
|
||||
"reserve": "保留",
|
||||
"allowlist": "白名單",
|
||||
"blocklist": "黑名單",
|
||||
"modelLimitations": "模型限制",
|
||||
"modelCatalog": "模型目錄",
|
||||
"refreshModel": "刷新模型",
|
||||
"modelRestrictionMode": "模型限制模式",
|
||||
"catalogHelpTitle": "模型目錄說明",
|
||||
"catalogObservedAtHelp": "時間表示最近一次成功取得上游目錄的時間",
|
||||
"catalogDescription": "目錄僅提供模型 ID,端點能力請透過實際請求確認,連線測試僅驗證測試所用端點",
|
||||
"searchOrAddModels": "搜索或添加模型",
|
||||
"searchOrEnterIdAndPressEnter": "搜索或輸入 ID,回車添加",
|
||||
"selectModel": "選擇模型",
|
||||
@@ -229,6 +230,7 @@
|
||||
"outboundTunnel": "出站隧道"
|
||||
},
|
||||
"scheduling": {
|
||||
"preserve": "保留",
|
||||
"scheduling": "調度",
|
||||
"switchAccountSchedule": "切換賬號調度",
|
||||
"concurrencyLimit": "併發限制",
|
||||
|
||||
@@ -481,7 +481,7 @@ watch(
|
||||
/>
|
||||
<ZPopover placement="top-start">
|
||||
<template #reference>
|
||||
<ZButton variant="text" size="small" square :aria-label="t('systemUpdate.channels.help')">
|
||||
<ZButton variant="link" size="small" square :aria-label="t('systemUpdate.channels.help')">
|
||||
<CircleHelp class="size-4" />
|
||||
</ZButton>
|
||||
</template>
|
||||
|
||||
@@ -3,7 +3,6 @@ import type { ApiKeyAccountForm } from '../utils/upstreamApiKey'
|
||||
|
||||
import { ZFormItem, ZInput, ZSegmented } from '@codex-proxy/ui'
|
||||
|
||||
import { computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
defineProps<{ disabled?: boolean, editing?: boolean }>()
|
||||
@@ -11,13 +10,10 @@ defineProps<{ disabled?: boolean, editing?: boolean }>()
|
||||
const { t } = useI18n({ useScope: 'global' })
|
||||
|
||||
const form = defineModel<ApiKeyAccountForm>({ required: true })
|
||||
const transportOptions = computed(() => ([
|
||||
{ label: 'HTTP / SSE', value: 'http' },
|
||||
{
|
||||
label: t('accounts.credentials.wsFirst'),
|
||||
value: 'prefer_websocket',
|
||||
},
|
||||
]))
|
||||
const transportOptions = [
|
||||
{ label: 'SSE', value: 'http' },
|
||||
{ label: 'WS', value: 'prefer_websocket' },
|
||||
]
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
<script setup lang="ts">
|
||||
import type { AccountGroupRef, AccountModelAccess } from '@/api'
|
||||
|
||||
import { ZButton, ZDialog } from '@codex-proxy/ui'
|
||||
import { ZButton, ZDialog, ZFormItem, ZSegmented } from '@codex-proxy/ui'
|
||||
|
||||
import { computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import AccountSettingsFields from './AccountSettingsFields.vue'
|
||||
|
||||
@@ -21,13 +22,37 @@ const emit = defineEmits<{
|
||||
const { t } = useI18n({ useScope: 'global' })
|
||||
|
||||
const open = defineModel<boolean>({ required: true })
|
||||
const enabled = defineModel<boolean>('enabled', { required: true })
|
||||
const enabled = defineModel<boolean | undefined>('enabled', { required: true })
|
||||
const concurrencyLimit = defineModel<string>('concurrencyLimit', { required: true })
|
||||
const modelAccess = defineModel<AccountModelAccess | undefined>('modelAccess', { required: true })
|
||||
const weight = defineModel<string>('weight', { required: true })
|
||||
const proxyMode = defineModel<string>('proxyMode', { required: true })
|
||||
const proxyId = defineModel<string>('proxyId', { required: true })
|
||||
const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: true })
|
||||
const schedulingModes = computed(() => [
|
||||
{
|
||||
value: 'preserve',
|
||||
label: t('accounts.scheduling.preserve'),
|
||||
},
|
||||
{
|
||||
value: 'enabled',
|
||||
label: t('accounts.actions.enableScheduling'),
|
||||
},
|
||||
{
|
||||
value: 'disabled',
|
||||
label: t('accounts.actions.disableScheduling'),
|
||||
},
|
||||
])
|
||||
const schedulingMode = computed({
|
||||
get: () => enabled.value === undefined
|
||||
? 'preserve'
|
||||
: enabled.value ? 'enabled' : 'disabled',
|
||||
set: (value: string) => {
|
||||
enabled.value = value === 'preserve'
|
||||
? undefined
|
||||
: value === 'enabled'
|
||||
},
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -40,7 +65,6 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
>
|
||||
<div class="grid gap-5">
|
||||
<AccountSettingsFields
|
||||
v-model:enabled="enabled"
|
||||
v-model:concurrency-limit="concurrencyLimit"
|
||||
v-model:weight="weight"
|
||||
v-model:model-access="modelAccess"
|
||||
@@ -52,7 +76,19 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
:groups="groups"
|
||||
:groups-loading="groupsLoading"
|
||||
:disabled="saving"
|
||||
/>
|
||||
>
|
||||
<template #scheduling>
|
||||
<ZFormItem :label="t('accounts.scheduling.scheduling')">
|
||||
<ZSegmented
|
||||
v-model="schedulingMode"
|
||||
class="w-80 min-w-0 max-w-full"
|
||||
:aria-label="t('accounts.scheduling.scheduling')"
|
||||
:options="schedulingModes"
|
||||
:disabled="saving"
|
||||
/>
|
||||
</ZFormItem>
|
||||
</template>
|
||||
</AccountSettingsFields>
|
||||
</div>
|
||||
|
||||
<template #footer>
|
||||
|
||||
@@ -58,7 +58,7 @@ function connectionLogClass(tone: string) {
|
||||
<section
|
||||
class="flex items-center justify-between gap-4 rounded-cp-card bg-cp-fill-quaternary px-4 py-3"
|
||||
>
|
||||
<AccountIdentityCell :account="account" size="lg" show-plan />
|
||||
<AccountIdentityCell :account="account" size="lg" show-plan show-notes />
|
||||
<AccountStatusBadge
|
||||
:status="account.status"
|
||||
:error-reason="account.errorReason"
|
||||
|
||||
@@ -59,6 +59,7 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
class="min-w-0 flex-1"
|
||||
:account="account"
|
||||
size="lg"
|
||||
show-notes
|
||||
/>
|
||||
<div class="flex shrink-0 items-center gap-3">
|
||||
<AccountPlanBadge :authentication-kind="account.authenticationKind" :plan-type="account.planType" :plan-type-display="account.planTypeDisplay" size="sm" />
|
||||
@@ -73,12 +74,7 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
<ZSegmented v-model="oauthTransport" :aria-label="t('accounts.editor.transportLabel')" :options="transportOptions" :disabled="saving || !configurationReady" />
|
||||
</ZFormItem>
|
||||
|
||||
<section v-if="isOpenAiApiKeyAccount(account)" class="grid gap-4">
|
||||
<h3 class="m-0 text-cp font-heavy text-cp-text">
|
||||
{{ t('accounts.editor.upstreamConnection') }}
|
||||
</h3>
|
||||
<AccountApiKeyFields v-model="apiKey" editing :disabled="saving || !configurationReady" />
|
||||
</section>
|
||||
<AccountApiKeyFields v-if="isOpenAiApiKeyAccount(account)" v-model="apiKey" editing :disabled="saving || !configurationReady" />
|
||||
|
||||
<AccountSettingsFields
|
||||
v-model:enabled="enabled"
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { AccountModelAccess } from '@/api'
|
||||
|
||||
import { ZCheckbox, ZEmpty, ZFormItem, ZIconButton, ZInput, ZPopover, ZScrollbar, ZSegmented } from '@codex-proxy/ui'
|
||||
|
||||
import { RefreshCw, Search, TriangleAlert } from '@lucide/vue'
|
||||
import { CircleAlert, RefreshCw, Search, TriangleAlert } from '@lucide/vue'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { getAccountModels, refreshAccountModels } from '@/api'
|
||||
@@ -21,6 +21,7 @@ const { t } = useI18n({ useScope: 'global' })
|
||||
const model = defineModel<AccountModelAccess | undefined>({ required: true })
|
||||
// 未取得目录时不能判断缺失,成功返回空目录则可以
|
||||
const catalog = ref<Array<{ id: string, label: string }>>()
|
||||
const catalogObservedAtDisplay = ref<string>()
|
||||
const search = ref('')
|
||||
const inputError = ref('')
|
||||
const request = useRequestState()
|
||||
@@ -113,6 +114,15 @@ function addModel() {
|
||||
search.value = ''
|
||||
}
|
||||
|
||||
function handleCatalogHelpKeydown(event: KeyboardEvent, open: boolean, close: () => void) {
|
||||
if (event.key === 'Escape' && open) {
|
||||
// 先关闭说明浮层,避免同一次按键也关闭编辑弹窗
|
||||
event.preventDefault()
|
||||
event.stopPropagation()
|
||||
close()
|
||||
}
|
||||
}
|
||||
|
||||
async function load(refresh = false) {
|
||||
const accountId = props.accountId
|
||||
if (!accountId || !restricted.value)
|
||||
@@ -123,8 +133,10 @@ async function load(refresh = false) {
|
||||
{ accountId },
|
||||
{ signal: request.signal },
|
||||
)
|
||||
if (request.isCurrent(requestId))
|
||||
if (request.isCurrent(requestId)) {
|
||||
catalog.value = result.models
|
||||
catalogObservedAtDisplay.value = result.observedAt ? result.observedAtDisplay : undefined
|
||||
}
|
||||
}
|
||||
catch (cause) {
|
||||
request.fail(requestId, cause)
|
||||
@@ -140,6 +152,7 @@ watch([
|
||||
], () => {
|
||||
request.invalidate()
|
||||
catalog.value = undefined
|
||||
catalogObservedAtDisplay.value = undefined
|
||||
search.value = ''
|
||||
inputError.value = ''
|
||||
error.value = ''
|
||||
@@ -153,7 +166,40 @@ watch(search, () => {
|
||||
|
||||
<template>
|
||||
<div class="grid gap-3">
|
||||
<ZFormItem :label="t('accounts.modelAccess.modelLimitations')">
|
||||
<ZFormItem>
|
||||
<template #label>
|
||||
<span class="inline-flex flex-wrap items-end gap-x-2 gap-y-1">
|
||||
<span>{{ t('accounts.modelAccess.modelCatalog') }}</span>
|
||||
<span v-if="restricted && catalogObservedAtDisplay" class="text-cp-xs leading-none font-normal text-cp-text-tertiary tabular-nums">
|
||||
{{ catalogObservedAtDisplay }}
|
||||
</span>
|
||||
</span>
|
||||
</template>
|
||||
<template v-if="restricted" #label-extra>
|
||||
<ZPopover class="-my-1 -ml-1.5 shrink-0" trigger="hover-click" placement="top-start" role="tooltip">
|
||||
<template #reference="{ open, id, close }">
|
||||
<ZIconButton
|
||||
variant="link"
|
||||
class="size-6!"
|
||||
:aria-label="t('accounts.modelAccess.catalogHelpTitle')"
|
||||
:aria-describedby="open ? id : undefined"
|
||||
:aria-expanded="open"
|
||||
:title="undefined"
|
||||
@keydown="handleCatalogHelpKeydown($event, open, close)"
|
||||
>
|
||||
<CircleAlert class="size-[1em] text-cp-xs" aria-hidden="true" />
|
||||
</ZIconButton>
|
||||
</template>
|
||||
<div class="max-w-72 space-y-2 px-3 py-2 text-cp-sm leading-relaxed text-cp-text-secondary">
|
||||
<p class="m-0">
|
||||
{{ t('accounts.modelAccess.catalogObservedAtHelp') }}
|
||||
</p>
|
||||
<p class="m-0">
|
||||
{{ t('accounts.modelAccess.catalogDescription') }}
|
||||
</p>
|
||||
</div>
|
||||
</ZPopover>
|
||||
</template>
|
||||
<div class="flex min-w-0 items-center justify-between gap-2">
|
||||
<ZSegmented v-model="mode" class="w-80 min-w-0 max-w-full" :aria-label="t('accounts.modelAccess.modelRestrictionMode')" :options="modes" :disabled="disabled" />
|
||||
<ZIconButton v-if="restricted && accountId" :aria-label="t('accounts.modelAccess.refreshModel')" variant="plain" :loading="loading" :disabled="disabled" @click="load(true)">
|
||||
@@ -196,7 +242,7 @@ watch(search, () => {
|
||||
<ZPopover v-if="item.unavailable" class="mr-1 shrink-0" trigger="hover-click" placement="top" role="tooltip">
|
||||
<template #reference="{ open, id }">
|
||||
<ZIconButton
|
||||
class="bg-transparent! hover:bg-transparent! active:bg-transparent!"
|
||||
variant="link"
|
||||
size="small"
|
||||
:aria-label="t('accounts.modelAccess.notInCurrentModelCatalog', { id: item.id })"
|
||||
:aria-describedby="open ? id : undefined"
|
||||
|
||||
@@ -45,8 +45,7 @@ watch(() => props.notes, recheckOpen, { flush: 'post' })
|
||||
<template #reference>
|
||||
<button
|
||||
type="button"
|
||||
class="inline-flex min-w-0 max-w-full touch-manipulation items-center rounded-cp-sm border-0 bg-transparent p-0 text-left text-cp-xs leading-4 text-cp-text-tertiary outline-none transition-colors hover:text-cp-primary-text focus-visible:ring-2 focus-visible:ring-cp-control-outline motion-reduce:transition-none"
|
||||
:class="open ? 'text-cp-primary-text' : ''"
|
||||
class="inline-flex min-w-0 max-w-full cursor-default touch-manipulation items-center rounded-cp-sm border-0 bg-transparent p-0 text-left text-cp-xs leading-4 text-cp-text-tertiary outline-none focus-visible:ring-2 focus-visible:ring-cp-control-outline"
|
||||
:aria-label="t('accounts.notes.viewAccountNotes')"
|
||||
aria-haspopup="dialog"
|
||||
:aria-expanded="open"
|
||||
|
||||
@@ -23,7 +23,7 @@ withDefaults(defineProps<{
|
||||
const { t } = useI18n({ useScope: 'global' })
|
||||
|
||||
const modelAccess = defineModel<AccountModelAccess | undefined>('modelAccess', { required: true })
|
||||
const enabled = defineModel<boolean>('enabled', { required: true })
|
||||
const enabled = defineModel<boolean>('enabled')
|
||||
const concurrencyLimit = defineModel<string>('concurrencyLimit', { required: true })
|
||||
const weight = defineModel<string>('weight', { required: true })
|
||||
const proxyMode = defineModel<string>('proxyMode', { required: true })
|
||||
@@ -34,14 +34,16 @@ const selectedGroupIds = defineModel<string[]>('selectedGroupIds', { required: t
|
||||
<template>
|
||||
<div class="grid gap-5">
|
||||
<AccountModelAccessField v-model="modelAccess" :account-id="accountId" :disabled="disabled" :allow-preserve="preserveModelAccess" />
|
||||
<div v-if="showScheduling" class="flex min-h-6 items-center justify-between gap-3">
|
||||
<span class="text-cp leading-none font-medium text-cp-text-secondary">{{ t('accounts.scheduling.scheduling') }}</span>
|
||||
<ZSwitch
|
||||
v-model="enabled"
|
||||
:aria-label="t('accounts.scheduling.switchAccountSchedule')"
|
||||
:disabled="disabled"
|
||||
/>
|
||||
</div>
|
||||
<slot name="scheduling">
|
||||
<div v-if="showScheduling" class="flex min-h-6 items-center justify-between gap-3">
|
||||
<span class="text-cp leading-none font-medium text-cp-text-secondary">{{ t('accounts.scheduling.scheduling') }}</span>
|
||||
<ZSwitch
|
||||
v-model="enabled"
|
||||
:aria-label="t('accounts.scheduling.switchAccountSchedule')"
|
||||
:disabled="disabled"
|
||||
/>
|
||||
</div>
|
||||
</slot>
|
||||
|
||||
<div class="grid gap-4 sm:grid-cols-2">
|
||||
<ZFormItem :label="t('accounts.scheduling.concurrencyLimit')">
|
||||
|
||||
@@ -69,6 +69,8 @@ function handleRetentionKeydown(event: KeyboardEvent) {
|
||||
<ZPopover v-model="retentionOpen" trigger="hover-click" placement="top-start" :hover-delay="240">
|
||||
<template #reference>
|
||||
<ZIconButton
|
||||
variant="link"
|
||||
:title="undefined"
|
||||
:aria-label="t('accounts.importTasks.taskRetentionInstructions')"
|
||||
size="small"
|
||||
:aria-expanded="retentionOpen"
|
||||
|
||||
@@ -116,6 +116,8 @@ function handleExplanationKeydown(event: KeyboardEvent) {
|
||||
<ZPopover v-model="explanationOpen" trigger="hover-click" placement="top-start" :hover-delay="240">
|
||||
<template #reference>
|
||||
<ZIconButton
|
||||
variant="link"
|
||||
:title="undefined"
|
||||
:aria-label="t('accounts.quotaForecast.forecastDescription')"
|
||||
size="small"
|
||||
:aria-expanded="explanationOpen"
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ watch(() => props.accountId, () => {
|
||||
</h3>
|
||||
<ZPopover trigger="click" placement="bottom-start" role="dialog">
|
||||
<template #reference="{ open, id }">
|
||||
<ZIconButton size="small" class="size-6! p-0!" :aria-label="t('accounts.quotaReset.automaticUsageRules')" :aria-expanded="open" :aria-controls="id">
|
||||
<ZIconButton variant="link" :title="undefined" size="small" class="size-6! p-0!" :aria-label="t('accounts.quotaReset.automaticUsageRules')" :aria-expanded="open" :aria-controls="id">
|
||||
<CircleHelp class="size-3.5" />
|
||||
</ZIconButton>
|
||||
</template>
|
||||
|
||||
@@ -18,7 +18,7 @@ export function useAccountBatchEditor(options: {
|
||||
const selectedAccountsById = new Map<string, Account>()
|
||||
const showBatchEditModal = shallowRef(false)
|
||||
const editingCount = shallowRef(0)
|
||||
const schedulingEnabled = shallowRef(false)
|
||||
const schedulingEnabled = shallowRef<boolean>()
|
||||
const concurrencyLimit = shallowRef('')
|
||||
const weight = shallowRef('1')
|
||||
const modelAccess = ref<AccountModelAccess | undefined>()
|
||||
@@ -30,7 +30,7 @@ export function useAccountBatchEditor(options: {
|
||||
const saveAction = useAsyncAction()
|
||||
const saving = saveAction.loading
|
||||
|
||||
// 调度按开关值统一更新,其他未操作字段保留每个账号原值
|
||||
// 未操作的批量字段保留每个账号原值
|
||||
watch([concurrencyLimit, weight, selectedGroupIds], (values, previous) => {
|
||||
if (!showBatchEditModal.value || saving.value)
|
||||
return
|
||||
@@ -50,7 +50,7 @@ export function useAccountBatchEditor(options: {
|
||||
modelAccess.value = undefined
|
||||
editedFields.value.clear()
|
||||
catalogAccountId.value = accounts[0]?.id
|
||||
schedulingEnabled.value = false
|
||||
schedulingEnabled.value = undefined
|
||||
proxyMode.value = 'preserve'
|
||||
proxyId.value = ''
|
||||
concurrencyLimit.value = sharedConcurrencyLimit(accounts)
|
||||
@@ -62,6 +62,11 @@ export function useAccountBatchEditor(options: {
|
||||
async function save() {
|
||||
if (saving.value || options.selectedIds.value.size === 0)
|
||||
return
|
||||
if (schedulingEnabled.value === undefined && modelAccess.value === undefined
|
||||
&& editedFields.value.size === 0 && proxyMode.value === 'preserve') {
|
||||
showBatchEditModal.value = false
|
||||
return
|
||||
}
|
||||
const modelError = accountModelAccessError(modelAccess.value)
|
||||
if (modelError) {
|
||||
ZNotification.warning({ message: modelError })
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user