mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-13 08:06:41 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6e1b347be | ||
|
|
1ac9803c34 | ||
|
|
352b781340 | ||
|
|
62a6dfc489 | ||
|
|
e91f91b542 | ||
|
|
380556369b |
@@ -424,6 +424,34 @@ impl From<AdminAccountRefreshResult> for AccountRefreshData {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountQuotaData {
|
||||
quota: Value,
|
||||
raw: Value,
|
||||
quota_data: AdminAccountQuotaData,
|
||||
plan_type: Option<String>,
|
||||
account: AdminAccountData,
|
||||
}
|
||||
|
||||
impl AccountQuotaData {
|
||||
fn from_account(
|
||||
quota: Value,
|
||||
raw: Value,
|
||||
quota_data: AdminAccountQuotaData,
|
||||
account: AdminAccountData,
|
||||
) -> Self {
|
||||
let plan_type = account.plan_type.clone();
|
||||
Self {
|
||||
quota,
|
||||
raw,
|
||||
quota_data,
|
||||
plan_type,
|
||||
account,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct AccountHealthCheckRequest {
|
||||
@@ -664,18 +692,13 @@ pub(crate) async fn account_quota(
|
||||
let quota_json = quota.to_string();
|
||||
let mut quota_data = quota_data("a_json, Some(Utc::now()));
|
||||
apply_account_quota_window_token_usage(&state, &account_id, &mut quota_data).await;
|
||||
let plan_type = quota
|
||||
.get("plan_type")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToString::to_string);
|
||||
let account =
|
||||
account_data_for_quota_refresh(&state, &account_id, quota_data.clone()).await?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(serde_json::json!({
|
||||
"quota": quota,
|
||||
"raw": raw,
|
||||
"quotaData": quota_data,
|
||||
"planType": plan_type,
|
||||
})),
|
||||
AdminEnvelope::ok(AccountQuotaData::from_account(
|
||||
quota, raw, quota_data, account,
|
||||
)),
|
||||
))
|
||||
}
|
||||
Err(AdminAccountError::NotFound) => Err(account_not_found()),
|
||||
@@ -690,6 +713,25 @@ pub(crate) async fn account_quota(
|
||||
}
|
||||
}
|
||||
|
||||
async fn account_data_for_quota_refresh(
|
||||
state: &AppState,
|
||||
account_id: &str,
|
||||
quota_data: AdminAccountQuotaData,
|
||||
) -> Result<AdminAccountData, AdminError> {
|
||||
let Some(account) = state
|
||||
.services
|
||||
.admin_accounts
|
||||
.get(account_id)
|
||||
.await
|
||||
.map_err(|error| account_error(&error))?
|
||||
else {
|
||||
return Err(account_not_found());
|
||||
};
|
||||
let quota_by_account = HashMap::from([(account_id.to_string(), quota_data)]);
|
||||
let stats = account_list_stats(state, std::slice::from_ref(&account), "a_by_account).await;
|
||||
Ok(stats.data_for(account))
|
||||
}
|
||||
|
||||
/// `POST /api/admin/accounts/import`
|
||||
pub(crate) async fn import_accounts(
|
||||
State(state): State<AppState>,
|
||||
|
||||
@@ -38,15 +38,11 @@ impl AdminAccountService {
|
||||
.await
|
||||
.map_err(|e| AdminAccountError::FetchQuota(e.to_string()))?;
|
||||
let normalized = crate::upstream::accounts::quota::quota_from_usage(&raw);
|
||||
if let Ok(json_str) = serde_json::to_string(&normalized) {
|
||||
if matches!(
|
||||
self.store.update_quota_json(account_id, &json_str).await,
|
||||
Ok(true)
|
||||
) {
|
||||
self.sync_account_pool_best_effort(account_id, "account quota refresh")
|
||||
.await;
|
||||
}
|
||||
}
|
||||
self.account_pool
|
||||
.apply_quota_snapshot(account_id, &normalized)
|
||||
.await;
|
||||
self.sync_account_pool_best_effort(account_id, "account quota refresh")
|
||||
.await;
|
||||
Ok(serde_json::json!({ "quota": normalized, "raw": raw }))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -463,7 +463,7 @@ fn log_error(error: &AdminUsageRecordError) -> AdminError {
|
||||
| AdminUsageRecordError::Get
|
||||
| AdminUsageRecordError::Clear
|
||||
| AdminUsageRecordError::Append
|
||||
| AdminUsageRecordError::Trim => AdminError::internal(error.to_string()),
|
||||
| AdminUsageRecordError::Retention => AdminError::internal(error.to_string()),
|
||||
AdminUsageRecordError::Accounts => {
|
||||
AdminError::usage_record_accounts_failed(error.to_string())
|
||||
}
|
||||
|
||||
@@ -117,9 +117,9 @@ pub enum AdminUsageRecordError {
|
||||
/// 写入失败。
|
||||
#[error("failed to append usage record")]
|
||||
Append,
|
||||
/// 裁剪失败。
|
||||
#[error("failed to trim usage records")]
|
||||
Trim,
|
||||
/// 保留期清理失败。
|
||||
#[error("failed to trim expired usage records")]
|
||||
Retention,
|
||||
/// 账号关联信息读取失败。
|
||||
#[error("failed to load usage record accounts")]
|
||||
Accounts,
|
||||
@@ -135,23 +135,16 @@ pub struct AdminUsageRecordService {
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct AdminUsageRecordSettings {
|
||||
enabled: bool,
|
||||
capacity: u32,
|
||||
capture_body: bool,
|
||||
}
|
||||
|
||||
impl AdminUsageRecordService {
|
||||
/// 构造管理端使用记录服务。
|
||||
pub fn new(
|
||||
store: SqliteUsageRecordStore,
|
||||
enabled: bool,
|
||||
capacity: u32,
|
||||
capture_body: bool,
|
||||
) -> Self {
|
||||
pub fn new(store: SqliteUsageRecordStore, enabled: bool, capture_body: bool) -> Self {
|
||||
Self {
|
||||
store,
|
||||
settings: Arc::new(tokio::sync::RwLock::new(AdminUsageRecordSettings {
|
||||
enabled,
|
||||
capacity,
|
||||
capture_body,
|
||||
})),
|
||||
}
|
||||
@@ -289,9 +282,9 @@ impl AdminUsageRecordService {
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Append)?;
|
||||
self.store
|
||||
.trim_to_capacity(settings.capacity)
|
||||
.trim_to_retention(Utc::now())
|
||||
.await
|
||||
.map_err(|_| AdminUsageRecordError::Trim)?;
|
||||
.map_err(|_| AdminUsageRecordError::Retention)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use serde_json::Value;
|
||||
use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool};
|
||||
use thiserror::Error;
|
||||
@@ -19,8 +19,8 @@ use crate::infra::{
|
||||
time::{china_quarter_hour_start, parse_rfc3339_utc as parse_rfc3339},
|
||||
};
|
||||
|
||||
/// 默认保留最新使用记录数量。
|
||||
pub const DEFAULT_USAGE_RECORD_CAPACITY: u32 = 2_000;
|
||||
/// 使用记录保留天数。
|
||||
pub const USAGE_RECORD_RETENTION_DAYS: i64 = 30;
|
||||
/// 默认不保存请求/响应体,避免把用户内容写入诊断日志。
|
||||
pub const DEFAULT_USAGE_RECORD_CAPTURE_BODY: bool = false;
|
||||
|
||||
@@ -105,9 +105,9 @@ impl SqliteUsageRecordStore {
|
||||
append_event(&self.pool, event).await
|
||||
}
|
||||
|
||||
/// 保留最新的指定数量使用记录。
|
||||
pub async fn trim_to_capacity(&self, capacity: u32) -> SqliteUsageRecordStoreResult<u64> {
|
||||
trim_to_capacity(&self.pool, capacity).await
|
||||
/// 按保留期清理使用记录。
|
||||
pub async fn trim_to_retention(&self, now: DateTime<Utc>) -> SqliteUsageRecordStoreResult<u64> {
|
||||
trim_to_retention(&self.pool, now).await
|
||||
}
|
||||
|
||||
/// 分页查询使用记录。
|
||||
@@ -286,7 +286,6 @@ fn usage_record_summary_fields(event: &UsageRecord) -> UsageRecordSummaryFields
|
||||
"upstreamRequestId",
|
||||
"upstream_request_id",
|
||||
"openaiRequestId",
|
||||
"cfRay",
|
||||
],
|
||||
)
|
||||
}),
|
||||
@@ -426,17 +425,15 @@ fn metadata_first_token_latency(metadata: &Value) -> Option<i64> {
|
||||
})
|
||||
}
|
||||
|
||||
async fn trim_to_capacity(pool: &SqlitePool, capacity: u32) -> SqliteUsageRecordStoreResult<u64> {
|
||||
let result = sqlx::query(
|
||||
"delete from usage_records where id in (
|
||||
select id from usage_records
|
||||
order by created_at desc, id desc
|
||||
limit -1 offset ?
|
||||
)",
|
||||
)
|
||||
.bind(i64::from(capacity))
|
||||
.execute(pool)
|
||||
.await?;
|
||||
async fn trim_to_retention(
|
||||
pool: &SqlitePool,
|
||||
now: DateTime<Utc>,
|
||||
) -> SqliteUsageRecordStoreResult<u64> {
|
||||
let cutoff = now - Duration::days(USAGE_RECORD_RETENTION_DAYS);
|
||||
let result = sqlx::query("delete from usage_records where created_at < ?")
|
||||
.bind(cutoff.to_rfc3339())
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
|
||||
@@ -90,60 +90,3 @@ fn parse_installation_id(raw: &str) -> Option<String> {
|
||||
fn generate_installation_id() -> String {
|
||||
Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::*;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn data_dir_should_use_xdg_data_home_directly() {
|
||||
let _guard = ENV_LOCK.lock().unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let previous = std::env::var_os("XDG_DATA_HOME");
|
||||
std::env::set_var("XDG_DATA_HOME", dir.path());
|
||||
|
||||
let resolved = data_dir();
|
||||
|
||||
restore_env("XDG_DATA_HOME", previous);
|
||||
assert_eq!(resolved, dir.path());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_persist_under_data_dir_root() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let data_path = dir.path().join(INSTALLATION_ID_FILE_NAME);
|
||||
|
||||
let installation_id = load_or_create_installation_id_from_paths(None, Some(&data_path))
|
||||
.expect("installation id should be generated");
|
||||
|
||||
assert_eq!(fs::read_to_string(data_path).unwrap(), installation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let codex_path = dir.path().join(".codex").join(INSTALLATION_ID_FILE_NAME);
|
||||
let data_path = dir.path().join("data").join(INSTALLATION_ID_FILE_NAME);
|
||||
let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a";
|
||||
persist_installation_id(&codex_path, desktop_id).unwrap();
|
||||
|
||||
let installation_id =
|
||||
load_or_create_installation_id_from_paths(Some(&codex_path), Some(&data_path))
|
||||
.expect("installation id should be seeded");
|
||||
|
||||
assert_eq!(installation_id, desktop_id);
|
||||
assert_eq!(fs::read_to_string(data_path).unwrap(), desktop_id);
|
||||
}
|
||||
|
||||
fn restore_env(key: &str, value: Option<std::ffi::OsString>) {
|
||||
if let Some(value) = value {
|
||||
std::env::set_var(key, value);
|
||||
} else {
|
||||
std::env::remove_var(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ use serde_json::{json, Map, Value};
|
||||
use crate::{
|
||||
proxy::dispatch::exhaustion::{ExhaustedAccountKind, ExhaustedAccountRef},
|
||||
upstream::accounts::model::AccountStatus,
|
||||
upstream::transport::{is_banned_auth_signal, CodexBackendTransport, CodexClientError},
|
||||
upstream::transport::{
|
||||
is_banned_auth_signal, CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
|
||||
},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -15,6 +17,7 @@ pub(crate) struct DispatchErrorMetadata {
|
||||
pub exhausted_count: Option<usize>,
|
||||
pub upstream_error: Option<String>,
|
||||
pub upstream_status: Option<u16>,
|
||||
pub diagnostics: Option<CodexUpstreamDiagnostics>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
@@ -63,6 +66,7 @@ impl DispatchErrorMetadata {
|
||||
exhausted_count: None,
|
||||
upstream_error: None,
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +76,7 @@ impl DispatchErrorMetadata {
|
||||
exhausted_count: Some(exhausted.count),
|
||||
upstream_error: Some(exhausted.upstream_error.to_string()),
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +89,7 @@ impl DispatchErrorMetadata {
|
||||
CodexClientError::Upstream { status, .. } => Some(status.as_u16()),
|
||||
_ => None,
|
||||
},
|
||||
diagnostics: upstream_error_diagnostics(error).cloned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +99,7 @@ impl DispatchErrorMetadata {
|
||||
exhausted_count: None,
|
||||
upstream_error: None,
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -129,6 +136,45 @@ pub(crate) fn insert_dispatch_error_metadata(
|
||||
if let Some(status) = metadata.upstream_status {
|
||||
object.insert("upstreamStatus".to_string(), json!(status));
|
||||
}
|
||||
if let Some(diagnostics) = metadata.diagnostics {
|
||||
insert_upstream_diagnostics_metadata(object, &diagnostics);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn insert_upstream_diagnostics_metadata(
|
||||
object: &mut Map<String, Value>,
|
||||
diagnostics: &CodexUpstreamDiagnostics,
|
||||
) {
|
||||
if diagnostics.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(status_code) = diagnostics.status_code {
|
||||
object.insert("upstreamStatus".to_string(), json!(status_code));
|
||||
}
|
||||
if let Some(request_id) = &diagnostics.request_id {
|
||||
object.insert(
|
||||
"upstreamRequestId".to_string(),
|
||||
Value::String(request_id.clone()),
|
||||
);
|
||||
}
|
||||
if !diagnostics.trace_headers.is_empty() {
|
||||
object.insert(
|
||||
"upstreamTraceHeaders".to_string(),
|
||||
json!(diagnostics.trace_headers),
|
||||
);
|
||||
}
|
||||
if let Some(cf_ray) = diagnostics.cf_ray() {
|
||||
object.insert("cfRay".to_string(), Value::String(cf_ray.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn upstream_error_diagnostics(
|
||||
error: &CodexClientError,
|
||||
) -> Option<&CodexUpstreamDiagnostics> {
|
||||
match error {
|
||||
CodexClientError::Upstream { diagnostics, .. } => Some(diagnostics),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
|
||||
|
||||
@@ -199,6 +199,7 @@ impl ResponseDispatchError {
|
||||
exhausted_count: None,
|
||||
upstream_error: Some(sse_failure_error_body(failure)),
|
||||
upstream_status: None,
|
||||
diagnostics: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::{
|
||||
admin::monitoring::{
|
||||
@@ -9,7 +9,11 @@ use crate::{
|
||||
},
|
||||
infra::time::elapsed_millis_i64,
|
||||
proxy::dispatch::{
|
||||
errors::{backend_transport_name, upstream_error_http_status},
|
||||
errors::{
|
||||
backend_transport_name, insert_dispatch_error_metadata,
|
||||
insert_upstream_diagnostics_metadata, upstream_error_http_status,
|
||||
DispatchErrorMetadata,
|
||||
},
|
||||
usage_events::{
|
||||
enrich_event_route_metadata, enrich_usage_record_identity,
|
||||
event_kind as response_event_kind, reasoning_effort_from_request,
|
||||
@@ -18,7 +22,10 @@ use crate::{
|
||||
},
|
||||
upstream::{
|
||||
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
|
||||
transport::{CodexBackendTransport, CodexClientError, WebSocketPoolDecision},
|
||||
transport::{
|
||||
CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
|
||||
WebSocketPoolDecision,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -28,6 +35,7 @@ use super::{
|
||||
},
|
||||
sse_failure::{status_code_for_stream_failure, stream_failure_metadata},
|
||||
stream_lifecycle::{latest_response_id, LiveResponseStreamContext},
|
||||
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
|
||||
};
|
||||
|
||||
pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
|
||||
@@ -40,7 +48,10 @@ pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
|
||||
pub(super) started_at: Instant,
|
||||
pub(super) stream: bool,
|
||||
pub(super) transport: CodexBackendTransport,
|
||||
pub(super) request: &'a CodexResponsesRequest,
|
||||
pub(super) error: &'a CodexClientError,
|
||||
pub(super) trace: &'a ResponseDispatchTrace,
|
||||
pub(super) attempt: Option<&'a ResponseDispatchAttempt>,
|
||||
}
|
||||
|
||||
pub(super) struct ResponseStreamFailureEventRecord<'a> {
|
||||
@@ -54,8 +65,11 @@ pub(super) struct ResponseStreamFailureEventRecord<'a> {
|
||||
pub(super) transport: CodexBackendTransport,
|
||||
pub(super) request: &'a CodexResponsesRequest,
|
||||
pub(super) failure: &'a ResponsesSseFailure,
|
||||
pub(super) diagnostics: &'a CodexUpstreamDiagnostics,
|
||||
pub(super) rate_limit_headers: &'a [(String, String)],
|
||||
pub(super) prefetched: &'a [u8],
|
||||
pub(super) trace: &'a ResponseDispatchTrace,
|
||||
pub(super) attempt: &'a ResponseDispatchAttempt,
|
||||
}
|
||||
|
||||
pub(super) struct ResponseDispatchErrorEventRecord<'a> {
|
||||
@@ -88,6 +102,14 @@ pub(super) async fn record_response_dispatch_error_event(
|
||||
record.transport,
|
||||
);
|
||||
enrich_response_dispatch_error_metadata(&mut metadata, record.error);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
i64::from(record.error.http_status_code()),
|
||||
dispatch_error_client_status_code(record.error, record.stream, record.compact),
|
||||
dispatch_error_upstream_status_code(record.error),
|
||||
);
|
||||
}
|
||||
record_dispatch_error_event(DispatchErrorUsageRecord {
|
||||
usage_records: record.usage_records,
|
||||
request_id: record.request_id,
|
||||
@@ -105,6 +127,7 @@ pub(super) async fn record_response_dispatch_error_event(
|
||||
pub(super) async fn record_response_upstream_error_event(
|
||||
record: ResponseUpstreamErrorEventRecord<'_>,
|
||||
) {
|
||||
let event_status_code = i64::from(upstream_error_http_status(record.error));
|
||||
let mut metadata = dispatch_error_metadata(
|
||||
record.error,
|
||||
record.stream,
|
||||
@@ -121,9 +144,18 @@ pub(super) async fn record_response_upstream_error_event(
|
||||
event.account_id = Some(record.account_id.to_string());
|
||||
event.route = Some(record.route.to_string());
|
||||
event.model = Some(record.model.to_string());
|
||||
event.status_code = Some(i64::from(upstream_error_http_status(record.error)));
|
||||
event.status_code = Some(event_status_code);
|
||||
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
insert_dispatch_error_metadata(object, DispatchErrorMetadata::upstream(record.error));
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
upstream_failure_client_status_code(record.stream),
|
||||
Some(event_status_code),
|
||||
);
|
||||
insert_response_request_summary_object(object, record.request, record.transport);
|
||||
insert_response_trace_metadata_object(object, record.trace.attempts(), record.attempt);
|
||||
if let Some(account_email) = record
|
||||
.account_email
|
||||
.map(str::trim)
|
||||
@@ -144,6 +176,7 @@ pub(super) async fn record_response_upstream_error_event(
|
||||
pub(super) async fn record_prefetched_response_stream_failure_event(
|
||||
record: ResponseStreamFailureEventRecord<'_>,
|
||||
) {
|
||||
let event_status_code = status_code_for_stream_failure(record.failure);
|
||||
let mut metadata = stream_failure_metadata(record.failure, None);
|
||||
if let Some(object) = metadata.as_object_mut() {
|
||||
if record.transport == CodexBackendTransport::WebSocket {
|
||||
@@ -152,11 +185,24 @@ pub(super) async fn record_prefetched_response_stream_failure_event(
|
||||
Value::String("websocket".to_string()),
|
||||
);
|
||||
}
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
200,
|
||||
diagnostics_status_code(record.diagnostics),
|
||||
);
|
||||
insert_response_request_summary_object(object, record.request, record.transport);
|
||||
object.insert("requestBody".to_string(), json!(record.request));
|
||||
object.insert(
|
||||
"responseBody".to_string(),
|
||||
Value::String(String::from_utf8_lossy(record.prefetched).to_string()),
|
||||
);
|
||||
insert_upstream_diagnostics_metadata(object, record.diagnostics);
|
||||
insert_response_trace_metadata_object(
|
||||
object,
|
||||
record.trace.attempts(),
|
||||
Some(record.attempt),
|
||||
);
|
||||
}
|
||||
record_response_event(ResponseUsageRecord {
|
||||
usage_records: record.usage_records,
|
||||
@@ -170,7 +216,7 @@ pub(super) async fn record_prefetched_response_stream_failure_event(
|
||||
reasoning_effort: reasoning_effort_from_request(record.request),
|
||||
service_tier: record.request.service_tier(),
|
||||
started_at: record.started_at,
|
||||
status_code: status_code_for_stream_failure(record.failure),
|
||||
status_code: event_status_code,
|
||||
level: UsageRecordLevel::Error,
|
||||
message: "v1 responses stream failed",
|
||||
metadata,
|
||||
@@ -193,6 +239,7 @@ fn enrich_live_response_stream_metadata(
|
||||
context: &LiveResponseStreamContext,
|
||||
rate_limit_headers: &[(String, String)],
|
||||
metadata: &mut Value,
|
||||
status_code: i64,
|
||||
body: &str,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
@@ -211,6 +258,15 @@ fn enrich_live_response_stream_metadata(
|
||||
.entry("websocketPool".to_string())
|
||||
.or_insert_with(|| decision.metadata_value());
|
||||
}
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
status_code,
|
||||
200,
|
||||
diagnostics_status_code(&context.diagnostics),
|
||||
);
|
||||
insert_response_request_summary_object(object, &context.request, context.transport);
|
||||
insert_upstream_diagnostics_metadata(object, &context.diagnostics);
|
||||
insert_response_trace_metadata_object(object, &context.attempts, Some(&context.attempt));
|
||||
object
|
||||
.entry("requestBody".to_string())
|
||||
.or_insert_with(|| serde_json::json!(context.request));
|
||||
@@ -219,6 +275,181 @@ fn enrich_live_response_stream_metadata(
|
||||
.or_insert_with(|| Value::String(body.to_string()));
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_upstream_diagnostics(
|
||||
metadata: &mut Value,
|
||||
diagnostics: &CodexUpstreamDiagnostics,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_upstream_diagnostics_metadata(object, diagnostics);
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_status_metadata(
|
||||
metadata: &mut Value,
|
||||
event_status_code: i64,
|
||||
client_status_code: i64,
|
||||
upstream_status_code: Option<i64>,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_response_status_metadata_object(
|
||||
object,
|
||||
event_status_code,
|
||||
client_status_code,
|
||||
upstream_status_code,
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn insert_response_trace_metadata(
|
||||
metadata: &mut Value,
|
||||
trace: &ResponseDispatchTrace,
|
||||
current_attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
let Some(object) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
insert_response_trace_metadata_object(object, trace.attempts(), current_attempt);
|
||||
}
|
||||
|
||||
fn insert_response_attempt_metadata_object(
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
let Some(attempt) = attempt else {
|
||||
return;
|
||||
};
|
||||
object.insert("attemptIndex".to_string(), json!(attempt.index()));
|
||||
object.insert(
|
||||
"attemptAccountId".to_string(),
|
||||
Value::String(attempt.account_id().to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_response_trace_metadata_object(
|
||||
object: &mut Map<String, Value>,
|
||||
attempts: &[ResponseDispatchAttempt],
|
||||
current_attempt: Option<&ResponseDispatchAttempt>,
|
||||
) {
|
||||
insert_response_attempt_metadata_object(object, current_attempt);
|
||||
object.insert("attemptCount".to_string(), json!(attempts.len()));
|
||||
object.insert(
|
||||
"attempts".to_string(),
|
||||
Value::Array(
|
||||
attempts
|
||||
.iter()
|
||||
.map(|attempt| {
|
||||
json!({
|
||||
"index": attempt.index(),
|
||||
"accountId": attempt.account_id(),
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
fn insert_response_status_metadata_object(
|
||||
object: &mut Map<String, Value>,
|
||||
event_status_code: i64,
|
||||
client_status_code: i64,
|
||||
upstream_status_code: Option<i64>,
|
||||
) {
|
||||
object.insert("eventStatusCode".to_string(), json!(event_status_code));
|
||||
object.insert("clientStatusCode".to_string(), json!(client_status_code));
|
||||
if let Some(upstream_status_code) = upstream_status_code {
|
||||
object.insert(
|
||||
"upstreamStatusCode".to_string(),
|
||||
json!(upstream_status_code),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_response_request_summary_object(
|
||||
object: &mut Map<String, Value>,
|
||||
request: &CodexResponsesRequest,
|
||||
transport: CodexBackendTransport,
|
||||
) {
|
||||
object.insert(
|
||||
"requestSummary".to_string(),
|
||||
response_request_summary(request, transport),
|
||||
);
|
||||
}
|
||||
|
||||
fn response_request_summary(
|
||||
request: &CodexResponsesRequest,
|
||||
transport: CodexBackendTransport,
|
||||
) -> Value {
|
||||
let body = request.body();
|
||||
let input = body.get("input");
|
||||
let tools = body.get("tools");
|
||||
json!({
|
||||
"model": request.model(),
|
||||
"stream": request.stream(),
|
||||
"store": request.store(),
|
||||
"transport": backend_transport_name(transport),
|
||||
"inputType": json_value_kind(input),
|
||||
"inputItemsCount": input.and_then(Value::as_array).map(Vec::len),
|
||||
"toolsType": json_value_kind(tools),
|
||||
"toolsCount": tools.and_then(Value::as_array).map(Vec::len),
|
||||
"topLevelFields": body.keys().cloned().collect::<Vec<_>>(),
|
||||
"previousResponseIdPresent": request.previous_response_id().is_some(),
|
||||
"serviceTier": request.service_tier(),
|
||||
"localTransport": {
|
||||
"useWebsocket": request.use_websocket,
|
||||
"forceHttpSse": request.force_http_sse,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn json_value_kind(value: Option<&Value>) -> &'static str {
|
||||
match value {
|
||||
None => "missing",
|
||||
Some(Value::Null) => "null",
|
||||
Some(Value::Bool(_)) => "boolean",
|
||||
Some(Value::Number(_)) => "number",
|
||||
Some(Value::String(_)) => "string",
|
||||
Some(Value::Array(_)) => "array",
|
||||
Some(Value::Object(_)) => "object",
|
||||
}
|
||||
}
|
||||
|
||||
fn diagnostics_status_code(diagnostics: &CodexUpstreamDiagnostics) -> Option<i64> {
|
||||
diagnostics.status_code.map(i64::from)
|
||||
}
|
||||
|
||||
fn dispatch_error_upstream_status_code(error: &ResponseDispatchError) -> Option<i64> {
|
||||
match error {
|
||||
ResponseDispatchError::Upstream(error) => {
|
||||
Some(i64::from(upstream_error_http_status(error)))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_error_client_status_code(
|
||||
error: &ResponseDispatchError,
|
||||
stream: bool,
|
||||
compact: bool,
|
||||
) -> i64 {
|
||||
if stream {
|
||||
return 200;
|
||||
}
|
||||
if matches!(error, ResponseDispatchError::Upstream(_)) && !compact {
|
||||
return 502;
|
||||
}
|
||||
i64::from(error.http_status_code())
|
||||
}
|
||||
|
||||
fn upstream_failure_client_status_code(stream: bool) -> i64 {
|
||||
if stream {
|
||||
200
|
||||
} else {
|
||||
502
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_live_response_stream_event(
|
||||
context: &LiveResponseStreamContext,
|
||||
status_code: i64,
|
||||
@@ -230,7 +461,13 @@ pub(super) async fn record_live_response_stream_event(
|
||||
) {
|
||||
ensure_stream_metadata_flag(&mut metadata);
|
||||
enrich_event_route_metadata(&mut metadata, &context.route);
|
||||
enrich_live_response_stream_metadata(context, rate_limit_headers, &mut metadata, body);
|
||||
enrich_live_response_stream_metadata(
|
||||
context,
|
||||
rate_limit_headers,
|
||||
&mut metadata,
|
||||
status_code,
|
||||
body,
|
||||
);
|
||||
log_live_response_stream_finalized(context, status_code, level, message, &metadata, body);
|
||||
let mut event = UsageRecord::new(response_event_kind(&context.route), level, message);
|
||||
event.request_id = Some(context.request_id.clone());
|
||||
|
||||
@@ -8,3 +8,4 @@ mod prefetch;
|
||||
pub(crate) mod service;
|
||||
mod sse_failure;
|
||||
mod stream_lifecycle;
|
||||
mod trace;
|
||||
|
||||
@@ -82,10 +82,12 @@ use super::{
|
||||
affinity::{evict_reasoning_replay, record_response_affinity},
|
||||
errors::{ResponseDispatchError, ResponseDispatchStreamError},
|
||||
event_recording::{
|
||||
insert_websocket_pool_decision, record_prefetched_response_stream_failure_event,
|
||||
record_response_dispatch_error_event, record_response_upstream_error_event,
|
||||
ResponseDispatchErrorDetails, ResponseDispatchErrorEventRecord,
|
||||
ResponseStreamFailureEventRecord, ResponseUpstreamErrorEventRecord,
|
||||
insert_response_status_metadata, insert_response_trace_metadata,
|
||||
insert_response_upstream_diagnostics, insert_websocket_pool_decision,
|
||||
record_prefetched_response_stream_failure_event, record_response_dispatch_error_event,
|
||||
record_response_upstream_error_event, ResponseDispatchErrorDetails,
|
||||
ResponseDispatchErrorEventRecord, ResponseStreamFailureEventRecord,
|
||||
ResponseUpstreamErrorEventRecord,
|
||||
},
|
||||
live_stream::spawn_live_response_stream,
|
||||
prefetch::prefetch_first_sse_chunk,
|
||||
@@ -96,6 +98,7 @@ use super::{
|
||||
sse_failure_invalid_reasoning_replay, stream_failure_http_status,
|
||||
},
|
||||
stream_lifecycle::LiveResponseStreamContext,
|
||||
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
|
||||
};
|
||||
|
||||
/// OpenAI Responses 调度服务。
|
||||
@@ -387,11 +390,13 @@ impl ResponseDispatchService {
|
||||
let mut history_recovery_used = false;
|
||||
let mut empty_response_retries = 0u8;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
const MAX_EMPTY_RESPONSE_RETRIES: u8 = 2;
|
||||
let (account, response, collected_response): (
|
||||
let (account, response, collected_response, attempt): (
|
||||
Account,
|
||||
CodexBackendResponse,
|
||||
CollectedResponse,
|
||||
ResponseDispatchAttempt,
|
||||
) = loop {
|
||||
let mut attempt_acquire_request = acquire_request
|
||||
.clone()
|
||||
@@ -487,6 +492,7 @@ impl ResponseDispatchService {
|
||||
self.account_pool.wait_for_request_interval(&acquired).await;
|
||||
let account = acquired.account;
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_response_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
@@ -626,7 +632,7 @@ impl ResponseDispatchService {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break (account, response, collected_response);
|
||||
break (account, response, collected_response, attempt);
|
||||
}
|
||||
Err(error) if is_rate_limit_upstream_error(&error) => {
|
||||
exhausted_accounts.record_rate_limited(
|
||||
@@ -750,7 +756,10 @@ impl ResponseDispatchService {
|
||||
started_at,
|
||||
stream: false,
|
||||
transport: backend_transport_for_response_request(&request),
|
||||
request: &request,
|
||||
error: &error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
@@ -787,6 +796,14 @@ impl ResponseDispatchService {
|
||||
"firstTokenMs": response.first_token_ms,
|
||||
"usage": response.usage,
|
||||
});
|
||||
insert_response_status_metadata(
|
||||
&mut metadata,
|
||||
200,
|
||||
200,
|
||||
response.diagnostics.status_code.map(i64::from),
|
||||
);
|
||||
insert_response_upstream_diagnostics(&mut metadata, &response.diagnostics);
|
||||
insert_response_trace_metadata(&mut metadata, &trace, Some(&attempt));
|
||||
insert_websocket_pool_decision(&mut metadata, response.websocket_pool_decision);
|
||||
record_response_event(ResponseUsageRecord {
|
||||
usage_records: &self.usage_records,
|
||||
@@ -916,6 +933,7 @@ impl ResponseDispatchService {
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut history_recovery_used = false;
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
macro_rules! return_stream_dispatch_error {
|
||||
($error:expr) => {{
|
||||
let error = $error;
|
||||
@@ -1024,6 +1042,7 @@ impl ResponseDispatchService {
|
||||
self.account_pool.wait_for_request_interval(&acquired).await;
|
||||
let account = acquired.account;
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_response_stream_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
@@ -1051,6 +1070,7 @@ impl ResponseDispatchService {
|
||||
let turn_state_update = response.turn_state_update;
|
||||
let websocket_pool_decision = response.websocket_pool_decision;
|
||||
let turn_state = response.turn_state;
|
||||
let diagnostics = response.diagnostics;
|
||||
self.cloudflare
|
||||
.capture_set_cookie_headers(&release_account_id, &set_cookie_headers)
|
||||
.await;
|
||||
@@ -1204,7 +1224,10 @@ impl ResponseDispatchService {
|
||||
started_at,
|
||||
stream: true,
|
||||
transport,
|
||||
request: &request,
|
||||
error: upstream_error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -1306,8 +1329,11 @@ impl ResponseDispatchService {
|
||||
transport,
|
||||
request: &request,
|
||||
failure: &failure,
|
||||
diagnostics: &diagnostics,
|
||||
rate_limit_headers: &rate_limit_headers,
|
||||
prefetched: &prefetched,
|
||||
trace: &trace,
|
||||
attempt: &attempt,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -1336,6 +1362,9 @@ impl ResponseDispatchService {
|
||||
turn_state_update,
|
||||
websocket_pool_decision,
|
||||
turn_state,
|
||||
diagnostics,
|
||||
attempt: attempt.clone(),
|
||||
attempts: trace.attempts().to_vec(),
|
||||
started_at,
|
||||
};
|
||||
return Ok(spawn_live_response_stream(context, prefetched, body));
|
||||
@@ -1460,7 +1489,10 @@ impl ResponseDispatchService {
|
||||
started_at,
|
||||
stream: true,
|
||||
transport: backend_transport_for_response_request(&request),
|
||||
request: &request,
|
||||
error: &error,
|
||||
trace: &trace,
|
||||
attempt: Some(&attempt),
|
||||
})
|
||||
.await;
|
||||
return Err(ResponseDispatchError::Upstream(error));
|
||||
@@ -1483,6 +1515,7 @@ impl ResponseDispatchService {
|
||||
let mut excluded_account_ids = Vec::new();
|
||||
let mut exhausted_accounts = AccountExhaustionTracker::default();
|
||||
let mut quota_verify_attempts = 0usize;
|
||||
let mut trace = ResponseDispatchTrace::default();
|
||||
|
||||
loop {
|
||||
let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now())
|
||||
@@ -1548,6 +1581,7 @@ impl ResponseDispatchService {
|
||||
};
|
||||
let account = acquired.account;
|
||||
let release_account_id = account.id.clone();
|
||||
let attempt = trace.start_attempt(&release_account_id);
|
||||
let response_result = create_compact_response_with_account_retrying_5xx(
|
||||
&self.codex,
|
||||
self.installation_id.as_deref(),
|
||||
@@ -1585,6 +1619,19 @@ impl ResponseDispatchService {
|
||||
.record_token_usage(&account.id, request.model(), &usage)
|
||||
.await;
|
||||
}
|
||||
let mut metadata = json!({
|
||||
"stream": false,
|
||||
"compact": true,
|
||||
"usage": usage,
|
||||
});
|
||||
insert_response_status_metadata(
|
||||
&mut metadata,
|
||||
200,
|
||||
200,
|
||||
response.diagnostics.status_code.map(i64::from),
|
||||
);
|
||||
insert_response_upstream_diagnostics(&mut metadata, &response.diagnostics);
|
||||
insert_response_trace_metadata(&mut metadata, &trace, Some(&attempt));
|
||||
record_response_event(ResponseUsageRecord {
|
||||
usage_records: &self.usage_records,
|
||||
request_id,
|
||||
@@ -1600,11 +1647,7 @@ impl ResponseDispatchService {
|
||||
status_code: 200,
|
||||
level: UsageRecordLevel::Info,
|
||||
message: "v1 responses compact completed",
|
||||
metadata: json!({
|
||||
"stream": false,
|
||||
"compact": true,
|
||||
"usage": usage,
|
||||
}),
|
||||
metadata,
|
||||
rate_limit_headers: &response.rate_limit_headers,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::{
|
||||
},
|
||||
transport::{
|
||||
CodexBackendTransport, CodexRateLimitHeaderUpdates, CodexTurnStateUpdate,
|
||||
WebSocketPoolDecision,
|
||||
CodexUpstreamDiagnostics, WebSocketPoolDecision,
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -40,6 +40,7 @@ use super::{
|
||||
stream_failure_metadata, stream_failure_source, synthetic_stream_disconnected_detail,
|
||||
STREAM_DISCONNECTED_CODE, STREAM_DISCONNECTED_MESSAGE,
|
||||
},
|
||||
trace::ResponseDispatchAttempt,
|
||||
};
|
||||
use crate::proxy::dispatch::cloudflare::CloudflareRecovery;
|
||||
|
||||
@@ -65,6 +66,9 @@ pub(super) struct LiveResponseStreamContext {
|
||||
pub(super) turn_state_update: Option<CodexTurnStateUpdate>,
|
||||
pub(super) websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
pub(super) turn_state: Option<String>,
|
||||
pub(super) diagnostics: CodexUpstreamDiagnostics,
|
||||
pub(super) attempt: ResponseDispatchAttempt,
|
||||
pub(super) attempts: Vec<ResponseDispatchAttempt>,
|
||||
pub(super) started_at: Instant,
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
//! Request-level trace state for Responses dispatch.
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct ResponseDispatchTrace {
|
||||
next_attempt_index: i64,
|
||||
attempts: Vec<ResponseDispatchAttempt>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(super) struct ResponseDispatchAttempt {
|
||||
index: i64,
|
||||
account_id: String,
|
||||
}
|
||||
|
||||
impl ResponseDispatchTrace {
|
||||
pub(super) fn start_attempt(&mut self, account_id: &str) -> ResponseDispatchAttempt {
|
||||
let attempt = ResponseDispatchAttempt {
|
||||
index: self.next_attempt_index,
|
||||
account_id: account_id.to_string(),
|
||||
};
|
||||
self.next_attempt_index += 1;
|
||||
self.attempts.push(attempt.clone());
|
||||
attempt
|
||||
}
|
||||
|
||||
pub(super) fn attempts(&self) -> &[ResponseDispatchAttempt] {
|
||||
&self.attempts
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseDispatchAttempt {
|
||||
pub(super) fn index(&self) -> i64 {
|
||||
self.index
|
||||
}
|
||||
|
||||
pub(super) fn account_id(&self) -> &str {
|
||||
&self.account_id
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,7 @@ use crate::{
|
||||
account_usage_service::AdminUsageService,
|
||||
account_usage_store::SqliteUsageStore,
|
||||
usage_record_service::AdminUsageRecordService,
|
||||
usage_record_store::{
|
||||
SqliteUsageRecordStore, DEFAULT_USAGE_RECORD_CAPACITY,
|
||||
DEFAULT_USAGE_RECORD_CAPTURE_BODY,
|
||||
},
|
||||
usage_record_store::{SqliteUsageRecordStore, DEFAULT_USAGE_RECORD_CAPTURE_BODY},
|
||||
},
|
||||
admin::{
|
||||
accounts::service::{AdminAccountService, AdminAccountServiceParts},
|
||||
@@ -110,7 +107,6 @@ pub struct Services {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct UsageRecordOptions {
|
||||
pub enabled: bool,
|
||||
pub capacity: u32,
|
||||
pub capture_body: bool,
|
||||
}
|
||||
|
||||
@@ -118,7 +114,6 @@ impl UsageRecordOptions {
|
||||
pub fn from_config(config: &AppConfig) -> Self {
|
||||
Self {
|
||||
enabled: config.logging.enabled,
|
||||
capacity: DEFAULT_USAGE_RECORD_CAPACITY,
|
||||
capture_body: DEFAULT_USAGE_RECORD_CAPTURE_BODY,
|
||||
}
|
||||
}
|
||||
@@ -189,6 +184,11 @@ impl Services {
|
||||
StdArc::new(client)
|
||||
}
|
||||
};
|
||||
let usage_records = StdArc::new(AdminUsageRecordService::new(
|
||||
stores.usage_records.clone(),
|
||||
usage_record_options.enabled,
|
||||
usage_record_options.capture_body,
|
||||
));
|
||||
let account_pool = StdArc::new(RuntimeAccountPoolService::new(
|
||||
account_store_trait.clone(),
|
||||
account_pool_options_from_config(config),
|
||||
@@ -251,12 +251,6 @@ impl Services {
|
||||
refresh_policy: refresh_policy.clone(),
|
||||
installation_id: installation_id.clone(),
|
||||
}));
|
||||
let usage_records = StdArc::new(AdminUsageRecordService::new(
|
||||
stores.usage_records.clone(),
|
||||
usage_record_options.enabled,
|
||||
usage_record_options.capacity,
|
||||
usage_record_options.capture_body,
|
||||
));
|
||||
let usage_store = SqliteUsageStore::new(stores.accounts.pool().clone());
|
||||
let usage = StdArc::new(AdminUsageService::new(usage_store));
|
||||
let session_affinity = StdArc::new(RuntimeSessionAffinityService::new(
|
||||
|
||||
@@ -562,9 +562,10 @@ impl AccountPool {
|
||||
}
|
||||
let after = RuntimeAccountStateSnapshot::from(&*account);
|
||||
if before != after {
|
||||
let sync_usage_window = before.usage_window_changed(&after);
|
||||
refreshed_accounts.push(RefreshedAccountState {
|
||||
account: account.clone(),
|
||||
sync_usage_window: before.usage_window_changed(&after),
|
||||
sync_usage_window,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1310,11 +1311,14 @@ impl RuntimeAccountPoolService {
|
||||
}
|
||||
}
|
||||
|
||||
let mut pool = self.pool.lock().await;
|
||||
let in_memory = pool.apply_quota_state(account_id, limit_reached, cooldown_until);
|
||||
if let Some(reset_at) = reset_at {
|
||||
pool.sync_rate_limit_window(account_id, reset_at, limit_window_seconds);
|
||||
}
|
||||
let in_memory = {
|
||||
let mut pool = self.pool.lock().await;
|
||||
let in_memory = pool.apply_quota_state(account_id, limit_reached, cooldown_until);
|
||||
if let Some(reset_at) = reset_at {
|
||||
pool.sync_rate_limit_window(account_id, reset_at, limit_window_seconds);
|
||||
}
|
||||
in_memory
|
||||
};
|
||||
|
||||
persisted || in_memory
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ use crate::upstream::protocol::websocket::{
|
||||
websocket_audit_artifact_from_attempt, websocket_payload_audit_snapshot,
|
||||
};
|
||||
|
||||
use super::diagnostics::CodexUpstreamDiagnostics;
|
||||
use super::endpoints::{endpoint_url, CODEX_RESPONSES_COMPACT_PATH, CODEX_RESPONSES_PATH};
|
||||
use super::headers::{
|
||||
build_ordered_codex_base_headers, insert_optional_header, insert_ordered_headers,
|
||||
@@ -129,6 +130,8 @@ pub enum CodexClientError {
|
||||
body: String,
|
||||
/// 推导出的重试秒数。
|
||||
retry_after_seconds: Option<u64>,
|
||||
/// 上游诊断元数据。
|
||||
diagnostics: CodexUpstreamDiagnostics,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
set_cookie_headers: Vec<String>,
|
||||
},
|
||||
@@ -255,6 +258,8 @@ pub struct CodexBackendResponse {
|
||||
pub first_token_ms: Option<i64>,
|
||||
/// WebSocket 连接池决策。
|
||||
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
/// Codex Responses 实际使用的上游传输。
|
||||
@@ -302,6 +307,8 @@ pub struct CodexBackendStreamingResponse {
|
||||
pub turn_state_update: Option<CodexTurnStateUpdate>,
|
||||
/// WebSocket 连接池决策。
|
||||
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
/// Codex compact 端点响应。
|
||||
@@ -313,6 +320,8 @@ pub struct CodexCompactResponse {
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
/// 上游透传的限流头。
|
||||
pub rate_limit_headers: Vec<(String, String)>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -376,6 +385,7 @@ impl CodexBackendClient {
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let turn_state = response_meta::turn_state(response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
@@ -388,6 +398,7 @@ impl CodexBackendClient {
|
||||
retry_after_seconds: retry_after_seconds
|
||||
.or_else(|| retry_after_seconds_from_body(&body)),
|
||||
body,
|
||||
diagnostics,
|
||||
set_cookie_headers,
|
||||
});
|
||||
}
|
||||
@@ -410,6 +421,7 @@ impl CodexBackendClient {
|
||||
rate_limit_headers,
|
||||
first_token_ms,
|
||||
websocket_pool_decision: None,
|
||||
diagnostics,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -428,6 +440,7 @@ impl CodexBackendClient {
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let turn_state = response_meta::turn_state(response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
@@ -440,6 +453,7 @@ impl CodexBackendClient {
|
||||
retry_after_seconds: retry_after_seconds
|
||||
.or_else(|| retry_after_seconds_from_body(&body)),
|
||||
body,
|
||||
diagnostics,
|
||||
set_cookie_headers,
|
||||
});
|
||||
}
|
||||
@@ -453,6 +467,7 @@ impl CodexBackendClient {
|
||||
rate_limit_header_updates: None,
|
||||
turn_state_update: None,
|
||||
websocket_pool_decision: None,
|
||||
diagnostics,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -634,6 +649,7 @@ impl CodexBackendClient {
|
||||
rate_limit_headers: exchange.rate_limit_headers,
|
||||
first_token_ms: exchange.first_token_ms,
|
||||
websocket_pool_decision: exchange.pool_decision,
|
||||
diagnostics: exchange.diagnostics,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -701,6 +717,7 @@ impl CodexBackendClient {
|
||||
rate_limit_header_updates: Some(exchange.rate_limit_header_updates),
|
||||
turn_state_update: Some(exchange.turn_state_update),
|
||||
websocket_pool_decision: exchange.pool_decision,
|
||||
diagnostics: exchange.diagnostics,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -738,6 +755,7 @@ impl CodexBackendClient {
|
||||
.await?;
|
||||
|
||||
let status = response.status();
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
|
||||
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
|
||||
let retry_after_seconds = retry_after_seconds(response.headers(), None);
|
||||
@@ -749,6 +767,7 @@ impl CodexBackendClient {
|
||||
retry_after_seconds: retry_after_seconds
|
||||
.or_else(|| retry_after_seconds_from_body(&body)),
|
||||
body,
|
||||
diagnostics,
|
||||
set_cookie_headers,
|
||||
});
|
||||
}
|
||||
@@ -761,12 +780,16 @@ impl CodexBackendClient {
|
||||
"Compact response is not valid JSON: {}",
|
||||
truncate_for_error(&body)
|
||||
),
|
||||
diagnostics: CodexUpstreamDiagnostics::with_status(
|
||||
StatusCode::BAD_GATEWAY.as_u16(),
|
||||
),
|
||||
set_cookie_headers: set_cookie_headers.clone(),
|
||||
})?;
|
||||
Ok(CodexCompactResponse {
|
||||
body: parsed,
|
||||
set_cookie_headers,
|
||||
rate_limit_headers,
|
||||
diagnostics,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -801,6 +824,7 @@ impl CodexBackendClient {
|
||||
status: StatusCode::BAD_GATEWAY,
|
||||
retry_after_seconds: None,
|
||||
body: "backend model catalog is unavailable".to_string(),
|
||||
diagnostics: CodexUpstreamDiagnostics::with_status(StatusCode::BAD_GATEWAY.as_u16()),
|
||||
set_cookie_headers: Vec::new(),
|
||||
})
|
||||
}
|
||||
@@ -1201,17 +1225,17 @@ fn websocket_exchange_error_to_client_error(
|
||||
error: CodexWebSocketExchangeError,
|
||||
) -> CodexClientError {
|
||||
match error {
|
||||
CodexWebSocketExchangeError::Upstream {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
set_cookie_headers,
|
||||
} => CodexClientError::Upstream {
|
||||
status: StatusCode::from_u16(status_code).unwrap_or(StatusCode::BAD_GATEWAY),
|
||||
body,
|
||||
retry_after_seconds,
|
||||
set_cookie_headers,
|
||||
},
|
||||
CodexWebSocketExchangeError::Upstream(upstream) => {
|
||||
let upstream = *upstream;
|
||||
CodexClientError::Upstream {
|
||||
status: StatusCode::from_u16(upstream.status_code)
|
||||
.unwrap_or(StatusCode::BAD_GATEWAY),
|
||||
body: upstream.body,
|
||||
retry_after_seconds: upstream.retry_after_seconds,
|
||||
diagnostics: upstream.diagnostics,
|
||||
set_cookie_headers: upstream.set_cookie_headers,
|
||||
}
|
||||
}
|
||||
error => CodexClientError::WebSocket(error),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
//! Upstream response diagnostics captured at the transport boundary.
|
||||
|
||||
use reqwest::header::HeaderMap;
|
||||
|
||||
const UPSTREAM_REQUEST_ID_HEADERS: &[&str] = &[
|
||||
"x-request-id",
|
||||
"x-openai-request-id",
|
||||
"openai-request-id",
|
||||
"request-id",
|
||||
];
|
||||
|
||||
const UPSTREAM_TRACE_HEADERS: &[&str] = &[
|
||||
"x-request-id",
|
||||
"x-openai-request-id",
|
||||
"openai-request-id",
|
||||
"request-id",
|
||||
"cf-ray",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CodexUpstreamDiagnostics {
|
||||
pub status_code: Option<u16>,
|
||||
pub request_id: Option<String>,
|
||||
pub trace_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl CodexUpstreamDiagnostics {
|
||||
pub fn from_headers(status_code: Option<u16>, headers: &HeaderMap) -> Self {
|
||||
Self {
|
||||
status_code,
|
||||
request_id: first_header(headers, UPSTREAM_REQUEST_ID_HEADERS),
|
||||
trace_headers: trace_headers(headers),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_status(status_code: u16) -> Self {
|
||||
Self {
|
||||
status_code: Some(status_code),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.status_code.is_none() && self.request_id.is_none() && self.trace_headers.is_empty()
|
||||
}
|
||||
|
||||
pub fn cf_ray(&self) -> Option<&str> {
|
||||
self.trace_headers
|
||||
.iter()
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("cf-ray"))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
fn first_header(headers: &HeaderMap, names: &[&str]) -> Option<String> {
|
||||
names.iter().find_map(|name| header_value(headers, name))
|
||||
}
|
||||
|
||||
fn trace_headers(headers: &HeaderMap) -> Vec<(String, String)> {
|
||||
UPSTREAM_TRACE_HEADERS
|
||||
.iter()
|
||||
.filter_map(|name| header_value(headers, name).map(|value| ((*name).to_string(), value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn header_value(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Codex HTTP/SSE/WebSocket 上游 transport。
|
||||
|
||||
pub mod client;
|
||||
pub mod diagnostics;
|
||||
pub mod endpoints;
|
||||
pub mod headers;
|
||||
mod response_meta;
|
||||
@@ -19,6 +20,7 @@ pub use self::{
|
||||
CodexModelCatalogRequest, CodexRateLimitHeaderUpdates, CodexRequestContext,
|
||||
CodexTurnStateUpdate,
|
||||
},
|
||||
diagnostics::CodexUpstreamDiagnostics,
|
||||
endpoints::{
|
||||
endpoint_request_path, endpoint_url, usage_endpoint_urls, CODEX_RESPONSES_COMPACT_PATH,
|
||||
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, CODEX_USAGE_PATH, WHAM_USAGE_PATH,
|
||||
|
||||
@@ -6,6 +6,15 @@ use reqwest::header::{HeaderMap, SET_COOKIE};
|
||||
|
||||
use crate::upstream::protocol::responses::update_first_response_event_ms;
|
||||
|
||||
use super::diagnostics::CodexUpstreamDiagnostics;
|
||||
|
||||
pub(super) fn diagnostics(
|
||||
status_code: Option<u16>,
|
||||
headers: &HeaderMap,
|
||||
) -> CodexUpstreamDiagnostics {
|
||||
CodexUpstreamDiagnostics::from_headers(status_code, headers)
|
||||
}
|
||||
|
||||
pub(super) fn turn_state(headers: &HeaderMap) -> Option<String> {
|
||||
headers
|
||||
.get("x-codex-turn-state")
|
||||
|
||||
@@ -6,7 +6,8 @@ use crate::upstream::protocol::events::retry_after_seconds_from_body;
|
||||
use super::{
|
||||
client::{read_capped_error_body, retry_after_seconds, truncate_for_error},
|
||||
endpoints::usage_endpoint_urls,
|
||||
CodexBackendClient, CodexClientError, CodexClientResult, CodexRequestContext,
|
||||
response_meta, CodexBackendClient, CodexClientError, CodexClientResult, CodexRequestContext,
|
||||
CodexUpstreamDiagnostics,
|
||||
};
|
||||
|
||||
impl CodexBackendClient {
|
||||
@@ -18,6 +19,7 @@ impl CodexBackendClient {
|
||||
let headers = self.usage_request_headers(context)?;
|
||||
let response = self.client.get(endpoint).headers(headers).send().await?;
|
||||
let status = response.status();
|
||||
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
|
||||
let retry_after_seconds = retry_after_seconds(response.headers(), None);
|
||||
|
||||
if status == StatusCode::NOT_FOUND {
|
||||
@@ -31,6 +33,7 @@ impl CodexBackendClient {
|
||||
retry_after_seconds: retry_after_seconds
|
||||
.or_else(|| retry_after_seconds_from_body(&body)),
|
||||
body,
|
||||
diagnostics,
|
||||
set_cookie_headers: Vec::new(),
|
||||
});
|
||||
}
|
||||
@@ -49,6 +52,7 @@ impl CodexBackendClient {
|
||||
|| "usage endpoint is unavailable".to_string(),
|
||||
|body| format!("invalid usage response: {}", truncate_for_error(&body)),
|
||||
),
|
||||
diagnostics: CodexUpstreamDiagnostics::with_status(StatusCode::BAD_GATEWAY.as_u16()),
|
||||
set_cookie_headers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
io,
|
||||
fmt, io,
|
||||
path::{Path, PathBuf},
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
@@ -37,12 +37,12 @@ use crate::upstream::protocol::websocket::{
|
||||
OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact,
|
||||
};
|
||||
|
||||
use super::response_meta;
|
||||
use super::websocket_pool::{
|
||||
CodexWebSocketConnectionMetadata, CodexWebSocketPool, CodexWebSocketPoolKey,
|
||||
PooledWebSocketConnection, WebSocketPoolAcquire, WebSocketPoolDecision,
|
||||
};
|
||||
use super::websocket_pump::{PumpKeepalive, PumpedWebSocket, RawWsStream};
|
||||
use super::{diagnostics::CodexUpstreamDiagnostics, response_meta};
|
||||
use uuid::Uuid;
|
||||
|
||||
const REDACTED_HEADER_VALUE: &str = "<redacted>";
|
||||
@@ -120,6 +120,8 @@ pub struct CodexWebSocketExchange {
|
||||
pub first_token_ms: Option<i64>,
|
||||
/// WebSocket 连接池决策。
|
||||
pub pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
/// Responses WebSocket live SSE exchange result.
|
||||
@@ -138,6 +140,8 @@ pub struct CodexWebSocketStreamingExchange {
|
||||
pub turn_state_update: CodexWebSocketTurnStateUpdate,
|
||||
/// WebSocket 连接池决策。
|
||||
pub pool_decision: Option<WebSocketPoolDecision>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
/// Responses WebSocket live SSE byte stream.
|
||||
@@ -161,17 +165,8 @@ pub enum CodexWebSocketExchangeError {
|
||||
#[error("invalid websocket SSE response: {0}")]
|
||||
InvalidSse(#[from] SseError),
|
||||
/// 上游 WebSocket 错误帧。
|
||||
#[error("websocket upstream error {status_code}: {body}")]
|
||||
Upstream {
|
||||
/// HTTP-style upstream status code.
|
||||
status_code: u16,
|
||||
/// 推导出的重试秒数。
|
||||
retry_after_seconds: Option<u64>,
|
||||
/// 原始错误帧。
|
||||
body: String,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
set_cookie_headers: Vec<String>,
|
||||
},
|
||||
#[error("{0}")]
|
||||
Upstream(Box<CodexWebSocketUpstreamError>),
|
||||
/// 上游返回 `response.incomplete`。
|
||||
#[error("Incomplete response returned, reason: {reason}")]
|
||||
IncompleteResponse {
|
||||
@@ -210,6 +205,49 @@ pub enum CodexWebSocketExchangeError {
|
||||
},
|
||||
}
|
||||
|
||||
/// WebSocket 上游错误帧载荷。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CodexWebSocketUpstreamError {
|
||||
/// HTTP-style upstream status code.
|
||||
pub status_code: u16,
|
||||
/// 推导出的重试秒数。
|
||||
pub retry_after_seconds: Option<u64>,
|
||||
/// 原始错误帧。
|
||||
pub body: String,
|
||||
/// 上游透传的 `set-cookie` 列表。
|
||||
pub set_cookie_headers: Vec<String>,
|
||||
/// 上游诊断元数据。
|
||||
pub diagnostics: CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
impl fmt::Display for CodexWebSocketUpstreamError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"websocket upstream error {}: {}",
|
||||
self.status_code, self.body
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexWebSocketExchangeError {
|
||||
fn upstream(
|
||||
status_code: u16,
|
||||
retry_after_seconds: Option<u64>,
|
||||
body: String,
|
||||
set_cookie_headers: Vec<String>,
|
||||
diagnostics: CodexUpstreamDiagnostics,
|
||||
) -> Self {
|
||||
Self::Upstream(Box::new(CodexWebSocketUpstreamError {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
set_cookie_headers,
|
||||
diagnostics,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexWebSocketRequest {
|
||||
/// 返回连接描述。
|
||||
pub fn connection(&self) -> &CodexWebSocketConnection {
|
||||
@@ -389,12 +427,13 @@ fn websocket_opening_error(response: &WsResponse<Option<Vec<u8>>>) -> CodexWebSo
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.filter(|seconds| *seconds > 0)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&body));
|
||||
CodexWebSocketExchangeError::Upstream {
|
||||
CodexWebSocketExchangeError::upstream(
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
|
||||
}
|
||||
response_meta::set_cookie_headers(response.headers()),
|
||||
response_meta::diagnostics(Some(status_code), response.headers()),
|
||||
)
|
||||
}
|
||||
|
||||
fn request_line_for_endpoint(endpoint: &str) -> String {
|
||||
@@ -940,12 +979,13 @@ async fn collect_websocket_response(
|
||||
if let Some(classified) = classify_websocket_error_frame(&raw) {
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
return Err(CodexWebSocketExchangeError::Upstream {
|
||||
status_code: classified.status_code,
|
||||
return Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
retry_after_seconds,
|
||||
body: raw,
|
||||
set_cookie_headers: Vec::new(),
|
||||
});
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
|
||||
@@ -986,6 +1026,7 @@ async fn collect_websocket_response(
|
||||
rate_limit_headers: metadata.rate_limit_headers.clone(),
|
||||
first_token_ms,
|
||||
pool_decision: None,
|
||||
diagnostics: metadata.diagnostics.clone(),
|
||||
};
|
||||
return Ok((exchange, websocket, metadata));
|
||||
}
|
||||
@@ -1035,12 +1076,13 @@ async fn prefetch_first_content_stream_frames(
|
||||
if let Some(classified) = classify_websocket_error_frame(&raw) {
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
return Err(CodexWebSocketExchangeError::Upstream {
|
||||
status_code: classified.status_code,
|
||||
return Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
retry_after_seconds,
|
||||
body: raw,
|
||||
set_cookie_headers: Vec::new(),
|
||||
});
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
));
|
||||
}
|
||||
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
|
||||
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
|
||||
@@ -1095,6 +1137,10 @@ fn websocket_connection_metadata(
|
||||
turn_state: response_meta::turn_state(response.headers()),
|
||||
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
|
||||
rate_limit_headers: response_meta::rate_limit_headers(response.headers()),
|
||||
diagnostics: response_meta::diagnostics(
|
||||
Some(response.status().as_u16()),
|
||||
response.headers(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1145,6 +1191,7 @@ fn stream_websocket_response(
|
||||
rate_limit_header_updates,
|
||||
turn_state_update,
|
||||
pool_decision: None,
|
||||
diagnostics: response_metadata.diagnostics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1212,12 +1259,13 @@ async fn forward_websocket_response_stream(
|
||||
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
|
||||
.or_else(|| events::retry_after_seconds_from_body(&raw));
|
||||
let _ = tx
|
||||
.send(Err(CodexWebSocketExchangeError::Upstream {
|
||||
status_code: classified.status_code,
|
||||
.send(Err(CodexWebSocketExchangeError::upstream(
|
||||
classified.status_code,
|
||||
retry_after_seconds,
|
||||
body: raw,
|
||||
set_cookie_headers: Vec::new(),
|
||||
}))
|
||||
raw,
|
||||
Vec::new(),
|
||||
CodexUpstreamDiagnostics::with_status(classified.status_code),
|
||||
)))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -352,6 +352,7 @@ pub(crate) struct CodexWebSocketConnectionMetadata {
|
||||
pub(crate) turn_state: Option<String>,
|
||||
pub(crate) set_cookie_headers: Vec<String>,
|
||||
pub(crate) rate_limit_headers: Vec<(String, String)>,
|
||||
pub(crate) diagnostics: super::diagnostics::CodexUpstreamDiagnostics,
|
||||
}
|
||||
|
||||
pub(crate) struct PooledWebSocketConnection {
|
||||
|
||||
@@ -46,11 +46,38 @@ async fn admin_account_quota_should_send_usage_cookie() {
|
||||
access_token: SecretString::new("access-quota-cookie".to_string().into()),
|
||||
refresh_token: None,
|
||||
access_token_expires_at: None,
|
||||
status: AccountStatus::Active,
|
||||
status: AccountStatus::QuotaExhausted,
|
||||
added_at: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
sqlx::query(
|
||||
"insert into account_usage (
|
||||
account_id,
|
||||
request_count,
|
||||
window_request_count,
|
||||
window_input_tokens,
|
||||
window_output_tokens,
|
||||
window_cached_tokens,
|
||||
window_started_at,
|
||||
window_reset_at,
|
||||
limit_window_seconds,
|
||||
last_used_at
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind("acct_quota_cookie")
|
||||
.bind(9)
|
||||
.bind(9)
|
||||
.bind(9_000)
|
||||
.bind(900)
|
||||
.bind(90)
|
||||
.bind("2027-01-01T00:00:00Z")
|
||||
.bind("2027-01-01T05:00:00Z")
|
||||
.bind(18_000)
|
||||
.bind("2027-01-15T04:00:00Z")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"insert into usage_time_buckets (
|
||||
bucket_start,
|
||||
@@ -74,6 +101,29 @@ async fn admin_account_quota_should_send_usage_cookie() {
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"insert into usage_time_buckets (
|
||||
bucket_start,
|
||||
account_id,
|
||||
model,
|
||||
request_count,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cached_tokens,
|
||||
updated_at
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
)
|
||||
.bind("2027-01-15T09:00:00+00:00")
|
||||
.bind("acct_quota_cookie")
|
||||
.bind("gpt-5")
|
||||
.bind(1)
|
||||
.bind(1_500)
|
||||
.bind(500)
|
||||
.bind(300)
|
||||
.bind("2027-01-15T09:00:00Z")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
SqliteCookieStore::new(pool.clone())
|
||||
.capture_set_cookie(
|
||||
"acct_quota_cookie",
|
||||
@@ -107,6 +157,15 @@ async fn admin_account_quota_should_send_usage_cookie() {
|
||||
);
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["data"]["planType"], "plus");
|
||||
assert_eq!(body["data"]["account"]["status"], "active");
|
||||
assert_eq!(body["data"]["account"]["usage"]["requestCount"], 1);
|
||||
assert_eq!(body["data"]["account"]["usage"]["inputTokens"], 1_500);
|
||||
assert_eq!(body["data"]["account"]["usage"]["outputTokens"], 500);
|
||||
assert_eq!(body["data"]["account"]["usage"]["cachedTokens"], 300);
|
||||
assert_eq!(
|
||||
body["data"]["account"]["usage"]["models"][0]["model"],
|
||||
"gpt-5"
|
||||
);
|
||||
assert_eq!(
|
||||
body["data"]["quotaData"]["windows"][0]["labelDisplay"],
|
||||
"5小时限额"
|
||||
@@ -125,6 +184,6 @@ async fn admin_account_quota_should_send_usage_cookie() {
|
||||
);
|
||||
assert_eq!(
|
||||
body["data"]["quotaData"]["windows"][1]["tokenUsageDisplay"],
|
||||
"0"
|
||||
"2K"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use codex_proxy_rs::{
|
||||
},
|
||||
infra::{
|
||||
database::connect_sqlite,
|
||||
time::{china_datetime, china_day_start, china_hour, china_hour_start},
|
||||
time::{china_datetime, china_day_start, china_hour},
|
||||
},
|
||||
proxy::dispatch::session_affinity::SqliteSessionAffinityStore,
|
||||
runtime::{
|
||||
@@ -222,20 +222,14 @@ async fn dashboard_latency_trend_should_use_first_token_latency() {
|
||||
)
|
||||
.await;
|
||||
let now = Utc::now();
|
||||
let current_hour_start = china_hour_start(now);
|
||||
let sample_hour_start = if now - current_hour_start > Duration::minutes(31) {
|
||||
current_hour_start
|
||||
} else {
|
||||
current_hour_start - Duration::hours(1)
|
||||
};
|
||||
let mut slow_completion =
|
||||
usage_record_with_tokens(sample_hour_start + Duration::minutes(1), 10);
|
||||
let mut slow_completion = usage_record_with_tokens(now, 10);
|
||||
slow_completion.latency_ms = Some(10_000);
|
||||
slow_completion.metadata["firstTokenMs"] = json!(100_i64);
|
||||
let mut fast_completion =
|
||||
usage_record_with_tokens(sample_hour_start + Duration::minutes(16), 10);
|
||||
slow_completion.metadata["serviceTier"] = json!("default");
|
||||
let mut fast_completion = usage_record_with_tokens(now, 10);
|
||||
fast_completion.latency_ms = Some(1_000);
|
||||
fast_completion.metadata["firstTokenMs"] = json!(500_i64);
|
||||
fast_completion.metadata["serviceTier"] = json!("priority");
|
||||
store.append(&slow_completion).await.unwrap();
|
||||
store.append(&fast_completion).await.unwrap();
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use chrono::{Duration, Utc};
|
||||
use codex_proxy_rs::{
|
||||
admin::monitoring::{
|
||||
usage_record_model::{UsageRecord, UsageRecordLevel},
|
||||
@@ -91,6 +92,22 @@ async fn usage_record_store_should_promote_diagnostic_metadata_fields() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_record_store_should_not_promote_cf_ray_as_upstream_request_id() {
|
||||
let (store, _dir) = usage_record_store("usage-records-cf-ray.sqlite").await;
|
||||
let mut event = UsageRecord::new("request", UsageRecordLevel::Error, "upstream failed");
|
||||
event.id = "usage_cf_ray".to_string();
|
||||
event.metadata = json!({
|
||||
"upstreamStatus": 403,
|
||||
"cfRay": "ray-1"
|
||||
});
|
||||
|
||||
store.append(&event).await.unwrap();
|
||||
let loaded = store.get("usage_cf_ray").await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(loaded.upstream_request_id, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_record_store_should_get_and_clear_events() {
|
||||
let (store, _dir) = usage_record_store("usage-records-clear.sqlite").await;
|
||||
@@ -108,6 +125,28 @@ async fn usage_record_store_should_get_and_clear_events() {
|
||||
assert!(page.items.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_record_store_should_trim_records_older_than_thirty_days() {
|
||||
let (store, _dir) = usage_record_store("usage-records-retention.sqlite").await;
|
||||
let now = Utc::now();
|
||||
let mut expired = UsageRecord::new("v1.response", UsageRecordLevel::Info, "expired");
|
||||
expired.id = "usage_expired".to_string();
|
||||
expired.route = Some("/v1/responses".to_string());
|
||||
expired.created_at = now - Duration::days(31);
|
||||
store.append(&expired).await.unwrap();
|
||||
|
||||
let mut retained = UsageRecord::new("v1.response", UsageRecordLevel::Info, "retained");
|
||||
retained.id = "usage_retained".to_string();
|
||||
retained.route = Some("/v1/responses".to_string());
|
||||
retained.created_at = now - Duration::days(29);
|
||||
store.append(&retained).await.unwrap();
|
||||
|
||||
assert_eq!(store.trim_to_retention(now).await.unwrap(), 1);
|
||||
|
||||
assert!(store.get("usage_expired").await.unwrap().is_none());
|
||||
assert!(store.get("usage_retained").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
async fn usage_record_store(db_name: &str) -> (SqliteUsageRecordStore, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = dir.path().join(db_name);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pub(crate) mod format;
|
||||
pub(crate) mod log_rotation;
|
||||
pub(crate) mod paths;
|
||||
pub(crate) mod storage_schema;
|
||||
pub(crate) mod time;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
use std::{ffi::OsString, fs, sync::Mutex};
|
||||
|
||||
use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_installation_id};
|
||||
|
||||
const INSTALLATION_ID_FILE_NAME: &str = "installation_id";
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn data_dir_should_use_xdg_data_home_directly() {
|
||||
let _guard = ENV_LOCK.lock().unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let _xdg_data_home = EnvVarGuard::set("XDG_DATA_HOME", dir.path().as_os_str().to_owned());
|
||||
|
||||
let resolved = ensure_data_dir().unwrap();
|
||||
|
||||
assert_eq!(resolved, dir.path());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_persist_under_data_dir_root() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let installation_id =
|
||||
load_or_create_installation_id(Some(dir.path())).expect("installation id should generate");
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(dir.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
|
||||
installation_id
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() {
|
||||
let _guard = ENV_LOCK.lock().unwrap();
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
let data = tempfile::tempdir().unwrap();
|
||||
let codex_path = home.path().join(".codex").join(INSTALLATION_ID_FILE_NAME);
|
||||
let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a";
|
||||
fs::create_dir_all(codex_path.parent().unwrap()).unwrap();
|
||||
fs::write(&codex_path, desktop_id).unwrap();
|
||||
let _home = EnvVarGuard::set("HOME", home.path().as_os_str().to_owned());
|
||||
|
||||
let installation_id =
|
||||
load_or_create_installation_id(Some(data.path())).expect("installation id should seed");
|
||||
|
||||
assert_eq!(installation_id, desktop_id);
|
||||
assert_eq!(
|
||||
fs::read_to_string(data.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
|
||||
desktop_id
|
||||
);
|
||||
}
|
||||
|
||||
struct EnvVarGuard {
|
||||
key: &'static str,
|
||||
previous: Option<OsString>,
|
||||
}
|
||||
|
||||
impl EnvVarGuard {
|
||||
fn set(key: &'static str, value: OsString) -> Self {
|
||||
let previous = std::env::var_os(key);
|
||||
std::env::set_var(key, value);
|
||||
Self { key, previous }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EnvVarGuard {
|
||||
fn drop(&mut self) {
|
||||
match &self.previous {
|
||||
Some(value) => std::env::set_var(self.key, value),
|
||||
None => std::env::remove_var(self.key),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,9 +16,7 @@ use codex_proxy_rs::{
|
||||
admin::keys::service::SqliteClientKeyStore,
|
||||
admin::monitoring::{
|
||||
usage_record_model::UsageRecordLevel,
|
||||
usage_record_store::{
|
||||
SqliteUsageRecordStore, UsageRecordFilter, DEFAULT_USAGE_RECORD_CAPACITY,
|
||||
},
|
||||
usage_record_store::{SqliteUsageRecordStore, UsageRecordFilter},
|
||||
},
|
||||
config::schema::AppConfig,
|
||||
http::router,
|
||||
@@ -388,7 +386,6 @@ async fn test_app_with_account_pool_and_logging_capture_body(
|
||||
config.logging.enabled = true;
|
||||
let usage_record_options = UsageRecordOptions {
|
||||
enabled: true,
|
||||
capacity: DEFAULT_USAGE_RECORD_CAPACITY,
|
||||
capture_body: true,
|
||||
};
|
||||
let state = test_app_state_with_pool_installation_id_and_usage_record_options(
|
||||
@@ -554,6 +551,17 @@ async fn test_app_with_two_accounts(
|
||||
(app, api_key, pool, dir)
|
||||
}
|
||||
|
||||
async fn test_app_with_two_accounts_and_logging(
|
||||
base_url: String,
|
||||
) -> (axum::Router, String, SqlitePool, tempfile::TempDir) {
|
||||
let (app, _state, api_key, pool, dir) =
|
||||
test_app_with_two_accounts_and_state_config(base_url, |config| {
|
||||
config.logging.enabled = true;
|
||||
})
|
||||
.await;
|
||||
(app, api_key, pool, dir)
|
||||
}
|
||||
|
||||
async fn test_app_with_two_accounts_and_state(
|
||||
base_url: String,
|
||||
) -> (
|
||||
@@ -562,6 +570,19 @@ async fn test_app_with_two_accounts_and_state(
|
||||
String,
|
||||
SqlitePool,
|
||||
tempfile::TempDir,
|
||||
) {
|
||||
test_app_with_two_accounts_and_state_config(base_url, |_| {}).await
|
||||
}
|
||||
|
||||
async fn test_app_with_two_accounts_and_state_config(
|
||||
base_url: String,
|
||||
configure: impl FnOnce(&mut AppConfig),
|
||||
) -> (
|
||||
axum::Router,
|
||||
AppState,
|
||||
String,
|
||||
SqlitePool,
|
||||
tempfile::TempDir,
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = dir.path().join("openai-responses-fallback.sqlite");
|
||||
@@ -576,8 +597,10 @@ async fn test_app_with_two_accounts_and_state(
|
||||
"chatgpt-secondary",
|
||||
)
|
||||
.await;
|
||||
let mut config = test_config(url, base_url);
|
||||
configure(&mut config);
|
||||
let state = test_app_state_with_pool_and_installation_id(
|
||||
&test_config(url, base_url),
|
||||
&config,
|
||||
pool.clone(),
|
||||
TEST_INSTALLATION_ID.to_string(),
|
||||
)
|
||||
|
||||
@@ -861,6 +861,82 @@ async fn responses_should_record_request_count_when_5xx_retries_are_exhausted()
|
||||
assert_eq!(metadata["accountEmail"], "user@example.com");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_record_attempt_trace_after_retrying_another_account() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-primary"))
|
||||
.respond_with(ResponseTemplate::new(429).set_body_json(json!({
|
||||
"error": {"message": "primary rate limited"}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.and(header("authorization", "Bearer access-secondary"))
|
||||
.respond_with(ResponseTemplate::new(500).set_body_json(json!({
|
||||
"error": {"message": "secondary failed"}
|
||||
})))
|
||||
.expect(3)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let secret_input = "do not persist this input";
|
||||
let (app, api_key, pool, _dir) = test_app_with_two_accounts_and_logging(server.uri()).await;
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/responses")
|
||||
.header("authorization", format!("Bearer {api_key}"))
|
||||
.header("content-type", "application/json")
|
||||
.header("x-request-id", "req_attempt_trace_retry")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "gpt-5.5",
|
||||
"input": secret_input,
|
||||
"stream": false,
|
||||
"use_websocket": false
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let event = latest_response_usage_record(&pool).await;
|
||||
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(event.level, "error");
|
||||
assert_eq!(event.request_id.as_deref(), Some("req_attempt_trace_retry"));
|
||||
assert_eq!(event.account_id.as_deref(), Some("acct_secondary"));
|
||||
assert_eq!(event.status_code, Some(500));
|
||||
assert_eq!(metadata["eventStatusCode"], 500);
|
||||
assert_eq!(metadata["clientStatusCode"], 502);
|
||||
assert_eq!(metadata["upstreamStatusCode"], 500);
|
||||
assert_eq!(metadata["attemptIndex"], 1);
|
||||
assert_eq!(metadata["attemptAccountId"], "acct_secondary");
|
||||
assert_eq!(metadata["attemptCount"], 2);
|
||||
assert_eq!(metadata["attempts"][0]["index"], 0);
|
||||
assert_eq!(metadata["attempts"][0]["accountId"], "acct_primary");
|
||||
assert_eq!(metadata["attempts"][1]["index"], 1);
|
||||
assert_eq!(metadata["attempts"][1]["accountId"], "acct_secondary");
|
||||
assert_eq!(metadata["requestSummary"]["inputType"], "string");
|
||||
assert!(metadata["requestSummary"]["inputItemsCount"].is_null());
|
||||
assert_eq!(metadata["requestSummary"]["transport"], "http_sse");
|
||||
assert_eq!(
|
||||
metadata["requestSummary"]["localTransport"]["useWebsocket"],
|
||||
false
|
||||
);
|
||||
assert!(
|
||||
!metadata.to_string().contains(secret_input),
|
||||
"request summary must not persist input content: {metadata:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_should_record_session_affinity_for_completed_response() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -352,6 +352,14 @@ async fn runtime_account_pool_should_persist_quota_window_reset_when_cooldown_ex
|
||||
);
|
||||
assert!(persisted_reset_at.is_some_and(|reset_at| reset_at > now));
|
||||
assert_eq!(stored_limit_window_seconds, Some(60));
|
||||
let usage_record_count: i64 =
|
||||
sqlx::query_scalar("select count(*) from usage_records where account_id = ?")
|
||||
.bind("acct_quota_reset")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(usage_record_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
use codex_proxy_rs::upstream::transport::CodexUpstreamDiagnostics;
|
||||
use reqwest::header::{HeaderMap, HeaderValue};
|
||||
|
||||
#[test]
|
||||
fn diagnostics_should_extract_request_id_and_trace_headers() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-request-id", HeaderValue::from_static("req_1"));
|
||||
headers.insert("cf-ray", HeaderValue::from_static("ray_1"));
|
||||
|
||||
let diagnostics = CodexUpstreamDiagnostics::from_headers(Some(429), &headers);
|
||||
|
||||
assert_eq!(diagnostics.status_code, Some(429));
|
||||
assert_eq!(diagnostics.request_id.as_deref(), Some("req_1"));
|
||||
assert_eq!(diagnostics.cf_ray(), Some("ray_1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_should_not_treat_cf_ray_as_request_id() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("cf-ray", HeaderValue::from_static("ray_1"));
|
||||
|
||||
let diagnostics = CodexUpstreamDiagnostics::from_headers(Some(403), &headers);
|
||||
|
||||
assert_eq!(diagnostics.request_id, None);
|
||||
assert_eq!(diagnostics.cf_ray(), Some("ray_1"));
|
||||
}
|
||||
@@ -37,6 +37,7 @@ use tokio_tungstenite::{
|
||||
use crate::support::assertions::assert_substrings_appear_in_order;
|
||||
|
||||
pub(crate) mod client;
|
||||
pub(crate) mod diagnostics;
|
||||
pub(crate) mod headers;
|
||||
pub(crate) mod http_client;
|
||||
pub(crate) mod websocket;
|
||||
|
||||
@@ -322,18 +322,12 @@ async fn websocket_execute_response_create_request_should_surface_response_faile
|
||||
.expect_err("response.failed should be surfaced as upstream error");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::Upstream {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
let CodexWebSocketExchangeError::Upstream(error) = error else {
|
||||
panic!("expected upstream websocket error");
|
||||
};
|
||||
assert_eq!(status_code, 429);
|
||||
assert_eq!(retry_after_seconds, Some(12));
|
||||
assert!(body.contains("rate_limit_exceeded"));
|
||||
assert_eq!(error.status_code, 429);
|
||||
assert_eq!(error.retry_after_seconds, Some(12));
|
||||
assert!(error.body.contains("rate_limit_exceeded"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -405,18 +399,12 @@ async fn websocket_execute_response_create_request_should_preserve_opening_error
|
||||
.expect_err("failed opening should surface upstream status");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::Upstream {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
let CodexWebSocketExchangeError::Upstream(error) = error else {
|
||||
panic!("expected upstream opening error");
|
||||
};
|
||||
assert_eq!(status_code, 429);
|
||||
assert_eq!(retry_after_seconds, Some(33));
|
||||
assert!(body.contains("rate limited"));
|
||||
assert_eq!(error.status_code, 429);
|
||||
assert_eq!(error.retry_after_seconds, Some(33));
|
||||
assert!(error.body.contains("rate limited"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -482,18 +470,12 @@ async fn websocket_execute_response_create_request_should_surface_wrapped_error_
|
||||
.expect_err("wrapped error should surface as upstream error");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::Upstream {
|
||||
status_code,
|
||||
retry_after_seconds,
|
||||
body,
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
let CodexWebSocketExchangeError::Upstream(error) = error else {
|
||||
panic!("expected wrapped upstream error");
|
||||
};
|
||||
assert_eq!(status_code, 409);
|
||||
assert_eq!(retry_after_seconds, Some(17));
|
||||
assert!(body.contains("wrapped conflict"));
|
||||
assert_eq!(error.status_code, 409);
|
||||
assert_eq!(error.retry_after_seconds, Some(17));
|
||||
assert!(error.body.contains("wrapped conflict"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -530,14 +512,11 @@ async fn websocket_execute_response_create_request_should_surface_connection_lim
|
||||
.expect_err("connection limit should surface as upstream error");
|
||||
server.await.unwrap();
|
||||
|
||||
let CodexWebSocketExchangeError::Upstream {
|
||||
status_code, body, ..
|
||||
} = error
|
||||
else {
|
||||
let CodexWebSocketExchangeError::Upstream(error) = error else {
|
||||
panic!("expected connection limit upstream error");
|
||||
};
|
||||
assert_eq!(status_code, 503);
|
||||
assert!(body.contains("websocket_connection_limit_reached"));
|
||||
assert_eq!(error.status_code, 503);
|
||||
assert!(error.body.contains("websocket_connection_limit_reached"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -20,6 +20,18 @@ import { useIdSet } from '@/composables/useIdSet'
|
||||
import { withMinimumDuration } from '@/utils/async'
|
||||
|
||||
type TokenImportAccount = { token: string } | { refreshToken: string }
|
||||
type AccountRow = {
|
||||
id: string
|
||||
status: string
|
||||
planType?: string | null
|
||||
quota: any
|
||||
usage: any
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
type AccountQuotaRefreshResult = {
|
||||
account: AccountRow
|
||||
}
|
||||
|
||||
export function useAccountMutations(options: {
|
||||
page: Ref<number>
|
||||
@@ -30,7 +42,7 @@ export function useAccountMutations(options: {
|
||||
}) {
|
||||
const { downloadJson } = useDownload()
|
||||
const loading = ref(true)
|
||||
const accounts = ref<any[]>([])
|
||||
const accounts = ref<AccountRow[]>([])
|
||||
const accountSummary = ref({
|
||||
total: 0,
|
||||
active: 0,
|
||||
@@ -341,18 +353,8 @@ export function useAccountMutations(options: {
|
||||
await refreshingQuotaAccounts.run(accountId, async () => {
|
||||
try {
|
||||
await withMinimumDuration(async () => {
|
||||
const result = await getAccountQuota({ id: accountId })
|
||||
if (result?.quotaData) {
|
||||
accounts.value = accounts.value.map((account) =>
|
||||
account.id === accountId
|
||||
? {
|
||||
...account,
|
||||
quota: result.quotaData,
|
||||
planType: result.planType ?? account.planType,
|
||||
}
|
||||
: account,
|
||||
)
|
||||
}
|
||||
const result = (await getAccountQuota({ id: accountId })) as AccountQuotaRefreshResult
|
||||
mergeAccountQuotaRefresh(accountId, result)
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Failed to refresh account quota:', error)
|
||||
@@ -360,6 +362,12 @@ export function useAccountMutations(options: {
|
||||
})
|
||||
}
|
||||
|
||||
function mergeAccountQuotaRefresh(accountId: string, result: AccountQuotaRefreshResult) {
|
||||
accounts.value = accounts.value.map((account) =>
|
||||
account.id === accountId ? { ...account, ...result.account } : account,
|
||||
)
|
||||
}
|
||||
|
||||
async function handleToggleSchedule(account: any) {
|
||||
const nextStatus = account.status === 'disabled' ? 'active' : 'disabled'
|
||||
await updatingStatusAccounts.run(account.id, async () => {
|
||||
|
||||
@@ -74,7 +74,7 @@ export const usageRecordColumns = [
|
||||
{
|
||||
key: 'clientIp',
|
||||
label: 'IP',
|
||||
width: '200px',
|
||||
width: '240px',
|
||||
ellipsis: false,
|
||||
},
|
||||
{
|
||||
@@ -111,7 +111,6 @@ export const usageTimeRangeOptions = [
|
||||
{ label: '今天', value: 'today' },
|
||||
{ label: '最近 7 天', value: '7d' },
|
||||
{ label: '最近 30 天', value: '30d' },
|
||||
{ label: '全部时间', value: 'all' },
|
||||
]
|
||||
|
||||
export const levelLabels: Record<string, string> = {
|
||||
|
||||
@@ -47,10 +47,6 @@ const timeRangeParams = computed<Record<string, string>>(() => {
|
||||
}
|
||||
}
|
||||
|
||||
if (timeRange.value === 'all') {
|
||||
return {} as Record<string, string>
|
||||
}
|
||||
|
||||
return {
|
||||
startTime: now.subtract(6, 'day').startOf('day').toISOString(),
|
||||
endTime: now.toISOString(),
|
||||
|
||||
@@ -1 +1 @@
|
||||
version: "1.0.4"
|
||||
version: "1.0.6"
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
---
|
||||
name: engineering-standards
|
||||
description: 面向长期维护的工程规范工作流。当用户要求代码审计、问题修复、重构、实现方案、生产问题追溯,或强调代码质量、干净架构、拒绝垃圾代码、禁止兼容写法、先审计再改代码、参考项目对比时使用。
|
||||
---
|
||||
|
||||
# 工程规范
|
||||
|
||||
使用这个 skill 时,先证明当前系统真实行为,再决定是否需要改代码。核心规则:先画出现有证据链,再谈方案。
|
||||
|
||||
## 硬门禁
|
||||
|
||||
提出方案或动代码前,必须先给出三段内容:
|
||||
|
||||
1. 现有机制:入口、调用链、数据变化、数据读取、接口/界面展示路径。
|
||||
2. 缺口证据:具体缺失或断裂的环节,并给出源码位置。
|
||||
3. 最小改动点:在不绕开现有设计的前提下,能修复问题的最小所有权边界。
|
||||
|
||||
在现有链路被追完并证明不足之前,不要引入新的机制、分支或旁路来绕开问题。
|
||||
|
||||
## 审计流程
|
||||
|
||||
以当前仓库的实时源码为唯一事实来源。只有在当前项目链路已经清楚后,才使用记忆或参考项目。
|
||||
|
||||
1. 先确认用户讨论的数据或行为是什么。
|
||||
2. 追踪它在哪里产生、转换、持久化、加载和展示。
|
||||
3. 同时查主动链路和被动链路。运行时行为要分开检查:触发条件、执行路径、持久化、读取和展示。
|
||||
4. 设计前先分类问题所在层级:
|
||||
- 展示层
|
||||
- 接口契约
|
||||
- 持久化层
|
||||
- 运行时行为
|
||||
- 外部依赖
|
||||
- 异步或周期性行为
|
||||
5. 说明哪些证据支持该分类,哪些仍未证明。
|
||||
6. 排查中发现代码异味、疑似死代码或结构性隐患时,先记录证据,不打断主流程;主问题排查完毕后再汇总报告,并询问是否纳入修复范围。
|
||||
|
||||
如果有 `codegraph`,先用它快速定位,再回到源码和测试验证。不要把 codegraph 摘要、记忆、参考项目当成强于当前源码的证据。
|
||||
|
||||
## 参考项目
|
||||
|
||||
参考项目只能在当前项目证据清楚后,用来理解可能的模式。
|
||||
|
||||
除非同时满足以下条件,否则不要把参考项目机制搬进当前项目:
|
||||
|
||||
1. 当前项目确实缺少所需行为。
|
||||
2. 当前架构存在匹配的所有权边界。
|
||||
3. 新设计不会绕开或复制现有链路。
|
||||
4. 当前任务范围允许进入实现。
|
||||
|
||||
参考项目和当前源码冲突时,以当前源码为准。
|
||||
|
||||
## 干净实现标准
|
||||
|
||||
优先在现有所有权边界内做最直接、最干净的修复。
|
||||
|
||||
- 除非用户明确要求,不要添加兼容代码。
|
||||
- 不要用补丁式分支或重复机制掩盖边界不清。
|
||||
- 不要用大范围重构掩盖不确定性。
|
||||
- 不要为解决局部问题改动无关模块。
|
||||
- 只有在确实降低复杂度或符合现有本地模式时,才添加抽象。
|
||||
- 必要时可以重构,但重构必须收敛职责、减少分支、让主链路更清晰。
|
||||
- 不要用外层绕路掩盖内层问题。
|
||||
- 除非证明现有设计不足,不要引入新的执行路径替代原链路。
|
||||
|
||||
如果方案会改变语义,动代码前必须明确说明。
|
||||
|
||||
## 用户质量要求
|
||||
|
||||
默认用户要求严格工程纪律:
|
||||
|
||||
- 简单问题先给结论。
|
||||
- 审计必须给具体源码证据。
|
||||
- 不要做推测性断言。
|
||||
- 不要虚构任何事实、数据来源或系统行为。
|
||||
- 新项目/新链路不要塞兼容垃圾。
|
||||
- 仓库约定测试在测试目录时,不要把测试写进源码模块。
|
||||
- 使用代码索引工具辅助理解时,改代码后必须同步索引。
|
||||
|
||||
当任务范围被限定为审计、文档、评审或方案阶段时,只交付对应产物,未经要求不实现。
|
||||
|
||||
## 动代码前检查
|
||||
|
||||
编辑文件前,确认:
|
||||
|
||||
1. 当前链路已经端到端追完。
|
||||
2. 问题已经按证据分类。
|
||||
3. 现有机制不能满足需求,而不是没查到。
|
||||
4. 改动点是最小的干净边界。
|
||||
5. 测试会通过合适的公开接口或集成测试面覆盖行为。
|
||||
|
||||
任何一项缺失,都继续调查,不要实现。
|
||||
|
||||
## 收尾检查
|
||||
|
||||
结束前确认:
|
||||
|
||||
1. 运行格式化和相关测试。
|
||||
2. 如果该语言有既定 lint 命令,运行 lint。
|
||||
3. 确认没有在应该放测试目录的项目里添加源码内测试。
|
||||
4. 确认没有用参考项目模式替代当前项目证据。
|
||||
5. 确认没有未经明确批准引入额外分支或旁路设计。
|
||||
6. 需要时同步 codegraph。
|
||||
7. 总结改了什么、验证了什么、明确没有改什么。
|
||||
|
||||
## 必须避免的反模式
|
||||
|
||||
不要重复这些错误:
|
||||
|
||||
1. 只看一个局部触发点,就断言完整系统行为。
|
||||
2. 没有追完已有链路,就新增另一套处理机制。
|
||||
3. 没有证明现有机制失败,就用旁路设计绕开问题。
|
||||
4. 把参考项目的做法直接搬进当前项目,跳过当前源码证据。
|
||||
5. 用补丁式分支或重复机制掩盖所有权边界不清。
|
||||
|
||||
正确做法:
|
||||
|
||||
1. 先追当前项目已经存在的真实链路。
|
||||
2. 证明数据在产生、转换、持久化、读取和展示中的断点。
|
||||
3. 以断点为依据选择最干净的修复边界;能局部修清楚就局部修,原链路本身不干净时就重构收敛。
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "工程规范"
|
||||
short_description: "长期维护的工程规范:证据优先、范围纪律、干净实现与审计工作流。"
|
||||
default_prompt: "使用 $engineering-standards 按工程规范先审计当前代码链路,再提出或实现干净修复。"
|
||||
Reference in New Issue
Block a user