Compare commits

..
6 Commits
Author SHA1 Message Date
zyycnandCodex c6e1b347be chore: bump version to 1.0.6
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:54:27 +08:00
zyycnandCodex 1ac9803c34 fix: stop recording account state usage events
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:54:14 +08:00
zyycnandCodex 352b781340 chore: add engineering standards skill
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 01:10:02 +08:00
zyycnandCodex 62a6dfc489 chore: bump version to 1.0.5
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 00:16:26 +08:00
zyycnandCodex e91f91b542 test: move source tests to integration suite
Co-authored-by: Codex <noreply@openai.com>
2026-07-07 00:15:30 +08:00
zyycnandCodex 380556369b fix: harden runtime diagnostics and quota refresh
Co-authored-by: Codex <noreply@openai.com>
2026-07-06 23:59:27 +08:00
39 changed files with 1169 additions and 262 deletions
+52 -10
View File
@@ -424,6 +424,34 @@ impl From<AdminAccountRefreshResult> for AccountRefreshData {
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
struct AccountQuotaData {
quota: Value,
raw: Value,
quota_data: AdminAccountQuotaData,
plan_type: Option<String>,
account: AdminAccountData,
}
impl AccountQuotaData {
fn from_account(
quota: Value,
raw: Value,
quota_data: AdminAccountQuotaData,
account: AdminAccountData,
) -> Self {
let plan_type = account.plan_type.clone();
Self {
quota,
raw,
quota_data,
plan_type,
account,
}
}
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(default)]
pub(crate) struct AccountHealthCheckRequest {
@@ -664,18 +692,13 @@ pub(crate) async fn account_quota(
let quota_json = quota.to_string();
let mut quota_data = quota_data(&quota_json, Some(Utc::now()));
apply_account_quota_window_token_usage(&state, &account_id, &mut quota_data).await;
let plan_type = quota
.get("plan_type")
.and_then(Value::as_str)
.map(ToString::to_string);
let account =
account_data_for_quota_refresh(&state, &account_id, quota_data.clone()).await?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(serde_json::json!({
"quota": quota,
"raw": raw,
"quotaData": quota_data,
"planType": plan_type,
})),
AdminEnvelope::ok(AccountQuotaData::from_account(
quota, raw, quota_data, account,
)),
))
}
Err(AdminAccountError::NotFound) => Err(account_not_found()),
@@ -690,6 +713,25 @@ pub(crate) async fn account_quota(
}
}
async fn account_data_for_quota_refresh(
state: &AppState,
account_id: &str,
quota_data: AdminAccountQuotaData,
) -> Result<AdminAccountData, AdminError> {
let Some(account) = state
.services
.admin_accounts
.get(account_id)
.await
.map_err(|error| account_error(&error))?
else {
return Err(account_not_found());
};
let quota_by_account = HashMap::from([(account_id.to_string(), quota_data)]);
let stats = account_list_stats(state, std::slice::from_ref(&account), &quota_by_account).await;
Ok(stats.data_for(account))
}
/// `POST /api/admin/accounts/import`
pub(crate) async fn import_accounts(
State(state): State<AppState>,
+5 -9
View File
@@ -38,15 +38,11 @@ impl AdminAccountService {
.await
.map_err(|e| AdminAccountError::FetchQuota(e.to_string()))?;
let normalized = crate::upstream::accounts::quota::quota_from_usage(&raw);
if let Ok(json_str) = serde_json::to_string(&normalized) {
if matches!(
self.store.update_quota_json(account_id, &json_str).await,
Ok(true)
) {
self.sync_account_pool_best_effort(account_id, "account quota refresh")
.await;
}
}
self.account_pool
.apply_quota_snapshot(account_id, &normalized)
.await;
self.sync_account_pool_best_effort(account_id, "account quota refresh")
.await;
Ok(serde_json::json!({ "quota": normalized, "raw": raw }))
}
}
@@ -463,7 +463,7 @@ fn log_error(error: &AdminUsageRecordError) -> AdminError {
| AdminUsageRecordError::Get
| AdminUsageRecordError::Clear
| AdminUsageRecordError::Append
| AdminUsageRecordError::Trim => AdminError::internal(error.to_string()),
| AdminUsageRecordError::Retention => AdminError::internal(error.to_string()),
AdminUsageRecordError::Accounts => {
AdminError::usage_record_accounts_failed(error.to_string())
}
@@ -117,9 +117,9 @@ pub enum AdminUsageRecordError {
/// 写入失败。
#[error("failed to append usage record")]
Append,
/// 裁剪失败。
#[error("failed to trim usage records")]
Trim,
/// 保留期清理失败。
#[error("failed to trim expired usage records")]
Retention,
/// 账号关联信息读取失败。
#[error("failed to load usage record accounts")]
Accounts,
@@ -135,23 +135,16 @@ pub struct AdminUsageRecordService {
#[derive(Debug, Clone, Copy)]
struct AdminUsageRecordSettings {
enabled: bool,
capacity: u32,
capture_body: bool,
}
impl AdminUsageRecordService {
/// 构造管理端使用记录服务。
pub fn new(
store: SqliteUsageRecordStore,
enabled: bool,
capacity: u32,
capture_body: bool,
) -> Self {
pub fn new(store: SqliteUsageRecordStore, enabled: bool, capture_body: bool) -> Self {
Self {
store,
settings: Arc::new(tokio::sync::RwLock::new(AdminUsageRecordSettings {
enabled,
capacity,
capture_body,
})),
}
@@ -289,9 +282,9 @@ impl AdminUsageRecordService {
.await
.map_err(|_| AdminUsageRecordError::Append)?;
self.store
.trim_to_capacity(settings.capacity)
.trim_to_retention(Utc::now())
.await
.map_err(|_| AdminUsageRecordError::Trim)?;
.map_err(|_| AdminUsageRecordError::Retention)?;
Ok(())
}
}
@@ -2,7 +2,7 @@
use std::collections::{BTreeMap, HashMap};
use chrono::{DateTime, Utc};
use chrono::{DateTime, Duration, Utc};
use serde_json::Value;
use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool};
use thiserror::Error;
@@ -19,8 +19,8 @@ use crate::infra::{
time::{china_quarter_hour_start, parse_rfc3339_utc as parse_rfc3339},
};
/// 默认保留最新使用记录数量。
pub const DEFAULT_USAGE_RECORD_CAPACITY: u32 = 2_000;
/// 使用记录保留天数。
pub const USAGE_RECORD_RETENTION_DAYS: i64 = 30;
/// 默认不保存请求/响应体,避免把用户内容写入诊断日志。
pub const DEFAULT_USAGE_RECORD_CAPTURE_BODY: bool = false;
@@ -105,9 +105,9 @@ impl SqliteUsageRecordStore {
append_event(&self.pool, event).await
}
/// 保留最新的指定数量使用记录。
pub async fn trim_to_capacity(&self, capacity: u32) -> SqliteUsageRecordStoreResult<u64> {
trim_to_capacity(&self.pool, capacity).await
/// 按保留期清理使用记录。
pub async fn trim_to_retention(&self, now: DateTime<Utc>) -> SqliteUsageRecordStoreResult<u64> {
trim_to_retention(&self.pool, now).await
}
/// 分页查询使用记录。
@@ -286,7 +286,6 @@ fn usage_record_summary_fields(event: &UsageRecord) -> UsageRecordSummaryFields
"upstreamRequestId",
"upstream_request_id",
"openaiRequestId",
"cfRay",
],
)
}),
@@ -426,17 +425,15 @@ fn metadata_first_token_latency(metadata: &Value) -> Option<i64> {
})
}
async fn trim_to_capacity(pool: &SqlitePool, capacity: u32) -> SqliteUsageRecordStoreResult<u64> {
let result = sqlx::query(
"delete from usage_records where id in (
select id from usage_records
order by created_at desc, id desc
limit -1 offset ?
)",
)
.bind(i64::from(capacity))
.execute(pool)
.await?;
async fn trim_to_retention(
pool: &SqlitePool,
now: DateTime<Utc>,
) -> SqliteUsageRecordStoreResult<u64> {
let cutoff = now - Duration::days(USAGE_RECORD_RETENTION_DAYS);
let result = sqlx::query("delete from usage_records where created_at < ?")
.bind(cutoff.to_rfc3339())
.execute(pool)
.await?;
Ok(result.rows_affected())
}
-57
View File
@@ -90,60 +90,3 @@ fn parse_installation_id(raw: &str) -> Option<String> {
fn generate_installation_id() -> String {
Uuid::new_v4().to_string()
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
static ENV_LOCK: Mutex<()> = Mutex::new(());
#[test]
fn data_dir_should_use_xdg_data_home_directly() {
let _guard = ENV_LOCK.lock().unwrap();
let dir = tempfile::tempdir().unwrap();
let previous = std::env::var_os("XDG_DATA_HOME");
std::env::set_var("XDG_DATA_HOME", dir.path());
let resolved = data_dir();
restore_env("XDG_DATA_HOME", previous);
assert_eq!(resolved, dir.path());
}
#[test]
fn load_or_create_installation_id_should_persist_under_data_dir_root() {
let dir = tempfile::tempdir().unwrap();
let data_path = dir.path().join(INSTALLATION_ID_FILE_NAME);
let installation_id = load_or_create_installation_id_from_paths(None, Some(&data_path))
.expect("installation id should be generated");
assert_eq!(fs::read_to_string(data_path).unwrap(), installation_id);
}
#[test]
fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() {
let dir = tempfile::tempdir().unwrap();
let codex_path = dir.path().join(".codex").join(INSTALLATION_ID_FILE_NAME);
let data_path = dir.path().join("data").join(INSTALLATION_ID_FILE_NAME);
let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a";
persist_installation_id(&codex_path, desktop_id).unwrap();
let installation_id =
load_or_create_installation_id_from_paths(Some(&codex_path), Some(&data_path))
.expect("installation id should be seeded");
assert_eq!(installation_id, desktop_id);
assert_eq!(fs::read_to_string(data_path).unwrap(), desktop_id);
}
fn restore_env(key: &str, value: Option<std::ffi::OsString>) {
if let Some(value) = value {
std::env::set_var(key, value);
} else {
std::env::remove_var(key);
}
}
}
+47 -1
View File
@@ -6,7 +6,9 @@ use serde_json::{json, Map, Value};
use crate::{
proxy::dispatch::exhaustion::{ExhaustedAccountKind, ExhaustedAccountRef},
upstream::accounts::model::AccountStatus,
upstream::transport::{is_banned_auth_signal, CodexBackendTransport, CodexClientError},
upstream::transport::{
is_banned_auth_signal, CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
},
};
#[derive(Clone, Debug)]
@@ -15,6 +17,7 @@ pub(crate) struct DispatchErrorMetadata {
pub exhausted_count: Option<usize>,
pub upstream_error: Option<String>,
pub upstream_status: Option<u16>,
pub diagnostics: Option<CodexUpstreamDiagnostics>,
}
#[derive(Clone, Copy, Debug)]
@@ -63,6 +66,7 @@ impl DispatchErrorMetadata {
exhausted_count: None,
upstream_error: None,
upstream_status: None,
diagnostics: None,
}
}
@@ -72,6 +76,7 @@ impl DispatchErrorMetadata {
exhausted_count: Some(exhausted.count),
upstream_error: Some(exhausted.upstream_error.to_string()),
upstream_status: None,
diagnostics: None,
}
}
@@ -84,6 +89,7 @@ impl DispatchErrorMetadata {
CodexClientError::Upstream { status, .. } => Some(status.as_u16()),
_ => None,
},
diagnostics: upstream_error_diagnostics(error).cloned(),
}
}
@@ -93,6 +99,7 @@ impl DispatchErrorMetadata {
exhausted_count: None,
upstream_error: None,
upstream_status: None,
diagnostics: None,
}
}
}
@@ -129,6 +136,45 @@ pub(crate) fn insert_dispatch_error_metadata(
if let Some(status) = metadata.upstream_status {
object.insert("upstreamStatus".to_string(), json!(status));
}
if let Some(diagnostics) = metadata.diagnostics {
insert_upstream_diagnostics_metadata(object, &diagnostics);
}
}
pub(crate) fn insert_upstream_diagnostics_metadata(
object: &mut Map<String, Value>,
diagnostics: &CodexUpstreamDiagnostics,
) {
if diagnostics.is_empty() {
return;
}
if let Some(status_code) = diagnostics.status_code {
object.insert("upstreamStatus".to_string(), json!(status_code));
}
if let Some(request_id) = &diagnostics.request_id {
object.insert(
"upstreamRequestId".to_string(),
Value::String(request_id.clone()),
);
}
if !diagnostics.trace_headers.is_empty() {
object.insert(
"upstreamTraceHeaders".to_string(),
json!(diagnostics.trace_headers),
);
}
if let Some(cf_ray) = diagnostics.cf_ray() {
object.insert("cfRay".to_string(), Value::String(cf_ray.to_string()));
}
}
pub(crate) fn upstream_error_diagnostics(
error: &CodexClientError,
) -> Option<&CodexUpstreamDiagnostics> {
match error {
CodexClientError::Upstream { diagnostics, .. } => Some(diagnostics),
_ => None,
}
}
pub(crate) fn is_rate_limit_upstream_error(error: &CodexClientError) -> bool {
@@ -199,6 +199,7 @@ impl ResponseDispatchError {
exhausted_count: None,
upstream_error: Some(sse_failure_error_body(failure)),
upstream_status: None,
diagnostics: None,
},
}
}
@@ -1,6 +1,6 @@
use std::time::Instant;
use serde_json::{json, Value};
use serde_json::{json, Map, Value};
use crate::{
admin::monitoring::{
@@ -9,7 +9,11 @@ use crate::{
},
infra::time::elapsed_millis_i64,
proxy::dispatch::{
errors::{backend_transport_name, upstream_error_http_status},
errors::{
backend_transport_name, insert_dispatch_error_metadata,
insert_upstream_diagnostics_metadata, upstream_error_http_status,
DispatchErrorMetadata,
},
usage_events::{
enrich_event_route_metadata, enrich_usage_record_identity,
event_kind as response_event_kind, reasoning_effort_from_request,
@@ -18,7 +22,10 @@ use crate::{
},
upstream::{
protocol::responses::{CodexResponsesRequest, ResponsesSseFailure},
transport::{CodexBackendTransport, CodexClientError, WebSocketPoolDecision},
transport::{
CodexBackendTransport, CodexClientError, CodexUpstreamDiagnostics,
WebSocketPoolDecision,
},
},
};
@@ -28,6 +35,7 @@ use super::{
},
sse_failure::{status_code_for_stream_failure, stream_failure_metadata},
stream_lifecycle::{latest_response_id, LiveResponseStreamContext},
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
};
pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
@@ -40,7 +48,10 @@ pub(super) struct ResponseUpstreamErrorEventRecord<'a> {
pub(super) started_at: Instant,
pub(super) stream: bool,
pub(super) transport: CodexBackendTransport,
pub(super) request: &'a CodexResponsesRequest,
pub(super) error: &'a CodexClientError,
pub(super) trace: &'a ResponseDispatchTrace,
pub(super) attempt: Option<&'a ResponseDispatchAttempt>,
}
pub(super) struct ResponseStreamFailureEventRecord<'a> {
@@ -54,8 +65,11 @@ pub(super) struct ResponseStreamFailureEventRecord<'a> {
pub(super) transport: CodexBackendTransport,
pub(super) request: &'a CodexResponsesRequest,
pub(super) failure: &'a ResponsesSseFailure,
pub(super) diagnostics: &'a CodexUpstreamDiagnostics,
pub(super) rate_limit_headers: &'a [(String, String)],
pub(super) prefetched: &'a [u8],
pub(super) trace: &'a ResponseDispatchTrace,
pub(super) attempt: &'a ResponseDispatchAttempt,
}
pub(super) struct ResponseDispatchErrorEventRecord<'a> {
@@ -88,6 +102,14 @@ pub(super) async fn record_response_dispatch_error_event(
record.transport,
);
enrich_response_dispatch_error_metadata(&mut metadata, record.error);
if let Some(object) = metadata.as_object_mut() {
insert_response_status_metadata_object(
object,
i64::from(record.error.http_status_code()),
dispatch_error_client_status_code(record.error, record.stream, record.compact),
dispatch_error_upstream_status_code(record.error),
);
}
record_dispatch_error_event(DispatchErrorUsageRecord {
usage_records: record.usage_records,
request_id: record.request_id,
@@ -105,6 +127,7 @@ pub(super) async fn record_response_dispatch_error_event(
pub(super) async fn record_response_upstream_error_event(
record: ResponseUpstreamErrorEventRecord<'_>,
) {
let event_status_code = i64::from(upstream_error_http_status(record.error));
let mut metadata = dispatch_error_metadata(
record.error,
record.stream,
@@ -121,9 +144,18 @@ pub(super) async fn record_response_upstream_error_event(
event.account_id = Some(record.account_id.to_string());
event.route = Some(record.route.to_string());
event.model = Some(record.model.to_string());
event.status_code = Some(i64::from(upstream_error_http_status(record.error)));
event.status_code = Some(event_status_code);
event.latency_ms = Some(elapsed_millis_i64(record.started_at));
if let Some(object) = metadata.as_object_mut() {
insert_dispatch_error_metadata(object, DispatchErrorMetadata::upstream(record.error));
insert_response_status_metadata_object(
object,
event_status_code,
upstream_failure_client_status_code(record.stream),
Some(event_status_code),
);
insert_response_request_summary_object(object, record.request, record.transport);
insert_response_trace_metadata_object(object, record.trace.attempts(), record.attempt);
if let Some(account_email) = record
.account_email
.map(str::trim)
@@ -144,6 +176,7 @@ pub(super) async fn record_response_upstream_error_event(
pub(super) async fn record_prefetched_response_stream_failure_event(
record: ResponseStreamFailureEventRecord<'_>,
) {
let event_status_code = status_code_for_stream_failure(record.failure);
let mut metadata = stream_failure_metadata(record.failure, None);
if let Some(object) = metadata.as_object_mut() {
if record.transport == CodexBackendTransport::WebSocket {
@@ -152,11 +185,24 @@ pub(super) async fn record_prefetched_response_stream_failure_event(
Value::String("websocket".to_string()),
);
}
insert_response_status_metadata_object(
object,
event_status_code,
200,
diagnostics_status_code(record.diagnostics),
);
insert_response_request_summary_object(object, record.request, record.transport);
object.insert("requestBody".to_string(), json!(record.request));
object.insert(
"responseBody".to_string(),
Value::String(String::from_utf8_lossy(record.prefetched).to_string()),
);
insert_upstream_diagnostics_metadata(object, record.diagnostics);
insert_response_trace_metadata_object(
object,
record.trace.attempts(),
Some(record.attempt),
);
}
record_response_event(ResponseUsageRecord {
usage_records: record.usage_records,
@@ -170,7 +216,7 @@ pub(super) async fn record_prefetched_response_stream_failure_event(
reasoning_effort: reasoning_effort_from_request(record.request),
service_tier: record.request.service_tier(),
started_at: record.started_at,
status_code: status_code_for_stream_failure(record.failure),
status_code: event_status_code,
level: UsageRecordLevel::Error,
message: "v1 responses stream failed",
metadata,
@@ -193,6 +239,7 @@ fn enrich_live_response_stream_metadata(
context: &LiveResponseStreamContext,
rate_limit_headers: &[(String, String)],
metadata: &mut Value,
status_code: i64,
body: &str,
) {
let Some(object) = metadata.as_object_mut() else {
@@ -211,6 +258,15 @@ fn enrich_live_response_stream_metadata(
.entry("websocketPool".to_string())
.or_insert_with(|| decision.metadata_value());
}
insert_response_status_metadata_object(
object,
status_code,
200,
diagnostics_status_code(&context.diagnostics),
);
insert_response_request_summary_object(object, &context.request, context.transport);
insert_upstream_diagnostics_metadata(object, &context.diagnostics);
insert_response_trace_metadata_object(object, &context.attempts, Some(&context.attempt));
object
.entry("requestBody".to_string())
.or_insert_with(|| serde_json::json!(context.request));
@@ -219,6 +275,181 @@ fn enrich_live_response_stream_metadata(
.or_insert_with(|| Value::String(body.to_string()));
}
pub(super) fn insert_response_upstream_diagnostics(
metadata: &mut Value,
diagnostics: &CodexUpstreamDiagnostics,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_upstream_diagnostics_metadata(object, diagnostics);
}
pub(super) fn insert_response_status_metadata(
metadata: &mut Value,
event_status_code: i64,
client_status_code: i64,
upstream_status_code: Option<i64>,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_response_status_metadata_object(
object,
event_status_code,
client_status_code,
upstream_status_code,
);
}
pub(super) fn insert_response_trace_metadata(
metadata: &mut Value,
trace: &ResponseDispatchTrace,
current_attempt: Option<&ResponseDispatchAttempt>,
) {
let Some(object) = metadata.as_object_mut() else {
return;
};
insert_response_trace_metadata_object(object, trace.attempts(), current_attempt);
}
fn insert_response_attempt_metadata_object(
object: &mut serde_json::Map<String, Value>,
attempt: Option<&ResponseDispatchAttempt>,
) {
let Some(attempt) = attempt else {
return;
};
object.insert("attemptIndex".to_string(), json!(attempt.index()));
object.insert(
"attemptAccountId".to_string(),
Value::String(attempt.account_id().to_string()),
);
}
fn insert_response_trace_metadata_object(
object: &mut Map<String, Value>,
attempts: &[ResponseDispatchAttempt],
current_attempt: Option<&ResponseDispatchAttempt>,
) {
insert_response_attempt_metadata_object(object, current_attempt);
object.insert("attemptCount".to_string(), json!(attempts.len()));
object.insert(
"attempts".to_string(),
Value::Array(
attempts
.iter()
.map(|attempt| {
json!({
"index": attempt.index(),
"accountId": attempt.account_id(),
})
})
.collect(),
),
);
}
fn insert_response_status_metadata_object(
object: &mut Map<String, Value>,
event_status_code: i64,
client_status_code: i64,
upstream_status_code: Option<i64>,
) {
object.insert("eventStatusCode".to_string(), json!(event_status_code));
object.insert("clientStatusCode".to_string(), json!(client_status_code));
if let Some(upstream_status_code) = upstream_status_code {
object.insert(
"upstreamStatusCode".to_string(),
json!(upstream_status_code),
);
}
}
fn insert_response_request_summary_object(
object: &mut Map<String, Value>,
request: &CodexResponsesRequest,
transport: CodexBackendTransport,
) {
object.insert(
"requestSummary".to_string(),
response_request_summary(request, transport),
);
}
fn response_request_summary(
request: &CodexResponsesRequest,
transport: CodexBackendTransport,
) -> Value {
let body = request.body();
let input = body.get("input");
let tools = body.get("tools");
json!({
"model": request.model(),
"stream": request.stream(),
"store": request.store(),
"transport": backend_transport_name(transport),
"inputType": json_value_kind(input),
"inputItemsCount": input.and_then(Value::as_array).map(Vec::len),
"toolsType": json_value_kind(tools),
"toolsCount": tools.and_then(Value::as_array).map(Vec::len),
"topLevelFields": body.keys().cloned().collect::<Vec<_>>(),
"previousResponseIdPresent": request.previous_response_id().is_some(),
"serviceTier": request.service_tier(),
"localTransport": {
"useWebsocket": request.use_websocket,
"forceHttpSse": request.force_http_sse,
},
})
}
fn json_value_kind(value: Option<&Value>) -> &'static str {
match value {
None => "missing",
Some(Value::Null) => "null",
Some(Value::Bool(_)) => "boolean",
Some(Value::Number(_)) => "number",
Some(Value::String(_)) => "string",
Some(Value::Array(_)) => "array",
Some(Value::Object(_)) => "object",
}
}
fn diagnostics_status_code(diagnostics: &CodexUpstreamDiagnostics) -> Option<i64> {
diagnostics.status_code.map(i64::from)
}
fn dispatch_error_upstream_status_code(error: &ResponseDispatchError) -> Option<i64> {
match error {
ResponseDispatchError::Upstream(error) => {
Some(i64::from(upstream_error_http_status(error)))
}
_ => None,
}
}
fn dispatch_error_client_status_code(
error: &ResponseDispatchError,
stream: bool,
compact: bool,
) -> i64 {
if stream {
return 200;
}
if matches!(error, ResponseDispatchError::Upstream(_)) && !compact {
return 502;
}
i64::from(error.http_status_code())
}
fn upstream_failure_client_status_code(stream: bool) -> i64 {
if stream {
200
} else {
502
}
}
pub(super) async fn record_live_response_stream_event(
context: &LiveResponseStreamContext,
status_code: i64,
@@ -230,7 +461,13 @@ pub(super) async fn record_live_response_stream_event(
) {
ensure_stream_metadata_flag(&mut metadata);
enrich_event_route_metadata(&mut metadata, &context.route);
enrich_live_response_stream_metadata(context, rate_limit_headers, &mut metadata, body);
enrich_live_response_stream_metadata(
context,
rate_limit_headers,
&mut metadata,
status_code,
body,
);
log_live_response_stream_finalized(context, status_code, level, message, &metadata, body);
let mut event = UsageRecord::new(response_event_kind(&context.route), level, message);
event.request_id = Some(context.request_id.clone());
@@ -8,3 +8,4 @@ mod prefetch;
pub(crate) mod service;
mod sse_failure;
mod stream_lifecycle;
mod trace;
+54 -11
View File
@@ -82,10 +82,12 @@ use super::{
affinity::{evict_reasoning_replay, record_response_affinity},
errors::{ResponseDispatchError, ResponseDispatchStreamError},
event_recording::{
insert_websocket_pool_decision, record_prefetched_response_stream_failure_event,
record_response_dispatch_error_event, record_response_upstream_error_event,
ResponseDispatchErrorDetails, ResponseDispatchErrorEventRecord,
ResponseStreamFailureEventRecord, ResponseUpstreamErrorEventRecord,
insert_response_status_metadata, insert_response_trace_metadata,
insert_response_upstream_diagnostics, insert_websocket_pool_decision,
record_prefetched_response_stream_failure_event, record_response_dispatch_error_event,
record_response_upstream_error_event, ResponseDispatchErrorDetails,
ResponseDispatchErrorEventRecord, ResponseStreamFailureEventRecord,
ResponseUpstreamErrorEventRecord,
},
live_stream::spawn_live_response_stream,
prefetch::prefetch_first_sse_chunk,
@@ -96,6 +98,7 @@ use super::{
sse_failure_invalid_reasoning_replay, stream_failure_http_status,
},
stream_lifecycle::LiveResponseStreamContext,
trace::{ResponseDispatchAttempt, ResponseDispatchTrace},
};
/// OpenAI Responses 调度服务。
@@ -387,11 +390,13 @@ impl ResponseDispatchService {
let mut history_recovery_used = false;
let mut empty_response_retries = 0u8;
let mut quota_verify_attempts = 0usize;
let mut trace = ResponseDispatchTrace::default();
const MAX_EMPTY_RESPONSE_RETRIES: u8 = 2;
let (account, response, collected_response): (
let (account, response, collected_response, attempt): (
Account,
CodexBackendResponse,
CollectedResponse,
ResponseDispatchAttempt,
) = loop {
let mut attempt_acquire_request = acquire_request
.clone()
@@ -487,6 +492,7 @@ impl ResponseDispatchService {
self.account_pool.wait_for_request_interval(&acquired).await;
let account = acquired.account;
let release_account_id = account.id.clone();
let attempt = trace.start_attempt(&release_account_id);
let response_result = create_response_with_account_retrying_5xx(
&self.codex,
self.installation_id.as_deref(),
@@ -626,7 +632,7 @@ impl ResponseDispatchService {
continue;
}
}
break (account, response, collected_response);
break (account, response, collected_response, attempt);
}
Err(error) if is_rate_limit_upstream_error(&error) => {
exhausted_accounts.record_rate_limited(
@@ -750,7 +756,10 @@ impl ResponseDispatchService {
started_at,
stream: false,
transport: backend_transport_for_response_request(&request),
request: &request,
error: &error,
trace: &trace,
attempt: Some(&attempt),
})
.await;
return Err(ResponseDispatchError::Upstream(error));
@@ -787,6 +796,14 @@ impl ResponseDispatchService {
"firstTokenMs": response.first_token_ms,
"usage": response.usage,
});
insert_response_status_metadata(
&mut metadata,
200,
200,
response.diagnostics.status_code.map(i64::from),
);
insert_response_upstream_diagnostics(&mut metadata, &response.diagnostics);
insert_response_trace_metadata(&mut metadata, &trace, Some(&attempt));
insert_websocket_pool_decision(&mut metadata, response.websocket_pool_decision);
record_response_event(ResponseUsageRecord {
usage_records: &self.usage_records,
@@ -916,6 +933,7 @@ impl ResponseDispatchService {
let mut exhausted_accounts = AccountExhaustionTracker::default();
let mut history_recovery_used = false;
let mut quota_verify_attempts = 0usize;
let mut trace = ResponseDispatchTrace::default();
macro_rules! return_stream_dispatch_error {
($error:expr) => {{
let error = $error;
@@ -1024,6 +1042,7 @@ impl ResponseDispatchService {
self.account_pool.wait_for_request_interval(&acquired).await;
let account = acquired.account;
let release_account_id = account.id.clone();
let attempt = trace.start_attempt(&release_account_id);
let response_result = create_response_stream_with_account_retrying_5xx(
&self.codex,
self.installation_id.as_deref(),
@@ -1051,6 +1070,7 @@ impl ResponseDispatchService {
let turn_state_update = response.turn_state_update;
let websocket_pool_decision = response.websocket_pool_decision;
let turn_state = response.turn_state;
let diagnostics = response.diagnostics;
self.cloudflare
.capture_set_cookie_headers(&release_account_id, &set_cookie_headers)
.await;
@@ -1204,7 +1224,10 @@ impl ResponseDispatchService {
started_at,
stream: true,
transport,
request: &request,
error: upstream_error,
trace: &trace,
attempt: Some(&attempt),
},
)
.await;
@@ -1306,8 +1329,11 @@ impl ResponseDispatchService {
transport,
request: &request,
failure: &failure,
diagnostics: &diagnostics,
rate_limit_headers: &rate_limit_headers,
prefetched: &prefetched,
trace: &trace,
attempt: &attempt,
},
)
.await;
@@ -1336,6 +1362,9 @@ impl ResponseDispatchService {
turn_state_update,
websocket_pool_decision,
turn_state,
diagnostics,
attempt: attempt.clone(),
attempts: trace.attempts().to_vec(),
started_at,
};
return Ok(spawn_live_response_stream(context, prefetched, body));
@@ -1460,7 +1489,10 @@ impl ResponseDispatchService {
started_at,
stream: true,
transport: backend_transport_for_response_request(&request),
request: &request,
error: &error,
trace: &trace,
attempt: Some(&attempt),
})
.await;
return Err(ResponseDispatchError::Upstream(error));
@@ -1483,6 +1515,7 @@ impl ResponseDispatchService {
let mut excluded_account_ids = Vec::new();
let mut exhausted_accounts = AccountExhaustionTracker::default();
let mut quota_verify_attempts = 0usize;
let mut trace = ResponseDispatchTrace::default();
loop {
let acquire_request = AccountAcquireRequest::new(request.model(), Utc::now())
@@ -1548,6 +1581,7 @@ impl ResponseDispatchService {
};
let account = acquired.account;
let release_account_id = account.id.clone();
let attempt = trace.start_attempt(&release_account_id);
let response_result = create_compact_response_with_account_retrying_5xx(
&self.codex,
self.installation_id.as_deref(),
@@ -1585,6 +1619,19 @@ impl ResponseDispatchService {
.record_token_usage(&account.id, request.model(), &usage)
.await;
}
let mut metadata = json!({
"stream": false,
"compact": true,
"usage": usage,
});
insert_response_status_metadata(
&mut metadata,
200,
200,
response.diagnostics.status_code.map(i64::from),
);
insert_response_upstream_diagnostics(&mut metadata, &response.diagnostics);
insert_response_trace_metadata(&mut metadata, &trace, Some(&attempt));
record_response_event(ResponseUsageRecord {
usage_records: &self.usage_records,
request_id,
@@ -1600,11 +1647,7 @@ impl ResponseDispatchService {
status_code: 200,
level: UsageRecordLevel::Info,
message: "v1 responses compact completed",
metadata: json!({
"stream": false,
"compact": true,
"usage": usage,
}),
metadata,
rate_limit_headers: &response.rate_limit_headers,
})
.await;
@@ -24,7 +24,7 @@ use crate::{
},
transport::{
CodexBackendTransport, CodexRateLimitHeaderUpdates, CodexTurnStateUpdate,
WebSocketPoolDecision,
CodexUpstreamDiagnostics, WebSocketPoolDecision,
},
},
};
@@ -40,6 +40,7 @@ use super::{
stream_failure_metadata, stream_failure_source, synthetic_stream_disconnected_detail,
STREAM_DISCONNECTED_CODE, STREAM_DISCONNECTED_MESSAGE,
},
trace::ResponseDispatchAttempt,
};
use crate::proxy::dispatch::cloudflare::CloudflareRecovery;
@@ -65,6 +66,9 @@ pub(super) struct LiveResponseStreamContext {
pub(super) turn_state_update: Option<CodexTurnStateUpdate>,
pub(super) websocket_pool_decision: Option<WebSocketPoolDecision>,
pub(super) turn_state: Option<String>,
pub(super) diagnostics: CodexUpstreamDiagnostics,
pub(super) attempt: ResponseDispatchAttempt,
pub(super) attempts: Vec<ResponseDispatchAttempt>,
pub(super) started_at: Instant,
}
@@ -0,0 +1,39 @@
//! Request-level trace state for Responses dispatch.
#[derive(Debug, Default)]
pub(super) struct ResponseDispatchTrace {
next_attempt_index: i64,
attempts: Vec<ResponseDispatchAttempt>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct ResponseDispatchAttempt {
index: i64,
account_id: String,
}
impl ResponseDispatchTrace {
pub(super) fn start_attempt(&mut self, account_id: &str) -> ResponseDispatchAttempt {
let attempt = ResponseDispatchAttempt {
index: self.next_attempt_index,
account_id: account_id.to_string(),
};
self.next_attempt_index += 1;
self.attempts.push(attempt.clone());
attempt
}
pub(super) fn attempts(&self) -> &[ResponseDispatchAttempt] {
&self.attempts
}
}
impl ResponseDispatchAttempt {
pub(super) fn index(&self) -> i64 {
self.index
}
pub(super) fn account_id(&self) -> &str {
&self.account_id
}
}
+6 -12
View File
@@ -7,10 +7,7 @@ use crate::{
account_usage_service::AdminUsageService,
account_usage_store::SqliteUsageStore,
usage_record_service::AdminUsageRecordService,
usage_record_store::{
SqliteUsageRecordStore, DEFAULT_USAGE_RECORD_CAPACITY,
DEFAULT_USAGE_RECORD_CAPTURE_BODY,
},
usage_record_store::{SqliteUsageRecordStore, DEFAULT_USAGE_RECORD_CAPTURE_BODY},
},
admin::{
accounts::service::{AdminAccountService, AdminAccountServiceParts},
@@ -110,7 +107,6 @@ pub struct Services {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UsageRecordOptions {
pub enabled: bool,
pub capacity: u32,
pub capture_body: bool,
}
@@ -118,7 +114,6 @@ impl UsageRecordOptions {
pub fn from_config(config: &AppConfig) -> Self {
Self {
enabled: config.logging.enabled,
capacity: DEFAULT_USAGE_RECORD_CAPACITY,
capture_body: DEFAULT_USAGE_RECORD_CAPTURE_BODY,
}
}
@@ -189,6 +184,11 @@ impl Services {
StdArc::new(client)
}
};
let usage_records = StdArc::new(AdminUsageRecordService::new(
stores.usage_records.clone(),
usage_record_options.enabled,
usage_record_options.capture_body,
));
let account_pool = StdArc::new(RuntimeAccountPoolService::new(
account_store_trait.clone(),
account_pool_options_from_config(config),
@@ -251,12 +251,6 @@ impl Services {
refresh_policy: refresh_policy.clone(),
installation_id: installation_id.clone(),
}));
let usage_records = StdArc::new(AdminUsageRecordService::new(
stores.usage_records.clone(),
usage_record_options.enabled,
usage_record_options.capacity,
usage_record_options.capture_body,
));
let usage_store = SqliteUsageStore::new(stores.accounts.pool().clone());
let usage = StdArc::new(AdminUsageService::new(usage_store));
let session_affinity = StdArc::new(RuntimeSessionAffinityService::new(
+10 -6
View File
@@ -562,9 +562,10 @@ impl AccountPool {
}
let after = RuntimeAccountStateSnapshot::from(&*account);
if before != after {
let sync_usage_window = before.usage_window_changed(&after);
refreshed_accounts.push(RefreshedAccountState {
account: account.clone(),
sync_usage_window: before.usage_window_changed(&after),
sync_usage_window,
});
}
}
@@ -1310,11 +1311,14 @@ impl RuntimeAccountPoolService {
}
}
let mut pool = self.pool.lock().await;
let in_memory = pool.apply_quota_state(account_id, limit_reached, cooldown_until);
if let Some(reset_at) = reset_at {
pool.sync_rate_limit_window(account_id, reset_at, limit_window_seconds);
}
let in_memory = {
let mut pool = self.pool.lock().await;
let in_memory = pool.apply_quota_state(account_id, limit_reached, cooldown_until);
if let Some(reset_at) = reset_at {
pool.sync_rate_limit_window(account_id, reset_at, limit_window_seconds);
}
in_memory
};
persisted || in_memory
}
+35 -11
View File
@@ -34,6 +34,7 @@ use crate::upstream::protocol::websocket::{
websocket_audit_artifact_from_attempt, websocket_payload_audit_snapshot,
};
use super::diagnostics::CodexUpstreamDiagnostics;
use super::endpoints::{endpoint_url, CODEX_RESPONSES_COMPACT_PATH, CODEX_RESPONSES_PATH};
use super::headers::{
build_ordered_codex_base_headers, insert_optional_header, insert_ordered_headers,
@@ -129,6 +130,8 @@ pub enum CodexClientError {
body: String,
/// 推导出的重试秒数。
retry_after_seconds: Option<u64>,
/// 上游诊断元数据。
diagnostics: CodexUpstreamDiagnostics,
/// 上游透传的 `set-cookie` 列表。
set_cookie_headers: Vec<String>,
},
@@ -255,6 +258,8 @@ pub struct CodexBackendResponse {
pub first_token_ms: Option<i64>,
/// WebSocket 连接池决策。
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
/// Codex Responses 实际使用的上游传输。
@@ -302,6 +307,8 @@ pub struct CodexBackendStreamingResponse {
pub turn_state_update: Option<CodexTurnStateUpdate>,
/// WebSocket 连接池决策。
pub websocket_pool_decision: Option<WebSocketPoolDecision>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
/// Codex compact 端点响应。
@@ -313,6 +320,8 @@ pub struct CodexCompactResponse {
pub set_cookie_headers: Vec<String>,
/// 上游透传的限流头。
pub rate_limit_headers: Vec<(String, String)>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
// ---------------------------------------------------------------------------
@@ -376,6 +385,7 @@ impl CodexBackendClient {
.send()
.await?;
let status = response.status();
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let turn_state = response_meta::turn_state(response.headers());
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
@@ -388,6 +398,7 @@ impl CodexBackendClient {
retry_after_seconds: retry_after_seconds
.or_else(|| retry_after_seconds_from_body(&body)),
body,
diagnostics,
set_cookie_headers,
});
}
@@ -410,6 +421,7 @@ impl CodexBackendClient {
rate_limit_headers,
first_token_ms,
websocket_pool_decision: None,
diagnostics,
})
}
@@ -428,6 +440,7 @@ impl CodexBackendClient {
.send()
.await?;
let status = response.status();
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let turn_state = response_meta::turn_state(response.headers());
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
@@ -440,6 +453,7 @@ impl CodexBackendClient {
retry_after_seconds: retry_after_seconds
.or_else(|| retry_after_seconds_from_body(&body)),
body,
diagnostics,
set_cookie_headers,
});
}
@@ -453,6 +467,7 @@ impl CodexBackendClient {
rate_limit_header_updates: None,
turn_state_update: None,
websocket_pool_decision: None,
diagnostics,
})
}
@@ -634,6 +649,7 @@ impl CodexBackendClient {
rate_limit_headers: exchange.rate_limit_headers,
first_token_ms: exchange.first_token_ms,
websocket_pool_decision: exchange.pool_decision,
diagnostics: exchange.diagnostics,
})
}
@@ -701,6 +717,7 @@ impl CodexBackendClient {
rate_limit_header_updates: Some(exchange.rate_limit_header_updates),
turn_state_update: Some(exchange.turn_state_update),
websocket_pool_decision: exchange.pool_decision,
diagnostics: exchange.diagnostics,
})
}
@@ -738,6 +755,7 @@ impl CodexBackendClient {
.await?;
let status = response.status();
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let set_cookie_headers = response_meta::set_cookie_headers(response.headers());
let rate_limit_headers = response_meta::rate_limit_headers(response.headers());
let retry_after_seconds = retry_after_seconds(response.headers(), None);
@@ -749,6 +767,7 @@ impl CodexBackendClient {
retry_after_seconds: retry_after_seconds
.or_else(|| retry_after_seconds_from_body(&body)),
body,
diagnostics,
set_cookie_headers,
});
}
@@ -761,12 +780,16 @@ impl CodexBackendClient {
"Compact response is not valid JSON: {}",
truncate_for_error(&body)
),
diagnostics: CodexUpstreamDiagnostics::with_status(
StatusCode::BAD_GATEWAY.as_u16(),
),
set_cookie_headers: set_cookie_headers.clone(),
})?;
Ok(CodexCompactResponse {
body: parsed,
set_cookie_headers,
rate_limit_headers,
diagnostics,
})
}
@@ -801,6 +824,7 @@ impl CodexBackendClient {
status: StatusCode::BAD_GATEWAY,
retry_after_seconds: None,
body: "backend model catalog is unavailable".to_string(),
diagnostics: CodexUpstreamDiagnostics::with_status(StatusCode::BAD_GATEWAY.as_u16()),
set_cookie_headers: Vec::new(),
})
}
@@ -1201,17 +1225,17 @@ fn websocket_exchange_error_to_client_error(
error: CodexWebSocketExchangeError,
) -> CodexClientError {
match error {
CodexWebSocketExchangeError::Upstream {
status_code,
retry_after_seconds,
body,
set_cookie_headers,
} => CodexClientError::Upstream {
status: StatusCode::from_u16(status_code).unwrap_or(StatusCode::BAD_GATEWAY),
body,
retry_after_seconds,
set_cookie_headers,
},
CodexWebSocketExchangeError::Upstream(upstream) => {
let upstream = *upstream;
CodexClientError::Upstream {
status: StatusCode::from_u16(upstream.status_code)
.unwrap_or(StatusCode::BAD_GATEWAY),
body: upstream.body,
retry_after_seconds: upstream.retry_after_seconds,
diagnostics: upstream.diagnostics,
set_cookie_headers: upstream.set_cookie_headers,
}
}
error => CodexClientError::WebSocket(error),
}
}
@@ -0,0 +1,73 @@
//! Upstream response diagnostics captured at the transport boundary.
use reqwest::header::HeaderMap;
const UPSTREAM_REQUEST_ID_HEADERS: &[&str] = &[
"x-request-id",
"x-openai-request-id",
"openai-request-id",
"request-id",
];
const UPSTREAM_TRACE_HEADERS: &[&str] = &[
"x-request-id",
"x-openai-request-id",
"openai-request-id",
"request-id",
"cf-ray",
];
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CodexUpstreamDiagnostics {
pub status_code: Option<u16>,
pub request_id: Option<String>,
pub trace_headers: Vec<(String, String)>,
}
impl CodexUpstreamDiagnostics {
pub fn from_headers(status_code: Option<u16>, headers: &HeaderMap) -> Self {
Self {
status_code,
request_id: first_header(headers, UPSTREAM_REQUEST_ID_HEADERS),
trace_headers: trace_headers(headers),
}
}
pub fn with_status(status_code: u16) -> Self {
Self {
status_code: Some(status_code),
..Self::default()
}
}
pub fn is_empty(&self) -> bool {
self.status_code.is_none() && self.request_id.is_none() && self.trace_headers.is_empty()
}
pub fn cf_ray(&self) -> Option<&str> {
self.trace_headers
.iter()
.find(|(name, _)| name.eq_ignore_ascii_case("cf-ray"))
.map(|(_, value)| value.as_str())
}
}
fn first_header(headers: &HeaderMap, names: &[&str]) -> Option<String> {
names.iter().find_map(|name| header_value(headers, name))
}
fn trace_headers(headers: &HeaderMap) -> Vec<(String, String)> {
UPSTREAM_TRACE_HEADERS
.iter()
.filter_map(|name| header_value(headers, name).map(|value| ((*name).to_string(), value)))
.collect()
}
fn header_value(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
+2
View File
@@ -1,6 +1,7 @@
//! Codex HTTP/SSE/WebSocket 上游 transport。
pub mod client;
pub mod diagnostics;
pub mod endpoints;
pub mod headers;
mod response_meta;
@@ -19,6 +20,7 @@ pub use self::{
CodexModelCatalogRequest, CodexRateLimitHeaderUpdates, CodexRequestContext,
CodexTurnStateUpdate,
},
diagnostics::CodexUpstreamDiagnostics,
endpoints::{
endpoint_request_path, endpoint_url, usage_endpoint_urls, CODEX_RESPONSES_COMPACT_PATH,
CODEX_RESPONSES_PATH, CODEX_USAGE_API_PATH, CODEX_USAGE_PATH, WHAM_USAGE_PATH,
@@ -6,6 +6,15 @@ use reqwest::header::{HeaderMap, SET_COOKIE};
use crate::upstream::protocol::responses::update_first_response_event_ms;
use super::diagnostics::CodexUpstreamDiagnostics;
pub(super) fn diagnostics(
status_code: Option<u16>,
headers: &HeaderMap,
) -> CodexUpstreamDiagnostics {
CodexUpstreamDiagnostics::from_headers(status_code, headers)
}
pub(super) fn turn_state(headers: &HeaderMap) -> Option<String> {
headers
.get("x-codex-turn-state")
+5 -1
View File
@@ -6,7 +6,8 @@ use crate::upstream::protocol::events::retry_after_seconds_from_body;
use super::{
client::{read_capped_error_body, retry_after_seconds, truncate_for_error},
endpoints::usage_endpoint_urls,
CodexBackendClient, CodexClientError, CodexClientResult, CodexRequestContext,
response_meta, CodexBackendClient, CodexClientError, CodexClientResult, CodexRequestContext,
CodexUpstreamDiagnostics,
};
impl CodexBackendClient {
@@ -18,6 +19,7 @@ impl CodexBackendClient {
let headers = self.usage_request_headers(context)?;
let response = self.client.get(endpoint).headers(headers).send().await?;
let status = response.status();
let diagnostics = response_meta::diagnostics(Some(status.as_u16()), response.headers());
let retry_after_seconds = retry_after_seconds(response.headers(), None);
if status == StatusCode::NOT_FOUND {
@@ -31,6 +33,7 @@ impl CodexBackendClient {
retry_after_seconds: retry_after_seconds
.or_else(|| retry_after_seconds_from_body(&body)),
body,
diagnostics,
set_cookie_headers: Vec::new(),
});
}
@@ -49,6 +52,7 @@ impl CodexBackendClient {
|| "usage endpoint is unavailable".to_string(),
|body| format!("invalid usage response: {}", truncate_for_error(&body)),
),
diagnostics: CodexUpstreamDiagnostics::with_status(StatusCode::BAD_GATEWAY.as_u16()),
set_cookie_headers: Vec::new(),
})
}
+79 -31
View File
@@ -2,7 +2,7 @@
use std::{
collections::VecDeque,
io,
fmt, io,
path::{Path, PathBuf},
pin::Pin,
sync::Arc,
@@ -37,12 +37,12 @@ use crate::upstream::protocol::websocket::{
OpeningAuditHeader, OpeningAuditSnapshot, WebSocketAuditArtifact,
};
use super::response_meta;
use super::websocket_pool::{
CodexWebSocketConnectionMetadata, CodexWebSocketPool, CodexWebSocketPoolKey,
PooledWebSocketConnection, WebSocketPoolAcquire, WebSocketPoolDecision,
};
use super::websocket_pump::{PumpKeepalive, PumpedWebSocket, RawWsStream};
use super::{diagnostics::CodexUpstreamDiagnostics, response_meta};
use uuid::Uuid;
const REDACTED_HEADER_VALUE: &str = "<redacted>";
@@ -120,6 +120,8 @@ pub struct CodexWebSocketExchange {
pub first_token_ms: Option<i64>,
/// WebSocket 连接池决策。
pub pool_decision: Option<WebSocketPoolDecision>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
/// Responses WebSocket live SSE exchange result.
@@ -138,6 +140,8 @@ pub struct CodexWebSocketStreamingExchange {
pub turn_state_update: CodexWebSocketTurnStateUpdate,
/// WebSocket 连接池决策。
pub pool_decision: Option<WebSocketPoolDecision>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
/// Responses WebSocket live SSE byte stream.
@@ -161,17 +165,8 @@ pub enum CodexWebSocketExchangeError {
#[error("invalid websocket SSE response: {0}")]
InvalidSse(#[from] SseError),
/// 上游 WebSocket 错误帧。
#[error("websocket upstream error {status_code}: {body}")]
Upstream {
/// HTTP-style upstream status code.
status_code: u16,
/// 推导出的重试秒数。
retry_after_seconds: Option<u64>,
/// 原始错误帧。
body: String,
/// 上游透传的 `set-cookie` 列表。
set_cookie_headers: Vec<String>,
},
#[error("{0}")]
Upstream(Box<CodexWebSocketUpstreamError>),
/// 上游返回 `response.incomplete`。
#[error("Incomplete response returned, reason: {reason}")]
IncompleteResponse {
@@ -210,6 +205,49 @@ pub enum CodexWebSocketExchangeError {
},
}
/// WebSocket 上游错误帧载荷。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CodexWebSocketUpstreamError {
/// HTTP-style upstream status code.
pub status_code: u16,
/// 推导出的重试秒数。
pub retry_after_seconds: Option<u64>,
/// 原始错误帧。
pub body: String,
/// 上游透传的 `set-cookie` 列表。
pub set_cookie_headers: Vec<String>,
/// 上游诊断元数据。
pub diagnostics: CodexUpstreamDiagnostics,
}
impl fmt::Display for CodexWebSocketUpstreamError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"websocket upstream error {}: {}",
self.status_code, self.body
)
}
}
impl CodexWebSocketExchangeError {
fn upstream(
status_code: u16,
retry_after_seconds: Option<u64>,
body: String,
set_cookie_headers: Vec<String>,
diagnostics: CodexUpstreamDiagnostics,
) -> Self {
Self::Upstream(Box::new(CodexWebSocketUpstreamError {
status_code,
retry_after_seconds,
body,
set_cookie_headers,
diagnostics,
}))
}
}
impl CodexWebSocketRequest {
/// 返回连接描述。
pub fn connection(&self) -> &CodexWebSocketConnection {
@@ -389,12 +427,13 @@ fn websocket_opening_error(response: &WsResponse<Option<Vec<u8>>>) -> CodexWebSo
.and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0)
.or_else(|| events::retry_after_seconds_from_body(&body));
CodexWebSocketExchangeError::Upstream {
CodexWebSocketExchangeError::upstream(
status_code,
retry_after_seconds,
body,
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
}
response_meta::set_cookie_headers(response.headers()),
response_meta::diagnostics(Some(status_code), response.headers()),
)
}
fn request_line_for_endpoint(endpoint: &str) -> String {
@@ -940,12 +979,13 @@ async fn collect_websocket_response(
if let Some(classified) = classify_websocket_error_frame(&raw) {
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
.or_else(|| events::retry_after_seconds_from_body(&raw));
return Err(CodexWebSocketExchangeError::Upstream {
status_code: classified.status_code,
return Err(CodexWebSocketExchangeError::upstream(
classified.status_code,
retry_after_seconds,
body: raw,
set_cookie_headers: Vec::new(),
});
raw,
Vec::new(),
CodexUpstreamDiagnostics::with_status(classified.status_code),
));
}
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
@@ -986,6 +1026,7 @@ async fn collect_websocket_response(
rate_limit_headers: metadata.rate_limit_headers.clone(),
first_token_ms,
pool_decision: None,
diagnostics: metadata.diagnostics.clone(),
};
return Ok((exchange, websocket, metadata));
}
@@ -1035,12 +1076,13 @@ async fn prefetch_first_content_stream_frames(
if let Some(classified) = classify_websocket_error_frame(&raw) {
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
.or_else(|| events::retry_after_seconds_from_body(&raw));
return Err(CodexWebSocketExchangeError::Upstream {
status_code: classified.status_code,
return Err(CodexWebSocketExchangeError::upstream(
classified.status_code,
retry_after_seconds,
body: raw,
set_cookie_headers: Vec::new(),
});
raw,
Vec::new(),
CodexUpstreamDiagnostics::with_status(classified.status_code),
));
}
if let Some(reason) = websocket_incomplete_response_reason(&raw) {
return Err(CodexWebSocketExchangeError::IncompleteResponse { reason });
@@ -1095,6 +1137,10 @@ fn websocket_connection_metadata(
turn_state: response_meta::turn_state(response.headers()),
set_cookie_headers: response_meta::set_cookie_headers(response.headers()),
rate_limit_headers: response_meta::rate_limit_headers(response.headers()),
diagnostics: response_meta::diagnostics(
Some(response.status().as_u16()),
response.headers(),
),
}
}
@@ -1145,6 +1191,7 @@ fn stream_websocket_response(
rate_limit_header_updates,
turn_state_update,
pool_decision: None,
diagnostics: response_metadata.diagnostics,
}
}
@@ -1212,12 +1259,13 @@ async fn forward_websocket_response_stream(
let retry_after_seconds = retry_after_seconds_from_wrapped_error_headers(&raw)
.or_else(|| events::retry_after_seconds_from_body(&raw));
let _ = tx
.send(Err(CodexWebSocketExchangeError::Upstream {
status_code: classified.status_code,
.send(Err(CodexWebSocketExchangeError::upstream(
classified.status_code,
retry_after_seconds,
body: raw,
set_cookie_headers: Vec::new(),
}))
raw,
Vec::new(),
CodexUpstreamDiagnostics::with_status(classified.status_code),
)))
.await;
return;
}
@@ -352,6 +352,7 @@ pub(crate) struct CodexWebSocketConnectionMetadata {
pub(crate) turn_state: Option<String>,
pub(crate) set_cookie_headers: Vec<String>,
pub(crate) rate_limit_headers: Vec<(String, String)>,
pub(crate) diagnostics: super::diagnostics::CodexUpstreamDiagnostics,
}
pub(crate) struct PooledWebSocketConnection {
+61 -2
View File
@@ -46,11 +46,38 @@ async fn admin_account_quota_should_send_usage_cookie() {
access_token: SecretString::new("access-quota-cookie".to_string().into()),
refresh_token: None,
access_token_expires_at: None,
status: AccountStatus::Active,
status: AccountStatus::QuotaExhausted,
added_at: None,
},
)
.await;
sqlx::query(
"insert into account_usage (
account_id,
request_count,
window_request_count,
window_input_tokens,
window_output_tokens,
window_cached_tokens,
window_started_at,
window_reset_at,
limit_window_seconds,
last_used_at
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
)
.bind("acct_quota_cookie")
.bind(9)
.bind(9)
.bind(9_000)
.bind(900)
.bind(90)
.bind("2027-01-01T00:00:00Z")
.bind("2027-01-01T05:00:00Z")
.bind(18_000)
.bind("2027-01-15T04:00:00Z")
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into usage_time_buckets (
bucket_start,
@@ -74,6 +101,29 @@ async fn admin_account_quota_should_send_usage_cookie() {
.execute(&pool)
.await
.unwrap();
sqlx::query(
"insert into usage_time_buckets (
bucket_start,
account_id,
model,
request_count,
input_tokens,
output_tokens,
cached_tokens,
updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?)",
)
.bind("2027-01-15T09:00:00+00:00")
.bind("acct_quota_cookie")
.bind("gpt-5")
.bind(1)
.bind(1_500)
.bind(500)
.bind(300)
.bind("2027-01-15T09:00:00Z")
.execute(&pool)
.await
.unwrap();
SqliteCookieStore::new(pool.clone())
.capture_set_cookie(
"acct_quota_cookie",
@@ -107,6 +157,15 @@ async fn admin_account_quota_should_send_usage_cookie() {
);
let body = response_json(response).await;
assert_eq!(body["data"]["planType"], "plus");
assert_eq!(body["data"]["account"]["status"], "active");
assert_eq!(body["data"]["account"]["usage"]["requestCount"], 1);
assert_eq!(body["data"]["account"]["usage"]["inputTokens"], 1_500);
assert_eq!(body["data"]["account"]["usage"]["outputTokens"], 500);
assert_eq!(body["data"]["account"]["usage"]["cachedTokens"], 300);
assert_eq!(
body["data"]["account"]["usage"]["models"][0]["model"],
"gpt-5"
);
assert_eq!(
body["data"]["quotaData"]["windows"][0]["labelDisplay"],
"5小时限额"
@@ -125,6 +184,6 @@ async fn admin_account_quota_should_send_usage_cookie() {
);
assert_eq!(
body["data"]["quotaData"]["windows"][1]["tokenUsageDisplay"],
"0"
"2K"
);
}
+5 -11
View File
@@ -14,7 +14,7 @@ use codex_proxy_rs::{
},
infra::{
database::connect_sqlite,
time::{china_datetime, china_day_start, china_hour, china_hour_start},
time::{china_datetime, china_day_start, china_hour},
},
proxy::dispatch::session_affinity::SqliteSessionAffinityStore,
runtime::{
@@ -222,20 +222,14 @@ async fn dashboard_latency_trend_should_use_first_token_latency() {
)
.await;
let now = Utc::now();
let current_hour_start = china_hour_start(now);
let sample_hour_start = if now - current_hour_start > Duration::minutes(31) {
current_hour_start
} else {
current_hour_start - Duration::hours(1)
};
let mut slow_completion =
usage_record_with_tokens(sample_hour_start + Duration::minutes(1), 10);
let mut slow_completion = usage_record_with_tokens(now, 10);
slow_completion.latency_ms = Some(10_000);
slow_completion.metadata["firstTokenMs"] = json!(100_i64);
let mut fast_completion =
usage_record_with_tokens(sample_hour_start + Duration::minutes(16), 10);
slow_completion.metadata["serviceTier"] = json!("default");
let mut fast_completion = usage_record_with_tokens(now, 10);
fast_completion.latency_ms = Some(1_000);
fast_completion.metadata["firstTokenMs"] = json!(500_i64);
fast_completion.metadata["serviceTier"] = json!("priority");
store.append(&slow_completion).await.unwrap();
store.append(&fast_completion).await.unwrap();
@@ -1,3 +1,4 @@
use chrono::{Duration, Utc};
use codex_proxy_rs::{
admin::monitoring::{
usage_record_model::{UsageRecord, UsageRecordLevel},
@@ -91,6 +92,22 @@ async fn usage_record_store_should_promote_diagnostic_metadata_fields() {
);
}
#[tokio::test]
async fn usage_record_store_should_not_promote_cf_ray_as_upstream_request_id() {
let (store, _dir) = usage_record_store("usage-records-cf-ray.sqlite").await;
let mut event = UsageRecord::new("request", UsageRecordLevel::Error, "upstream failed");
event.id = "usage_cf_ray".to_string();
event.metadata = json!({
"upstreamStatus": 403,
"cfRay": "ray-1"
});
store.append(&event).await.unwrap();
let loaded = store.get("usage_cf_ray").await.unwrap().unwrap();
assert_eq!(loaded.upstream_request_id, None);
}
#[tokio::test]
async fn usage_record_store_should_get_and_clear_events() {
let (store, _dir) = usage_record_store("usage-records-clear.sqlite").await;
@@ -108,6 +125,28 @@ async fn usage_record_store_should_get_and_clear_events() {
assert!(page.items.is_empty());
}
#[tokio::test]
async fn usage_record_store_should_trim_records_older_than_thirty_days() {
let (store, _dir) = usage_record_store("usage-records-retention.sqlite").await;
let now = Utc::now();
let mut expired = UsageRecord::new("v1.response", UsageRecordLevel::Info, "expired");
expired.id = "usage_expired".to_string();
expired.route = Some("/v1/responses".to_string());
expired.created_at = now - Duration::days(31);
store.append(&expired).await.unwrap();
let mut retained = UsageRecord::new("v1.response", UsageRecordLevel::Info, "retained");
retained.id = "usage_retained".to_string();
retained.route = Some("/v1/responses".to_string());
retained.created_at = now - Duration::days(29);
store.append(&retained).await.unwrap();
assert_eq!(store.trim_to_retention(now).await.unwrap(), 1);
assert!(store.get("usage_expired").await.unwrap().is_none());
assert!(store.get("usage_retained").await.unwrap().is_some());
}
async fn usage_record_store(db_name: &str) -> (SqliteUsageRecordStore, tempfile::TempDir) {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join(db_name);
+1
View File
@@ -1,4 +1,5 @@
pub(crate) mod format;
pub(crate) mod log_rotation;
pub(crate) mod paths;
pub(crate) mod storage_schema;
pub(crate) mod time;
+74
View File
@@ -0,0 +1,74 @@
use std::{ffi::OsString, fs, sync::Mutex};
use codex_proxy_rs::infra::paths::{ensure_data_dir, load_or_create_installation_id};
const INSTALLATION_ID_FILE_NAME: &str = "installation_id";
static ENV_LOCK: Mutex<()> = Mutex::new(());
#[test]
fn data_dir_should_use_xdg_data_home_directly() {
let _guard = ENV_LOCK.lock().unwrap();
let dir = tempfile::tempdir().unwrap();
let _xdg_data_home = EnvVarGuard::set("XDG_DATA_HOME", dir.path().as_os_str().to_owned());
let resolved = ensure_data_dir().unwrap();
assert_eq!(resolved, dir.path());
}
#[test]
fn load_or_create_installation_id_should_persist_under_data_dir_root() {
let dir = tempfile::tempdir().unwrap();
let installation_id =
load_or_create_installation_id(Some(dir.path())).expect("installation id should generate");
assert_eq!(
fs::read_to_string(dir.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
installation_id
);
}
#[test]
fn load_or_create_installation_id_should_seed_data_dir_from_codex_desktop_id() {
let _guard = ENV_LOCK.lock().unwrap();
let home = tempfile::tempdir().unwrap();
let data = tempfile::tempdir().unwrap();
let codex_path = home.path().join(".codex").join(INSTALLATION_ID_FILE_NAME);
let desktop_id = "018f8f6b-1d7b-7b7c-b9c8-8c9f4c6d0e1a";
fs::create_dir_all(codex_path.parent().unwrap()).unwrap();
fs::write(&codex_path, desktop_id).unwrap();
let _home = EnvVarGuard::set("HOME", home.path().as_os_str().to_owned());
let installation_id =
load_or_create_installation_id(Some(data.path())).expect("installation id should seed");
assert_eq!(installation_id, desktop_id);
assert_eq!(
fs::read_to_string(data.path().join(INSTALLATION_ID_FILE_NAME)).unwrap(),
desktop_id
);
}
struct EnvVarGuard {
key: &'static str,
previous: Option<OsString>,
}
impl EnvVarGuard {
fn set(key: &'static str, value: OsString) -> Self {
let previous = std::env::var_os(key);
std::env::set_var(key, value);
Self { key, previous }
}
}
impl Drop for EnvVarGuard {
fn drop(&mut self) {
match &self.previous {
Some(value) => std::env::set_var(self.key, value),
None => std::env::remove_var(self.key),
}
}
}
@@ -16,9 +16,7 @@ use codex_proxy_rs::{
admin::keys::service::SqliteClientKeyStore,
admin::monitoring::{
usage_record_model::UsageRecordLevel,
usage_record_store::{
SqliteUsageRecordStore, UsageRecordFilter, DEFAULT_USAGE_RECORD_CAPACITY,
},
usage_record_store::{SqliteUsageRecordStore, UsageRecordFilter},
},
config::schema::AppConfig,
http::router,
@@ -388,7 +386,6 @@ async fn test_app_with_account_pool_and_logging_capture_body(
config.logging.enabled = true;
let usage_record_options = UsageRecordOptions {
enabled: true,
capacity: DEFAULT_USAGE_RECORD_CAPACITY,
capture_body: true,
};
let state = test_app_state_with_pool_installation_id_and_usage_record_options(
@@ -554,6 +551,17 @@ async fn test_app_with_two_accounts(
(app, api_key, pool, dir)
}
async fn test_app_with_two_accounts_and_logging(
base_url: String,
) -> (axum::Router, String, SqlitePool, tempfile::TempDir) {
let (app, _state, api_key, pool, dir) =
test_app_with_two_accounts_and_state_config(base_url, |config| {
config.logging.enabled = true;
})
.await;
(app, api_key, pool, dir)
}
async fn test_app_with_two_accounts_and_state(
base_url: String,
) -> (
@@ -562,6 +570,19 @@ async fn test_app_with_two_accounts_and_state(
String,
SqlitePool,
tempfile::TempDir,
) {
test_app_with_two_accounts_and_state_config(base_url, |_| {}).await
}
async fn test_app_with_two_accounts_and_state_config(
base_url: String,
configure: impl FnOnce(&mut AppConfig),
) -> (
axum::Router,
AppState,
String,
SqlitePool,
tempfile::TempDir,
) {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("openai-responses-fallback.sqlite");
@@ -576,8 +597,10 @@ async fn test_app_with_two_accounts_and_state(
"chatgpt-secondary",
)
.await;
let mut config = test_config(url, base_url);
configure(&mut config);
let state = test_app_state_with_pool_and_installation_id(
&test_config(url, base_url),
&config,
pool.clone(),
TEST_INSTALLATION_ID.to_string(),
)
@@ -861,6 +861,82 @@ async fn responses_should_record_request_count_when_5xx_retries_are_exhausted()
assert_eq!(metadata["accountEmail"], "user@example.com");
}
#[tokio::test]
async fn responses_should_record_attempt_trace_after_retrying_another_account() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.and(header("authorization", "Bearer access-primary"))
.respond_with(ResponseTemplate::new(429).set_body_json(json!({
"error": {"message": "primary rate limited"}
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/codex/responses"))
.and(header("authorization", "Bearer access-secondary"))
.respond_with(ResponseTemplate::new(500).set_body_json(json!({
"error": {"message": "secondary failed"}
})))
.expect(3)
.mount(&server)
.await;
let secret_input = "do not persist this input";
let (app, api_key, pool, _dir) = test_app_with_two_accounts_and_logging(server.uri()).await;
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri("/v1/responses")
.header("authorization", format!("Bearer {api_key}"))
.header("content-type", "application/json")
.header("x-request-id", "req_attempt_trace_retry")
.body(Body::from(
json!({
"model": "gpt-5.5",
"input": secret_input,
"stream": false,
"use_websocket": false
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
let event = latest_response_usage_record(&pool).await;
let metadata: Value = serde_json::from_str(&event.metadata_json).unwrap();
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
assert_eq!(event.level, "error");
assert_eq!(event.request_id.as_deref(), Some("req_attempt_trace_retry"));
assert_eq!(event.account_id.as_deref(), Some("acct_secondary"));
assert_eq!(event.status_code, Some(500));
assert_eq!(metadata["eventStatusCode"], 500);
assert_eq!(metadata["clientStatusCode"], 502);
assert_eq!(metadata["upstreamStatusCode"], 500);
assert_eq!(metadata["attemptIndex"], 1);
assert_eq!(metadata["attemptAccountId"], "acct_secondary");
assert_eq!(metadata["attemptCount"], 2);
assert_eq!(metadata["attempts"][0]["index"], 0);
assert_eq!(metadata["attempts"][0]["accountId"], "acct_primary");
assert_eq!(metadata["attempts"][1]["index"], 1);
assert_eq!(metadata["attempts"][1]["accountId"], "acct_secondary");
assert_eq!(metadata["requestSummary"]["inputType"], "string");
assert!(metadata["requestSummary"]["inputItemsCount"].is_null());
assert_eq!(metadata["requestSummary"]["transport"], "http_sse");
assert_eq!(
metadata["requestSummary"]["localTransport"]["useWebsocket"],
false
);
assert!(
!metadata.to_string().contains(secret_input),
"request summary must not persist input content: {metadata:?}",
);
}
#[tokio::test]
async fn responses_should_record_session_affinity_for_completed_response() {
let server = MockServer::start().await;
@@ -352,6 +352,14 @@ async fn runtime_account_pool_should_persist_quota_window_reset_when_cooldown_ex
);
assert!(persisted_reset_at.is_some_and(|reset_at| reset_at > now));
assert_eq!(stored_limit_window_seconds, Some(60));
let usage_record_count: i64 =
sqlx::query_scalar("select count(*) from usage_records where account_id = ?")
.bind("acct_quota_reset")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(usage_record_count, 0);
}
#[test]
@@ -0,0 +1,26 @@
use codex_proxy_rs::upstream::transport::CodexUpstreamDiagnostics;
use reqwest::header::{HeaderMap, HeaderValue};
#[test]
fn diagnostics_should_extract_request_id_and_trace_headers() {
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("req_1"));
headers.insert("cf-ray", HeaderValue::from_static("ray_1"));
let diagnostics = CodexUpstreamDiagnostics::from_headers(Some(429), &headers);
assert_eq!(diagnostics.status_code, Some(429));
assert_eq!(diagnostics.request_id.as_deref(), Some("req_1"));
assert_eq!(diagnostics.cf_ray(), Some("ray_1"));
}
#[test]
fn diagnostics_should_not_treat_cf_ray_as_request_id() {
let mut headers = HeaderMap::new();
headers.insert("cf-ray", HeaderValue::from_static("ray_1"));
let diagnostics = CodexUpstreamDiagnostics::from_headers(Some(403), &headers);
assert_eq!(diagnostics.request_id, None);
assert_eq!(diagnostics.cf_ray(), Some("ray_1"));
}
+1
View File
@@ -37,6 +37,7 @@ use tokio_tungstenite::{
use crate::support::assertions::assert_substrings_appear_in_order;
pub(crate) mod client;
pub(crate) mod diagnostics;
pub(crate) mod headers;
pub(crate) mod http_client;
pub(crate) mod websocket;
+15 -36
View File
@@ -322,18 +322,12 @@ async fn websocket_execute_response_create_request_should_surface_response_faile
.expect_err("response.failed should be surfaced as upstream error");
server.await.unwrap();
let CodexWebSocketExchangeError::Upstream {
status_code,
retry_after_seconds,
body,
..
} = error
else {
let CodexWebSocketExchangeError::Upstream(error) = error else {
panic!("expected upstream websocket error");
};
assert_eq!(status_code, 429);
assert_eq!(retry_after_seconds, Some(12));
assert!(body.contains("rate_limit_exceeded"));
assert_eq!(error.status_code, 429);
assert_eq!(error.retry_after_seconds, Some(12));
assert!(error.body.contains("rate_limit_exceeded"));
}
#[tokio::test]
@@ -405,18 +399,12 @@ async fn websocket_execute_response_create_request_should_preserve_opening_error
.expect_err("failed opening should surface upstream status");
server.await.unwrap();
let CodexWebSocketExchangeError::Upstream {
status_code,
retry_after_seconds,
body,
..
} = error
else {
let CodexWebSocketExchangeError::Upstream(error) = error else {
panic!("expected upstream opening error");
};
assert_eq!(status_code, 429);
assert_eq!(retry_after_seconds, Some(33));
assert!(body.contains("rate limited"));
assert_eq!(error.status_code, 429);
assert_eq!(error.retry_after_seconds, Some(33));
assert!(error.body.contains("rate limited"));
}
#[tokio::test]
@@ -482,18 +470,12 @@ async fn websocket_execute_response_create_request_should_surface_wrapped_error_
.expect_err("wrapped error should surface as upstream error");
server.await.unwrap();
let CodexWebSocketExchangeError::Upstream {
status_code,
retry_after_seconds,
body,
..
} = error
else {
let CodexWebSocketExchangeError::Upstream(error) = error else {
panic!("expected wrapped upstream error");
};
assert_eq!(status_code, 409);
assert_eq!(retry_after_seconds, Some(17));
assert!(body.contains("wrapped conflict"));
assert_eq!(error.status_code, 409);
assert_eq!(error.retry_after_seconds, Some(17));
assert!(error.body.contains("wrapped conflict"));
}
#[tokio::test]
@@ -530,14 +512,11 @@ async fn websocket_execute_response_create_request_should_surface_connection_lim
.expect_err("connection limit should surface as upstream error");
server.await.unwrap();
let CodexWebSocketExchangeError::Upstream {
status_code, body, ..
} = error
else {
let CodexWebSocketExchangeError::Upstream(error) = error else {
panic!("expected connection limit upstream error");
};
assert_eq!(status_code, 503);
assert!(body.contains("websocket_connection_limit_reached"));
assert_eq!(error.status_code, 503);
assert!(error.body.contains("websocket_connection_limit_reached"));
}
#[tokio::test]
@@ -20,6 +20,18 @@ import { useIdSet } from '@/composables/useIdSet'
import { withMinimumDuration } from '@/utils/async'
type TokenImportAccount = { token: string } | { refreshToken: string }
type AccountRow = {
id: string
status: string
planType?: string | null
quota: any
usage: any
[key: string]: any
}
type AccountQuotaRefreshResult = {
account: AccountRow
}
export function useAccountMutations(options: {
page: Ref<number>
@@ -30,7 +42,7 @@ export function useAccountMutations(options: {
}) {
const { downloadJson } = useDownload()
const loading = ref(true)
const accounts = ref<any[]>([])
const accounts = ref<AccountRow[]>([])
const accountSummary = ref({
total: 0,
active: 0,
@@ -341,18 +353,8 @@ export function useAccountMutations(options: {
await refreshingQuotaAccounts.run(accountId, async () => {
try {
await withMinimumDuration(async () => {
const result = await getAccountQuota({ id: accountId })
if (result?.quotaData) {
accounts.value = accounts.value.map((account) =>
account.id === accountId
? {
...account,
quota: result.quotaData,
planType: result.planType ?? account.planType,
}
: account,
)
}
const result = (await getAccountQuota({ id: accountId })) as AccountQuotaRefreshResult
mergeAccountQuotaRefresh(accountId, result)
})
} catch (error) {
console.error('Failed to refresh account quota:', error)
@@ -360,6 +362,12 @@ export function useAccountMutations(options: {
})
}
function mergeAccountQuotaRefresh(accountId: string, result: AccountQuotaRefreshResult) {
accounts.value = accounts.value.map((account) =>
account.id === accountId ? { ...account, ...result.account } : account,
)
}
async function handleToggleSchedule(account: any) {
const nextStatus = account.status === 'disabled' ? 'active' : 'disabled'
await updatingStatusAccounts.run(account.id, async () => {
+1 -2
View File
@@ -74,7 +74,7 @@ export const usageRecordColumns = [
{
key: 'clientIp',
label: 'IP',
width: '200px',
width: '240px',
ellipsis: false,
},
{
@@ -111,7 +111,6 @@ export const usageTimeRangeOptions = [
{ label: '今天', value: 'today' },
{ label: '最近 7 天', value: '7d' },
{ label: '最近 30 天', value: '30d' },
{ label: '全部时间', value: 'all' },
]
export const levelLabels: Record<string, string> = {
-4
View File
@@ -47,10 +47,6 @@ const timeRangeParams = computed<Record<string, string>>(() => {
}
}
if (timeRange.value === 'all') {
return {} as Record<string, string>
}
return {
startTime: now.subtract(6, 'day').startOf('day').toISOString(),
endTime: now.toISOString(),
+1 -1
View File
@@ -1 +1 @@
version: "1.0.4"
version: "1.0.6"
+119
View File
@@ -0,0 +1,119 @@
---
name: engineering-standards
description: 面向长期维护的工程规范工作流。当用户要求代码审计、问题修复、重构、实现方案、生产问题追溯,或强调代码质量、干净架构、拒绝垃圾代码、禁止兼容写法、先审计再改代码、参考项目对比时使用。
---
# 工程规范
使用这个 skill 时,先证明当前系统真实行为,再决定是否需要改代码。核心规则:先画出现有证据链,再谈方案。
## 硬门禁
提出方案或动代码前,必须先给出三段内容:
1. 现有机制:入口、调用链、数据变化、数据读取、接口/界面展示路径。
2. 缺口证据:具体缺失或断裂的环节,并给出源码位置。
3. 最小改动点:在不绕开现有设计的前提下,能修复问题的最小所有权边界。
在现有链路被追完并证明不足之前,不要引入新的机制、分支或旁路来绕开问题。
## 审计流程
以当前仓库的实时源码为唯一事实来源。只有在当前项目链路已经清楚后,才使用记忆或参考项目。
1. 先确认用户讨论的数据或行为是什么。
2. 追踪它在哪里产生、转换、持久化、加载和展示。
3. 同时查主动链路和被动链路。运行时行为要分开检查:触发条件、执行路径、持久化、读取和展示。
4. 设计前先分类问题所在层级:
- 展示层
- 接口契约
- 持久化层
- 运行时行为
- 外部依赖
- 异步或周期性行为
5. 说明哪些证据支持该分类,哪些仍未证明。
6. 排查中发现代码异味、疑似死代码或结构性隐患时,先记录证据,不打断主流程;主问题排查完毕后再汇总报告,并询问是否纳入修复范围。
如果有 `codegraph`,先用它快速定位,再回到源码和测试验证。不要把 codegraph 摘要、记忆、参考项目当成强于当前源码的证据。
## 参考项目
参考项目只能在当前项目证据清楚后,用来理解可能的模式。
除非同时满足以下条件,否则不要把参考项目机制搬进当前项目:
1. 当前项目确实缺少所需行为。
2. 当前架构存在匹配的所有权边界。
3. 新设计不会绕开或复制现有链路。
4. 当前任务范围允许进入实现。
参考项目和当前源码冲突时,以当前源码为准。
## 干净实现标准
优先在现有所有权边界内做最直接、最干净的修复。
- 除非用户明确要求,不要添加兼容代码。
- 不要用补丁式分支或重复机制掩盖边界不清。
- 不要用大范围重构掩盖不确定性。
- 不要为解决局部问题改动无关模块。
- 只有在确实降低复杂度或符合现有本地模式时,才添加抽象。
- 必要时可以重构,但重构必须收敛职责、减少分支、让主链路更清晰。
- 不要用外层绕路掩盖内层问题。
- 除非证明现有设计不足,不要引入新的执行路径替代原链路。
如果方案会改变语义,动代码前必须明确说明。
## 用户质量要求
默认用户要求严格工程纪律:
- 简单问题先给结论。
- 审计必须给具体源码证据。
- 不要做推测性断言。
- 不要虚构任何事实、数据来源或系统行为。
- 新项目/新链路不要塞兼容垃圾。
- 仓库约定测试在测试目录时,不要把测试写进源码模块。
- 使用代码索引工具辅助理解时,改代码后必须同步索引。
当任务范围被限定为审计、文档、评审或方案阶段时,只交付对应产物,未经要求不实现。
## 动代码前检查
编辑文件前,确认:
1. 当前链路已经端到端追完。
2. 问题已经按证据分类。
3. 现有机制不能满足需求,而不是没查到。
4. 改动点是最小的干净边界。
5. 测试会通过合适的公开接口或集成测试面覆盖行为。
任何一项缺失,都继续调查,不要实现。
## 收尾检查
结束前确认:
1. 运行格式化和相关测试。
2. 如果该语言有既定 lint 命令,运行 lint。
3. 确认没有在应该放测试目录的项目里添加源码内测试。
4. 确认没有用参考项目模式替代当前项目证据。
5. 确认没有未经明确批准引入额外分支或旁路设计。
6. 需要时同步 codegraph。
7. 总结改了什么、验证了什么、明确没有改什么。
## 必须避免的反模式
不要重复这些错误:
1. 只看一个局部触发点,就断言完整系统行为。
2. 没有追完已有链路,就新增另一套处理机制。
3. 没有证明现有机制失败,就用旁路设计绕开问题。
4. 把参考项目的做法直接搬进当前项目,跳过当前源码证据。
5. 用补丁式分支或重复机制掩盖所有权边界不清。
正确做法:
1. 先追当前项目已经存在的真实链路。
2. 证明数据在产生、转换、持久化、读取和展示中的断点。
3. 以断点为依据选择最干净的修复边界;能局部修清楚就局部修,原链路本身不干净时就重构收敛。
@@ -0,0 +1,4 @@
interface:
display_name: "工程规范"
short_description: "长期维护的工程规范:证据优先、范围纪律、干净实现与审计工作流。"
default_prompt: "使用 $engineering-standards 按工程规范先审计当前代码链路,再提出或实现干净修复。"