mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-14 08:06:40 +00:00
Persist provider account name, email, and authentication kind at write time so request, attempt, and error views survive account deletion; add stable diagnostic keys with display-name resolution (account email/name, API key name); count only errored requests in failure diagnostics. Co-authored-by: Codex <noreply@openai.com>
1114 lines
42 KiB
Rust
1114 lines
42 KiB
Rust
//! 单行 `model_requests` 生命周期与最终 usage/cost 的 PostgreSQL owner。
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Utc};
|
|
use serde_json::Value;
|
|
use sqlx::PgPool;
|
|
use uuid::Uuid;
|
|
|
|
use gateway_core::accounting::{CostSource as CoreCostSource, Usage as CoreUsage};
|
|
use gateway_core::engine::{
|
|
AttemptRecord as CoreAttemptRecord, ExecutionStore, IntermediateFailure,
|
|
ModelRequestFinalization as CoreModelRequestFinalization, ModelRequestId,
|
|
NewModelRequest as CoreNewModelRequest, ProbeFailure, RecoveryReport as CoreRecoveryReport,
|
|
UpstreamSendState as CoreUpstreamSendState,
|
|
};
|
|
use gateway_core::error::{StoreError as CoreStoreError, StoreErrorKind as CoreStoreErrorKind};
|
|
|
|
use crate::{
|
|
ConflictKind, DecimalAmount, StoreError, StoreResult, postgres_unavailable, require_nonempty,
|
|
};
|
|
|
|
const ENTITY: &str = "model request";
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum UpstreamSendState {
|
|
NotSent,
|
|
Sent,
|
|
Ambiguous,
|
|
}
|
|
|
|
impl UpstreamSendState {
|
|
#[must_use]
|
|
pub const fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::NotSent => "not_sent",
|
|
Self::Sent => "sent",
|
|
Self::Ambiguous => "ambiguous",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum ModelRequestOutcome {
|
|
Succeeded,
|
|
Failed,
|
|
Cancelled,
|
|
Incomplete,
|
|
}
|
|
|
|
impl ModelRequestOutcome {
|
|
#[must_use]
|
|
pub const fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Succeeded => "succeeded",
|
|
Self::Failed => "failed",
|
|
Self::Cancelled => "cancelled",
|
|
Self::Incomplete => "incomplete",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum CostSource {
|
|
ProviderReported,
|
|
Calculated,
|
|
Unavailable,
|
|
}
|
|
|
|
impl CostSource {
|
|
#[must_use]
|
|
pub const fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::ProviderReported => "provider_reported",
|
|
Self::Calculated => "calculated",
|
|
Self::Unavailable => "unavailable",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct NewModelRequest {
|
|
pub id: String,
|
|
pub client_api_key_id: Option<String>,
|
|
pub client_api_key_ref: String,
|
|
pub config_revision: u64,
|
|
pub protocol: String,
|
|
pub operation: String,
|
|
pub endpoint: String,
|
|
pub client_transport: String,
|
|
pub requested_model_id: String,
|
|
pub client_ip: Option<String>,
|
|
pub user_agent: Option<String>,
|
|
pub reasoning_effort: Option<String>,
|
|
pub reasoning_preset: Option<String>,
|
|
pub request_kind: Option<String>,
|
|
pub subagent_kind: Option<String>,
|
|
pub compact: bool,
|
|
pub image_generation_requested: bool,
|
|
pub started_at: DateTime<Utc>,
|
|
pub deadline_at: DateTime<Utc>,
|
|
}
|
|
|
|
impl NewModelRequest {
|
|
pub fn validate(&self) -> StoreResult<()> {
|
|
require_nonempty(ENTITY, "id", &self.id)?;
|
|
require_nonempty(ENTITY, "client_api_key_ref", &self.client_api_key_ref)?;
|
|
require_nonempty(ENTITY, "protocol", &self.protocol)?;
|
|
require_nonempty(ENTITY, "operation", &self.operation)?;
|
|
require_nonempty(ENTITY, "endpoint", &self.endpoint)?;
|
|
require_nonempty(ENTITY, "client_transport", &self.client_transport)?;
|
|
require_nonempty(ENTITY, "requested_model_id", &self.requested_model_id)?;
|
|
if self.config_revision == 0 || self.started_at > self.deadline_at {
|
|
return Err(invalid(
|
|
"revision and deadline violate the frozen constraints",
|
|
));
|
|
}
|
|
if self
|
|
.client_api_key_id
|
|
.as_ref()
|
|
.is_some_and(|id| id != &self.client_api_key_ref)
|
|
{
|
|
return Err(invalid("live client key ID must equal its historical ref"));
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct ModelRequestAttemptStart {
|
|
pub model_request_id: String,
|
|
pub attempt_count: u32,
|
|
pub provider_kind: String,
|
|
pub provider_account_id: Option<String>,
|
|
pub provider_account_ref: Option<String>,
|
|
pub upstream_model_id: String,
|
|
pub upstream_transport: String,
|
|
pub http_version: Option<String>,
|
|
}
|
|
|
|
impl ModelRequestAttemptStart {
|
|
pub fn validate(&self) -> StoreResult<()> {
|
|
for (field, value) in [
|
|
("model_request_id", self.model_request_id.as_str()),
|
|
("provider_kind", self.provider_kind.as_str()),
|
|
("upstream_model_id", self.upstream_model_id.as_str()),
|
|
("upstream_transport", self.upstream_transport.as_str()),
|
|
] {
|
|
require_nonempty(ENTITY, field, value)?;
|
|
}
|
|
if self.attempt_count == 0 {
|
|
return Err(invalid("attempt_count must be positive"));
|
|
}
|
|
if self
|
|
.provider_account_id
|
|
.as_ref()
|
|
.is_some_and(|id| Some(id) != self.provider_account_ref.as_ref())
|
|
{
|
|
return Err(invalid(
|
|
"live provider account ID must equal its historical ref",
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct ModelRequestUsage {
|
|
pub input_tokens: Option<u64>,
|
|
pub output_tokens: Option<u64>,
|
|
pub cached_tokens: Option<u64>,
|
|
pub cache_write_tokens: Option<u64>,
|
|
pub reasoning_tokens: Option<u64>,
|
|
pub image_input_tokens: Option<u64>,
|
|
pub image_output_tokens: Option<u64>,
|
|
pub total_tokens: Option<u64>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
|
pub struct ModelRequestTimings {
|
|
pub transport_decision_wait_ms: Option<u64>,
|
|
pub connect_ms: Option<u64>,
|
|
pub headers_ms: Option<u64>,
|
|
pub first_event_ms: Option<u64>,
|
|
pub first_reasoning_ms: Option<u64>,
|
|
pub first_text_ms: Option<u64>,
|
|
pub first_token_ms: Option<u64>,
|
|
pub provider_processing_ms: Option<u64>,
|
|
pub latency_ms: Option<u64>,
|
|
}
|
|
|
|
impl ModelRequestTimings {
|
|
fn validate(&self) -> StoreResult<()> {
|
|
if let Some(total) = self.latency_ms {
|
|
let phases = [
|
|
self.transport_decision_wait_ms,
|
|
self.connect_ms,
|
|
self.headers_ms,
|
|
self.first_event_ms,
|
|
self.first_reasoning_ms,
|
|
self.first_text_ms,
|
|
self.first_token_ms,
|
|
self.provider_processing_ms,
|
|
];
|
|
if phases.into_iter().flatten().any(|phase| phase > total) {
|
|
return Err(invalid("timing phase exceeds total latency"));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct ModelRequestFinalization {
|
|
pub model_request_id: String,
|
|
pub outcome: ModelRequestOutcome,
|
|
pub upstream_send_state: UpstreamSendState,
|
|
pub attempt_count: u32,
|
|
pub downstream_committed_at: Option<DateTime<Utc>>,
|
|
pub client_status_code: Option<u16>,
|
|
pub upstream_status_code: Option<u16>,
|
|
pub client_response_id: Option<String>,
|
|
pub upstream_request_id: Option<String>,
|
|
pub upstream_response_id: Option<String>,
|
|
pub upstream_transport: Option<String>,
|
|
pub http_version: Option<String>,
|
|
pub websocket_pool: Option<String>,
|
|
pub service_tier: Option<String>,
|
|
pub provider_metadata_json: Option<Value>,
|
|
pub error_kind: Option<String>,
|
|
pub provider_error_code: Option<String>,
|
|
pub error_message: Option<String>,
|
|
pub retry_after_ms: Option<u64>,
|
|
pub usage: ModelRequestUsage,
|
|
pub image_generation_succeeded: Option<bool>,
|
|
pub cost_source: CostSource,
|
|
pub cost_amount: Option<DecimalAmount>,
|
|
pub cost_currency: Option<String>,
|
|
pub timings: ModelRequestTimings,
|
|
pub completed_at: DateTime<Utc>,
|
|
}
|
|
|
|
impl ModelRequestFinalization {
|
|
pub fn validate(&self) -> StoreResult<()> {
|
|
require_nonempty(ENTITY, "model_request_id", &self.model_request_id)?;
|
|
for status in [self.client_status_code, self.upstream_status_code]
|
|
.into_iter()
|
|
.flatten()
|
|
{
|
|
if !(100..=599).contains(&status) {
|
|
return Err(invalid("HTTP status must be between 100 and 599"));
|
|
}
|
|
}
|
|
let cost_is_absent = self.cost_amount.is_none() && self.cost_currency.is_none();
|
|
let cost_is_complete = self.cost_amount.is_some() && self.cost_currency.is_some();
|
|
if (self.cost_source == CostSource::Unavailable && !cost_is_absent)
|
|
|| (self.cost_source != CostSource::Unavailable && !cost_is_complete)
|
|
{
|
|
return Err(invalid("cost source, amount, and currency do not agree"));
|
|
}
|
|
if let Some(currency) = &self.cost_currency
|
|
&& (currency.len() != 3 || !currency.bytes().all(|byte| byte.is_ascii_uppercase()))
|
|
{
|
|
return Err(invalid("cost currency must be three uppercase characters"));
|
|
}
|
|
for (field, value) in [
|
|
("upstream_transport", self.upstream_transport.as_deref()),
|
|
("http_version", self.http_version.as_deref()),
|
|
] {
|
|
if let Some(value) = value {
|
|
require_nonempty(ENTITY, field, value)?;
|
|
}
|
|
}
|
|
if self
|
|
.websocket_pool
|
|
.as_deref()
|
|
.is_some_and(|kind| !matches!(kind, "new" | "reuse"))
|
|
{
|
|
return Err(invalid("websocket pool must be new or reuse"));
|
|
}
|
|
if let Some(service_tier) = self.service_tier.as_deref() {
|
|
require_nonempty(ENTITY, "service_tier", service_tier)?;
|
|
if service_tier.len() > 64 || service_tier.chars().any(char::is_control) {
|
|
return Err(invalid("service tier is invalid"));
|
|
}
|
|
}
|
|
if self
|
|
.provider_metadata_json
|
|
.as_ref()
|
|
.is_some_and(|metadata| !metadata.is_object())
|
|
{
|
|
return Err(invalid("provider observation must be a JSON object"));
|
|
}
|
|
self.timings.validate()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
|
pub struct ModelRequestRecoveryReport {
|
|
pub requests: u64,
|
|
}
|
|
|
|
#[async_trait]
|
|
pub trait ModelRequestRepository: Send + Sync {
|
|
async fn insert_model_request(&self, request: NewModelRequest) -> StoreResult<()>;
|
|
/// 请求行与首次 attempt 列一次插入;`attempt.attempt_count` 必须为 1。
|
|
async fn insert_model_request_with_first_attempt(
|
|
&self,
|
|
request: NewModelRequest,
|
|
attempt: ModelRequestAttemptStart,
|
|
) -> StoreResult<()>;
|
|
async fn begin_model_request_attempt(
|
|
&self,
|
|
attempt: ModelRequestAttemptStart,
|
|
) -> StoreResult<u32>;
|
|
async fn mark_upstream_send_state(
|
|
&self,
|
|
model_request_id: &str,
|
|
state: UpstreamSendState,
|
|
) -> StoreResult<bool>;
|
|
async fn mark_downstream_committed(
|
|
&self,
|
|
model_request_id: &str,
|
|
committed_at: DateTime<Utc>,
|
|
client_status_code: Option<u16>,
|
|
) -> StoreResult<bool>;
|
|
async fn record_client_status_code(
|
|
&self,
|
|
model_request_id: &str,
|
|
client_status_code: u16,
|
|
) -> StoreResult<bool>;
|
|
async fn finalize_model_request(
|
|
&self,
|
|
finalization: ModelRequestFinalization,
|
|
) -> StoreResult<bool>;
|
|
async fn recover_expired_model_requests(
|
|
&self,
|
|
now: DateTime<Utc>,
|
|
) -> StoreResult<ModelRequestRecoveryReport>;
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct PgExecutionStore {
|
|
pool: PgPool,
|
|
}
|
|
|
|
impl PgExecutionStore {
|
|
#[must_use]
|
|
pub const fn new(pool: PgPool) -> Self {
|
|
Self { pool }
|
|
}
|
|
|
|
#[must_use]
|
|
pub const fn pool(&self) -> &PgPool {
|
|
&self.pool
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl ModelRequestRepository for PgExecutionStore {
|
|
async fn insert_model_request(&self, request: NewModelRequest) -> StoreResult<()> {
|
|
request.validate()?;
|
|
sqlx::query(
|
|
"insert into model_requests (
|
|
id, client_api_key_id, client_api_key_ref, config_revision, protocol,
|
|
operation, endpoint, client_transport, requested_model_id,
|
|
client_ip, user_agent, reasoning_effort,
|
|
reasoning_preset, request_kind, subagent_kind, compact,
|
|
image_generation_requested, started_at, deadline_at
|
|
) values (
|
|
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
|
|
$13, $14, $15, $16, $17, $18, $19
|
|
)",
|
|
)
|
|
.bind(request.id)
|
|
.bind(request.client_api_key_id)
|
|
.bind(request.client_api_key_ref)
|
|
.bind(to_i64(request.config_revision, "config_revision")?)
|
|
.bind(request.protocol)
|
|
.bind(request.operation)
|
|
.bind(request.endpoint)
|
|
.bind(request.client_transport)
|
|
.bind(request.requested_model_id)
|
|
.bind(request.client_ip)
|
|
.bind(request.user_agent)
|
|
.bind(request.reasoning_effort)
|
|
.bind(request.reasoning_preset)
|
|
.bind(request.request_kind)
|
|
.bind(request.subagent_kind)
|
|
.bind(request.compact)
|
|
.bind(request.image_generation_requested)
|
|
.bind(request.started_at)
|
|
.bind(request.deadline_at)
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("insert model request"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn insert_model_request_with_first_attempt(
|
|
&self,
|
|
request: NewModelRequest,
|
|
attempt: ModelRequestAttemptStart,
|
|
) -> StoreResult<()> {
|
|
request.validate()?;
|
|
attempt.validate()?;
|
|
if attempt.attempt_count != 1 || attempt.model_request_id != request.id {
|
|
return Err(invalid("first attempt must target the inserted request"));
|
|
}
|
|
sqlx::query(
|
|
"insert into model_requests (
|
|
id, client_api_key_id, client_api_key_ref, config_revision, protocol,
|
|
operation, endpoint, client_transport, requested_model_id,
|
|
client_ip, user_agent, reasoning_effort,
|
|
reasoning_preset, request_kind, subagent_kind, compact,
|
|
image_generation_requested, started_at, deadline_at,
|
|
provider_kind, provider_account_id, provider_account_ref,
|
|
provider_account_name_snapshot, provider_account_email_snapshot,
|
|
provider_account_authentication_kind_snapshot,
|
|
upstream_model_id, upstream_transport, http_version,
|
|
attempt_count, upstream_send_state
|
|
) values (
|
|
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
|
|
$13, $14, $15, $16, $17, $18, $19,
|
|
$20, $21, $22,
|
|
(select name from provider_accounts where id = $21),
|
|
(select email from provider_accounts where id = $21),
|
|
(select authentication_kind from provider_accounts where id = $21),
|
|
$23, $24, $25, 1, 'not_sent'
|
|
)",
|
|
)
|
|
.bind(request.id)
|
|
.bind(request.client_api_key_id)
|
|
.bind(request.client_api_key_ref)
|
|
.bind(to_i64(request.config_revision, "config_revision")?)
|
|
.bind(request.protocol)
|
|
.bind(request.operation)
|
|
.bind(request.endpoint)
|
|
.bind(request.client_transport)
|
|
.bind(request.requested_model_id)
|
|
.bind(request.client_ip)
|
|
.bind(request.user_agent)
|
|
.bind(request.reasoning_effort)
|
|
.bind(request.reasoning_preset)
|
|
.bind(request.request_kind)
|
|
.bind(request.subagent_kind)
|
|
.bind(request.compact)
|
|
.bind(request.image_generation_requested)
|
|
.bind(request.started_at)
|
|
.bind(request.deadline_at)
|
|
.bind(attempt.provider_kind)
|
|
.bind(attempt.provider_account_id)
|
|
.bind(attempt.provider_account_ref)
|
|
.bind(attempt.upstream_model_id)
|
|
.bind(attempt.upstream_transport)
|
|
.bind(attempt.http_version)
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("insert model request with first attempt"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn begin_model_request_attempt(
|
|
&self,
|
|
attempt: ModelRequestAttemptStart,
|
|
) -> StoreResult<u32> {
|
|
attempt.validate()?;
|
|
// upstream_send_state 是请求级单调水位(sent > ambiguous > not_sent),
|
|
// 由 mark/finalize 抬升;开启新 attempt 不得把已持久化的 sent 重置回
|
|
// not_sent——崩溃恢复的终态写回会原样继承本列。
|
|
let count = sqlx::query_scalar::<_, i32>(
|
|
"update model_requests
|
|
set provider_kind = $2,
|
|
provider_account_id = $3,
|
|
provider_account_ref = $4,
|
|
provider_account_name_snapshot = (
|
|
select name from provider_accounts where id = $3
|
|
),
|
|
provider_account_email_snapshot = (
|
|
select email from provider_accounts where id = $3
|
|
),
|
|
provider_account_authentication_kind_snapshot = (
|
|
select authentication_kind from provider_accounts where id = $3
|
|
),
|
|
upstream_model_id = $5,
|
|
upstream_transport = $6,
|
|
http_version = $7,
|
|
attempt_count = $8
|
|
where id = $1 and outcome = 'running' and downstream_committed_at is null
|
|
and $8 = attempt_count + 1
|
|
returning attempt_count",
|
|
)
|
|
.bind(&attempt.model_request_id)
|
|
.bind(attempt.provider_kind)
|
|
.bind(attempt.provider_account_id)
|
|
.bind(attempt.provider_account_ref)
|
|
.bind(attempt.upstream_model_id)
|
|
.bind(attempt.upstream_transport)
|
|
.bind(attempt.http_version)
|
|
.bind(
|
|
i32::try_from(attempt.attempt_count)
|
|
.map_err(|_| invalid("attempt_count is too large"))?,
|
|
)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("begin model request attempt"))?
|
|
.ok_or(StoreError::Conflict {
|
|
entity: ENTITY,
|
|
id: attempt.model_request_id,
|
|
kind: ConflictKind::DownstreamAlreadyCommitted,
|
|
})?;
|
|
u32::try_from(count).map_err(|_| invalid("attempt_count is invalid"))
|
|
}
|
|
|
|
async fn mark_upstream_send_state(
|
|
&self,
|
|
model_request_id: &str,
|
|
state: UpstreamSendState,
|
|
) -> StoreResult<bool> {
|
|
require_nonempty(ENTITY, "id", model_request_id)?;
|
|
let result = sqlx::query(
|
|
"update model_requests set upstream_send_state = $2
|
|
where id = $1 and outcome = 'running' and attempt_count > 0",
|
|
)
|
|
.bind(model_request_id)
|
|
.bind(state.as_str())
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("mark upstream send state"))?;
|
|
Ok(result.rows_affected() == 1)
|
|
}
|
|
|
|
async fn mark_downstream_committed(
|
|
&self,
|
|
model_request_id: &str,
|
|
committed_at: DateTime<Utc>,
|
|
client_status_code: Option<u16>,
|
|
) -> StoreResult<bool> {
|
|
require_nonempty(ENTITY, "id", model_request_id)?;
|
|
validate_status_code(client_status_code)?;
|
|
let result = sqlx::query(
|
|
"update model_requests
|
|
set downstream_committed_at = $2, client_status_code = $3
|
|
where id = $1 and outcome = 'running' and downstream_committed_at is null
|
|
and client_status_code is null",
|
|
)
|
|
.bind(model_request_id)
|
|
.bind(committed_at)
|
|
.bind(client_status_code.map(i32::from))
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("mark downstream committed"))?;
|
|
Ok(result.rows_affected() == 1)
|
|
}
|
|
|
|
async fn record_client_status_code(
|
|
&self,
|
|
model_request_id: &str,
|
|
client_status_code: u16,
|
|
) -> StoreResult<bool> {
|
|
require_nonempty(ENTITY, "id", model_request_id)?;
|
|
validate_status_code(Some(client_status_code))?;
|
|
let result = sqlx::query(
|
|
"update model_requests set client_status_code = $2
|
|
where id = $1 and client_status_code is null",
|
|
)
|
|
.bind(model_request_id)
|
|
.bind(i32::from(client_status_code))
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("record client status code"))?;
|
|
Ok(result.rows_affected() == 1)
|
|
}
|
|
|
|
async fn finalize_model_request(
|
|
&self,
|
|
finalization: ModelRequestFinalization,
|
|
) -> StoreResult<bool> {
|
|
finalization.validate()?;
|
|
let result = sqlx::query(
|
|
"update model_requests
|
|
set outcome = $2, upstream_send_state = $3, attempt_count = $4,
|
|
downstream_committed_at = $5,
|
|
client_status_code = coalesce(client_status_code, $6),
|
|
upstream_status_code = $7,
|
|
client_response_id = $8, upstream_request_id = $9, upstream_response_id = $10,
|
|
error_kind = $11, provider_error_code = $12, error_message = $13,
|
|
retry_after_ms = $14, input_tokens = $15, output_tokens = $16,
|
|
cached_tokens = $17, cache_write_tokens = $18, reasoning_tokens = $19,
|
|
image_input_tokens = $20, image_output_tokens = $21, total_tokens = $22,
|
|
image_generation_succeeded = $23, cost_source = $24,
|
|
cost_amount = $25::numeric, cost_currency = $26,
|
|
transport_decision_wait_ms = $27, connect_ms = $28,
|
|
headers_ms = $29, first_event_ms = $30, first_reasoning_ms = $31,
|
|
first_text_ms = $32, first_token_ms = $33, provider_processing_ms = $34,
|
|
latency_ms = $35, completed_at = $36,
|
|
upstream_transport = coalesce($37, upstream_transport),
|
|
http_version = coalesce($38, http_version), websocket_pool = $39,
|
|
service_tier = $40, provider_observation_json = $41
|
|
where id = $1 and outcome = 'running'",
|
|
)
|
|
.bind(&finalization.model_request_id)
|
|
.bind(finalization.outcome.as_str())
|
|
.bind(finalization.upstream_send_state.as_str())
|
|
.bind(
|
|
i32::try_from(finalization.attempt_count)
|
|
.map_err(|_| invalid("attempt_count is too large"))?,
|
|
)
|
|
.bind(finalization.downstream_committed_at)
|
|
.bind(finalization.client_status_code.map(i32::from))
|
|
.bind(finalization.upstream_status_code.map(i32::from))
|
|
.bind(finalization.client_response_id.map(String::into_bytes))
|
|
.bind(finalization.upstream_request_id)
|
|
.bind(finalization.upstream_response_id.map(String::into_bytes))
|
|
.bind(finalization.error_kind)
|
|
.bind(finalization.provider_error_code)
|
|
.bind(finalization.error_message)
|
|
.bind(optional_i64(finalization.retry_after_ms, "retry_after_ms")?)
|
|
.bind(optional_i64(
|
|
finalization.usage.input_tokens,
|
|
"input_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.output_tokens,
|
|
"output_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.cached_tokens,
|
|
"cached_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.cache_write_tokens,
|
|
"cache_write_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.reasoning_tokens,
|
|
"reasoning_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.image_input_tokens,
|
|
"image_input_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.image_output_tokens,
|
|
"image_output_tokens",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.usage.total_tokens,
|
|
"total_tokens",
|
|
)?)
|
|
.bind(finalization.image_generation_succeeded)
|
|
.bind(finalization.cost_source.as_str())
|
|
.bind(finalization.cost_amount.map(|amount| amount.to_string()))
|
|
.bind(finalization.cost_currency)
|
|
.bind(optional_i64(
|
|
finalization.timings.transport_decision_wait_ms,
|
|
"transport_decision_wait_ms",
|
|
)?)
|
|
.bind(optional_i64(finalization.timings.connect_ms, "connect_ms")?)
|
|
.bind(optional_i64(finalization.timings.headers_ms, "headers_ms")?)
|
|
.bind(optional_i64(
|
|
finalization.timings.first_event_ms,
|
|
"first_event_ms",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.timings.first_reasoning_ms,
|
|
"first_reasoning_ms",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.timings.first_text_ms,
|
|
"first_text_ms",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.timings.first_token_ms,
|
|
"first_token_ms",
|
|
)?)
|
|
.bind(optional_i64(
|
|
finalization.timings.provider_processing_ms,
|
|
"provider_processing_ms",
|
|
)?)
|
|
.bind(optional_i64(finalization.timings.latency_ms, "latency_ms")?)
|
|
.bind(finalization.completed_at)
|
|
.bind(finalization.upstream_transport)
|
|
.bind(finalization.http_version)
|
|
.bind(finalization.websocket_pool)
|
|
.bind(finalization.service_tier)
|
|
.bind(finalization.provider_metadata_json.map(sqlx::types::Json))
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("finalize model request"))?;
|
|
Ok(result.rows_affected() == 1)
|
|
}
|
|
|
|
async fn recover_expired_model_requests(
|
|
&self,
|
|
now: DateTime<Utc>,
|
|
) -> StoreResult<ModelRequestRecoveryReport> {
|
|
let result = sqlx::query(
|
|
"update model_requests
|
|
set outcome = 'incomplete', error_kind = 'process_interrupted',
|
|
error_message = 'request did not reach a terminal state',
|
|
image_generation_succeeded = case
|
|
when image_generation_requested then false else null
|
|
end,
|
|
completed_at = $1
|
|
where outcome = 'running' and deadline_at <= $1",
|
|
)
|
|
.bind(now)
|
|
.execute(&self.pool)
|
|
.await
|
|
.map_err(|_| postgres_unavailable("recover expired model requests"))?;
|
|
Ok(ModelRequestRecoveryReport {
|
|
requests: result.rows_affected(),
|
|
})
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl ExecutionStore for PgExecutionStore {
|
|
async fn create_model_request(
|
|
&self,
|
|
request: CoreNewModelRequest,
|
|
) -> Result<(), CoreStoreError> {
|
|
self.insert_model_request(new_model_request_row(request))
|
|
.await
|
|
.map_err(core_store_error)
|
|
}
|
|
|
|
async fn record_attempt(&self, attempt: CoreAttemptRecord) -> Result<(), CoreStoreError> {
|
|
let expected_count = attempt.attempt_count.get();
|
|
let persisted = self
|
|
.begin_model_request_attempt(attempt_start_row(attempt))
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
if persisted == expected_count {
|
|
Ok(())
|
|
} else {
|
|
Err(CoreStoreError::new(CoreStoreErrorKind::InvalidState))
|
|
}
|
|
}
|
|
|
|
async fn create_model_request_with_attempt(
|
|
&self,
|
|
request: CoreNewModelRequest,
|
|
attempt: CoreAttemptRecord,
|
|
) -> Result<(), CoreStoreError> {
|
|
// 合并写只对首次 attempt 成立(插入即带 attempt 列);其余走基础两步。
|
|
if attempt.attempt_count.get() != 1 {
|
|
self.create_model_request(request).await?;
|
|
return self.record_attempt(attempt).await;
|
|
}
|
|
self.insert_model_request_with_first_attempt(
|
|
new_model_request_row(request),
|
|
attempt_start_row(attempt),
|
|
)
|
|
.await
|
|
.map_err(core_store_error)
|
|
}
|
|
|
|
async fn mark_send_state(
|
|
&self,
|
|
request_id: &ModelRequestId,
|
|
state: CoreUpstreamSendState,
|
|
) -> Result<(), CoreStoreError> {
|
|
let updated = self
|
|
.mark_upstream_send_state(request_id.as_str(), send_state_from_core(state))
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
require_core_update(updated)
|
|
}
|
|
|
|
async fn mark_downstream_committed(
|
|
&self,
|
|
request_id: &ModelRequestId,
|
|
committed_at: std::time::SystemTime,
|
|
client_status_code: Option<u16>,
|
|
) -> Result<(), CoreStoreError> {
|
|
let updated = ModelRequestRepository::mark_downstream_committed(
|
|
self,
|
|
request_id.as_str(),
|
|
DateTime::<Utc>::from(committed_at),
|
|
client_status_code,
|
|
)
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
require_core_update(updated)
|
|
}
|
|
|
|
async fn record_client_status(
|
|
&self,
|
|
request_id: &ModelRequestId,
|
|
client_status_code: u16,
|
|
) -> Result<(), CoreStoreError> {
|
|
let updated = ModelRequestRepository::record_client_status_code(
|
|
self,
|
|
request_id.as_str(),
|
|
client_status_code,
|
|
)
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
require_core_update(updated)
|
|
}
|
|
|
|
async fn record_intermediate_failure(
|
|
&self,
|
|
failure: IntermediateFailure,
|
|
) -> Result<(), CoreStoreError> {
|
|
let error = failure.error;
|
|
let retry_after_ms = error
|
|
.retry_after()
|
|
.map(|duration| u64::try_from(duration.as_millis()))
|
|
.transpose()
|
|
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
|
super::OpsEventRepository::append_ops_event(
|
|
&super::PgOpsEventRepository::new(self.pool.clone()),
|
|
super::OpsEvent {
|
|
id: Uuid::now_v7().to_string(),
|
|
model_request_id: Some(failure.request_id.as_str().to_owned()),
|
|
attempt_index: Some(failure.attempt_index.get()),
|
|
level: super::OpsEventLevel::Warning,
|
|
component: "routing".to_owned(),
|
|
operation: failure.trigger.as_str().to_owned(),
|
|
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
|
provider_account_id: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
|
|
provider_account_ref: failure.account_id.as_ref().map(|id| id.as_str().to_owned()),
|
|
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
|
|
failure_kind: error.kind().as_str().to_owned(),
|
|
status_code: error.upstream_status().or(failure.upstream_status_code),
|
|
provider_error_code: error.upstream_code().map(|code| code.as_str().to_owned()),
|
|
retry_after_ms,
|
|
upstream_request_id: error
|
|
.upstream_request_id()
|
|
.map(|id| id.as_str().to_owned())
|
|
.or(failure.upstream_request_id),
|
|
latency_ms: Some(
|
|
u64::try_from(failure.latency.as_millis())
|
|
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?,
|
|
),
|
|
message: "intermediate upstream failure".to_owned(),
|
|
occurrence_count: 1,
|
|
created_at: Utc::now(),
|
|
},
|
|
)
|
|
.await
|
|
.map_err(core_store_error)
|
|
}
|
|
|
|
async fn record_probe_failure(&self, failure: ProbeFailure) -> Result<(), CoreStoreError> {
|
|
let error = failure.error;
|
|
let retry_after_ms = error
|
|
.retry_after()
|
|
.map(|duration| u64::try_from(duration.as_millis()))
|
|
.transpose()
|
|
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
|
let latency_ms = u64::try_from(failure.latency.as_millis())
|
|
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
|
let provider_error_code = error.upstream_code().map(|code| code.as_str().to_owned());
|
|
super::OpsEventRepository::append_ops_event(
|
|
&super::PgOpsEventRepository::new(self.pool.clone()),
|
|
super::OpsEvent {
|
|
id: Uuid::now_v7().to_string(),
|
|
model_request_id: None,
|
|
attempt_index: None,
|
|
level: super::OpsEventLevel::Warning,
|
|
component: "account_probe".to_owned(),
|
|
operation: "connection_test".to_owned(),
|
|
provider_kind: Some(failure.provider_kind.as_str().to_owned()),
|
|
provider_account_id: Some(failure.account_id.as_str().to_owned()),
|
|
provider_account_ref: Some(failure.account_id.as_str().to_owned()),
|
|
upstream_model_id: Some(failure.upstream_model_id.as_str().to_owned()),
|
|
failure_kind: error.kind().as_str().to_owned(),
|
|
status_code: error.upstream_status(),
|
|
provider_error_code,
|
|
retry_after_ms,
|
|
upstream_request_id: error.upstream_request_id().map(|id| id.as_str().to_owned()),
|
|
latency_ms: Some(latency_ms),
|
|
message: "account connection test failed".to_owned(),
|
|
occurrence_count: 1,
|
|
created_at: Utc::now(),
|
|
},
|
|
)
|
|
.await
|
|
.map_err(core_store_error)
|
|
}
|
|
|
|
async fn finalize_model_request(
|
|
&self,
|
|
finalization: CoreModelRequestFinalization,
|
|
) -> Result<(), CoreStoreError> {
|
|
let provider_metadata_json = finalization
|
|
.provider_metadata_json
|
|
.as_deref()
|
|
.map(serde_json::from_str::<Value>)
|
|
.transpose()
|
|
.map_err(|_| CoreStoreError::new(CoreStoreErrorKind::InvalidData))?;
|
|
if provider_metadata_json
|
|
.as_ref()
|
|
.is_some_and(|metadata| !metadata.is_object())
|
|
{
|
|
return Err(CoreStoreError::new(CoreStoreErrorKind::InvalidData));
|
|
}
|
|
let (cost_source, cost_amount, cost_currency) = match finalization.cost.total() {
|
|
Some(total) => (
|
|
cost_source_from_core(finalization.cost.source()),
|
|
Some(
|
|
total
|
|
.amount()
|
|
.to_string()
|
|
.parse()
|
|
.map_err(core_store_error)?,
|
|
),
|
|
Some(total.currency().as_str().to_owned()),
|
|
),
|
|
None => (CostSource::Unavailable, None, None),
|
|
};
|
|
let error_kind = finalization
|
|
.error
|
|
.as_ref()
|
|
.map(|error| error.kind().as_str().to_owned());
|
|
let error_message = finalization
|
|
.error
|
|
.as_ref()
|
|
.map(|error| error.safe_message().to_owned());
|
|
let completed = ModelRequestRepository::finalize_model_request(
|
|
self,
|
|
ModelRequestFinalization {
|
|
model_request_id: finalization.request_id.as_str().to_owned(),
|
|
outcome: outcome_from_core(finalization.outcome)?,
|
|
upstream_send_state: send_state_from_core(finalization.send_state),
|
|
attempt_count: finalization.attempt_count,
|
|
downstream_committed_at: finalization
|
|
.downstream_committed_at
|
|
.map(DateTime::<Utc>::from),
|
|
client_status_code: finalization.client_status_code,
|
|
upstream_status_code: finalization.upstream_status_code,
|
|
client_response_id: finalization.client_response_id,
|
|
upstream_request_id: finalization.upstream_request_id,
|
|
upstream_response_id: finalization.upstream_response_id,
|
|
upstream_transport: finalization.upstream_transport,
|
|
http_version: finalization.http_version,
|
|
websocket_pool: finalization.websocket_pool,
|
|
service_tier: finalization.service_tier,
|
|
provider_metadata_json,
|
|
error_kind,
|
|
provider_error_code: finalization.provider_error_code,
|
|
error_message,
|
|
retry_after_ms: finalization.retry_after_ms,
|
|
usage: usage_from_core(finalization.usage),
|
|
image_generation_succeeded: finalization.image_generation_succeeded,
|
|
cost_source,
|
|
cost_amount,
|
|
cost_currency,
|
|
timings: ModelRequestTimings {
|
|
transport_decision_wait_ms: finalization.timings.transport_decision_wait_ms,
|
|
connect_ms: finalization.timings.connect_ms,
|
|
headers_ms: finalization.timings.headers_ms,
|
|
first_event_ms: finalization.timings.first_event_ms,
|
|
first_reasoning_ms: finalization.timings.first_reasoning_ms,
|
|
first_text_ms: finalization.timings.first_text_ms,
|
|
first_token_ms: finalization.timings.first_token_ms,
|
|
provider_processing_ms: finalization.timings.provider_processing_ms,
|
|
latency_ms: finalization.timings.latency_ms,
|
|
},
|
|
completed_at: DateTime::<Utc>::from(finalization.completed_at),
|
|
},
|
|
)
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
require_core_update(completed)
|
|
}
|
|
|
|
async fn recover_expired(
|
|
&self,
|
|
now: std::time::SystemTime,
|
|
) -> Result<CoreRecoveryReport, CoreStoreError> {
|
|
let report = self
|
|
.recover_expired_model_requests(DateTime::<Utc>::from(now))
|
|
.await
|
|
.map_err(core_store_error)?;
|
|
Ok(CoreRecoveryReport {
|
|
requests: report.requests,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn usage_from_core(usage: CoreUsage) -> ModelRequestUsage {
|
|
ModelRequestUsage {
|
|
input_tokens: usage.input_tokens,
|
|
output_tokens: usage.output_tokens,
|
|
cached_tokens: usage.cached_tokens,
|
|
cache_write_tokens: usage.cache_write_tokens,
|
|
reasoning_tokens: usage.reasoning_tokens,
|
|
image_input_tokens: usage.image_input_tokens,
|
|
image_output_tokens: usage.image_output_tokens,
|
|
total_tokens: usage.total_tokens,
|
|
}
|
|
}
|
|
|
|
const fn send_state_from_core(value: CoreUpstreamSendState) -> UpstreamSendState {
|
|
match value {
|
|
CoreUpstreamSendState::NotSent => UpstreamSendState::NotSent,
|
|
CoreUpstreamSendState::Sent => UpstreamSendState::Sent,
|
|
CoreUpstreamSendState::Ambiguous => UpstreamSendState::Ambiguous,
|
|
}
|
|
}
|
|
|
|
fn outcome_from_core(
|
|
value: gateway_core::engine::ExecutionOutcome,
|
|
) -> Result<ModelRequestOutcome, CoreStoreError> {
|
|
match value {
|
|
gateway_core::engine::ExecutionOutcome::Running => {
|
|
Err(CoreStoreError::new(CoreStoreErrorKind::InvalidState))
|
|
}
|
|
gateway_core::engine::ExecutionOutcome::Succeeded => Ok(ModelRequestOutcome::Succeeded),
|
|
gateway_core::engine::ExecutionOutcome::Failed => Ok(ModelRequestOutcome::Failed),
|
|
gateway_core::engine::ExecutionOutcome::Cancelled => Ok(ModelRequestOutcome::Cancelled),
|
|
gateway_core::engine::ExecutionOutcome::Incomplete => Ok(ModelRequestOutcome::Incomplete),
|
|
}
|
|
}
|
|
|
|
const fn cost_source_from_core(value: CoreCostSource) -> CostSource {
|
|
match value {
|
|
CoreCostSource::ProviderReported => CostSource::ProviderReported,
|
|
CoreCostSource::Calculated => CostSource::Calculated,
|
|
CoreCostSource::Unavailable => CostSource::Unavailable,
|
|
}
|
|
}
|
|
|
|
fn require_core_update(updated: bool) -> Result<(), CoreStoreError> {
|
|
if updated {
|
|
Ok(())
|
|
} else {
|
|
Err(CoreStoreError::new(CoreStoreErrorKind::InvalidState))
|
|
}
|
|
}
|
|
|
|
fn core_store_error(error: StoreError) -> CoreStoreError {
|
|
let kind = match error {
|
|
StoreError::Unavailable { .. } => CoreStoreErrorKind::Unavailable,
|
|
StoreError::Conflict { .. } => CoreStoreErrorKind::Conflict,
|
|
StoreError::NotFound { .. } | StoreError::InvalidData { .. } => {
|
|
CoreStoreErrorKind::InvalidData
|
|
}
|
|
};
|
|
CoreStoreError::new(kind)
|
|
}
|
|
|
|
fn new_model_request_row(request: CoreNewModelRequest) -> NewModelRequest {
|
|
NewModelRequest {
|
|
id: request.id.as_str().to_owned(),
|
|
client_api_key_id: request
|
|
.client_api_key_id
|
|
.as_ref()
|
|
.map(|id| id.as_str().to_owned()),
|
|
client_api_key_ref: request.client_api_key_ref.as_str().to_owned(),
|
|
config_revision: request.config_revision.get(),
|
|
protocol: request.protocol,
|
|
operation: request.operation.as_str().to_owned(),
|
|
endpoint: request.endpoint,
|
|
client_transport: request.client_transport,
|
|
requested_model_id: request.requested_model.as_str().to_owned(),
|
|
client_ip: request.client_ip.map(|address| address.to_string()),
|
|
user_agent: request.user_agent,
|
|
reasoning_effort: request.reasoning_effort,
|
|
reasoning_preset: request.reasoning_preset,
|
|
request_kind: request.request_kind,
|
|
subagent_kind: request.subagent_kind,
|
|
compact: request.compact,
|
|
image_generation_requested: request.image_generation_requested,
|
|
started_at: DateTime::<Utc>::from(request.started_at),
|
|
deadline_at: DateTime::<Utc>::from(request.deadline_at),
|
|
}
|
|
}
|
|
|
|
fn attempt_start_row(attempt: CoreAttemptRecord) -> ModelRequestAttemptStart {
|
|
ModelRequestAttemptStart {
|
|
model_request_id: attempt.request_id.as_str().to_owned(),
|
|
attempt_count: attempt.attempt_count.get(),
|
|
provider_kind: attempt.provider_kind.as_str().to_owned(),
|
|
provider_account_id: attempt
|
|
.provider_account_id
|
|
.as_ref()
|
|
.map(|id| id.as_str().to_owned()),
|
|
provider_account_ref: attempt
|
|
.provider_account_ref
|
|
.as_ref()
|
|
.map(|id| id.as_str().to_owned()),
|
|
upstream_model_id: attempt.upstream_model_id.as_str().to_owned(),
|
|
upstream_transport: attempt.upstream_transport,
|
|
http_version: attempt.http_version,
|
|
}
|
|
}
|
|
|
|
fn optional_i64(value: Option<u64>, field: &'static str) -> StoreResult<Option<i64>> {
|
|
value.map(|value| to_i64(value, field)).transpose()
|
|
}
|
|
|
|
fn validate_status_code(status: Option<u16>) -> StoreResult<()> {
|
|
if status.is_some_and(|status| !(100..=599).contains(&status)) {
|
|
return Err(invalid("HTTP status must be between 100 and 599"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn to_i64(value: u64, field: &'static str) -> StoreResult<i64> {
|
|
i64::try_from(value).map_err(|_| invalid(field))
|
|
}
|
|
|
|
fn invalid(message: &str) -> StoreError {
|
|
StoreError::InvalidData {
|
|
entity: ENTITY,
|
|
message: message.to_owned(),
|
|
}
|
|
}
|