mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 16:05:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0242bd3193 | ||
|
|
229ef48470 | ||
|
|
19c35ba901 | ||
|
|
3c06ae97e8 | ||
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab | ||
|
|
95f6434443 | ||
|
|
8734b0d3d5 | ||
|
|
790d0d9628 | ||
|
|
3c562563c5 | ||
|
|
f19785d0e1 | ||
|
|
a3d098a87c | ||
|
|
449d7c4218 | ||
|
|
d199141a91 | ||
|
|
df83ffdafb | ||
|
|
cf03c56cfa | ||
|
|
3d1e33558a | ||
|
|
f314ebb3a2 | ||
|
|
31d6ef9585 | ||
|
|
fdfb77c4a2 | ||
|
|
a93e3853f7 | ||
|
|
34da9d7cc8 | ||
|
|
b3dd6bbe46 | ||
|
|
9ab3e45288 | ||
|
|
d255a86644 | ||
|
|
f3f50a818f | ||
|
|
1ff520659f | ||
|
|
33381fc472 | ||
|
|
e3d767bf54 | ||
|
|
8f82d0e100 | ||
|
|
3085e5aabe | ||
|
|
36dce639a9 | ||
|
|
6e15757150 | ||
|
|
365d389a14 | ||
|
|
e79d57eaf9 | ||
|
|
8d5dfea6de | ||
|
|
e02e5b186e | ||
|
|
2dc670875a | ||
|
|
9768921ff7 | ||
|
|
db90f0631a | ||
|
|
8a0a8f9af4 | ||
|
|
8ca2c68e90 | ||
|
|
d26fc44fe6 | ||
|
|
7ed2a6668a | ||
|
|
3c4ac99ae4 | ||
|
|
63d276767e | ||
|
|
60dcc5b0db | ||
|
|
0d2c648422 | ||
|
|
56f7ddd1dc | ||
|
|
930e311f51 | ||
|
|
5df8c71b89 | ||
|
|
dfedb5069d | ||
|
|
7016c8f401 | ||
|
|
5a14b74ff2 | ||
|
|
00bab8d390 | ||
|
|
f6e364ca33 | ||
|
|
4cd98889a7 | ||
|
|
c9b613659c | ||
|
|
01b4029998 | ||
|
|
f583e50e88 | ||
|
|
a7ea957843 | ||
|
|
c92c95a733 | ||
|
|
ae7608440e | ||
|
|
5c9da8a39d | ||
|
|
e012f24ef2 | ||
|
|
756c9c7158 | ||
|
|
ab4bf75d0e | ||
|
|
7a9715ed41 | ||
|
|
c353a6159e | ||
|
|
f5dca4848a | ||
|
|
36321a9af4 | ||
|
|
b485b8bd4a | ||
|
|
cae2c8656a | ||
|
|
acf574129f | ||
|
|
6426e0d3de | ||
|
|
eedc39b33a | ||
|
|
9672a2e000 | ||
|
|
0fffd2c550 |
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -15,7 +12,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
@@ -43,14 +40,14 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
ref: ${{ inputs.tag }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Read release metadata
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
RELEASE_TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
ruby <<'RUBY' >> "$GITHUB_OUTPUT"
|
||||
require "json"
|
||||
@@ -188,9 +185,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -253,9 +250,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -378,7 +375,7 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
@@ -454,7 +451,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -471,8 +468,8 @@ jobs:
|
||||
push: true
|
||||
build-args: |
|
||||
CPR_PREBUILT_DIR=dist/docker
|
||||
cache-from: type=gha,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
labels: |
|
||||
@@ -525,7 +522,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -584,7 +581,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
@@ -626,7 +623,7 @@ jobs:
|
||||
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
|
||||
body: |
|
||||
> [!WARNING]
|
||||
> **Upgrading across major versions?** Database schema and configuration are only compatible within a major version, and the one-click self-update refuses cross-major targets (releases before 3.0 lack this gate — do not use their built-in update to reach a newer major). Deploy the new major fresh (new `.runtime/` data directory) and re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
> **Upgrading across major versions?** Online upgrades across major versions are not supported. Deploy the new major fresh with a new `.runtime/` data directory, then re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
|
||||
## 📥 Installation
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ Thumbs.db
|
||||
/.codex-ws-audit/
|
||||
/.agents/
|
||||
/.reasonix/
|
||||
/.claude
|
||||
/.worktrees/
|
||||
/worktrees/
|
||||
/.cargo/
|
||||
|
||||
@@ -168,10 +168,8 @@ docker compose -f deploy/compose.yaml up -d
|
||||
> 运行状态。
|
||||
|
||||
> [!WARNING]
|
||||
> 跨大版本不是原地升级:数据库 schema 与配置只在同一大版本内兼容,内置一键自更新
|
||||
> 会拒绝跨大版本目标(3.0 之前的旧版没有这道门禁,不要用旧版界面的一键更新跨大版本)。
|
||||
> 升级到新的大版本需要全新部署(全新 `.runtime/` 数据目录),并重新导入或重新授权
|
||||
> Provider 账号与客户端 Key。
|
||||
> 不支持跨大版本在线升级。升级到新的大版本时,请使用全新的 `.runtime/` 数据目录重新部署,
|
||||
> 并重新导入或重新授权 Provider 账号与客户端 Key。
|
||||
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
|
||||
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
@@ -184,9 +182,10 @@ upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。该脚本只发布仓库版本和触发 Release 构建,不会登录服务器,也不会
|
||||
改变任何运行实例;源码提交、仓库发版和实例升级是三个独立状态。
|
||||
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
|
||||
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
|
||||
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
|
||||
仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
Generated
+723
-209
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-th
|
||||
|
||||
[dev-dependencies]
|
||||
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
|
||||
serde_json = "1"
|
||||
syn = { version = "3.0.3", features = ["full"] }
|
||||
tempfile = "3.27.0"
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
let providers = ProviderRegistry::new([openai.core_provider(), xai.core_provider()])?;
|
||||
let mut core = gateway_core::initialize(store.core_ports(), providers).await?;
|
||||
host.report_startup_ready("Core");
|
||||
let admin = gateway_admin::initialize(
|
||||
let mut admin = gateway_admin::initialize(
|
||||
admin,
|
||||
store.admin_ports(),
|
||||
vec![openai.admin_provider(), xai.admin_provider()],
|
||||
@@ -99,6 +99,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
plan.extend(core.take_worker_contributions());
|
||||
plan.extend(openai.take_worker_contributions());
|
||||
plan.extend(xai.take_worker_contributions());
|
||||
plan.extend(admin.take_worker_contributions());
|
||||
host.start_workers(plan, store.worker_leader_lease())?;
|
||||
host.report_startup_ready("Workers");
|
||||
host.serve(api.router()).await?;
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
use std::{env, error::Error, io};
|
||||
|
||||
use provider_openai::transport::tls::ensure_rustls_provider;
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
ensure_rustls_provider();
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
match arguments.next().as_deref().and_then(|value| value.to_str()) {
|
||||
None | Some("serve") => {
|
||||
|
||||
@@ -82,6 +82,91 @@ fn gateway_admin_stays_free_of_infrastructure_dependencies() {
|
||||
);
|
||||
}
|
||||
|
||||
/// workspace 包名到冻结成员路径的映射。
|
||||
const PACKAGE_TO_MEMBER: &[(&str, &str)] = &[
|
||||
("codex-proxy-rs", "apps/gateway"),
|
||||
("gateway-admin", "crates/gateway-admin"),
|
||||
("gateway-api", "crates/gateway-api"),
|
||||
("gateway-core", "crates/gateway-core"),
|
||||
("gateway-host", "crates/gateway-host"),
|
||||
("gateway-protocol", "crates/gateway-protocol"),
|
||||
("gateway-store", "crates/gateway-store"),
|
||||
("provider-openai", "crates/providers/openai"),
|
||||
("provider-xai", "crates/providers/xai"),
|
||||
];
|
||||
|
||||
/// 冻结的 workspace 内部运行时依赖边;新增/删除任何边都必须同步本表。
|
||||
const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
("codex-proxy-rs", "gateway-admin"),
|
||||
("codex-proxy-rs", "gateway-api"),
|
||||
("codex-proxy-rs", "gateway-core"),
|
||||
("codex-proxy-rs", "gateway-host"),
|
||||
("codex-proxy-rs", "gateway-store"),
|
||||
("codex-proxy-rs", "provider-openai"),
|
||||
("codex-proxy-rs", "provider-xai"),
|
||||
("gateway-admin", "gateway-core"),
|
||||
("gateway-api", "gateway-admin"),
|
||||
("gateway-api", "gateway-core"),
|
||||
("gateway-api", "gateway-protocol"),
|
||||
("gateway-host", "gateway-admin"),
|
||||
("gateway-host", "gateway-core"),
|
||||
("gateway-store", "gateway-admin"),
|
||||
("gateway-store", "gateway-core"),
|
||||
("provider-openai", "gateway-admin"),
|
||||
("provider-openai", "gateway-core"),
|
||||
("provider-openai", "gateway-protocol"),
|
||||
("provider-xai", "gateway-admin"),
|
||||
("provider-xai", "gateway-core"),
|
||||
("provider-xai", "gateway-protocol"),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn workspace_internal_dependency_edges_match_frozen_dag() {
|
||||
let metadata = cargo_metadata_json();
|
||||
let packages = metadata["packages"]
|
||||
.as_array()
|
||||
.expect("cargo metadata packages");
|
||||
let internal_names: std::collections::BTreeSet<&str> =
|
||||
PACKAGE_TO_MEMBER.iter().map(|(name, _)| *name).collect();
|
||||
|
||||
let mut actual: Vec<(String, String)> = Vec::new();
|
||||
for package in packages {
|
||||
let name = package["name"].as_str().expect("package name");
|
||||
if !internal_names.contains(name) {
|
||||
continue;
|
||||
}
|
||||
for dependency in package["dependencies"].as_array().into_iter().flatten() {
|
||||
let dep_name = dependency["name"].as_str().expect("dependency name");
|
||||
if !internal_names.contains(dep_name) {
|
||||
continue;
|
||||
}
|
||||
let is_runtime_edge = dependency["dep_kinds"]
|
||||
.as_array()
|
||||
.map(|kinds| {
|
||||
kinds
|
||||
.iter()
|
||||
.any(|kind| kind["kind"].as_str() == Some("normal"))
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if is_runtime_edge {
|
||||
actual.push((name.to_owned(), dep_name.to_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut expected: Vec<(String, String)> = ALLOWED_INTERNAL_EDGES
|
||||
.iter()
|
||||
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
|
||||
.collect();
|
||||
actual.sort();
|
||||
expected.sort();
|
||||
|
||||
assert_eq!(
|
||||
actual, expected,
|
||||
"workspace internal dependency edges diverged from the frozen DAG"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
@@ -107,6 +192,20 @@ fn backend_root() -> PathBuf {
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
fn cargo_metadata_json() -> serde_json::Value {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束。
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
|
||||
@@ -135,7 +135,7 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_core_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.145.0'", "codex_version: 'latest'"));
|
||||
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -154,7 +154,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '5828'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '6119'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -26,9 +26,14 @@ subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
cron = "0.17"
|
||||
chrono-tz = "0.10"
|
||||
tokio = { version = "1.52", features = ["rt", "time", "macros"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
//! 备份纯策略与 Worker 任务。
|
||||
|
||||
pub mod policy;
|
||||
pub mod task;
|
||||
@@ -0,0 +1,161 @@
|
||||
//! 备份纯策略:Cron 推进与保留候选。
|
||||
//!
|
||||
//! 本模块不持有任何端口或对象,是 `BackupService` 与 `BackupTask` 共用的纯函数集。
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use chrono::{DateTime, Duration, TimeZone as _, Utc};
|
||||
use chrono_tz::Tz;
|
||||
use cron::Schedule;
|
||||
|
||||
use crate::model::backup::{BackupError, BackupRecord, BackupStatus, code};
|
||||
|
||||
/// 向历史回溯的最大范围(分钟)。覆盖 Feb 29 + 特定星期等罕见组合的最长间隔。
|
||||
const MAX_BACKDATE_MINUTES: i64 = 40 * 366 * 24 * 60;
|
||||
|
||||
/// 已解析并校验的备份计划。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupSchedule {
|
||||
schedule: Schedule,
|
||||
timezone: Tz,
|
||||
}
|
||||
|
||||
impl BackupSchedule {
|
||||
/// 解析并校验 5 段 Cron 与 IANA 时区。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Cron 非法或时区不是 IANA 名称时返回 [`BackupError`]。
|
||||
pub fn parse(cron_expression: &str, schedule_timezone: &str) -> Result<Self, BackupError> {
|
||||
let expression = format!("0 {cron_expression}");
|
||||
let schedule = Schedule::from_str(&expression).map_err(|_| {
|
||||
BackupError::new(code::INVALID_CRON, "Cron 表达式必须是 5 段格式".to_owned())
|
||||
})?;
|
||||
let timezone = schedule_timezone.parse::<Tz>().map_err(|_| {
|
||||
BackupError::new(code::INVALID_TIMEZONE, "时区必须是 IANA 名称".to_owned())
|
||||
})?;
|
||||
Ok(Self { schedule, timezone })
|
||||
}
|
||||
|
||||
/// 返回最接近 `at`(含 `at` 所在分钟)的最近一次触发时间。
|
||||
///
|
||||
/// 若过去 `MAX_BACKDATE_MINUTES` 内没有任何触发(极罕见的稀疏计划或超长停机),
|
||||
/// 返回 `None`,调用方不应为历史点补任务。
|
||||
#[must_use]
|
||||
pub fn last_firing_at_or_before(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
|
||||
let high_min = at.timestamp().div_euclid(60);
|
||||
let low_min = high_min.saturating_sub(MAX_BACKDATE_MINUTES);
|
||||
|
||||
// 检查窗口内是否存在触发;不存在则直接返回 None。
|
||||
let low_instant = instant(low_min, self.timezone)?;
|
||||
if self
|
||||
.schedule
|
||||
.after(&low_instant)
|
||||
.next()
|
||||
.is_none_or(|firing| firing.timestamp() > high_min * 60 + 59)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
// 二分查找最小的 u(分钟)使 after(u) > high;prev = after(u - 1 分钟)。
|
||||
let mut lo = low_min;
|
||||
let mut hi = high_min;
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
if self
|
||||
.firing_after(mid)
|
||||
.is_some_and(|firing| firing > high_min)
|
||||
{
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
self.firing_after(lo - 1)
|
||||
.filter(|&firing| firing <= high_min)
|
||||
.and_then(|firing| Utc.timestamp_opt(firing * 60, 0).single())
|
||||
}
|
||||
|
||||
/// 返回第一个严格晚于 `at` 的触发时间。
|
||||
#[must_use]
|
||||
pub fn next_after(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
|
||||
let local = at.with_timezone(&self.timezone);
|
||||
self.schedule
|
||||
.after(&local)
|
||||
.next()
|
||||
.map(|firing| firing.with_timezone(&Utc))
|
||||
}
|
||||
|
||||
/// `after(u 分钟)` 返回的第一个触发时间(分钟索引),用于二分搜索。
|
||||
fn firing_after(&self, minute_index: i64) -> Option<i64> {
|
||||
let instant = instant(minute_index, self.timezone)?;
|
||||
self.schedule
|
||||
.after(&instant)
|
||||
.next()
|
||||
.map(|firing| firing.timestamp().div_euclid(60))
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造指定分钟索引(UTC epoch 分钟)在目标时区下的 `DateTime`。
|
||||
fn instant(minute_index: i64, timezone: Tz) -> Option<DateTime<Tz>> {
|
||||
timezone.timestamp_opt(minute_index * 60, 0).single()
|
||||
}
|
||||
|
||||
/// 一条记录被纳入删除流程的原因。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RetentionReason {
|
||||
/// 超过 `retention_days`。
|
||||
Days,
|
||||
/// 超过 `retention_count`。
|
||||
Count,
|
||||
}
|
||||
|
||||
/// 保留决策。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RetentionDecision {
|
||||
pub record_id: String,
|
||||
pub reason: RetentionReason,
|
||||
}
|
||||
|
||||
/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。
|
||||
///
|
||||
/// 规则(`docs/s3-backup-design-audit.md` §4.4):
|
||||
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理。
|
||||
/// - 自动计划备份满足任一启用阈值即可进入删除。
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)。
|
||||
/// - 手动备份不受自动阈值影响;无 TTL,仅由管理员手动删除。
|
||||
#[must_use]
|
||||
pub fn decide_retention(
|
||||
retention_days: u32,
|
||||
retention_count: u32,
|
||||
now: DateTime<Utc>,
|
||||
scheduled_completed_desc: &[BackupRecord],
|
||||
) -> Vec<RetentionDecision> {
|
||||
let mut decisions = Vec::new();
|
||||
for (index, record) in scheduled_completed_desc.iter().enumerate() {
|
||||
if record.status != BackupStatus::Completed {
|
||||
continue;
|
||||
}
|
||||
// 最近一个成功的计划备份永远保留。
|
||||
if index == 0 {
|
||||
continue;
|
||||
}
|
||||
let expired_by_days = retention_days > 0
|
||||
&& record.completed_at.is_some_and(|completed_at| {
|
||||
completed_at <= now - Duration::days(i64::from(retention_days))
|
||||
});
|
||||
let exceeded_count =
|
||||
retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX);
|
||||
if expired_by_days || exceeded_count {
|
||||
decisions.push(RetentionDecision {
|
||||
record_id: record.id.clone(),
|
||||
reason: if expired_by_days {
|
||||
RetentionReason::Days
|
||||
} else {
|
||||
RetentionReason::Count
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
decisions
|
||||
}
|
||||
@@ -0,0 +1,648 @@
|
||||
//! Backup Worker 贡献:单个可取消 `DaemonTask` 承担调度、执行、删除收敛与保留清理。
|
||||
//!
|
||||
//! 当前部署边界是单副本,因此不需要 Redis lease、fencing token 或 heartbeat。长时间
|
||||
//! `pg_dump` 与上传由本 Daemon 自身持有;Host 只负责 panic 后重启、健康与关闭。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, build_backup_seed, code,
|
||||
},
|
||||
ports::backup::{
|
||||
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpRequest,
|
||||
StatusTransitionUpdate, UploadObjectRequest,
|
||||
},
|
||||
};
|
||||
|
||||
impl From<BackupError> for WorkerTaskError {
|
||||
fn from(error: BackupError) -> Self {
|
||||
WorkerTaskError::safe(error.message())
|
||||
}
|
||||
}
|
||||
|
||||
use super::policy::{BackupSchedule, decide_retention};
|
||||
|
||||
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待。
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量。
|
||||
const RETENTION_BATCH_SIZE: usize = 10;
|
||||
/// 保留扫描一次读取的 completed 计划备份上限。
|
||||
const RETENTION_SCAN_LIMIT: u32 = 1000;
|
||||
/// 每个循环最多完成的待删除记录数量。
|
||||
const PENDING_DELETION_BATCH: u32 = 20;
|
||||
|
||||
/// 备份 Daemon 任务。
|
||||
pub struct BackupTask {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
}
|
||||
|
||||
impl BackupTask {
|
||||
/// 组合仓储、导出器与对象存储适配器。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
dump,
|
||||
object_store,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move { self.run_daemon(cancellation).await })
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupTask {
|
||||
async fn run_daemon(&self, cancellation: CancellationToken) -> Result<(), WorkerTaskError> {
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle(&cancellation).await?;
|
||||
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Ok(()),
|
||||
_ = tokio::time::sleep(BACKUP_LOOP_INTERVAL) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一个周期:推进计划、恢复中间态、删除收敛、领取执行一个任务、保留清理。
|
||||
///
|
||||
/// 对外暴露以便集成测试逐周期驱动;daemon 循环内部复用。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 仓储或基础设施不可用、任务执行失败时返回 [`WorkerTaskError`]。
|
||||
pub async fn run_cycle(&self, cancellation: &CancellationToken) -> Result<(), WorkerTaskError> {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.advance_schedule(Utc::now()).await?;
|
||||
self.recover_intermediate(cancellation).await?;
|
||||
self.finish_pending_deletions().await?;
|
||||
self.process_next_task(cancellation).await?;
|
||||
self.run_retention_batch(Utc::now()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 推进到期计划并幂等插入 scheduled 任务。
|
||||
async fn advance_schedule(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
if !settings.schedule_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let cron = settings.cron_expression.as_deref().unwrap_or_default();
|
||||
let timezone = settings.schedule_timezone.as_deref().unwrap_or_default();
|
||||
if cron.is_empty() || timezone.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let schedule = BackupSchedule::parse(cron, timezone)
|
||||
.map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?;
|
||||
|
||||
let due = match settings.next_run_at {
|
||||
Some(next_run_at) if next_run_at <= now => true,
|
||||
Some(_) => false,
|
||||
None => {
|
||||
// 刚启用计划但尚无游标:直接初始化。
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
if !due {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let scheduled_at = match schedule.last_firing_at_or_before(now) {
|
||||
Some(scheduled_at) => scheduled_at,
|
||||
None => {
|
||||
// 无法从计划推导最近到期时间:推进游标避免反复触发。
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let prefix = settings.prefix.as_deref().unwrap_or_default();
|
||||
let expires_at = if settings.retention_days > 0 {
|
||||
Some(now + chrono::Duration::days(i64::from(settings.retention_days)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let seed = build_backup_seed(
|
||||
BackupTriggerKind::Scheduled,
|
||||
Some(scheduled_at),
|
||||
prefix,
|
||||
now,
|
||||
expires_at,
|
||||
)
|
||||
.map_err(infra_error)?;
|
||||
let inserted = self
|
||||
.repository
|
||||
.insert_scheduled_record(seed)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
if inserted {
|
||||
info!(scheduled_at = %scheduled_at, "计划备份任务已创建");
|
||||
} else {
|
||||
warn!(scheduled_at = %scheduled_at, "计划时间点冲突,跳过并推进游标");
|
||||
}
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 恢复中间状态(dumping/uploading/deleting)。
|
||||
async fn recover_intermediate(
|
||||
&self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let records = self
|
||||
.repository
|
||||
.list_intermediate_records()
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in records {
|
||||
match record.status {
|
||||
BackupStatus::Dumping => self.recover_dumping(&record, cancellation).await?,
|
||||
BackupStatus::Uploading => self.recover_uploading(&record, cancellation).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// dumping:有完整暂存则继续上传,否则清理暂存并标记失败。
|
||||
async fn recover_dumping(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
match self.dump.inspect_staging(&record.id).await? {
|
||||
Some(artifact) => {
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
size_bytes: Some(artifact.size_bytes),
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
update,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
self.upload_and_verify(
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
now,
|
||||
cancellation,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
self.fail_task(record, code::PG_DUMP_FAILED, "导出进程中断,暂存归档不完整")
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// uploading:远端匹配则补记完成,有完整暂存则重试上传,否则失败。
|
||||
async fn recover_uploading(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(remote) = self
|
||||
.object_store
|
||||
.head_object(&config, &record.object_key)
|
||||
.await?
|
||||
&& record.size_bytes == Some(remote.size_bytes)
|
||||
&& record.sha256.as_deref() == Some(remote.sha256.as_str())
|
||||
{
|
||||
let now = Utc::now();
|
||||
self.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
StatusTransitionUpdate::default(),
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
return Ok(());
|
||||
}
|
||||
match self.dump.inspect_staging(&record.id).await? {
|
||||
Some(artifact) => {
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
Utc::now(),
|
||||
cancellation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.fail_task(
|
||||
record,
|
||||
code::S3_UPLOAD_FAILED,
|
||||
"远端对象缺失且本地无完整暂存",
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 完成待删除记录(deleting → DeleteObject → 硬删除)。
|
||||
async fn finish_pending_deletions(&self) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
let records = self
|
||||
.repository
|
||||
.list_pending_deletions(PENDING_DELETION_BATCH)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in records {
|
||||
if self
|
||||
.object_store
|
||||
.delete_object(&config, &record.object_key)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.repository
|
||||
.delete_record(&record.id)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 领取一个 queued 任务并执行导出、上传与远端校验。
|
||||
async fn process_next_task(
|
||||
&self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let now = Utc::now();
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.claim_next_queued(now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
info!(
|
||||
backup_id = %record.id,
|
||||
attempt = record.attempt_count,
|
||||
"领取备份任务"
|
||||
);
|
||||
if let Err(error) = self.execute_task(&record, cancellation).await {
|
||||
warn!(backup_id = %record.id, error = %error, "备份任务执行失败");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 执行单个任务;内部处理全部状态迁移。
|
||||
async fn execute_task(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
if BackupStorageConfig::from_settings(&settings).is_none() {
|
||||
return self
|
||||
.fail_task(record, code::INVALID_CONFIG, "备份存储配置不完整")
|
||||
.await;
|
||||
}
|
||||
if !settings.storage_verified() {
|
||||
return self
|
||||
.fail_task(record, code::INVALID_CONFIG, "备份存储配置尚未通过连接测试")
|
||||
.await;
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
let artifact = match self
|
||||
.dump
|
||||
.dump(DumpRequest {
|
||||
backup_id: record.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(artifact) => artifact,
|
||||
Err(error) => {
|
||||
let (error_code, message) = classify_dump_error(error);
|
||||
return self.fail_task(record, error_code, message).await;
|
||||
}
|
||||
};
|
||||
|
||||
// dumping → uploading,持久化 size/sha256。
|
||||
let update = StatusTransitionUpdate {
|
||||
size_bytes: Some(artifact.size_bytes),
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
update,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.upload_and_verify(
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
now,
|
||||
cancellation,
|
||||
)
|
||||
.await?;
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 上传 + 远端校验,内部完成终态迁移。
|
||||
async fn upload_and_verify(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
source: &std::path::Path,
|
||||
size_bytes: u64,
|
||||
sha256: &str,
|
||||
now: DateTime<Utc>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
let store = self.object_store.clone();
|
||||
let metadata =
|
||||
BackupObjectMetadata::new(record.id.clone(), sha256.to_owned(), now, size_bytes);
|
||||
if let Err(error) = store
|
||||
.upload_file(
|
||||
&config,
|
||||
UploadObjectRequest {
|
||||
object_key: record.object_key.clone(),
|
||||
source: source.to_path_buf(),
|
||||
metadata,
|
||||
cancellation: cancellation.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match store.head_object(&config, &record.object_key).await {
|
||||
Ok(Some(remote)) if remote.size_bytes == size_bytes && remote.sha256 == sha256 => {
|
||||
if self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
StatusTransitionUpdate::default(),
|
||||
Utc::now(),
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
.is_some()
|
||||
{
|
||||
info!(
|
||||
backup_id = %record.id,
|
||||
size_bytes,
|
||||
"备份完成"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(Some(remote)) => {
|
||||
let size_matches = remote.size_bytes == size_bytes;
|
||||
let sha256_matches = remote.sha256 == sha256;
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
expected_size_bytes = size_bytes,
|
||||
remote_size_bytes = remote.size_bytes,
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"上传完成后未找到远端对象",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 把活跃任务写入失败终态并清理暂存。
|
||||
async fn fail_task(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let message = message.into();
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
error_code,
|
||||
error_message = %message,
|
||||
"备份任务失败"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
error_code: Some(error_code.to_string()),
|
||||
error_message: Some(message),
|
||||
completed_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = self
|
||||
.repository
|
||||
.transition_status(&record.id, record.status, BackupStatus::Failed, update, now)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 执行一小批到期保留清理:先处理 `expires_at` 已到期的记录,再按天数/份数清理计划备份。
|
||||
async fn run_retention_batch(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
|
||||
// 1. expires_at 已到期的记录(手动或计划)无条件进入删除。
|
||||
let expired = self
|
||||
.repository
|
||||
.list_expired_records(RETENTION_BATCH_SIZE as u32)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in expired {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&record.id, now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
self.finalize_deletion(&record).await?;
|
||||
}
|
||||
|
||||
// 2. 计划备份按 retentionDays / retentionCount 清理。
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(RETENTION_SCAN_LIMIT)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let decisions = decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
&records,
|
||||
);
|
||||
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&decision.record_id, now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
self.finalize_deletion(&record).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 对已进入 deleting 的记录执行对象删除并硬删除记录行。
|
||||
async fn finalize_deletion(&self, record: &BackupRecord) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
if self
|
||||
.object_store
|
||||
.delete_object(&config, &record.object_key)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.repository
|
||||
.delete_record(&record.id)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
|
||||
match error.code() {
|
||||
code::CANCELLED => (code::CANCELLED, "备份已取消"),
|
||||
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
|
||||
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(format!("backup store failed: {error}"))
|
||||
}
|
||||
|
||||
fn infra_error(error: BackupError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(error.message())
|
||||
}
|
||||
@@ -2,23 +2,28 @@
|
||||
//!
|
||||
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现。
|
||||
|
||||
use std::{fmt, path::Path, sync::Arc};
|
||||
use std::{fmt, path::Path, sync::Arc, time::Duration};
|
||||
|
||||
use gateway_core::{
|
||||
engine::probe::AccountProbe,
|
||||
routing::{ProviderKind, snapshot::SnapshotControl},
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
|
||||
WorkerRunnable,
|
||||
},
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
mod use_case;
|
||||
|
||||
pub use use_case::{
|
||||
accounts::AccountsService, auth::AuthService, client_keys::ClientKeyService,
|
||||
observability::ObservabilityService, openai::OpenAiService, settings::SettingsService,
|
||||
system::SystemService, xai::XaiService,
|
||||
accounts::AccountsService, auth::AuthService, backup::BackupService,
|
||||
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
|
||||
settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
};
|
||||
|
||||
use model::{AdminError, AdminErrorKind};
|
||||
@@ -28,7 +33,7 @@ use ports::{
|
||||
system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
accounts::DefaultAccountsService, auth::DefaultAuthService,
|
||||
accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_keys::DefaultClientKeyService, observability::DefaultObservabilityService,
|
||||
openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService,
|
||||
xai::DefaultXaiService,
|
||||
@@ -48,6 +53,8 @@ const WEAK_INITIAL_PASSWORDS: &[&str] = &[
|
||||
"codex-proxy-rs",
|
||||
];
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
|
||||
/// 只用于首次幂等创建默认管理员的启动密码。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
@@ -146,6 +153,7 @@ pub struct AdminServices {
|
||||
system: Arc<dyn SystemService>,
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
backups: Arc<dyn BackupService>,
|
||||
}
|
||||
|
||||
impl AdminServices {
|
||||
@@ -188,11 +196,17 @@ impl AdminServices {
|
||||
pub fn xai(&self) -> &dyn XaiService {
|
||||
self.xai.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn backups(&self) -> &dyn BackupService {
|
||||
self.backups.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Admin 初始化完成后的封闭能力包。
|
||||
pub struct AdminBundle {
|
||||
services: AdminServices,
|
||||
worker_contributions: Vec<WorkerContribution>,
|
||||
}
|
||||
|
||||
impl AdminBundle {
|
||||
@@ -200,6 +214,11 @@ impl AdminBundle {
|
||||
pub fn services(&self) -> AdminServices {
|
||||
self.services.clone()
|
||||
}
|
||||
|
||||
/// 取出 Backup Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验配置、建立动态 Provider 注册表并完成默认管理员幂等初始化。
|
||||
@@ -241,6 +260,18 @@ pub async fn initialize(
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
));
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
);
|
||||
let services = AdminServices {
|
||||
auth,
|
||||
accounts,
|
||||
@@ -268,8 +299,32 @@ pub async fn initialize(
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
backups,
|
||||
};
|
||||
Ok(AdminBundle { services })
|
||||
let worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`。
|
||||
fn backup_worker_contribution(
|
||||
task: backup::task::BackupTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("Backup worker id is invalid"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("Backup worker restart policy is invalid"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("Backup worker registration is invalid"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -13,13 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
Expired,
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -75,9 +208,9 @@ pub struct AccountRecord {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -169,12 +302,16 @@ pub struct AccountPage {
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
|
||||
@@ -0,0 +1,566 @@
|
||||
//! S3 备份领域模型、状态机与稳定错误。
|
||||
//!
|
||||
//! 领域类型只携带事实,不承载基础设施细节。Secret 使用 [`secrecy::SecretString`],
|
||||
//! 全流程不实现 `Serialize`,避免被错误地写入 wire 或日志。管理员身份等审计事实
|
||||
//! 只存在于 `admin_audit_events`,本模型不重复保存。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use secrecy::SecretString;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::PageSize;
|
||||
|
||||
/// 备份触发来源。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupTriggerKind {
|
||||
Manual,
|
||||
Scheduled,
|
||||
}
|
||||
|
||||
impl BackupTriggerKind {
|
||||
/// 稳定持久化值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Manual => "manual",
|
||||
Self::Scheduled => "scheduled",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析持久化值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"manual" => Some(Self::Manual),
|
||||
"scheduled" => Some(Self::Scheduled),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for BackupTriggerKind {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份任务状态机(六状态)。
|
||||
///
|
||||
/// 迁移 `backup_records_lifecycle_ck` 与字段空值语义共同定义合法持久化行;
|
||||
/// [`Self::allows_transition_to`] 只约束领域内的显式状态迁移,两层都必须满足。
|
||||
/// 删除成功后记录行被硬删除,因此不存在 `deleted` 终态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BackupStatus {
|
||||
Queued,
|
||||
Dumping,
|
||||
Uploading,
|
||||
Completed,
|
||||
Failed,
|
||||
Deleting,
|
||||
}
|
||||
|
||||
impl BackupStatus {
|
||||
/// 稳定持久化值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Queued => "queued",
|
||||
Self::Dumping => "dumping",
|
||||
Self::Uploading => "uploading",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Deleting => "deleting",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析持久化值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"queued" => Some(Self::Queued),
|
||||
"dumping" => Some(Self::Dumping),
|
||||
"uploading" => Some(Self::Uploading),
|
||||
"completed" => Some(Self::Completed),
|
||||
"failed" => Some(Self::Failed),
|
||||
"deleting" => Some(Self::Deleting),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 该状态是否消耗全局活跃名额。
|
||||
#[must_use]
|
||||
pub const fn is_active(self) -> bool {
|
||||
matches!(self, Self::Queued | Self::Dumping | Self::Uploading)
|
||||
}
|
||||
|
||||
/// 该状态是否允许进入删除流程。
|
||||
#[must_use]
|
||||
pub const fn can_be_deleted(self) -> bool {
|
||||
matches!(self, Self::Completed | Self::Failed)
|
||||
}
|
||||
|
||||
/// 领域内允许的显式迁移。
|
||||
#[must_use]
|
||||
pub fn allows_transition_to(self, target: BackupStatus) -> bool {
|
||||
matches!(
|
||||
(self, target),
|
||||
(Self::Queued, Self::Dumping)
|
||||
| (Self::Queued, Self::Failed)
|
||||
| (Self::Dumping, Self::Uploading)
|
||||
| (Self::Dumping, Self::Failed)
|
||||
| (Self::Uploading, Self::Completed)
|
||||
| (Self::Uploading, Self::Failed)
|
||||
| (Self::Completed, Self::Deleting)
|
||||
| (Self::Failed, Self::Deleting)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for BackupStatus {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 存储配置更新命令;`secret_access_key` 为 `None` 表示保留旧值。
|
||||
#[derive(Clone)]
|
||||
pub struct UpdateBackupStorageCommand {
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: Option<SecretString>,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for UpdateBackupStorageCommand {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("UpdateBackupStorageCommand")
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &self.secret_access_key.is_some())
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 调度配置更新命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateBackupScheduleCommand {
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: String,
|
||||
pub schedule_timezone: String,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
}
|
||||
|
||||
/// 完整保存的备份设置(含 Secret)。
|
||||
///
|
||||
/// 只从仓储读取;不实现 `Serialize`,防止 Secret 进入 wire 或日志。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupSettings {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub bucket: Option<String>,
|
||||
pub access_key_id: Option<String>,
|
||||
pub secret_access_key: Option<SecretString>,
|
||||
pub prefix: Option<String>,
|
||||
pub force_path_style: bool,
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: Option<String>,
|
||||
pub schedule_timezone: Option<String>,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
pub next_run_at: Option<DateTime<Utc>>,
|
||||
pub last_verified_at: Option<DateTime<Utc>>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BackupSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BackupSettings")
|
||||
.field("storage_revision", &self.storage_revision)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &self.secret_access_key.is_some())
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.field("schedule_enabled", &self.schedule_enabled)
|
||||
.field("cron_expression", &self.cron_expression)
|
||||
.field("schedule_timezone", &self.schedule_timezone)
|
||||
.field("retention_days", &self.retention_days)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.field("next_run_at", &self.next_run_at)
|
||||
.field("last_verified_at", &self.last_verified_at)
|
||||
.field("updated_at", &self.updated_at)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupSettings {
|
||||
/// S3 存储配置是否完整(bucket/region/密钥/前缀齐全)。
|
||||
#[must_use]
|
||||
pub fn storage_configured(&self) -> bool {
|
||||
self.endpoint.is_some()
|
||||
&& self.region.is_some()
|
||||
&& self.bucket.is_some()
|
||||
&& self.access_key_id.is_some()
|
||||
&& self.secret_access_key.is_some()
|
||||
&& self.prefix.is_some()
|
||||
}
|
||||
|
||||
/// 探测是否已对当前 revision 成功;`last_verified_at` 非空即表示当前配置已通过探针。
|
||||
#[must_use]
|
||||
pub fn storage_verified(&self) -> bool {
|
||||
self.last_verified_at.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// 已经校验并可用于对象存储操作的存储配置快照。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupStorageConfig {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: SecretString,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BackupStorageConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BackupStorageConfig")
|
||||
.field("storage_revision", &self.storage_revision)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &"[REDACTED]")
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupStorageConfig {
|
||||
/// 从完整设置推导对象存储配置;设置不完整时返回 `None`。
|
||||
#[must_use]
|
||||
pub fn from_settings(settings: &BackupSettings) -> Option<Self> {
|
||||
Some(Self {
|
||||
storage_revision: settings.storage_revision,
|
||||
endpoint: settings.endpoint.clone()?,
|
||||
region: settings.region.clone()?,
|
||||
bucket: settings.bucket.clone()?,
|
||||
access_key_id: settings.access_key_id.clone()?,
|
||||
secret_access_key: settings.secret_access_key.clone()?,
|
||||
prefix: settings.prefix.clone()?,
|
||||
force_path_style: settings.force_path_style,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 一条备份记录。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecord {
|
||||
pub id: String,
|
||||
pub trigger_kind: BackupTriggerKind,
|
||||
pub status: BackupStatus,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
/// 手动过期时间;创建时确定,到期自动清理。
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 备份记录分页查询。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordListQuery {
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub status: Option<BackupStatus>,
|
||||
pub trigger: Option<BackupTriggerKind>,
|
||||
}
|
||||
|
||||
/// 备份记录分页结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordPage {
|
||||
pub items: Vec<BackupRecord>,
|
||||
pub total: u64,
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
}
|
||||
|
||||
impl BackupRecordPage {
|
||||
/// 计算总页数;空集合视为一页。
|
||||
#[must_use]
|
||||
pub fn total_pages(&self) -> u32 {
|
||||
let size = u64::from(self.page_size.get());
|
||||
if self.total == 0 {
|
||||
1
|
||||
} else {
|
||||
self.total.div_ceil(size).min(u64::from(u32::MAX)) as u32
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建备份任务的持久化数据;id 与对象 key 由领域生成。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordSeed {
|
||||
pub id: String,
|
||||
pub trigger_kind: BackupTriggerKind,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 连接测试结果;只携带稳定阶段与脱敏消息。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ConnectionTestResult {
|
||||
pub ok: bool,
|
||||
pub stage: &'static str,
|
||||
pub code: Option<&'static str>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// 对象存储探测阶段。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ConnectionTestStage {
|
||||
PutObject,
|
||||
HeadObject,
|
||||
GetObject,
|
||||
DeleteObject,
|
||||
}
|
||||
|
||||
impl ConnectionTestStage {
|
||||
/// 稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::PutObject => "putObject",
|
||||
Self::HeadObject => "headObject",
|
||||
Self::GetObject => "getObject",
|
||||
Self::DeleteObject => "deleteObject",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 上传到对象存储并期望远端校验的归档 metadata。
|
||||
///
|
||||
/// `size_bytes` 在上传时不写入 metadata,但 `HeadObject` 会返回远端大小,
|
||||
/// 用于与本地归档校验一致。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupObjectMetadata {
|
||||
pub backup_id: String,
|
||||
pub sha256: String,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
impl BackupObjectMetadata {
|
||||
/// 构造上传 metadata。
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
backup_id: String,
|
||||
sha256: String,
|
||||
created_at: DateTime<Utc>,
|
||||
size_bytes: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
backup_id,
|
||||
sha256,
|
||||
created_at,
|
||||
size_bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 短时下载地址结果。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct DownloadUrlResult {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub expires_in: std::time::Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for DownloadUrlResult {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("DownloadUrlResult")
|
||||
.field("url", &"[REDACTED]")
|
||||
.field("file_name", &self.file_name)
|
||||
.field("expires_in", &self.expires_in)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份基础设施错误:只携带稳定错误码与脱敏消息。
|
||||
///
|
||||
/// 该类型不构成第二套 kind 枚举;`gateway-admin` 边界统一映射为既有
|
||||
/// `AdminErrorKind`。稳定 `error_code` 同时用于任务记录、日志与 UI。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct BackupError {
|
||||
code: &'static str,
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl BackupError {
|
||||
/// 构造基础设施错误。
|
||||
#[must_use]
|
||||
pub const fn new(code: &'static str, message: String) -> Self {
|
||||
Self { code, message }
|
||||
}
|
||||
|
||||
/// 稳定错误码,例如 `backup.s3_auth_failed`。
|
||||
#[must_use]
|
||||
pub const fn code(&self) -> &'static str {
|
||||
self.code
|
||||
}
|
||||
|
||||
/// 脱敏后的可操作消息。
|
||||
#[must_use]
|
||||
pub fn message(&self) -> &str {
|
||||
&self.message
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造稳定错误码常量。
|
||||
pub mod code {
|
||||
/// 配置或参数非法。
|
||||
pub const INVALID_CONFIG: &str = "backup.invalid_config";
|
||||
/// Cron 表达式非法。
|
||||
pub const INVALID_CRON: &str = "backup.invalid_cron";
|
||||
/// IANA 时区非法。
|
||||
pub const INVALID_TIMEZONE: &str = "backup.invalid_timezone";
|
||||
/// 记录不存在。
|
||||
pub const RECORD_NOT_FOUND: &str = "backup.record_not_found";
|
||||
/// 已存在活跃任务,拒绝创建。
|
||||
pub const ACTIVE_TASK_CONFLICT: &str = "backup.active_task_conflict";
|
||||
/// 记录状态与期望不符,无法迁移。
|
||||
pub const STATE_CONFLICT: &str = "backup.state_conflict";
|
||||
/// 已有记录时禁止切换存储目标。
|
||||
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
|
||||
/// `pg_dump` 非零退出或失败。
|
||||
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
|
||||
/// 暂存磁盘空间不足。
|
||||
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
|
||||
/// S3 认证失败。
|
||||
pub const S3_AUTH_FAILED: &str = "backup.s3_auth_failed";
|
||||
/// S3 权限拒绝。
|
||||
pub const S3_PERMISSION_DENIED: &str = "backup.s3_permission_denied";
|
||||
/// S3 上传失败。
|
||||
pub const S3_UPLOAD_FAILED: &str = "backup.s3_upload_failed";
|
||||
/// 远端对象校验失败。
|
||||
pub const REMOTE_VERIFICATION_FAILED: &str = "backup.remote_verification_failed";
|
||||
/// 任务被取消。
|
||||
pub const CANCELLED: &str = "backup.cancelled";
|
||||
/// 仓储或对象存储暂不可用。
|
||||
pub const STORE_UNAVAILABLE: &str = "backup.store_unavailable";
|
||||
/// 上游 S3 服务返回无效或失败响应。
|
||||
pub const UPSTREAM_FAILURE: &str = "backup.upstream_failure";
|
||||
}
|
||||
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`。
|
||||
///
|
||||
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
|
||||
/// 输出统一使用 `/` 分隔且不含空段。
|
||||
pub fn build_object_key(
|
||||
prefix: &str,
|
||||
backup_id: &str,
|
||||
at: DateTime<Utc>,
|
||||
) -> Result<String, BackupError> {
|
||||
let prefix = normalize_prefix(prefix)?;
|
||||
Ok(format!(
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
|
||||
at.format("%Y"),
|
||||
at.format("%m"),
|
||||
at.format("%d"),
|
||||
at.format("%Y%m%d_%H%M%S"),
|
||||
short_backup_id(backup_id),
|
||||
))
|
||||
}
|
||||
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致。
|
||||
#[must_use]
|
||||
pub fn build_download_file_name(backup_id: &str) -> String {
|
||||
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
|
||||
}
|
||||
|
||||
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex。
|
||||
#[must_use]
|
||||
fn short_backup_id(backup_id: &str) -> String {
|
||||
let hex = backup_id.strip_prefix("backup_").unwrap_or(backup_id);
|
||||
hex.get(..8).unwrap_or(hex).to_owned()
|
||||
}
|
||||
|
||||
/// 规范化并校验对象 prefix。
|
||||
///
|
||||
/// 返回去掉尾部 `/` 的规范 prefix;空串、前导 `/`、`..` 段、反斜杠和控制字符均拒绝。
|
||||
pub fn normalize_prefix(prefix: &str) -> Result<String, BackupError> {
|
||||
if prefix.is_empty() || prefix.starts_with('/') {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀必须非空且不能以前导 / 开头".to_owned(),
|
||||
));
|
||||
}
|
||||
if prefix.bytes().any(|byte| byte.is_ascii_control()) || prefix.contains('\\') {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀包含非法字符".to_owned(),
|
||||
));
|
||||
}
|
||||
if prefix.split('/').any(|segment| segment == "..") {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀不能包含 .. 路径段".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(prefix.trim_end_matches('/').to_owned())
|
||||
}
|
||||
|
||||
/// 生成新的备份记录 id。
|
||||
#[must_use]
|
||||
pub fn new_backup_id() -> String {
|
||||
format!("backup_{}", Uuid::now_v7().simple())
|
||||
}
|
||||
|
||||
/// 从触发类型与计划时间构造可持久化 seed;id 与对象 key 一次生成保持一致。
|
||||
pub fn build_backup_seed(
|
||||
trigger_kind: BackupTriggerKind,
|
||||
scheduled_at: Option<DateTime<Utc>>,
|
||||
prefix: &str,
|
||||
at: DateTime<Utc>,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecordSeed, BackupError> {
|
||||
let id = new_backup_id();
|
||||
let object_key = build_object_key(prefix, &id, at)?;
|
||||
Ok(BackupRecordSeed {
|
||||
id,
|
||||
trigger_kind,
|
||||
scheduled_at,
|
||||
object_key,
|
||||
expires_at,
|
||||
})
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroU64};
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod provider_credentials;
|
||||
@@ -50,6 +51,11 @@ impl AdminError {
|
||||
Self::new(AdminErrorKind::Invalid, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn bad_request(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::Invalid, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn not_found(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::NotFound, message)
|
||||
@@ -60,6 +66,11 @@ impl AdminError {
|
||||
Self::new(AdminErrorKind::Conflict, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn bad_gateway(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::BadGateway, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::Unavailable, message)
|
||||
|
||||
@@ -204,42 +204,22 @@ impl DecimalAmount {
|
||||
/// 精确相加两个 `numeric(20,10)` 金额。
|
||||
#[must_use]
|
||||
pub fn checked_add(&self, other: &Self) -> Option<Self> {
|
||||
let sum = scaled_amount(&self.0)?.checked_add(scaled_amount(&other.0)?)?;
|
||||
Self::from_scaled(sum)
|
||||
Self::from_decimal(self.to_decimal()?.checked_add(other.to_decimal()?)?)
|
||||
}
|
||||
|
||||
/// 将金额按非零请求数均分,保留最多十位小数。
|
||||
#[must_use]
|
||||
pub fn checked_div_u64(&self, divisor: u64) -> Option<Self> {
|
||||
let divisor = u128::from(divisor);
|
||||
(divisor != 0)
|
||||
.then(|| scaled_amount(&self.0)?.checked_div(divisor))
|
||||
.flatten()
|
||||
.and_then(Self::from_scaled)
|
||||
Self::from_decimal(self.to_decimal()?.checked_div_u64(divisor)?)
|
||||
}
|
||||
|
||||
fn from_scaled(value: u128) -> Option<Self> {
|
||||
if value >= 10_u128.pow(20) {
|
||||
return None;
|
||||
}
|
||||
let whole = value / 10_u128.pow(10);
|
||||
let fraction = value % 10_u128.pow(10);
|
||||
let value = if fraction == 0 {
|
||||
whole.to_string()
|
||||
} else {
|
||||
format!("{whole}.{fraction:010}")
|
||||
.trim_end_matches('0')
|
||||
.to_owned()
|
||||
};
|
||||
Some(Self(value))
|
||||
fn to_decimal(&self) -> Option<gateway_core::accounting::Decimal> {
|
||||
self.0.parse().ok()
|
||||
}
|
||||
}
|
||||
|
||||
fn scaled_amount(value: &str) -> Option<u128> {
|
||||
let (whole, fraction) = value.split_once('.').unwrap_or((value, ""));
|
||||
let whole = whole.parse::<u128>().ok()?;
|
||||
let fraction = format!("{fraction:0<10}").parse::<u128>().ok()?;
|
||||
whole.checked_mul(10_u128.pow(10))?.checked_add(fraction)
|
||||
fn from_decimal(value: gateway_core::accounting::Decimal) -> Option<Self> {
|
||||
Some(Self(value.canonical()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for DecimalAmount {
|
||||
@@ -460,13 +440,20 @@ pub struct RequestMetricPoint {
|
||||
}
|
||||
|
||||
/// 账号池容量统计。
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
@@ -645,6 +632,9 @@ pub struct UsageAttempt {
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_send_state: Option<String>,
|
||||
@@ -705,6 +695,7 @@ pub struct UsageSummary {
|
||||
/// 单个诊断维度值的聚合结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DiagnosticObservation {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -712,6 +703,7 @@ pub struct DiagnosticObservation {
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
}
|
||||
@@ -726,8 +718,12 @@ pub struct OpsError {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
@@ -1021,6 +1017,7 @@ pub struct UsageInsights {
|
||||
/// 已完成分母计算的诊断项。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsItem {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -1028,6 +1025,7 @@ pub struct DiagnosticsItem {
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
|
||||
@@ -11,7 +11,10 @@ use gateway_core::{
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -153,8 +156,6 @@ pub struct PreparedCredentialCreate {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -573,6 +574,7 @@ pub struct ProviderQuotaWindow {
|
||||
pub window_seconds: Option<u64>,
|
||||
pub used_percent: Option<f64>,
|
||||
pub reset_at: Option<DateTime<Utc>>,
|
||||
pub limit_reached: bool,
|
||||
pub local_usage: Option<AccountUsage>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
@@ -591,6 +593,11 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
@@ -687,6 +694,8 @@ impl fmt::Debug for ProviderExport {
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -11,14 +11,8 @@ use super::Revision;
|
||||
/// 客户端模型到上游模型的全局精确映射。
|
||||
pub type ModelMappings = BTreeMap<PublicModelId, UpstreamModelId>;
|
||||
|
||||
/// 账号调度策略。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RotationStrategy {
|
||||
Smart,
|
||||
QuotaResetPriority,
|
||||
RoundRobin,
|
||||
Sticky,
|
||||
}
|
||||
/// 账号调度策略;由 Core 拥有稳定值与 wire 映射。
|
||||
pub use gateway_core::engine::credential::RotationStrategy;
|
||||
|
||||
/// 完整运行设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
//! 备份控制面所需的持久化与基础设施能力。
|
||||
//!
|
||||
//! 端口方法只接受/返回领域类型与既有错误体系:`BackupRepository` 复用
|
||||
//! `AdminStoreError`;`DatabaseDumpPort` 与 `BackupObjectStorePort` 返回只携带稳定
|
||||
//! 错误码的 [`BackupError`]。`gateway-store` 实现全部端口;`gateway-admin` 拥有
|
||||
//! 状态机、Cron、保留策略与 BackupTask 算法。
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::model::Revision;
|
||||
use crate::model::{
|
||||
MutationContext,
|
||||
backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
|
||||
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, ConnectionTestResult,
|
||||
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
},
|
||||
};
|
||||
use crate::ports::store::AdminStoreResult;
|
||||
|
||||
/// Worker 状态迁移携带的字段更新;`None` 字段不写入。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct StatusTransitionUpdate {
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 备份配置、任务与计划游标的 PostgreSQL 仓储端口。
|
||||
#[async_trait]
|
||||
pub trait BackupRepository: Send + Sync {
|
||||
/// 读取完整设置(含 Secret)。
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings>;
|
||||
|
||||
/// 同事务更新 S3 存储配置、递增 revision、清空验证状态并写审计事件。
|
||||
///
|
||||
/// 已有记录时禁止切换 endpoint/region/bucket/path-style(存储身份锁定)。
|
||||
/// 返回更新后的设置与新的全局 `config_revision`。
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
command: UpdateBackupStorageCommand,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)>;
|
||||
|
||||
/// 同事务更新调度配置、推进计划游标并写审计事件。
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
next_run_at: Option<DateTime<Utc>>,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings>;
|
||||
|
||||
/// CAS 记录探针成功;`storage_revision` 与当前配置不一致时丢弃结果(返回 `false`)。
|
||||
async fn record_verification(
|
||||
&self,
|
||||
storage_revision: u64,
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 插入 queued 任务;存在活跃任务时返回 `backup.active_task_conflict`。
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord>;
|
||||
|
||||
/// 幂等插入 scheduled 任务;同一时间点或活跃冲突时返回 `false`(调用方推进游标即可)。
|
||||
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 分页查询记录。
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage>;
|
||||
|
||||
/// 读取单条记录。
|
||||
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 读取中间状态记录(dumping/uploading),供启动恢复。
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 读取等待完成墓碑的记录(status = deleting)。
|
||||
async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 读取 `expires_at` 已到期且非活跃的记录(completed/failed),供保留清理。
|
||||
async fn list_expired_records(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 原子领取最旧的 queued 任务并迁移到 dumping;无任务返回 `None`。
|
||||
async fn claim_next_queued(&self, now: DateTime<Utc>)
|
||||
-> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 条件迁移状态:要求当前状态一致;不匹配返回 `None`。
|
||||
async fn transition_status(
|
||||
&self,
|
||||
id: &str,
|
||||
from: BackupStatus,
|
||||
to: BackupStatus,
|
||||
update: StatusTransitionUpdate,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 迁移到删除流程(completed/failed → deleting);供管理员与保留策略调用。
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 远端删除成功后硬删除记录行。
|
||||
async fn delete_record(&self, id: &str) -> AdminStoreResult<()>;
|
||||
|
||||
/// 条件推进计划游标;仅当计划表达式和时区仍是 `expected_*` 时才写入,返回是否推进。
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
next_run_at: DateTime<Utc>,
|
||||
expected_cron: &str,
|
||||
expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 按完成时间倒序读取 completed 计划备份,供保留策略扫描(最多 `limit` 条)。
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
}
|
||||
|
||||
/// `pg_dump` 导出与本地暂存端口。
|
||||
#[async_trait]
|
||||
pub trait DatabaseDumpPort: Send + Sync {
|
||||
/// 创建归档;完成后原子落盘。取消时终止子进程并清理部分文件。
|
||||
async fn dump(&self, request: DumpRequest) -> Result<DumpArtifact, BackupError>;
|
||||
|
||||
/// 检查暂存目录是否存在完整归档。
|
||||
async fn inspect_staging(&self, backup_id: &str)
|
||||
-> Result<Option<StagedArtifact>, BackupError>;
|
||||
|
||||
/// 清理暂存目录中该备份的全部文件。
|
||||
async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError>;
|
||||
}
|
||||
|
||||
/// 一次 `pg_dump` 请求。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DumpRequest {
|
||||
pub backup_id: String,
|
||||
pub cancellation: gateway_core::engine::CancellationToken,
|
||||
}
|
||||
|
||||
/// 已完成并落盘的归档事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DumpArtifact {
|
||||
pub path: PathBuf,
|
||||
pub size_bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
/// 暂存目录中已存在的完整归档。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StagedArtifact {
|
||||
pub path: PathBuf,
|
||||
pub size_bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
/// S3 兼容对象存储适配器端口。
|
||||
///
|
||||
/// 方法接受已经校验的存储配置;SDK client 的构造与缓存属于 adapter 实现细节。
|
||||
#[async_trait]
|
||||
pub trait BackupObjectStorePort: Send + Sync {
|
||||
/// 对专用探针对象执行 Put/Head/Get/Delete。
|
||||
async fn test_connection(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, BackupError>;
|
||||
|
||||
/// 分片流式上传本地归档;取消时中止 multipart 并返回 `backup.cancelled`。
|
||||
async fn upload_file(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), BackupError>;
|
||||
|
||||
/// 读取对象大小与 metadata;对象不存在返回 `Ok(None)`。
|
||||
async fn head_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, BackupError>;
|
||||
|
||||
/// 删除对象;对象不存在视为成功。
|
||||
async fn delete_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), BackupError>;
|
||||
|
||||
/// 创建短时预签名下载 URL。
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
file_name: &str,
|
||||
ttl: Duration,
|
||||
) -> Result<String, BackupError>;
|
||||
}
|
||||
|
||||
/// 一次对象上传请求。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UploadObjectRequest {
|
||||
pub object_key: String,
|
||||
pub source: PathBuf,
|
||||
pub metadata: BackupObjectMetadata,
|
||||
pub cancellation: gateway_core::engine::CancellationToken,
|
||||
}
|
||||
|
||||
/// 备份控制面所需的全部 Store 能力集合。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupStorePorts {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
}
|
||||
|
||||
impl BackupStorePorts {
|
||||
/// 组合备份端口。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
dump,
|
||||
object_store,
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份任务/配置仓储。
|
||||
#[must_use]
|
||||
pub fn repository(&self) -> Arc<dyn BackupRepository> {
|
||||
self.repository.clone()
|
||||
}
|
||||
|
||||
/// `pg_dump` 导出与暂存。
|
||||
#[must_use]
|
||||
pub fn dump(&self) -> Arc<dyn DatabaseDumpPort> {
|
||||
self.dump.clone()
|
||||
}
|
||||
|
||||
/// S3/R2 对象存储适配器。
|
||||
#[must_use]
|
||||
pub fn object_store(&self) -> Arc<dyn BackupObjectStorePort> {
|
||||
self.object_store.clone()
|
||||
}
|
||||
|
||||
/// 测试与不可用路径使用的空实现;任何调用都返回不可用错误。
|
||||
#[doc(hidden)]
|
||||
#[must_use]
|
||||
pub fn disabled() -> Self {
|
||||
Self {
|
||||
repository: Arc::new(UnavailableBackupRepository),
|
||||
dump: Arc::new(UnavailableDumpPort),
|
||||
object_store: Arc::new(UnavailableObjectStore),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableBackupRepository;
|
||||
|
||||
#[async_trait]
|
||||
impl BackupRepository for UnavailableBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
_command: UpdateBackupStorageCommand,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
_command: UpdateBackupScheduleCommand,
|
||||
_next_run_at: Option<DateTime<Utc>>,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn record_verification(
|
||||
&self,
|
||||
_storage_revision: u64,
|
||||
_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn insert_backup_record(
|
||||
&self,
|
||||
_seed: BackupRecordSeed,
|
||||
) -> AdminStoreResult<BackupRecord> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn insert_scheduled_record(&self, _seed: BackupRecordSeed) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
_query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn load_backup_record(&self, _id: &str) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn claim_next_queued(
|
||||
&self,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn transition_status(
|
||||
&self,
|
||||
_id: &str,
|
||||
_from: BackupStatus,
|
||||
_to: BackupStatus,
|
||||
_update: StatusTransitionUpdate,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
_id: &str,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn delete_record(&self, _id: &str) -> AdminStoreResult<()> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
_next_run_at: DateTime<Utc>,
|
||||
_expected_cron: &str,
|
||||
_expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableDumpPort;
|
||||
|
||||
#[async_trait]
|
||||
impl DatabaseDumpPort for UnavailableDumpPort {
|
||||
async fn dump(&self, _request: DumpRequest) -> Result<DumpArtifact, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn inspect_staging(
|
||||
&self,
|
||||
_backup_id: &str,
|
||||
) -> Result<Option<StagedArtifact>, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn cleanup_staging(&self, _backup_id: &str) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableObjectStore;
|
||||
|
||||
#[async_trait]
|
||||
impl BackupObjectStorePort for UnavailableObjectStore {
|
||||
async fn test_connection(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn upload_file(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_request: UploadObjectRequest,
|
||||
) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn head_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn delete_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
_file_name: &str,
|
||||
_ttl: std::time::Duration,
|
||||
) -> Result<String, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
}
|
||||
|
||||
fn disabled_store() -> crate::ports::store::AdminStoreError {
|
||||
crate::ports::store::AdminStoreError::new(
|
||||
crate::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"backup",
|
||||
"backup store is disabled in this configuration",
|
||||
)
|
||||
}
|
||||
|
||||
fn disabled_infra() -> BackupError {
|
||||
BackupError::new(
|
||||
crate::model::backup::code::STORE_UNAVAILABLE,
|
||||
"backup infrastructure is disabled in this configuration".to_owned(),
|
||||
)
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 管理用例依赖的外部能力端口。
|
||||
|
||||
pub mod backup;
|
||||
pub mod provider;
|
||||
pub mod store;
|
||||
pub mod system;
|
||||
|
||||
@@ -7,6 +7,7 @@ use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
@@ -300,6 +301,7 @@ pub struct AdminStorePorts {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
}
|
||||
|
||||
impl AdminStorePorts {
|
||||
@@ -310,6 +312,7 @@ impl AdminStorePorts {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
@@ -317,6 +320,7 @@ impl AdminStorePorts {
|
||||
client_keys,
|
||||
observability,
|
||||
settings,
|
||||
backup,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,4 +348,9 @@ impl AdminStorePorts {
|
||||
pub fn settings(&self) -> Arc<dyn SettingsStore> {
|
||||
self.settings.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn backup(&self) -> BackupStorePorts {
|
||||
self.backup.clone()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,7 +18,8 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
@@ -216,8 +217,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -260,18 +264,37 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
provider
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
// 该账号降级为空额度投影,其余账号与页面状态照常返回。
|
||||
let provider = match self.providers.require(&account.provider_kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Ok(empty_quota());
|
||||
}
|
||||
};
|
||||
match provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: rolling_usage.get(&account.id).cloned(),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))
|
||||
{
|
||||
Ok(quota) => Ok(quota),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
Ok(empty_quota())
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -282,11 +305,16 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -407,7 +435,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, Utc::now());
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -441,7 +469,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, Utc::now())
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
@@ -486,31 +514,48 @@ impl AccountsService for DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted
|
||||
|| (account.availability == crate::model::accounts::AccountAvailability::Cooldown
|
||||
&& account.availability_reason.as_deref() == Some("usage_limit_exhausted")
|
||||
&& account.cooldown_until.is_none_or(|until| until > now))
|
||||
{
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else {
|
||||
AccountStatus::Active
|
||||
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
//! 备份配置、手动创建、下载与删除用例。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::routing::snapshot::SnapshotControl;
|
||||
use secrecy::ExposeSecret as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind, MutationContext,
|
||||
auth::{AdminAuditEvent, AuditActorKind},
|
||||
backup::{
|
||||
BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings,
|
||||
BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult,
|
||||
DownloadUrlResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
build_backup_seed, code,
|
||||
},
|
||||
};
|
||||
use crate::ports::{
|
||||
backup::{BackupObjectStorePort, BackupRepository},
|
||||
store::AuthStore,
|
||||
};
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
/// 默认下载地址有效期。
|
||||
const DOWNLOAD_TTL: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// API 消费的备份管理服务。
|
||||
#[async_trait]
|
||||
pub trait BackupService: Send + Sync {
|
||||
async fn load_settings(&self) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn update_storage(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupStorageCommand,
|
||||
) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn update_schedule(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<ConnectionTestResult, AdminError>;
|
||||
|
||||
async fn create_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecord, AdminError>;
|
||||
|
||||
async fn list_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> Result<BackupRecordPage, AdminError>;
|
||||
|
||||
async fn download_url(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<DownloadUrlResult, AdminError>;
|
||||
|
||||
async fn delete_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<BackupRecord, AdminError>;
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultBackupService {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
}
|
||||
|
||||
impl DefaultBackupService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
object_store,
|
||||
auth,
|
||||
snapshot,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupService for DefaultBackupService {
|
||||
async fn load_settings(&self) -> Result<BackupSettings, AdminError> {
|
||||
self.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))
|
||||
}
|
||||
|
||||
async fn update_storage(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupStorageCommand,
|
||||
) -> Result<BackupSettings, AdminError> {
|
||||
validate_storage(&command)?;
|
||||
let (settings, revision) = self
|
||||
.repository
|
||||
.update_storage_settings(command, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup storage"))?;
|
||||
super::publish_committed(self.snapshot.as_ref(), revision).await?;
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
async fn update_schedule(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
) -> Result<BackupSettings, AdminError> {
|
||||
validate_schedule(&command)?;
|
||||
let next_run_at = if command.schedule_enabled {
|
||||
Some(
|
||||
crate::backup::policy::BackupSchedule::parse(
|
||||
&command.cron_expression,
|
||||
&command.schedule_timezone,
|
||||
)
|
||||
.map_err(map_backup_error)?
|
||||
.next_after(Utc::now())
|
||||
.ok_or_else(|| AdminError::bad_request("无法从计划推导下次执行时间"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
self.repository
|
||||
.update_schedule_settings(command, next_run_at, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup schedule"))
|
||||
}
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<ConnectionTestResult, AdminError> {
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: "putObject",
|
||||
code: Some(code::INVALID_CONFIG),
|
||||
message: "备份存储配置不完整,无法执行连接测试".to_owned(),
|
||||
});
|
||||
};
|
||||
let result = self
|
||||
.object_store
|
||||
.test_connection(&config)
|
||||
.await
|
||||
.map_err(map_backup_error)?;
|
||||
if result.ok {
|
||||
let _ = self
|
||||
.repository
|
||||
.record_verification(config.storage_revision, Utc::now())
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
}
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.s3_connection_tested",
|
||||
"backup_settings",
|
||||
vec![],
|
||||
)
|
||||
.await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn create_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecord, AdminError> {
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
if !settings.storage_configured() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储配置不完整,请先保存存储配置",
|
||||
));
|
||||
}
|
||||
if !settings.storage_verified() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储尚未通过连接测试,请先测试存储配置",
|
||||
));
|
||||
}
|
||||
let prefix = settings.prefix.as_deref().unwrap_or_default();
|
||||
let seed = build_backup_seed(
|
||||
BackupTriggerKind::Manual,
|
||||
None,
|
||||
prefix,
|
||||
Utc::now(),
|
||||
expires_at,
|
||||
)
|
||||
.map_err(map_backup_error)?;
|
||||
let record = self
|
||||
.repository
|
||||
.insert_backup_record(seed)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?;
|
||||
self.append_audit(context, "backup.created", "backup", vec![record.id.clone()])
|
||||
.await?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn list_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> Result<BackupRecordPage, AdminError> {
|
||||
self.repository
|
||||
.list_backup_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup records"))
|
||||
}
|
||||
|
||||
async fn download_url(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<DownloadUrlResult, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if record.status != BackupStatus::Completed {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 的备份才能创建下载地址",
|
||||
));
|
||||
}
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Err(AdminError::bad_request("备份存储配置不完整"));
|
||||
};
|
||||
let file_name = crate::model::backup::build_download_file_name(backup_id);
|
||||
let url = self
|
||||
.object_store
|
||||
.presigned_download(&config, &record.object_key, &file_name, DOWNLOAD_TTL)
|
||||
.await
|
||||
.map_err(map_backup_error)?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.download_url_created",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
)
|
||||
.await?;
|
||||
Ok(DownloadUrlResult {
|
||||
url,
|
||||
file_name,
|
||||
expires_in: DOWNLOAD_TTL,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<BackupRecord, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if !record.status.can_be_deleted() {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 或 failed 的备份才能删除",
|
||||
));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let deleting = self
|
||||
.repository
|
||||
.transition_to_deleting(backup_id, now)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.delete_requested",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
)
|
||||
.await?;
|
||||
Ok(deleting)
|
||||
}
|
||||
}
|
||||
|
||||
impl DefaultBackupService {
|
||||
async fn require_record(&self, backup_id: &str) -> Result<BackupRecord, AdminError> {
|
||||
self.repository
|
||||
.load_backup_record(backup_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::not_found("备份记录不存在"))
|
||||
}
|
||||
|
||||
async fn append_audit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
entity_ref: &str,
|
||||
changed_fields: Vec<String>,
|
||||
) -> Result<(), AdminError> {
|
||||
let actor_kind = match context.actor {
|
||||
crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession,
|
||||
crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey,
|
||||
crate::model::MutationActor::System => AuditActorKind::System,
|
||||
};
|
||||
let actor_ref = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => admin_user_id.clone(),
|
||||
crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(),
|
||||
crate::model::MutationActor::System => "system".to_owned(),
|
||||
};
|
||||
let actor_admin_user_id = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => {
|
||||
Some(admin_user_id.clone())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let event = AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
actor_kind,
|
||||
actor_admin_user_id,
|
||||
actor_ref,
|
||||
request_id: Some(context.request_id.clone()),
|
||||
action: action.to_owned(),
|
||||
entity_kind: "backup".to_owned(),
|
||||
entity_ref: entity_ref.to_owned(),
|
||||
config_revision: None,
|
||||
changed_fields,
|
||||
occurred_at: Utc::now(),
|
||||
};
|
||||
self.auth
|
||||
.append_audit_event(event)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup audit"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验存储配置;空串视为非法,Secret 缺省表示保留旧值。
|
||||
fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminError> {
|
||||
validate_endpoint(&command.endpoint)?;
|
||||
validate_nonempty("region", &command.region)?;
|
||||
validate_nonempty("bucket", &command.bucket)?;
|
||||
validate_nonempty("accessKeyId", &command.access_key_id)?;
|
||||
if let Some(secret) = command.secret_access_key.as_ref() {
|
||||
let value = secret.expose_secret();
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request(
|
||||
"secretAccessKey 不能为空字符串;缺省表示保留旧值",
|
||||
));
|
||||
}
|
||||
validate_length("secretAccessKey", value, 512)?;
|
||||
}
|
||||
crate::model::backup::normalize_prefix(&command.prefix).map_err(map_backup_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_endpoint(value: &str) -> Result<(), AdminError> {
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request("endpoint 不能为空"));
|
||||
}
|
||||
if !value.starts_with("https://") {
|
||||
return Err(AdminError::bad_request(
|
||||
"endpoint 必须使用 HTTPS;仅测试或显式开发配置允许 HTTP",
|
||||
));
|
||||
}
|
||||
if value.contains('@') || value.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(AdminError::bad_request("endpoint 格式非法"));
|
||||
}
|
||||
validate_length("endpoint", value, 2048)
|
||||
}
|
||||
|
||||
fn validate_nonempty(field: &str, value: &str) -> Result<(), AdminError> {
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request(format!("{field} 不能为空")));
|
||||
}
|
||||
validate_length(field, value, 255)
|
||||
}
|
||||
|
||||
fn validate_length(field: &str, value: &str, max: usize) -> Result<(), AdminError> {
|
||||
if value.len() > max || value.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(AdminError::bad_request(format!(
|
||||
"{field} 超出长度或包含非法字符"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验调度配置并解析 Cron 与时区。
|
||||
fn validate_schedule(command: &UpdateBackupScheduleCommand) -> Result<(), AdminError> {
|
||||
if command.cron_expression.trim().is_empty() {
|
||||
return Err(AdminError::bad_request("cronExpression 不能为空"));
|
||||
}
|
||||
crate::backup::policy::BackupSchedule::parse(
|
||||
&command.cron_expression,
|
||||
&command.schedule_timezone,
|
||||
)
|
||||
.map_err(map_backup_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 备份基础设施错误到既有 `AdminErrorKind` 的唯一边界映射。
|
||||
fn map_backup_error(error: BackupError) -> AdminError {
|
||||
match error.code() {
|
||||
code::INVALID_CONFIG | code::INVALID_CRON | code::INVALID_TIMEZONE => {
|
||||
AdminError::bad_request(error.message().to_owned())
|
||||
}
|
||||
code::RECORD_NOT_FOUND => AdminError::not_found(error.message().to_owned()),
|
||||
code::ACTIVE_TASK_CONFLICT | code::STATE_CONFLICT | code::STORAGE_IDENTITY_LOCKED => {
|
||||
AdminError::conflict(error.message().to_owned())
|
||||
}
|
||||
code::STORE_UNAVAILABLE => {
|
||||
AdminError::new(AdminErrorKind::Unavailable, error.message().to_owned())
|
||||
}
|
||||
_ => AdminError::bad_gateway(error.message().to_owned()),
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod openai;
|
||||
|
||||
@@ -247,6 +247,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
items: items
|
||||
.into_iter()
|
||||
.map(|item| DiagnosticsItem {
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
request_count: item.request_count,
|
||||
success_count: item.request_count.saturating_sub(item.failure_count),
|
||||
@@ -254,6 +255,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
error_rate: rate_or_zero(item.failure_count, item.request_count),
|
||||
request_share: rate_or_zero(item.request_count, total_requests),
|
||||
average_latency_ms: item.average_latency_ms,
|
||||
latency_p95_ms: item.latency_p95_ms,
|
||||
estimated_cost: usd_cost(&item.costs),
|
||||
attempt_count: item.attempt_count,
|
||||
total_tokens: item.total_tokens,
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
//! 备份模块测试:镜像 src/backup/ 的 policy 与 task。
|
||||
|
||||
mod policy;
|
||||
pub mod support;
|
||||
mod task;
|
||||
@@ -0,0 +1,171 @@
|
||||
//! 备份调度与保留策略(backup/policy.rs)的纯逻辑测试。
|
||||
|
||||
use chrono::{Duration, TimeZone as _, Utc};
|
||||
|
||||
use gateway_admin::{
|
||||
backup::policy::{BackupSchedule, RetentionReason, decide_retention},
|
||||
model::backup::{BackupRecord, BackupStatus, BackupTriggerKind},
|
||||
};
|
||||
|
||||
fn record(
|
||||
id: &str,
|
||||
trigger: BackupTriggerKind,
|
||||
status: BackupStatus,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
completed_at: Option<chrono::DateTime<Utc>>,
|
||||
) -> BackupRecord {
|
||||
BackupRecord {
|
||||
id: id.to_owned(),
|
||||
trigger_kind: trigger,
|
||||
status,
|
||||
scheduled_at: if trigger == BackupTriggerKind::Scheduled {
|
||||
Some(created_at)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
object_key: format!("k/{id}"),
|
||||
size_bytes: Some(1),
|
||||
sha256: Some("a".repeat(64)),
|
||||
attempt_count: 0,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
started_at: Some(created_at),
|
||||
completed_at,
|
||||
expires_at: None,
|
||||
created_at,
|
||||
updated_at: created_at,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_bad_cron_and_timezone() {
|
||||
assert!(BackupSchedule::parse("61 * * * *", "Asia/Shanghai").is_err());
|
||||
assert!(BackupSchedule::parse("0 2 * * *", "Not/AZone").is_err());
|
||||
assert!(BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_after_is_strictly_later() {
|
||||
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
|
||||
let at = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
|
||||
let next = schedule.next_after(at).unwrap();
|
||||
assert_eq!(next, Utc.with_ymd_and_hms(2026, 8, 2, 18, 0, 0).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn last_firing_is_at_or_before() {
|
||||
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
|
||||
// 北京时间 02:00 = 前一日 18:00 UTC。
|
||||
let before = Utc.with_ymd_and_hms(2026, 8, 1, 17, 59, 0).unwrap();
|
||||
assert_eq!(
|
||||
schedule.last_firing_at_or_before(before).unwrap(),
|
||||
Utc.with_ymd_and_hms(2026, 7, 31, 18, 0, 0).unwrap()
|
||||
);
|
||||
let exact = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
|
||||
assert_eq!(schedule.last_firing_at_or_before(exact).unwrap(), exact);
|
||||
let after = Utc.with_ymd_and_hms(2026, 8, 1, 19, 0, 0).unwrap();
|
||||
assert_eq!(schedule.last_firing_at_or_before(after).unwrap(), exact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dst_spring_forward_skips_missing_hour() {
|
||||
let schedule = BackupSchedule::parse("30 2 * * *", "America/New_York").unwrap();
|
||||
let at = Utc.with_ymd_and_hms(2026, 3, 8, 6, 29, 0).unwrap(); // 01:29 EST
|
||||
let next = schedule.next_after(at).unwrap();
|
||||
assert_eq!(next, Utc.with_ymd_and_hms(2026, 3, 9, 6, 30, 0).unwrap());
|
||||
let at_after_spring = Utc.with_ymd_and_hms(2026, 3, 8, 10, 0, 0).unwrap(); // 06:00 EDT
|
||||
let last = schedule.last_firing_at_or_before(at_after_spring).unwrap();
|
||||
assert_eq!(last, Utc.with_ymd_and_hms(2026, 3, 7, 7, 30, 0).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_count_keeps_newest_n() {
|
||||
let now = Utc::now();
|
||||
let base = now - Duration::days(10);
|
||||
let records = vec![
|
||||
record(
|
||||
"newest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base + Duration::days(2),
|
||||
Some(base + Duration::days(2)),
|
||||
),
|
||||
record(
|
||||
"middle",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base + Duration::days(1),
|
||||
Some(base + Duration::days(1)),
|
||||
),
|
||||
record(
|
||||
"oldest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base,
|
||||
Some(base),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(0, 2, now, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["oldest"]);
|
||||
assert_eq!(decisions[0].reason, RetentionReason::Count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_days_deletes_old_scheduled_but_keeps_newest() {
|
||||
let now = Utc::now();
|
||||
let records = vec![
|
||||
record(
|
||||
"recent",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(1),
|
||||
Some(now - Duration::days(1)),
|
||||
),
|
||||
record(
|
||||
"ancient",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(40),
|
||||
Some(now - Duration::days(40)),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(30, 0, now, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["ancient"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newest_scheduled_never_auto_deleted_even_when_ancient() {
|
||||
let now = Utc::now();
|
||||
let records = vec![record(
|
||||
"sole",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(400),
|
||||
Some(now - Duration::days(400)),
|
||||
)];
|
||||
assert!(decide_retention(30, 1, now, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_thresholds_delete_nothing() {
|
||||
let now = Utc::now();
|
||||
let records = vec![
|
||||
record(
|
||||
"newest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(2),
|
||||
Some(now - Duration::days(2)),
|
||||
),
|
||||
record(
|
||||
"old",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(90),
|
||||
Some(now - Duration::days(90)),
|
||||
),
|
||||
];
|
||||
assert!(decide_retention(0, 0, now, &records).is_empty());
|
||||
}
|
||||
@@ -0,0 +1,664 @@
|
||||
//! 备份测试共享 fixture:内存 fake 端口与辅助函数。
|
||||
//!
|
||||
//! 用内存 fake 端口驱动编排层,不依赖 MinIO/真实 S3。
|
||||
//! s3.rs / pg_dump.rs 是 SDK 薄适配器,协议正确性交给 aws-sdk / pg_dump 保证。
|
||||
//! 供 backup::task 与 use_case::backup 的测试复用。
|
||||
|
||||
use std::sync::Mutex;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
MutationActor, MutationContext, Revision,
|
||||
auth::AdminAuditEvent,
|
||||
backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
|
||||
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
|
||||
UpdateBackupStorageCommand,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
backup::{
|
||||
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpArtifact, DumpRequest,
|
||||
StagedArtifact, StatusTransitionUpdate, UploadObjectRequest,
|
||||
},
|
||||
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult, AuthStore},
|
||||
},
|
||||
};
|
||||
use secrecy::SecretString;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// 假归档内容;dump 与上传校验共用同一字节与校验值。
|
||||
pub(crate) const DUMP_CONTENT: &[u8] = b"custom-format-archive-bytes-0123456789";
|
||||
|
||||
/// 计算一段内容的 SHA-256 十六进制。
|
||||
pub(crate) fn sha256_hex(content: &[u8]) -> String {
|
||||
hex::encode(Sha256::digest(content))
|
||||
}
|
||||
|
||||
/// 返回 32 位十六进制的合法备份记录 id。
|
||||
pub(crate) fn backup_id(suffix: &str) -> String {
|
||||
format!("backup_{}", sha256_hex(suffix.as_bytes()))
|
||||
}
|
||||
|
||||
/// 已配置且已通过连接测试的设置快照。
|
||||
pub(crate) fn configured_settings() -> BackupSettings {
|
||||
let now = Utc::now();
|
||||
BackupSettings {
|
||||
storage_revision: 1,
|
||||
endpoint: Some("https://s3.example.com".to_owned()),
|
||||
region: Some("auto".to_owned()),
|
||||
bucket: Some("backup-bucket".to_owned()),
|
||||
access_key_id: Some("ak".to_owned()),
|
||||
secret_access_key: Some(SecretString::from("sk-value")),
|
||||
prefix: Some("codex".to_owned()),
|
||||
force_path_style: false,
|
||||
schedule_enabled: false,
|
||||
cron_expression: None,
|
||||
schedule_timezone: None,
|
||||
retention_days: 0,
|
||||
retention_count: 0,
|
||||
next_run_at: None,
|
||||
last_verified_at: Some(now),
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 seed 构造 queued 记录。
|
||||
pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord {
|
||||
let now = Utc::now();
|
||||
BackupRecord {
|
||||
id: seed.id.clone(),
|
||||
trigger_kind: seed.trigger_kind,
|
||||
status: BackupStatus::Queued,
|
||||
scheduled_at: seed.scheduled_at,
|
||||
object_key: seed.object_key.clone(),
|
||||
size_bytes: None,
|
||||
sha256: None,
|
||||
attempt_count: 0,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
started_at: None,
|
||||
completed_at: None,
|
||||
expires_at: seed.expires_at,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// 内存版备份仓储。
|
||||
pub(crate) struct FakeBackupRepository {
|
||||
settings: Mutex<BackupSettings>,
|
||||
records: Mutex<Vec<BackupRecord>>,
|
||||
}
|
||||
|
||||
impl FakeBackupRepository {
|
||||
pub(crate) fn new(settings: BackupSettings) -> Self {
|
||||
Self {
|
||||
settings: Mutex::new(settings),
|
||||
records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 测试辅助:读取当前全部记录。
|
||||
pub(crate) fn all_records(&self) -> Vec<BackupRecord> {
|
||||
self.records.lock().expect("records").clone()
|
||||
}
|
||||
|
||||
/// 测试辅助:强制把一条记录标记为指定时间完成的 completed。
|
||||
pub(crate) fn set_completed(&self, id: &str, completed_at: DateTime<Utc>) {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
|
||||
record.status = BackupStatus::Completed;
|
||||
record.completed_at = Some(completed_at);
|
||||
record.updated_at = Utc::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// 测试辅助:强制把一条记录迁移到指定状态。
|
||||
pub(crate) fn force_status(&self, id: &str, status: BackupStatus) {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
|
||||
record.status = status;
|
||||
if matches!(status, BackupStatus::Completed | BackupStatus::Failed) {
|
||||
record.completed_at = Some(Utc::now());
|
||||
}
|
||||
record.updated_at = Utc::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupRepository for FakeBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
Ok(self.settings.lock().expect("settings").clone())
|
||||
}
|
||||
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
command: UpdateBackupStorageCommand,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.storage_revision += 1;
|
||||
settings.endpoint = Some(command.endpoint);
|
||||
settings.region = Some(command.region);
|
||||
settings.bucket = Some(command.bucket);
|
||||
settings.access_key_id = Some(command.access_key_id);
|
||||
if let Some(secret) = command.secret_access_key {
|
||||
settings.secret_access_key = Some(secret);
|
||||
}
|
||||
settings.prefix = Some(command.prefix);
|
||||
settings.force_path_style = command.force_path_style;
|
||||
settings.last_verified_at = None;
|
||||
settings.updated_at = Utc::now();
|
||||
let revision = Revision::new(settings.storage_revision).map_err(|_| {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", "zero revision")
|
||||
})?;
|
||||
Ok((settings.clone(), revision))
|
||||
}
|
||||
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
next_run_at: Option<DateTime<Utc>>,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.schedule_enabled = command.schedule_enabled;
|
||||
settings.cron_expression = Some(command.cron_expression);
|
||||
settings.schedule_timezone = Some(command.schedule_timezone);
|
||||
settings.retention_days = command.retention_days;
|
||||
settings.retention_count = command.retention_count;
|
||||
settings.next_run_at = next_run_at;
|
||||
settings.updated_at = Utc::now();
|
||||
Ok(settings.clone())
|
||||
}
|
||||
|
||||
async fn record_verification(
|
||||
&self,
|
||||
storage_revision: u64,
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
if settings.storage_revision != storage_revision {
|
||||
return Ok(false);
|
||||
}
|
||||
settings.last_verified_at = Some(at);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if records.iter().any(|record| record.status.is_active()) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"backup record",
|
||||
"an active backup task already exists",
|
||||
));
|
||||
}
|
||||
let record = queued_record(&seed);
|
||||
records.push(record.clone());
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if records.iter().any(|record| record.status.is_active())
|
||||
|| records.iter().any(|record| {
|
||||
record.trigger_kind == BackupTriggerKind::Scheduled
|
||||
&& record.scheduled_at == seed.scheduled_at
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
records.push(queued_record(&seed));
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage> {
|
||||
let records = self.records.lock().expect("records");
|
||||
let mut items = records
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
query.status.is_none_or(|status| record.status == status)
|
||||
&& query
|
||||
.trigger
|
||||
.is_none_or(|trigger| record.trigger_kind == trigger)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_by(|a, b| {
|
||||
b.created_at
|
||||
.cmp(&a.created_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
let total = items.len() as u64;
|
||||
let offset =
|
||||
(u64::from(query.page.saturating_sub(1)) * u64::from(query.page_size.get())) as usize;
|
||||
let limit = usize::from(query.page_size.get());
|
||||
items = items.into_iter().skip(offset).take(limit).collect();
|
||||
Ok(BackupRecordPage {
|
||||
items,
|
||||
total,
|
||||
page: query.page,
|
||||
page_size: query.page_size,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.find(|record| record.id == id)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
matches!(
|
||||
record.status,
|
||||
BackupStatus::Dumping | BackupStatus::Uploading
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| record.status == BackupStatus::Deleting)
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let now = Utc::now();
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record
|
||||
.expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
&& matches!(
|
||||
record.status,
|
||||
BackupStatus::Completed | BackupStatus::Failed
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn claim_next_queued(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.status == BackupStatus::Queued)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = BackupStatus::Dumping;
|
||||
record.started_at = Some(now);
|
||||
record.attempt_count += 1;
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn transition_status(
|
||||
&self,
|
||||
id: &str,
|
||||
from: BackupStatus,
|
||||
to: BackupStatus,
|
||||
update: StatusTransitionUpdate,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.id == id && record.status == from)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = to;
|
||||
if let Some(size) = update.size_bytes {
|
||||
record.size_bytes = Some(size);
|
||||
}
|
||||
if let Some(sha256) = update.sha256 {
|
||||
record.sha256 = Some(sha256);
|
||||
}
|
||||
if let Some(code) = update.error_code {
|
||||
record.error_code = Some(code);
|
||||
}
|
||||
if let Some(message) = update.error_message {
|
||||
record.error_message = Some(message);
|
||||
}
|
||||
record.completed_at = update.completed_at.or_else(|| {
|
||||
matches!(to, BackupStatus::Completed | BackupStatus::Failed).then_some(now)
|
||||
});
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.id == id && record.status.can_be_deleted())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = BackupStatus::Deleting;
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn delete_record(&self, id: &str) -> AdminStoreResult<()> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
records.retain(|record| record.id != id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
next_run_at: DateTime<Utc>,
|
||||
_expected_cron: &str,
|
||||
_expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.next_run_at = Some(next_run_at);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let mut records = self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record.trigger_kind == BackupTriggerKind::Scheduled
|
||||
&& record.status == BackupStatus::Completed
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
records.sort_by(|a, b| {
|
||||
b.completed_at
|
||||
.cmp(&a.completed_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
Ok(records)
|
||||
}
|
||||
}
|
||||
|
||||
/// 假 `pg_dump` 端口:把固定内容写入临时文件作为归档;可注入失败。
|
||||
pub(crate) struct FakeDumpPort {
|
||||
dir: TempDir,
|
||||
fail_dump: bool,
|
||||
}
|
||||
|
||||
impl FakeDumpPort {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
dir: tempfile::tempdir().expect("temp dir"),
|
||||
fail_dump: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让下一次 `dump` 返回 `backup.pg_dump_failed`。
|
||||
pub(crate) fn fail_next_dump(mut self) -> Self {
|
||||
self.fail_dump = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn artifact(&self, backup_id: &str) -> DumpArtifact {
|
||||
let path = self.dir.path().join(format!("{backup_id}.dump"));
|
||||
std::fs::write(&path, DUMP_CONTENT).expect("write artifact");
|
||||
DumpArtifact {
|
||||
path,
|
||||
size_bytes: DUMP_CONTENT.len() as u64,
|
||||
sha256: sha256_hex(DUMP_CONTENT),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DatabaseDumpPort for FakeDumpPort {
|
||||
async fn dump(
|
||||
&self,
|
||||
request: DumpRequest,
|
||||
) -> Result<DumpArtifact, gateway_admin::model::backup::BackupError> {
|
||||
if self.fail_dump {
|
||||
return Err(gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::PG_DUMP_FAILED,
|
||||
"pg_dump failed".to_owned(),
|
||||
));
|
||||
}
|
||||
if request.cancellation.is_cancelled() {
|
||||
return Err(gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::CANCELLED,
|
||||
"cancelled".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(self.artifact(&request.backup_id))
|
||||
}
|
||||
|
||||
async fn inspect_staging(
|
||||
&self,
|
||||
backup_id: &str,
|
||||
) -> Result<Option<StagedArtifact>, gateway_admin::model::backup::BackupError> {
|
||||
let path = self.dir.path().join(format!("{backup_id}.dump"));
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(StagedArtifact {
|
||||
path,
|
||||
size_bytes: DUMP_CONTENT.len() as u64,
|
||||
sha256: sha256_hex(DUMP_CONTENT),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_staging(
|
||||
&self,
|
||||
_backup_id: &str,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让上传返回指定的脱敏 S3 错误。
|
||||
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
|
||||
self.upload_error = Some(BackupError::new(code, message.into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupObjectStorePort for FakeObjectStore {
|
||||
async fn test_connection(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, gateway_admin::model::backup::BackupError> {
|
||||
Ok(ConnectionTestResult {
|
||||
ok: true,
|
||||
stage: "deleteObject",
|
||||
code: None,
|
||||
message: "连接测试成功".to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn upload_file(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.upload_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let content = std::fs::read(&request.source).map_err(|_| {
|
||||
gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
|
||||
"read source".to_owned(),
|
||||
)
|
||||
})?;
|
||||
self.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(request.object_key, content);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn head_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, gateway_admin::model::backup::BackupError> {
|
||||
let Some(content) = self.object(object_key) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(BackupObjectMetadata::new(
|
||||
"backup-test".to_owned(),
|
||||
sha256_hex(&content),
|
||||
Utc::now(),
|
||||
content.len() as u64,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn delete_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
self.objects.lock().expect("objects").remove(object_key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
_file_name: &str,
|
||||
_ttl: std::time::Duration,
|
||||
) -> Result<String, gateway_admin::model::backup::BackupError> {
|
||||
Ok(format!("https://presigned.example.com/{object_key}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录审计事件的假 AuthStore。
|
||||
pub(crate) struct FakeAuthStore {
|
||||
audit: Mutex<Vec<AdminAuditEvent>>,
|
||||
}
|
||||
|
||||
impl FakeAuthStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
audit: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn audit_actions(&self) -> Vec<String> {
|
||||
self.audit
|
||||
.lock()
|
||||
.expect("audit")
|
||||
.iter()
|
||||
.map(|event| event.action.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AuthStore for FakeAuthStore {
|
||||
async fn load_password_hash(&self, _admin_user_id: &str) -> AdminStoreResult<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn create_password_hash_if_absent(
|
||||
&self,
|
||||
_admin_user_id: &str,
|
||||
_password_hash: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn load_admin_api_key(
|
||||
&self,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::settings::AdminApiKey>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn store_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
_session: &gateway_admin::model::auth::AdminSession,
|
||||
) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
self.audit.lock().expect("audit").push(event);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 系统发起者上下文。
|
||||
pub(crate) fn system_context() -> MutationContext {
|
||||
MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "backup-runtime-test".to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//! BackupTask 执行链测试:驱动内存 fake 端口跑完整 dump/upload/verify/retention 流程。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::{
|
||||
backup::task::BackupTask,
|
||||
model::backup::{
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
|
||||
},
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
|
||||
use super::support::{
|
||||
DUMP_CONTENT, FakeBackupRepository, FakeDumpPort, FakeObjectStore, backup_id,
|
||||
configured_settings, sha256_hex, system_context,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_runs_full_dump_upload_verify_pipeline() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("manual"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/manual.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
assert_eq!(records.len(), 1);
|
||||
let finished = &records[0];
|
||||
assert_eq!(finished.status, BackupStatus::Completed);
|
||||
assert_eq!(finished.size_bytes, Some(DUMP_CONTENT.len() as u64));
|
||||
assert_eq!(
|
||||
finished.sha256.as_deref(),
|
||||
Some(sha256_hex(DUMP_CONTENT).as_str())
|
||||
);
|
||||
|
||||
// 对象已上传且内容与本地归档一致。
|
||||
let stored = object_store
|
||||
.object(&seed.object_key)
|
||||
.expect("uploaded object exists");
|
||||
assert_eq!(stored.as_slice(), DUMP_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new().fail_next_dump());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("fail"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/fail.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some("backup.pg_dump_failed"));
|
||||
assert!(failed.completed_at.is_some());
|
||||
// 归档未上传,对象不存在。
|
||||
assert!(object_store.object(&seed.object_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("recovered-upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
dump.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("stage dump");
|
||||
repository.force_status(&seed.id, BackupStatus::Dumping);
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&cancellation).await.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_cleans_expired_scheduled_backups() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
// 造一条旧 completed 计划备份(超过 retention_days)与一条新备份。
|
||||
let old_seed = BackupRecordSeed {
|
||||
id: backup_id("old"),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(Utc::now() - Duration::days(40)),
|
||||
object_key: "codex/2026/06/01/old.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
let new_seed = BackupRecordSeed {
|
||||
id: backup_id("new"),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(Utc::now() - Duration::days(1)),
|
||||
object_key: "codex/2026/07/01/new.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_scheduled_record(old_seed.clone())
|
||||
.await
|
||||
.expect("insert old");
|
||||
// 旧备份 40 天前完成(释放活跃名额),再插入新备份。
|
||||
let now = Utc::now();
|
||||
repository.set_completed(&backup_id("old"), now - Duration::days(40));
|
||||
repository
|
||||
.insert_scheduled_record(new_seed.clone())
|
||||
.await
|
||||
.expect("insert new");
|
||||
// 新备份 1 天前完成。
|
||||
repository.set_completed(&backup_id("new"), now - Duration::days(1));
|
||||
// 两个对象都已上传。
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(old_seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(new_seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
|
||||
// 设置 retention_days = 30。
|
||||
repository
|
||||
.update_schedule_settings(
|
||||
UpdateBackupScheduleCommand {
|
||||
schedule_enabled: false,
|
||||
cron_expression: "0 2 * * *".to_owned(),
|
||||
schedule_timezone: "Asia/Shanghai".to_owned(),
|
||||
retention_days: 30,
|
||||
retention_count: 0,
|
||||
},
|
||||
None,
|
||||
&system_context(),
|
||||
)
|
||||
.await
|
||||
.expect("set retention");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
// 旧备份被硬删除,对象被删除;新备份保留。
|
||||
let records = repository.all_records();
|
||||
let ids: Vec<_> = records.iter().map(|record| record.id.as_str()).collect();
|
||||
assert_eq!(ids, vec![backup_id("new").as_str()]);
|
||||
assert!(object_store.object(&old_seed.object_key).is_none());
|
||||
assert!(object_store.object(&new_seed.object_key).is_some());
|
||||
}
|
||||
@@ -17,6 +17,8 @@ use gateway_admin::{
|
||||
ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore},
|
||||
};
|
||||
|
||||
mod backup;
|
||||
mod model;
|
||||
mod use_case;
|
||||
|
||||
#[test]
|
||||
@@ -125,6 +127,8 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(18_000), Some(30.0)),
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
@@ -144,6 +148,7 @@ fn quota_window(
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! 备份领域模型(model/backup.rs)的纯逻辑测试。
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use secrecy::SecretString;
|
||||
|
||||
use gateway_admin::model::backup::{
|
||||
BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, build_object_key,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn status_machine_rejects_invalid_transitions() {
|
||||
assert!(BackupStatus::Queued.allows_transition_to(BackupStatus::Dumping));
|
||||
assert!(BackupStatus::Dumping.allows_transition_to(BackupStatus::Uploading));
|
||||
assert!(BackupStatus::Uploading.allows_transition_to(BackupStatus::Completed));
|
||||
assert!(BackupStatus::Completed.allows_transition_to(BackupStatus::Deleting));
|
||||
assert!(!BackupStatus::Queued.allows_transition_to(BackupStatus::Completed));
|
||||
assert!(!BackupStatus::Completed.allows_transition_to(BackupStatus::Dumping));
|
||||
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Failed));
|
||||
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_classification() {
|
||||
assert!(BackupStatus::Queued.is_active());
|
||||
assert!(BackupStatus::Uploading.is_active());
|
||||
assert!(!BackupStatus::Completed.is_active());
|
||||
assert!(BackupStatus::Completed.can_be_deleted());
|
||||
assert!(BackupStatus::Failed.can_be_deleted());
|
||||
assert!(!BackupStatus::Queued.can_be_deleted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_key_shape() {
|
||||
let at = Utc.with_ymd_and_hms(2026, 8, 1, 12, 30, 0).unwrap();
|
||||
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
|
||||
assert_eq!(
|
||||
key,
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_key_rejects_dangerous_prefixes() {
|
||||
let at = Utc::now();
|
||||
assert!(build_object_key("/leading", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a/../b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a\\b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a\u{0001}b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("", "backup_x", at).is_err());
|
||||
assert!(build_object_key("trailing//", "backup_x", at).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trigger_and_status_roundtrip() {
|
||||
for status in [
|
||||
BackupStatus::Queued,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
BackupStatus::Failed,
|
||||
BackupStatus::Deleting,
|
||||
] {
|
||||
assert_eq!(BackupStatus::parse(status.as_str()), Some(status));
|
||||
}
|
||||
assert_eq!(
|
||||
BackupTriggerKind::parse("manual"),
|
||||
Some(BackupTriggerKind::Manual)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupTriggerKind::parse("scheduled"),
|
||||
Some(BackupTriggerKind::Scheduled)
|
||||
);
|
||||
assert_eq!(BackupTriggerKind::parse("unknown"), None);
|
||||
assert_eq!(BackupStatus::parse("skipped"), None);
|
||||
assert_eq!(BackupStatus::parse("deleted"), None);
|
||||
assert_eq!(BackupStatus::parse("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_from_settings_redacts_secret_in_debug() {
|
||||
let mut settings = BackupSettings {
|
||||
storage_revision: 3,
|
||||
endpoint: Some("https://example.com".to_owned()),
|
||||
region: Some("auto".to_owned()),
|
||||
bucket: Some("b".to_owned()),
|
||||
access_key_id: Some("ak".to_owned()),
|
||||
secret_access_key: Some(SecretString::from("sk-secret")),
|
||||
prefix: Some("p".to_owned()),
|
||||
force_path_style: false,
|
||||
schedule_enabled: false,
|
||||
cron_expression: None,
|
||||
schedule_timezone: None,
|
||||
retention_days: 0,
|
||||
retention_count: 0,
|
||||
next_run_at: None,
|
||||
last_verified_at: None,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
assert!(!format!("{:?}", settings).contains("sk-secret"));
|
||||
let config = BackupStorageConfig::from_settings(&settings).unwrap();
|
||||
assert!(!format!("{config:?}").contains("sk-secret"));
|
||||
settings.endpoint = None;
|
||||
assert!(BackupStorageConfig::from_settings(&settings).is_none());
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
//! 模型模块测试:镜像 src/model/ 中带逻辑的模块。
|
||||
|
||||
mod backup;
|
||||
@@ -373,7 +373,8 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
accounts: Mutex<Vec<AccountRecord>>,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
@@ -387,7 +388,8 @@ impl FakeAccountStore {
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
@@ -406,6 +408,17 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe") = Some(account);
|
||||
}
|
||||
|
||||
fn record(&self, event: &'static str) {
|
||||
self.events.lock().expect("store events").push(event);
|
||||
}
|
||||
@@ -430,22 +443,41 @@ impl FakeAccountStore {
|
||||
#[async_trait]
|
||||
impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: vec![self.account.clone()],
|
||||
total: 1,
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
summary: AccountSummary {
|
||||
total: 1,
|
||||
active: 1,
|
||||
total: accounts.len() as u64,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
Ok((account_id == self.account.id).then(|| self.account.clone()))
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.filter(|account| account.id == account_id)
|
||||
.or_else(|| {
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -473,11 +505,12 @@ impl AccountStore for FakeAccountStore {
|
||||
_: CredentialListQuery,
|
||||
) -> AdminStoreResult<CredentialPage> {
|
||||
self.record("store.list_credentials");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(CredentialPage {
|
||||
config_revision: revision(1),
|
||||
items: (provider_kind == &self.account.provider_kind)
|
||||
.then(|| self.account.clone())
|
||||
items: accounts
|
||||
.into_iter()
|
||||
.filter(|account| &account.provider_kind == provider_kind)
|
||||
.collect(),
|
||||
next_cursor: None,
|
||||
})
|
||||
@@ -489,14 +522,19 @@ impl AccountStore for FakeAccountStore {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> AdminStoreResult<Option<CredentialDetails>> {
|
||||
self.record("store.credential_details");
|
||||
Ok(
|
||||
(provider_kind == &self.account.provider_kind
|
||||
&& account_id.as_str() == self.account.id)
|
||||
.then(|| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential: self.account.clone(),
|
||||
}),
|
||||
)
|
||||
let account = self
|
||||
.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| {
|
||||
&account.provider_kind == provider_kind && account.id == account_id.as_str()
|
||||
})
|
||||
.cloned();
|
||||
Ok(account.map(|credential| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn load_credentials_for_export(
|
||||
@@ -505,21 +543,26 @@ impl AccountStore for FakeAccountStore {
|
||||
account_ids: &[ProviderAccountId],
|
||||
) -> AdminStoreResult<Vec<ProviderExportCredentialInput>> {
|
||||
self.record("store.load_credentials_for_export");
|
||||
if provider_kind != &self.account.provider_kind
|
||||
|| account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() != self.account.id)
|
||||
{
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
if accounts.iter().any(|account| {
|
||||
&account.provider_kind != provider_kind
|
||||
|| !account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() == account.id)
|
||||
}) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::NotFound,
|
||||
"test account",
|
||||
"credential not found",
|
||||
));
|
||||
}
|
||||
Ok(vec![ProviderExportCredentialInput {
|
||||
account: self.account.clone(),
|
||||
provider_material: document(),
|
||||
}])
|
||||
Ok(accounts
|
||||
.into_iter()
|
||||
.map(|account| ProviderExportCredentialInput {
|
||||
account,
|
||||
provider_material: document(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn commit_credential_import(
|
||||
@@ -802,6 +845,36 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
let probe = Arc::new(FreeModelQuotaProbe {
|
||||
store: Arc::clone(&store),
|
||||
});
|
||||
let services = accounts_service_with_probe(provider, store, probe).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
@@ -835,7 +908,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let expected = store.account.clone();
|
||||
let expected = store.accounts.lock().expect("accounts")[0].clone();
|
||||
accounts_service(provider.clone(), store)
|
||||
.await
|
||||
.accounts()
|
||||
@@ -877,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -913,22 +986,87 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_expiry() {
|
||||
async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() {
|
||||
let events = events();
|
||||
let openai = FakeProviderAdmin::new("openai", events.clone());
|
||||
openai.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_bad_xai".to_owned();
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(openai)
|
||||
.provider(failing)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("one failing quota must not fail the directory page");
|
||||
|
||||
assert_eq!(page.items.len(), 2);
|
||||
let by_id = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| (item.account.id.as_str(), item))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
let healthy = by_id.get("acct_test").expect("healthy account item");
|
||||
assert_eq!(healthy.quota.windows.len(), 1);
|
||||
assert_eq!(healthy.quota.windows[0].key, "primary");
|
||||
let degraded = by_id.get("acct_bad_xai").expect("degraded account item");
|
||||
assert!(degraded.quota.windows.is_empty());
|
||||
assert!(degraded.quota.observed_at.is_none());
|
||||
assert!(degraded.quota.provider_data.is_none());
|
||||
assert_eq!(page.summary.total, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Cooldown;
|
||||
account.availability_reason = Some("usage_limit_exhausted".to_owned());
|
||||
account.cooldown_until = Some(Utc::now() + TimeDelta::minutes(5));
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
@@ -952,6 +1090,125 @@ async fn accounts_list_should_prefer_usage_limit_exhaustion_over_stale_token_exp
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -967,9 +1224,12 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -1077,6 +1337,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1121,9 +1383,8 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -1159,8 +1420,6 @@ fn prepared_create_with_id(
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
}
|
||||
}
|
||||
@@ -1234,6 +1493,28 @@ impl AccountProbe for FailingAccountProbe {
|
||||
}
|
||||
}
|
||||
|
||||
struct FreeModelQuotaProbe {
|
||||
store: Arc<FakeAccountStore>,
|
||||
}
|
||||
|
||||
impl AccountProbe for FreeModelQuotaProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable() -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! BackupService 用例测试:通过 AdminHarness 注入 fake 备份端口构造真实服务。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_admin::model::backup::BackupStatus;
|
||||
use gateway_admin::ports::backup::BackupStorePorts;
|
||||
|
||||
use crate::backup::support::{
|
||||
FakeAuthStore, FakeBackupRepository, FakeDumpPort, FakeObjectStore, configured_settings,
|
||||
system_context,
|
||||
};
|
||||
|
||||
use super::AdminHarness;
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_service_creates_downloads_and_deletes_with_audit() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let auth = Arc::new(FakeAuthStore::new());
|
||||
|
||||
let services = AdminHarness::new()
|
||||
.auth(auth.clone())
|
||||
.backup(BackupStorePorts::new(
|
||||
repository.clone(),
|
||||
dump,
|
||||
object_store,
|
||||
))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// 创建手动备份 → queued + 审计。
|
||||
let created = services
|
||||
.backups()
|
||||
.create_backup(&system_context(), None)
|
||||
.await
|
||||
.expect("create backup");
|
||||
assert_eq!(created.status, BackupStatus::Queued);
|
||||
assert!(created.object_key.starts_with("codex/"));
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.created")
|
||||
);
|
||||
|
||||
// 下载地址要求 completed 记录。
|
||||
repository.force_status(&created.id, BackupStatus::Completed);
|
||||
let download = services
|
||||
.backups()
|
||||
.download_url(&system_context(), &created.id)
|
||||
.await
|
||||
.expect("download url");
|
||||
assert!(download.url.starts_with("https://presigned.example.com/"));
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.download_url_created")
|
||||
);
|
||||
|
||||
// 删除 → deleting + 审计。
|
||||
let deleting = services
|
||||
.backups()
|
||||
.delete_backup(&system_context(), &created.id)
|
||||
.await
|
||||
.expect("delete backup");
|
||||
assert_eq!(deleting.status, BackupStatus::Deleting);
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.delete_requested")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_service_rejects_download_of_non_completed_record() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let auth = Arc::new(FakeAuthStore::new());
|
||||
|
||||
let services = AdminHarness::new()
|
||||
.auth(auth.clone())
|
||||
.backup(BackupStorePorts::new(
|
||||
repository.clone(),
|
||||
dump,
|
||||
object_store,
|
||||
))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let created = services
|
||||
.backups()
|
||||
.create_backup(&system_context(), None)
|
||||
.await
|
||||
.expect("create backup");
|
||||
|
||||
// queued 记录不能创建下载地址。
|
||||
let error = services
|
||||
.backups()
|
||||
.download_url(&system_context(), &created.id)
|
||||
.await
|
||||
.expect_err("non-completed record must be rejected");
|
||||
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
mod accounts;
|
||||
mod auth;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod observability;
|
||||
mod openai;
|
||||
@@ -46,6 +47,7 @@ use gateway_admin::{
|
||||
system::{SystemOperationAccepted, SystemUpdateDetail, SystemUpdateStatus, SystemVersion},
|
||||
},
|
||||
ports::{
|
||||
backup::BackupStorePorts,
|
||||
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
|
||||
store::{
|
||||
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
|
||||
@@ -75,6 +77,7 @@ pub(super) struct AdminHarness {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
providers: Vec<Arc<dyn ProviderAdmin>>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
system: Arc<dyn SystemOperations>,
|
||||
@@ -91,6 +94,7 @@ impl AdminHarness {
|
||||
client_keys: unavailable.clone(),
|
||||
observability: unavailable.clone(),
|
||||
settings: unavailable,
|
||||
backup: BackupStorePorts::disabled(),
|
||||
providers: vec![
|
||||
Arc::new(UnavailableProvider::new("openai")),
|
||||
Arc::new(UnavailableProvider::new("xai")),
|
||||
@@ -135,6 +139,11 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn backup(mut self, backup: BackupStorePorts) -> Self {
|
||||
self.backup = backup;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn provider(mut self, provider: Arc<dyn ProviderAdmin>) -> Self {
|
||||
self.providers
|
||||
.retain(|registered| registered.provider_kind() != provider.provider_kind());
|
||||
@@ -165,6 +174,7 @@ impl AdminHarness {
|
||||
self.client_keys,
|
||||
self.observability,
|
||||
self.settings,
|
||||
self.backup,
|
||||
),
|
||||
self.providers,
|
||||
Arc::new(NoopSnapshot),
|
||||
|
||||
@@ -695,6 +695,7 @@ fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) ->
|
||||
|
||||
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
DiagnosticObservation {
|
||||
key: name.to_owned(),
|
||||
name: name.to_owned(),
|
||||
request_count,
|
||||
success_count: request_count,
|
||||
@@ -702,6 +703,7 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
attempt_count: request_count,
|
||||
total_tokens: request_count.saturating_mul(100),
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(200),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ futures = "0.3.32"
|
||||
gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
|
||||
@@ -16,7 +16,7 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use gateway_admin::model::{
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize,
|
||||
AdminError as AdminServiceError, PageSize,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
|
||||
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
|
||||
@@ -37,6 +37,7 @@ use gateway_core::{
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError,
|
||||
@@ -90,7 +91,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -126,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -171,9 +162,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -193,7 +186,10 @@ pub struct AccountView {
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
pub token_refreshing: bool,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
@@ -213,6 +209,9 @@ pub struct AccountView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -226,6 +225,7 @@ pub struct AccountQuotaWindowView {
|
||||
pub label_display: String,
|
||||
pub used_percent: Option<f64>,
|
||||
pub used_percent_display: String,
|
||||
pub limit_reached: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_usage: Option<serde_json::Value>,
|
||||
pub reset_at_display: String,
|
||||
@@ -525,7 +525,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -543,7 +543,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -1331,9 +1331,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1364,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = status.as_str().to_owned();
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -1386,7 +1389,8 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
token_refreshing: false,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
access_token_expires_at: expires_at,
|
||||
@@ -1416,9 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1434,6 +1441,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
local_usage,
|
||||
provider_data: _,
|
||||
} = window;
|
||||
@@ -1445,6 +1453,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
used_percent,
|
||||
used_percent_display: used_percent
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
|
||||
limit_reached,
|
||||
local_usage: local_usage.as_ref().map(quota_local_usage),
|
||||
reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)),
|
||||
}
|
||||
@@ -1468,7 +1477,7 @@ fn quota_local_usage(usage: &AccountUsage) -> Value {
|
||||
"imageRequestCount": usage.image_request_count,
|
||||
"imageRequestFailedCount": usage.image_request_failed_count,
|
||||
"totalTokens": total_tokens,
|
||||
"totalTokensDisplay": format_tokens(total_tokens),
|
||||
"totalTokensDisplay": format_compact_number(total_tokens),
|
||||
"requestBuckets": usage.request_buckets.iter().map(|bucket| serde_json::json!({
|
||||
"bucketStart": bucket.bucket_start,
|
||||
"requestCount": bucket.request_count,
|
||||
@@ -1601,7 +1610,7 @@ fn account_currency_cost_view(cost: &AccountCost) -> CurrencyCostView {
|
||||
}
|
||||
|
||||
fn display_optional_tokens(value: Option<u64>) -> String {
|
||||
value.map_or_else(|| "—".to_owned(), format_tokens)
|
||||
value.map_or_else(|| "—".to_owned(), format_compact_number)
|
||||
}
|
||||
|
||||
fn empty_account_usage() -> AccountUsageView {
|
||||
@@ -1709,47 +1718,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn format_number(value: u64) -> String {
|
||||
let text = value.to_string();
|
||||
let mut output = String::with_capacity(text.len() + text.len() / 3);
|
||||
for (index, character) in text.chars().rev().enumerate() {
|
||||
if index > 0 && index % 3 == 0 {
|
||||
output.push(',');
|
||||
}
|
||||
output.push(character);
|
||||
}
|
||||
output.chars().rev().collect()
|
||||
}
|
||||
|
||||
fn format_tokens(value: u64) -> String {
|
||||
if value < 1_000 {
|
||||
return format_number(value);
|
||||
}
|
||||
for (suffix, threshold) in [
|
||||
("P", 1_000_000_000_000_000_u64),
|
||||
("T", 1_000_000_000_000_u64),
|
||||
("B", 1_000_000_000_u64),
|
||||
("M", 1_000_000_u64),
|
||||
("K", 1_000_u64),
|
||||
] {
|
||||
if value >= threshold {
|
||||
let scaled = value as f64 / threshold as f64;
|
||||
return format!("{scaled:.1}{suffix}").replace(".0", "");
|
||||
}
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
@@ -1787,16 +1755,5 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
}
|
||||
|
||||
fn map_service_error(error: AdminServiceError) -> AdminError {
|
||||
match error.kind() {
|
||||
AdminErrorKind::Invalid => AdminError::bad_request(error.to_string()),
|
||||
AdminErrorKind::Unauthorized => AdminError::admin_session_required(),
|
||||
AdminErrorKind::NotFound => AdminError::not_found(error.to_string()),
|
||||
AdminErrorKind::Conflict => AdminError::conflict(error.to_string()),
|
||||
AdminErrorKind::RateLimited => AdminError::too_many_login_attempts(),
|
||||
AdminErrorKind::BadGateway => AdminError::bad_gateway(error.to_string()),
|
||||
AdminErrorKind::Unavailable => {
|
||||
AdminError::service_unavailable("Account directory unavailable")
|
||||
}
|
||||
AdminErrorKind::Internal => AdminError::internal(error.to_string()),
|
||||
}
|
||||
super::wire::map_admin_service_error(error, "Account directory unavailable")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
//! 备份设置、计划、手动创建、记录、下载与删除路由。
|
||||
//!
|
||||
//! 路径统一位于 `/api/admin/settings/backups/*`,内部由独立 `BackupService` 承担业务。
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::backup::{
|
||||
BackupRecord, BackupSettings, BackupStatus, BackupTriggerKind, ConnectionTestResult,
|
||||
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminPageData, AdminResponse, AdminSessionState,
|
||||
PageMeta, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
/// 备份设置视图;`secretAccessKey` 返回已保存的明文凭据,由前端掩码显示。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupSettingsView {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub bucket: Option<String>,
|
||||
pub access_key_id: Option<String>,
|
||||
pub secret_access_key: Option<String>,
|
||||
pub prefix: Option<String>,
|
||||
pub force_path_style: bool,
|
||||
pub verified: bool,
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: Option<String>,
|
||||
pub schedule_timezone: Option<String>,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
pub next_run_at: Option<DateTime<Utc>>,
|
||||
pub last_verified_at: Option<DateTime<Utc>>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl From<BackupSettings> for BackupSettingsView {
|
||||
fn from(settings: BackupSettings) -> Self {
|
||||
let verified = settings.storage_verified();
|
||||
Self {
|
||||
storage_revision: settings.storage_revision,
|
||||
endpoint: settings.endpoint,
|
||||
region: settings.region,
|
||||
bucket: settings.bucket,
|
||||
access_key_id: settings.access_key_id,
|
||||
secret_access_key: settings
|
||||
.secret_access_key
|
||||
.map(|secret| secret.expose_secret().to_owned()),
|
||||
prefix: settings.prefix,
|
||||
force_path_style: settings.force_path_style,
|
||||
verified,
|
||||
schedule_enabled: settings.schedule_enabled,
|
||||
cron_expression: settings.cron_expression,
|
||||
schedule_timezone: settings.schedule_timezone,
|
||||
retention_days: settings.retention_days,
|
||||
retention_count: settings.retention_count,
|
||||
next_run_at: settings.next_run_at,
|
||||
last_verified_at: settings.last_verified_at,
|
||||
updated_at: settings.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 更新 S3 存储配置请求;`secretAccessKey` 缺省表示保留旧值。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateBackupStorageRequest {
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
#[serde(default)]
|
||||
pub secret_access_key: Option<String>,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
/// 更新调度配置请求。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateBackupScheduleRequest {
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: String,
|
||||
pub schedule_timezone: String,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
}
|
||||
|
||||
/// 备份记录 wire 视图。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupRecordView {
|
||||
pub id: String,
|
||||
pub trigger_kind: String,
|
||||
pub status: String,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl From<BackupRecord> for BackupRecordView {
|
||||
fn from(record: BackupRecord) -> Self {
|
||||
Self {
|
||||
id: record.id,
|
||||
trigger_kind: record.trigger_kind.to_string(),
|
||||
status: record.status.to_string(),
|
||||
scheduled_at: record.scheduled_at,
|
||||
object_key: record.object_key,
|
||||
size_bytes: record.size_bytes,
|
||||
sha256: record.sha256,
|
||||
attempt_count: record.attempt_count,
|
||||
error_code: record.error_code,
|
||||
error_message: record.error_message,
|
||||
started_at: record.started_at,
|
||||
completed_at: record.completed_at,
|
||||
expires_at: record.expires_at,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录列表查询参数。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupRecordsQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub status: Option<String>,
|
||||
pub trigger: Option<String>,
|
||||
}
|
||||
|
||||
/// 下载地址请求。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct BackupIdRequest {
|
||||
pub backup_id: String,
|
||||
}
|
||||
|
||||
/// 下载地址响应。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DownloadUrlView {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub expires_in_seconds: u64,
|
||||
}
|
||||
|
||||
impl From<gateway_admin::model::backup::DownloadUrlResult> for DownloadUrlView {
|
||||
fn from(result: gateway_admin::model::backup::DownloadUrlResult) -> Self {
|
||||
Self {
|
||||
url: result.url,
|
||||
file_name: result.file_name,
|
||||
expires_in_seconds: result.expires_in.as_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 连接测试响应。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ConnectionTestView {
|
||||
pub ok: bool,
|
||||
pub stage: String,
|
||||
pub code: Option<String>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl From<ConnectionTestResult> for ConnectionTestView {
|
||||
fn from(result: ConnectionTestResult) -> Self {
|
||||
Self {
|
||||
ok: result.ok,
|
||||
stage: result.stage.to_owned(),
|
||||
code: result.code.map(str::to_owned),
|
||||
message: result.message,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造固定 GET/POST 备份路由。
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/api/admin/settings/backups", get(backup_settings::<S>))
|
||||
.route(
|
||||
"/api/admin/settings/backups/storage/update",
|
||||
post(update_backup_storage::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/storage/test",
|
||||
post(test_backup_storage::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/schedule/update",
|
||||
post(update_backup_schedule::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/records",
|
||||
get(backup_records::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/create",
|
||||
post(create_backup::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/download-url",
|
||||
post(download_backup_url::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/delete",
|
||||
post(delete_backup::<S>),
|
||||
)
|
||||
}
|
||||
|
||||
async fn backup_settings<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_backup_storage<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<UpdateBackupStorageRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = UpdateBackupStorageCommand {
|
||||
endpoint: request.endpoint,
|
||||
region: request.region,
|
||||
bucket: request.bucket,
|
||||
access_key_id: request.access_key_id,
|
||||
secret_access_key: request.secret_access_key.map(SecretString::from),
|
||||
prefix: request.prefix,
|
||||
force_path_style: request.force_path_style,
|
||||
};
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.update_storage(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn test_backup_storage<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.test_connection(&auth.context().mutation_context())
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(ConnectionTestView::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_backup_schedule<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<UpdateBackupScheduleRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = UpdateBackupScheduleCommand {
|
||||
schedule_enabled: request.schedule_enabled,
|
||||
cron_expression: request.cron_expression,
|
||||
schedule_timezone: request.schedule_timezone,
|
||||
retention_days: request.retention_days,
|
||||
retention_count: request.retention_count,
|
||||
};
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.update_schedule(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn backup_records<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<BackupRecordsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let page_size = gateway_admin::model::PageSize::new(query.page_size.unwrap_or(20))
|
||||
.map_err(|_| AdminError::bad_request("Invalid pageSize"))?;
|
||||
let page = query.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(AdminError::bad_request("Invalid page"));
|
||||
}
|
||||
let status = match query.status.as_deref() {
|
||||
None => None,
|
||||
Some(value) => BackupStatus::parse(value)
|
||||
.map(Some)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid status"))?,
|
||||
};
|
||||
let trigger = match query.trigger.as_deref() {
|
||||
None => None,
|
||||
Some(value) => BackupTriggerKind::parse(value)
|
||||
.map(Some)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid trigger"))?,
|
||||
};
|
||||
let domain_query = gateway_admin::model::backup::BackupRecordListQuery {
|
||||
page,
|
||||
page_size,
|
||||
status,
|
||||
trigger,
|
||||
};
|
||||
let page_result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.list_records(domain_query)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let meta = PageMeta::new(
|
||||
page_result.page,
|
||||
u32::from(page_result.page_size.get()),
|
||||
page_result.total,
|
||||
page_result.total_pages(),
|
||||
);
|
||||
let items: Vec<BackupRecordView> = page_result
|
||||
.items
|
||||
.into_iter()
|
||||
.map(BackupRecordView::from)
|
||||
.collect();
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminPageData::new(items, meta)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn create_backup<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
body: Option<Json<CreateBackupRequest>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let expires_at = body
|
||||
.and_then(|body| body.0.expires_in_days)
|
||||
.filter(|&days| days > 0)
|
||||
.map(|days| Utc::now() + chrono::Duration::days(i64::from(days)));
|
||||
let record = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.create_backup(&auth.context().mutation_context(), expires_at)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::ACCEPTED,
|
||||
AdminEnvelope::ok(BackupRecordView::from(record)),
|
||||
))
|
||||
}
|
||||
|
||||
/// 创建备份请求;`expiresInDays` 为 0 或缺省表示不设置过期时间。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CreateBackupRequest {
|
||||
pub expires_in_days: Option<u32>,
|
||||
}
|
||||
|
||||
async fn download_backup_url<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<BackupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.download_url(&auth.context().mutation_context(), &request.backup_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DownloadUrlView::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_backup<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<BackupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let record = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.delete_backup(&auth.context().mutation_context(), &request.backup_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupRecordView::from(record)),
|
||||
))
|
||||
}
|
||||
|
||||
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
|
||||
map_admin_service_error(error, "Backup service unavailable")
|
||||
}
|
||||
@@ -9,8 +9,10 @@ use axum::{
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backups;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod presenter;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod wire;
|
||||
@@ -29,6 +31,7 @@ where
|
||||
Router::new()
|
||||
.merge(accounts::router::<S>())
|
||||
.merge(auth::router::<S>())
|
||||
.merge(backups::router::<S>())
|
||||
.merge(client_keys::router::<S>())
|
||||
.merge(observability::router::<S>())
|
||||
.merge(settings::router::<S>())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
//! Dashboard、用量与诊断查询的 wire 映射和固定路由。
|
||||
|
||||
use std::{collections::BTreeMap, num::NonZeroU32};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::get,
|
||||
};
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
mod presenter;
|
||||
mod query;
|
||||
mod routes;
|
||||
mod wire;
|
||||
|
||||
pub(crate) use presenter::*;
|
||||
pub use query::*;
|
||||
pub use routes::*;
|
||||
pub use wire::*;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,463 @@
|
||||
//! 查询 wire 类型、校验与领域命令映射。
|
||||
|
||||
use super::*;
|
||||
|
||||
use base64::Engine as _;
|
||||
|
||||
/// 观测列表默认页大小。
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小。
|
||||
pub const MAX_PAGE_SIZE: u16 = 100;
|
||||
/// 游标编码在 HTTP 层解码前允许的最大长度。
|
||||
pub const MAX_CURSOR_BYTES: usize = 1024;
|
||||
|
||||
/// Dashboard 查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DashboardQuery {
|
||||
pub kind: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl DashboardQuery {
|
||||
/// 解析 dashboard 趋势类型。
|
||||
pub fn trend_kind(&self) -> Result<TrendKind, WireValidationError> {
|
||||
TrendKind::parse(self.kind.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 逻辑请求列表、汇总和洞察查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub outcome: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl UsageQuery {
|
||||
/// 校验分页字段,不改变仓储层的分页 owner。
|
||||
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let page = self.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(WireValidationError::new("page"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((page, page_size))
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界;编码和排序语义由应用层负责。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 详情查询参数。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DetailQuery {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl DetailQuery {
|
||||
/// 校验详情 ID,错误不回显输入值。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_text(&self.id, "id")
|
||||
}
|
||||
}
|
||||
|
||||
/// 诊断聚合查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DiagnosticsQuery {
|
||||
pub dimension: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
impl DiagnosticsQuery {
|
||||
/// 解析诊断维度。
|
||||
pub fn dimension(&self) -> Result<DiagnosticDimension, WireValidationError> {
|
||||
DiagnosticDimension::parse(self.dimension.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 运维错误查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct OpsQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub failure_class: Option<String>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl OpsQuery {
|
||||
/// 校验分页字段。
|
||||
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let page = self.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(WireValidationError::new("page"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((page, page_size))
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Dashboard 趋势类型。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TrendKind {
|
||||
Usage,
|
||||
Latency,
|
||||
Errors,
|
||||
}
|
||||
|
||||
impl TrendKind {
|
||||
/// 从 query 值解析趋势类型。
|
||||
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
|
||||
match trimmed(value) {
|
||||
None | Some("usage") => Ok(Self::Usage),
|
||||
Some("latency") => Ok(Self::Latency),
|
||||
Some("errors") => Ok(Self::Errors),
|
||||
Some(_) => Err(WireValidationError::new("kind")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 诊断聚合维度。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DiagnosticDimension {
|
||||
Model,
|
||||
Account,
|
||||
ApiKey,
|
||||
Provider,
|
||||
Transport,
|
||||
Failure,
|
||||
Status,
|
||||
}
|
||||
|
||||
impl DiagnosticDimension {
|
||||
/// 从 query 值解析诊断维度。
|
||||
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
|
||||
match trimmed(value) {
|
||||
None | Some("model") => Ok(Self::Model),
|
||||
Some("account") => Ok(Self::Account),
|
||||
Some("apiKey" | "api_key") => Ok(Self::ApiKey),
|
||||
Some("provider") => Ok(Self::Provider),
|
||||
Some("transport") => Ok(Self::Transport),
|
||||
Some("failureClass" | "failure_class") => Ok(Self::Failure),
|
||||
Some("status") => Ok(Self::Status),
|
||||
Some(_) => Err(WireValidationError::new("dimension")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回终态响应中的稳定维度名称。
|
||||
#[must_use]
|
||||
pub const fn display_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Model => "model",
|
||||
Self::Account => "account",
|
||||
Self::ApiKey => "apiKey",
|
||||
Self::Provider => "provider",
|
||||
Self::Transport => "transport",
|
||||
Self::Failure => "failureClass",
|
||||
Self::Status => "status",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值。
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = trimmed(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
/// 解析 HTTP 状态码。
|
||||
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
|
||||
value
|
||||
.map(|value| {
|
||||
u16::try_from(value)
|
||||
.ok()
|
||||
.filter(|value| (100..=599).contains(value))
|
||||
.ok_or_else(|| WireValidationError::new("statusCode"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// 解析尝试序号。
|
||||
pub fn parse_attempt_index(value: Option<i64>) -> Result<Option<u32>, WireValidationError> {
|
||||
value
|
||||
.map(|value| {
|
||||
u32::try_from(value)
|
||||
.ok()
|
||||
.filter(|value| *value > 0 && *value <= i32::MAX as u32)
|
||||
.ok_or_else(|| WireValidationError::new("attemptIndex"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn require_text(value: &str, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value.trim().is_empty() {
|
||||
Err(WireValidationError::new(field))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn domain_trend_kind(kind: TrendKind) -> domain::TrendKind {
|
||||
match kind {
|
||||
TrendKind::Usage => domain::TrendKind::Usage,
|
||||
TrendKind::Latency => domain::TrendKind::Latency,
|
||||
TrendKind::Errors => domain::TrendKind::Errors,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn domain_diagnostic_dimension(
|
||||
dimension: DiagnosticDimension,
|
||||
) -> domain::DiagnosticDimension {
|
||||
match dimension {
|
||||
DiagnosticDimension::Model => domain::DiagnosticDimension::Model,
|
||||
DiagnosticDimension::Account => domain::DiagnosticDimension::Account,
|
||||
DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey,
|
||||
DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider,
|
||||
DiagnosticDimension::Transport => domain::DiagnosticDimension::Transport,
|
||||
DiagnosticDimension::Failure => domain::DiagnosticDimension::Failure,
|
||||
DiagnosticDimension::Status => domain::DiagnosticDimension::Status,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_outcome(
|
||||
value: Option<String>,
|
||||
) -> Result<Option<domain::RequestOutcome>, WireValidationError> {
|
||||
non_empty(value)
|
||||
.map(|value| {
|
||||
domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn usage_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start = parse_datetime(start)?.unwrap_or(end - Duration::days(7));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start =
|
||||
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_today_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start = parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, WireValidationError> {
|
||||
let outcome = query.outcome.clone().or_else(|| {
|
||||
non_empty(query.kind.clone()).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"running" | "succeeded" | "failed" | "cancelled" | "incomplete"
|
||||
)
|
||||
})
|
||||
});
|
||||
Ok(domain::UsageFilter {
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
model: non_empty(query.model.clone()),
|
||||
outcome: request_outcome(outcome)?,
|
||||
status_code: parse_status(query.status_code)?,
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
search: non_empty(query.search.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn usage_command(
|
||||
query: &UsageQuery,
|
||||
) -> Result<(domain::UsageQuery, u32, u16), WireValidationError> {
|
||||
let (page, page_size) = query.validate_page()?;
|
||||
query.validate_cursor()?;
|
||||
let page_number = NonZeroU32::new(page)
|
||||
.map(domain::PageNumber::new)
|
||||
.ok_or_else(|| WireValidationError::new("page"))?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
Ok((
|
||||
domain::UsageQuery {
|
||||
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
|
||||
filter: usage_filter(query)?,
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
page: page_number,
|
||||
page_size: page_size_value,
|
||||
},
|
||||
page,
|
||||
page_size,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn ops_command(
|
||||
query: &OpsQuery,
|
||||
) -> Result<(domain::OpsErrorQuery, u32, u16), WireValidationError> {
|
||||
let (page, page_size) = query.validate_page()?;
|
||||
query.validate_cursor()?;
|
||||
let page_number = NonZeroU32::new(page)
|
||||
.map(domain::PageNumber::new)
|
||||
.ok_or_else(|| WireValidationError::new("page"))?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
let status_code = parse_status(
|
||||
query
|
||||
.upstream_status_code
|
||||
.or(query.client_status_code)
|
||||
.or(query.status_code),
|
||||
)?;
|
||||
Ok((
|
||||
domain::OpsErrorQuery {
|
||||
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
|
||||
filter: domain::OpsErrorFilter {
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())),
|
||||
model: non_empty(query.model.clone()),
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
failure_kind: non_empty(query.failure_class.clone()),
|
||||
status_code,
|
||||
search: non_empty(query.search.clone()),
|
||||
},
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
page: page_number,
|
||||
page_size: page_size_value,
|
||||
},
|
||||
page,
|
||||
page_size,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn decode_observability_cursor(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<domain::ObservabilityCursor>, WireValidationError> {
|
||||
value
|
||||
.map(|encoded| {
|
||||
if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
let bytes = URL_SAFE_NO_PAD
|
||||
.decode(encoded)
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
let wire: CursorWire =
|
||||
serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?;
|
||||
if wire.stable_id.trim().is_empty() {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(domain::ObservabilityCursor {
|
||||
observed_at: wire.observed_at,
|
||||
stable_id: wire.stable_id,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn encode_observability_cursor(
|
||||
cursor: &domain::ObservabilityCursor,
|
||||
) -> Result<String, WireValidationError> {
|
||||
let bytes = serde_json::to_vec(&CursorWire {
|
||||
observed_at: cursor.observed_at,
|
||||
stable_id: cursor.stable_id.clone(),
|
||||
})
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
Ok(URL_SAFE_NO_PAD.encode(bytes))
|
||||
}
|
||||
|
||||
pub(crate) fn non_empty(value: Option<String>) -> Option<String> {
|
||||
value
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//! 固定 `/api/admin` 观测路由与 handler。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/api/admin/dashboard/summary", get(dashboard_summary::<S>))
|
||||
.route("/api/admin/dashboard/trend", get(dashboard_trend::<S>))
|
||||
.route("/api/admin/usage/records", get(usage_records::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/records/detail",
|
||||
get(usage_record_detail::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/records/summary",
|
||||
get(usage_records_summary::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/insights/overview",
|
||||
get(usage_insights_overview::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
get(usage_insights_diagnostics::<S>),
|
||||
)
|
||||
.route("/api/admin/operations/errors", get(ops_errors::<S>))
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_summary<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DashboardQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.dashboard_summary(range, domain_trend_kind(kind))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(dashboard_view(result, kind)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_trend<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DashboardQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.dashboard_trend(range, domain_trend_kind(kind))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(trend_view(result, kind)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_records<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (command, page, page_size) = usage_command(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_records(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = usage_page_view(result, page, page_size)
|
||||
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_record_detail<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DetailQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
query.validate().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_record_detail(query.id.trim())
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_detail_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_records_summary<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_summary_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_overview<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_insights(range, filter)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_insights_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_diagnostics<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DiagnosticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let dimension = query.dimension().map_err(map_wire_error)?;
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
model: non_empty(query.model),
|
||||
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
|
||||
search: non_empty(query.search),
|
||||
..domain::UsageFilter::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.diagnostics(range, filter, domain_diagnostic_dimension(dimension))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(diagnostics_view(result, dimension)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn ops_errors<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<OpsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (command, page, page_size) = ops_command(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.ops_errors(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = ops_page_view(result, page, page_size)
|
||||
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
@@ -0,0 +1,712 @@
|
||||
//! 响应 DTO 与固定 wire 形状。
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 观测列表游标的稳定 wire 形状。
|
||||
#[derive(Clone, Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CursorWire {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
/// 观测列表响应数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PageData<T> {
|
||||
pub items: Vec<T>,
|
||||
pub page: PageMeta,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
/// Token 详情展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TokenDetailsView {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub input_tokens_display: String,
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens_display: String,
|
||||
pub cache_write_tokens_display: String,
|
||||
pub reasoning_tokens_display: String,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens_display: String,
|
||||
pub total_tokens_display: String,
|
||||
}
|
||||
|
||||
/// 按货币展示的成本。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CostView {
|
||||
pub currency: String,
|
||||
pub estimated_amount: String,
|
||||
}
|
||||
|
||||
/// 成本覆盖状态计数。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CostCoverageView {
|
||||
pub known: u64,
|
||||
pub partial: u64,
|
||||
pub unknown: u64,
|
||||
pub not_billable: u64,
|
||||
}
|
||||
|
||||
/// Provider 受控价格规则生成的单次请求费用明细展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BillingView {
|
||||
pub input_amount_display: String,
|
||||
pub output_amount_display: String,
|
||||
pub cache_read_amount_display: String,
|
||||
pub cache_write_amount_display: String,
|
||||
pub standard_amount_display: String,
|
||||
pub total_amount_display: String,
|
||||
pub input_price_display: String,
|
||||
pub output_price_display: String,
|
||||
pub cache_read_price_display: String,
|
||||
pub cache_write_price_display: String,
|
||||
pub service_tier_display: String,
|
||||
pub multiplier_display: String,
|
||||
}
|
||||
|
||||
/// 单条逻辑请求展示。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordView {
|
||||
pub id: String,
|
||||
pub request_id: String,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub route: String,
|
||||
pub model: String,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub protocol: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub http_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub client_status_code: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub websocket_pool: Option<WebSocketPoolMetadataView>,
|
||||
pub image_generation_requested: bool,
|
||||
pub image_generation_succeeded: Option<bool>,
|
||||
pub latency_details: UsageLatencyDetailsView,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub attempt_count: u64,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub message: String,
|
||||
/// Provider 安全观测;Core 字段由顶层字段提供,不再复制进 metadata。
|
||||
pub metadata: BTreeMap<String, Value>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_at_display: String,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub token_details: TokenDetailsView,
|
||||
pub billing: Option<BillingView>,
|
||||
pub costs: Vec<CostView>,
|
||||
pub cost_coverage: CostCoverageView,
|
||||
pub first_token_latency_ms: Option<u64>,
|
||||
pub first_token_latency_ms_display: String,
|
||||
pub latency_ms_display: String,
|
||||
pub logical_outcome: String,
|
||||
}
|
||||
|
||||
/// WebSocket 池决策的稳定形状;与 Provider 选择逻辑无关。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct WebSocketPoolMetadataView {
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
/// 逻辑请求在上游和输出阶段测得的时延事实。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageLatencyDetailsView {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub transport_decision_wait_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ws_connect_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_headers_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_event_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_reasoning_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_text_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_token_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub openai_processing_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// 单次上游尝试展示。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageAttemptView {
|
||||
pub id: String,
|
||||
pub attempt_index: u32,
|
||||
pub trigger: String,
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub transport: String,
|
||||
pub send_state: String,
|
||||
pub outcome: String,
|
||||
pub downstream_committed: bool,
|
||||
pub status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub failure_class: Option<String>,
|
||||
pub cost_estimate_status: String,
|
||||
pub estimated_cost_amount: Option<String>,
|
||||
pub estimated_cost_currency: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub credential_name: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 逻辑请求详情与其尝试列表。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordDetailView {
|
||||
#[serde(flatten)]
|
||||
pub request: UsageRecordView,
|
||||
pub attempts: Vec<UsageAttemptView>,
|
||||
/// 尝试列表是否完整;best-effort 下恒为 false。
|
||||
pub attempts_complete: bool,
|
||||
}
|
||||
|
||||
/// Dashboard 趋势数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrendData {
|
||||
pub kind: TrendKind,
|
||||
pub points: Vec<TrendPointView>,
|
||||
pub summary: Vec<TrendSummaryView>,
|
||||
}
|
||||
|
||||
/// Dashboard 单个趋势桶。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrendPointView {
|
||||
pub time: String,
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub requests: String,
|
||||
pub requests_value: u64,
|
||||
pub input_tokens: String,
|
||||
pub input_tokens_value: u64,
|
||||
pub output_tokens: String,
|
||||
pub output_tokens_value: u64,
|
||||
pub cached_tokens: String,
|
||||
pub cached_tokens_value: u64,
|
||||
pub cache_hit_rate_value: f64,
|
||||
pub tokens_value: u64,
|
||||
pub errors: String,
|
||||
pub errors_value: u64,
|
||||
pub latency: String,
|
||||
pub latency_value: Option<u64>,
|
||||
pub max_latency: String,
|
||||
pub max_latency_value: Option<u64>,
|
||||
pub min_latency: String,
|
||||
pub min_latency_value: Option<u64>,
|
||||
pub success_rate: String,
|
||||
pub success_rate_value: Option<f64>,
|
||||
}
|
||||
|
||||
/// Dashboard 趋势摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct TrendSummaryView {
|
||||
pub label: String,
|
||||
pub value: String,
|
||||
pub ratio: Option<String>,
|
||||
}
|
||||
|
||||
/// Dashboard 卡片集合。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCardsView {
|
||||
pub credentials: DashboardCredentialsCardView,
|
||||
pub traffic: DashboardTrafficCardView,
|
||||
pub tokens: DashboardTokensCardView,
|
||||
pub cache: DashboardCacheCardView,
|
||||
}
|
||||
|
||||
/// Dashboard 上游凭据卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCredentialsCardView {
|
||||
pub total: String,
|
||||
pub total_value: u64,
|
||||
pub available: String,
|
||||
pub available_value: u64,
|
||||
pub unavailable: String,
|
||||
pub unavailable_value: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 流量卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardTrafficCardView {
|
||||
pub today_requests: String,
|
||||
pub today_requests_value: u64,
|
||||
pub yesterday_requests_value: u64,
|
||||
pub total_requests: String,
|
||||
}
|
||||
|
||||
/// Dashboard token 卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardTokensCardView {
|
||||
pub today_tokens: String,
|
||||
pub today_tokens_value: u64,
|
||||
pub yesterday_tokens_value: u64,
|
||||
pub total_tokens: String,
|
||||
pub total_billing_amount_usd: String,
|
||||
}
|
||||
|
||||
/// Dashboard 缓存卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCacheCardView {
|
||||
pub today_hit_rate: String,
|
||||
pub today_hit_rate_value: Option<f64>,
|
||||
pub yesterday_hit_rate_value: Option<f64>,
|
||||
pub total_hit_rate: String,
|
||||
pub total_cached_tokens: String,
|
||||
pub average_first_token_latency_ms: String,
|
||||
}
|
||||
|
||||
/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardAccountUsageView {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
pub authentication_kind: String,
|
||||
pub email: String,
|
||||
pub plan_type: Option<String>,
|
||||
pub tokens: String,
|
||||
pub request_count: u64,
|
||||
pub request_buckets: Vec<DashboardAccountRequestBucketView>,
|
||||
pub quota_used_percent: Option<f64>,
|
||||
pub last_used: String,
|
||||
}
|
||||
|
||||
/// Dashboard 账号单小时请求数。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardAccountRequestBucketView {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// Provider 账号池的持久事实汇总。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
}
|
||||
|
||||
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCapacityInfoView {
|
||||
pub max_concurrent_per_account: u64,
|
||||
pub total_slots: u64,
|
||||
pub used_slots: Option<u64>,
|
||||
pub available_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 逻辑请求指标展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RequestMetricsView {
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 上游尝试指标展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AttemptMetricsView {
|
||||
pub attempt_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub rate_limited_count: u64,
|
||||
pub auth_failure_count: u64,
|
||||
pub provider5xx_count: u64,
|
||||
pub cost_coverage: CostCoverageView,
|
||||
pub costs: Vec<CostView>,
|
||||
}
|
||||
|
||||
/// 健康时间线单点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealthTimelinePointView {
|
||||
pub time: String,
|
||||
pub status: String,
|
||||
pub reliability_display: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 健康时间线。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealthTimelineView {
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub reliability_display: String,
|
||||
pub status: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub points: Vec<HealthTimelinePointView>,
|
||||
}
|
||||
|
||||
/// Dashboard 展示的实际 Provider 上游请求身份。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireProfileView {
|
||||
pub provider: String,
|
||||
pub product: String,
|
||||
pub version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub build: Option<String>,
|
||||
pub target: DashboardWireTargetView,
|
||||
pub user_agent: String,
|
||||
pub attributes: Vec<DashboardWireAttributeView>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verified_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub release: Option<DashboardDesktopReleaseView>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireAttributeView {
|
||||
pub label: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireTargetView {
|
||||
pub os_type: String,
|
||||
pub os_version: String,
|
||||
pub arch: String,
|
||||
pub terminal: String,
|
||||
}
|
||||
|
||||
/// 发布检查与启动画像分离;未检查时使用明确的 `unchecked` 状态。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardDesktopReleaseView {
|
||||
pub status: DashboardDesktopReleaseStatusView,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub checked_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latest_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latest_build: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Dashboard 发布检查的稳定 wire 状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum DashboardDesktopReleaseStatusView {
|
||||
Unchecked,
|
||||
Aligned,
|
||||
ReviewRequired,
|
||||
CheckFailed,
|
||||
}
|
||||
|
||||
impl From<domain::DesktopReleaseStatus> for DashboardDesktopReleaseStatusView {
|
||||
fn from(status: domain::DesktopReleaseStatus) -> Self {
|
||||
match status {
|
||||
domain::DesktopReleaseStatus::Unchecked => Self::Unchecked,
|
||||
domain::DesktopReleaseStatus::Current => Self::Aligned,
|
||||
domain::DesktopReleaseStatus::UpdateAvailable => Self::ReviewRequired,
|
||||
domain::DesktopReleaseStatus::Failed => Self::CheckFailed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dashboard 汇总响应数据。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardDataView {
|
||||
pub cards: DashboardCardsView,
|
||||
pub trend: TrendData,
|
||||
pub health_timeline: HealthTimelineView,
|
||||
pub wire_profiles: Vec<DashboardWireProfileView>,
|
||||
pub account_usage: Vec<DashboardAccountUsageView>,
|
||||
pub usage_records: Vec<UsageRecordView>,
|
||||
pub pool_summary: DashboardPoolSummaryView,
|
||||
pub capacity_info: DashboardCapacityInfoView,
|
||||
pub rotation_strategy: String,
|
||||
}
|
||||
|
||||
/// 用量汇总响应数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageSummaryView {
|
||||
pub total_requests: String,
|
||||
pub input_tokens: String,
|
||||
pub output_tokens: String,
|
||||
pub cached_tokens: String,
|
||||
pub cache_write_tokens: String,
|
||||
pub total_tokens: String,
|
||||
pub average_latency_ms: String,
|
||||
pub logical_requests: RequestMetricsView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
}
|
||||
|
||||
/// 洞察健康趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewHealthPointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub error_rate: f64,
|
||||
}
|
||||
|
||||
/// 洞察健康摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewHealthView {
|
||||
pub total_requests: u64,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub success_rate: f64,
|
||||
pub request_change_rate: Option<f64>,
|
||||
pub success_rate_change: Option<f64>,
|
||||
pub points: Vec<OverviewHealthPointView>,
|
||||
}
|
||||
|
||||
/// 洞察性能趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewPerformancePointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub latency_p50_ms: Option<f64>,
|
||||
pub latency_p95_ms: Option<f64>,
|
||||
pub latency_p99_ms: Option<f64>,
|
||||
pub first_token_p50_ms: Option<f64>,
|
||||
pub first_token_p95_ms: Option<f64>,
|
||||
pub first_token_p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// 洞察性能摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewPerformanceView {
|
||||
pub latency_p50_ms: Option<f64>,
|
||||
pub latency_p95_ms: Option<f64>,
|
||||
pub latency_p99_ms: Option<f64>,
|
||||
pub first_token_p50_ms: Option<f64>,
|
||||
pub first_token_p95_ms: Option<f64>,
|
||||
pub first_token_p99_ms: Option<f64>,
|
||||
pub latency_coverage: f64,
|
||||
pub first_token_coverage: f64,
|
||||
pub points: Vec<OverviewPerformancePointView>,
|
||||
}
|
||||
|
||||
/// 洞察成本趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewCostPointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub standard_cost: Option<String>,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 洞察成本摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewCostView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub standard_cost: Option<String>,
|
||||
pub cost_per_request: Option<String>,
|
||||
pub tokens_per_request: f64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub points: Vec<OverviewCostPointView>,
|
||||
pub costs: Vec<CostView>,
|
||||
pub coverage: CostCoverageView,
|
||||
}
|
||||
|
||||
/// Provider 维度洞察摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ProviderOverviewView {
|
||||
pub provider: String,
|
||||
pub request_count: u64,
|
||||
pub attempt_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 用量洞察总响应。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageInsightsOverviewView {
|
||||
pub granularity: String,
|
||||
pub health: OverviewHealthView,
|
||||
pub performance: OverviewPerformanceView,
|
||||
pub cost: OverviewCostView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
pub providers: Vec<ProviderOverviewView>,
|
||||
}
|
||||
|
||||
/// 诊断聚合项目。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticItemView {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub error_count: u64,
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 诊断聚合响应。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticsView {
|
||||
pub dimension: String,
|
||||
pub items: Vec<DiagnosticItemView>,
|
||||
}
|
||||
|
||||
/// 运维错误项目。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OpsErrorView {
|
||||
pub id: String,
|
||||
pub request_id: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub failure_class: String,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub metadata: OpsErrorMetadataView,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_at_display: String,
|
||||
}
|
||||
|
||||
/// 运维错误安全元数据;只保留可查询的标识,不回显秘密材料。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OpsErrorMetadataView {
|
||||
pub source: String,
|
||||
pub component: String,
|
||||
pub attempt_id: Option<String>,
|
||||
pub account_label: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
//! Admin API 的展示层辅助:数值格式化。
|
||||
//!
|
||||
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
|
||||
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
|
||||
|
||||
/// 千分位格式化。
|
||||
#[must_use]
|
||||
pub fn format_number(value: u64) -> String {
|
||||
let text = value.to_string();
|
||||
let mut output = String::with_capacity(text.len() + text.len() / 3);
|
||||
for (index, character) in text.chars().rev().enumerate() {
|
||||
if index > 0 && index % 3 == 0 {
|
||||
output.push(',');
|
||||
}
|
||||
output.push(character);
|
||||
}
|
||||
output.chars().rev().collect()
|
||||
}
|
||||
|
||||
/// 紧凑格式化:≥1000 时用 K/M/B/T/P 后缀,否则千分位。
|
||||
#[must_use]
|
||||
pub fn format_compact_number(value: u64) -> String {
|
||||
if value < 1_000 {
|
||||
return format_number(value);
|
||||
}
|
||||
for (suffix, threshold) in [
|
||||
("P", 1_000_000_000_000_000_u64),
|
||||
("T", 1_000_000_000_000_u64),
|
||||
("B", 1_000_000_000_u64),
|
||||
("M", 1_000_000_u64),
|
||||
("K", 1_000_u64),
|
||||
] {
|
||||
if value >= threshold {
|
||||
let scaled = value as f64 / threshold as f64;
|
||||
return format!("{scaled:.1}{suffix}").replace(".0", "");
|
||||
}
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
@@ -72,10 +72,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
if i64::try_from(self.request_interval_ms).is_err() {
|
||||
return Err(WireValidationError::new("requestIntervalMs"));
|
||||
}
|
||||
if !matches!(
|
||||
self.rotation_strategy.as_str(),
|
||||
"smart" | "quota_reset_priority" | "round_robin" | "sticky"
|
||||
) {
|
||||
if RotationStrategy::parse(&self.rotation_strategy).is_none() {
|
||||
return Err(WireValidationError::new("rotationStrategy"));
|
||||
}
|
||||
Ok(())
|
||||
@@ -91,7 +88,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
|
||||
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
|
||||
request_interval_ms: self.request_interval_ms,
|
||||
rotation_strategy: parse_rotation_strategy(&self.rotation_strategy)
|
||||
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
|
||||
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
|
||||
@@ -111,7 +108,7 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
rotation_strategy: rotation_strategy_name(settings.rotation_strategy).to_owned(),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
@@ -319,25 +316,6 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_rotation_strategy(value: &str) -> Option<RotationStrategy> {
|
||||
match value {
|
||||
"smart" => Some(RotationStrategy::Smart),
|
||||
"quota_reset_priority" => Some(RotationStrategy::QuotaResetPriority),
|
||||
"round_robin" => Some(RotationStrategy::RoundRobin),
|
||||
"sticky" => Some(RotationStrategy::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const fn rotation_strategy_name(strategy: RotationStrategy) -> &'static str {
|
||||
match strategy {
|
||||
RotationStrategy::Smart => "smart",
|
||||
RotationStrategy::QuotaResetPriority => "quota_reset_priority",
|
||||
RotationStrategy::RoundRobin => "round_robin",
|
||||
RotationStrategy::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= max_len
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderName, HeaderValue, Method, StatusCode};
|
||||
use axum::http::{HeaderName, HeaderValue, Method, Request, StatusCode};
|
||||
use axum::routing::get;
|
||||
use gateway_admin::AdminServices;
|
||||
use gateway_core::engine::execution::ExecutionService;
|
||||
@@ -166,10 +166,32 @@ pub fn initialize(
|
||||
Duration::from_secs(seconds),
|
||||
));
|
||||
}
|
||||
// Trace 必须在 SetRequestId 内侧,span 才能捕获本服务生成的 request_id;
|
||||
// 默认 DEBUG span 会被 info 日志过滤器丢弃,这里显式用 info_span。
|
||||
let trace_layer = TraceLayer::new_for_http().make_span_with({
|
||||
let request_id_header = request_id_header.clone();
|
||||
move |request: &Request<_>| {
|
||||
let request_id = request
|
||||
.headers()
|
||||
.get(&request_id_header)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
tracing::info_span!(
|
||||
"request",
|
||||
request_id = %request_id,
|
||||
method = %request.method().as_str(),
|
||||
uri = %request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(axum::http::uri::PathAndQuery::as_str)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
});
|
||||
let router = router
|
||||
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
|
||||
.layer(trace_layer)
|
||||
.layer(SetRequestIdLayer::new(request_id_header, MakeRequestUuid))
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state);
|
||||
Ok(ApiBundle { router })
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::{EngineError, UpstreamSendState},
|
||||
error::{GatewayError, GatewayErrorKind, ProviderErrorKind},
|
||||
engine::EngineError,
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -72,19 +72,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
#[must_use]
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
match error {
|
||||
EngineError::Provider(error)
|
||||
if error.kind() == ProviderErrorKind::Unavailable
|
||||
&& error.send_state() == UpstreamSendState::NotSent =>
|
||||
{
|
||||
let gateway = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
);
|
||||
match error.client_visible_upstream_error() {
|
||||
Some(upstream) => gateway.with_client_visible_upstream_error(upstream.clone()),
|
||||
None => gateway,
|
||||
}
|
||||
}
|
||||
EngineError::Provider(error) => GatewayError::from_provider(error),
|
||||
EngineError::Cancelled => {
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
@@ -185,6 +172,16 @@ pub const fn gateway_error_contract(
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
|
||||
@@ -470,6 +470,15 @@ fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent
|
||||
// 不能透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
// 设备 attestation/integrity 头只能由官方客户端或网关自身生成,
|
||||
// 客户端注入的 x-oai-attestation / X-OAI-IS 不得透传上游。
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -331,7 +344,7 @@ mod actions {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
@@ -340,7 +353,7 @@ mod actions {
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -365,7 +378,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
@@ -374,7 +387,7 @@ mod actions {
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "active"
|
||||
"accountStatus": "quota_exhausted"
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -23,9 +23,9 @@ use gateway_admin::{
|
||||
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
|
||||
},
|
||||
observability::{
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsErrorPage,
|
||||
OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter, UsageOverview,
|
||||
UsagePage, UsageQuery, UsageRecord,
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
|
||||
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageRecord,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -79,6 +79,10 @@ pub(super) struct AdminTestFixture {
|
||||
pub auth: Arc<MemoryAuthStore>,
|
||||
pub settings: Arc<MemorySettingsStore>,
|
||||
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
pub usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -92,8 +96,16 @@ impl AdminTestFixture {
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
let client_keys = Arc::new(MemoryClientKeyStore);
|
||||
let usage_records = Arc::new(Mutex::new(Vec::new()));
|
||||
let usage_detail = Arc::new(Mutex::new(None));
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
@@ -101,6 +113,7 @@ impl AdminTestFixture {
|
||||
client_keys.clone(),
|
||||
unused,
|
||||
settings.clone(),
|
||||
gateway_admin::ports::backup::BackupStorePorts::disabled(),
|
||||
);
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new("openai")),
|
||||
@@ -125,6 +138,10 @@ impl AdminTestFixture {
|
||||
auth,
|
||||
settings,
|
||||
usage_records,
|
||||
usage_detail,
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -392,6 +409,10 @@ impl ClientKeyStore for MemoryClientKeyStore {
|
||||
|
||||
struct UnusedStore {
|
||||
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -503,7 +524,11 @@ impl AccountStore for UnusedStore {
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
Err(unavailable("dashboard"))
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("dashboard"))
|
||||
}
|
||||
|
||||
async fn dashboard_trend(&self, _: TimeRange) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
@@ -537,7 +562,11 @@ impl ObservabilityStore for UnusedStore {
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
Err(unavailable("usage detail"))
|
||||
self.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
@@ -550,11 +579,16 @@ impl ObservabilityStore for UnusedStore {
|
||||
_: UsageFilter,
|
||||
_: DiagnosticDimension,
|
||||
) -> AdminStoreResult<Vec<DiagnosticObservation>> {
|
||||
Err(unavailable("usage diagnostics"))
|
||||
Ok(self.diagnostics.lock().expect("diagnostics").clone())
|
||||
}
|
||||
|
||||
async fn list_ops_errors(&self, _: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
Err(unavailable("ops errors"))
|
||||
let items = self.ops_errors.lock().expect("ops errors").clone();
|
||||
Ok(OpsErrorPage {
|
||||
total: items.len() as u64,
|
||||
items,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
mod query {
|
||||
use gateway_api::admin::observability::{
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
|
||||
parse_attempt_index, parse_datetime, parse_status,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_bound_page_size_and_cursor() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"page": 2,
|
||||
"pageSize": 100,
|
||||
"cursor": "opaque"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(query.validate_page().unwrap(), (2, 100));
|
||||
assert!(query.validate_cursor().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_zero_page() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
|
||||
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
|
||||
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
|
||||
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
|
||||
assert_eq!(
|
||||
parse_attempt_index(Some(0)).unwrap_err().field(),
|
||||
"attemptIndex"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_datetime(Some("not-a-time")).unwrap_err().field(),
|
||||
"timeRange"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod response {
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::observability::DesktopReleaseStatus;
|
||||
use gateway_api::admin::observability::{
|
||||
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
|
||||
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
|
||||
DashboardWireProfileView, DashboardWireTargetView, PageData, PageMeta, TokenDetailsView,
|
||||
TrendData, TrendKind, TrendPointView, TrendSummaryView,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn usage_page_should_keep_terminal_camel_case_shape() {
|
||||
let data = PageData {
|
||||
items: vec![json!({"id": "request_1"})],
|
||||
page: PageMeta::new(1, 50, 1, 1),
|
||||
next_cursor: Some("cursor".to_owned()),
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["page"]["pageSize"], 50);
|
||||
assert_eq!(value["nextCursor"], "cursor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
|
||||
let value = serde_json::to_value(DashboardWireProfileView {
|
||||
provider: "xai".to_owned(),
|
||||
product: "Grok Build".to_owned(),
|
||||
version: "0.2.106".to_owned(),
|
||||
build: None,
|
||||
target: DashboardWireTargetView {
|
||||
os_type: "linux".to_owned(),
|
||||
os_version: "—".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "headless".to_owned(),
|
||||
},
|
||||
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
|
||||
attributes: vec![DashboardWireAttributeView {
|
||||
label: "客户端标识".to_owned(),
|
||||
value: "grok-shell".to_owned(),
|
||||
}],
|
||||
verified_at: None,
|
||||
release: None,
|
||||
})
|
||||
.expect("dashboard profile");
|
||||
|
||||
assert_eq!(value["provider"], "xai");
|
||||
assert_eq!(value["version"], "0.2.106");
|
||||
assert_eq!(value["attributes"][0]["label"], "客户端标识");
|
||||
assert!(value.get("release").is_none());
|
||||
assert!(value.get("verifiedAt").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_account_usage_should_keep_daily_request_timeline() {
|
||||
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let value = serde_json::to_value(DashboardAccountUsageView {
|
||||
id: "account_1".to_owned(),
|
||||
provider: "xai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
email: "account@example.com".to_owned(),
|
||||
plan_type: Some("free".to_owned()),
|
||||
tokens: "—".to_owned(),
|
||||
request_count: 3,
|
||||
request_buckets: vec![DashboardAccountRequestBucketView {
|
||||
bucket_start,
|
||||
request_count: 3,
|
||||
}],
|
||||
quota_used_percent: None,
|
||||
last_used: "刚刚".to_owned(),
|
||||
})
|
||||
.expect("dashboard account usage");
|
||||
|
||||
assert_eq!(value["requestCount"], 3);
|
||||
assert_eq!(
|
||||
value["requestBuckets"][0]["bucketStart"],
|
||||
"1970-01-01T00:00:00Z"
|
||||
);
|
||||
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
|
||||
let now = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let data = TrendData {
|
||||
kind: TrendKind::Usage,
|
||||
points: vec![TrendPointView {
|
||||
time: "08:00".to_owned(),
|
||||
bucket: now,
|
||||
label: "01-01 08:00".to_owned(),
|
||||
requests: "1".to_owned(),
|
||||
requests_value: 1,
|
||||
input_tokens: "2".to_owned(),
|
||||
input_tokens_value: 2,
|
||||
output_tokens: "3".to_owned(),
|
||||
output_tokens_value: 3,
|
||||
cached_tokens: "0".to_owned(),
|
||||
cached_tokens_value: 0,
|
||||
cache_hit_rate_value: 0.0,
|
||||
tokens_value: 5,
|
||||
errors: "0".to_owned(),
|
||||
errors_value: 0,
|
||||
latency: "1 ms".to_owned(),
|
||||
latency_value: Some(1),
|
||||
first_token_latency: "1 ms".to_owned(),
|
||||
first_token_latency_value: Some(1),
|
||||
max_latency: "1 ms".to_owned(),
|
||||
max_latency_value: Some(1),
|
||||
min_latency: "1 ms".to_owned(),
|
||||
min_latency_value: Some(1),
|
||||
success_rate: "100.0%".to_owned(),
|
||||
success_rate_value: Some(100.0),
|
||||
}],
|
||||
summary: vec![TrendSummaryView {
|
||||
label: "输入".to_owned(),
|
||||
value: "2".to_owned(),
|
||||
ratio: None,
|
||||
}],
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["kind"], "usage");
|
||||
assert_eq!(value["points"][0]["requestsValue"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
|
||||
let coverage = CostCoverageView {
|
||||
known: 1,
|
||||
partial: 0,
|
||||
unknown: 0,
|
||||
not_billable: 0,
|
||||
};
|
||||
let token_details = TokenDetailsView {
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(2),
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(3),
|
||||
input_tokens_display: "1".to_owned(),
|
||||
output_tokens_display: "2".to_owned(),
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
cache_write_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens_display: "-".to_owned(),
|
||||
total_tokens_display: "3".to_owned(),
|
||||
};
|
||||
let cursor = CursorWire {
|
||||
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
|
||||
stable_id: "request_1".to_owned(),
|
||||
};
|
||||
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
|
||||
assert!(value.to_string().contains("known"));
|
||||
assert!(!value.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_view_should_preserve_the_original_detail_contract() {
|
||||
let value = serde_json::to_value(BillingView {
|
||||
input_amount_display: "$0.03".to_owned(),
|
||||
output_amount_display: "$0.00".to_owned(),
|
||||
cache_read_amount_display: "$0.14".to_owned(),
|
||||
cache_write_amount_display: "$0.00".to_owned(),
|
||||
standard_amount_display: "$0.17".to_owned(),
|
||||
total_amount_display: "$0.17".to_owned(),
|
||||
input_price_display: "$10.0000 / 1M Token".to_owned(),
|
||||
output_price_display: "$60.0000 / 1M Token".to_owned(),
|
||||
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
|
||||
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
|
||||
service_tier_display: "Fast".to_owned(),
|
||||
multiplier_display: "1.00x".to_owned(),
|
||||
})
|
||||
.expect("billing view");
|
||||
|
||||
assert_eq!(value["inputAmountDisplay"], "$0.03");
|
||||
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
|
||||
assert_eq!(value["serviceTierDisplay"], "Fast");
|
||||
assert_eq!(value["multiplierDisplay"], "1.00x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
|
||||
for (domain, expected) in [
|
||||
(DesktopReleaseStatus::Unchecked, "unchecked"),
|
||||
(DesktopReleaseStatus::Current, "aligned"),
|
||||
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
|
||||
(DesktopReleaseStatus::Failed, "check_failed"),
|
||||
] {
|
||||
let status = DashboardDesktopReleaseStatusView::from(domain);
|
||||
assert_eq!(serde_json::to_value(status).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?outcome=provider_future_state")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_other_outcome")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageRecord {
|
||||
id: "request_endpoint".to_owned(),
|
||||
client_api_key_ref: "key_endpoint".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
provider_account_email: None,
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
"transport": "must-not-overwrite-core",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: Some("resp_usage_metadata".to_owned()),
|
||||
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(31),
|
||||
image_output_tokens: Some(9),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
first_event_ms: Some(17),
|
||||
first_reasoning_ms: Some(19),
|
||||
first_text_ms: Some(23),
|
||||
first_token_ms: Some(19),
|
||||
provider_processing_ms: Some(29),
|
||||
latency_ms: Some(31),
|
||||
client_ip: Some("127.0.0.1".to_owned()),
|
||||
user_agent: Some("usage-metadata-test".to_owned()),
|
||||
reasoning_effort: Some("max".to_owned()),
|
||||
reasoning_preset: Some("ultra".to_owned()),
|
||||
request_kind: Some("review".to_owned()),
|
||||
subagent_kind: Some("worker".to_owned()),
|
||||
compact: true,
|
||||
image_generation_requested: true,
|
||||
image_generation_succeeded: Some(true),
|
||||
started_at: Utc::now(),
|
||||
deadline_at: Utc::now(),
|
||||
completed_at: Some(Utc::now()),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_endpoint")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
"websocketPool": value["data"]["items"][0]["metadata"]["websocketPool"],
|
||||
"imageGenerationRequested": value["data"]["items"][0]["metadata"]["imageGenerationRequested"],
|
||||
"imageGenerationSucceeded": value["data"]["items"][0]["metadata"]["imageGenerationSucceeded"],
|
||||
"requestedModel": value["data"]["items"][0]["metadata"]["requestedModel"],
|
||||
"upstreamModel": value["data"]["items"][0]["metadata"]["upstreamModel"],
|
||||
"clientIp": value["data"]["items"][0]["metadata"]["clientIp"],
|
||||
"userAgent": value["data"]["items"][0]["metadata"]["userAgent"],
|
||||
"reasoningEffort": value["data"]["items"][0]["metadata"]["reasoningEffort"],
|
||||
"reasoningPreset": value["data"]["items"][0]["metadata"]["reasoningPreset"],
|
||||
"requestKind": value["data"]["items"][0]["metadata"]["requestKind"],
|
||||
"subagentKind": value["data"]["items"][0]["metadata"]["subagentKind"],
|
||||
"compact": value["data"]["items"][0]["metadata"]["compact"],
|
||||
"transport": value["data"]["items"][0]["metadata"]["transport"],
|
||||
"transportDecisionWaitMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["transportDecisionWaitMs"],
|
||||
"wsConnectMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["wsConnectMs"],
|
||||
"firstReasoningMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstReasoningMs"],
|
||||
"firstTextMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTextMs"],
|
||||
"firstTokenMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTokenMs"],
|
||||
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
|
||||
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
"websocketPool": {"kind": "reuse"},
|
||||
"imageGenerationRequested": true,
|
||||
"imageGenerationSucceeded": true,
|
||||
"requestedModel": "grok-4.5",
|
||||
"upstreamModel": "grok-4.5",
|
||||
"clientIp": "127.0.0.1",
|
||||
"userAgent": "usage-metadata-test",
|
||||
"reasoningEffort": "max",
|
||||
"reasoningPreset": "ultra",
|
||||
"requestKind": "review",
|
||||
"subagentKind": "worker",
|
||||
"compact": true,
|
||||
"transport": "http_sse",
|
||||
"transportDecisionWaitMs": 7,
|
||||
"wsConnectMs": 11,
|
||||
"firstReasoningMs": 19,
|
||||
"firstTextMs": 23,
|
||||
"firstTokenMs": 19,
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
//! Admin 观测 wire/路由的固定合同测试。
|
||||
|
||||
mod query;
|
||||
mod response;
|
||||
@@ -0,0 +1,64 @@
|
||||
//! 查询 wire 校验的固定合同测试。
|
||||
|
||||
use gateway_api::admin::observability::{
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
|
||||
parse_attempt_index, parse_datetime, parse_status,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_bound_page_size_and_cursor() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"page": 2,
|
||||
"pageSize": 100,
|
||||
"cursor": "opaque"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(query.validate_page().unwrap(), (2, 100));
|
||||
assert!(query.validate_cursor().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_zero_page() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
|
||||
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
|
||||
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
|
||||
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
|
||||
assert_eq!(
|
||||
parse_attempt_index(Some(0)).unwrap_err().field(),
|
||||
"attemptIndex"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_datetime(Some("not-a-time")).unwrap_err().field(),
|
||||
"timeRange"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,848 @@
|
||||
//! 响应 DTO 与固定 wire 形状的合同测试。
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::observability::DesktopReleaseStatus;
|
||||
use gateway_api::admin::PageMeta;
|
||||
use gateway_api::admin::observability::{
|
||||
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
|
||||
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
|
||||
DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData,
|
||||
TrendKind, TrendPointView, TrendSummaryView,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn usage_page_should_keep_terminal_camel_case_shape() {
|
||||
let data = PageData {
|
||||
items: vec![json!({"id": "request_1"})],
|
||||
page: PageMeta::new(1, 50, 1, 1),
|
||||
next_cursor: Some("cursor".to_owned()),
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["page"]["pageSize"], 50);
|
||||
assert_eq!(value["nextCursor"], "cursor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
|
||||
let value = serde_json::to_value(DashboardWireProfileView {
|
||||
provider: "xai".to_owned(),
|
||||
product: "Grok Build".to_owned(),
|
||||
version: "0.2.106".to_owned(),
|
||||
build: None,
|
||||
target: DashboardWireTargetView {
|
||||
os_type: "linux".to_owned(),
|
||||
os_version: "—".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "headless".to_owned(),
|
||||
},
|
||||
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
|
||||
attributes: vec![DashboardWireAttributeView {
|
||||
label: "客户端标识".to_owned(),
|
||||
value: "grok-shell".to_owned(),
|
||||
}],
|
||||
verified_at: None,
|
||||
release: None,
|
||||
})
|
||||
.expect("dashboard profile");
|
||||
|
||||
assert_eq!(value["provider"], "xai");
|
||||
assert_eq!(value["version"], "0.2.106");
|
||||
assert_eq!(value["attributes"][0]["label"], "客户端标识");
|
||||
assert!(value.get("release").is_none());
|
||||
assert!(value.get("verifiedAt").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_account_usage_should_keep_daily_request_timeline() {
|
||||
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let value = serde_json::to_value(DashboardAccountUsageView {
|
||||
id: "account_1".to_owned(),
|
||||
provider: "xai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
email: "account@example.com".to_owned(),
|
||||
plan_type: Some("free".to_owned()),
|
||||
tokens: "—".to_owned(),
|
||||
request_count: 3,
|
||||
request_buckets: vec![DashboardAccountRequestBucketView {
|
||||
bucket_start,
|
||||
request_count: 3,
|
||||
}],
|
||||
quota_used_percent: None,
|
||||
last_used: "刚刚".to_owned(),
|
||||
})
|
||||
.expect("dashboard account usage");
|
||||
|
||||
assert_eq!(value["requestCount"], 3);
|
||||
assert_eq!(
|
||||
value["requestBuckets"][0]["bucketStart"],
|
||||
"1970-01-01T00:00:00Z"
|
||||
);
|
||||
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
|
||||
let now = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let data = TrendData {
|
||||
kind: TrendKind::Usage,
|
||||
points: vec![TrendPointView {
|
||||
time: "08:00".to_owned(),
|
||||
bucket: now,
|
||||
label: "01-01 08:00".to_owned(),
|
||||
requests: "1".to_owned(),
|
||||
requests_value: 1,
|
||||
input_tokens: "2".to_owned(),
|
||||
input_tokens_value: 2,
|
||||
output_tokens: "3".to_owned(),
|
||||
output_tokens_value: 3,
|
||||
cached_tokens: "0".to_owned(),
|
||||
cached_tokens_value: 0,
|
||||
cache_hit_rate_value: 0.0,
|
||||
tokens_value: 5,
|
||||
errors: "0".to_owned(),
|
||||
errors_value: 0,
|
||||
latency: "1 ms".to_owned(),
|
||||
latency_value: Some(1),
|
||||
max_latency: "1 ms".to_owned(),
|
||||
max_latency_value: Some(1),
|
||||
min_latency: "1 ms".to_owned(),
|
||||
min_latency_value: Some(1),
|
||||
success_rate: "100.0%".to_owned(),
|
||||
success_rate_value: Some(100.0),
|
||||
}],
|
||||
summary: vec![TrendSummaryView {
|
||||
label: "输入".to_owned(),
|
||||
value: "2".to_owned(),
|
||||
ratio: None,
|
||||
}],
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["kind"], "usage");
|
||||
assert_eq!(value["points"][0]["requestsValue"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
|
||||
let coverage = CostCoverageView {
|
||||
known: 1,
|
||||
partial: 0,
|
||||
unknown: 0,
|
||||
not_billable: 0,
|
||||
};
|
||||
let token_details = TokenDetailsView {
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(2),
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(3),
|
||||
input_tokens_display: "1".to_owned(),
|
||||
output_tokens_display: "2".to_owned(),
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
cache_write_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens_display: "-".to_owned(),
|
||||
total_tokens_display: "3".to_owned(),
|
||||
};
|
||||
let cursor = CursorWire {
|
||||
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
|
||||
stable_id: "request_1".to_owned(),
|
||||
};
|
||||
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
|
||||
assert!(value.to_string().contains("known"));
|
||||
assert!(!value.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_view_should_preserve_the_original_detail_contract() {
|
||||
let value = serde_json::to_value(BillingView {
|
||||
input_amount_display: "$0.03".to_owned(),
|
||||
output_amount_display: "$0.00".to_owned(),
|
||||
cache_read_amount_display: "$0.14".to_owned(),
|
||||
cache_write_amount_display: "$0.00".to_owned(),
|
||||
standard_amount_display: "$0.17".to_owned(),
|
||||
total_amount_display: "$0.17".to_owned(),
|
||||
input_price_display: "$10.0000 / 1M Token".to_owned(),
|
||||
output_price_display: "$60.0000 / 1M Token".to_owned(),
|
||||
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
|
||||
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
|
||||
service_tier_display: "Fast".to_owned(),
|
||||
multiplier_display: "1.00x".to_owned(),
|
||||
})
|
||||
.expect("billing view");
|
||||
|
||||
assert_eq!(value["inputAmountDisplay"], "$0.03");
|
||||
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
|
||||
assert_eq!(value["serviceTierDisplay"], "Fast");
|
||||
assert_eq!(value["multiplierDisplay"], "1.00x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
|
||||
for (domain, expected) in [
|
||||
(DesktopReleaseStatus::Unchecked, "unchecked"),
|
||||
(DesktopReleaseStatus::Current, "aligned"),
|
||||
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
|
||||
(DesktopReleaseStatus::Failed, "check_failed"),
|
||||
] {
|
||||
let status = DashboardDesktopReleaseStatusView::from(domain);
|
||||
assert_eq!(serde_json::to_value(status).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
banned: 2,
|
||||
},
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_account_summary")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["credentials"]["totalValue"],
|
||||
&value["data"]["cards"]["credentials"]["availableValue"],
|
||||
&value["data"]["cards"]["credentials"]["unavailableValue"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!(807),
|
||||
&serde_json::json!(15),
|
||||
&serde_json::json!(792)
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{UsageAttempt, UsageDetail};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
fixture
|
||||
.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_detail",
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
"alpha@example.invalid",
|
||||
"oauth",
|
||||
now,
|
||||
),
|
||||
attempts: vec![UsageAttempt {
|
||||
source: "ops_event".to_owned(),
|
||||
id: "ops_detail".to_owned(),
|
||||
attempt_index: 1,
|
||||
component: "routing".to_owned(),
|
||||
operation: "fallback".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_snap_b".to_owned()),
|
||||
provider_account_name: Some("Snapshot Beta".to_owned()),
|
||||
provider_account_email: Some("beta@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-b".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
upstream_send_state: Some("sent".to_owned()),
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Failed,
|
||||
downstream_committed: false,
|
||||
status_code: Some(429),
|
||||
provider_error_code: Some("rate_limit".to_owned()),
|
||||
failure_kind: Some("rate_limited".to_owned()),
|
||||
retry_after_ms: Some(1_000),
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(120),
|
||||
message: Some("limited".to_owned()),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
total_tokens: None,
|
||||
cost_source: Some("unavailable".to_owned()),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
occurred_at: now,
|
||||
}],
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records/detail?id=req_detail")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_detail_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("usage detail request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage detail response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage detail body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage detail JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"accountId": value["data"]["accountId"],
|
||||
"accountName": value["data"]["accountName"],
|
||||
"accountEmail": value["data"]["accountEmail"],
|
||||
"authenticationKind": value["data"]["authenticationKind"],
|
||||
"attemptCredentialName": value["data"]["attempts"][0]["credentialName"],
|
||||
"attemptAccountId": value["data"]["attempts"][0]["accountId"],
|
||||
"attemptAccountName": value["data"]["attempts"][0]["accountName"],
|
||||
"attemptAccountEmail": value["data"]["attempts"][0]["accountEmail"],
|
||||
"attemptAuthenticationKind": value["data"]["attempts"][0]["authenticationKind"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"accountId": "acct_snap_a",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"attemptCredentialName": "Snapshot Beta",
|
||||
"attemptAccountId": "acct_snap_b",
|
||||
"attemptAccountName": "Snapshot Beta",
|
||||
"attemptAccountEmail": "beta@example.invalid",
|
||||
"attemptAuthenticationKind": "api_key",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::OpsError;
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.ops_errors
|
||||
.lock()
|
||||
.expect("ops errors")
|
||||
.push(OpsError {
|
||||
source: "model_request".to_owned(),
|
||||
event_id: "err_snapshot".to_owned(),
|
||||
request_id: Some("req_err".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
provider_account_name: None,
|
||||
provider_account_email: Some("err@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-err".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
failure_kind: "upstream_error".to_owned(),
|
||||
client_status_code: Some(502),
|
||||
upstream_status_code: Some(502),
|
||||
provider_error_code: Some("upstream".to_owned()),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
occurrence_count: 1,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/operations/errors")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_ops_errors_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("ops errors request"),
|
||||
)
|
||||
.await
|
||||
.expect("ops errors response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("ops errors body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("ops errors JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"provider": value["data"]["items"][0]["provider"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"kind": value["data"]["items"][0]["kind"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
|
||||
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"provider": "openai",
|
||||
"authenticationKind": "api_key",
|
||||
"kind": "model_request",
|
||||
"accountId": "acct_err",
|
||||
"accountLabel": "err@example.invalid",
|
||||
"clientStatusCode": 502,
|
||||
"route": "/v1/responses",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_admin::model::observability::{CostCoverage, DiagnosticObservation};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.diagnostics
|
||||
.lock()
|
||||
.expect("diagnostics")
|
||||
.push(DiagnosticObservation {
|
||||
key: "acct_diag".to_owned(),
|
||||
name: "diag@example.invalid".to_owned(),
|
||||
request_count: 2,
|
||||
success_count: 2,
|
||||
failure_count: 0,
|
||||
attempt_count: 2,
|
||||
total_tokens: 200,
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(3800),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/insights/diagnostics?dimension=account")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_diagnostics_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("diagnostics request"),
|
||||
)
|
||||
.await
|
||||
.expect("diagnostics response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["items"][0]["key"],
|
||||
&value["data"]["items"][0]["name"],
|
||||
&value["data"]["items"][0]["latencyP95Ms"],
|
||||
&value["data"]["dimension"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("acct_diag"),
|
||||
&serde_json::json!("diag@example.invalid"),
|
||||
&serde_json::json!(3800),
|
||||
&serde_json::json!("account"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
fn usage_record_with_account(
|
||||
id: &str,
|
||||
account_id: &str,
|
||||
name: &str,
|
||||
email: &str,
|
||||
authentication_kind: &str,
|
||||
started_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> gateway_admin::model::observability::UsageRecord {
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
|
||||
UsageRecord {
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_detail".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "coding".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some(account_id.to_owned()),
|
||||
provider_account_name: Some(name.to_owned()),
|
||||
provider_account_email: Some(email.to_owned()),
|
||||
provider_account_authentication_kind: Some(authentication_kind.to_owned()),
|
||||
upstream_model_id: Some("upstream-model".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: Some(started_at),
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
transport_decision_wait_ms: None,
|
||||
connect_ms: None,
|
||||
headers_ms: None,
|
||||
first_event_ms: None,
|
||||
first_reasoning_ms: None,
|
||||
first_text_ms: None,
|
||||
first_token_ms: None,
|
||||
provider_processing_ms: None,
|
||||
latency_ms: None,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
image_generation_succeeded: None,
|
||||
started_at,
|
||||
deadline_at: started_at + chrono::Duration::seconds(30),
|
||||
completed_at: Some(started_at),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?outcome=provider_future_state")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_other_outcome")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{
|
||||
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
|
||||
UsageRecord,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let usd = |amount: &str| CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: DecimalAmount::from_str(amount).expect("USD amount"),
|
||||
};
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageRecord {
|
||||
id: "request_endpoint".to_owned(),
|
||||
client_api_key_ref: "key_endpoint".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: Some("acct_snapshot".to_owned()),
|
||||
provider_account_name: Some("Snapshot Alpha".to_owned()),
|
||||
provider_account_email: Some("alpha@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
"transport": "must-not-overwrite-core",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: Some("resp_usage_metadata".to_owned()),
|
||||
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(31),
|
||||
image_output_tokens: Some(9),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Calculated(Box::new(
|
||||
CalculatedBillingBreakdown {
|
||||
input_amount: usd("0.03"),
|
||||
output_amount: usd("0.07"),
|
||||
cache_read_amount: usd("0.00"),
|
||||
cache_write_amount: usd("0.00"),
|
||||
standard_amount: usd("0.10"),
|
||||
total_amount: usd("0.10"),
|
||||
input_price_per_million: usd("10.0000"),
|
||||
output_price_per_million: usd("60.0000"),
|
||||
cache_read_price_per_million: usd("1.0000"),
|
||||
cache_write_price_per_million: usd("12.5000"),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
multiplier_percent: 100,
|
||||
},
|
||||
))),
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
first_event_ms: Some(17),
|
||||
first_reasoning_ms: Some(19),
|
||||
first_text_ms: Some(23),
|
||||
first_token_ms: Some(19),
|
||||
provider_processing_ms: Some(29),
|
||||
latency_ms: Some(31),
|
||||
client_ip: Some("127.0.0.1".to_owned()),
|
||||
user_agent: Some("usage-metadata-test".to_owned()),
|
||||
reasoning_effort: Some("max".to_owned()),
|
||||
reasoning_preset: Some("ultra".to_owned()),
|
||||
request_kind: Some("review".to_owned()),
|
||||
subagent_kind: Some("worker".to_owned()),
|
||||
compact: true,
|
||||
image_generation_requested: true,
|
||||
image_generation_succeeded: Some(true),
|
||||
started_at: Utc::now(),
|
||||
deadline_at: Utc::now(),
|
||||
completed_at: Some(Utc::now()),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_endpoint")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
|
||||
"$10 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
|
||||
"$60 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
|
||||
"$12.5 / 1M Token"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountName": value["data"]["items"][0]["accountName"],
|
||||
"accountEmail": value["data"]["items"][0]["accountEmail"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
"websocketPool": value["data"]["items"][0]["websocketPool"],
|
||||
"imageGenerationRequested": value["data"]["items"][0]["imageGenerationRequested"],
|
||||
"imageGenerationSucceeded": value["data"]["items"][0]["imageGenerationSucceeded"],
|
||||
"requestedModel": value["data"]["items"][0]["requestedModel"],
|
||||
"upstreamModel": value["data"]["items"][0]["upstreamModel"],
|
||||
"clientIp": value["data"]["items"][0]["clientIp"],
|
||||
"userAgent": value["data"]["items"][0]["userAgent"],
|
||||
"reasoningEffort": value["data"]["items"][0]["reasoningEffort"],
|
||||
"reasoningPreset": value["data"]["items"][0]["reasoningPreset"],
|
||||
"requestKind": value["data"]["items"][0]["requestKind"],
|
||||
"subagentKind": value["data"]["items"][0]["subagentKind"],
|
||||
"compact": value["data"]["items"][0]["compact"],
|
||||
"transport": value["data"]["items"][0]["transport"],
|
||||
"transportDecisionWaitMs": value["data"]["items"][0]["latencyDetails"]["transportDecisionWaitMs"],
|
||||
"wsConnectMs": value["data"]["items"][0]["latencyDetails"]["wsConnectMs"],
|
||||
"firstReasoningMs": value["data"]["items"][0]["latencyDetails"]["firstReasoningMs"],
|
||||
"firstTextMs": value["data"]["items"][0]["latencyDetails"]["firstTextMs"],
|
||||
"firstTokenMs": value["data"]["items"][0]["latencyDetails"]["firstTokenMs"],
|
||||
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
|
||||
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"accountId": "acct_snapshot",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
"websocketPool": {"kind": "reuse"},
|
||||
"imageGenerationRequested": true,
|
||||
"imageGenerationSucceeded": true,
|
||||
"requestedModel": "grok-4.5",
|
||||
"upstreamModel": "grok-4.5",
|
||||
"clientIp": "127.0.0.1",
|
||||
"userAgent": "usage-metadata-test",
|
||||
"reasoningEffort": "max",
|
||||
"reasoningPreset": "ultra",
|
||||
"requestKind": "review",
|
||||
"subagentKind": "worker",
|
||||
"compact": true,
|
||||
"transport": "http_sse",
|
||||
"transportDecisionWaitMs": 7,
|
||||
"wsConnectMs": 11,
|
||||
"firstReasoningMs": 19,
|
||||
"firstTextMs": 23,
|
||||
"firstTokenMs": 19,
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -62,6 +62,123 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
assert_eq!(request.validate().unwrap_err().field(), "rotationStrategy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::engine::credential::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
PublicModelId::new("gpt-5.4").expect("public model"),
|
||||
UpstreamModelId::new("gpt-5.5").expect("upstream model"),
|
||||
),
|
||||
(
|
||||
PublicModelId::new("grok-latest").expect("public model"),
|
||||
UpstreamModelId::new("grok-4.5").expect("upstream model"),
|
||||
),
|
||||
]),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 25,
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
.expect("timestamp"),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(RuntimeSettingsView::from(settings)).expect("serialize view");
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
},
|
||||
"refreshMarginSeconds": 1800,
|
||||
"refreshConcurrency": 4,
|
||||
"maxConcurrentPerAccount": 5,
|
||||
"requestIntervalMs": 25,
|
||||
"rotationStrategy": "round_robin",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"updatedAt": "2026-08-02T10:30:00Z"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::engine::credential::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(update_body()).expect("decode settings");
|
||||
request.validate().expect("fixture settings must validate");
|
||||
|
||||
let request_fields: BTreeSet<String> = update_body()
|
||||
.as_object()
|
||||
.expect("request body object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
.iter()
|
||||
.map(|(public, upstream)| {
|
||||
Ok((
|
||||
PublicModelId::new(public.clone())?,
|
||||
UpstreamModelId::new(upstream.clone())?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
|
||||
.expect("valid model mappings"),
|
||||
refresh_margin_seconds: request.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.max_concurrent_per_account).expect("u32"),
|
||||
request_interval_ms: request.request_interval_ms,
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
updated_at: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
let response_fields: BTreeSet<String> =
|
||||
serde_json::to_value(RuntimeSettingsView::from(settings))
|
||||
.expect("serialize view")
|
||||
.as_object()
|
||||
.expect("view object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut expected_fields = request_fields;
|
||||
expected_fields.insert("updatedAt".to_owned());
|
||||
|
||||
assert_eq!(response_fields, expected_fields);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
let mut body = update_body();
|
||||
|
||||
@@ -35,9 +35,15 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
let mut expected = vec![
|
||||
"src/admin/accounts.rs",
|
||||
"src/admin/auth.rs",
|
||||
"src/admin/backups.rs",
|
||||
"src/admin/client_keys.rs",
|
||||
"src/admin/mod.rs",
|
||||
"src/admin/observability.rs",
|
||||
"src/admin/observability/mod.rs",
|
||||
"src/admin/observability/presenter.rs",
|
||||
"src/admin/observability/query.rs",
|
||||
"src/admin/observability/routes.rs",
|
||||
"src/admin/observability/wire.rs",
|
||||
"src/admin/presenter.rs",
|
||||
"src/admin/settings.rs",
|
||||
"src/admin/system.rs",
|
||||
"src/admin/wire.rs",
|
||||
@@ -71,7 +77,9 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/admin/auth.rs",
|
||||
"tests/admin/client_keys.rs",
|
||||
"tests/admin/mod.rs",
|
||||
"tests/admin/observability.rs",
|
||||
"tests/admin/observability/mod.rs",
|
||||
"tests/admin/observability/query.rs",
|
||||
"tests/admin/observability/response.rs",
|
||||
"tests/admin/settings.rs",
|
||||
"tests/admin/system.rs",
|
||||
"tests/admin/wire.rs",
|
||||
|
||||
@@ -58,6 +58,30 @@ fn no_available_provider_error_should_map_to_service_unavailable() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_capacity_error_should_map_to_service_unavailable() {
|
||||
assert_eq!(
|
||||
gateway_error_contract(GatewayErrorKind::AccountCapacityUnavailable),
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_infrastructure_error_should_map_to_service_unavailable() {
|
||||
assert_eq!(
|
||||
gateway_error_contract(GatewayErrorKind::ProviderInfrastructureUnavailable),
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_error_should_map_to_openai_retryable_status() {
|
||||
assert_eq!(
|
||||
@@ -122,29 +146,32 @@ fn engine_provider_error_should_preserve_retry_classification() {
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
ProviderErrorKind::AccountCapacityUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request"
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
fn no_eligible_provider_account_should_preserve_safe_provider_detail() {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"no account is eligible for the requested model",
|
||||
Some("no_eligible_account".to_owned()),
|
||||
Some("account_unavailable_error".to_owned()),
|
||||
);
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
);
|
||||
let gateway = gateway_error_from_engine(&error);
|
||||
|
||||
@@ -164,6 +191,22 @@ fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
@@ -181,16 +224,18 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sent_provider_unavailability_should_remain_bad_gateway() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
UpstreamSendState::Sent,
|
||||
));
|
||||
fn provider_unavailability_should_remain_bad_gateway_regardless_of_send_state() {
|
||||
for send_state in [UpstreamSendState::NotSent, UpstreamSendState::Sent] {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
send_state,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error).kind(),
|
||||
GatewayErrorKind::UpstreamUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error).kind(),
|
||||
GatewayErrorKind::UpstreamUnavailable
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -533,9 +533,8 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
operation_kind: OperationKind::Generate,
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
protocol_context: Some(json!({
|
||||
"opaque_request_headers": [["user-agent", "IENvZGV4LUNMSS8xLjAg"]]
|
||||
})),
|
||||
// 客户端 User-Agent 仅用于本地展示,不再透传给上游指纹上下文。
|
||||
protocol_context: None,
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
|
||||
@@ -288,6 +288,15 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
"x-codex-installation-id",
|
||||
HeaderValue::from_static("client-installation"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-oai-attestation",
|
||||
HeaderValue::from_static("client-attestation"),
|
||||
);
|
||||
headers.insert("x-oai-is", HeaderValue::from_static("client-is"));
|
||||
headers.insert(
|
||||
"x-oai-is-update",
|
||||
HeaderValue::from_static("client-is-update"),
|
||||
);
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
@@ -321,8 +330,6 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
"user-agent",
|
||||
"originator",
|
||||
"x-openai-internal-codex-residency",
|
||||
"x-codex-turn-state",
|
||||
] {
|
||||
@@ -339,6 +346,12 @@ fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
"x-oai-is-update",
|
||||
] {
|
||||
assert!(values(excluded).is_empty(), "unexpected {excluded}");
|
||||
}
|
||||
|
||||
@@ -39,6 +39,30 @@ impl Decimal {
|
||||
.filter(|value| *value <= MAX_SCALED_DECIMAL)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
/// 除以非零整数,保留最多十位小数。
|
||||
#[must_use]
|
||||
pub fn checked_div_u64(self, divisor: u64) -> Option<Self> {
|
||||
let divisor = u128::from(divisor);
|
||||
(divisor != 0)
|
||||
.then(|| self.0.checked_div(divisor))
|
||||
.flatten()
|
||||
.and_then(|value| Self::from_scaled(value).ok())
|
||||
}
|
||||
|
||||
/// 去尾零的 canonical 字符串,用于 wire 序列化。
|
||||
#[must_use]
|
||||
pub fn canonical(self) -> String {
|
||||
let integer = self.0 / DECIMAL_SCALE;
|
||||
let fraction = self.0 % DECIMAL_SCALE;
|
||||
if fraction == 0 {
|
||||
integer.to_string()
|
||||
} else {
|
||||
let fraction = format!("{fraction:010}");
|
||||
let trimmed = fraction.trim_end_matches('0');
|
||||
format!("{integer}.{trimmed}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Decimal {
|
||||
|
||||
@@ -190,9 +190,26 @@ impl fmt::Debug for ContinuationBinding {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable")]
|
||||
pub struct NativeContinuationStoreError;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable: {detail}")]
|
||||
pub struct NativeContinuationStoreError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
impl NativeContinuationStoreError {
|
||||
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
|
||||
#[must_use]
|
||||
pub fn new(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn detail(&self) -> &str {
|
||||
&self.detail
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的 previous-response 亲和存储端口。
|
||||
///
|
||||
|
||||
@@ -165,8 +165,8 @@ where
|
||||
timings: ModelRequestTimings::default(),
|
||||
client_response_id: None,
|
||||
upstream_response_id: None,
|
||||
last_account_exhaustion: None,
|
||||
last_account_exhaustion_events: Vec::new(),
|
||||
last_retryable_failure: None,
|
||||
last_retryable_failure_events: Vec::new(),
|
||||
provider_attempt_outcomes: Vec::new(),
|
||||
pending_terminal_failure: None,
|
||||
};
|
||||
@@ -245,9 +245,10 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
timings: ModelRequestTimings,
|
||||
client_response_id: Option<String>,
|
||||
upstream_response_id: Option<String>,
|
||||
last_account_exhaustion: Option<ProviderError>,
|
||||
/// 最近一次账号耗尽在重试时被丢弃的原始失败批次;只用于后续空选路终态。
|
||||
last_account_exhaustion_events: Vec<ProviderEvent>,
|
||||
/// 最近一次为无感恢复而被丢弃的原始上游错误;只在后续空选路时成为终态。
|
||||
last_retryable_failure: Option<ProviderError>,
|
||||
/// 与 `last_retryable_failure` 同属一个 attempt 的原始失败批次。
|
||||
last_retryable_failure_events: Vec<ProviderEvent>,
|
||||
provider_attempt_outcomes: Vec<ProviderAttemptOutcome>,
|
||||
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误。
|
||||
pending_terminal_failure: Option<PendingTerminalFailure>,
|
||||
@@ -659,25 +660,33 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if error.kind() == ProviderErrorKind::NoEligibleAccount
|
||||
&& let Some(exhaustion) = self.last_account_exhaustion.take()
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::NoEligibleAccount
|
||||
) && let Some(last_failure) = self.last_retryable_failure.take()
|
||||
{
|
||||
let send_state = self.current_send_state();
|
||||
let mut events = std::mem::take(&mut self.last_account_exhaustion_events);
|
||||
let mut events = std::mem::take(&mut self.last_retryable_failure_events);
|
||||
if !events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut events);
|
||||
return Ok(Some(PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error: exhaustion,
|
||||
error: last_failure,
|
||||
send_state,
|
||||
}));
|
||||
}
|
||||
self.finish_provider_error(&exhaustion).await?;
|
||||
return Err(provider_engine_error(exhaustion));
|
||||
self.finish_provider_error(&last_failure).await?;
|
||||
return Err(provider_engine_error(last_failure));
|
||||
}
|
||||
if !(matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::Unavailable | ProviderErrorKind::NoEligibleAccount
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::ProviderInfrastructureUnavailable
|
||||
) && error.send_state() == UpstreamSendState::NotSent)
|
||||
{
|
||||
self.record_provider_failure(candidate.provider().clone(), error.kind());
|
||||
@@ -909,14 +918,10 @@ where
|
||||
&mut self,
|
||||
mut error: ProviderError,
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
// 原始 wire 只能活在本次决策栈内;后续 clone、attempt 记录与终态对象均只
|
||||
// 接触已剥离的稳定错误字段。
|
||||
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
|
||||
// 均只接触已剥离的稳定错误字段。
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
let account_exhausted = matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted
|
||||
);
|
||||
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
|
||||
// attempt_send_state 是本 attempt 自身的发送事实,驱动重试门;
|
||||
// 持久化与终态用请求级水位,二者不可混用(水位会把早先 attempt 的
|
||||
@@ -976,8 +981,8 @@ where
|
||||
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
|
||||
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始失败响应只能随最后一次客户端可见
|
||||
// 的账号耗尽错误继续,不能进入中间失败记录。
|
||||
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
|
||||
// 所有权保留到下一次 attempt 成功,或最终空选路时返回客户端。
|
||||
let persistence_error = self.request_persisted.then(|| error.clone());
|
||||
if same_account_retry {
|
||||
if let Some(account) = current.metadata.provider_account_id() {
|
||||
@@ -992,10 +997,8 @@ where
|
||||
{
|
||||
self.excluded_accounts.insert(account.clone());
|
||||
}
|
||||
if account_exhausted {
|
||||
self.last_account_exhaustion_events = atomic_client_events;
|
||||
self.last_account_exhaustion = Some(error);
|
||||
}
|
||||
self.last_retryable_failure_events = atomic_client_events;
|
||||
self.last_retryable_failure = Some(error);
|
||||
if let Some(error) = persistence_error {
|
||||
best_effort_store_write(
|
||||
"record_intermediate_failure",
|
||||
@@ -1096,6 +1099,41 @@ where
|
||||
true
|
||||
}
|
||||
|
||||
/// 原生续写的原账号在本地调度阶段已不可用时,改用完整输入重放。
|
||||
///
|
||||
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
|
||||
/// 对应 Provider 的可携带会话状态时,才可安全移除 native handle 并换号。
|
||||
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt != ContinuationAttempt::Native
|
||||
|| !matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(pin) = self
|
||||
.continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if self
|
||||
.operation
|
||||
.provider_session_state(pin.provider().as_str())
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn reset_uncommitted_observations(&mut self) {
|
||||
self.usage = Usage::new();
|
||||
self.cost = CostEstimate::unavailable();
|
||||
|
||||
@@ -188,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData {
|
||||
}
|
||||
|
||||
/// 数据库中固定的账号运行状态。
|
||||
///
|
||||
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
|
||||
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
@@ -206,7 +208,6 @@ impl AccountAvailability {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Ready => "ready",
|
||||
Self::Cooldown => "cooldown",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Expired => "expired",
|
||||
Self::Banned => "banned",
|
||||
@@ -220,7 +221,6 @@ impl AccountAvailability {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"ready" => Some(Self::Ready),
|
||||
"cooldown" => Some(Self::Cooldown),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"expired" => Some(Self::Expired),
|
||||
"banned" => Some(Self::Banned),
|
||||
@@ -244,7 +244,6 @@ pub struct ProviderAccount {
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
@@ -274,7 +273,6 @@ impl ProviderAccount {
|
||||
revision,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Unknown,
|
||||
cooldown_until: None,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
@@ -299,11 +297,9 @@ impl ProviderAccount {
|
||||
mut self,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.availability = availability;
|
||||
self.cooldown_until = cooldown_until;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -373,11 +369,6 @@ impl ProviderAccount {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cooldown_until(&self) -> Option<SystemTime> {
|
||||
self.cooldown_until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn access_token_expires_at(&self) -> Option<SystemTime> {
|
||||
self.access_token_expires_at
|
||||
@@ -398,7 +389,6 @@ impl ProviderAccount {
|
||||
pub fn is_schedulable(&self, now: SystemTime) -> bool {
|
||||
let available = match self.availability {
|
||||
AccountAvailability::Ready => true,
|
||||
AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
@@ -588,6 +578,8 @@ pub struct QuotaObservation {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: Option<OpaqueProviderData>,
|
||||
pub observed_at: Option<SystemTime>,
|
||||
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
|
||||
pub limit_reached: Option<bool>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for QuotaObservation {
|
||||
@@ -614,9 +606,9 @@ pub struct AccountStateChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub reason: Option<String>,
|
||||
pub cooldown_until: Option<SystemTime>,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口。
|
||||
@@ -698,6 +690,17 @@ impl RotationStrategy {
|
||||
Self::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"smart" => Some(Self::Smart),
|
||||
"quota_reset_priority" => Some(Self::QuotaResetPriority),
|
||||
"round_robin" => Some(Self::RoundRobin),
|
||||
"sticky" => Some(Self::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `runtime_settings` 冻结到一次请求计划的账号调度策略。
|
||||
@@ -745,6 +748,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub quota_limit_reached: bool,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -892,6 +896,7 @@ impl AccountRuntimeSignals {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
self.quota_limit_reached = quota.limit_reached;
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -913,14 +918,20 @@ impl AccountRuntimeSignals {
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl AccountQuotaSignals {
|
||||
#[must_use]
|
||||
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
|
||||
pub const fn new(
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -933,6 +944,12 @@ impl AccountQuotaSignals {
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
|
||||
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实与可重建运行信号的请求级组合。
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::net::IpAddr;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
@@ -39,7 +40,7 @@ use crate::routing::{
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const REDIS_COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
@@ -173,6 +174,22 @@ pub enum ProviderCircuitDecision {
|
||||
#[error("provider circuit store is unavailable")]
|
||||
pub struct ProviderCircuitError;
|
||||
|
||||
/// Provider circuit 的可重建协调策略;由 Core 拥有并交给 Store adapter 执行。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderCircuitPolicy {
|
||||
pub failure_threshold: NonZeroU32,
|
||||
pub open_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for ProviderCircuitPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
failure_threshold: NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
|
||||
open_duration: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCircuitPort: Send + Sync {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
@@ -254,10 +271,20 @@ impl DefaultExecutionService {
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => result.ok().flatten(),
|
||||
result = resolve => match result {
|
||||
Ok(pin) => pin,
|
||||
Err(error) => {
|
||||
tracing::debug!(
|
||||
request_id = request_id.as_str(),
|
||||
%error,
|
||||
"Continuation affinity 查询失败,退化为外部续接"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
_ = timeout => None,
|
||||
};
|
||||
match pin {
|
||||
@@ -393,7 +420,7 @@ impl DefaultExecutionService {
|
||||
provider_kind: &ProviderKind,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
@@ -933,17 +960,28 @@ async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let provider = pin.provider().as_str().to_owned();
|
||||
let account = pin.account().as_str().to_owned();
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(REDIS_COORDINATION_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "Continuation affinity 写入失败,后续请求将退化为外部续接");
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
%error,
|
||||
"Continuation affinity 写入失败,后续请求将退化为外部续接"
|
||||
);
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!("Continuation affinity 后台写入超时,已丢弃本次亲和记录");
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
"Continuation affinity 后台写入超时,已丢弃本次亲和记录"
|
||||
);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,8 +129,12 @@ pub enum ProviderErrorKind {
|
||||
RateLimited,
|
||||
/// Credential 配额耗尽。
|
||||
QuotaExhausted,
|
||||
/// 仍有符合条件的账号,但它们暂时没有可调度容量。
|
||||
AccountCapacityUnavailable,
|
||||
/// Provider 已确认当前请求无法选出可用账号。
|
||||
NoEligibleAccount,
|
||||
/// Provider 的账号存储、租约协调或本地凭据数据不可用。
|
||||
ProviderInfrastructureUnavailable,
|
||||
/// 请求超时。
|
||||
Timeout,
|
||||
/// 网络或 transport 失败。
|
||||
@@ -156,7 +160,9 @@ impl ProviderErrorKind {
|
||||
Self::PermissionDenied => "permission_denied",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
|
||||
Self::NoEligibleAccount => "no_eligible_account",
|
||||
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
|
||||
Self::Timeout => "timeout",
|
||||
Self::Transport => "transport",
|
||||
Self::Protocol => "protocol",
|
||||
@@ -711,6 +717,10 @@ pub enum GatewayErrorKind {
|
||||
ModelNotFound,
|
||||
/// 当前没有可用 target。
|
||||
NoAvailableProvider,
|
||||
/// 符合条件的上游账号暂时没有调度容量。
|
||||
AccountCapacityUnavailable,
|
||||
/// Provider 的本地账号基础设施不可用。
|
||||
ProviderInfrastructureUnavailable,
|
||||
/// 上游限流。
|
||||
RateLimited,
|
||||
/// 上游暂不可用。
|
||||
@@ -734,6 +744,8 @@ impl GatewayErrorKind {
|
||||
Self::PolicyDenied => "policy_denied",
|
||||
Self::ModelNotFound => "model_not_found",
|
||||
Self::NoAvailableProvider => "no_available_provider",
|
||||
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
|
||||
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::UpstreamUnavailable => "upstream_unavailable",
|
||||
Self::Timeout => "timeout",
|
||||
@@ -782,10 +794,18 @@ impl GatewayError {
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
),
|
||||
ProviderErrorKind::AccountCapacityUnavailable => Self::new(
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy",
|
||||
),
|
||||
ProviderErrorKind::NoEligibleAccount => Self::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
),
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable => Self::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable",
|
||||
),
|
||||
ProviderErrorKind::Timeout => {
|
||||
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
|
||||
}
|
||||
|
||||
@@ -635,6 +635,13 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
/// 生命周期与 credential revision 无关;不要求调用方持有 revision。
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -244,13 +244,8 @@ async fn compile_runtime_snapshot(
|
||||
}
|
||||
|
||||
let model_mappings = facts.settings.model_mappings;
|
||||
let rotation_strategy = match facts.settings.rotation_strategy.as_str() {
|
||||
"smart" => RotationStrategy::Smart,
|
||||
"quota_reset_priority" => RotationStrategy::QuotaResetPriority,
|
||||
"round_robin" => RotationStrategy::RoundRobin,
|
||||
"sticky" => RotationStrategy::Sticky,
|
||||
_ => return Err(RuntimeSnapshotCompileError::InvalidData),
|
||||
};
|
||||
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
let selection_policy = AccountSelectionPolicy::new(
|
||||
rotation_strategy,
|
||||
NonZeroU32::new(facts.settings.max_concurrent_per_account)
|
||||
|
||||
@@ -21,10 +21,11 @@ pub enum WorkerKind {
|
||||
StaleModelRequestRecovery,
|
||||
Retention,
|
||||
OpsFlush,
|
||||
Backup,
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 7] = [
|
||||
pub const ALL: [Self; 8] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
@@ -32,6 +33,7 @@ impl WorkerKind {
|
||||
Self::StaleModelRequestRecovery,
|
||||
Self::Retention,
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
];
|
||||
|
||||
#[must_use]
|
||||
@@ -44,6 +46,7 @@ impl WorkerKind {
|
||||
Self::StaleModelRequestRecovery => "stale_model_request_recovery",
|
||||
Self::Retention => "retention",
|
||||
Self::OpsFlush => "ops_flush",
|
||||
Self::Backup => "backup",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,22 @@ fn decimal_should_render_with_database_scale() {
|
||||
assert_eq!(decimal.to_string(), "12.3400000000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_canonical_should_strip_trailing_zeroes() {
|
||||
assert_eq!(Decimal::from_str("12.34").unwrap().canonical(), "12.34");
|
||||
assert_eq!(Decimal::from_str("12").unwrap().canonical(), "12");
|
||||
assert_eq!(Decimal::from_str("0").unwrap().canonical(), "0");
|
||||
assert_eq!(Decimal::from_str("1.0000000000").unwrap().canonical(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_div_u64_should_keep_up_to_ten_fraction_digits() {
|
||||
let decimal = Decimal::from_str("10").unwrap();
|
||||
let third = decimal.checked_div_u64(3).unwrap();
|
||||
assert_eq!(third.canonical(), "3.3333333333");
|
||||
assert_eq!(decimal.checked_div_u64(0), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_usd_ticks_should_map_exactly_to_decimal_scale() {
|
||||
let estimate = ProviderReportedCost::from_usd_ticks(12_345_678_901)
|
||||
|
||||
@@ -29,8 +29,9 @@ use gateway_core::engine::{
|
||||
ProviderAttemptOutcome, RecoveryReport, UpstreamSendState,
|
||||
};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, ContinuationFailure, GatewayErrorKind, OpaqueUpstreamValue,
|
||||
ProviderError, ProviderErrorKind, StoreError, StoreErrorKind,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, ContinuationFailure,
|
||||
GatewayErrorKind, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, StoreError,
|
||||
StoreErrorKind,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -512,17 +513,27 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
|
||||
}
|
||||
|
||||
fn plan(operation: &Operation) -> RoutingPlan {
|
||||
plan_with_policy(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
account_selection_policy,
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -1513,6 +1524,70 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_native_continuation_replays_with_the_configured_account_policy() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let policy = AccountSelectionPolicy::new(
|
||||
RotationStrategy::RoundRobin,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
);
|
||||
let route_plan = plan_with_policy(&operation, policy);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let original = ProviderAccountId::new("acct_one").expect("account");
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.clone(),
|
||||
);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].account_selection_policy().strategy(),
|
||||
RotationStrategy::RoundRobin
|
||||
);
|
||||
assert!(contexts[1].excluded_accounts().contains(&original));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
@@ -2554,6 +2629,128 @@ fn pre_delivery_transport_failure_rotates_account_for_non_idempotent_generation(
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_capacity_exhaustion_returns_the_last_retryable_upstream_failure() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let raw_body = Bytes::from_static(b"{\"error\":\"last upstream body\"}\x00");
|
||||
let upstream_error =
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_retry()
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"message too big",
|
||||
Some("1009".to_owned()),
|
||||
Some("websocket_close_error".to_owned()),
|
||||
))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
502,
|
||||
Some(b"application/json".to_vec()),
|
||||
raw_body.clone(),
|
||||
));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(upstream_error)],
|
||||
},
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::AccountCapacityUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("empty selection must return the retained upstream failure");
|
||||
let EngineError::Provider(error) = error else {
|
||||
panic!("expected retained provider error");
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
let detail = error
|
||||
.client_visible_upstream_error()
|
||||
.expect("structured upstream detail");
|
||||
assert_eq!(detail.message(), "message too big");
|
||||
assert_eq!(detail.code(), Some("1009"));
|
||||
assert_eq!(detail.error_type(), Some("websocket_close_error"));
|
||||
let response = error
|
||||
.client_visible_upstream_response()
|
||||
.expect("original upstream response");
|
||||
assert_eq!(response.status(), 502);
|
||||
assert_eq!(response.body(), &raw_body);
|
||||
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
assert_eq!(
|
||||
state.finalizations[0].send_state,
|
||||
UpstreamSendState::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_infrastructure_failure_does_not_restore_the_last_upstream_failure() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
let route_plan = plan(&operation);
|
||||
let upstream_error =
|
||||
ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_pre_delivery_retry()
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"upstream marker",
|
||||
Some("upstream_marker".to_owned()),
|
||||
Some("upstream_error".to_owned()),
|
||||
))
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
502,
|
||||
Some(b"application/json".to_vec()),
|
||||
Bytes::from_static(b"{\"error\":\"upstream marker\"}"),
|
||||
));
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(upstream_error)],
|
||||
},
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
]);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
let error = block_on(session.collect_uncommitted())
|
||||
.expect_err("provider infrastructure failure must remain terminal");
|
||||
let EngineError::Provider(error) = error else {
|
||||
panic!("expected provider error");
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable
|
||||
);
|
||||
assert!(error.client_visible_upstream_error().is_none());
|
||||
assert!(error.client_visible_upstream_response().is_none());
|
||||
assert_eq!(provider.contexts.lock().expect("contexts lock").len(), 2);
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.intermediate_failures, 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Failed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_delivery_retry_marker_is_ignored_after_downstream_commit() {
|
||||
let operation = operation(RetrySafety::NonIdempotent);
|
||||
|
||||
@@ -24,7 +24,7 @@ fn account(id: &str) -> ProviderAccount {
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready, None)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
}
|
||||
|
||||
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
|
||||
@@ -35,6 +35,7 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -56,6 +57,19 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotation_strategy_parse_should_round_trip_stable_wire_values() {
|
||||
for strategy in [
|
||||
RotationStrategy::Smart,
|
||||
RotationStrategy::QuotaResetPriority,
|
||||
RotationStrategy::RoundRobin,
|
||||
RotationStrategy::Sticky,
|
||||
] {
|
||||
assert_eq!(RotationStrategy::parse(strategy.as_str()), Some(strategy));
|
||||
}
|
||||
assert_eq!(RotationStrategy::parse("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plaintext_credential_debug_should_redact_values() {
|
||||
let mut object = Map::new();
|
||||
@@ -97,31 +111,27 @@ fn availability_should_round_trip_all_database_values() {
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_exhausted")
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled").with_runtime_state(
|
||||
false,
|
||||
AccountAvailability::QuotaExhausted,
|
||||
None,
|
||||
),
|
||||
account: account("acct_disabled")
|
||||
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
@@ -145,33 +155,11 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account =
|
||||
account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready, None);
|
||||
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
|
||||
|
||||
assert!(!account.is_schedulable(SystemTime::now()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_become_schedulable_at_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account =
|
||||
account("acct_cooldown").with_runtime_state(true, AccountAvailability::Cooldown, Some(now));
|
||||
|
||||
assert!(account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_account_should_remain_isolated_before_its_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let account = account("acct_cooldown").with_runtime_state(
|
||||
true,
|
||||
AccountAvailability::Cooldown,
|
||||
Some(now + Duration::from_secs(1)),
|
||||
);
|
||||
|
||||
assert!(!account.is_schedulable(now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_prefer_lower_inflight_count() {
|
||||
let candidates = vec![
|
||||
@@ -298,10 +286,15 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(
|
||||
Some(reset_at),
|
||||
Some(75),
|
||||
false,
|
||||
)));
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -616,3 +609,48 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
// availability 终态(Banned/Expired/Invalid)始终排除,
|
||||
// 不受 quota 信号或 skip_exhausted 放宽影响。
|
||||
let mut banned = candidate("acct_banned", 0, Some(100));
|
||||
banned.account = banned
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Banned);
|
||||
let mut expired = candidate("acct_expired", 0, Some(100));
|
||||
expired.account = expired
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Expired);
|
||||
let mut invalid = candidate("acct_invalid", 0, Some(100));
|
||||
invalid.account = invalid
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Invalid);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![banned, expired, invalid, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
|
||||
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
|
||||
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
|
||||
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![exhausted, healthy];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
quota_limit_reached: false,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -17,13 +17,14 @@ use gateway_core::task::{
|
||||
use gateway_host::workers::{WorkerStartError, WorkerSupervisor};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
const ACTIVE_KINDS: [WorkerKind; 6] = [
|
||||
const ACTIVE_KINDS: [WorkerKind; 7] = [
|
||||
WorkerKind::OAuthRefresh,
|
||||
WorkerKind::QuotaCatalogHealth,
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
WorkerKind::RuntimeChangeSubscription,
|
||||
WorkerKind::StaleModelRequestRecovery,
|
||||
WorkerKind::Retention,
|
||||
WorkerKind::Backup,
|
||||
];
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -13,24 +13,31 @@ path = "tests/main.rs"
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1.89"
|
||||
aws-credential-types = "1.2"
|
||||
aws-sdk-s3 = { version = "1.140", default-features = false, features = ["rt-tokio", "default-https-client"] }
|
||||
aws-smithy-runtime-api = "1.14"
|
||||
aws-smithy-types = "1.6"
|
||||
chrono = "0.4.45"
|
||||
fs2 = "0.4"
|
||||
futures = "0.3.32"
|
||||
gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
hex = "0.4.3"
|
||||
redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json", "macros", "migrate"] }
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["rt", "time"] }
|
||||
tokio = { version = "1.52", default-features = false, features = ["fs", "io-std", "io-util", "macros", "process", "rt", "sync", "time"] }
|
||||
tracing = "0.1.44"
|
||||
url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread"] }
|
||||
tempfile = "3.23"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "net", "process", "time"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Admin 认证与设置 adapter。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) struct AdminAuthStoreAdapter {
|
||||
pub(crate) security: postgres::PgAdminSecurityAuditRepository,
|
||||
pub(crate) settings: postgres::PgRuntimeSettingsRepository,
|
||||
pub(crate) state: redis::RedisAdminAuthStateRepository,
|
||||
}
|
||||
|
||||
pub(crate) struct AdminSettingsStoreAdapter {
|
||||
pub(crate) control_plane: postgres::PgControlPlaneRepository,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SettingsStore for AdminSettingsStoreAdapter {
|
||||
async fn load_runtime_settings(&self) -> AdminStoreResult<AdminRuntimeSettings> {
|
||||
let snapshot = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("runtime settings", error))?;
|
||||
admin_runtime_settings(snapshot.settings)
|
||||
}
|
||||
|
||||
async fn admin_api_key_exists(&self) -> AdminStoreResult<bool> {
|
||||
postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
|
||||
.await
|
||||
.map(|snapshot| snapshot.settings.admin_api_key.is_some())
|
||||
.map_err(|error| admin_store_error("admin API key", error))
|
||||
}
|
||||
|
||||
async fn replace_runtime_settings(
|
||||
&self,
|
||||
command: ReplaceRuntimeSettings,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminRuntimeSettings> {
|
||||
let current = postgres::ControlPlaneRepository::load_control_plane(&self.control_plane)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("runtime settings", error))?;
|
||||
let replacement = postgres::ControlPlaneReplacement {
|
||||
settings: postgres::RuntimeSettingsUpdate {
|
||||
admin_api_key: current.settings.admin_api_key,
|
||||
refresh_margin_seconds: command.refresh_margin_seconds,
|
||||
refresh_concurrency: command.refresh_concurrency,
|
||||
max_concurrent_per_account: command.max_concurrent_per_account,
|
||||
request_interval_ms: command.request_interval_ms,
|
||||
rotation_strategy: command.rotation_strategy.as_str().to_owned(),
|
||||
model_mappings: store_model_mappings(command.model_mappings),
|
||||
usage_retention_days: command.usage_retention_days,
|
||||
ops_event_retention_days: command.ops_event_retention_days,
|
||||
audit_retention_days: command.audit_retention_days,
|
||||
},
|
||||
audit: mutation_audit(
|
||||
context,
|
||||
"settings.replace",
|
||||
"runtime_settings",
|
||||
"1",
|
||||
vec![
|
||||
"model_mappings_json".to_owned(),
|
||||
"refresh_margin_seconds".to_owned(),
|
||||
"refresh_concurrency".to_owned(),
|
||||
"max_concurrent_per_account".to_owned(),
|
||||
"request_interval_ms".to_owned(),
|
||||
"rotation_strategy".to_owned(),
|
||||
"retention".to_owned(),
|
||||
],
|
||||
),
|
||||
};
|
||||
let snapshot = postgres::ControlPlaneRepository::replace_control_plane(
|
||||
&self.control_plane,
|
||||
replacement,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("runtime settings", error))?;
|
||||
admin_runtime_settings(snapshot.settings)
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
key: AdminApiKey,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
self.replace_admin_api_key_value(Some(key.expose_for_auth().to_owned()), context)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_admin_api_key(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
self.replace_admin_api_key_value(None, context).await
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminSettingsStoreAdapter {
|
||||
async fn replace_admin_api_key_value(
|
||||
&self,
|
||||
admin_api_key: Option<String>,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AdminApiKeyMutation> {
|
||||
let exists = admin_api_key.is_some();
|
||||
let revision = postgres::ControlPlaneRepository::replace_admin_api_key(
|
||||
&self.control_plane,
|
||||
admin_api_key,
|
||||
mutation_audit(
|
||||
context,
|
||||
if exists {
|
||||
"admin_api_key.replace"
|
||||
} else {
|
||||
"admin_api_key.delete"
|
||||
},
|
||||
"runtime_settings",
|
||||
"1",
|
||||
vec!["admin_api_key".to_owned()],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin API key", error))?;
|
||||
Ok(AdminApiKeyMutation {
|
||||
config_revision: admin_revision(revision)?,
|
||||
exists,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_runtime_settings(
|
||||
settings: postgres::RuntimeSettings,
|
||||
) -> AdminStoreResult<AdminRuntimeSettings> {
|
||||
let rotation_strategy = AdminRotationStrategy::parse(settings.rotation_strategy.as_str())
|
||||
.ok_or_else(|| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"runtime settings",
|
||||
"rotation strategy is invalid",
|
||||
)
|
||||
})?;
|
||||
let model_mappings = settings
|
||||
.model_mappings
|
||||
.into_iter()
|
||||
.map(|(public, upstream)| {
|
||||
let public = gateway_core::routing::PublicModelId::new(public).map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"runtime settings",
|
||||
"public model mapping is invalid",
|
||||
)
|
||||
})?;
|
||||
let upstream = gateway_core::routing::UpstreamModelId::new(upstream).map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"runtime settings",
|
||||
"upstream model mapping is invalid",
|
||||
)
|
||||
})?;
|
||||
Ok((public, upstream))
|
||||
})
|
||||
.collect::<AdminStoreResult<ModelMappings>>()?;
|
||||
Ok(AdminRuntimeSettings {
|
||||
config_revision: admin_revision(settings.config_revision)?,
|
||||
model_mappings,
|
||||
refresh_margin_seconds: settings.refresh_margin_seconds,
|
||||
refresh_concurrency: settings.refresh_concurrency,
|
||||
max_concurrent_per_account: settings.max_concurrent_per_account,
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
rotation_strategy,
|
||||
usage_retention_days: settings.usage_retention_days,
|
||||
ops_event_retention_days: settings.ops_event_retention_days,
|
||||
audit_retention_days: settings.audit_retention_days,
|
||||
updated_at: settings.updated_at,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn store_model_mappings(
|
||||
mappings: ModelMappings,
|
||||
) -> std::collections::BTreeMap<String, String> {
|
||||
mappings
|
||||
.into_iter()
|
||||
.map(|(public, upstream)| (public.as_str().to_owned(), upstream.as_str().to_owned()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthStore for AdminAuthStoreAdapter {
|
||||
async fn load_password_hash(&self, admin_user_id: &str) -> AdminStoreResult<Option<String>> {
|
||||
postgres::AdminSecurityAuditRepository::password_hash(&self.security, admin_user_id)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin authentication", error))
|
||||
}
|
||||
|
||||
async fn create_password_hash_if_absent(
|
||||
&self,
|
||||
admin_user_id: &str,
|
||||
password_hash: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
postgres::AdminSecurityAuditRepository::create_password_hash_if_absent(
|
||||
&self.security,
|
||||
admin_user_id,
|
||||
password_hash,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin authentication", error))
|
||||
}
|
||||
|
||||
async fn load_admin_api_key(&self) -> AdminStoreResult<Option<AdminApiKey>> {
|
||||
postgres::RuntimeSettingsRepository::load_runtime_settings(&self.settings)
|
||||
.await
|
||||
.map(|settings| settings.admin_api_key.map(AdminApiKey::new))
|
||||
.map_err(|error| admin_store_error("admin API key", error))
|
||||
}
|
||||
|
||||
async fn load_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>> {
|
||||
redis::AdminAuthStateRepository::load_admin_session(&self.state, session_id)
|
||||
.await
|
||||
.map(|session| {
|
||||
session.map(|record| AdminSession {
|
||||
admin_user_id: record.admin_user_id,
|
||||
expires_at: record.expires_at,
|
||||
})
|
||||
})
|
||||
.map_err(|error| admin_store_error("admin session", error))
|
||||
}
|
||||
|
||||
async fn store_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
session: &AdminSession,
|
||||
) -> AdminStoreResult<()> {
|
||||
redis::AdminAuthStateRepository::store_admin_session(
|
||||
&self.state,
|
||||
session_id,
|
||||
&redis::AdminSessionRecord {
|
||||
admin_user_id: session.admin_user_id.clone(),
|
||||
expires_at: session.expires_at,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin session", error))
|
||||
}
|
||||
|
||||
async fn delete_session(&self, session_id: &str) -> AdminStoreResult<Option<AdminSession>> {
|
||||
redis::AdminAuthStateRepository::delete_admin_session(&self.state, session_id)
|
||||
.await
|
||||
.map(|session| {
|
||||
session.map(|record| AdminSession {
|
||||
admin_user_id: record.admin_user_id,
|
||||
expires_at: record.expires_at,
|
||||
})
|
||||
})
|
||||
.map_err(|error| admin_store_error("admin session", error))
|
||||
}
|
||||
|
||||
async fn append_audit_event(&self, event: AdminAuditModel) -> AdminStoreResult<()> {
|
||||
let config_revision = event
|
||||
.config_revision
|
||||
.map(|revision| i64::try_from(revision.get()))
|
||||
.transpose()
|
||||
.map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"admin audit",
|
||||
"config revision is outside the supported range",
|
||||
)
|
||||
})?;
|
||||
let actor_kind = match event.actor_kind {
|
||||
gateway_admin::model::auth::AuditActorKind::AdminSession => {
|
||||
postgres::AdminAuditActorKind::AdminSession
|
||||
}
|
||||
gateway_admin::model::auth::AuditActorKind::AdminApiKey => {
|
||||
postgres::AdminAuditActorKind::AdminApiKey
|
||||
}
|
||||
gateway_admin::model::auth::AuditActorKind::System => {
|
||||
postgres::AdminAuditActorKind::System
|
||||
}
|
||||
gateway_admin::model::auth::AuditActorKind::Anonymous => {
|
||||
postgres::AdminAuditActorKind::Anonymous
|
||||
}
|
||||
};
|
||||
postgres::AdminSecurityAuditRepository::append_admin_audit_event(
|
||||
&self.security,
|
||||
postgres::AdminAuditEvent {
|
||||
id: event.id,
|
||||
actor_kind,
|
||||
actor_admin_user_id: event.actor_admin_user_id,
|
||||
actor_ref: event.actor_ref,
|
||||
admin_request_id: event.request_id,
|
||||
action: event.action,
|
||||
entity_kind: event.entity_kind,
|
||||
entity_ref: event.entity_ref,
|
||||
config_revision,
|
||||
changed_fields: event.changed_fields,
|
||||
created_at: event.occurred_at,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| admin_store_error("admin audit", error))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! S3 对象存储与 `pg_dump` 基础设施适配器。
|
||||
|
||||
pub mod pg_dump;
|
||||
pub mod s3;
|
||||
pub mod staging;
|
||||
@@ -0,0 +1,205 @@
|
||||
//! `pg_dump` 子进程与本地暂存适配器。
|
||||
//!
|
||||
//! 数据库密码只通过 `PGPASSWORD` 环境变量传给子进程,绝不进入命令行参数或日志。
|
||||
//! 导出以有界内存流式写入暂存文件并计算 SHA-256;取消时终止子进程并清理部分文件。
|
||||
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
use tokio::process::Command;
|
||||
|
||||
use gateway_admin::model::backup::{BackupError, code};
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpArtifact, DumpRequest, StagedArtifact};
|
||||
|
||||
use super::staging::StagingArea;
|
||||
|
||||
/// `pg_dump` 导出适配器。
|
||||
pub struct PgDumpAdapter {
|
||||
staging: Arc<StagingArea>,
|
||||
/// 不含密码的 PostgreSQL URL,用于 `--dbname`。
|
||||
database_url: String,
|
||||
/// 数据库密码,仅通过 `PGPASSWORD` 注入。
|
||||
database_password: String,
|
||||
}
|
||||
|
||||
impl PgDumpAdapter {
|
||||
/// 组合暂存区与数据库连接事实。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
staging: Arc<StagingArea>,
|
||||
database_url: impl Into<String>,
|
||||
database_password: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
staging,
|
||||
database_url: database_url.into(),
|
||||
database_password: database_password.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DatabaseDumpPort for PgDumpAdapter {
|
||||
async fn dump(&self, request: DumpRequest) -> Result<DumpArtifact, BackupError> {
|
||||
self.staging
|
||||
.ensure_capacity()
|
||||
.map_err(|_| staging_space_exhausted())?;
|
||||
let partial = self.staging.partial_path(&request.backup_id);
|
||||
let final_path = self.staging.final_path(&request.backup_id);
|
||||
|
||||
let mut child = Command::new("pg_dump")
|
||||
.arg("--format=custom")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--dbname")
|
||||
.arg(&self.database_url)
|
||||
.env("PGPASSWORD", &self.database_password)
|
||||
.env("PGCONNECT_TIMEOUT", "10")
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.map_err(|_| pg_dump_failed("无法启动 pg_dump 进程"))?;
|
||||
let stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| pg_dump_failed("无法读取 pg_dump 输出"))?;
|
||||
|
||||
let file = tokio::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.mode(0o600)
|
||||
.open(&partial)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法创建暂存文件"))?;
|
||||
let mut writer = tokio::io::BufWriter::new(file);
|
||||
let mut reader = tokio::io::BufReader::new(stdout);
|
||||
let mut hasher = Sha256::new();
|
||||
let mut size: u64 = 0;
|
||||
|
||||
let copy = async {
|
||||
let mut buffer = vec![0_u8; 64 * 1024];
|
||||
loop {
|
||||
let n = reader
|
||||
.read(&mut buffer)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("读取 pg_dump 输出失败"))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
size = size.saturating_add(n as u64);
|
||||
if size > self.staging.max_archive_bytes() {
|
||||
return Err(staging_space_exhausted());
|
||||
}
|
||||
hasher.update(&buffer[..n]);
|
||||
writer
|
||||
.write_all(&buffer[..n])
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("写入暂存文件失败"))?;
|
||||
}
|
||||
writer
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("刷新暂存文件失败"))?;
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let copy_result = tokio::select! {
|
||||
result = copy => result,
|
||||
_ = request.cancellation.cancelled() => {
|
||||
let _ = child.kill().await;
|
||||
Err(cancelled())
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(error) = copy_result {
|
||||
let _ = child.kill().await;
|
||||
let _ = child.wait().await;
|
||||
self.staging.cleanup(&request.backup_id);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let status = child
|
||||
.wait()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("等待 pg_dump 退出失败"))?;
|
||||
if !status.success() {
|
||||
self.staging.cleanup(&request.backup_id);
|
||||
return Err(pg_dump_failed("pg_dump 非零退出"));
|
||||
}
|
||||
|
||||
tokio::fs::rename(&partial, &final_path)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("暂存归档改名失败"))?;
|
||||
let sha256 = hex::encode(hasher.finalize());
|
||||
Ok(DumpArtifact {
|
||||
path: final_path,
|
||||
size_bytes: size,
|
||||
sha256,
|
||||
})
|
||||
}
|
||||
|
||||
async fn inspect_staging(
|
||||
&self,
|
||||
backup_id: &str,
|
||||
) -> Result<Option<StagedArtifact>, BackupError> {
|
||||
let path = self.staging.final_path(backup_id);
|
||||
let metadata = match tokio::fs::metadata(&path).await {
|
||||
Ok(metadata) => metadata,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
if !metadata.is_file() {
|
||||
return Ok(None);
|
||||
}
|
||||
let size_bytes = metadata.len();
|
||||
let sha256 = hash_file(&path).await?;
|
||||
Ok(Some(StagedArtifact {
|
||||
path,
|
||||
size_bytes,
|
||||
sha256,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError> {
|
||||
self.staging.cleanup(backup_id);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 以有界内存读取文件并计算 SHA-256。
|
||||
async fn hash_file(path: &std::path::Path) -> Result<String, BackupError> {
|
||||
let mut file = tokio::fs::File::open(path)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("打开暂存归档失败"))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = vec![0_u8; 64 * 1024];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buffer)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("读取暂存归档失败"))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..n]);
|
||||
}
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
fn pg_dump_failed(message: &'static str) -> BackupError {
|
||||
BackupError::new(code::PG_DUMP_FAILED, message.to_owned())
|
||||
}
|
||||
|
||||
fn staging_space_exhausted() -> BackupError {
|
||||
BackupError::new(
|
||||
code::STAGING_SPACE_EXHAUSTED,
|
||||
"暂存磁盘空间不足或归档超过单任务上限".to_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn cancelled() -> BackupError {
|
||||
BackupError::new(code::CANCELLED, "备份导出已取消".to_owned())
|
||||
}
|
||||
@@ -0,0 +1,549 @@
|
||||
//! S3 兼容对象存储适配器(含 Cloudflare R2)。
|
||||
//!
|
||||
//! 单个 `BackupObjectStorePort` 实现:方法接受已经校验的存储配置,内部按
|
||||
//! `storage_revision` 缓存 SDK client。上传采用分片流式读取,内存占用有界;
|
||||
//! 取消时主动 `AbortMultipartUpload`。
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use aws_credential_types::Credentials;
|
||||
use aws_sdk_s3::config::{BehaviorVersion, Builder as S3ConfigBuilder, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use aws_sdk_s3::{Client as S3Client, presigning::PresigningConfig};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
|
||||
use aws_smithy_runtime_api::client::result::SdkError;
|
||||
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
|
||||
use aws_smithy_types::retry::RetryConfig;
|
||||
use aws_smithy_types::timeout::TimeoutConfig;
|
||||
use chrono::{DateTime, Utc};
|
||||
use secrecy::ExposeSecret as _;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncSeekExt as _, SeekFrom};
|
||||
use tokio::sync::Semaphore;
|
||||
use uuid::Uuid;
|
||||
|
||||
use gateway_admin::model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupStorageConfig, ConnectionTestResult,
|
||||
ConnectionTestStage, code,
|
||||
};
|
||||
use gateway_admin::ports::backup::{BackupObjectStorePort, UploadObjectRequest};
|
||||
|
||||
/// 分片大小:16 MiB。
|
||||
const PART_SIZE: usize = 16 * 1024 * 1024;
|
||||
/// 并发上传分片数;最大额外内存约为 `PART_SIZE * 并发数`。
|
||||
const CONCURRENCY: usize = 4;
|
||||
/// SDK 单次请求超时。
|
||||
const OPERATION_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// 探针对象的固定 metadata 值。
|
||||
const PROBE_METADATA_KEY: &str = "probe";
|
||||
const PROBE_METADATA_VALUE: &str = "backup-connection-test";
|
||||
|
||||
/// S3 兼容对象存储适配器。
|
||||
pub struct S3ObjectStoreAdapter {
|
||||
clients: Mutex<HashMap<u64, S3Client>>,
|
||||
}
|
||||
|
||||
impl Default for S3ObjectStoreAdapter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl S3ObjectStoreAdapter {
|
||||
/// 创建适配器;client 按 `storage_revision` 惰性构建并缓存。
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
clients: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn client(&self, config: &BackupStorageConfig) -> S3Client {
|
||||
if let Some(client) = self
|
||||
.clients
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|guard| guard.get(&config.storage_revision).cloned())
|
||||
{
|
||||
return client;
|
||||
}
|
||||
let client = build_client(config);
|
||||
if let Ok(mut guard) = self.clients.lock() {
|
||||
guard.insert(config.storage_revision, client.clone());
|
||||
}
|
||||
client
|
||||
}
|
||||
}
|
||||
|
||||
fn build_client(config: &BackupStorageConfig) -> S3Client {
|
||||
let retry = RetryConfig::standard().with_max_attempts(3);
|
||||
let timeout = TimeoutConfig::builder()
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.read_timeout(OPERATION_TIMEOUT)
|
||||
.build();
|
||||
let s3_config = S3ConfigBuilder::new()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(config.region.clone()))
|
||||
.endpoint_url(&config.endpoint)
|
||||
.force_path_style(config.force_path_style)
|
||||
.credentials_provider(Credentials::new(
|
||||
config.access_key_id.clone(),
|
||||
config.secret_access_key.expose_secret().to_string(),
|
||||
None,
|
||||
None,
|
||||
"static",
|
||||
))
|
||||
.retry_config(retry)
|
||||
.timeout_config(timeout)
|
||||
.build();
|
||||
S3Client::from_conf(s3_config)
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl BackupObjectStorePort for S3ObjectStoreAdapter {
|
||||
async fn test_connection(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, BackupError> {
|
||||
let client = self.client(config);
|
||||
let probe_key = format!(
|
||||
"{}/.probe/{}",
|
||||
config.prefix.trim_end_matches('/'),
|
||||
Uuid::new_v4()
|
||||
);
|
||||
let body: Vec<u8> = (0_u8..64).collect();
|
||||
|
||||
if let Err(error) = client
|
||||
.put_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(&probe_key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.metadata(PROBE_METADATA_KEY, PROBE_METADATA_VALUE)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(failed(ConnectionTestStage::PutObject, &error));
|
||||
}
|
||||
|
||||
match client
|
||||
.head_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(&probe_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(head) => {
|
||||
let length_matches = head.content_length() == Some(body.len() as i64);
|
||||
let metadata_matches = head
|
||||
.metadata()
|
||||
.and_then(|metadata| metadata.get(PROBE_METADATA_KEY))
|
||||
.is_some_and(|value| value == PROBE_METADATA_VALUE);
|
||||
if !length_matches || !metadata_matches {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: ConnectionTestStage::HeadObject.as_str(),
|
||||
code: Some(code::UPSTREAM_FAILURE),
|
||||
message: "探针对象校验不一致".to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(failed(ConnectionTestStage::HeadObject, &error));
|
||||
}
|
||||
}
|
||||
|
||||
match client
|
||||
.get_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(&probe_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(get) => {
|
||||
let received = get
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|_| {
|
||||
BackupError::new(code::UPSTREAM_FAILURE, "读取探针对象响应失败".to_owned())
|
||||
})?
|
||||
.into_bytes();
|
||||
if received.as_ref() != body.as_slice() {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: ConnectionTestStage::GetObject.as_str(),
|
||||
code: Some(code::UPSTREAM_FAILURE),
|
||||
message: "探针对象内容不一致".to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
return Ok(failed(ConnectionTestStage::GetObject, &error));
|
||||
}
|
||||
}
|
||||
|
||||
let _ = best_effort_delete(&client, config, &probe_key).await;
|
||||
Ok(ConnectionTestResult {
|
||||
ok: true,
|
||||
stage: ConnectionTestStage::DeleteObject.as_str(),
|
||||
code: None,
|
||||
message: "连接测试成功".to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn upload_file(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), BackupError> {
|
||||
let client = self.client(config);
|
||||
let file_size = tokio::fs::metadata(&request.source)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?
|
||||
.len();
|
||||
if file_size == 0 {
|
||||
return Err(BackupError::new(
|
||||
code::S3_UPLOAD_FAILED,
|
||||
"暂存归档为空,拒绝上传".to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
let created_at = request.metadata.created_at.to_rfc3339();
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(&config.bucket)
|
||||
.key(&request.object_key)
|
||||
.metadata("backup-id", &request.metadata.backup_id)
|
||||
.metadata("sha256", &request.metadata.sha256)
|
||||
.metadata("created-at", &created_at)
|
||||
.send()
|
||||
.await
|
||||
.map_err(map_s3_error)?;
|
||||
let upload_id = create
|
||||
.upload_id()
|
||||
.ok_or_else(|| {
|
||||
BackupError::new(
|
||||
code::S3_UPLOAD_FAILED,
|
||||
"S3 未返回 multipart upload id".to_owned(),
|
||||
)
|
||||
})?
|
||||
.to_owned();
|
||||
|
||||
let upload = upload_parts(&client, config, &request, upload_id.clone(), file_size);
|
||||
let outcome = tokio::select! {
|
||||
result = upload => result,
|
||||
_ = request.cancellation.cancelled() => {
|
||||
Err(BackupError::new(code::CANCELLED, "上传已取消".to_owned()))
|
||||
}
|
||||
};
|
||||
if let Err(error) = &outcome {
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(&config.bucket)
|
||||
.key(&request.object_key)
|
||||
.upload_id(&upload_id)
|
||||
.send()
|
||||
.await;
|
||||
if error.code() != code::CANCELLED {
|
||||
let _ = client
|
||||
.delete_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(&request.object_key)
|
||||
.send()
|
||||
.await;
|
||||
}
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn head_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, BackupError> {
|
||||
let client = self.client(config);
|
||||
match client
|
||||
.head_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(head) => {
|
||||
let size_bytes = head.content_length().filter(|&value| value >= 0);
|
||||
let metadata = head.metadata().cloned().unwrap_or_default();
|
||||
let sha256 = metadata.get("sha256").cloned().ok_or_else(|| {
|
||||
BackupError::new(
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"远端对象缺少 sha256 metadata".to_owned(),
|
||||
)
|
||||
})?;
|
||||
let backup_id = metadata.get("backup-id").cloned().unwrap_or_default();
|
||||
let created_at = metadata
|
||||
.get("created-at")
|
||||
.and_then(|value| DateTime::parse_from_rfc3339(value).ok())
|
||||
.map(|value| value.with_timezone(&Utc))
|
||||
.unwrap_or_else(Utc::now);
|
||||
let Some(size_bytes) = size_bytes else {
|
||||
return Err(BackupError::new(
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"远端对象缺少大小".to_owned(),
|
||||
));
|
||||
};
|
||||
Ok(Some(BackupObjectMetadata::new(
|
||||
backup_id,
|
||||
sha256,
|
||||
created_at,
|
||||
size_bytes as u64,
|
||||
)))
|
||||
}
|
||||
Err(error) if is_not_found(&error) => Ok(None),
|
||||
Err(error) => Err(map_s3_error(error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), BackupError> {
|
||||
let client = self.client(config);
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(map_s3_error)
|
||||
}
|
||||
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
file_name: &str,
|
||||
ttl: Duration,
|
||||
) -> Result<String, BackupError> {
|
||||
let client = self.client(config);
|
||||
let presigning = PresigningConfig::expires_in(ttl)
|
||||
.map_err(|_| BackupError::new(code::INVALID_CONFIG, "下载地址有效期非法".to_owned()))?;
|
||||
let request = client
|
||||
.get_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(object_key)
|
||||
.response_content_disposition(format!("attachment; filename=\"{file_name}\""))
|
||||
.presigned(presigning)
|
||||
.await
|
||||
.map_err(map_s3_error)?;
|
||||
Ok(request.uri().to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
/// 并发分片上传 + CompleteMultipartUpload。
|
||||
async fn upload_parts(
|
||||
client: &S3Client,
|
||||
config: &BackupStorageConfig,
|
||||
request: &UploadObjectRequest,
|
||||
upload_id: String,
|
||||
file_size: u64,
|
||||
) -> Result<(), BackupError> {
|
||||
let total_parts = file_size.div_ceil(u64::try_from(PART_SIZE).unwrap_or(u64::MAX));
|
||||
let next_part = Arc::new(AtomicU64::new(1));
|
||||
let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel::<(i32, String)>(CONCURRENCY * 2);
|
||||
|
||||
let mut workers = Vec::with_capacity(CONCURRENCY);
|
||||
for _ in 0..CONCURRENCY {
|
||||
let client = client.clone();
|
||||
let semaphore = Arc::clone(&semaphore);
|
||||
let next_part = Arc::clone(&next_part);
|
||||
let sender = sender.clone();
|
||||
let source = request.source.clone();
|
||||
let bucket = config.bucket.clone();
|
||||
let key = request.object_key.clone();
|
||||
let upload_id = upload_id.clone();
|
||||
workers.push(tokio::spawn(async move {
|
||||
loop {
|
||||
let part = next_part.fetch_add(1, Ordering::Relaxed);
|
||||
if part > total_parts {
|
||||
break;
|
||||
}
|
||||
let _permit = semaphore.acquire().await.map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "上传并发闸门关闭".to_owned())
|
||||
})?;
|
||||
let offset = (part - 1) * (PART_SIZE as u64);
|
||||
let remaining = file_size.checked_sub(offset).ok_or_else(|| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片偏移超出归档大小".to_owned())
|
||||
})?;
|
||||
let part_len = usize::try_from(remaining.min(PART_SIZE as u64)).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片大小溢出".to_owned())
|
||||
})?;
|
||||
let bytes = read_part(&source, offset, part_len).await?;
|
||||
let part_number = i32::try_from(part).map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "分片号溢出".to_owned())
|
||||
})?;
|
||||
let response = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(bytes))
|
||||
.send()
|
||||
.await
|
||||
.map_err(map_s3_error)?;
|
||||
let etag = response
|
||||
.e_tag()
|
||||
.ok_or_else(|| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "S3 未返回分片 ETag".to_owned())
|
||||
})?
|
||||
.to_owned();
|
||||
if sender.send((part_number, etag)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok::<(), BackupError>(())
|
||||
}));
|
||||
}
|
||||
drop(sender);
|
||||
|
||||
let mut parts = Vec::new();
|
||||
while let Some((part, etag)) = receiver.recv().await {
|
||||
parts.push((part, etag));
|
||||
}
|
||||
for worker in workers {
|
||||
worker.await.map_err(|_| {
|
||||
BackupError::new(code::S3_UPLOAD_FAILED, "上传分片任务崩溃".to_owned())
|
||||
})??;
|
||||
}
|
||||
parts.sort_by_key(|(part, _)| *part);
|
||||
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(
|
||||
parts
|
||||
.into_iter()
|
||||
.map(|(part, etag)| {
|
||||
CompletedPart::builder()
|
||||
.part_number(part)
|
||||
.e_tag(etag)
|
||||
.build()
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
.build();
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&config.bucket)
|
||||
.key(&request.object_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(completed)
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(map_s3_error)
|
||||
}
|
||||
|
||||
/// 从文件中读取指定长度的完整分片。
|
||||
async fn read_part(source: &Path, offset: u64, len: usize) -> Result<Vec<u8>, BackupError> {
|
||||
let mut file = tokio::fs::File::open(source)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "打开暂存归档失败".to_owned()))?;
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "定位暂存归档失败".to_owned()))?;
|
||||
let mut buffer = vec![0_u8; len];
|
||||
// Tokio 默认单次文件读取最多 2 MiB;分片必须读取完整,不能把短读当作末尾分片上传。
|
||||
file.read_exact(&mut buffer)
|
||||
.await
|
||||
.map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?;
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
async fn best_effort_delete(
|
||||
client: &S3Client,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), BackupError> {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(&config.bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(map_s3_error)
|
||||
}
|
||||
|
||||
/// 探测失败结果。
|
||||
fn failed(
|
||||
stage: ConnectionTestStage,
|
||||
error: &SdkError<impl std::fmt::Debug + ProvideErrorMetadata, HttpResponse>,
|
||||
) -> ConnectionTestResult {
|
||||
let result = ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: stage.as_str(),
|
||||
code: Some(s3_error_code(error)),
|
||||
message: s3_error_message(error),
|
||||
};
|
||||
tracing::warn!(
|
||||
stage = result.stage,
|
||||
code = result.code.unwrap_or_default(),
|
||||
message = %result.message,
|
||||
"S3 连接探针失败"
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
/// 稳定错误码。
|
||||
fn s3_error_code<E: ProvideErrorMetadata>(error: &SdkError<E, HttpResponse>) -> &'static str {
|
||||
match status_of(error) {
|
||||
Some(401) => code::S3_AUTH_FAILED,
|
||||
Some(403) => code::S3_PERMISSION_DENIED,
|
||||
Some(404) => code::UPSTREAM_FAILURE,
|
||||
Some(500..=599) => code::UPSTREAM_FAILURE,
|
||||
_ => code::UPSTREAM_FAILURE,
|
||||
}
|
||||
}
|
||||
|
||||
fn s3_error_message<E: ProvideErrorMetadata>(error: &SdkError<E, HttpResponse>) -> String {
|
||||
// 只取稳定的 S3 错误码(如 AccessDenied / NoSuchBucket / SignatureDoesNotMatch),
|
||||
// 不拼入原始 message,避免泄露请求细节。
|
||||
let detail = error.code().unwrap_or("unknown");
|
||||
match status_of(error) {
|
||||
Some(401) => format!("当前凭据认证失败({detail})"),
|
||||
Some(403) => format!("当前凭据没有目标前缀的读写权限({detail})"),
|
||||
Some(404) => format!("目标存储或对象不存在({detail})"),
|
||||
Some(500..=599) => format!("上游对象存储服务返回失败响应({detail})"),
|
||||
_ => format!("对象存储请求失败({detail})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// 提取 HTTP 状态码;非 ServiceError 返回 `None`。
|
||||
fn status_of<E>(error: &SdkError<E, HttpResponse>) -> Option<u16> {
|
||||
match error {
|
||||
SdkError::ServiceError(service) => Some(service.raw().status().as_u16()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 对象不存在的判定(HeadObject 的 404)。
|
||||
fn is_not_found<E>(error: &SdkError<E, HttpResponse>) -> bool {
|
||||
status_of(error) == Some(404)
|
||||
}
|
||||
|
||||
fn map_s3_error<E: ProvideErrorMetadata>(error: SdkError<E, HttpResponse>) -> BackupError {
|
||||
BackupError::new(s3_error_code(&error), s3_error_message(&error))
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! 受控暂存目录:权限校验、磁盘空间检查与原子改名。
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::{StoreError, StoreResult};
|
||||
|
||||
/// 单任务暂存归档上限(默认 64 GiB)。
|
||||
pub const DEFAULT_MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024 * 1024;
|
||||
/// 暂存目录所需的最小剩余空间;与单归档上限解耦,避免小磁盘环境被 64 GiB 门槛误拒。
|
||||
const MIN_STAGING_FREE_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
/// 备份暂存区。只负责文件系统;归档校验与上传由调用方负责。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StagingArea {
|
||||
base_dir: PathBuf,
|
||||
max_archive_bytes: u64,
|
||||
}
|
||||
|
||||
impl StagingArea {
|
||||
/// 创建暂存区并确保目录存在、权限为仅运行用户可读写(0700)。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 目录创建或权限设置失败时返回 [`StoreError`]。
|
||||
pub fn open(base_dir: PathBuf, max_archive_bytes: u64) -> StoreResult<Self> {
|
||||
std::fs::create_dir_all(&base_dir).map_err(|_| unavailable("create staging directory"))?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
std::fs::set_permissions(&base_dir, std::fs::Permissions::from_mode(0o700))
|
||||
.map_err(|_| unavailable("set staging directory permissions"))?;
|
||||
}
|
||||
let metadata = std::fs::metadata(&base_dir)
|
||||
.map_err(|_| unavailable("read staging directory metadata"))?;
|
||||
if !metadata.is_dir() {
|
||||
return Err(invalid("staging path is not a directory"));
|
||||
}
|
||||
Ok(Self {
|
||||
base_dir,
|
||||
max_archive_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
/// 未完成归档的暂存路径。
|
||||
#[must_use]
|
||||
pub fn partial_path(&self, backup_id: &str) -> PathBuf {
|
||||
self.base_dir.join(format!("{backup_id}.partial"))
|
||||
}
|
||||
|
||||
/// 完成归档的暂存路径。
|
||||
#[must_use]
|
||||
pub fn final_path(&self, backup_id: &str) -> PathBuf {
|
||||
self.base_dir.join(format!("{backup_id}.dump"))
|
||||
}
|
||||
|
||||
/// 单任务暂存归档上限。
|
||||
#[must_use]
|
||||
pub const fn max_archive_bytes(&self) -> u64 {
|
||||
self.max_archive_bytes
|
||||
}
|
||||
|
||||
/// 检查剩余磁盘空间是否足够暂存一个归档。
|
||||
///
|
||||
/// 只要求保留一个基本工作余量;单个归档的硬上限由
|
||||
/// [`Self::max_archive_bytes`] 在写入时兜底。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 剩余空间不足或读取失败时返回 [`crate::StoreError`]。
|
||||
pub fn ensure_capacity(&self) -> StoreResult<()> {
|
||||
let free = fs2::available_space(&self.base_dir)
|
||||
.map_err(|_| unavailable("read staging free space"))?;
|
||||
if free < MIN_STAGING_FREE_BYTES {
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "backup staging",
|
||||
message: "staging disk space is below 1 GiB".to_owned(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 清理该备份在暂存区的全部文件;不存在视为成功。
|
||||
pub fn cleanup(&self, backup_id: &str) {
|
||||
let _ = std::fs::remove_file(self.partial_path(backup_id));
|
||||
let _ = std::fs::remove_file(self.final_path(backup_id));
|
||||
}
|
||||
|
||||
/// 校验一个已完成归档的暂存路径归属本暂存区。
|
||||
#[must_use]
|
||||
pub fn owns(&self, path: &Path) -> bool {
|
||||
path.parent() == Some(self.base_dir.as_path())
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable(operation: &'static str) -> StoreError {
|
||||
crate::postgres_unavailable(operation)
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> StoreError {
|
||||
StoreError::InvalidData {
|
||||
entity: "backup staging",
|
||||
message: message.to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//! 完成连接、迁移与 hydration 的 Store 能力集合与启动屏障。
|
||||
|
||||
use gateway_core::engine::credential::ProviderAccountStore;
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 已完成连接、迁移与 hydration 的 Store 能力集合。
|
||||
pub struct StoreBundle {
|
||||
admin_ports: AdminStorePorts,
|
||||
core_ports: CoreStorePorts,
|
||||
provider_ports: ProviderStorePorts,
|
||||
worker_leader_lease: Arc<dyn WorkerLeaderLeasePort>,
|
||||
health_probes: Vec<Arc<dyn HealthProbe>>,
|
||||
worker_contributions: Vec<WorkerContribution>,
|
||||
}
|
||||
|
||||
impl StoreBundle {
|
||||
#[must_use]
|
||||
pub fn admin_ports(&self) -> AdminStorePorts {
|
||||
self.admin_ports.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn core_ports(&self) -> CoreStorePorts {
|
||||
self.core_ports.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn provider_ports(&self) -> ProviderStorePorts {
|
||||
self.provider_ports.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn worker_leader_lease(&self) -> Arc<dyn WorkerLeaderLeasePort> {
|
||||
Arc::clone(&self.worker_leader_lease)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn health_probes(&self) -> Vec<Arc<dyn HealthProbe>> {
|
||||
self.health_probes.clone()
|
||||
}
|
||||
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
}
|
||||
|
||||
/// 在返回 Bundle 前完成全部 Store 启动屏障。
|
||||
pub async fn initialize(mut config: StoreConfig) -> StoreResult<StoreBundle> {
|
||||
const REDIS_NAMESPACE: &str = "codex-proxy-rs";
|
||||
|
||||
config.resolve_and_validate(std::path::Path::new("."))?;
|
||||
let pool = postgres::connect_and_migrate(&config.database_url()?, config.pool).await?;
|
||||
let redis_client = ::redis::Client::open(config.redis_url()?)
|
||||
.map_err(|_| redis_unavailable("create Redis client"))?;
|
||||
let redis_connection = redis_client
|
||||
.get_connection_manager()
|
||||
.await
|
||||
.map_err(|_| redis_unavailable("connect Redis manager"))?;
|
||||
|
||||
let provider_accounts = Arc::new(postgres::PgProviderAccountRepository::new(pool.clone()));
|
||||
let cooldowns = Arc::new(redis::RedisCredentialCooldownRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let account_store: Arc<dyn ProviderAccountStore> = provider_accounts;
|
||||
|
||||
let credential_leases =
|
||||
redis::RedisCredentialLeaseRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?;
|
||||
let provider_leases = Arc::new(redis::RedisProviderLeaseCoordinator::new(
|
||||
credential_leases.clone(),
|
||||
));
|
||||
let provider_session_affinity = Arc::new(redis::RedisProviderSessionAffinityRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let credential_state = Arc::new(redis::RedisCredentialStateRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let runtime_policy = Arc::new(postgres::PgRuntimeSettingsRepository::new(pool.clone()));
|
||||
let oauth_pending = Arc::new(redis::RedisOAuthPendingFlowRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
|
||||
let admin_ports = AdminStorePorts::new(
|
||||
Arc::new(postgres::PgAdminAccountStore::new(
|
||||
pool.clone(),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminAuthStoreAdapter {
|
||||
security: postgres::PgAdminSecurityAuditRepository::new(pool.clone()),
|
||||
settings: postgres::PgRuntimeSettingsRepository::new(pool.clone()),
|
||||
state: redis::RedisAdminAuthStateRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?,
|
||||
}),
|
||||
Arc::new(postgres::PgAdminClientKeyStore::new(pool.clone())),
|
||||
Arc::new(postgres::PgAdminObservabilityStore::new(
|
||||
pool.clone(),
|
||||
Some(credential_leases.clone()),
|
||||
Some(Arc::clone(&cooldowns) as Arc<dyn ProviderCooldownPort>),
|
||||
)),
|
||||
Arc::new(AdminSettingsStoreAdapter {
|
||||
control_plane: postgres::PgControlPlaneRepository::new(pool.clone()),
|
||||
}),
|
||||
backup_ports(pool.clone(), &config)?,
|
||||
);
|
||||
|
||||
let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone()));
|
||||
let (execution, execution_writer) =
|
||||
postgres::BufferedExecutionStore::new(Arc::clone(&execution_repository));
|
||||
let execution = Arc::new(execution);
|
||||
let retention = Arc::new(postgres::PgRetentionRepository::new(pool.clone()));
|
||||
let admissions: Arc<dyn gateway_core::engine::admission::ClientAdmissionPort> = Arc::new(
|
||||
redis::RedisClientAdmissionRepository::new(redis_connection.clone(), REDIS_NAMESPACE)?,
|
||||
);
|
||||
let circuits: Arc<dyn gateway_core::engine::execution::ProviderCircuitPort> =
|
||||
Arc::new(redis::RedisProviderCircuitRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
gateway_core::engine::execution::ProviderCircuitPolicy::default(),
|
||||
)?);
|
||||
// Continuation affinity 是下一轮请求的路由事实,Core 必须直接等待 Redis 确认。
|
||||
let continuation: Arc<dyn gateway_core::engine::continuation::NativeContinuationPort> =
|
||||
Arc::new(redis::RedisNativeContinuationRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?);
|
||||
let (admissions, admission_release_writer) =
|
||||
redis::BufferedClientAdmissionPort::new(admissions);
|
||||
let (circuits, circuit_feedback_writer) = redis::BufferedProviderCircuitPort::new(circuits);
|
||||
let core_ports = CoreStorePorts::new(
|
||||
execution,
|
||||
(
|
||||
Arc::new(admissions),
|
||||
Arc::new(postgres::PgClientAdmissionRecoveryRepository::new(
|
||||
pool.clone(),
|
||||
)),
|
||||
),
|
||||
Arc::new(circuits),
|
||||
continuation,
|
||||
(
|
||||
Arc::new(postgres::PgRuntimeSnapshotRepository::new(pool.clone())),
|
||||
Arc::new(redis::RedisRuntimeChangeRepository::new(
|
||||
redis_client,
|
||||
REDIS_NAMESPACE,
|
||||
)?),
|
||||
),
|
||||
Arc::new(postgres::PgClientApiKeyUsageSink::new(pool.clone())),
|
||||
);
|
||||
|
||||
let provider_ports = ProviderStorePorts::new(
|
||||
account_store,
|
||||
provider_leases,
|
||||
provider_session_affinity,
|
||||
Arc::new(redis::RedisProviderSessionExclusionRepository::new(
|
||||
redis_connection.clone(),
|
||||
REDIS_NAMESPACE,
|
||||
)?),
|
||||
credential_state.clone(),
|
||||
credential_state,
|
||||
cooldowns,
|
||||
runtime_policy,
|
||||
oauth_pending,
|
||||
);
|
||||
let worker_leader_lease = Arc::new(redis::worker_lease::RedisWorkerLeaderLeasePort::new(
|
||||
credential_leases,
|
||||
));
|
||||
let health_probes: Vec<Arc<dyn HealthProbe>> = vec![
|
||||
Arc::new(PostgresHealthProbe { pool: pool.clone() }),
|
||||
Arc::new(RedisHealthProbe {
|
||||
connection: redis_connection,
|
||||
}),
|
||||
];
|
||||
let worker_contributions = store_worker_contributions(
|
||||
execution_repository,
|
||||
execution_writer,
|
||||
admission_release_writer,
|
||||
circuit_feedback_writer,
|
||||
retention,
|
||||
)?;
|
||||
Ok(StoreBundle {
|
||||
admin_ports,
|
||||
core_ports,
|
||||
provider_ports,
|
||||
worker_leader_lease,
|
||||
health_probes,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// 构造备份控制面的仓储、导出器与对象存储适配器。
|
||||
pub(crate) fn backup_ports(
|
||||
pool: sqlx::PgPool,
|
||||
config: &StoreConfig,
|
||||
) -> StoreResult<BackupStorePorts> {
|
||||
let staging = Arc::new(backup::staging::StagingArea::open(
|
||||
std::path::PathBuf::from(".runtime/data/backup-staging"),
|
||||
backup::staging::DEFAULT_MAX_ARCHIVE_BYTES,
|
||||
)?);
|
||||
let repository = Arc::new(postgres::PgBackupRepository::new(pool));
|
||||
let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new(
|
||||
staging,
|
||||
config.database.url.clone(),
|
||||
config.database.password.clone(),
|
||||
));
|
||||
let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new());
|
||||
Ok(BackupStorePorts::new(repository, dump, object_store))
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Store 启动配置、环境变量解析与校验。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) const DATABASE_URL_ENV: &str = "CPR_DATABASE_URL";
|
||||
pub(crate) const REDIS_URL_ENV: &str = "CPR_REDIS_URL";
|
||||
pub(crate) const DATABASE_PASSWORD_ENV: &str = "CPR_DATABASE_PASSWORD";
|
||||
pub(crate) const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD";
|
||||
|
||||
/// Store 自己拥有并校验的启动配置。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StoreConfig {
|
||||
pub(crate) database: StoreConnectionConfig,
|
||||
pub(crate) redis: StoreConnectionConfig,
|
||||
#[serde(default)]
|
||||
pub(crate) pool: StorePoolConfig,
|
||||
}
|
||||
|
||||
/// PostgreSQL 连接池预算;acquire 超时决定池耗尽时快速失败而非排队积压。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct StorePoolConfig {
|
||||
pub max_connections: u32,
|
||||
pub acquire_timeout_seconds: u64,
|
||||
}
|
||||
|
||||
impl Default for StorePoolConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_connections: 10,
|
||||
acquire_timeout_seconds: 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StorePoolConfig {
|
||||
fn validate(&self) -> StoreResult<()> {
|
||||
if self.max_connections == 0 || self.acquire_timeout_seconds == 0 {
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: "pool limits must be positive".to_owned(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StoreConfig {
|
||||
pub fn resolve_and_validate(&mut self, _source_dir: &std::path::Path) -> StoreResult<()> {
|
||||
if let Some(url) = optional_environment_value(DATABASE_URL_ENV)? {
|
||||
self.database.url = url;
|
||||
}
|
||||
if let Some(url) = optional_environment_value(REDIS_URL_ENV)? {
|
||||
self.redis.url = url;
|
||||
}
|
||||
if let Some(password) = optional_environment_value(DATABASE_PASSWORD_ENV)? {
|
||||
self.database.password = password;
|
||||
}
|
||||
if let Some(password) = optional_environment_value(REDIS_PASSWORD_ENV)? {
|
||||
self.redis.password = password;
|
||||
}
|
||||
self.database.validate("database")?;
|
||||
self.redis.validate("redis")?;
|
||||
self.pool.validate()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn database_url(&self) -> StoreResult<String> {
|
||||
self.database.connection_url("database")
|
||||
}
|
||||
|
||||
pub(crate) fn redis_url(&self) -> StoreResult<String> {
|
||||
self.redis.connection_url("redis")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn optional_environment_value(name: &'static str) -> StoreResult<Option<String>> {
|
||||
match std::env::var(name) {
|
||||
Ok(value) if value.trim().is_empty() => Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("environment variable {name} is empty"),
|
||||
}),
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(std::env::VarError::NotPresent) => Ok(None),
|
||||
Err(std::env::VarError::NotUnicode(_)) => Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("environment variable {name} is not Unicode"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for StoreConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("StoreConfig")
|
||||
.field("database", &"[REDACTED]")
|
||||
.field("redis", &"[REDACTED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct StoreConnectionConfig {
|
||||
pub(crate) url: String,
|
||||
pub(crate) password: String,
|
||||
}
|
||||
|
||||
impl StoreConnectionConfig {
|
||||
fn validate(&self, field: &'static str) -> StoreResult<()> {
|
||||
require_nonempty("store config", field, &self.url)?;
|
||||
if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("{field}.password must be exactly 48 hexadecimal characters"),
|
||||
});
|
||||
}
|
||||
self.connection_url(field).map(|_| ())
|
||||
}
|
||||
|
||||
fn connection_url(&self, field: &'static str) -> StoreResult<String> {
|
||||
let mut url = url::Url::parse(&self.url).map_err(|_| StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("{field}.url is invalid"),
|
||||
})?;
|
||||
if url.password().is_some() {
|
||||
return Err(StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("{field}.url must not contain a password"),
|
||||
});
|
||||
}
|
||||
url.set_password(Some(&self.password))
|
||||
.map_err(|()| StoreError::InvalidData {
|
||||
entity: "store config",
|
||||
message: format!("{field}.url cannot carry credentials"),
|
||||
})?;
|
||||
Ok(url.to_string())
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,870 @@
|
||||
//! `backup_settings` 与 `backup_records` 的 PostgreSQL owner。
|
||||
//!
|
||||
//! 实现 `gateway-admin::ports::backup::BackupRepository`。本模块只执行事务与
|
||||
//! 条件更新,不决定保留策略或状态机语义。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use sqlx::{PgPool, Postgres, Row as _, Transaction};
|
||||
|
||||
use gateway_admin::model::backup::{
|
||||
BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupRecordSeed, BackupSettings,
|
||||
BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
};
|
||||
use gateway_admin::model::{MutationContext, Revision as AdminRevision};
|
||||
use gateway_admin::ports::backup::{BackupRepository, StatusTransitionUpdate};
|
||||
use gateway_admin::ports::store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult};
|
||||
|
||||
use crate::{
|
||||
StoreError, StoreResult,
|
||||
postgres::{
|
||||
admin_security_audit::AdminAuditEvent,
|
||||
runtime_settings::bump_config_revision_in_transaction,
|
||||
},
|
||||
};
|
||||
|
||||
/// `backup_records` 的最大分页大小。
|
||||
const BACKUP_PAGE_LIMIT: u32 = 200;
|
||||
|
||||
/// 备份任务/配置仓储。
|
||||
#[derive(Clone)]
|
||||
pub struct PgBackupRepository {
|
||||
pool: PgPool,
|
||||
}
|
||||
|
||||
impl PgBackupRepository {
|
||||
/// 包装连接池。
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupRepository for PgBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
load_backup_settings(&self.pool).await
|
||||
}
|
||||
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
command: UpdateBackupStorageCommand,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, AdminRevision)> {
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("begin backup storage update"))?;
|
||||
let result = async {
|
||||
let current = lock_settings_in_transaction(&mut transaction).await?;
|
||||
ensure_storage_identity_stable(¤t, &command, &mut transaction).await?;
|
||||
|
||||
let secret = command
|
||||
.secret_access_key
|
||||
.as_ref()
|
||||
.map(|secret| secret.expose_secret().to_owned());
|
||||
sqlx::query(
|
||||
"update backup_settings
|
||||
set endpoint = $1,
|
||||
region = $2,
|
||||
bucket = $3,
|
||||
access_key_id = $4,
|
||||
secret_access_key = coalesce($5, secret_access_key),
|
||||
prefix = $6,
|
||||
force_path_style = $7,
|
||||
storage_revision = storage_revision + 1,
|
||||
last_verified_at = null,
|
||||
updated_at = now()
|
||||
where id = 1",
|
||||
)
|
||||
.bind(&command.endpoint)
|
||||
.bind(&command.region)
|
||||
.bind(&command.bucket)
|
||||
.bind(&command.access_key_id)
|
||||
.bind(secret.as_deref())
|
||||
.bind(&command.prefix)
|
||||
.bind(command.force_path_style)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("update backup storage"))?;
|
||||
|
||||
let store_revision = bump_config_revision_in_transaction(&mut transaction)
|
||||
.await
|
||||
.map_err(map_admin_error)?;
|
||||
let revision = admin_revision(store_revision)?;
|
||||
let changed = storage_changed_fields(¤t, &command);
|
||||
let audit = crate::mutation_audit(
|
||||
context,
|
||||
"backup.s3_config_updated",
|
||||
"backup_settings",
|
||||
"1",
|
||||
changed,
|
||||
);
|
||||
append_audit_in_transaction(&mut transaction, audit, Some(revision))
|
||||
.await
|
||||
.map_err(map_admin_error)?;
|
||||
|
||||
let settings = load_settings_in_transaction(&mut transaction).await?;
|
||||
Ok((settings, revision))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(result) => {
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("commit backup storage update"))?;
|
||||
Ok(result)
|
||||
}
|
||||
Err(error) => {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("rollback backup storage update"))?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
next_run_at: Option<DateTime<Utc>>,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("begin backup schedule update"))?;
|
||||
let result = async {
|
||||
sqlx::query(
|
||||
"update backup_settings
|
||||
set schedule_enabled = $1,
|
||||
cron_expression = $2,
|
||||
schedule_timezone = $3,
|
||||
retention_days = $4,
|
||||
retention_count = $5,
|
||||
next_run_at = $6,
|
||||
updated_at = now()
|
||||
where id = 1",
|
||||
)
|
||||
.bind(command.schedule_enabled)
|
||||
.bind(&command.cron_expression)
|
||||
.bind(&command.schedule_timezone)
|
||||
.bind(i64::from(command.retention_days))
|
||||
.bind(i64::from(command.retention_count))
|
||||
.bind(next_run_at)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("update backup schedule"))?;
|
||||
|
||||
let audit = crate::mutation_audit(
|
||||
context,
|
||||
"backup.schedule_updated",
|
||||
"backup_settings",
|
||||
"1",
|
||||
vec![
|
||||
"schedule_enabled".to_owned(),
|
||||
"cron_expression".to_owned(),
|
||||
"schedule_timezone".to_owned(),
|
||||
"retention_days".to_owned(),
|
||||
"retention_count".to_owned(),
|
||||
],
|
||||
);
|
||||
append_audit_in_transaction(&mut transaction, audit, None)
|
||||
.await
|
||||
.map_err(map_admin_error)?;
|
||||
load_settings_in_transaction(&mut transaction).await
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(settings) => {
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("commit backup schedule update"))?;
|
||||
Ok(settings)
|
||||
}
|
||||
Err(error) => {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("rollback backup schedule update"))?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_verification(
|
||||
&self,
|
||||
storage_revision: u64,
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let result = sqlx::query(
|
||||
"update backup_settings
|
||||
set last_verified_at = $2, updated_at = now()
|
||||
where id = 1 and storage_revision = $1",
|
||||
)
|
||||
.bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?)
|
||||
.bind(at)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("record backup storage verification"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("begin backup record insert"))?;
|
||||
let result = async {
|
||||
let _ = lock_settings_in_transaction(&mut transaction).await?;
|
||||
insert_queued_in_transaction(&mut transaction, &seed).await
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(record) => {
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("commit backup record insert"))?;
|
||||
Ok(record)
|
||||
}
|
||||
Err(error) => {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("rollback backup record insert"))?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool> {
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("begin scheduled insert"))?;
|
||||
match insert_queued_in_transaction(&mut transaction, &seed).await {
|
||||
Ok(_) => {
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("commit scheduled insert"))?;
|
||||
Ok(true)
|
||||
}
|
||||
Err(error) if is_active_or_scheduled_conflict(&error) => {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("rollback scheduled insert"))?;
|
||||
Ok(false)
|
||||
}
|
||||
Err(error) => {
|
||||
transaction
|
||||
.rollback()
|
||||
.await
|
||||
.map_err(|_| store_unavailable("rollback scheduled insert"))?;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage> {
|
||||
let limit = u32::from(query.page_size.get()).min(BACKUP_PAGE_LIMIT);
|
||||
let offset = u64::from(query.page.saturating_sub(1)) * u64::from(limit);
|
||||
|
||||
let status = query.status.map(BackupStatus::as_str);
|
||||
let trigger = query.trigger.map(BackupTriggerKind::as_str);
|
||||
|
||||
let total = sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from backup_records
|
||||
where ($1::text is null or status = $1)
|
||||
and ($2::text is null or trigger_kind = $2)",
|
||||
)
|
||||
.bind(status)
|
||||
.bind(trigger)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("count backup records"))?;
|
||||
|
||||
let rows = sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records
|
||||
where ($1::text is null or status = $1)
|
||||
and ($2::text is null or trigger_kind = $2)
|
||||
order by created_at desc, id desc
|
||||
limit $3 offset $4",
|
||||
)
|
||||
.bind(status)
|
||||
.bind(trigger)
|
||||
.bind(i64::from(limit))
|
||||
.bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("list backup records"))?;
|
||||
|
||||
let items = rows
|
||||
.iter()
|
||||
.map(backup_record_from_row)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(BackupRecordPage {
|
||||
items,
|
||||
total: u64::try_from(total).map_err(|_| store_invalid("record count"))?,
|
||||
page: query.page,
|
||||
page_size: query.page_size,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records where id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("load backup record"))?
|
||||
.map(|row| backup_record_from_row(&row))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let rows = sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records
|
||||
where status in ('dumping', 'uploading')
|
||||
order by created_at, id",
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("list intermediate backup records"))?;
|
||||
rows.iter()
|
||||
.map(backup_record_from_row)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
}
|
||||
|
||||
async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let limit = limit.clamp(1, 100);
|
||||
let rows = sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records
|
||||
where status = 'deleting'
|
||||
order by created_at, id
|
||||
limit $1",
|
||||
)
|
||||
.bind(i64::from(limit))
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("list pending deletions"))?;
|
||||
rows.iter()
|
||||
.map(backup_record_from_row)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
}
|
||||
|
||||
async fn list_expired_records(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let limit = limit.clamp(1, 1000);
|
||||
let now = Utc::now();
|
||||
let rows = sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records
|
||||
where expires_at is not null
|
||||
and expires_at <= $1
|
||||
and status in ('completed', 'failed')
|
||||
order by created_at, id
|
||||
limit $2",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(i64::from(limit))
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("list expired backups"))?;
|
||||
rows.iter()
|
||||
.map(backup_record_from_row)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
}
|
||||
|
||||
async fn claim_next_queued(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
sqlx::query(
|
||||
"with claimed as (
|
||||
select backup_records.id from backup_records
|
||||
where backup_records.status = 'queued'
|
||||
order by backup_records.created_at, backup_records.id
|
||||
limit 1
|
||||
for update skip locked
|
||||
)
|
||||
update backup_records br
|
||||
set status = 'dumping',
|
||||
started_at = $1,
|
||||
completed_at = null,
|
||||
attempt_count = br.attempt_count + 1,
|
||||
updated_at = $1
|
||||
from claimed
|
||||
where br.id = claimed.id
|
||||
returning br.id, br.trigger_kind, br.status, br.scheduled_at, br.object_key,
|
||||
br.size_bytes, br.sha256, br.attempt_count, br.error_code,
|
||||
br.error_message, br.started_at, br.completed_at, br.expires_at,
|
||||
br.created_at, br.updated_at",
|
||||
)
|
||||
.bind(now)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("claim queued backup task"))?
|
||||
.map(|row| backup_record_from_row(&row))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn transition_status(
|
||||
&self,
|
||||
id: &str,
|
||||
from: BackupStatus,
|
||||
to: BackupStatus,
|
||||
update: StatusTransitionUpdate,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let size_bytes = update
|
||||
.size_bytes
|
||||
.map(i64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| store_invalid("size bytes"))?;
|
||||
sqlx::query(
|
||||
"update backup_records
|
||||
set status = $1,
|
||||
size_bytes = coalesce($2, size_bytes),
|
||||
sha256 = coalesce($3, sha256),
|
||||
error_code = coalesce($4, error_code),
|
||||
error_message = coalesce($5, error_message),
|
||||
completed_at = coalesce(
|
||||
$6,
|
||||
case when $1 in ('completed', 'failed') then $7 else completed_at end
|
||||
),
|
||||
updated_at = $7
|
||||
where id = $8 and status = $9
|
||||
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at",
|
||||
)
|
||||
.bind(to.as_str())
|
||||
.bind(size_bytes)
|
||||
.bind(update.sha256.as_deref())
|
||||
.bind(update.error_code.as_deref())
|
||||
.bind(update.error_message.as_deref())
|
||||
.bind(update.completed_at)
|
||||
.bind(now)
|
||||
.bind(id)
|
||||
.bind(from.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("transition backup task"))?
|
||||
.map(|row| backup_record_from_row(&row))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
sqlx::query(
|
||||
"update backup_records
|
||||
set status = 'deleting', updated_at = $1
|
||||
where id = $2 and status in ('completed', 'failed')
|
||||
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("transition backup to deleting"))?
|
||||
.map(|row| backup_record_from_row(&row))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
async fn delete_record(&self, id: &str) -> AdminStoreResult<()> {
|
||||
sqlx::query("delete from backup_records where id = $1")
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|_| store_unavailable("delete backup record"))
|
||||
}
|
||||
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
next_run_at: DateTime<Utc>,
|
||||
expected_cron: &str,
|
||||
expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let result = sqlx::query(
|
||||
"update backup_settings
|
||||
set next_run_at = $1
|
||||
where id = 1
|
||||
and schedule_enabled
|
||||
and cron_expression = $2
|
||||
and schedule_timezone = $3",
|
||||
)
|
||||
.bind(next_run_at)
|
||||
.bind(expected_cron)
|
||||
.bind(expected_timezone)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("advance backup schedule cursor"))?;
|
||||
Ok(result.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let limit = limit.clamp(1, 10000);
|
||||
let rows = sqlx::query(
|
||||
"select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at
|
||||
from backup_records
|
||||
where trigger_kind = 'scheduled' and status = 'completed'
|
||||
order by completed_at desc, id desc
|
||||
limit $1",
|
||||
)
|
||||
.bind(i64::from(limit))
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("list scheduled completed backups"))?;
|
||||
rows.iter()
|
||||
.map(backup_record_from_row)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
}
|
||||
}
|
||||
|
||||
/// Store 内部 Revision → Admin Revision 转换。
|
||||
fn admin_revision(revision: crate::Revision) -> AdminStoreResult<AdminRevision> {
|
||||
AdminRevision::new(revision.get()).map_err(|_| {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Invalid, "revision", "zero revision")
|
||||
})
|
||||
}
|
||||
|
||||
/// 锁定并读取单例配置行。
|
||||
async fn lock_settings_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
sqlx::query(
|
||||
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
|
||||
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
|
||||
retention_days, retention_count, next_run_at, last_verified_at, updated_at
|
||||
from backup_settings where id = 1 for update",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("lock backup settings"))?
|
||||
.map(|row| backup_settings_from_row(&row))
|
||||
.transpose()?
|
||||
.ok_or_else(|| store_not_found("backup settings"))
|
||||
}
|
||||
|
||||
async fn load_settings_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
sqlx::query(
|
||||
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
|
||||
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
|
||||
retention_days, retention_count, next_run_at, last_verified_at, updated_at
|
||||
from backup_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("load backup settings in transaction"))?
|
||||
.map(|row| backup_settings_from_row(&row))
|
||||
.transpose()?
|
||||
.ok_or_else(|| store_not_found("backup settings"))
|
||||
}
|
||||
|
||||
async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult<BackupSettings> {
|
||||
sqlx::query(
|
||||
"select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key,
|
||||
prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone,
|
||||
retention_days, retention_count, next_run_at, last_verified_at, updated_at
|
||||
from backup_settings where id = 1",
|
||||
)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("load backup settings"))?
|
||||
.map(|row| backup_settings_from_row(&row))
|
||||
.transpose()?
|
||||
.ok_or_else(|| store_not_found("backup settings"))
|
||||
}
|
||||
|
||||
/// 插入 queued 记录;返回记录或唯一约束冲突。
|
||||
async fn insert_queued_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
seed: &BackupRecordSeed,
|
||||
) -> AdminStoreResult<BackupRecord> {
|
||||
let row = sqlx::query(
|
||||
"insert into backup_records (
|
||||
id, trigger_kind, status, scheduled_at, object_key, expires_at, created_at, updated_at
|
||||
) values ($1, $2, 'queued', $3, $4, $5, now(), now())
|
||||
returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256,
|
||||
attempt_count, error_code, error_message, started_at, completed_at, expires_at,
|
||||
created_at, updated_at",
|
||||
)
|
||||
.bind(&seed.id)
|
||||
.bind(seed.trigger_kind.as_str())
|
||||
.bind(seed.scheduled_at)
|
||||
.bind(&seed.object_key)
|
||||
.bind(seed.expires_at)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(map_insert_error)?;
|
||||
backup_record_from_row(&row)
|
||||
}
|
||||
|
||||
/// 把唯一约束冲突映射为活跃任务/计划冲突。
|
||||
fn map_insert_error(error: sqlx::Error) -> AdminStoreError {
|
||||
match &error {
|
||||
sqlx::Error::Database(database_error)
|
||||
if database_error.constraint() == Some("backup_records_active_uq") =>
|
||||
{
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"backup record",
|
||||
"an active backup task already exists",
|
||||
)
|
||||
}
|
||||
sqlx::Error::Database(database_error)
|
||||
if database_error.constraint() == Some("backup_records_scheduled_uq") =>
|
||||
{
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"backup record",
|
||||
"a scheduled backup already exists at this time",
|
||||
)
|
||||
}
|
||||
_ => AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"backup record",
|
||||
"backup record insert failed",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_active_or_scheduled_conflict(error: &AdminStoreError) -> bool {
|
||||
error.kind() == AdminStoreErrorKind::Conflict
|
||||
}
|
||||
|
||||
/// 已有记录时锁定存储身份:endpoint/region/bucket/path-style 不允许变化。
|
||||
async fn ensure_storage_identity_stable(
|
||||
current: &BackupSettings,
|
||||
command: &UpdateBackupStorageCommand,
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> AdminStoreResult<()> {
|
||||
let identity_changed = current.endpoint.as_deref() != Some(command.endpoint.as_str())
|
||||
|| current.region.as_deref() != Some(command.region.as_str())
|
||||
|| current.bucket.as_deref() != Some(command.bucket.as_str())
|
||||
|| current.force_path_style != command.force_path_style;
|
||||
if !identity_changed {
|
||||
return Ok(());
|
||||
}
|
||||
let records_exist =
|
||||
sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)")
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| store_unavailable("check backup records existence"))?;
|
||||
if records_exist {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"backup storage",
|
||||
"storage identity cannot change while backup records exist",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn storage_changed_fields(
|
||||
current: &BackupSettings,
|
||||
command: &UpdateBackupStorageCommand,
|
||||
) -> Vec<String> {
|
||||
let mut fields = Vec::new();
|
||||
if current.endpoint.as_deref() != Some(command.endpoint.as_str()) {
|
||||
fields.push("endpoint".to_owned());
|
||||
}
|
||||
if current.region.as_deref() != Some(command.region.as_str()) {
|
||||
fields.push("region".to_owned());
|
||||
}
|
||||
if current.bucket.as_deref() != Some(command.bucket.as_str()) {
|
||||
fields.push("bucket".to_owned());
|
||||
}
|
||||
if current.access_key_id.as_deref() != Some(command.access_key_id.as_str()) {
|
||||
fields.push("access_key_id".to_owned());
|
||||
}
|
||||
if command.secret_access_key.is_some() {
|
||||
fields.push("secret_access_key".to_owned());
|
||||
}
|
||||
if current.prefix.as_deref() != Some(command.prefix.as_str()) {
|
||||
fields.push("prefix".to_owned());
|
||||
}
|
||||
if current.force_path_style != command.force_path_style {
|
||||
fields.push("force_path_style".to_owned());
|
||||
}
|
||||
fields
|
||||
}
|
||||
|
||||
/// 在同事务追加审计事件;`revision` 为空时不写 config_revision。
|
||||
async fn append_audit_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
mut event: AdminAuditEvent,
|
||||
revision: Option<AdminRevision>,
|
||||
) -> StoreResult<()> {
|
||||
event.config_revision = revision
|
||||
.map(|revision| {
|
||||
i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData {
|
||||
entity: "backup",
|
||||
message: "revision overflow".to_owned(),
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
event.validate()?;
|
||||
sqlx::query(
|
||||
"insert into admin_audit_events (
|
||||
id, actor_kind, actor_admin_user_id, actor_ref, admin_request_id,
|
||||
action, entity_kind, entity_ref, config_revision, changed_fields, created_at
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)",
|
||||
)
|
||||
.bind(event.id)
|
||||
.bind(event.actor_kind.as_str())
|
||||
.bind(event.actor_admin_user_id)
|
||||
.bind(event.actor_ref)
|
||||
.bind(event.admin_request_id)
|
||||
.bind(event.action)
|
||||
.bind(event.entity_kind)
|
||||
.bind(event.entity_ref)
|
||||
.bind(event.config_revision)
|
||||
.bind(event.changed_fields)
|
||||
.bind(event.created_at)
|
||||
.execute(&mut **transaction)
|
||||
.await
|
||||
.map_err(|_| crate::postgres_unavailable("append backup audit event"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<BackupRecord> {
|
||||
let size_bytes = decode(row.try_get::<Option<i64>, _>("size_bytes"))?
|
||||
.map(u64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| store_invalid("size_bytes"))?;
|
||||
Ok(BackupRecord {
|
||||
id: decode(row.try_get("id"))?,
|
||||
trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?)
|
||||
.ok_or_else(|| store_invalid("trigger_kind"))?,
|
||||
status: BackupStatus::parse(decode(row.try_get::<&str, _>("status"))?)
|
||||
.ok_or_else(|| store_invalid("status"))?,
|
||||
scheduled_at: decode(row.try_get("scheduled_at"))?,
|
||||
object_key: decode(row.try_get("object_key"))?,
|
||||
size_bytes,
|
||||
sha256: decode(row.try_get("sha256"))?,
|
||||
attempt_count: u32::try_from(decode(row.try_get::<i32, _>("attempt_count"))?)
|
||||
.map_err(|_| store_invalid("attempt_count"))?,
|
||||
error_code: decode(row.try_get("error_code"))?,
|
||||
error_message: decode(row.try_get("error_message"))?,
|
||||
started_at: decode(row.try_get("started_at"))?,
|
||||
completed_at: decode(row.try_get("completed_at"))?,
|
||||
expires_at: decode(row.try_get("expires_at"))?,
|
||||
created_at: decode(row.try_get("created_at"))?,
|
||||
updated_at: decode(row.try_get("updated_at"))?,
|
||||
})
|
||||
}
|
||||
|
||||
fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult<BackupSettings> {
|
||||
let storage_revision = u64::try_from(decode(row.try_get::<i64, _>("storage_revision"))?)
|
||||
.map_err(|_| store_invalid("storage_revision"))?;
|
||||
let secret = decode(row.try_get::<Option<String>, _>("secret_access_key"))?;
|
||||
Ok(BackupSettings {
|
||||
storage_revision,
|
||||
endpoint: decode(row.try_get("endpoint"))?,
|
||||
region: decode(row.try_get("region"))?,
|
||||
bucket: decode(row.try_get("bucket"))?,
|
||||
access_key_id: decode(row.try_get("access_key_id"))?,
|
||||
secret_access_key: secret.map(SecretString::from),
|
||||
prefix: decode(row.try_get("prefix"))?,
|
||||
force_path_style: decode(row.try_get("force_path_style"))?,
|
||||
schedule_enabled: decode(row.try_get("schedule_enabled"))?,
|
||||
cron_expression: decode(row.try_get("cron_expression"))?,
|
||||
schedule_timezone: decode(row.try_get("schedule_timezone"))?,
|
||||
retention_days: u32::try_from(decode(row.try_get::<i64, _>("retention_days"))?)
|
||||
.map_err(|_| store_invalid("retention_days"))?,
|
||||
retention_count: u32::try_from(decode(row.try_get::<i64, _>("retention_count"))?)
|
||||
.map_err(|_| store_invalid("retention_count"))?,
|
||||
next_run_at: decode(row.try_get("next_run_at"))?,
|
||||
last_verified_at: decode(row.try_get("last_verified_at"))?,
|
||||
updated_at: decode(row.try_get("updated_at"))?,
|
||||
})
|
||||
}
|
||||
|
||||
/// sqlx 解码错误统一映射为备份 InvalidData。
|
||||
fn decode<T>(result: Result<T, sqlx::Error>) -> AdminStoreResult<T> {
|
||||
result.map_err(|_| {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Invalid,
|
||||
"backup",
|
||||
"backup record decode failed",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// StoreError → AdminStoreError 的显式映射。
|
||||
fn map_admin_error(error: StoreError) -> AdminStoreError {
|
||||
match error {
|
||||
StoreError::Unavailable { .. } => AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"backup",
|
||||
"backup store unavailable",
|
||||
),
|
||||
StoreError::NotFound { entity, .. } => {
|
||||
AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found")
|
||||
}
|
||||
StoreError::Conflict { entity, .. } => {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict")
|
||||
}
|
||||
StoreError::InvalidData { entity, message } => {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn store_unavailable(operation: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"backup",
|
||||
format!("{operation}: store unavailable"),
|
||||
)
|
||||
}
|
||||
|
||||
fn store_not_found(entity: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found")
|
||||
}
|
||||
|
||||
fn store_invalid(message: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", message)
|
||||
}
|
||||
@@ -414,12 +414,18 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
reasoning_preset, request_kind, subagent_kind, compact,
|
||||
image_generation_requested, started_at, deadline_at,
|
||||
provider_kind, provider_account_id, provider_account_ref,
|
||||
provider_account_name_snapshot, provider_account_email_snapshot,
|
||||
provider_account_authentication_kind_snapshot,
|
||||
upstream_model_id, upstream_transport, http_version,
|
||||
attempt_count, upstream_send_state
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10::inet, $11, $12,
|
||||
$13, $14, $15, $16, $17, $18, $19,
|
||||
$20, $21, $22, $23, $24, $25, 1, 'not_sent'
|
||||
$20, $21, $22,
|
||||
(select name from provider_accounts where id = $21),
|
||||
(select email from provider_accounts where id = $21),
|
||||
(select authentication_kind from provider_accounts where id = $21),
|
||||
$23, $24, $25, 1, 'not_sent'
|
||||
)",
|
||||
)
|
||||
.bind(request.id)
|
||||
@@ -466,6 +472,15 @@ impl ModelRequestRepository for PgExecutionStore {
|
||||
set provider_kind = $2,
|
||||
provider_account_id = $3,
|
||||
provider_account_ref = $4,
|
||||
provider_account_name_snapshot = (
|
||||
select name from provider_accounts where id = $3
|
||||
),
|
||||
provider_account_email_snapshot = (
|
||||
select email from provider_accounts where id = $3
|
||||
),
|
||||
provider_account_authentication_kind_snapshot = (
|
||||
select authentication_kind from provider_accounts where id = $3
|
||||
),
|
||||
upstream_model_id = $5,
|
||||
upstream_transport = $6,
|
||||
http_version = $7,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 八张终态业务表的 PostgreSQL adapters。
|
||||
//! 九张终态业务表的 PostgreSQL adapters。
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::{PgPool, postgres::PgPoolOptions};
|
||||
@@ -9,6 +9,7 @@ use crate::{
|
||||
|
||||
mod admin_security_audit;
|
||||
mod admission_recovery;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod execution;
|
||||
mod execution_buffer;
|
||||
@@ -21,6 +22,7 @@ mod snapshot;
|
||||
|
||||
pub use admin_security_audit::*;
|
||||
pub use admission_recovery::*;
|
||||
pub use backup::*;
|
||||
pub use client_keys::*;
|
||||
pub use execution::*;
|
||||
pub use execution_buffer::*;
|
||||
@@ -79,6 +81,13 @@ pub trait ControlPlaneRepository: Send + Sync {
|
||||
replacement: ControlPlaneReplacement,
|
||||
) -> StoreResult<ControlPlaneSnapshot>;
|
||||
|
||||
/// 更新 admin_api_key 字段并推进 config revision。
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
admin_api_key: Option<String>,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision>;
|
||||
|
||||
async fn create_client_api_key(
|
||||
&self,
|
||||
key: NewClientApiKey,
|
||||
@@ -174,6 +183,15 @@ impl ControlPlaneRepository for PgControlPlaneRepository {
|
||||
}
|
||||
}
|
||||
|
||||
async fn replace_admin_api_key(
|
||||
&self,
|
||||
admin_api_key: Option<String>,
|
||||
audit: AdminAuditEvent,
|
||||
) -> StoreResult<Revision> {
|
||||
self.apply_targeted_mutation(ControlPlaneMutation::SetAdminApiKey(admin_api_key), audit)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_client_api_key(
|
||||
&self,
|
||||
key: NewClientApiKey,
|
||||
@@ -226,6 +244,7 @@ enum ControlPlaneMutation {
|
||||
UpdateClientApiKey(UpdateClientApiKeyDetails),
|
||||
SetClientApiKeyEnabled { id: String, enabled: bool },
|
||||
DeleteClientApiKey(String),
|
||||
SetAdminApiKey(Option<String>),
|
||||
}
|
||||
|
||||
impl PgControlPlaneRepository {
|
||||
@@ -255,6 +274,9 @@ impl PgControlPlaneRepository {
|
||||
ControlPlaneMutation::DeleteClientApiKey(id) => {
|
||||
delete_client_api_key_in_transaction(&mut transaction, &id).await?;
|
||||
}
|
||||
ControlPlaneMutation::SetAdminApiKey(key) => {
|
||||
update_admin_api_key_in_transaction(&mut transaction, key).await?;
|
||||
}
|
||||
}
|
||||
append_admin_audit_event_in_transaction(&mut transaction, audit, revision).await?;
|
||||
Ok(revision)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,403 @@
|
||||
//! Pg 观测 adapter:实现 `ObservabilityRepository` 与 `AdminObservabilityStore`。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use gateway_core::provider_ports::ProviderCooldownPort;
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::redis::{CredentialLeaseRepository as _, RedisCredentialLeaseRepository};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PgObservabilityRepository {
|
||||
pool: PgPool,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
}
|
||||
|
||||
impl PgObservabilityRepository {
|
||||
#[must_use]
|
||||
pub const fn new(pool: PgPool, cooldowns: Option<Arc<dyn ProviderCooldownPort>>) -> Self {
|
||||
Self { pool, cooldowns }
|
||||
}
|
||||
|
||||
/// 账号池指标:SQL 聚合后,仅用 Redis 429 冷却重分类仍可调度的 ready 账号。
|
||||
/// 冷却中从 `active` 或 `quota_exhausted` 移到 `rate_limited`;冷却不可用时保留 SQL 分类。
|
||||
async fn provider_account_metrics_with_cooldowns(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let base = provider_account_metrics(&self.pool, observed_at).await?;
|
||||
let Some(cooldowns) = &self.cooldowns else {
|
||||
return Ok(base);
|
||||
};
|
||||
let candidates = schedulable_metric_candidates(&self.pool, observed_at).await?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(base);
|
||||
}
|
||||
// Redis 只能把 SQL 已归入 active/quota_exhausted 的候选移到冷却桶;
|
||||
// 其余状态不重建,避免 Redis 失败或脏 ID 时丢失持久化计数。
|
||||
let mut active = base.active;
|
||||
let mut rate_limited = base.rate_limited;
|
||||
let mut quota_exhausted = base.quota_exhausted;
|
||||
let now = SystemTime::now();
|
||||
for (account_id, revision, quota_reached) in candidates {
|
||||
let Ok(account_id) =
|
||||
gateway_core::engine::credential::ProviderAccountId::new(account_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let cooling = cooldowns
|
||||
.read(&account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|cooldown| cooldown.credential_revision().get() as i64 == revision)
|
||||
.filter(|cooldown| cooldown.until() > now)
|
||||
.is_some();
|
||||
if !cooling {
|
||||
continue;
|
||||
}
|
||||
if quota_reached {
|
||||
if quota_exhausted == 0 {
|
||||
continue;
|
||||
}
|
||||
quota_exhausted -= 1;
|
||||
} else {
|
||||
if active == 0 {
|
||||
continue;
|
||||
}
|
||||
active -= 1;
|
||||
}
|
||||
rate_limited += 1;
|
||||
}
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: base.total,
|
||||
enabled: base.enabled,
|
||||
unavailable: rate_limited
|
||||
+ quota_exhausted
|
||||
+ base.expired
|
||||
+ base.invalid
|
||||
+ base.disabled
|
||||
+ base.banned,
|
||||
active,
|
||||
rate_limited,
|
||||
expired: base.expired,
|
||||
invalid: base.invalid,
|
||||
quota_exhausted,
|
||||
disabled: base.disabled,
|
||||
banned: base.banned,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// `gateway-admin` 观测端口的 PostgreSQL adapter。
|
||||
///
|
||||
/// SQL 查询与内部投影继续由 [`PgObservabilityRepository`] 唯一拥有;本类型只负责
|
||||
/// Admin UTC 领域模型与持久化投影之间的无格式化转换。
|
||||
#[derive(Clone)]
|
||||
pub struct PgAdminObservabilityStore {
|
||||
repository: PgObservabilityRepository,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
}
|
||||
|
||||
impl PgAdminObservabilityStore {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
pool: PgPool,
|
||||
runtime_signals: Option<RedisCredentialLeaseRepository>,
|
||||
cooldowns: Option<Arc<dyn ProviderCooldownPort>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository: PgObservabilityRepository::new(pool, cooldowns),
|
||||
runtime_signals,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityRepository for PgObservabilityRepository {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<DashboardObservation> {
|
||||
let filter = UsageRecordFilter::default();
|
||||
let account_usage_range = ObservabilityRange::new(
|
||||
range.end - TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS),
|
||||
range.end,
|
||||
)?;
|
||||
let account_usage_query =
|
||||
ProviderAccountUsageQuery::recent(account_usage_range, DASHBOARD_ACCOUNT_LIMIT)?
|
||||
.with_hourly_request_buckets()?;
|
||||
let recent_query = UsageRecordQuery {
|
||||
range,
|
||||
filter: UsageRecordFilter {
|
||||
outcome: Some("succeeded".to_owned()),
|
||||
..UsageRecordFilter::default()
|
||||
},
|
||||
cursor: None,
|
||||
page: ObservabilityPageNumber::new(1)?,
|
||||
page_size: ObservabilityPageSize::new(10)?,
|
||||
};
|
||||
// 分两批并发:页面延迟从六路之和降为两批各自最慢者之和,同时单次
|
||||
// 渲染最多占用 3 个池连接,不与数据面写路径争抢整个连接池。
|
||||
let (requests, attempts) = futures::try_join!(
|
||||
request_metrics(&self.pool, range, &filter),
|
||||
attempt_metrics(&self.pool, range, &filter),
|
||||
)?;
|
||||
let provider_accounts = self
|
||||
.provider_account_metrics_with_cooldowns(range.end)
|
||||
.await?;
|
||||
let (trend, account_usage, recent_requests) = futures::try_join!(
|
||||
request_metric_series(&self.pool, range, &filter),
|
||||
provider_account_usage(&self.pool, account_usage_query),
|
||||
async { Ok(list_usage_records(&self.pool, recent_query).await?.items) },
|
||||
)?;
|
||||
Ok(DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
provider_accounts,
|
||||
trend,
|
||||
account_usage,
|
||||
recent_requests,
|
||||
})
|
||||
}
|
||||
|
||||
async fn dashboard_trend(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
request_metric_series(&self.pool, range, &UsageRecordFilter::default()).await
|
||||
}
|
||||
|
||||
async fn usage_trend(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
request_metric_series(&self.pool, range, &filter).await
|
||||
}
|
||||
|
||||
async fn usage_calculated_billing_facts(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<Vec<CalculatedUsageBillingFact>> {
|
||||
calculated_usage_billing_facts(&self.pool, range, &filter).await
|
||||
}
|
||||
|
||||
async fn provider_account_usage(
|
||||
&self,
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
|
||||
provider_account_usage(&self.pool, query).await
|
||||
}
|
||||
|
||||
async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult<UsageRecordPage> {
|
||||
list_usage_records(&self.pool, query).await
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, request_id: &str) -> StoreResult<UsageRecordDetail> {
|
||||
usage_record_detail(&self.pool, request_id).await
|
||||
}
|
||||
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<UsageOverview> {
|
||||
filter.validate()?;
|
||||
let requests = request_metrics(&self.pool, range, &filter).await?;
|
||||
let attempts = attempt_metrics(&self.pool, range, &filter).await?;
|
||||
let providers = provider_observations(&self.pool, range, &filter).await?;
|
||||
Ok(UsageOverview {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
providers,
|
||||
})
|
||||
}
|
||||
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
) -> StoreResult<Vec<DiagnosticObservation>> {
|
||||
usage_diagnostics(&self.pool, range, &filter, dimension).await
|
||||
}
|
||||
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult<OpsErrorPage> {
|
||||
list_ops_errors(&self.pool, query).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AdminObservabilityStore for PgAdminObservabilityStore {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
) -> AdminStoreResult<admin_observability::DashboardObservation> {
|
||||
let observation = self
|
||||
.repository
|
||||
.dashboard_summary(store_range(range)?)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
admin_dashboard_observation(observation)
|
||||
}
|
||||
|
||||
async fn dashboard_runtime_slots(
|
||||
&self,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<admin_observability::DashboardRuntimeSlots>> {
|
||||
let active_account_ids = active_provider_account_ids(&self.repository.pool, observed_at)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
let active_accounts = u64::try_from(active_account_ids.len())
|
||||
.map_err(|_| observability_error(invalid("active account count overflows u64")))?;
|
||||
if active_account_ids.is_empty() {
|
||||
return Ok(Some(admin_observability::DashboardRuntimeSlots {
|
||||
active_accounts,
|
||||
used_slots: Some(0),
|
||||
}));
|
||||
}
|
||||
let Some(runtime_signals) = &self.runtime_signals else {
|
||||
return Ok(None);
|
||||
};
|
||||
let signals = match runtime_signals
|
||||
.credential_runtime_signals(&active_account_ids)
|
||||
.await
|
||||
{
|
||||
Ok(signals) => signals,
|
||||
Err(_) => {
|
||||
return Ok(Some(admin_observability::DashboardRuntimeSlots {
|
||||
active_accounts,
|
||||
used_slots: None,
|
||||
}));
|
||||
}
|
||||
};
|
||||
let used_slots = signals.into_iter().fold(0_u64, |total, signal| {
|
||||
total.saturating_add(u64::from(signal.in_flight))
|
||||
});
|
||||
Ok(Some(admin_observability::DashboardRuntimeSlots {
|
||||
active_accounts,
|
||||
used_slots: Some(used_slots),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn dashboard_trend(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
) -> AdminStoreResult<Vec<admin_observability::RequestMetricPoint>> {
|
||||
self.repository
|
||||
.dashboard_trend(store_range(range)?)
|
||||
.await
|
||||
.map_err(observability_error)?
|
||||
.into_iter()
|
||||
.map(admin_request_metric_point)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn usage_trend(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
) -> AdminStoreResult<Vec<admin_observability::RequestMetricPoint>> {
|
||||
self.repository
|
||||
.usage_trend(store_range(range)?, store_usage_filter(filter))
|
||||
.await
|
||||
.map_err(observability_error)?
|
||||
.into_iter()
|
||||
.map(admin_request_metric_point)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn usage_calculated_billing_facts(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
) -> AdminStoreResult<Vec<admin_observability::UsageCalculatedBillingFact>> {
|
||||
self.repository
|
||||
.usage_calculated_billing_facts(store_range(range)?, store_usage_filter(filter))
|
||||
.await
|
||||
.map_err(observability_error)?
|
||||
.into_iter()
|
||||
.map(admin_calculated_usage_billing_fact)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn list_usage_records(
|
||||
&self,
|
||||
query: admin_observability::UsageQuery,
|
||||
) -> AdminStoreResult<admin_observability::UsagePage> {
|
||||
let page = self
|
||||
.repository
|
||||
.list_usage_records(store_usage_query(query)?)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
admin_usage_page(page)
|
||||
}
|
||||
|
||||
async fn usage_record_detail(
|
||||
&self,
|
||||
request_id: &str,
|
||||
) -> AdminStoreResult<admin_observability::UsageDetail> {
|
||||
let detail = self
|
||||
.repository
|
||||
.usage_record_detail(request_id)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
admin_usage_detail(detail)
|
||||
}
|
||||
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
) -> AdminStoreResult<admin_observability::UsageOverview> {
|
||||
let overview = self
|
||||
.repository
|
||||
.usage_summary(store_range(range)?, store_usage_filter(filter))
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
admin_usage_overview(overview)
|
||||
}
|
||||
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
range: admin_observability::TimeRange,
|
||||
filter: admin_observability::UsageFilter,
|
||||
dimension: admin_observability::DiagnosticDimension,
|
||||
) -> AdminStoreResult<Vec<admin_observability::DiagnosticObservation>> {
|
||||
self.repository
|
||||
.usage_diagnostics(
|
||||
store_range(range)?,
|
||||
store_usage_filter(filter),
|
||||
store_diagnostic_dimension(dimension),
|
||||
)
|
||||
.await
|
||||
.map_err(observability_error)?
|
||||
.into_iter()
|
||||
.map(admin_diagnostic_observation)
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn list_ops_errors(
|
||||
&self,
|
||||
query: admin_observability::OpsErrorQuery,
|
||||
) -> AdminStoreResult<admin_observability::OpsErrorPage> {
|
||||
let page = self
|
||||
.repository
|
||||
.list_ops_errors(store_ops_error_query(query)?)
|
||||
.await
|
||||
.map_err(observability_error)?;
|
||||
admin_ops_error_page(page)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn observability_error(
|
||||
error: StoreError,
|
||||
) -> gateway_admin::ports::store::AdminStoreError {
|
||||
admin_store_error("observability", error)
|
||||
}
|
||||
@@ -0,0 +1,864 @@
|
||||
//! Store 与 Admin 领域之间的无格式化转换及 row 解析。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn store_range(
|
||||
range: admin_observability::TimeRange,
|
||||
) -> AdminStoreResult<ObservabilityRange> {
|
||||
ObservabilityRange::new(range.start, range.end).map_err(observability_error)
|
||||
}
|
||||
|
||||
pub(crate) fn store_usage_filter(filter: admin_observability::UsageFilter) -> UsageRecordFilter {
|
||||
UsageRecordFilter {
|
||||
client_api_key_ref: filter.client_api_key_ref,
|
||||
request_id: filter.request_id,
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
operation: filter.operation,
|
||||
provider_kind: filter.provider_kind,
|
||||
model: filter.model,
|
||||
outcome: filter.outcome.map(store_request_outcome),
|
||||
status_code: filter.status_code,
|
||||
transport: filter.transport,
|
||||
attempt_index: filter.attempt_index,
|
||||
response_id: filter.response_id,
|
||||
upstream_request_id: filter.upstream_request_id,
|
||||
search: filter.search,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn store_request_outcome(outcome: admin_observability::RequestOutcome) -> String {
|
||||
outcome.as_str().to_owned()
|
||||
}
|
||||
|
||||
pub(crate) fn store_cursor(
|
||||
cursor: admin_observability::ObservabilityCursor,
|
||||
) -> AdminStoreResult<ObservabilityCursor> {
|
||||
ObservabilityCursor::new(cursor.observed_at, cursor.stable_id).map_err(observability_error)
|
||||
}
|
||||
|
||||
pub(crate) fn store_usage_query(
|
||||
query: admin_observability::UsageQuery,
|
||||
) -> AdminStoreResult<UsageRecordQuery> {
|
||||
Ok(UsageRecordQuery {
|
||||
range: store_range(query.range)?,
|
||||
filter: store_usage_filter(query.filter),
|
||||
cursor: query.cursor.map(store_cursor).transpose()?,
|
||||
page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?,
|
||||
page_size: ObservabilityPageSize::new(query.page_size.get())
|
||||
.map_err(observability_error)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn store_ops_error_filter(
|
||||
filter: admin_observability::OpsErrorFilter,
|
||||
) -> OpsErrorFilter {
|
||||
OpsErrorFilter {
|
||||
client_api_key_ref: filter.client_api_key_ref,
|
||||
request_id: filter.request_id,
|
||||
provider_account_ref: filter.provider_account_ref,
|
||||
provider_kind: filter.provider_kind,
|
||||
operation: filter.operation,
|
||||
model: filter.model,
|
||||
transport: filter.transport,
|
||||
attempt_index: filter.attempt_index,
|
||||
response_id: filter.response_id,
|
||||
upstream_request_id: filter.upstream_request_id,
|
||||
failure_kind: filter.failure_kind,
|
||||
status_code: filter.status_code,
|
||||
search: filter.search,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn store_ops_error_query(
|
||||
query: admin_observability::OpsErrorQuery,
|
||||
) -> AdminStoreResult<OpsErrorQuery> {
|
||||
Ok(OpsErrorQuery {
|
||||
range: store_range(query.range)?,
|
||||
filter: store_ops_error_filter(query.filter),
|
||||
cursor: query.cursor.map(store_cursor).transpose()?,
|
||||
page: ObservabilityPageNumber::new(query.page.get()).map_err(observability_error)?,
|
||||
page_size: ObservabilityPageSize::new(query.page_size.get())
|
||||
.map_err(observability_error)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn store_diagnostic_dimension(
|
||||
dimension: admin_observability::DiagnosticDimension,
|
||||
) -> DiagnosticDimension {
|
||||
match dimension {
|
||||
admin_observability::DiagnosticDimension::Provider => DiagnosticDimension::Provider,
|
||||
admin_observability::DiagnosticDimension::Model => DiagnosticDimension::Model,
|
||||
admin_observability::DiagnosticDimension::Account => DiagnosticDimension::Account,
|
||||
admin_observability::DiagnosticDimension::ApiKey => DiagnosticDimension::ApiKey,
|
||||
admin_observability::DiagnosticDimension::Transport => DiagnosticDimension::Transport,
|
||||
admin_observability::DiagnosticDimension::Failure => DiagnosticDimension::Failure,
|
||||
admin_observability::DiagnosticDimension::Status => DiagnosticDimension::Status,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_dashboard_observation(
|
||||
observation: DashboardObservation,
|
||||
) -> AdminStoreResult<admin_observability::DashboardObservation> {
|
||||
let DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
attempts,
|
||||
provider_accounts,
|
||||
trend,
|
||||
account_usage,
|
||||
recent_requests,
|
||||
} = observation;
|
||||
Ok(admin_observability::DashboardObservation {
|
||||
range: admin_range(range),
|
||||
requests: admin_request_metrics(requests)?,
|
||||
attempts: admin_attempt_metrics(attempts)?,
|
||||
provider_accounts: admin_account_pool_metrics(provider_accounts),
|
||||
trend: trend
|
||||
.into_iter()
|
||||
.map(admin_request_metric_point)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
account_usage: account_usage
|
||||
.into_iter()
|
||||
.map(admin_dashboard_account_usage)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
recent_requests: recent_requests
|
||||
.into_iter()
|
||||
.map(admin_usage_record)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_range(range: ObservabilityRange) -> admin_observability::TimeRange {
|
||||
admin_observability::TimeRange {
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_request_metrics(
|
||||
metrics: RequestMetrics,
|
||||
) -> AdminStoreResult<admin_observability::RequestMetrics> {
|
||||
Ok(admin_observability::RequestMetrics {
|
||||
request_count: metrics.request_count,
|
||||
success_count: metrics.success_count,
|
||||
failure_count: metrics.failure_count,
|
||||
cancelled_count: metrics.cancelled_count,
|
||||
incomplete_count: metrics.incomplete_count,
|
||||
caller_error_count: metrics.caller_error_count,
|
||||
input_tokens: metrics.input_tokens,
|
||||
output_tokens: metrics.output_tokens,
|
||||
cached_tokens: metrics.cached_tokens,
|
||||
cache_write_tokens: metrics.cache_write_tokens,
|
||||
reasoning_tokens: metrics.reasoning_tokens,
|
||||
total_tokens: metrics.total_tokens,
|
||||
first_token_latency_sum_ms: metrics.first_token_latency_sum,
|
||||
first_token_latency_count: metrics.first_token_latency_count,
|
||||
latency_sum_ms: metrics.latency_sum,
|
||||
latency_count: metrics.latency_count,
|
||||
min_latency_ms: metrics.min_latency_ms,
|
||||
max_latency_ms: metrics.max_latency_ms,
|
||||
latency_percentiles: admin_latency_percentiles(metrics.latency_percentiles)?,
|
||||
first_token_latency_percentiles: admin_latency_percentiles(
|
||||
metrics.first_token_latency_percentiles,
|
||||
)?,
|
||||
cache_eligible_request_count: metrics.cache_eligible_request_count,
|
||||
cache_hit_request_count: metrics.cache_hit_request_count,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_latency_percentiles(
|
||||
percentiles: LatencyPercentiles,
|
||||
) -> AdminStoreResult<admin_observability::LatencyPercentiles> {
|
||||
Ok(admin_observability::LatencyPercentiles {
|
||||
p50_ms: percentiles
|
||||
.p50_ms
|
||||
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
|
||||
.transpose()
|
||||
.map_err(|_| invalid_admin_percentile())?,
|
||||
p95_ms: percentiles
|
||||
.p95_ms
|
||||
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
|
||||
.transpose()
|
||||
.map_err(|_| invalid_admin_percentile())?,
|
||||
p99_ms: percentiles
|
||||
.p99_ms
|
||||
.map(|value| admin_observability::PercentileMilliseconds::new(value.as_f64()))
|
||||
.transpose()
|
||||
.map_err(|_| invalid_admin_percentile())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn invalid_admin_percentile() -> gateway_admin::ports::store::AdminStoreError {
|
||||
observability_error(StoreError::InvalidData {
|
||||
entity: "observability latency percentile",
|
||||
message: "latency percentile is outside the admin contract".to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_attempt_metrics(
|
||||
metrics: AttemptMetrics,
|
||||
) -> AdminStoreResult<admin_observability::AttemptMetrics> {
|
||||
Ok(admin_observability::AttemptMetrics {
|
||||
attempt_count: metrics.attempt_count,
|
||||
success_count: metrics.success_count,
|
||||
failure_count: metrics.failure_count,
|
||||
cancelled_count: metrics.cancelled_count,
|
||||
incomplete_count: metrics.incomplete_count,
|
||||
rate_limited_count: metrics.rate_limited_count,
|
||||
auth_failure_count: metrics.auth_failure_count,
|
||||
provider_5xx_count: metrics.provider_5xx_count,
|
||||
cost_coverage: admin_cost_coverage(metrics.cost_coverage),
|
||||
costs: admin_currency_costs(metrics.costs)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_cost_coverage(
|
||||
coverage: CostCoverage,
|
||||
) -> admin_observability::CostCoverage {
|
||||
admin_observability::CostCoverage {
|
||||
provider_reported_count: coverage.provider_reported_count,
|
||||
calculated_count: coverage.calculated_count,
|
||||
partial_count: 0,
|
||||
unavailable_count: coverage.unavailable_count,
|
||||
not_billable_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_currency_costs(
|
||||
costs: Vec<CurrencyCostTotal>,
|
||||
) -> AdminStoreResult<Vec<admin_observability::CurrencyCost>> {
|
||||
costs
|
||||
.into_iter()
|
||||
.map(|cost| {
|
||||
Ok(admin_observability::CurrencyCost {
|
||||
currency: cost.currency,
|
||||
amount: admin_decimal_amount(cost.amount)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn admin_decimal_amount(
|
||||
amount: DecimalAmount,
|
||||
) -> AdminStoreResult<admin_observability::DecimalAmount> {
|
||||
admin_observability::DecimalAmount::from_str(amount.as_str()).map_err(|_| {
|
||||
observability_error(StoreError::InvalidData {
|
||||
entity: "observability decimal amount",
|
||||
message: "amount is outside the admin numeric contract".to_owned(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_optional_decimal_amount(
|
||||
amount: Option<DecimalAmount>,
|
||||
) -> AdminStoreResult<Option<admin_observability::DecimalAmount>> {
|
||||
amount.map(admin_decimal_amount).transpose()
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_account_pool_metrics(
|
||||
metrics: ProviderAccountMetrics,
|
||||
) -> admin_observability::AccountPoolMetrics {
|
||||
admin_observability::AccountPoolMetrics {
|
||||
total: metrics.total,
|
||||
enabled: metrics.enabled,
|
||||
unavailable: metrics.unavailable,
|
||||
active: metrics.active,
|
||||
rate_limited: metrics.rate_limited,
|
||||
expired: metrics.expired,
|
||||
invalid: metrics.invalid,
|
||||
quota_exhausted: metrics.quota_exhausted,
|
||||
refreshing: None,
|
||||
disabled: metrics.disabled,
|
||||
banned: metrics.banned,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_dashboard_account_usage(
|
||||
usage: ProviderAccountUsageObservation,
|
||||
) -> AdminStoreResult<admin_observability::DashboardAccountUsage> {
|
||||
Ok(admin_observability::DashboardAccountUsage {
|
||||
account_id: usage.account_id,
|
||||
provider_kind: usage.provider_kind,
|
||||
authentication_kind: usage.authentication_kind,
|
||||
name: usage.name,
|
||||
email: usage.email,
|
||||
plan_type: usage.plan_type,
|
||||
enabled: usage.enabled,
|
||||
availability: usage.availability,
|
||||
request_count: usage.request_count,
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
image_request_count: usage.image_request_count,
|
||||
image_request_failed_count: usage.image_request_failed_count,
|
||||
total_tokens: usage.total_tokens,
|
||||
cost_coverage: admin_cost_coverage(usage.cost_coverage),
|
||||
costs: admin_currency_costs(usage.costs)?,
|
||||
last_used_at: usage.last_used_at,
|
||||
request_buckets: usage
|
||||
.request_buckets
|
||||
.into_iter()
|
||||
.map(
|
||||
|bucket| admin_observability::DashboardAccountRequestBucket {
|
||||
bucket_start: bucket.bucket_start,
|
||||
request_count: bucket.request_count,
|
||||
},
|
||||
)
|
||||
.collect(),
|
||||
quota_used_percent: None,
|
||||
models: usage
|
||||
.models
|
||||
.into_iter()
|
||||
.map(admin_dashboard_account_model_usage)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_dashboard_account_model_usage(
|
||||
usage: ProviderAccountModelUsageObservation,
|
||||
) -> AdminStoreResult<admin_observability::DashboardAccountModelUsage> {
|
||||
Ok(admin_observability::DashboardAccountModelUsage {
|
||||
model: usage.model,
|
||||
request_count: usage.request_count,
|
||||
success_count: usage.success_count,
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
cached_tokens: usage.cached_tokens,
|
||||
cache_write_tokens: usage.cache_write_tokens,
|
||||
reasoning_tokens: usage.reasoning_tokens,
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
image_request_count: usage.image_request_count,
|
||||
image_request_failed_count: usage.image_request_failed_count,
|
||||
total_tokens: usage.total_tokens,
|
||||
cost_coverage: admin_cost_coverage(usage.cost_coverage),
|
||||
costs: admin_currency_costs(usage.costs)?,
|
||||
last_used_at: usage.last_used_at,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_request_metric_point(
|
||||
point: RequestMetricPoint,
|
||||
) -> AdminStoreResult<admin_observability::RequestMetricPoint> {
|
||||
Ok(admin_observability::RequestMetricPoint {
|
||||
bucket_start: point.bucket_start,
|
||||
granularity: admin_granularity(point.granularity),
|
||||
metrics: admin_request_metrics(point.metrics)?,
|
||||
cost_coverage: admin_cost_coverage(point.cost_coverage),
|
||||
costs: admin_currency_costs(point.costs)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_calculated_usage_billing_fact(
|
||||
fact: CalculatedUsageBillingFact,
|
||||
) -> AdminStoreResult<admin_observability::UsageCalculatedBillingFact> {
|
||||
Ok(admin_observability::UsageCalculatedBillingFact {
|
||||
bucket_start: fact.bucket_start,
|
||||
provider_kind: fact.provider_kind,
|
||||
upstream_model_id: fact.upstream_model_id,
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
cache_write_tokens: fact.cache_write_tokens,
|
||||
total: admin_observability::CurrencyCost {
|
||||
currency: fact.total.currency,
|
||||
amount: admin_decimal_amount(fact.total.amount)?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) const fn admin_granularity(
|
||||
granularity: ObservationGranularity,
|
||||
) -> admin_observability::Granularity {
|
||||
match granularity {
|
||||
ObservationGranularity::FifteenMinutes => admin_observability::Granularity::FifteenMinutes,
|
||||
ObservationGranularity::Hour => admin_observability::Granularity::Hour,
|
||||
ObservationGranularity::Day => admin_observability::Granularity::Day,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_page(
|
||||
page: UsageRecordPage,
|
||||
) -> AdminStoreResult<admin_observability::UsagePage> {
|
||||
Ok(admin_observability::UsagePage {
|
||||
items: page
|
||||
.items
|
||||
.into_iter()
|
||||
.map(admin_usage_record)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
total: page.total,
|
||||
next_cursor: page.next_cursor.map(admin_cursor),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_cursor(
|
||||
cursor: ObservabilityCursor,
|
||||
) -> admin_observability::ObservabilityCursor {
|
||||
admin_observability::ObservabilityCursor {
|
||||
observed_at: cursor.observed_at,
|
||||
stable_id: cursor.stable_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_record(
|
||||
record: UsageRecord,
|
||||
) -> AdminStoreResult<admin_observability::UsageRecord> {
|
||||
let billing = match (&record.cost_amount, &record.cost_currency) {
|
||||
(Some(amount), Some(currency)) => Some(admin_observability::UsageBilling::Total {
|
||||
source: record.cost_source.clone(),
|
||||
total: admin_observability::CurrencyCost {
|
||||
currency: currency.clone(),
|
||||
amount: admin_decimal_amount(amount.clone())?,
|
||||
},
|
||||
}),
|
||||
(None, None) => None,
|
||||
_ => {
|
||||
return Err(observability_error(StoreError::InvalidData {
|
||||
entity: "observability request billing",
|
||||
message: "cost amount and currency must be present together".to_owned(),
|
||||
}));
|
||||
}
|
||||
};
|
||||
Ok(admin_observability::UsageRecord {
|
||||
id: record.id,
|
||||
client_api_key_ref: record.client_api_key_ref,
|
||||
config_revision: record.config_revision,
|
||||
protocol: record.protocol,
|
||||
operation: record.operation,
|
||||
endpoint: record.endpoint,
|
||||
client_transport: record.client_transport,
|
||||
requested_model_id: record.requested_model_id,
|
||||
provider_kind: record.provider_kind,
|
||||
provider_account_ref: record.provider_account_ref,
|
||||
provider_account_name: record.provider_account_name,
|
||||
provider_account_email: record.provider_account_email,
|
||||
provider_account_authentication_kind: record.provider_account_authentication_kind,
|
||||
upstream_model_id: record.upstream_model_id,
|
||||
upstream_transport: record.upstream_transport,
|
||||
http_version: record.http_version,
|
||||
websocket_pool: record.websocket_pool,
|
||||
service_tier: record.service_tier,
|
||||
provider_metadata_json: record.provider_metadata_json,
|
||||
attempt_count: record.attempt_count,
|
||||
upstream_send_state: record.upstream_send_state,
|
||||
downstream_committed_at: record.downstream_committed_at,
|
||||
outcome: admin_request_outcome(&record.outcome)?,
|
||||
client_status_code: record.client_status_code,
|
||||
upstream_status_code: record.upstream_status_code,
|
||||
client_response_id: record.client_response_id,
|
||||
upstream_request_id: record.upstream_request_id,
|
||||
upstream_response_id: record.upstream_response_id,
|
||||
error_kind: record.error_kind,
|
||||
provider_error_code: record.provider_error_code,
|
||||
error_message: record.error_message,
|
||||
retry_after_ms: record.retry_after_ms,
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
cache_write_tokens: record.cache_write_tokens,
|
||||
reasoning_tokens: record.reasoning_tokens,
|
||||
image_input_tokens: record.image_input_tokens,
|
||||
image_output_tokens: record.image_output_tokens,
|
||||
total_tokens: record.total_tokens,
|
||||
cost_source: record.cost_source,
|
||||
cost_amount: admin_optional_decimal_amount(record.cost_amount)?,
|
||||
cost_currency: record.cost_currency,
|
||||
billing,
|
||||
transport_decision_wait_ms: record.transport_decision_wait_ms,
|
||||
connect_ms: record.connect_ms,
|
||||
headers_ms: record.headers_ms,
|
||||
first_event_ms: record.first_event_ms,
|
||||
first_reasoning_ms: record.first_reasoning_ms,
|
||||
first_text_ms: record.first_text_ms,
|
||||
first_token_ms: record.first_token_ms,
|
||||
provider_processing_ms: record.provider_processing_ms,
|
||||
latency_ms: record.latency_ms,
|
||||
client_ip: record.client_ip,
|
||||
user_agent: record.user_agent,
|
||||
reasoning_effort: record.reasoning_effort,
|
||||
reasoning_preset: record.reasoning_preset,
|
||||
request_kind: record.request_kind,
|
||||
subagent_kind: record.subagent_kind,
|
||||
compact: record.compact,
|
||||
image_generation_requested: record.image_generation_requested,
|
||||
image_generation_succeeded: record.image_generation_succeeded,
|
||||
started_at: record.started_at,
|
||||
deadline_at: record.deadline_at,
|
||||
completed_at: record.completed_at,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_request_outcome(
|
||||
outcome: &str,
|
||||
) -> AdminStoreResult<admin_observability::RequestOutcome> {
|
||||
admin_observability::RequestOutcome::new(outcome.to_owned()).map_err(|_| {
|
||||
observability_error(StoreError::InvalidData {
|
||||
entity: "observability request outcome",
|
||||
message: "invalid request outcome".to_owned(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_detail(
|
||||
detail: UsageRecordDetail,
|
||||
) -> AdminStoreResult<admin_observability::UsageDetail> {
|
||||
Ok(admin_observability::UsageDetail {
|
||||
request: admin_usage_record(detail.request)?,
|
||||
attempts: detail
|
||||
.attempts
|
||||
.into_iter()
|
||||
.map(admin_usage_attempt)
|
||||
.collect::<AdminStoreResult<_>>()?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_attempt(
|
||||
attempt: UsageAttemptObservation,
|
||||
) -> AdminStoreResult<admin_observability::UsageAttempt> {
|
||||
Ok(admin_observability::UsageAttempt {
|
||||
source: attempt.source,
|
||||
id: attempt.id,
|
||||
attempt_index: attempt.attempt_index,
|
||||
component: attempt.component,
|
||||
operation: attempt.operation,
|
||||
provider_kind: attempt.provider_kind,
|
||||
provider_account_ref: attempt.provider_account_ref,
|
||||
provider_account_name: attempt.provider_account_name,
|
||||
provider_account_email: attempt.provider_account_email,
|
||||
provider_account_authentication_kind: attempt.provider_account_authentication_kind,
|
||||
upstream_model_id: attempt.upstream_model_id,
|
||||
upstream_transport: attempt.upstream_transport,
|
||||
upstream_send_state: attempt.upstream_send_state,
|
||||
outcome: admin_request_outcome(&attempt.outcome)?,
|
||||
downstream_committed: attempt.downstream_committed,
|
||||
status_code: attempt.status_code,
|
||||
provider_error_code: attempt.provider_error_code,
|
||||
failure_kind: attempt.failure_kind,
|
||||
retry_after_ms: attempt.retry_after_ms,
|
||||
upstream_request_id: attempt.upstream_request_id,
|
||||
latency_ms: attempt.latency_ms,
|
||||
message: attempt.message,
|
||||
input_tokens: attempt.input_tokens,
|
||||
output_tokens: attempt.output_tokens,
|
||||
cached_tokens: attempt.cached_tokens,
|
||||
cache_write_tokens: attempt.cache_write_tokens,
|
||||
reasoning_tokens: attempt.reasoning_tokens,
|
||||
total_tokens: attempt.total_tokens,
|
||||
cost_source: attempt.cost_source,
|
||||
cost_amount: admin_optional_decimal_amount(attempt.cost_amount)?,
|
||||
cost_currency: attempt.cost_currency,
|
||||
occurred_at: attempt.occurred_at,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_usage_overview(
|
||||
overview: UsageOverview,
|
||||
) -> AdminStoreResult<admin_observability::UsageOverview> {
|
||||
Ok(admin_observability::UsageOverview {
|
||||
range: admin_range(overview.range),
|
||||
requests: admin_request_metrics(overview.requests)?,
|
||||
attempts: admin_attempt_metrics(overview.attempts)?,
|
||||
providers: overview
|
||||
.providers
|
||||
.into_iter()
|
||||
.map(admin_provider_observation)
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_provider_observation(
|
||||
observation: ProviderObservation,
|
||||
) -> admin_observability::ProviderObservation {
|
||||
admin_observability::ProviderObservation {
|
||||
provider_kind: observation.provider_kind,
|
||||
request_count: observation.request_count,
|
||||
attempt_count: observation.attempt_count,
|
||||
failure_count: observation.failure_count,
|
||||
total_tokens: observation.total_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn admin_diagnostic_observation(
|
||||
observation: DiagnosticObservation,
|
||||
) -> AdminStoreResult<admin_observability::DiagnosticObservation> {
|
||||
Ok(admin_observability::DiagnosticObservation {
|
||||
key: observation.key,
|
||||
name: observation.name,
|
||||
request_count: observation.request_count,
|
||||
success_count: observation.success_count,
|
||||
failure_count: observation.failure_count,
|
||||
attempt_count: observation.attempt_count,
|
||||
total_tokens: observation.total_tokens,
|
||||
average_latency_ms: observation.average_latency_ms,
|
||||
latency_p95_ms: observation.latency_p95_ms,
|
||||
cost_coverage: admin_cost_coverage(observation.cost_coverage),
|
||||
costs: admin_currency_costs(observation.costs)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_ops_error_page(
|
||||
page: OpsErrorPage,
|
||||
) -> AdminStoreResult<admin_observability::OpsErrorPage> {
|
||||
Ok(admin_observability::OpsErrorPage {
|
||||
items: page.items.into_iter().map(admin_ops_error).collect(),
|
||||
total: page.total,
|
||||
next_cursor: page.next_cursor.map(admin_cursor),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn admin_ops_error(error: OpsErrorRecord) -> admin_observability::OpsError {
|
||||
admin_observability::OpsError {
|
||||
source: error.source,
|
||||
event_id: error.event_id,
|
||||
request_id: error.request_id,
|
||||
attempt_index: error.attempt_index,
|
||||
client_api_key_ref: error.client_api_key_ref,
|
||||
component: error.component,
|
||||
operation: error.operation,
|
||||
endpoint: error.endpoint,
|
||||
provider_kind: error.provider_kind,
|
||||
provider_account_ref: error.provider_account_ref,
|
||||
provider_account_name: error.provider_account_name,
|
||||
provider_account_email: error.provider_account_email,
|
||||
provider_account_authentication_kind: error.provider_account_authentication_kind,
|
||||
upstream_model_id: error.upstream_model_id,
|
||||
upstream_transport: error.upstream_transport,
|
||||
failure_kind: error.failure_kind,
|
||||
client_status_code: error.client_status_code,
|
||||
upstream_status_code: error.upstream_status_code,
|
||||
provider_error_code: error.provider_error_code,
|
||||
client_response_id: error.client_response_id,
|
||||
upstream_request_id: error.upstream_request_id,
|
||||
latency_ms: error.latency_ms,
|
||||
message: error.message,
|
||||
occurrence_count: error.occurrence_count,
|
||||
occurred_at: error.occurred_at,
|
||||
stable_sort_id: error.stable_sort_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<UsageRecord> {
|
||||
Ok(UsageRecord {
|
||||
id: get(row, "id")?,
|
||||
client_api_key_ref: get(row, "client_api_key_ref")?,
|
||||
config_revision: unsigned(row, "config_revision")?,
|
||||
protocol: get(row, "protocol")?,
|
||||
operation: get(row, "operation")?,
|
||||
endpoint: get(row, "endpoint")?,
|
||||
client_transport: get(row, "client_transport")?,
|
||||
requested_model_id: get(row, "requested_model_id")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
provider_account_ref: get(row, "provider_account_ref")?,
|
||||
provider_account_name: get(row, "provider_account_name")?,
|
||||
provider_account_email: get(row, "provider_account_email")?,
|
||||
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
upstream_transport: get(row, "upstream_transport")?,
|
||||
http_version: get(row, "http_version")?,
|
||||
websocket_pool: get(row, "websocket_pool")?,
|
||||
service_tier: get(row, "service_tier")?,
|
||||
provider_metadata_json: get::<Option<serde_json::Value>>(row, "provider_observation_json")?
|
||||
.map(|value| serde_json::to_string(&value))
|
||||
.transpose()
|
||||
.map_err(|_| postgres_unavailable("encode provider observation"))?,
|
||||
attempt_count: to_u32(get(row, "attempt_count")?)?,
|
||||
upstream_send_state: get(row, "upstream_send_state")?,
|
||||
downstream_committed_at: get(row, "downstream_committed_at")?,
|
||||
outcome: get(row, "outcome")?,
|
||||
client_status_code: optional_status(row, "client_status_code")?,
|
||||
upstream_status_code: optional_status(row, "upstream_status_code")?,
|
||||
client_response_id: opaque_response_id(row, "client_response_id")?,
|
||||
upstream_request_id: get(row, "upstream_request_id")?,
|
||||
upstream_response_id: opaque_response_id(row, "upstream_response_id")?,
|
||||
error_kind: get(row, "error_kind")?,
|
||||
provider_error_code: get(row, "provider_error_code")?,
|
||||
error_message: get(row, "error_message")?,
|
||||
retry_after_ms: optional_unsigned(row, "retry_after_ms")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
cost_source: get(row, "cost_source")?,
|
||||
cost_amount: optional_decimal(row, "cost_amount")?,
|
||||
cost_currency: get(row, "cost_currency")?,
|
||||
transport_decision_wait_ms: optional_unsigned(row, "transport_decision_wait_ms")?,
|
||||
connect_ms: optional_unsigned(row, "connect_ms")?,
|
||||
headers_ms: optional_unsigned(row, "headers_ms")?,
|
||||
first_event_ms: optional_unsigned(row, "first_event_ms")?,
|
||||
first_reasoning_ms: optional_unsigned(row, "first_reasoning_ms")?,
|
||||
first_text_ms: optional_unsigned(row, "first_text_ms")?,
|
||||
first_token_ms: optional_unsigned(row, "first_token_ms")?,
|
||||
provider_processing_ms: optional_unsigned(row, "provider_processing_ms")?,
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
client_ip: get(row, "client_ip")?,
|
||||
user_agent: get(row, "user_agent")?,
|
||||
reasoning_effort: get(row, "reasoning_effort")?,
|
||||
reasoning_preset: get(row, "reasoning_preset")?,
|
||||
request_kind: get(row, "request_kind")?,
|
||||
subagent_kind: get(row, "subagent_kind")?,
|
||||
compact: get(row, "compact")?,
|
||||
image_generation_requested: get(row, "image_generation_requested")?,
|
||||
image_generation_succeeded: get(row, "image_generation_succeeded")?,
|
||||
started_at: get(row, "started_at")?,
|
||||
deadline_at: get(row, "deadline_at")?,
|
||||
completed_at: get(row, "completed_at")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn ops_error_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<OpsErrorRecord> {
|
||||
Ok(OpsErrorRecord {
|
||||
source: get(row, "source")?,
|
||||
event_id: get(row, "event_id")?,
|
||||
request_id: get(row, "request_id")?,
|
||||
attempt_index: get::<Option<i32>>(row, "attempt_index")?
|
||||
.map(to_u32)
|
||||
.transpose()?,
|
||||
client_api_key_ref: get(row, "client_api_key_ref")?,
|
||||
component: get(row, "component")?,
|
||||
operation: get(row, "operation")?,
|
||||
endpoint: get(row, "endpoint")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
provider_account_ref: get(row, "provider_account_ref")?,
|
||||
provider_account_name: get(row, "provider_account_name")?,
|
||||
provider_account_email: get(row, "provider_account_email")?,
|
||||
provider_account_authentication_kind: get(row, "provider_account_authentication_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
upstream_transport: get(row, "upstream_transport")?,
|
||||
failure_kind: get(row, "failure_kind")?,
|
||||
client_status_code: optional_status(row, "client_status_code")?,
|
||||
upstream_status_code: optional_status(row, "upstream_status_code")?,
|
||||
provider_error_code: get(row, "provider_error_code")?,
|
||||
client_response_id: opaque_response_id(row, "client_response_id")?,
|
||||
upstream_request_id: get(row, "upstream_request_id")?,
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
message: get(row, "message")?,
|
||||
occurrence_count: to_u32(get(row, "occurrence_count")?)?,
|
||||
occurred_at: get(row, "occurred_at")?,
|
||||
stable_sort_id: get(row, "stable_sort_id")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn cost_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CurrencyCostTotal> {
|
||||
Ok(CurrencyCostTotal {
|
||||
currency: get(row, "cost_currency")?,
|
||||
amount: DecimalAmount::from_str(&get::<String>(row, "amount")?)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn calculated_usage_billing_fact_from_row(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<CalculatedUsageBillingFact> {
|
||||
Ok(CalculatedUsageBillingFact {
|
||||
bucket_start: get(row, "bucket_start")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
upstream_model_id: get(row, "upstream_model_id")?,
|
||||
service_tier: get(row, "service_tier")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
total: cost_from_row(row)?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn optional_decimal(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> StoreResult<Option<DecimalAmount>> {
|
||||
get::<Option<String>>(row, column)?
|
||||
.map(|value| DecimalAmount::from_str(&value))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn opaque_response_id(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> StoreResult<Option<String>> {
|
||||
get::<Option<Vec<u8>>>(row, column)?
|
||||
.map(String::from_utf8)
|
||||
.transpose()
|
||||
.map_err(|_| invalid(column))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_account_ids(account_ids: &[String]) -> StoreResult<()> {
|
||||
let mut unique = BTreeSet::new();
|
||||
for account_id in account_ids {
|
||||
validate_text(account_id, MAX_FILTER_BYTES, "provider account ID")?;
|
||||
if !unique.insert(account_id.as_str()) {
|
||||
return Err(invalid("account usage query contains duplicate IDs"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_optional_text(
|
||||
value: Option<&str>,
|
||||
max_bytes: usize,
|
||||
field: &'static str,
|
||||
) -> StoreResult<()> {
|
||||
value.map_or(Ok(()), |value| validate_text(value, max_bytes, field))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_text(value: &str, max_bytes: usize, field: &'static str) -> StoreResult<()> {
|
||||
if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) {
|
||||
return Err(invalid(&format!("{field} is invalid")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn get<'r, T>(row: &'r sqlx::postgres::PgRow, column: &'static str) -> StoreResult<T>
|
||||
where
|
||||
T: sqlx::Decode<'r, Postgres> + sqlx::Type<Postgres>,
|
||||
{
|
||||
row.try_get(column).map_err(|_| invalid(column))
|
||||
}
|
||||
|
||||
pub(crate) fn unsigned(row: &sqlx::postgres::PgRow, column: &'static str) -> StoreResult<u64> {
|
||||
to_u64(get(row, column)?)
|
||||
}
|
||||
|
||||
pub(crate) fn optional_unsigned(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> StoreResult<Option<u64>> {
|
||||
get::<Option<i64>>(row, column)?.map(to_u64).transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn optional_status(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &'static str,
|
||||
) -> StoreResult<Option<u16>> {
|
||||
get::<Option<i32>>(row, column)?
|
||||
.map(|value| {
|
||||
u16::try_from(value)
|
||||
.ok()
|
||||
.filter(|value| (100..=599).contains(value))
|
||||
.ok_or_else(|| invalid("status code is outside its supported range"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn to_u64(value: i64) -> StoreResult<u64> {
|
||||
u64::try_from(value).map_err(|_| invalid("numeric observation is negative"))
|
||||
}
|
||||
|
||||
pub(crate) fn to_u32(value: i32) -> StoreResult<u32> {
|
||||
u32::try_from(value).map_err(|_| invalid("integer observation is negative"))
|
||||
}
|
||||
|
||||
pub(crate) fn invalid(message: &str) -> StoreError {
|
||||
StoreError::InvalidData {
|
||||
entity: "observability",
|
||||
message: message.to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//! 从 `model_requests`、`ops_events` 与账号公共投影读取观测事实。
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, HashMap},
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_admin::{
|
||||
model::observability as admin_observability,
|
||||
ports::store::{AdminStoreResult, ObservabilityStore as AdminObservabilityStore},
|
||||
};
|
||||
use sqlx::{PgPool, Postgres, QueryBuilder, Row};
|
||||
|
||||
use crate::{
|
||||
DecimalAmount, StoreError, StoreResult, admin_store_error, postgres_unavailable,
|
||||
require_nonempty,
|
||||
};
|
||||
|
||||
mod admin_adapter;
|
||||
mod mapping;
|
||||
mod model;
|
||||
mod queries;
|
||||
|
||||
pub use admin_adapter::*;
|
||||
pub(crate) use mapping::*;
|
||||
pub use model::*;
|
||||
pub(crate) use queries::*;
|
||||
@@ -0,0 +1,715 @@
|
||||
//! 查询模型、校验与观测端口契约。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(crate) const MAX_PAGE_SIZE: u16 = 100;
|
||||
pub(crate) const MAX_FILTER_BYTES: usize = 256;
|
||||
pub(crate) const MAX_SEARCH_BYTES: usize = 512;
|
||||
pub(crate) const MAX_ACCOUNT_IDS: usize = 200;
|
||||
/// 概览卡只展示最近使用的四个账号;完整账号用量由账号管理页单独查询。
|
||||
pub(crate) const DASHBOARD_ACCOUNT_LIMIT: u16 = 4;
|
||||
pub(crate) const DIAGNOSTIC_LIMIT: i64 = 100;
|
||||
pub(crate) const ACCOUNT_USAGE_TIMELINE_HOURS: i64 = 24;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ObservabilityRange {
|
||||
pub start: DateTime<Utc>,
|
||||
pub end: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl ObservabilityRange {
|
||||
pub fn new(start: DateTime<Utc>, end: DateTime<Utc>) -> StoreResult<Self> {
|
||||
if end.signed_duration_since(start) <= TimeDelta::zero() {
|
||||
return Err(invalid("time range must be positive"));
|
||||
}
|
||||
Ok(Self { start, end })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ObservabilityPageSize(u16);
|
||||
|
||||
impl ObservabilityPageSize {
|
||||
pub fn new(value: u16) -> StoreResult<Self> {
|
||||
if value == 0 || value > MAX_PAGE_SIZE {
|
||||
return Err(invalid("page size must be between 1 and 100"));
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u16 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ObservabilityPageNumber(u32);
|
||||
|
||||
impl ObservabilityPageNumber {
|
||||
pub fn new(value: u32) -> StoreResult<Self> {
|
||||
if value == 0 {
|
||||
return Err(invalid("page number must be positive"));
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn get(self) -> u32 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObservabilityCursor {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
impl ObservabilityCursor {
|
||||
pub fn new(observed_at: DateTime<Utc>, stable_id: impl Into<String>) -> StoreResult<Self> {
|
||||
let stable_id = stable_id.into();
|
||||
validate_text(&stable_id, MAX_FILTER_BYTES, "cursor ID")?;
|
||||
Ok(Self {
|
||||
observed_at,
|
||||
stable_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct UsageRecordFilter {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub outcome: Option<String>,
|
||||
pub status_code: Option<u16>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
impl UsageRecordFilter {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
for (value, field) in [
|
||||
(self.client_api_key_ref.as_deref(), "client API key filter"),
|
||||
(self.request_id.as_deref(), "request ID filter"),
|
||||
(
|
||||
self.provider_account_ref.as_deref(),
|
||||
"provider account filter",
|
||||
),
|
||||
(self.operation.as_deref(), "operation filter"),
|
||||
(self.provider_kind.as_deref(), "provider filter"),
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.outcome.as_deref(), "outcome filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(
|
||||
self.upstream_request_id.as_deref(),
|
||||
"upstream request ID filter",
|
||||
),
|
||||
] {
|
||||
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
|
||||
}
|
||||
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
|
||||
if self
|
||||
.status_code
|
||||
.is_some_and(|status| !(100..=599).contains(&status))
|
||||
{
|
||||
return Err(invalid("status code filter must be between 100 and 599"));
|
||||
}
|
||||
if self
|
||||
.attempt_index
|
||||
.is_some_and(|index| index == 0 || i32::try_from(index).is_err())
|
||||
{
|
||||
return Err(invalid("attempt index filter is out of range"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecordQuery {
|
||||
pub range: ObservabilityRange,
|
||||
pub filter: UsageRecordFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub page: ObservabilityPageNumber,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct OpsErrorFilter {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub failure_kind: Option<String>,
|
||||
pub status_code: Option<u16>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
impl OpsErrorFilter {
|
||||
pub fn validate(&self) -> StoreResult<()> {
|
||||
for (value, field) in [
|
||||
(self.client_api_key_ref.as_deref(), "client API key filter"),
|
||||
(self.request_id.as_deref(), "request ID filter"),
|
||||
(
|
||||
self.provider_account_ref.as_deref(),
|
||||
"provider account filter",
|
||||
),
|
||||
(self.provider_kind.as_deref(), "provider filter"),
|
||||
(self.operation.as_deref(), "operation filter"),
|
||||
(self.model.as_deref(), "model filter"),
|
||||
(self.transport.as_deref(), "transport filter"),
|
||||
(
|
||||
self.upstream_request_id.as_deref(),
|
||||
"upstream request ID filter",
|
||||
),
|
||||
(self.failure_kind.as_deref(), "failure kind filter"),
|
||||
] {
|
||||
validate_optional_text(value, MAX_FILTER_BYTES, field)?;
|
||||
}
|
||||
validate_optional_text(self.search.as_deref(), MAX_SEARCH_BYTES, "search filter")?;
|
||||
if self
|
||||
.status_code
|
||||
.is_some_and(|status| !(100..=599).contains(&status))
|
||||
{
|
||||
return Err(invalid("status code filter must be between 100 and 599"));
|
||||
}
|
||||
if self
|
||||
.attempt_index
|
||||
.is_some_and(|index| index == 0 || i32::try_from(index).is_err())
|
||||
{
|
||||
return Err(invalid("attempt index filter is out of range"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpsErrorQuery {
|
||||
pub range: ObservabilityRange,
|
||||
pub filter: OpsErrorFilter,
|
||||
pub cursor: Option<ObservabilityCursor>,
|
||||
pub page: ObservabilityPageNumber,
|
||||
pub page_size: ObservabilityPageSize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DiagnosticDimension {
|
||||
Provider,
|
||||
Model,
|
||||
Account,
|
||||
ApiKey,
|
||||
Transport,
|
||||
Failure,
|
||||
Status,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CurrencyCostTotal {
|
||||
pub currency: String,
|
||||
pub amount: DecimalAmount,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CostCoverage {
|
||||
pub provider_reported_count: u64,
|
||||
pub calculated_count: u64,
|
||||
pub unavailable_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RequestMetrics {
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub first_token_latency_sum: u64,
|
||||
pub first_token_latency_count: u64,
|
||||
pub latency_sum: u64,
|
||||
pub latency_count: u64,
|
||||
pub max_latency_ms: Option<u64>,
|
||||
pub min_latency_ms: Option<u64>,
|
||||
/// 分母:`input_tokens is not null`,即上游确实报告过 input token 事实的请求。
|
||||
pub cache_eligible_request_count: u64,
|
||||
/// 分子:分母集合中 `cached_tokens > 0` 的请求。
|
||||
pub cache_hit_request_count: u64,
|
||||
pub latency_percentiles: LatencyPercentiles,
|
||||
pub first_token_latency_percentiles: LatencyPercentiles,
|
||||
}
|
||||
|
||||
impl RequestMetrics {
|
||||
/// 请求级 cache hit rate;没有 input token 事实时返回 `None`。
|
||||
#[must_use]
|
||||
pub fn cache_hit_request_rate(&self) -> Option<f64> {
|
||||
(self.cache_eligible_request_count > 0)
|
||||
.then(|| self.cache_hit_request_count as f64 / self.cache_eligible_request_count as f64)
|
||||
}
|
||||
}
|
||||
|
||||
/// PostgreSQL `percentile_cont` 的非负、有限毫秒值;bits 保留插值小数且可安全比较。
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct PercentileMilliseconds(u64);
|
||||
|
||||
impl PercentileMilliseconds {
|
||||
pub(crate) fn new(value: f64) -> StoreResult<Self> {
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return Err(postgres_unavailable("decode latency percentile"));
|
||||
}
|
||||
Ok(Self(value.to_bits()))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn as_f64(self) -> f64 {
|
||||
f64::from_bits(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PercentileMilliseconds {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
self.as_f64().fmt(formatter)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct LatencyPercentiles {
|
||||
pub p50_ms: Option<PercentileMilliseconds>,
|
||||
pub p95_ms: Option<PercentileMilliseconds>,
|
||||
pub p99_ms: Option<PercentileMilliseconds>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AttemptMetrics {
|
||||
pub attempt_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub rate_limited_count: u64,
|
||||
pub auth_failure_count: u64,
|
||||
pub provider_5xx_count: u64,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ObservationGranularity {
|
||||
FifteenMinutes,
|
||||
Hour,
|
||||
Day,
|
||||
}
|
||||
|
||||
impl ObservationGranularity {
|
||||
#[must_use]
|
||||
pub const fn seconds(self) -> i64 {
|
||||
match self {
|
||||
Self::FifteenMinutes => 15 * 60,
|
||||
Self::Hour => 60 * 60,
|
||||
Self::Day => 24 * 60 * 60,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn sql_interval(self) -> &'static str {
|
||||
match self {
|
||||
Self::FifteenMinutes => "15 minutes",
|
||||
Self::Hour => "1 hour",
|
||||
Self::Day => "1 day",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestMetricPoint {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub granularity: ObservationGranularity,
|
||||
pub metrics: RequestMetrics,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
}
|
||||
|
||||
/// 已完整交付且由 Provider 计算费用的请求事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CalculatedUsageBillingFact {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub provider_kind: String,
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub total: CurrencyCostTotal,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProviderAccountMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountRequestBucket {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountUsageObservation {
|
||||
pub account_id: String,
|
||||
pub provider_kind: String,
|
||||
pub authentication_kind: String,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub enabled: bool,
|
||||
pub availability: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
pub image_request_failed_count: u64,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
pub request_buckets: Vec<ProviderAccountRequestBucket>,
|
||||
pub models: Vec<ProviderAccountModelUsageObservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountModelUsageObservation {
|
||||
pub model: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub image_request_count: u64,
|
||||
pub image_request_failed_count: u64,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
pub last_used_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountUsageQuery {
|
||||
pub range: ObservabilityRange,
|
||||
pub account_ids: Option<Vec<String>>,
|
||||
pub limit: u16,
|
||||
pub(crate) include_hourly_request_buckets: bool,
|
||||
}
|
||||
|
||||
impl ProviderAccountUsageQuery {
|
||||
pub fn for_accounts(range: ObservabilityRange, account_ids: Vec<String>) -> StoreResult<Self> {
|
||||
if account_ids.is_empty() || account_ids.len() > MAX_ACCOUNT_IDS {
|
||||
return Err(invalid(
|
||||
"account usage query requires between 1 and 200 IDs",
|
||||
));
|
||||
}
|
||||
validate_account_ids(&account_ids)?;
|
||||
Ok(Self {
|
||||
range,
|
||||
limit: u16::try_from(account_ids.len())
|
||||
.map_err(|_| invalid("account usage query is too large"))?,
|
||||
account_ids: Some(account_ids),
|
||||
include_hourly_request_buckets: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn recent(range: ObservabilityRange, limit: u16) -> StoreResult<Self> {
|
||||
if limit == 0 || usize::from(limit) > MAX_ACCOUNT_IDS {
|
||||
return Err(invalid("account usage limit must be between 1 and 200"));
|
||||
}
|
||||
Ok(Self {
|
||||
range,
|
||||
account_ids: None,
|
||||
limit,
|
||||
include_hourly_request_buckets: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn with_hourly_request_buckets(mut self) -> StoreResult<Self> {
|
||||
if self.range.end.signed_duration_since(self.range.start)
|
||||
> TimeDelta::hours(ACCOUNT_USAGE_TIMELINE_HOURS)
|
||||
{
|
||||
return Err(invalid("account request timeline cannot exceed 24 hours"));
|
||||
}
|
||||
self.include_hourly_request_buckets = true;
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardObservation {
|
||||
pub range: ObservabilityRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub provider_accounts: ProviderAccountMetrics,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub account_usage: Vec<ProviderAccountUsageObservation>,
|
||||
pub recent_requests: Vec<UsageRecord>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecord {
|
||||
pub id: String,
|
||||
pub client_api_key_ref: String,
|
||||
pub config_revision: u64,
|
||||
pub protocol: String,
|
||||
pub operation: String,
|
||||
pub endpoint: String,
|
||||
pub client_transport: String,
|
||||
pub requested_model_id: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub upstream_send_state: String,
|
||||
pub downstream_committed_at: Option<DateTime<Utc>>,
|
||||
pub outcome: String,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub upstream_response_id: Option<String>,
|
||||
pub error_kind: Option<String>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_source: String,
|
||||
pub cost_amount: Option<DecimalAmount>,
|
||||
pub cost_currency: Option<String>,
|
||||
pub transport_decision_wait_ms: Option<u64>,
|
||||
pub connect_ms: Option<u64>,
|
||||
pub headers_ms: Option<u64>,
|
||||
pub first_event_ms: Option<u64>,
|
||||
pub first_reasoning_ms: Option<u64>,
|
||||
pub first_text_ms: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub provider_processing_ms: Option<u64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub compact: bool,
|
||||
pub image_generation_requested: bool,
|
||||
pub image_generation_succeeded: Option<bool>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub deadline_at: DateTime<Utc>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecordPage {
|
||||
pub items: Vec<UsageRecord>,
|
||||
pub total: u64,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageAttemptObservation {
|
||||
pub source: String,
|
||||
pub id: String,
|
||||
pub attempt_index: u32,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_send_state: Option<String>,
|
||||
pub outcome: String,
|
||||
pub downstream_committed: bool,
|
||||
pub status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub failure_kind: Option<String>,
|
||||
pub retry_after_ms: Option<u64>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cost_source: Option<String>,
|
||||
pub cost_amount: Option<DecimalAmount>,
|
||||
pub cost_currency: Option<String>,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageRecordDetail {
|
||||
pub request: UsageRecord,
|
||||
pub attempts: Vec<UsageAttemptObservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderObservation {
|
||||
pub provider_kind: String,
|
||||
pub request_count: u64,
|
||||
pub attempt_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageOverview {
|
||||
pub range: ObservabilityRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub providers: Vec<ProviderObservation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DiagnosticObservation {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCostTotal>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpsErrorRecord {
|
||||
pub source: String,
|
||||
pub event_id: String,
|
||||
pub request_id: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
pub client_status_code: Option<u16>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub client_response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub occurrence_count: u32,
|
||||
pub occurred_at: DateTime<Utc>,
|
||||
pub stable_sort_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct OpsErrorPage {
|
||||
pub items: Vec<OpsErrorRecord>,
|
||||
pub total: u64,
|
||||
pub next_cursor: Option<ObservabilityCursor>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ObservabilityRepository: Send + Sync {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<DashboardObservation>;
|
||||
async fn dashboard_trend(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>>;
|
||||
async fn usage_trend(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>>;
|
||||
async fn usage_calculated_billing_facts(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<Vec<CalculatedUsageBillingFact>>;
|
||||
async fn provider_account_usage(
|
||||
&self,
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>>;
|
||||
async fn list_usage_records(&self, query: UsageRecordQuery) -> StoreResult<UsageRecordPage>;
|
||||
async fn usage_record_detail(&self, request_id: &str) -> StoreResult<UsageRecordDetail>;
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
) -> StoreResult<UsageOverview>;
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
range: ObservabilityRange,
|
||||
filter: UsageRecordFilter,
|
||||
dimension: DiagnosticDimension,
|
||||
) -> StoreResult<Vec<DiagnosticObservation>>;
|
||||
async fn list_ops_errors(&self, query: OpsErrorQuery) -> StoreResult<OpsErrorPage>;
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
//! 账号用量查询族。
|
||||
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) async fn provider_account_usage(
|
||||
pool: &PgPool,
|
||||
query: ProviderAccountUsageQuery,
|
||||
) -> StoreResult<Vec<ProviderAccountUsageObservation>> {
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
validate_account_ids(account_ids)?;
|
||||
}
|
||||
if query.limit == 0 || usize::from(query.limit) > MAX_ACCOUNT_IDS {
|
||||
return Err(invalid("account usage limit must be between 1 and 200"));
|
||||
}
|
||||
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, pa.enabled, pa.availability,
|
||||
count(mr.id)::bigint as request_count,
|
||||
count(mr.id) filter (where mr.outcome = 'succeeded')::bigint as success_count,
|
||||
sum(mr.input_tokens)::bigint as input_tokens,
|
||||
sum(mr.output_tokens)::bigint as output_tokens,
|
||||
sum(mr.cached_tokens)::bigint as cached_tokens,
|
||||
sum(mr.cache_write_tokens)::bigint as cache_write_tokens,
|
||||
sum(mr.reasoning_tokens)::bigint as reasoning_tokens,
|
||||
sum(mr.image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(mr.image_output_tokens)::bigint as image_output_tokens,
|
||||
count(mr.id) filter (where mr.image_generation_succeeded is true)::bigint
|
||||
as image_request_count,
|
||||
count(mr.id) filter (where mr.image_generation_succeeded is false)::bigint
|
||||
as image_request_failed_count,
|
||||
sum(mr.total_tokens)::bigint as total_tokens,
|
||||
count(mr.id) filter (where mr.cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(mr.id) filter (where mr.cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(mr.id) filter (where mr.cost_source = 'unavailable')::bigint
|
||||
as unavailable_count,
|
||||
max(mr.started_at) as last_used_at
|
||||
from provider_accounts pa
|
||||
left join model_requests mr
|
||||
on mr.provider_account_ref = pa.id and mr.started_at >= ",
|
||||
);
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push(" and mr.started_at < ");
|
||||
statement.push_bind(query.range.end);
|
||||
push_completed_usage_fact_filter(&mut statement, "mr");
|
||||
if let Some(account_ids) = &query.account_ids {
|
||||
statement.push(" where pa.id = any(");
|
||||
statement.push_bind(account_ids.clone());
|
||||
statement.push("::text[])");
|
||||
}
|
||||
statement.push(
|
||||
" group by pa.id, pa.provider_kind, pa.authentication_kind, pa.name, pa.email,
|
||||
pa.plan_type, pa.enabled, pa.availability
|
||||
order by max(mr.started_at) desc nulls last, pa.name, pa.id limit ",
|
||||
);
|
||||
statement.push_bind(i64::from(query.limit));
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account usage"))?;
|
||||
|
||||
let mut observations = rows
|
||||
.iter()
|
||||
.map(provider_account_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
if observations.is_empty() {
|
||||
return Ok(observations);
|
||||
}
|
||||
let account_ids = observations
|
||||
.iter()
|
||||
.map(|item| item.account_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let mut request_buckets = if query.include_hourly_request_buckets {
|
||||
provider_account_request_buckets(pool, query.range, &account_ids).await?
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let account_costs = account_costs(pool, query.range, &account_ids).await?;
|
||||
let model_rows = account_model_rows(pool, query.range, &account_ids).await?;
|
||||
let model_costs = account_model_costs(pool, query.range, &account_ids).await?;
|
||||
let account_positions = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, item)| (item.account_id.clone(), index))
|
||||
.collect::<HashMap<_, _>>();
|
||||
for observation in &mut observations {
|
||||
observation.costs = account_costs
|
||||
.get(&observation.account_id)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
observation.request_buckets = request_buckets
|
||||
.remove(&observation.account_id)
|
||||
.unwrap_or_default();
|
||||
}
|
||||
for row in model_rows {
|
||||
let account_id: String = get(&row, "provider_account_ref")?;
|
||||
let Some(position) = account_positions.get(&account_id).copied() else {
|
||||
continue;
|
||||
};
|
||||
let model: String = get(&row, "model")?;
|
||||
observations[position]
|
||||
.models
|
||||
.push(ProviderAccountModelUsageObservation {
|
||||
costs: model_costs
|
||||
.get(&(account_id, model.clone()))
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
model,
|
||||
request_count: unsigned(&row, "request_count")?,
|
||||
success_count: unsigned(&row, "success_count")?,
|
||||
input_tokens: optional_unsigned(&row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(&row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(&row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(&row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(&row, "reasoning_tokens")?,
|
||||
image_input_tokens: optional_unsigned(&row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(&row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(&row, "image_request_count")?,
|
||||
image_request_failed_count: unsigned(&row, "image_request_failed_count")?,
|
||||
total_tokens: optional_unsigned(&row, "total_tokens")?,
|
||||
cost_coverage: coverage_from_row(&row)?,
|
||||
last_used_at: get(&row, "last_used_at")?,
|
||||
});
|
||||
}
|
||||
Ok(observations)
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_account_request_buckets(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, Vec<ProviderAccountRequestBucket>>> {
|
||||
let rows = sqlx::query(
|
||||
"select provider_account_ref,
|
||||
floor(extract(epoch from (started_at - $1)) / 3600)::bigint as bucket_index,
|
||||
count(*)::bigint as request_count
|
||||
from model_requests mr
|
||||
where mr.provider_account_ref = any($2::text[])
|
||||
and mr.started_at >= $1 and mr.started_at < $3
|
||||
and mr.outcome = 'succeeded'
|
||||
and mr.downstream_committed_at is not null
|
||||
and mr.client_status_code between 200 and 399
|
||||
group by mr.provider_account_ref, bucket_index
|
||||
order by mr.provider_account_ref, bucket_index",
|
||||
)
|
||||
.bind(range.start)
|
||||
.bind(account_ids)
|
||||
.bind(range.end)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account request timeline"))?;
|
||||
|
||||
let mut observed = HashMap::<String, BTreeMap<u64, u64>>::new();
|
||||
for row in rows {
|
||||
observed
|
||||
.entry(get(&row, "provider_account_ref")?)
|
||||
.or_default()
|
||||
.insert(
|
||||
unsigned(&row, "bucket_index")?,
|
||||
unsigned(&row, "request_count")?,
|
||||
);
|
||||
}
|
||||
|
||||
let step = TimeDelta::hours(1);
|
||||
let mut timelines = HashMap::with_capacity(account_ids.len());
|
||||
for account_id in account_ids {
|
||||
let mut account_observed = observed.remove(account_id).unwrap_or_default();
|
||||
let mut bucket_start = range.start;
|
||||
let mut bucket_index = 0_u64;
|
||||
let mut buckets = Vec::new();
|
||||
while bucket_start < range.end {
|
||||
buckets.push(ProviderAccountRequestBucket {
|
||||
bucket_start,
|
||||
request_count: account_observed.remove(&bucket_index).unwrap_or_default(),
|
||||
});
|
||||
bucket_start = bucket_start
|
||||
.checked_add_signed(step)
|
||||
.ok_or_else(|| invalid("account request timeline exceeds supported timestamps"))?;
|
||||
bucket_index = bucket_index
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| invalid("account request timeline exceeds supported buckets"))?;
|
||||
}
|
||||
timelines.insert(account_id.clone(), buckets);
|
||||
}
|
||||
Ok(timelines)
|
||||
}
|
||||
|
||||
pub(crate) async fn account_costs(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<String, Vec<CurrencyCostTotal>>> {
|
||||
let rows = sqlx::query(
|
||||
"select mr.provider_account_ref, mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_requests mr
|
||||
where mr.provider_account_ref = any($1::text[])
|
||||
and mr.started_at >= $2 and mr.started_at < $3
|
||||
and mr.outcome = 'succeeded'
|
||||
and mr.downstream_committed_at is not null
|
||||
and mr.client_status_code between 200 and 399
|
||||
and mr.cost_amount is not null and mr.cost_currency is not null
|
||||
group by mr.provider_account_ref, mr.cost_currency
|
||||
order by mr.provider_account_ref, mr.cost_currency",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account costs"))?;
|
||||
let mut costs = HashMap::<String, Vec<CurrencyCostTotal>>::new();
|
||||
for row in rows {
|
||||
costs
|
||||
.entry(get(&row, "provider_account_ref")?)
|
||||
.or_default()
|
||||
.push(cost_from_row(&row)?);
|
||||
}
|
||||
Ok(costs)
|
||||
}
|
||||
|
||||
pub(crate) async fn account_model_rows(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<Vec<sqlx::postgres::PgRow>> {
|
||||
sqlx::query(
|
||||
"select mr.provider_account_ref,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
count(*)::bigint as request_count,
|
||||
count(*) filter (where mr.outcome = 'succeeded')::bigint as success_count,
|
||||
sum(mr.input_tokens)::bigint as input_tokens,
|
||||
sum(mr.output_tokens)::bigint as output_tokens,
|
||||
sum(mr.cached_tokens)::bigint as cached_tokens,
|
||||
sum(mr.cache_write_tokens)::bigint as cache_write_tokens,
|
||||
sum(mr.reasoning_tokens)::bigint as reasoning_tokens,
|
||||
sum(mr.image_input_tokens)::bigint as image_input_tokens,
|
||||
sum(mr.image_output_tokens)::bigint as image_output_tokens,
|
||||
count(*) filter (where mr.image_generation_succeeded is true)::bigint
|
||||
as image_request_count,
|
||||
count(*) filter (where mr.image_generation_succeeded is false)::bigint
|
||||
as image_request_failed_count,
|
||||
sum(mr.total_tokens)::bigint as total_tokens,
|
||||
count(*) filter (where mr.cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (where mr.cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(*) filter (where mr.cost_source = 'unavailable')::bigint
|
||||
as unavailable_count,
|
||||
max(mr.started_at) as last_used_at
|
||||
from model_requests mr
|
||||
where mr.provider_account_ref = any($1::text[])
|
||||
and mr.started_at >= $2 and mr.started_at < $3
|
||||
and mr.outcome = 'succeeded'
|
||||
and mr.downstream_committed_at is not null
|
||||
and mr.client_status_code between 200 and 399
|
||||
group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id)
|
||||
order by mr.provider_account_ref, request_count desc, model",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account model usage"))
|
||||
}
|
||||
|
||||
pub(crate) async fn account_model_costs(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
account_ids: &[String],
|
||||
) -> StoreResult<HashMap<(String, String), Vec<CurrencyCostTotal>>> {
|
||||
let rows = sqlx::query(
|
||||
"select mr.provider_account_ref,
|
||||
coalesce(mr.upstream_model_id, mr.requested_model_id) as model,
|
||||
mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_requests mr
|
||||
where mr.provider_account_ref = any($1::text[])
|
||||
and mr.started_at >= $2 and mr.started_at < $3
|
||||
and mr.outcome = 'succeeded'
|
||||
and mr.downstream_committed_at is not null
|
||||
and mr.client_status_code between 200 and 399
|
||||
and mr.cost_amount is not null and mr.cost_currency is not null
|
||||
group by mr.provider_account_ref, coalesce(mr.upstream_model_id, mr.requested_model_id),
|
||||
mr.cost_currency
|
||||
order by mr.provider_account_ref, model, mr.cost_currency",
|
||||
)
|
||||
.bind(account_ids)
|
||||
.bind(range.start)
|
||||
.bind(range.end)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account model costs"))?;
|
||||
let mut costs = HashMap::<(String, String), Vec<CurrencyCostTotal>>::new();
|
||||
for row in rows {
|
||||
costs
|
||||
.entry((get(&row, "provider_account_ref")?, get(&row, "model")?))
|
||||
.or_default()
|
||||
.push(cost_from_row(&row)?);
|
||||
}
|
||||
Ok(costs)
|
||||
}
|
||||
|
||||
pub(crate) fn provider_account_from_row(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
) -> StoreResult<ProviderAccountUsageObservation> {
|
||||
Ok(ProviderAccountUsageObservation {
|
||||
account_id: get(row, "id")?,
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
authentication_kind: get(row, "authentication_kind")?,
|
||||
name: get(row, "name")?,
|
||||
email: get(row, "email")?,
|
||||
plan_type: get(row, "plan_type")?,
|
||||
enabled: get(row, "enabled")?,
|
||||
availability: get(row, "availability")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
input_tokens: optional_unsigned(row, "input_tokens")?,
|
||||
output_tokens: optional_unsigned(row, "output_tokens")?,
|
||||
cached_tokens: optional_unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: optional_unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: optional_unsigned(row, "reasoning_tokens")?,
|
||||
image_input_tokens: optional_unsigned(row, "image_input_tokens")?,
|
||||
image_output_tokens: optional_unsigned(row, "image_output_tokens")?,
|
||||
image_request_count: unsigned(row, "image_request_count")?,
|
||||
image_request_failed_count: unsigned(row, "image_request_failed_count")?,
|
||||
total_tokens: optional_unsigned(row, "total_tokens")?,
|
||||
cost_coverage: coverage_from_row(row)?,
|
||||
costs: Vec::new(),
|
||||
last_used_at: get(row, "last_used_at")?,
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,582 @@
|
||||
//! Dashboard 与趋势查询族。
|
||||
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) async fn request_metrics(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<RequestMetrics> {
|
||||
filter.validate()?;
|
||||
// 结果计数覆盖范围内全部请求(成功率/失败率的分母分子);用量、缓存、
|
||||
// 延迟与成本聚合仅统计用量事实(已完整交付客户端的成功响应)。
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new(format!(
|
||||
"select count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p99_ms
|
||||
from model_requests mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load request metrics"))?;
|
||||
request_metrics_from_row(&row)
|
||||
}
|
||||
|
||||
pub(crate) async fn request_metric_series(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
filter.validate()?;
|
||||
let granularity = granularity_for(range);
|
||||
// 与 request_metrics 同一契约:结果计数覆盖全部请求,用量/延迟/成本
|
||||
// 聚合仅统计用量事实。
|
||||
let fact = completed_usage_fact_predicate("mr");
|
||||
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
|
||||
query.push_bind(granularity.sql_interval());
|
||||
query.push(format!(
|
||||
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
|
||||
count(*)::bigint as request_count,
|
||||
count(*) filter (where outcome = 'succeeded')::bigint as success_count,
|
||||
count(*) filter (where outcome = 'failed')::bigint as failure_count,
|
||||
count(*) filter (where outcome = 'cancelled')::bigint as cancelled_count,
|
||||
count(*) filter (where outcome = 'incomplete')::bigint as incomplete_count,
|
||||
count(*) filter (where client_status_code between 400 and 499)::bigint
|
||||
as caller_error_count,
|
||||
coalesce(sum(input_tokens) filter (where {fact}), 0)::bigint as input_tokens,
|
||||
coalesce(sum(output_tokens) filter (where {fact}), 0)::bigint as output_tokens,
|
||||
coalesce(sum(cached_tokens) filter (where {fact}), 0)::bigint as cached_tokens,
|
||||
coalesce(sum(cache_write_tokens) filter (where {fact}), 0)::bigint
|
||||
as cache_write_tokens,
|
||||
coalesce(sum(reasoning_tokens) filter (where {fact}), 0)::bigint
|
||||
as reasoning_tokens,
|
||||
coalesce(sum(total_tokens) filter (where {fact}), 0)::bigint as total_tokens,
|
||||
coalesce(sum(first_token_ms) filter (where {fact}), 0)::bigint
|
||||
as first_token_latency_sum,
|
||||
count(first_token_ms) filter (where {fact})::bigint as first_token_latency_count,
|
||||
coalesce(sum(latency_ms) filter (where {fact}), 0)::bigint as latency_sum,
|
||||
count(latency_ms) filter (where {fact})::bigint as latency_count,
|
||||
max(latency_ms) filter (where {fact})::bigint as max_latency_ms,
|
||||
min(latency_ms) filter (where {fact})::bigint as min_latency_ms,
|
||||
count(*) filter (where {fact} and input_tokens is not null)::bigint
|
||||
as cache_eligible_request_count,
|
||||
count(*) filter (where {fact} and input_tokens is not null
|
||||
and cached_tokens > 0)::bigint
|
||||
as cache_hit_request_count,
|
||||
percentile_cont(0.50) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p50_ms,
|
||||
percentile_cont(0.95) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p95_ms,
|
||||
percentile_cont(0.99) within group (order by latency_ms)
|
||||
filter (where {fact}) as latency_p99_ms,
|
||||
percentile_cont(0.50) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p50_ms,
|
||||
percentile_cont(0.95) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p95_ms,
|
||||
percentile_cont(0.99) within group (order by first_token_ms)
|
||||
filter (where {fact}) as first_token_p99_ms,
|
||||
count(*) filter (where {fact} and cost_source = 'provider_reported')::bigint
|
||||
as provider_reported_count,
|
||||
count(*) filter (where {fact} and cost_source = 'calculated')::bigint
|
||||
as calculated_count,
|
||||
count(*) filter (where {fact} and cost_source = 'unavailable')::bigint
|
||||
as unavailable_count
|
||||
from model_requests mr where mr.started_at >= "
|
||||
));
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start order by bucket_start");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load request metric series"))?;
|
||||
|
||||
let mut points = BTreeMap::new();
|
||||
for row in rows {
|
||||
let bucket_start = get(&row, "bucket_start")?;
|
||||
points.insert(
|
||||
bucket_start,
|
||||
RequestMetricPoint {
|
||||
bucket_start,
|
||||
granularity,
|
||||
metrics: request_metrics_from_row(&row)?,
|
||||
cost_coverage: coverage_from_row(&row)?,
|
||||
costs: Vec::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
let bucket_costs = request_costs_by_bucket(pool, range, filter, granularity).await?;
|
||||
for (bucket, costs) in bucket_costs {
|
||||
if let Some(point) = points.get_mut(&bucket) {
|
||||
point.costs = costs;
|
||||
}
|
||||
}
|
||||
fill_metric_gaps(range, granularity, points)
|
||||
}
|
||||
|
||||
pub(crate) async fn calculated_usage_billing_facts(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<CalculatedUsageBillingFact>> {
|
||||
filter.validate()?;
|
||||
let granularity = granularity_for(range);
|
||||
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
|
||||
query.push_bind(granularity.sql_interval());
|
||||
query.push(
|
||||
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
|
||||
mr.provider_kind, mr.upstream_model_id, mr.service_tier,
|
||||
mr.input_tokens, mr.output_tokens, mr.cached_tokens, mr.cache_write_tokens,
|
||||
mr.cost_currency, mr.cost_amount::text as amount
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(
|
||||
" and mr.cost_source = 'calculated'
|
||||
and mr.cost_amount is not null and mr.cost_currency is not null
|
||||
and nullif(mr.provider_kind, '') is not null
|
||||
and nullif(mr.upstream_model_id, '') is not null",
|
||||
);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" order by bucket_start, mr.id");
|
||||
query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load calculated usage billing facts"))?
|
||||
.iter()
|
||||
.map(calculated_usage_billing_fact_from_row)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn request_costs_by_bucket(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
granularity: ObservationGranularity,
|
||||
) -> StoreResult<BTreeMap<DateTime<Utc>, Vec<CurrencyCostTotal>>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new("select date_bin(");
|
||||
query.push_bind(granularity.sql_interval());
|
||||
query.push(
|
||||
"::interval, mr.started_at, timestamptz '1970-01-01 00:00:00+00') as bucket_start,
|
||||
mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_requests mr
|
||||
where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by bucket_start, mr.cost_currency order by bucket_start, mr.cost_currency");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load request costs by bucket"))?;
|
||||
let mut result = BTreeMap::<DateTime<Utc>, Vec<CurrencyCostTotal>>::new();
|
||||
for row in rows {
|
||||
result
|
||||
.entry(get(&row, "bucket_start")?)
|
||||
.or_default()
|
||||
.push(cost_from_row(&row)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub(crate) async fn attempt_metrics(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<AttemptMetrics> {
|
||||
filter.validate()?;
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"with selected_requests as (
|
||||
select mr.* from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(
|
||||
"), usage_facts as (
|
||||
select * from selected_requests
|
||||
where outcome = 'succeeded'
|
||||
and downstream_committed_at is not null
|
||||
and client_status_code between 200 and 399
|
||||
), failures as (
|
||||
select coalesce(oe.occurrence_count, 1)::bigint as occurrences,
|
||||
oe.failure_kind, oe.status_code
|
||||
from ops_events oe
|
||||
join selected_requests sr on sr.id = oe.model_request_id
|
||||
union all
|
||||
select 1::bigint, coalesce(sr.error_kind, 'failed'),
|
||||
coalesce(sr.upstream_status_code, sr.client_status_code)
|
||||
from selected_requests sr
|
||||
where sr.outcome = 'failed' and sr.attempt_count > 0
|
||||
)
|
||||
select coalesce((select sum(attempt_count) from selected_requests), 0)::bigint
|
||||
as attempt_count,
|
||||
coalesce((select count(*) from selected_requests
|
||||
where outcome = 'succeeded' and attempt_count > 0), 0)::bigint
|
||||
as success_count,
|
||||
coalesce((select sum(occurrences) from failures), 0)::bigint as failure_count,
|
||||
coalesce((select count(*) from selected_requests
|
||||
where outcome = 'cancelled' and attempt_count > 0), 0)::bigint
|
||||
as cancelled_count,
|
||||
coalesce((select count(*) from selected_requests
|
||||
where outcome = 'incomplete' and attempt_count > 0), 0)::bigint
|
||||
as incomplete_count,
|
||||
coalesce((select sum(occurrences) from failures
|
||||
where failure_kind in ('rate_limited', 'quota_exhausted')
|
||||
or status_code = 429), 0)::bigint as rate_limited_count,
|
||||
coalesce((select sum(occurrences) from failures
|
||||
where failure_kind in ('authentication', 'authorization',
|
||||
'invalid_credential')
|
||||
or status_code in (401, 403)), 0)::bigint as auth_failure_count,
|
||||
coalesce((select sum(occurrences) from failures
|
||||
where status_code between 500 and 599), 0)::bigint
|
||||
as provider_5xx_count,
|
||||
coalesce((select count(*) from usage_facts
|
||||
where cost_source = 'provider_reported'), 0)::bigint
|
||||
as provider_reported_count,
|
||||
coalesce((select count(*) from usage_facts
|
||||
where cost_source = 'calculated'), 0)::bigint
|
||||
as calculated_count,
|
||||
coalesce((select count(*) from usage_facts
|
||||
where cost_source = 'unavailable'), 0)::bigint
|
||||
as unavailable_count",
|
||||
);
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load attempt metrics"))?;
|
||||
Ok(AttemptMetrics {
|
||||
attempt_count: unsigned(&row, "attempt_count")?,
|
||||
success_count: unsigned(&row, "success_count")?,
|
||||
failure_count: unsigned(&row, "failure_count")?,
|
||||
cancelled_count: unsigned(&row, "cancelled_count")?,
|
||||
incomplete_count: unsigned(&row, "incomplete_count")?,
|
||||
rate_limited_count: unsigned(&row, "rate_limited_count")?,
|
||||
auth_failure_count: unsigned(&row, "auth_failure_count")?,
|
||||
provider_5xx_count: unsigned(&row, "provider_5xx_count")?,
|
||||
cost_coverage: coverage_from_row(&row)?,
|
||||
costs: request_costs(pool, range, filter).await?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn request_costs(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<CurrencyCostTotal>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select mr.cost_currency, sum(mr.cost_amount)::text as amount
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
query.push(" and mr.cost_amount is not null and mr.cost_currency is not null");
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by mr.cost_currency order by mr.cost_currency");
|
||||
query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load request costs"))?
|
||||
.iter()
|
||||
.map(cost_from_row)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_account_metrics(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<ProviderAccountMetrics> {
|
||||
let row = sqlx::query(
|
||||
"with normalized as (
|
||||
select enabled, availability, quota_limit_reached,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability = 'invalid' then 'invalid'
|
||||
when availability = 'expired'
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
when availability = 'unknown' then 'invalid'
|
||||
when quota_limit_reached then 'quota_exhausted'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select count(*)::bigint as total,
|
||||
count(*) filter (where enabled)::bigint as enabled,
|
||||
count(*) filter (where status in ('rate_limited','quota_exhausted','expired','invalid','disabled','banned'))::bigint as unavailable,
|
||||
count(*) filter (where status = 'active')::bigint as active,
|
||||
count(*) filter (where status = 'rate_limited')::bigint as rate_limited,
|
||||
count(*) filter (where status = 'expired')::bigint as expired,
|
||||
count(*) filter (where status = 'invalid')::bigint as invalid,
|
||||
count(*) filter (where status = 'quota_exhausted')::bigint as quota_exhausted,
|
||||
count(*) filter (where status = 'disabled')::bigint as disabled,
|
||||
count(*) filter (where status = 'banned')::bigint as banned
|
||||
from normalized",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider account metrics"))?;
|
||||
Ok(ProviderAccountMetrics {
|
||||
total: unsigned(&row, "total")?,
|
||||
enabled: unsigned(&row, "enabled")?,
|
||||
unavailable: unsigned(&row, "unavailable")?,
|
||||
active: unsigned(&row, "active")?,
|
||||
rate_limited: unsigned(&row, "rate_limited")?,
|
||||
expired: unsigned(&row, "expired")?,
|
||||
invalid: unsigned(&row, "invalid")?,
|
||||
quota_exhausted: unsigned(&row, "quota_exhausted")?,
|
||||
disabled: unsigned(&row, "disabled")?,
|
||||
banned: unsigned(&row, "banned")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn active_provider_account_ids(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<String>> {
|
||||
sqlx::query(
|
||||
"with normalized as (
|
||||
select id,
|
||||
case
|
||||
when not enabled then 'disabled'
|
||||
when availability = 'banned' then 'banned'
|
||||
when availability = 'quota_exhausted' then 'quota_exhausted'
|
||||
when availability in ('expired', 'invalid')
|
||||
or (access_token_expires_at is not null and access_token_expires_at <= $1)
|
||||
then 'expired'
|
||||
else 'active'
|
||||
end as status
|
||||
from provider_accounts
|
||||
)
|
||||
select id
|
||||
from normalized
|
||||
where status = 'active'
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load active provider account IDs"))?
|
||||
.iter()
|
||||
.map(|row| get(row, "id"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 查询可被 Redis 冷却重新归类的 ready 账号:id、credential revision 与 quota 标记。
|
||||
/// 已过期或非 ready 的账号由 [`provider_account_metrics`] 直接计数。
|
||||
pub(crate) async fn schedulable_metric_candidates(
|
||||
pool: &PgPool,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> StoreResult<Vec<(String, i64, bool)>> {
|
||||
sqlx::query(
|
||||
"select id, credential_revision, quota_limit_reached
|
||||
from provider_accounts
|
||||
where enabled
|
||||
and availability = 'ready'
|
||||
and (access_token_expires_at is null or access_token_expires_at > $1)
|
||||
order by id",
|
||||
)
|
||||
.bind(observed_at)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load schedulable metric candidates"))?
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
get(&row, "id")?,
|
||||
get(&row, "credential_revision")?,
|
||||
get(&row, "quota_limit_reached")?,
|
||||
))
|
||||
})
|
||||
.collect::<StoreResult<Vec<_>>>()
|
||||
}
|
||||
|
||||
pub(crate) async fn provider_observations(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &UsageRecordFilter,
|
||||
) -> StoreResult<Vec<ProviderObservation>> {
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"select coalesce(mr.provider_kind, 'unrouted') as provider_kind,
|
||||
count(*)::bigint as request_count,
|
||||
coalesce(sum(mr.attempt_count), 0)::bigint as attempt_count,
|
||||
count(*) filter (where mr.outcome = 'failed')::bigint as failure_count,
|
||||
coalesce(sum(mr.total_tokens), 0)::bigint as total_tokens
|
||||
from model_requests mr where mr.started_at >= ",
|
||||
);
|
||||
query.push_bind(range.start);
|
||||
query.push(" and mr.started_at < ");
|
||||
query.push_bind(range.end);
|
||||
push_completed_usage_fact_filter(&mut query, "mr");
|
||||
push_usage_filter(&mut query, filter, "mr");
|
||||
query.push(" group by coalesce(mr.provider_kind, 'unrouted') order by request_count desc");
|
||||
let rows = query
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("load provider observations"))?;
|
||||
rows.iter()
|
||||
.map(|row| {
|
||||
Ok(ProviderObservation {
|
||||
provider_kind: get(row, "provider_kind")?,
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
attempt_count: unsigned(row, "attempt_count")?,
|
||||
failure_count: unsigned(row, "failure_count")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn request_metrics_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<RequestMetrics> {
|
||||
Ok(RequestMetrics {
|
||||
request_count: unsigned(row, "request_count")?,
|
||||
success_count: unsigned(row, "success_count")?,
|
||||
failure_count: unsigned(row, "failure_count")?,
|
||||
cancelled_count: unsigned(row, "cancelled_count")?,
|
||||
incomplete_count: unsigned(row, "incomplete_count")?,
|
||||
caller_error_count: unsigned(row, "caller_error_count")?,
|
||||
input_tokens: unsigned(row, "input_tokens")?,
|
||||
output_tokens: unsigned(row, "output_tokens")?,
|
||||
cached_tokens: unsigned(row, "cached_tokens")?,
|
||||
cache_write_tokens: unsigned(row, "cache_write_tokens")?,
|
||||
reasoning_tokens: unsigned(row, "reasoning_tokens")?,
|
||||
total_tokens: unsigned(row, "total_tokens")?,
|
||||
first_token_latency_sum: unsigned(row, "first_token_latency_sum")?,
|
||||
first_token_latency_count: unsigned(row, "first_token_latency_count")?,
|
||||
latency_sum: unsigned(row, "latency_sum")?,
|
||||
latency_count: unsigned(row, "latency_count")?,
|
||||
max_latency_ms: optional_unsigned(row, "max_latency_ms")?,
|
||||
min_latency_ms: optional_unsigned(row, "min_latency_ms")?,
|
||||
cache_eligible_request_count: unsigned(row, "cache_eligible_request_count")?,
|
||||
cache_hit_request_count: unsigned(row, "cache_hit_request_count")?,
|
||||
latency_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "latency_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "latency_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "latency_p99_ms")?,
|
||||
},
|
||||
first_token_latency_percentiles: LatencyPercentiles {
|
||||
p50_ms: optional_percentile(row, "first_token_p50_ms")?,
|
||||
p95_ms: optional_percentile(row, "first_token_p95_ms")?,
|
||||
p99_ms: optional_percentile(row, "first_token_p99_ms")?,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn optional_percentile(
|
||||
row: &sqlx::postgres::PgRow,
|
||||
column: &str,
|
||||
) -> StoreResult<Option<PercentileMilliseconds>> {
|
||||
row.try_get::<Option<f64>, _>(column)
|
||||
.map_err(|_| postgres_unavailable("decode latency percentile"))?
|
||||
.map(PercentileMilliseconds::new)
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn coverage_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<CostCoverage> {
|
||||
Ok(CostCoverage {
|
||||
provider_reported_count: unsigned(row, "provider_reported_count")?,
|
||||
calculated_count: unsigned(row, "calculated_count")?,
|
||||
unavailable_count: unsigned(row, "unavailable_count")?,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn granularity_for(range: ObservabilityRange) -> ObservationGranularity {
|
||||
let seconds = range.end.signed_duration_since(range.start).num_seconds();
|
||||
if seconds <= 2 * 24 * 60 * 60 {
|
||||
ObservationGranularity::FifteenMinutes
|
||||
} else if seconds <= 31 * 24 * 60 * 60 {
|
||||
ObservationGranularity::Hour
|
||||
} else {
|
||||
ObservationGranularity::Day
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn fill_metric_gaps(
|
||||
range: ObservabilityRange,
|
||||
granularity: ObservationGranularity,
|
||||
mut points: BTreeMap<DateTime<Utc>, RequestMetricPoint>,
|
||||
) -> StoreResult<Vec<RequestMetricPoint>> {
|
||||
let seconds = granularity.seconds();
|
||||
let start_epoch = range.start.timestamp().div_euclid(seconds) * seconds;
|
||||
let mut bucket = DateTime::from_timestamp(start_epoch, 0)
|
||||
.ok_or_else(|| invalid("metric range start is outside supported timestamps"))?;
|
||||
let step = TimeDelta::seconds(seconds);
|
||||
let mut result = Vec::new();
|
||||
while bucket < range.end {
|
||||
result.push(points.remove(&bucket).unwrap_or(RequestMetricPoint {
|
||||
bucket_start: bucket,
|
||||
granularity,
|
||||
metrics: RequestMetrics::default(),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
}));
|
||||
bucket = bucket
|
||||
.checked_add_signed(step)
|
||||
.ok_or_else(|| invalid("metric range exceeds supported timestamps"))?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! 观测查询族:dashboard / usage / ops / accounts。
|
||||
|
||||
mod accounts;
|
||||
mod dashboard;
|
||||
mod ops;
|
||||
mod usage;
|
||||
|
||||
pub(crate) use accounts::*;
|
||||
pub(crate) use dashboard::*;
|
||||
pub(crate) use ops::*;
|
||||
pub(crate) use usage::*;
|
||||
@@ -0,0 +1,171 @@
|
||||
//! Ops 错误查询族。
|
||||
|
||||
use super::super::*;
|
||||
|
||||
pub(crate) const OPS_ERRORS_CTE: &str = "with errors as (
|
||||
select 'model_request'::text as source,
|
||||
mr.id as event_id, mr.id as request_id,
|
||||
nullif(mr.attempt_count, 0) as attempt_index,
|
||||
mr.client_api_key_ref, 'model_request'::text as component, mr.operation,
|
||||
mr.endpoint,
|
||||
mr.provider_kind,
|
||||
mr.provider_account_ref,
|
||||
mr.provider_account_name_snapshot as provider_account_name,
|
||||
mr.provider_account_email_snapshot as provider_account_email,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport,
|
||||
coalesce(mr.error_kind, 'failed') as failure_kind,
|
||||
mr.client_status_code, mr.upstream_status_code,
|
||||
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
|
||||
mr.latency_ms, coalesce(mr.error_message, mr.error_kind, 'request failed') as message,
|
||||
1::integer as occurrence_count,
|
||||
coalesce(mr.completed_at, mr.started_at) as occurred_at,
|
||||
'model_request:' || mr.id as stable_sort_id
|
||||
from model_requests mr where mr.outcome = 'failed'
|
||||
union all
|
||||
select 'ops_event'::text, oe.id, oe.model_request_id, oe.attempt_index,
|
||||
mr.client_api_key_ref, oe.component, oe.operation,
|
||||
null::text as endpoint,
|
||||
oe.provider_kind, oe.provider_account_ref,
|
||||
oe.provider_account_name_snapshot as provider_account_name,
|
||||
oe.provider_account_email_snapshot as provider_account_email,
|
||||
oe.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
oe.upstream_model_id, null::text, oe.failure_kind,
|
||||
null::integer as client_status_code, oe.status_code as upstream_status_code,
|
||||
oe.provider_error_code, mr.client_response_id, oe.upstream_request_id,
|
||||
oe.latency_ms, oe.message, oe.occurrence_count, oe.created_at,
|
||||
'ops_event:' || oe.id
|
||||
from ops_events oe left join model_requests mr on mr.id = oe.model_request_id
|
||||
)";
|
||||
|
||||
pub(crate) async fn list_ops_errors(
|
||||
pool: &PgPool,
|
||||
query: OpsErrorQuery,
|
||||
) -> StoreResult<OpsErrorPage> {
|
||||
query.filter.validate()?;
|
||||
let total = count_ops_errors(pool, query.range, &query.filter).await?;
|
||||
let mut statement = QueryBuilder::<Postgres>::new(OPS_ERRORS_CTE);
|
||||
statement.push(
|
||||
" select source, event_id, request_id, attempt_index, client_api_key_ref,
|
||||
component, operation, endpoint, provider_kind, provider_account_ref,
|
||||
provider_account_name, provider_account_email,
|
||||
provider_account_authentication_kind,
|
||||
upstream_model_id, upstream_transport,
|
||||
failure_kind, client_status_code, upstream_status_code,
|
||||
provider_error_code, client_response_id,
|
||||
upstream_request_id, latency_ms, message, occurrence_count, occurred_at,
|
||||
stable_sort_id
|
||||
from errors e where e.occurred_at >= ",
|
||||
);
|
||||
statement.push_bind(query.range.start);
|
||||
statement.push(" and e.occurred_at < ");
|
||||
statement.push_bind(query.range.end);
|
||||
push_ops_filter(&mut statement, &query.filter);
|
||||
if let Some(cursor) = &query.cursor {
|
||||
statement.push(" and (e.occurred_at, e.stable_sort_id) < (");
|
||||
statement.push_bind(cursor.observed_at);
|
||||
statement.push(", ");
|
||||
statement.push_bind(cursor.stable_id.clone());
|
||||
statement.push(")");
|
||||
}
|
||||
statement.push(" order by e.occurred_at desc, e.stable_sort_id desc limit ");
|
||||
statement.push_bind(i64::from(query.page_size.get()) + 1);
|
||||
if query.cursor.is_none() {
|
||||
let offset = u64::from(query.page.get() - 1)
|
||||
.checked_mul(u64::from(query.page_size.get()))
|
||||
.and_then(|value| i64::try_from(value).ok())
|
||||
.ok_or_else(|| invalid("ops error page offset is too large"))?;
|
||||
statement.push(" offset ");
|
||||
statement.push_bind(offset);
|
||||
}
|
||||
let rows = statement
|
||||
.build()
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("list ops errors"))?;
|
||||
let mut items = rows
|
||||
.iter()
|
||||
.map(ops_error_from_row)
|
||||
.collect::<StoreResult<Vec<_>>>()?;
|
||||
let has_more = items.len() > usize::from(query.page_size.get());
|
||||
if has_more {
|
||||
items.pop();
|
||||
}
|
||||
let next_cursor = if has_more {
|
||||
items
|
||||
.last()
|
||||
.map(|item| ObservabilityCursor::new(item.occurred_at, item.stable_sort_id.clone()))
|
||||
.transpose()?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(OpsErrorPage {
|
||||
items,
|
||||
total,
|
||||
next_cursor,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn count_ops_errors(
|
||||
pool: &PgPool,
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) -> StoreResult<u64> {
|
||||
let mut statement = QueryBuilder::<Postgres>::new(OPS_ERRORS_CTE);
|
||||
statement.push(" select count(*)::bigint from errors e where e.occurred_at >= ");
|
||||
statement.push_bind(range.start);
|
||||
statement.push(" and e.occurred_at < ");
|
||||
statement.push_bind(range.end);
|
||||
push_ops_filter(&mut statement, filter);
|
||||
let total = statement
|
||||
.build_query_scalar::<i64>()
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|_| postgres_unavailable("count ops errors"))?;
|
||||
to_u64(total)
|
||||
}
|
||||
|
||||
pub(crate) fn push_ops_filter(statement: &mut QueryBuilder<Postgres>, filter: &OpsErrorFilter) {
|
||||
for (column, value) in [
|
||||
("client_api_key_ref", &filter.client_api_key_ref),
|
||||
("request_id", &filter.request_id),
|
||||
("provider_account_ref", &filter.provider_account_ref),
|
||||
("provider_kind", &filter.provider_kind),
|
||||
("operation", &filter.operation),
|
||||
("upstream_transport", &filter.transport),
|
||||
("upstream_request_id", &filter.upstream_request_id),
|
||||
("failure_kind", &filter.failure_kind),
|
||||
] {
|
||||
if let Some(value) = value {
|
||||
statement.push(format!(" and e.{column} = "));
|
||||
statement.push_bind(value.clone());
|
||||
}
|
||||
}
|
||||
if let Some(value) = &filter.response_id {
|
||||
statement.push(" and e.client_response_id = ");
|
||||
statement.push_bind(value.as_bytes().to_vec());
|
||||
}
|
||||
if let Some(model) = &filter.model {
|
||||
statement.push(" and e.upstream_model_id = ");
|
||||
statement.push_bind(model.clone());
|
||||
}
|
||||
if let Some(index) = filter.attempt_index {
|
||||
statement.push(" and e.attempt_index = ");
|
||||
statement.push_bind(i32::try_from(index).unwrap_or(i32::MAX));
|
||||
}
|
||||
if let Some(status) = filter.status_code {
|
||||
statement.push(" and e.upstream_status_code = ");
|
||||
statement.push_bind(i32::from(status));
|
||||
}
|
||||
if let Some(search) = &filter.search {
|
||||
statement.push(
|
||||
" and lower(concat_ws(' ', e.event_id, e.request_id, e.client_api_key_ref,
|
||||
e.component, e.operation, e.provider_kind, e.provider_account_ref,
|
||||
e.upstream_model_id, e.failure_kind, e.provider_error_code,
|
||||
e.upstream_request_id, e.message)) like ",
|
||||
);
|
||||
statement.push_bind(format!("%{}%", search.to_lowercase()));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user