mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4127886584 | ||
|
|
ee3e5db6c2 | ||
|
|
153cb67b9c | ||
|
|
f370d355b6 | ||
|
|
a31dad992c | ||
|
|
55a88f5f6b | ||
|
|
5d90a92582 | ||
|
|
93abe8efcd | ||
|
|
f4b75c42ed | ||
|
|
36ff038c78 | ||
|
|
545009590c | ||
|
|
5b7b9b21ee | ||
|
|
44a2a87b69 | ||
|
|
184e88083b | ||
|
|
53b40aea46 | ||
|
|
aa328b6dbf | ||
|
|
64231ff403 | ||
|
|
d7b80a2066 | ||
|
|
08d63ea3f3 | ||
|
|
ac9c4c85ad | ||
|
|
492c401599 | ||
|
|
a063258a40 | ||
|
|
dc590f2b5f | ||
|
|
ca52e2bfd9 | ||
|
|
43604e36dc | ||
|
|
a4a997e351 | ||
|
|
73ca463924 | ||
|
|
9eeec2c78c | ||
|
|
90890be2b4 | ||
|
|
2e4bab17a8 | ||
|
|
b052e79a16 | ||
|
|
9775451e83 | ||
|
|
c22a2ad144 | ||
|
|
7bd362d544 | ||
|
|
e7036a616f | ||
|
|
98652d4c68 | ||
|
|
d0dc7ef92b | ||
|
|
194fb20d5f | ||
|
|
30f528f084 | ||
|
|
28d024ceda | ||
|
|
6b9c98c59f | ||
|
|
1294e50b76 | ||
|
|
a0e540642b | ||
|
|
14d6a7de31 | ||
|
|
cd7b60a3ba | ||
|
|
ebbed22f36 | ||
|
|
f6e5d8f673 | ||
|
|
bf1c6211b8 | ||
|
|
d388fa344a | ||
|
|
eabb61b08c | ||
|
|
a5c3438927 | ||
|
|
b6948542bb | ||
|
|
0242bd3193 | ||
|
|
229ef48470 | ||
|
|
19c35ba901 | ||
|
|
3c06ae97e8 | ||
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab | ||
|
|
95f6434443 | ||
|
|
8734b0d3d5 | ||
|
|
790d0d9628 | ||
|
|
3c562563c5 | ||
|
|
f19785d0e1 | ||
|
|
a3d098a87c | ||
|
|
449d7c4218 | ||
|
|
d199141a91 | ||
|
|
df83ffdafb | ||
|
|
cf03c56cfa | ||
|
|
3d1e33558a | ||
|
|
f314ebb3a2 | ||
|
|
31d6ef9585 | ||
|
|
fdfb77c4a2 | ||
|
|
a93e3853f7 | ||
|
|
34da9d7cc8 | ||
|
|
b3dd6bbe46 | ||
|
|
9ab3e45288 | ||
|
|
d255a86644 | ||
|
|
f3f50a818f | ||
|
|
1ff520659f | ||
|
|
33381fc472 | ||
|
|
e3d767bf54 | ||
|
|
8f82d0e100 | ||
|
|
3085e5aabe | ||
|
|
36dce639a9 | ||
|
|
6e15757150 | ||
|
|
365d389a14 | ||
|
|
e79d57eaf9 | ||
|
|
8d5dfea6de | ||
|
|
e02e5b186e | ||
|
|
2dc670875a | ||
|
|
9768921ff7 | ||
|
|
db90f0631a | ||
|
|
8a0a8f9af4 | ||
|
|
8ca2c68e90 | ||
|
|
d26fc44fe6 | ||
|
|
7ed2a6668a | ||
|
|
3c4ac99ae4 | ||
|
|
63d276767e | ||
|
|
60dcc5b0db | ||
|
|
0d2c648422 | ||
|
|
56f7ddd1dc | ||
|
|
930e311f51 | ||
|
|
5df8c71b89 | ||
|
|
dfedb5069d | ||
|
|
7016c8f401 | ||
|
|
5a14b74ff2 | ||
|
|
00bab8d390 | ||
|
|
f6e364ca33 | ||
|
|
4cd98889a7 | ||
|
|
c9b613659c | ||
|
|
01b4029998 | ||
|
|
f583e50e88 | ||
|
|
a7ea957843 | ||
|
|
c92c95a733 | ||
|
|
ae7608440e | ||
|
|
5c9da8a39d | ||
|
|
e012f24ef2 | ||
|
|
756c9c7158 | ||
|
|
ab4bf75d0e | ||
|
|
7a9715ed41 | ||
|
|
c353a6159e | ||
|
|
f5dca4848a | ||
|
|
36321a9af4 | ||
|
|
b485b8bd4a | ||
|
|
cae2c8656a | ||
|
|
acf574129f | ||
|
|
6426e0d3de | ||
|
|
eedc39b33a | ||
|
|
9672a2e000 | ||
|
|
0fffd2c550 | ||
|
|
5671a57dc8 | ||
|
|
166e8754a2 | ||
|
|
fbaf6c1bdd | ||
|
|
7cdc2dba6f | ||
|
|
a321ccf737 | ||
|
|
4533ae8850 | ||
|
|
7542ed365e | ||
|
|
bf4da7a3e7 | ||
|
|
0fe354b083 | ||
|
|
6d0c583599 | ||
|
|
2e861c03c4 | ||
|
|
d5fc778ea7 | ||
|
|
404cd41ff4 | ||
|
|
851f473b88 | ||
|
|
e97da4f551 | ||
|
|
5544b0fb29 | ||
|
|
428e744710 | ||
|
|
7b406765d6 | ||
|
|
92ae7df115 | ||
|
|
cc42e78334 | ||
|
|
0c90ed30fb | ||
|
|
5a81856510 | ||
|
|
3110b71512 | ||
|
|
eac1b2c39e | ||
|
|
337bb80230 | ||
|
|
82f2a0da06 | ||
|
|
f0d8dad71e | ||
|
|
f5c1d773d7 |
@@ -21,6 +21,11 @@ updates:
|
||||
day: monday
|
||||
time: '09:10'
|
||||
timezone: Asia/Shanghai
|
||||
# 固定官方 Codex Desktop 的 TLS 指纹;升级前必须抓取握手并比对扩展顺序。
|
||||
ignore:
|
||||
- dependency-name: hyper-rustls
|
||||
- dependency-name: reqwest
|
||||
- dependency-name: rustls
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
components: rustfmt,clippy
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
- name: Detect affected checks
|
||||
id: filter
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
@@ -15,7 +12,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
@@ -43,14 +40,14 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
|
||||
ref: ${{ inputs.tag }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Read release metadata
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
RELEASE_TAG: ${{ inputs.tag }}
|
||||
run: |
|
||||
ruby <<'RUBY' >> "$GITHUB_OUTPUT"
|
||||
require "json"
|
||||
@@ -188,9 +185,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -253,9 +250,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-${{ matrix.target }}-
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
shell: bash
|
||||
@@ -378,7 +375,7 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
@@ -454,7 +451,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -471,8 +468,8 @@ jobs:
|
||||
push: true
|
||||
build-args: |
|
||||
CPR_PREBUILT_DIR=dist/docker
|
||||
cache-from: type=gha,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=gha,mode=max,scope=runtime-prebuilt-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
labels: |
|
||||
@@ -525,7 +522,7 @@ jobs:
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
@@ -584,7 +581,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
@@ -626,7 +623,7 @@ jobs:
|
||||
make_latest: ${{ needs.release-info.outputs.prerelease == 'true' && 'false' || 'true' }}
|
||||
body: |
|
||||
> [!WARNING]
|
||||
> **Upgrading across major versions?** Database schema and configuration are only compatible within a major version, and the one-click self-update refuses cross-major targets (releases before 3.0 lack this gate — do not use their built-in update to reach a newer major). Deploy the new major fresh (new `.runtime/` data directory) and re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
> **Upgrading across major versions?** Online upgrades across major versions are not supported. Deploy the new major fresh with a new `.runtime/` data directory, then re-import or re-authorize provider accounts and client keys. See the [upgrade notes](https://github.com/${{ github.repository }}/blob/main/deploy/README.md#镜像升级与源码构建).
|
||||
|
||||
## 📥 Installation
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ Thumbs.db
|
||||
/.codex-ws-audit/
|
||||
/.agents/
|
||||
/.reasonix/
|
||||
/.claude
|
||||
/.worktrees/
|
||||
/worktrees/
|
||||
/.cargo/
|
||||
|
||||
@@ -21,15 +21,16 @@
|
||||
|
||||
## 能力
|
||||
|
||||
| 领域 | 实现 |
|
||||
| -------- | --------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI SSE 与 WebSocket 均原样转发上游字节;xAI 在 `providers/xai` 内做 Grok 与 Responses 协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因 |
|
||||
| 领域 | 实现 |
|
||||
| -------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -93,6 +94,11 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
|
||||
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
|
||||
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
|
||||
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
|
||||
[账号接口](docs/api.md#5-账号)。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
|
||||
@@ -145,7 +151,7 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
## 运维
|
||||
|
||||
```bash
|
||||
# 升级
|
||||
# 实例升级
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build
|
||||
|
||||
@@ -162,10 +168,8 @@ docker compose -f deploy/compose.yaml up -d
|
||||
> 运行状态。
|
||||
|
||||
> [!WARNING]
|
||||
> 跨大版本不是原地升级:数据库 schema 与配置只在同一大版本内兼容,内置一键自更新
|
||||
> 会拒绝跨大版本目标(3.0 之前的旧版没有这道门禁,不要用旧版界面的一键更新跨大版本)。
|
||||
> 升级到新的大版本需要全新部署(全新 `.runtime/` 数据目录),并重新导入或重新授权
|
||||
> Provider 账号与客户端 Key。
|
||||
> 不支持跨大版本在线升级。升级到新的大版本时,请使用全新的 `.runtime/` 数据目录重新部署,
|
||||
> 并重新导入或重新授权 Provider 账号与客户端 Key。
|
||||
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
|
||||
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
@@ -178,8 +182,10 @@ upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
release/publish <version>
|
||||
```
|
||||
|
||||
脚本会更新 `release/version.yaml`、创建约定提交与带注释的 `v<version>` tag,并原子推送分支和 tag。
|
||||
不要手工修改版本后单独打 tag。
|
||||
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
|
||||
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
|
||||
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
|
||||
仓库发版和实例升级是三个独立状态。
|
||||
|
||||
## 文档
|
||||
|
||||
|
||||
Generated
+730
-248
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ tokio = { version = "1.52.3", default-features = false, features = ["rt-multi-th
|
||||
|
||||
[dev-dependencies]
|
||||
config = { version = "0.15.23", default-features = false, features = ["yaml"] }
|
||||
serde_json = "1"
|
||||
syn = { version = "3.0.3", features = ["full"] }
|
||||
tempfile = "3.27.0"
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
let providers = ProviderRegistry::new([openai.core_provider(), xai.core_provider()])?;
|
||||
let mut core = gateway_core::initialize(store.core_ports(), providers).await?;
|
||||
host.report_startup_ready("Core");
|
||||
let admin = gateway_admin::initialize(
|
||||
let mut admin = gateway_admin::initialize(
|
||||
admin,
|
||||
store.admin_ports(),
|
||||
vec![openai.admin_provider(), xai.admin_provider()],
|
||||
@@ -99,6 +99,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
plan.extend(core.take_worker_contributions());
|
||||
plan.extend(openai.take_worker_contributions());
|
||||
plan.extend(xai.take_worker_contributions());
|
||||
plan.extend(admin.take_worker_contributions());
|
||||
host.start_workers(plan, store.worker_leader_lease())?;
|
||||
host.report_startup_ready("Workers");
|
||||
host.serve(api.router()).await?;
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
use std::{env, error::Error, io};
|
||||
|
||||
use provider_openai::transport::tls::ensure_rustls_provider;
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
ensure_rustls_provider();
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
match arguments.next().as_deref().and_then(|value| value.to_str()) {
|
||||
None | Some("serve") => {
|
||||
|
||||
@@ -82,6 +82,91 @@ fn gateway_admin_stays_free_of_infrastructure_dependencies() {
|
||||
);
|
||||
}
|
||||
|
||||
/// workspace 包名到冻结成员路径的映射。
|
||||
const PACKAGE_TO_MEMBER: &[(&str, &str)] = &[
|
||||
("codex-proxy-rs", "apps/gateway"),
|
||||
("gateway-admin", "crates/gateway-admin"),
|
||||
("gateway-api", "crates/gateway-api"),
|
||||
("gateway-core", "crates/gateway-core"),
|
||||
("gateway-host", "crates/gateway-host"),
|
||||
("gateway-protocol", "crates/gateway-protocol"),
|
||||
("gateway-store", "crates/gateway-store"),
|
||||
("provider-openai", "crates/providers/openai"),
|
||||
("provider-xai", "crates/providers/xai"),
|
||||
];
|
||||
|
||||
/// 冻结的 workspace 内部运行时依赖边;新增/删除任何边都必须同步本表。
|
||||
const ALLOWED_INTERNAL_EDGES: &[(&str, &str)] = &[
|
||||
("codex-proxy-rs", "gateway-admin"),
|
||||
("codex-proxy-rs", "gateway-api"),
|
||||
("codex-proxy-rs", "gateway-core"),
|
||||
("codex-proxy-rs", "gateway-host"),
|
||||
("codex-proxy-rs", "gateway-store"),
|
||||
("codex-proxy-rs", "provider-openai"),
|
||||
("codex-proxy-rs", "provider-xai"),
|
||||
("gateway-admin", "gateway-core"),
|
||||
("gateway-api", "gateway-admin"),
|
||||
("gateway-api", "gateway-core"),
|
||||
("gateway-api", "gateway-protocol"),
|
||||
("gateway-host", "gateway-admin"),
|
||||
("gateway-host", "gateway-core"),
|
||||
("gateway-store", "gateway-admin"),
|
||||
("gateway-store", "gateway-core"),
|
||||
("provider-openai", "gateway-admin"),
|
||||
("provider-openai", "gateway-core"),
|
||||
("provider-openai", "gateway-protocol"),
|
||||
("provider-xai", "gateway-admin"),
|
||||
("provider-xai", "gateway-core"),
|
||||
("provider-xai", "gateway-protocol"),
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn workspace_internal_dependency_edges_match_frozen_dag() {
|
||||
let metadata = cargo_metadata_json();
|
||||
let packages = metadata["packages"]
|
||||
.as_array()
|
||||
.expect("cargo metadata packages");
|
||||
let internal_names: std::collections::BTreeSet<&str> =
|
||||
PACKAGE_TO_MEMBER.iter().map(|(name, _)| *name).collect();
|
||||
|
||||
let mut actual: Vec<(String, String)> = Vec::new();
|
||||
for package in packages {
|
||||
let name = package["name"].as_str().expect("package name");
|
||||
if !internal_names.contains(name) {
|
||||
continue;
|
||||
}
|
||||
for dependency in package["dependencies"].as_array().into_iter().flatten() {
|
||||
let dep_name = dependency["name"].as_str().expect("dependency name");
|
||||
if !internal_names.contains(dep_name) {
|
||||
continue;
|
||||
}
|
||||
let is_runtime_edge = dependency["dep_kinds"]
|
||||
.as_array()
|
||||
.map(|kinds| {
|
||||
kinds
|
||||
.iter()
|
||||
.any(|kind| kind["kind"].as_str() == Some("normal"))
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if is_runtime_edge {
|
||||
actual.push((name.to_owned(), dep_name.to_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut expected: Vec<(String, String)> = ALLOWED_INTERNAL_EDGES
|
||||
.iter()
|
||||
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
|
||||
.collect();
|
||||
actual.sort();
|
||||
expected.sort();
|
||||
|
||||
assert_eq!(
|
||||
actual, expected,
|
||||
"workspace internal dependency edges diverged from the frozen DAG"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_sources_do_not_host_tests() {
|
||||
for member in WORKSPACE_MEMBERS {
|
||||
@@ -107,6 +192,20 @@ fn backend_root() -> PathBuf {
|
||||
.expect("backend workspace root")
|
||||
}
|
||||
|
||||
fn cargo_metadata_json() -> serde_json::Value {
|
||||
let output = std::process::Command::new("cargo")
|
||||
.args(["metadata", "--format-version", "1", "--no-deps"])
|
||||
.current_dir(backend_root())
|
||||
.output()
|
||||
.expect("run cargo metadata");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"cargo metadata failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
serde_json::from_slice(&output.stdout).expect("parse cargo metadata")
|
||||
}
|
||||
|
||||
/// 提取成员 `[dependencies]` 段内声明的依赖名;段落以下一个 `[` 表头结束。
|
||||
fn dependency_names(member: &str) -> Vec<String> {
|
||||
let manifest = fs::read_to_string(backend_root().join(member).join("Cargo.toml"))
|
||||
|
||||
@@ -134,8 +134,8 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_core_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.145.0'", "codex_version: 'latest'"));
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -154,7 +154,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '5828'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '6119'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -26,9 +26,14 @@ subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
cron = "0.17"
|
||||
chrono-tz = "0.10"
|
||||
tokio = { version = "1.52", features = ["rt", "time", "macros"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
//! 备份纯策略与 Worker 任务。
|
||||
|
||||
pub mod policy;
|
||||
pub mod task;
|
||||
@@ -0,0 +1,161 @@
|
||||
//! 备份纯策略:Cron 推进与保留候选。
|
||||
//!
|
||||
//! 本模块不持有任何端口或对象,是 `BackupService` 与 `BackupTask` 共用的纯函数集。
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use chrono::{DateTime, Duration, TimeZone as _, Utc};
|
||||
use chrono_tz::Tz;
|
||||
use cron::Schedule;
|
||||
|
||||
use crate::model::backup::{BackupError, BackupRecord, BackupStatus, code};
|
||||
|
||||
/// 向历史回溯的最大范围(分钟)。覆盖 Feb 29 + 特定星期等罕见组合的最长间隔。
|
||||
const MAX_BACKDATE_MINUTES: i64 = 40 * 366 * 24 * 60;
|
||||
|
||||
/// 已解析并校验的备份计划。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupSchedule {
|
||||
schedule: Schedule,
|
||||
timezone: Tz,
|
||||
}
|
||||
|
||||
impl BackupSchedule {
|
||||
/// 解析并校验 5 段 Cron 与 IANA 时区。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Cron 非法或时区不是 IANA 名称时返回 [`BackupError`]。
|
||||
pub fn parse(cron_expression: &str, schedule_timezone: &str) -> Result<Self, BackupError> {
|
||||
let expression = format!("0 {cron_expression}");
|
||||
let schedule = Schedule::from_str(&expression).map_err(|_| {
|
||||
BackupError::new(code::INVALID_CRON, "Cron 表达式必须是 5 段格式".to_owned())
|
||||
})?;
|
||||
let timezone = schedule_timezone.parse::<Tz>().map_err(|_| {
|
||||
BackupError::new(code::INVALID_TIMEZONE, "时区必须是 IANA 名称".to_owned())
|
||||
})?;
|
||||
Ok(Self { schedule, timezone })
|
||||
}
|
||||
|
||||
/// 返回最接近 `at`(含 `at` 所在分钟)的最近一次触发时间。
|
||||
///
|
||||
/// 若过去 `MAX_BACKDATE_MINUTES` 内没有任何触发(极罕见的稀疏计划或超长停机),
|
||||
/// 返回 `None`,调用方不应为历史点补任务。
|
||||
#[must_use]
|
||||
pub fn last_firing_at_or_before(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
|
||||
let high_min = at.timestamp().div_euclid(60);
|
||||
let low_min = high_min.saturating_sub(MAX_BACKDATE_MINUTES);
|
||||
|
||||
// 检查窗口内是否存在触发;不存在则直接返回 None。
|
||||
let low_instant = instant(low_min, self.timezone)?;
|
||||
if self
|
||||
.schedule
|
||||
.after(&low_instant)
|
||||
.next()
|
||||
.is_none_or(|firing| firing.timestamp() > high_min * 60 + 59)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
// 二分查找最小的 u(分钟)使 after(u) > high;prev = after(u - 1 分钟)。
|
||||
let mut lo = low_min;
|
||||
let mut hi = high_min;
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
if self
|
||||
.firing_after(mid)
|
||||
.is_some_and(|firing| firing > high_min)
|
||||
{
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
self.firing_after(lo - 1)
|
||||
.filter(|&firing| firing <= high_min)
|
||||
.and_then(|firing| Utc.timestamp_opt(firing * 60, 0).single())
|
||||
}
|
||||
|
||||
/// 返回第一个严格晚于 `at` 的触发时间。
|
||||
#[must_use]
|
||||
pub fn next_after(&self, at: DateTime<Utc>) -> Option<DateTime<Utc>> {
|
||||
let local = at.with_timezone(&self.timezone);
|
||||
self.schedule
|
||||
.after(&local)
|
||||
.next()
|
||||
.map(|firing| firing.with_timezone(&Utc))
|
||||
}
|
||||
|
||||
/// `after(u 分钟)` 返回的第一个触发时间(分钟索引),用于二分搜索。
|
||||
fn firing_after(&self, minute_index: i64) -> Option<i64> {
|
||||
let instant = instant(minute_index, self.timezone)?;
|
||||
self.schedule
|
||||
.after(&instant)
|
||||
.next()
|
||||
.map(|firing| firing.timestamp().div_euclid(60))
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造指定分钟索引(UTC epoch 分钟)在目标时区下的 `DateTime`。
|
||||
fn instant(minute_index: i64, timezone: Tz) -> Option<DateTime<Tz>> {
|
||||
timezone.timestamp_opt(minute_index * 60, 0).single()
|
||||
}
|
||||
|
||||
/// 一条记录被纳入删除流程的原因。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RetentionReason {
|
||||
/// 超过 `retention_days`。
|
||||
Days,
|
||||
/// 超过 `retention_count`。
|
||||
Count,
|
||||
}
|
||||
|
||||
/// 保留决策。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RetentionDecision {
|
||||
pub record_id: String,
|
||||
pub reason: RetentionReason,
|
||||
}
|
||||
|
||||
/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。
|
||||
///
|
||||
/// 规则(`docs/s3-backup-design-audit.md` §4.4):
|
||||
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理。
|
||||
/// - 自动计划备份满足任一启用阈值即可进入删除。
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)。
|
||||
/// - 手动备份不受自动阈值影响;无 TTL,仅由管理员手动删除。
|
||||
#[must_use]
|
||||
pub fn decide_retention(
|
||||
retention_days: u32,
|
||||
retention_count: u32,
|
||||
now: DateTime<Utc>,
|
||||
scheduled_completed_desc: &[BackupRecord],
|
||||
) -> Vec<RetentionDecision> {
|
||||
let mut decisions = Vec::new();
|
||||
for (index, record) in scheduled_completed_desc.iter().enumerate() {
|
||||
if record.status != BackupStatus::Completed {
|
||||
continue;
|
||||
}
|
||||
// 最近一个成功的计划备份永远保留。
|
||||
if index == 0 {
|
||||
continue;
|
||||
}
|
||||
let expired_by_days = retention_days > 0
|
||||
&& record.completed_at.is_some_and(|completed_at| {
|
||||
completed_at <= now - Duration::days(i64::from(retention_days))
|
||||
});
|
||||
let exceeded_count =
|
||||
retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX);
|
||||
if expired_by_days || exceeded_count {
|
||||
decisions.push(RetentionDecision {
|
||||
record_id: record.id.clone(),
|
||||
reason: if expired_by_days {
|
||||
RetentionReason::Days
|
||||
} else {
|
||||
RetentionReason::Count
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
decisions
|
||||
}
|
||||
@@ -0,0 +1,648 @@
|
||||
//! Backup Worker 贡献:单个可取消 `DaemonTask` 承担调度、执行、删除收敛与保留清理。
|
||||
//!
|
||||
//! 当前部署边界是单副本,因此不需要 Redis lease、fencing token 或 heartbeat。长时间
|
||||
//! `pg_dump` 与上传由本 Daemon 自身持有;Host 只负责 panic 后重启、健康与关闭。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::{
|
||||
model::backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, build_backup_seed, code,
|
||||
},
|
||||
ports::backup::{
|
||||
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpRequest,
|
||||
StatusTransitionUpdate, UploadObjectRequest,
|
||||
},
|
||||
};
|
||||
|
||||
impl From<BackupError> for WorkerTaskError {
|
||||
fn from(error: BackupError) -> Self {
|
||||
WorkerTaskError::safe(error.message())
|
||||
}
|
||||
}
|
||||
|
||||
use super::policy::{BackupSchedule, decide_retention};
|
||||
|
||||
/// 循环间隔:管理员修改 Cron 后该时长内生效;无工作时可取消等待。
|
||||
const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// 每个循环最多执行的保留删除数量。
|
||||
const RETENTION_BATCH_SIZE: usize = 10;
|
||||
/// 保留扫描一次读取的 completed 计划备份上限。
|
||||
const RETENTION_SCAN_LIMIT: u32 = 1000;
|
||||
/// 每个循环最多完成的待删除记录数量。
|
||||
const PENDING_DELETION_BATCH: u32 = 20;
|
||||
|
||||
/// 备份 Daemon 任务。
|
||||
pub struct BackupTask {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
}
|
||||
|
||||
impl BackupTask {
|
||||
/// 组合仓储、导出器与对象存储适配器。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
dump,
|
||||
object_store,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move { self.run_daemon(cancellation).await })
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupTask {
|
||||
async fn run_daemon(&self, cancellation: CancellationToken) -> Result<(), WorkerTaskError> {
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle(&cancellation).await?;
|
||||
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Ok(()),
|
||||
_ = tokio::time::sleep(BACKUP_LOOP_INTERVAL) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 执行一个周期:推进计划、恢复中间态、删除收敛、领取执行一个任务、保留清理。
|
||||
///
|
||||
/// 对外暴露以便集成测试逐周期驱动;daemon 循环内部复用。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 仓储或基础设施不可用、任务执行失败时返回 [`WorkerTaskError`]。
|
||||
pub async fn run_cycle(&self, cancellation: &CancellationToken) -> Result<(), WorkerTaskError> {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.advance_schedule(Utc::now()).await?;
|
||||
self.recover_intermediate(cancellation).await?;
|
||||
self.finish_pending_deletions().await?;
|
||||
self.process_next_task(cancellation).await?;
|
||||
self.run_retention_batch(Utc::now()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 推进到期计划并幂等插入 scheduled 任务。
|
||||
async fn advance_schedule(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
if !settings.schedule_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let cron = settings.cron_expression.as_deref().unwrap_or_default();
|
||||
let timezone = settings.schedule_timezone.as_deref().unwrap_or_default();
|
||||
if cron.is_empty() || timezone.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let schedule = BackupSchedule::parse(cron, timezone)
|
||||
.map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?;
|
||||
|
||||
let due = match settings.next_run_at {
|
||||
Some(next_run_at) if next_run_at <= now => true,
|
||||
Some(_) => false,
|
||||
None => {
|
||||
// 刚启用计划但尚无游标:直接初始化。
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
if !due {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let scheduled_at = match schedule.last_firing_at_or_before(now) {
|
||||
Some(scheduled_at) => scheduled_at,
|
||||
None => {
|
||||
// 无法从计划推导最近到期时间:推进游标避免反复触发。
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let prefix = settings.prefix.as_deref().unwrap_or_default();
|
||||
let expires_at = if settings.retention_days > 0 {
|
||||
Some(now + chrono::Duration::days(i64::from(settings.retention_days)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let seed = build_backup_seed(
|
||||
BackupTriggerKind::Scheduled,
|
||||
Some(scheduled_at),
|
||||
prefix,
|
||||
now,
|
||||
expires_at,
|
||||
)
|
||||
.map_err(infra_error)?;
|
||||
let inserted = self
|
||||
.repository
|
||||
.insert_scheduled_record(seed)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
if inserted {
|
||||
info!(scheduled_at = %scheduled_at, "计划备份任务已创建");
|
||||
} else {
|
||||
warn!(scheduled_at = %scheduled_at, "计划时间点冲突,跳过并推进游标");
|
||||
}
|
||||
if let Some(next_run_at) = schedule.next_after(now) {
|
||||
let _ = self
|
||||
.repository
|
||||
.advance_schedule_cursor(next_run_at, cron, timezone)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 恢复中间状态(dumping/uploading/deleting)。
|
||||
async fn recover_intermediate(
|
||||
&self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let records = self
|
||||
.repository
|
||||
.list_intermediate_records()
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in records {
|
||||
match record.status {
|
||||
BackupStatus::Dumping => self.recover_dumping(&record, cancellation).await?,
|
||||
BackupStatus::Uploading => self.recover_uploading(&record, cancellation).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// dumping:有完整暂存则继续上传,否则清理暂存并标记失败。
|
||||
async fn recover_dumping(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
match self.dump.inspect_staging(&record.id).await? {
|
||||
Some(artifact) => {
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
size_bytes: Some(artifact.size_bytes),
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
update,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
self.upload_and_verify(
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
now,
|
||||
cancellation,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
self.fail_task(record, code::PG_DUMP_FAILED, "导出进程中断,暂存归档不完整")
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// uploading:远端匹配则补记完成,有完整暂存则重试上传,否则失败。
|
||||
async fn recover_uploading(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(remote) = self
|
||||
.object_store
|
||||
.head_object(&config, &record.object_key)
|
||||
.await?
|
||||
&& record.size_bytes == Some(remote.size_bytes)
|
||||
&& record.sha256.as_deref() == Some(remote.sha256.as_str())
|
||||
{
|
||||
let now = Utc::now();
|
||||
self.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
StatusTransitionUpdate::default(),
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
return Ok(());
|
||||
}
|
||||
match self.dump.inspect_staging(&record.id).await? {
|
||||
Some(artifact) => {
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
Utc::now(),
|
||||
cancellation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.fail_task(
|
||||
record,
|
||||
code::S3_UPLOAD_FAILED,
|
||||
"远端对象缺失且本地无完整暂存",
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 完成待删除记录(deleting → DeleteObject → 硬删除)。
|
||||
async fn finish_pending_deletions(&self) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
let records = self
|
||||
.repository
|
||||
.list_pending_deletions(PENDING_DELETION_BATCH)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in records {
|
||||
if self
|
||||
.object_store
|
||||
.delete_object(&config, &record.object_key)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.repository
|
||||
.delete_record(&record.id)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 领取一个 queued 任务并执行导出、上传与远端校验。
|
||||
async fn process_next_task(
|
||||
&self,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let now = Utc::now();
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.claim_next_queued(now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
info!(
|
||||
backup_id = %record.id,
|
||||
attempt = record.attempt_count,
|
||||
"领取备份任务"
|
||||
);
|
||||
if let Err(error) = self.execute_task(&record, cancellation).await {
|
||||
warn!(backup_id = %record.id, error = %error, "备份任务执行失败");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 执行单个任务;内部处理全部状态迁移。
|
||||
async fn execute_task(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
if BackupStorageConfig::from_settings(&settings).is_none() {
|
||||
return self
|
||||
.fail_task(record, code::INVALID_CONFIG, "备份存储配置不完整")
|
||||
.await;
|
||||
}
|
||||
if !settings.storage_verified() {
|
||||
return self
|
||||
.fail_task(record, code::INVALID_CONFIG, "备份存储配置尚未通过连接测试")
|
||||
.await;
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
let artifact = match self
|
||||
.dump
|
||||
.dump(DumpRequest {
|
||||
backup_id: record.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(artifact) => artifact,
|
||||
Err(error) => {
|
||||
let (error_code, message) = classify_dump_error(error);
|
||||
return self.fail_task(record, error_code, message).await;
|
||||
}
|
||||
};
|
||||
|
||||
// dumping → uploading,持久化 size/sha256。
|
||||
let update = StatusTransitionUpdate {
|
||||
size_bytes: Some(artifact.size_bytes),
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
update,
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.upload_and_verify(
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
now,
|
||||
cancellation,
|
||||
)
|
||||
.await?;
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 上传 + 远端校验,内部完成终态迁移。
|
||||
async fn upload_and_verify(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
source: &std::path::Path,
|
||||
size_bytes: u64,
|
||||
sha256: &str,
|
||||
now: DateTime<Utc>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
let store = self.object_store.clone();
|
||||
let metadata =
|
||||
BackupObjectMetadata::new(record.id.clone(), sha256.to_owned(), now, size_bytes);
|
||||
if let Err(error) = store
|
||||
.upload_file(
|
||||
&config,
|
||||
UploadObjectRequest {
|
||||
object_key: record.object_key.clone(),
|
||||
source: source.to_path_buf(),
|
||||
metadata,
|
||||
cancellation: cancellation.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match store.head_object(&config, &record.object_key).await {
|
||||
Ok(Some(remote)) if remote.size_bytes == size_bytes && remote.sha256 == sha256 => {
|
||||
if self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
StatusTransitionUpdate::default(),
|
||||
Utc::now(),
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
.is_some()
|
||||
{
|
||||
info!(
|
||||
backup_id = %record.id,
|
||||
size_bytes,
|
||||
"备份完成"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(Some(remote)) => {
|
||||
let size_matches = remote.size_bytes == size_bytes;
|
||||
let sha256_matches = remote.sha256 == sha256;
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
expected_size_bytes = size_bytes,
|
||||
remote_size_bytes = remote.size_bytes,
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"上传完成后未找到远端对象",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 把活跃任务写入失败终态并清理暂存。
|
||||
async fn fail_task(
|
||||
&self,
|
||||
record: &BackupRecord,
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let message = message.into();
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
error_code,
|
||||
error_message = %message,
|
||||
"备份任务失败"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
error_code: Some(error_code.to_string()),
|
||||
error_message: Some(message),
|
||||
completed_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = self
|
||||
.repository
|
||||
.transition_status(&record.id, record.status, BackupStatus::Failed, update, now)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let _ = self.dump.cleanup_staging(&record.id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 执行一小批到期保留清理:先处理 `expires_at` 已到期的记录,再按天数/份数清理计划备份。
|
||||
async fn run_retention_batch(&self, now: DateTime<Utc>) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
|
||||
// 1. expires_at 已到期的记录(手动或计划)无条件进入删除。
|
||||
let expired = self
|
||||
.repository
|
||||
.list_expired_records(RETENTION_BATCH_SIZE as u32)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
for record in expired {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&record.id, now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
self.finalize_deletion(&record).await?;
|
||||
}
|
||||
|
||||
// 2. 计划备份按 retentionDays / retentionCount 清理。
|
||||
let records = self
|
||||
.repository
|
||||
.list_scheduled_completed_desc(RETENTION_SCAN_LIMIT)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
let decisions = decide_retention(
|
||||
settings.retention_days,
|
||||
settings.retention_count,
|
||||
now,
|
||||
&records,
|
||||
);
|
||||
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&decision.record_id, now)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
self.finalize_deletion(&record).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 对已进入 deleting 的记录执行对象删除并硬删除记录行。
|
||||
async fn finalize_deletion(&self, record: &BackupRecord) -> Result<(), WorkerTaskError> {
|
||||
let settings = self.repository.load_settings().await.map_err(repo_error)?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(());
|
||||
};
|
||||
if self
|
||||
.object_store
|
||||
.delete_object(&config, &record.object_key)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
self.repository
|
||||
.delete_record(&record.id)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
|
||||
match error.code() {
|
||||
code::CANCELLED => (code::CANCELLED, "备份已取消"),
|
||||
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
|
||||
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(format!("backup store failed: {error}"))
|
||||
}
|
||||
|
||||
fn infra_error(error: BackupError) -> WorkerTaskError {
|
||||
WorkerTaskError::safe(error.message())
|
||||
}
|
||||
@@ -2,23 +2,28 @@
|
||||
//!
|
||||
//! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现。
|
||||
|
||||
use std::{fmt, path::Path, sync::Arc};
|
||||
use std::{fmt, path::Path, sync::Arc, time::Duration};
|
||||
|
||||
use gateway_core::{
|
||||
engine::probe::AccountProbe,
|
||||
routing::{ProviderKind, snapshot::SnapshotControl},
|
||||
task::{
|
||||
DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration,
|
||||
WorkerRunnable,
|
||||
},
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
pub mod backup;
|
||||
pub mod model;
|
||||
pub mod ports;
|
||||
mod use_case;
|
||||
|
||||
pub use use_case::{
|
||||
accounts::AccountsService, auth::AuthService, client_keys::ClientKeyService,
|
||||
observability::ObservabilityService, openai::OpenAiService, settings::SettingsService,
|
||||
system::SystemService, xai::XaiService,
|
||||
accounts::AccountsService, auth::AuthService, backup::BackupService,
|
||||
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
|
||||
settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
};
|
||||
|
||||
use model::{AdminError, AdminErrorKind};
|
||||
@@ -28,7 +33,7 @@ use ports::{
|
||||
system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
accounts::DefaultAccountsService, auth::DefaultAuthService,
|
||||
accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_keys::DefaultClientKeyService, observability::DefaultObservabilityService,
|
||||
openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService,
|
||||
xai::DefaultXaiService,
|
||||
@@ -48,6 +53,8 @@ const WEAK_INITIAL_PASSWORDS: &[&str] = &[
|
||||
"codex-proxy-rs",
|
||||
];
|
||||
|
||||
const BACKUP_WORKER_OWNER: &str = "backup";
|
||||
|
||||
/// 只用于首次幂等创建默认管理员的启动密码。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
@@ -146,6 +153,7 @@ pub struct AdminServices {
|
||||
system: Arc<dyn SystemService>,
|
||||
openai: Arc<dyn OpenAiService>,
|
||||
xai: Arc<dyn XaiService>,
|
||||
backups: Arc<dyn BackupService>,
|
||||
}
|
||||
|
||||
impl AdminServices {
|
||||
@@ -188,11 +196,17 @@ impl AdminServices {
|
||||
pub fn xai(&self) -> &dyn XaiService {
|
||||
self.xai.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn backups(&self) -> &dyn BackupService {
|
||||
self.backups.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Admin 初始化完成后的封闭能力包。
|
||||
pub struct AdminBundle {
|
||||
services: AdminServices,
|
||||
worker_contributions: Vec<WorkerContribution>,
|
||||
}
|
||||
|
||||
impl AdminBundle {
|
||||
@@ -200,6 +214,11 @@ impl AdminBundle {
|
||||
pub fn services(&self) -> AdminServices {
|
||||
self.services.clone()
|
||||
}
|
||||
|
||||
/// 取出 Backup Worker 贡献;只能调用一次,与其它 Bundle 的贡献一并交给 Host。
|
||||
pub fn take_worker_contributions(&mut self) -> Vec<WorkerContribution> {
|
||||
std::mem::take(&mut self.worker_contributions)
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验配置、建立动态 Provider 注册表并完成默认管理员幂等初始化。
|
||||
@@ -241,6 +260,18 @@ pub async fn initialize(
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
));
|
||||
let backup_ports = store.backup();
|
||||
let backups = Arc::new(DefaultBackupService::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.object_store(),
|
||||
store.auth(),
|
||||
snapshot.clone(),
|
||||
));
|
||||
let backup_task = backup::task::BackupTask::new(
|
||||
backup_ports.repository(),
|
||||
backup_ports.dump(),
|
||||
backup_ports.object_store(),
|
||||
);
|
||||
let services = AdminServices {
|
||||
auth,
|
||||
accounts,
|
||||
@@ -268,8 +299,32 @@ pub async fn initialize(
|
||||
store.accounts(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
backups,
|
||||
};
|
||||
Ok(AdminBundle { services })
|
||||
let worker_contributions = backup_worker_contribution(backup_task)?;
|
||||
Ok(AdminBundle {
|
||||
services,
|
||||
worker_contributions,
|
||||
})
|
||||
}
|
||||
|
||||
/// Backup Worker 注册:单个可取消 Daemon,owner 固定为 `backup`。
|
||||
fn backup_worker_contribution(
|
||||
task: backup::task::BackupTask,
|
||||
) -> Result<Vec<WorkerContribution>, AdminError> {
|
||||
let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER)
|
||||
.map_err(|_| AdminError::internal("Backup worker id is invalid"))?;
|
||||
let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60))
|
||||
.map_err(|_| AdminError::internal("Backup worker restart policy is invalid"))?;
|
||||
let registration = WorkerRegistration::try_new(
|
||||
id,
|
||||
WorkerRunnable::Daemon {
|
||||
restart,
|
||||
task: Box::new(task),
|
||||
},
|
||||
)
|
||||
.map_err(|_| AdminError::internal("Backup worker registration is invalid"))?;
|
||||
Ok(vec![WorkerContribution::Registration(registration)])
|
||||
}
|
||||
|
||||
fn provider_kind(value: &'static str) -> Result<ProviderKind, AdminError> {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 多 Provider 账号目录与连接测试的公共事实。
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::{pin::Pin, time::SystemTime};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
@@ -13,13 +13,146 @@ use super::{PageSize, Revision, observability::TimeRange};
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
///
|
||||
/// 只回答两个问题:是否可调度、是否需要人工干预。具体错误原因不进入此枚举,
|
||||
/// 由 [`AccountErrorReason`] 承载;新增原因只扩展 reason,不改变分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
Expired,
|
||||
/// 正常,可调度。
|
||||
Normal,
|
||||
/// 配额耗尽(持久化终态),等待额度恢复,无需干预。
|
||||
QuotaExhausted,
|
||||
/// 限流中(quota 快照级信号),自动恢复,无需干预。
|
||||
RateLimited,
|
||||
/// 管理员手动停用(`enabled = false`)。
|
||||
Disabled,
|
||||
/// 错误(过期 / 失效 / 封禁等),需要人工干预。具体原因见 [`AccountErrorReason`]。
|
||||
Error,
|
||||
}
|
||||
|
||||
impl AccountStatus {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Normal => "normal",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::Disabled => "disabled",
|
||||
Self::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"normal" => Some(Self::Normal),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"rate_limited" => Some(Self::RateLimited),
|
||||
"disabled" => Some(Self::Disabled),
|
||||
"error" => Some(Self::Error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 状态派生所需的最小事实;由 store / use_case 各自构造,派生规则唯一。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountStatusSignals {
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub access_token_expired: bool,
|
||||
pub quota_limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 表示账号当前处于临时限流。
|
||||
pub rate_limited_until: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl AccountStatusSignals {
|
||||
/// 派生产出运营分类。终态优先于瞬时信号:停用 > 配额耗尽 > 错误 > 限流。
|
||||
#[must_use]
|
||||
pub const fn derive(self) -> AccountStatus {
|
||||
if !self.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if matches!(self.availability, AccountAvailability::QuotaExhausted) {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if self.access_token_expired
|
||||
|| matches!(
|
||||
self.availability,
|
||||
AccountAvailability::Expired
|
||||
| AccountAvailability::Invalid
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Unknown
|
||||
)
|
||||
{
|
||||
AccountStatus::Error
|
||||
} else if self.rate_limited_until.is_some() {
|
||||
AccountStatus::RateLimited
|
||||
} else if self.quota_limit_reached {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else {
|
||||
AccountStatus::Normal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 错误分类下的具体原因;仅在 `AccountStatus::Error` 时存在。
|
||||
///
|
||||
/// 受控枚举用于展示与聚合,扩展不改变状态分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountErrorReason {
|
||||
Expired,
|
||||
TokenExpired,
|
||||
Invalid,
|
||||
Banned,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AccountErrorReason {
|
||||
/// 返回稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Expired => "expired",
|
||||
Self::TokenExpired => "token_expired",
|
||||
Self::Invalid => "invalid",
|
||||
Self::Banned => "banned",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析稳定 wire 值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"expired" => Some(Self::Expired),
|
||||
"token_expired" => Some(Self::TokenExpired),
|
||||
"invalid" => Some(Self::Invalid),
|
||||
"banned" => Some(Self::Banned),
|
||||
"unknown" => Some(Self::Unknown),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 由信号派生错误原因;非错误分类(正常 / 配额耗尽 / 限流 / 停用)返回 `None`。
|
||||
///
|
||||
/// 与 [`AccountStatusSignals::derive`] 保持同一判定:`Error` 当且仅当此处返回 `Some`。
|
||||
#[must_use]
|
||||
pub fn derive_error_reason(signals: &AccountStatusSignals) -> Option<AccountErrorReason> {
|
||||
if !signals.enabled {
|
||||
return None;
|
||||
}
|
||||
match signals.availability {
|
||||
AccountAvailability::QuotaExhausted => None,
|
||||
AccountAvailability::Ready => signals
|
||||
.access_token_expired
|
||||
.then_some(AccountErrorReason::TokenExpired),
|
||||
AccountAvailability::Expired => Some(AccountErrorReason::Expired),
|
||||
AccountAvailability::Invalid => Some(AccountErrorReason::Invalid),
|
||||
AccountAvailability::Banned => Some(AccountErrorReason::Banned),
|
||||
AccountAvailability::Unknown => Some(AccountErrorReason::Unknown),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号列表排序字段。
|
||||
@@ -65,7 +198,7 @@ pub struct AccountRecord {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -75,9 +208,9 @@ pub struct AccountRecord {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
/// 最近一次非 ready 状态变更的原始原因(上游错误原文 / 原因码);恢复 ready 后清空。
|
||||
pub last_error_message: Option<String>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
@@ -169,12 +302,16 @@ pub struct AccountPage {
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
@@ -212,6 +349,9 @@ pub enum AccountConnectionTestEvent {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
provider_error_type: Option<String>,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,566 @@
|
||||
//! S3 备份领域模型、状态机与稳定错误。
|
||||
//!
|
||||
//! 领域类型只携带事实,不承载基础设施细节。Secret 使用 [`secrecy::SecretString`],
|
||||
//! 全流程不实现 `Serialize`,避免被错误地写入 wire 或日志。管理员身份等审计事实
|
||||
//! 只存在于 `admin_audit_events`,本模型不重复保存。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use secrecy::SecretString;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::PageSize;
|
||||
|
||||
/// 备份触发来源。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupTriggerKind {
|
||||
Manual,
|
||||
Scheduled,
|
||||
}
|
||||
|
||||
impl BackupTriggerKind {
|
||||
/// 稳定持久化值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Manual => "manual",
|
||||
Self::Scheduled => "scheduled",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析持久化值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"manual" => Some(Self::Manual),
|
||||
"scheduled" => Some(Self::Scheduled),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for BackupTriggerKind {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份任务状态机(六状态)。
|
||||
///
|
||||
/// 迁移 `backup_records_lifecycle_ck` 与字段空值语义共同定义合法持久化行;
|
||||
/// [`Self::allows_transition_to`] 只约束领域内的显式状态迁移,两层都必须满足。
|
||||
/// 删除成功后记录行被硬删除,因此不存在 `deleted` 终态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum BackupStatus {
|
||||
Queued,
|
||||
Dumping,
|
||||
Uploading,
|
||||
Completed,
|
||||
Failed,
|
||||
Deleting,
|
||||
}
|
||||
|
||||
impl BackupStatus {
|
||||
/// 稳定持久化值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Queued => "queued",
|
||||
Self::Dumping => "dumping",
|
||||
Self::Uploading => "uploading",
|
||||
Self::Completed => "completed",
|
||||
Self::Failed => "failed",
|
||||
Self::Deleting => "deleting",
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析持久化值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"queued" => Some(Self::Queued),
|
||||
"dumping" => Some(Self::Dumping),
|
||||
"uploading" => Some(Self::Uploading),
|
||||
"completed" => Some(Self::Completed),
|
||||
"failed" => Some(Self::Failed),
|
||||
"deleting" => Some(Self::Deleting),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 该状态是否消耗全局活跃名额。
|
||||
#[must_use]
|
||||
pub const fn is_active(self) -> bool {
|
||||
matches!(self, Self::Queued | Self::Dumping | Self::Uploading)
|
||||
}
|
||||
|
||||
/// 该状态是否允许进入删除流程。
|
||||
#[must_use]
|
||||
pub const fn can_be_deleted(self) -> bool {
|
||||
matches!(self, Self::Completed | Self::Failed)
|
||||
}
|
||||
|
||||
/// 领域内允许的显式迁移。
|
||||
#[must_use]
|
||||
pub fn allows_transition_to(self, target: BackupStatus) -> bool {
|
||||
matches!(
|
||||
(self, target),
|
||||
(Self::Queued, Self::Dumping)
|
||||
| (Self::Queued, Self::Failed)
|
||||
| (Self::Dumping, Self::Uploading)
|
||||
| (Self::Dumping, Self::Failed)
|
||||
| (Self::Uploading, Self::Completed)
|
||||
| (Self::Uploading, Self::Failed)
|
||||
| (Self::Completed, Self::Deleting)
|
||||
| (Self::Failed, Self::Deleting)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for BackupStatus {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 存储配置更新命令;`secret_access_key` 为 `None` 表示保留旧值。
|
||||
#[derive(Clone)]
|
||||
pub struct UpdateBackupStorageCommand {
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: Option<SecretString>,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for UpdateBackupStorageCommand {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("UpdateBackupStorageCommand")
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &self.secret_access_key.is_some())
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 调度配置更新命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateBackupScheduleCommand {
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: String,
|
||||
pub schedule_timezone: String,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
}
|
||||
|
||||
/// 完整保存的备份设置(含 Secret)。
|
||||
///
|
||||
/// 只从仓储读取;不实现 `Serialize`,防止 Secret 进入 wire 或日志。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupSettings {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub bucket: Option<String>,
|
||||
pub access_key_id: Option<String>,
|
||||
pub secret_access_key: Option<SecretString>,
|
||||
pub prefix: Option<String>,
|
||||
pub force_path_style: bool,
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: Option<String>,
|
||||
pub schedule_timezone: Option<String>,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
pub next_run_at: Option<DateTime<Utc>>,
|
||||
pub last_verified_at: Option<DateTime<Utc>>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BackupSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BackupSettings")
|
||||
.field("storage_revision", &self.storage_revision)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &self.secret_access_key.is_some())
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.field("schedule_enabled", &self.schedule_enabled)
|
||||
.field("cron_expression", &self.cron_expression)
|
||||
.field("schedule_timezone", &self.schedule_timezone)
|
||||
.field("retention_days", &self.retention_days)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.field("next_run_at", &self.next_run_at)
|
||||
.field("last_verified_at", &self.last_verified_at)
|
||||
.field("updated_at", &self.updated_at)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupSettings {
|
||||
/// S3 存储配置是否完整(bucket/region/密钥/前缀齐全)。
|
||||
#[must_use]
|
||||
pub fn storage_configured(&self) -> bool {
|
||||
self.endpoint.is_some()
|
||||
&& self.region.is_some()
|
||||
&& self.bucket.is_some()
|
||||
&& self.access_key_id.is_some()
|
||||
&& self.secret_access_key.is_some()
|
||||
&& self.prefix.is_some()
|
||||
}
|
||||
|
||||
/// 探测是否已对当前 revision 成功;`last_verified_at` 非空即表示当前配置已通过探针。
|
||||
#[must_use]
|
||||
pub fn storage_verified(&self) -> bool {
|
||||
self.last_verified_at.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// 已经校验并可用于对象存储操作的存储配置快照。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupStorageConfig {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: SecretString,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
impl fmt::Debug for BackupStorageConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BackupStorageConfig")
|
||||
.field("storage_revision", &self.storage_revision)
|
||||
.field("endpoint", &self.endpoint)
|
||||
.field("region", &self.region)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("access_key_id", &self.access_key_id)
|
||||
.field("secret_access_key", &"[REDACTED]")
|
||||
.field("prefix", &self.prefix)
|
||||
.field("force_path_style", &self.force_path_style)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupStorageConfig {
|
||||
/// 从完整设置推导对象存储配置;设置不完整时返回 `None`。
|
||||
#[must_use]
|
||||
pub fn from_settings(settings: &BackupSettings) -> Option<Self> {
|
||||
Some(Self {
|
||||
storage_revision: settings.storage_revision,
|
||||
endpoint: settings.endpoint.clone()?,
|
||||
region: settings.region.clone()?,
|
||||
bucket: settings.bucket.clone()?,
|
||||
access_key_id: settings.access_key_id.clone()?,
|
||||
secret_access_key: settings.secret_access_key.clone()?,
|
||||
prefix: settings.prefix.clone()?,
|
||||
force_path_style: settings.force_path_style,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 一条备份记录。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecord {
|
||||
pub id: String,
|
||||
pub trigger_kind: BackupTriggerKind,
|
||||
pub status: BackupStatus,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
/// 手动过期时间;创建时确定,到期自动清理。
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// 备份记录分页查询。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordListQuery {
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub status: Option<BackupStatus>,
|
||||
pub trigger: Option<BackupTriggerKind>,
|
||||
}
|
||||
|
||||
/// 备份记录分页结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordPage {
|
||||
pub items: Vec<BackupRecord>,
|
||||
pub total: u64,
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
}
|
||||
|
||||
impl BackupRecordPage {
|
||||
/// 计算总页数;空集合视为一页。
|
||||
#[must_use]
|
||||
pub fn total_pages(&self) -> u32 {
|
||||
let size = u64::from(self.page_size.get());
|
||||
if self.total == 0 {
|
||||
1
|
||||
} else {
|
||||
self.total.div_ceil(size).min(u64::from(u32::MAX)) as u32
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建备份任务的持久化数据;id 与对象 key 由领域生成。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupRecordSeed {
|
||||
pub id: String,
|
||||
pub trigger_kind: BackupTriggerKind,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 连接测试结果;只携带稳定阶段与脱敏消息。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ConnectionTestResult {
|
||||
pub ok: bool,
|
||||
pub stage: &'static str,
|
||||
pub code: Option<&'static str>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// 对象存储探测阶段。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ConnectionTestStage {
|
||||
PutObject,
|
||||
HeadObject,
|
||||
GetObject,
|
||||
DeleteObject,
|
||||
}
|
||||
|
||||
impl ConnectionTestStage {
|
||||
/// 稳定 wire 值。
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::PutObject => "putObject",
|
||||
Self::HeadObject => "headObject",
|
||||
Self::GetObject => "getObject",
|
||||
Self::DeleteObject => "deleteObject",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 上传到对象存储并期望远端校验的归档 metadata。
|
||||
///
|
||||
/// `size_bytes` 在上传时不写入 metadata,但 `HeadObject` 会返回远端大小,
|
||||
/// 用于与本地归档校验一致。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BackupObjectMetadata {
|
||||
pub backup_id: String,
|
||||
pub sha256: String,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
impl BackupObjectMetadata {
|
||||
/// 构造上传 metadata。
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
backup_id: String,
|
||||
sha256: String,
|
||||
created_at: DateTime<Utc>,
|
||||
size_bytes: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
backup_id,
|
||||
sha256,
|
||||
created_at,
|
||||
size_bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 短时下载地址结果。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct DownloadUrlResult {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub expires_in: std::time::Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for DownloadUrlResult {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("DownloadUrlResult")
|
||||
.field("url", &"[REDACTED]")
|
||||
.field("file_name", &self.file_name)
|
||||
.field("expires_in", &self.expires_in)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份基础设施错误:只携带稳定错误码与脱敏消息。
|
||||
///
|
||||
/// 该类型不构成第二套 kind 枚举;`gateway-admin` 边界统一映射为既有
|
||||
/// `AdminErrorKind`。稳定 `error_code` 同时用于任务记录、日志与 UI。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct BackupError {
|
||||
code: &'static str,
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl BackupError {
|
||||
/// 构造基础设施错误。
|
||||
#[must_use]
|
||||
pub const fn new(code: &'static str, message: String) -> Self {
|
||||
Self { code, message }
|
||||
}
|
||||
|
||||
/// 稳定错误码,例如 `backup.s3_auth_failed`。
|
||||
#[must_use]
|
||||
pub const fn code(&self) -> &'static str {
|
||||
self.code
|
||||
}
|
||||
|
||||
/// 脱敏后的可操作消息。
|
||||
#[must_use]
|
||||
pub fn message(&self) -> &str {
|
||||
&self.message
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造稳定错误码常量。
|
||||
pub mod code {
|
||||
/// 配置或参数非法。
|
||||
pub const INVALID_CONFIG: &str = "backup.invalid_config";
|
||||
/// Cron 表达式非法。
|
||||
pub const INVALID_CRON: &str = "backup.invalid_cron";
|
||||
/// IANA 时区非法。
|
||||
pub const INVALID_TIMEZONE: &str = "backup.invalid_timezone";
|
||||
/// 记录不存在。
|
||||
pub const RECORD_NOT_FOUND: &str = "backup.record_not_found";
|
||||
/// 已存在活跃任务,拒绝创建。
|
||||
pub const ACTIVE_TASK_CONFLICT: &str = "backup.active_task_conflict";
|
||||
/// 记录状态与期望不符,无法迁移。
|
||||
pub const STATE_CONFLICT: &str = "backup.state_conflict";
|
||||
/// 已有记录时禁止切换存储目标。
|
||||
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
|
||||
/// `pg_dump` 非零退出或失败。
|
||||
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
|
||||
/// 暂存磁盘空间不足。
|
||||
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
|
||||
/// S3 认证失败。
|
||||
pub const S3_AUTH_FAILED: &str = "backup.s3_auth_failed";
|
||||
/// S3 权限拒绝。
|
||||
pub const S3_PERMISSION_DENIED: &str = "backup.s3_permission_denied";
|
||||
/// S3 上传失败。
|
||||
pub const S3_UPLOAD_FAILED: &str = "backup.s3_upload_failed";
|
||||
/// 远端对象校验失败。
|
||||
pub const REMOTE_VERIFICATION_FAILED: &str = "backup.remote_verification_failed";
|
||||
/// 任务被取消。
|
||||
pub const CANCELLED: &str = "backup.cancelled";
|
||||
/// 仓储或对象存储暂不可用。
|
||||
pub const STORE_UNAVAILABLE: &str = "backup.store_unavailable";
|
||||
/// 上游 S3 服务返回无效或失败响应。
|
||||
pub const UPSTREAM_FAILURE: &str = "backup.upstream_failure";
|
||||
}
|
||||
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`。
|
||||
///
|
||||
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
|
||||
/// 输出统一使用 `/` 分隔且不含空段。
|
||||
pub fn build_object_key(
|
||||
prefix: &str,
|
||||
backup_id: &str,
|
||||
at: DateTime<Utc>,
|
||||
) -> Result<String, BackupError> {
|
||||
let prefix = normalize_prefix(prefix)?;
|
||||
Ok(format!(
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
|
||||
at.format("%Y"),
|
||||
at.format("%m"),
|
||||
at.format("%d"),
|
||||
at.format("%Y%m%d_%H%M%S"),
|
||||
short_backup_id(backup_id),
|
||||
))
|
||||
}
|
||||
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致。
|
||||
#[must_use]
|
||||
pub fn build_download_file_name(backup_id: &str) -> String {
|
||||
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
|
||||
}
|
||||
|
||||
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex。
|
||||
#[must_use]
|
||||
fn short_backup_id(backup_id: &str) -> String {
|
||||
let hex = backup_id.strip_prefix("backup_").unwrap_or(backup_id);
|
||||
hex.get(..8).unwrap_or(hex).to_owned()
|
||||
}
|
||||
|
||||
/// 规范化并校验对象 prefix。
|
||||
///
|
||||
/// 返回去掉尾部 `/` 的规范 prefix;空串、前导 `/`、`..` 段、反斜杠和控制字符均拒绝。
|
||||
pub fn normalize_prefix(prefix: &str) -> Result<String, BackupError> {
|
||||
if prefix.is_empty() || prefix.starts_with('/') {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀必须非空且不能以前导 / 开头".to_owned(),
|
||||
));
|
||||
}
|
||||
if prefix.bytes().any(|byte| byte.is_ascii_control()) || prefix.contains('\\') {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀包含非法字符".to_owned(),
|
||||
));
|
||||
}
|
||||
if prefix.split('/').any(|segment| segment == "..") {
|
||||
return Err(BackupError::new(
|
||||
code::INVALID_CONFIG,
|
||||
"存储对象前缀不能包含 .. 路径段".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(prefix.trim_end_matches('/').to_owned())
|
||||
}
|
||||
|
||||
/// 生成新的备份记录 id。
|
||||
#[must_use]
|
||||
pub fn new_backup_id() -> String {
|
||||
format!("backup_{}", Uuid::now_v7().simple())
|
||||
}
|
||||
|
||||
/// 从触发类型与计划时间构造可持久化 seed;id 与对象 key 一次生成保持一致。
|
||||
pub fn build_backup_seed(
|
||||
trigger_kind: BackupTriggerKind,
|
||||
scheduled_at: Option<DateTime<Utc>>,
|
||||
prefix: &str,
|
||||
at: DateTime<Utc>,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecordSeed, BackupError> {
|
||||
let id = new_backup_id();
|
||||
let object_key = build_object_key(prefix, &id, at)?;
|
||||
Ok(BackupRecordSeed {
|
||||
id,
|
||||
trigger_kind,
|
||||
scheduled_at,
|
||||
object_key,
|
||||
expires_at,
|
||||
})
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroU64};
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod provider_credentials;
|
||||
@@ -50,6 +51,11 @@ impl AdminError {
|
||||
Self::new(AdminErrorKind::Invalid, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn bad_request(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::Invalid, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn not_found(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::NotFound, message)
|
||||
@@ -60,6 +66,11 @@ impl AdminError {
|
||||
Self::new(AdminErrorKind::Conflict, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn bad_gateway(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::BadGateway, message)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn unavailable(message: impl Into<String>) -> Self {
|
||||
Self::new(AdminErrorKind::Unavailable, message)
|
||||
|
||||
@@ -204,42 +204,22 @@ impl DecimalAmount {
|
||||
/// 精确相加两个 `numeric(20,10)` 金额。
|
||||
#[must_use]
|
||||
pub fn checked_add(&self, other: &Self) -> Option<Self> {
|
||||
let sum = scaled_amount(&self.0)?.checked_add(scaled_amount(&other.0)?)?;
|
||||
Self::from_scaled(sum)
|
||||
Self::from_decimal(self.to_decimal()?.checked_add(other.to_decimal()?)?)
|
||||
}
|
||||
|
||||
/// 将金额按非零请求数均分,保留最多十位小数。
|
||||
#[must_use]
|
||||
pub fn checked_div_u64(&self, divisor: u64) -> Option<Self> {
|
||||
let divisor = u128::from(divisor);
|
||||
(divisor != 0)
|
||||
.then(|| scaled_amount(&self.0)?.checked_div(divisor))
|
||||
.flatten()
|
||||
.and_then(Self::from_scaled)
|
||||
Self::from_decimal(self.to_decimal()?.checked_div_u64(divisor)?)
|
||||
}
|
||||
|
||||
fn from_scaled(value: u128) -> Option<Self> {
|
||||
if value >= 10_u128.pow(20) {
|
||||
return None;
|
||||
}
|
||||
let whole = value / 10_u128.pow(10);
|
||||
let fraction = value % 10_u128.pow(10);
|
||||
let value = if fraction == 0 {
|
||||
whole.to_string()
|
||||
} else {
|
||||
format!("{whole}.{fraction:010}")
|
||||
.trim_end_matches('0')
|
||||
.to_owned()
|
||||
};
|
||||
Some(Self(value))
|
||||
fn to_decimal(&self) -> Option<gateway_core::accounting::Decimal> {
|
||||
self.0.parse().ok()
|
||||
}
|
||||
}
|
||||
|
||||
fn scaled_amount(value: &str) -> Option<u128> {
|
||||
let (whole, fraction) = value.split_once('.').unwrap_or((value, ""));
|
||||
let whole = whole.parse::<u128>().ok()?;
|
||||
let fraction = format!("{fraction:0<10}").parse::<u128>().ok()?;
|
||||
whole.checked_mul(10_u128.pow(10))?.checked_add(fraction)
|
||||
fn from_decimal(value: gateway_core::accounting::Decimal) -> Option<Self> {
|
||||
Some(Self(value.canonical()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for DecimalAmount {
|
||||
@@ -330,6 +310,7 @@ pub struct LatencyPercentiles {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderBillingInput {
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -345,6 +326,7 @@ pub struct UsageCalculatedBillingFact {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub provider_kind: String,
|
||||
pub upstream_model_id: String,
|
||||
pub service_tier: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
@@ -458,13 +440,20 @@ pub struct RequestMetricPoint {
|
||||
}
|
||||
|
||||
/// 账号池容量统计。
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
@@ -542,12 +531,23 @@ pub struct DashboardRuntimeSlots {
|
||||
pub used_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的公共观测事实。
|
||||
/// 仪表盘卡片脚注展示的全历史累计。
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的当前区间事实及全历史累计。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: AccountPoolMetrics,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub account_usage: Vec<DashboardAccountUsage>,
|
||||
@@ -574,6 +574,7 @@ pub struct UsageRecord {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;管理领域保持不透明。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
@@ -642,6 +643,9 @@ pub struct UsageAttempt {
|
||||
pub operation: String,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub upstream_send_state: Option<String>,
|
||||
@@ -702,6 +706,7 @@ pub struct UsageSummary {
|
||||
/// 单个诊断维度值的聚合结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DiagnosticObservation {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -709,6 +714,7 @@ pub struct DiagnosticObservation {
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub cost_coverage: CostCoverage,
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
}
|
||||
@@ -723,8 +729,12 @@ pub struct OpsError {
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub component: String,
|
||||
pub operation: String,
|
||||
pub endpoint: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub provider_account_email: Option<String>,
|
||||
pub provider_account_authentication_kind: Option<String>,
|
||||
pub upstream_model_id: Option<String>,
|
||||
pub upstream_transport: Option<String>,
|
||||
pub failure_kind: String,
|
||||
@@ -1018,6 +1028,7 @@ pub struct UsageInsights {
|
||||
/// 已完成分母计算的诊断项。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DiagnosticsItem {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
@@ -1025,6 +1036,7 @@ pub struct DiagnosticsItem {
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<DecimalAmount>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
|
||||
@@ -5,13 +5,16 @@ use std::fmt;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId},
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
};
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{
|
||||
AccountAvailability, AccountErrorReason, AccountRecord, AccountStatus, AccountSummary,
|
||||
AccountUsage,
|
||||
},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -143,7 +146,7 @@ pub struct PreparedCredentialCreate {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -153,8 +156,6 @@ pub struct PreparedCredentialCreate {
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_reason: Option<String>,
|
||||
pub cooldown_until: Option<DateTime<Utc>>,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -171,13 +172,6 @@ pub struct CredentialImportCommit {
|
||||
pub prepared: PreparedCredentialImport,
|
||||
}
|
||||
|
||||
/// 可选的重新授权目标。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ReauthorizationTarget {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub credential_revision: Revision,
|
||||
}
|
||||
|
||||
/// OAuth pending owner 的中立身份;不编码具体 Provider 的 Redis key 或 JSON。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationOwner {
|
||||
@@ -238,15 +232,13 @@ impl fmt::Debug for AuthorizationOwnerBinding {
|
||||
}
|
||||
|
||||
/// OAuth 完成时应创建新账号还是 CAS 更新既有 credential。
|
||||
///
|
||||
/// 重新授权只绑定稳定的账号身份:credential revision 由服务端在临近写入时读取,
|
||||
/// 长流程期间的后台刷新不得让恢复操作失效。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AuthorizationMutationTarget {
|
||||
Create {
|
||||
name: String,
|
||||
},
|
||||
Reauthorize {
|
||||
account_id: ProviderAccountId,
|
||||
expected_credential_revision: Revision,
|
||||
},
|
||||
Create { name: String },
|
||||
Reauthorize { account_id: ProviderAccountId },
|
||||
}
|
||||
|
||||
/// 必须完整进入 Provider opaque pending payload、并在 complete 后原样恢复的事务信封。
|
||||
@@ -292,7 +284,7 @@ impl PendingAuthorizationMutation {
|
||||
pub struct StartAuthorization {
|
||||
pub context: MutationContext,
|
||||
pub name: String,
|
||||
pub reauthorization: Option<ReauthorizationTarget>,
|
||||
pub reauthorization: Option<ProviderAccountId>,
|
||||
}
|
||||
|
||||
/// OAuth 流程启动结果。
|
||||
@@ -480,14 +472,12 @@ pub struct CredentialDeletionResult {
|
||||
/// Provider-owned token 轮换命令。
|
||||
pub struct RotateCredential {
|
||||
pub mutation: CredentialMutation,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
/// Provider 校验手工轮换材料时所需的非事务输入。
|
||||
pub struct PrepareCredentialRotation {
|
||||
pub account: AccountRecord,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub provider_material: ProviderDocument,
|
||||
}
|
||||
|
||||
@@ -496,10 +486,6 @@ impl fmt::Debug for PrepareCredentialRotation {
|
||||
formatter
|
||||
.debug_struct("PrepareCredentialRotation")
|
||||
.field("account", &self.account)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
@@ -511,6 +497,7 @@ pub struct PreparedCredentialRotationFacts {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub expected_credential_revision: Revision,
|
||||
pub replacement_identity: Option<ProviderAccountIdentity>,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
@@ -572,15 +559,20 @@ impl fmt::Debug for RotateCredential {
|
||||
formatter
|
||||
.debug_struct("RotateCredential")
|
||||
.field("mutation", &self.mutation)
|
||||
.field(
|
||||
"expected_credential_revision",
|
||||
&self.expected_credential_revision,
|
||||
)
|
||||
.field("provider_material", &self.provider_material)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用账号用量能否可靠归属到一个 Provider quota 窗口。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum QuotaLocalUsageAttribution {
|
||||
/// 窗口覆盖账号的全部请求,可按账号与时间范围聚合。
|
||||
AccountWide,
|
||||
/// 窗口需要 Provider / 模型级归属,通用账号聚合不可用。
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
/// 一个 Provider quota 窗口的公共投影。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderQuotaWindow {
|
||||
@@ -588,9 +580,11 @@ pub struct ProviderQuotaWindow {
|
||||
pub group: String,
|
||||
pub label: String,
|
||||
pub source: Option<String>,
|
||||
pub local_usage_attribution: QuotaLocalUsageAttribution,
|
||||
pub window_seconds: Option<u64>,
|
||||
pub used_percent: Option<f64>,
|
||||
pub reset_at: Option<DateTime<Utc>>,
|
||||
pub limit_reached: bool,
|
||||
pub local_usage: Option<AccountUsage>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
@@ -609,6 +603,11 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间;`Some` 时账号处于临时限流,到期自动恢复。
|
||||
/// 与 `limit_reached`(配额耗尽)是独立维度,不污染额度窗口。
|
||||
pub rate_limited_until: Option<DateTime<Utc>>,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
@@ -705,6 +704,8 @@ impl fmt::Debug for ProviderExport {
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
/// `status == Error` 时的具体原因;其余状态为 `None`。
|
||||
pub error_reason: Option<AccountErrorReason>,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -11,14 +11,8 @@ use super::Revision;
|
||||
/// 客户端模型到上游模型的全局精确映射。
|
||||
pub type ModelMappings = BTreeMap<PublicModelId, UpstreamModelId>;
|
||||
|
||||
/// 账号调度策略。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RotationStrategy {
|
||||
Smart,
|
||||
QuotaResetPriority,
|
||||
RoundRobin,
|
||||
Sticky,
|
||||
}
|
||||
/// 账号调度策略;由 Core 拥有稳定值与 wire 映射。
|
||||
pub use gateway_core::engine::credential::RotationStrategy;
|
||||
|
||||
/// 完整运行设置事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
//! 备份控制面所需的持久化与基础设施能力。
|
||||
//!
|
||||
//! 端口方法只接受/返回领域类型与既有错误体系:`BackupRepository` 复用
|
||||
//! `AdminStoreError`;`DatabaseDumpPort` 与 `BackupObjectStorePort` 返回只携带稳定
|
||||
//! 错误码的 [`BackupError`]。`gateway-store` 实现全部端口;`gateway-admin` 拥有
|
||||
//! 状态机、Cron、保留策略与 BackupTask 算法。
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::model::Revision;
|
||||
use crate::model::{
|
||||
MutationContext,
|
||||
backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
|
||||
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, ConnectionTestResult,
|
||||
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
},
|
||||
};
|
||||
use crate::ports::store::AdminStoreResult;
|
||||
|
||||
/// Worker 状态迁移携带的字段更新;`None` 字段不写入。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct StatusTransitionUpdate {
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 备份配置、任务与计划游标的 PostgreSQL 仓储端口。
|
||||
#[async_trait]
|
||||
pub trait BackupRepository: Send + Sync {
|
||||
/// 读取完整设置(含 Secret)。
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings>;
|
||||
|
||||
/// 同事务更新 S3 存储配置、递增 revision、清空验证状态并写审计事件。
|
||||
///
|
||||
/// 已有记录时禁止切换 endpoint/region/bucket/path-style(存储身份锁定)。
|
||||
/// 返回更新后的设置与新的全局 `config_revision`。
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
command: UpdateBackupStorageCommand,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)>;
|
||||
|
||||
/// 同事务更新调度配置、推进计划游标并写审计事件。
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
next_run_at: Option<DateTime<Utc>>,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings>;
|
||||
|
||||
/// CAS 记录探针成功;`storage_revision` 与当前配置不一致时丢弃结果(返回 `false`)。
|
||||
async fn record_verification(
|
||||
&self,
|
||||
storage_revision: u64,
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 插入 queued 任务;存在活跃任务时返回 `backup.active_task_conflict`。
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord>;
|
||||
|
||||
/// 幂等插入 scheduled 任务;同一时间点或活跃冲突时返回 `false`(调用方推进游标即可)。
|
||||
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 分页查询记录。
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage>;
|
||||
|
||||
/// 读取单条记录。
|
||||
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 读取中间状态记录(dumping/uploading),供启动恢复。
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 读取等待完成墓碑的记录(status = deleting)。
|
||||
async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 读取 `expires_at` 已到期且非活跃的记录(completed/failed),供保留清理。
|
||||
async fn list_expired_records(&self, limit: u32) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
|
||||
/// 原子领取最旧的 queued 任务并迁移到 dumping;无任务返回 `None`。
|
||||
async fn claim_next_queued(&self, now: DateTime<Utc>)
|
||||
-> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 条件迁移状态:要求当前状态一致;不匹配返回 `None`。
|
||||
async fn transition_status(
|
||||
&self,
|
||||
id: &str,
|
||||
from: BackupStatus,
|
||||
to: BackupStatus,
|
||||
update: StatusTransitionUpdate,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 迁移到删除流程(completed/failed → deleting);供管理员与保留策略调用。
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 远端删除成功后硬删除记录行。
|
||||
async fn delete_record(&self, id: &str) -> AdminStoreResult<()>;
|
||||
|
||||
/// 条件推进计划游标;仅当计划表达式和时区仍是 `expected_*` 时才写入,返回是否推进。
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
next_run_at: DateTime<Utc>,
|
||||
expected_cron: &str,
|
||||
expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 按完成时间倒序读取 completed 计划备份,供保留策略扫描(最多 `limit` 条)。
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>>;
|
||||
}
|
||||
|
||||
/// `pg_dump` 导出与本地暂存端口。
|
||||
#[async_trait]
|
||||
pub trait DatabaseDumpPort: Send + Sync {
|
||||
/// 创建归档;完成后原子落盘。取消时终止子进程并清理部分文件。
|
||||
async fn dump(&self, request: DumpRequest) -> Result<DumpArtifact, BackupError>;
|
||||
|
||||
/// 检查暂存目录是否存在完整归档。
|
||||
async fn inspect_staging(&self, backup_id: &str)
|
||||
-> Result<Option<StagedArtifact>, BackupError>;
|
||||
|
||||
/// 清理暂存目录中该备份的全部文件。
|
||||
async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError>;
|
||||
}
|
||||
|
||||
/// 一次 `pg_dump` 请求。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DumpRequest {
|
||||
pub backup_id: String,
|
||||
pub cancellation: gateway_core::engine::CancellationToken,
|
||||
}
|
||||
|
||||
/// 已完成并落盘的归档事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DumpArtifact {
|
||||
pub path: PathBuf,
|
||||
pub size_bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
/// 暂存目录中已存在的完整归档。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StagedArtifact {
|
||||
pub path: PathBuf,
|
||||
pub size_bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
/// S3 兼容对象存储适配器端口。
|
||||
///
|
||||
/// 方法接受已经校验的存储配置;SDK client 的构造与缓存属于 adapter 实现细节。
|
||||
#[async_trait]
|
||||
pub trait BackupObjectStorePort: Send + Sync {
|
||||
/// 对专用探针对象执行 Put/Head/Get/Delete。
|
||||
async fn test_connection(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, BackupError>;
|
||||
|
||||
/// 分片流式上传本地归档;取消时中止 multipart 并返回 `backup.cancelled`。
|
||||
async fn upload_file(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), BackupError>;
|
||||
|
||||
/// 读取对象大小与 metadata;对象不存在返回 `Ok(None)`。
|
||||
async fn head_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, BackupError>;
|
||||
|
||||
/// 删除对象;对象不存在视为成功。
|
||||
async fn delete_object(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), BackupError>;
|
||||
|
||||
/// 创建短时预签名下载 URL。
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
file_name: &str,
|
||||
ttl: Duration,
|
||||
) -> Result<String, BackupError>;
|
||||
}
|
||||
|
||||
/// 一次对象上传请求。
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UploadObjectRequest {
|
||||
pub object_key: String,
|
||||
pub source: PathBuf,
|
||||
pub metadata: BackupObjectMetadata,
|
||||
pub cancellation: gateway_core::engine::CancellationToken,
|
||||
}
|
||||
|
||||
/// 备份控制面所需的全部 Store 能力集合。
|
||||
#[derive(Clone)]
|
||||
pub struct BackupStorePorts {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
}
|
||||
|
||||
impl BackupStorePorts {
|
||||
/// 组合备份端口。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
dump: Arc<dyn DatabaseDumpPort>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
dump,
|
||||
object_store,
|
||||
}
|
||||
}
|
||||
|
||||
/// 备份任务/配置仓储。
|
||||
#[must_use]
|
||||
pub fn repository(&self) -> Arc<dyn BackupRepository> {
|
||||
self.repository.clone()
|
||||
}
|
||||
|
||||
/// `pg_dump` 导出与暂存。
|
||||
#[must_use]
|
||||
pub fn dump(&self) -> Arc<dyn DatabaseDumpPort> {
|
||||
self.dump.clone()
|
||||
}
|
||||
|
||||
/// S3/R2 对象存储适配器。
|
||||
#[must_use]
|
||||
pub fn object_store(&self) -> Arc<dyn BackupObjectStorePort> {
|
||||
self.object_store.clone()
|
||||
}
|
||||
|
||||
/// 测试与不可用路径使用的空实现;任何调用都返回不可用错误。
|
||||
#[doc(hidden)]
|
||||
#[must_use]
|
||||
pub fn disabled() -> Self {
|
||||
Self {
|
||||
repository: Arc::new(UnavailableBackupRepository),
|
||||
dump: Arc::new(UnavailableDumpPort),
|
||||
object_store: Arc::new(UnavailableObjectStore),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableBackupRepository;
|
||||
|
||||
#[async_trait]
|
||||
impl BackupRepository for UnavailableBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
_command: UpdateBackupStorageCommand,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
_command: UpdateBackupScheduleCommand,
|
||||
_next_run_at: Option<DateTime<Utc>>,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn record_verification(
|
||||
&self,
|
||||
_storage_revision: u64,
|
||||
_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn insert_backup_record(
|
||||
&self,
|
||||
_seed: BackupRecordSeed,
|
||||
) -> AdminStoreResult<BackupRecord> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn insert_scheduled_record(&self, _seed: BackupRecordSeed) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
_query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn load_backup_record(&self, _id: &str) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn claim_next_queued(
|
||||
&self,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn transition_status(
|
||||
&self,
|
||||
_id: &str,
|
||||
_from: BackupStatus,
|
||||
_to: BackupStatus,
|
||||
_update: StatusTransitionUpdate,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
_id: &str,
|
||||
_now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn delete_record(&self, _id: &str) -> AdminStoreResult<()> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
_next_run_at: DateTime<Utc>,
|
||||
_expected_cron: &str,
|
||||
_expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableDumpPort;
|
||||
|
||||
#[async_trait]
|
||||
impl DatabaseDumpPort for UnavailableDumpPort {
|
||||
async fn dump(&self, _request: DumpRequest) -> Result<DumpArtifact, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn inspect_staging(
|
||||
&self,
|
||||
_backup_id: &str,
|
||||
) -> Result<Option<StagedArtifact>, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn cleanup_staging(&self, _backup_id: &str) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
}
|
||||
|
||||
struct UnavailableObjectStore;
|
||||
|
||||
#[async_trait]
|
||||
impl BackupObjectStorePort for UnavailableObjectStore {
|
||||
async fn test_connection(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn upload_file(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_request: UploadObjectRequest,
|
||||
) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn head_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn delete_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
) -> Result<(), BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
_object_key: &str,
|
||||
_file_name: &str,
|
||||
_ttl: std::time::Duration,
|
||||
) -> Result<String, BackupError> {
|
||||
Err(disabled_infra())
|
||||
}
|
||||
}
|
||||
|
||||
fn disabled_store() -> crate::ports::store::AdminStoreError {
|
||||
crate::ports::store::AdminStoreError::new(
|
||||
crate::ports::store::AdminStoreErrorKind::Unavailable,
|
||||
"backup",
|
||||
"backup store is disabled in this configuration",
|
||||
)
|
||||
}
|
||||
|
||||
fn disabled_infra() -> BackupError {
|
||||
BackupError::new(
|
||||
crate::model::backup::code::STORE_UNAVAILABLE,
|
||||
"backup infrastructure is disabled in this configuration".to_owned(),
|
||||
)
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 管理用例依赖的外部能力端口。
|
||||
|
||||
pub mod backup;
|
||||
pub mod provider;
|
||||
pub mod store;
|
||||
pub mod system;
|
||||
|
||||
@@ -64,6 +64,12 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
/// 成功之后,不参与事务成败,也不得恢复或改写已经提交的账号状态。
|
||||
async fn account_unavailable(&self, account_id: &ProviderAccountId);
|
||||
|
||||
/// 账号资格事实已经由控制面提交,失效 Provider 持有的可重建派生状态。
|
||||
///
|
||||
/// 该通知发生在 Store 事务成功之后、下一份 RuntimeSnapshot 编译之前;通知
|
||||
/// 不参与已提交事务成败。没有账号派生状态的 Provider 可使用默认空实现。
|
||||
async fn account_facts_changed(&self, _account_ids: &[ProviderAccountId]) {}
|
||||
|
||||
/// 生成一次连接测试所需的 Provider-owned operation;Core 负责实际执行与落账。
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
|
||||
@@ -7,6 +7,7 @@ use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
MutationContext, Revision,
|
||||
accounts::{
|
||||
@@ -300,6 +301,7 @@ pub struct AdminStorePorts {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
}
|
||||
|
||||
impl AdminStorePorts {
|
||||
@@ -310,6 +312,7 @@ impl AdminStorePorts {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
@@ -317,6 +320,7 @@ impl AdminStorePorts {
|
||||
client_keys,
|
||||
observability,
|
||||
settings,
|
||||
backup,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,4 +348,9 @@ impl AdminStorePorts {
|
||||
pub fn settings(&self) -> Arc<dyn SettingsStore> {
|
||||
self.settings.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn backup(&self) -> BackupStorePorts {
|
||||
self.backup.clone()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! 统一账号目录与跨 Provider 动态分派。
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::SystemTime};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
@@ -18,13 +18,14 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountRecord, AccountStatus, AccountStatusSignals, AccountUsageWindowQuery,
|
||||
derive_error_reason,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
PrepareCredentialRefresh, ProviderModels, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderQuotaWindow,
|
||||
ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -159,7 +160,9 @@ impl DefaultAccountsService {
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (account, quota) in accounts.iter().zip(quotas) {
|
||||
for window in &mut quota.windows {
|
||||
if window.local_usage.is_none() {
|
||||
if window.local_usage.is_none()
|
||||
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
let key = (account.id.clone(), window.key.clone());
|
||||
if let Some(usage) = usage_by_window.get(&key) {
|
||||
window.local_usage = Some(usage.clone());
|
||||
@@ -216,8 +219,11 @@ impl DefaultAccountsService {
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
quota,
|
||||
@@ -260,18 +266,37 @@ impl AccountsService for DefaultAccountsService {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
provider
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
// 该账号降级为空额度投影,其余账号与页面状态照常返回。
|
||||
let provider = match self.providers.require(&account.provider_kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Ok(empty_quota());
|
||||
}
|
||||
};
|
||||
match provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: rolling_usage.get(&account.id).cloned(),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))
|
||||
{
|
||||
Ok(quota) => Ok(quota),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
Ok(empty_quota())
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
@@ -282,11 +307,16 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(account, quota)| {
|
||||
let signals = account_signals(&account, "a, now);
|
||||
let error_reason = derive_error_reason(&signals);
|
||||
AccountDirectoryItem {
|
||||
status: signals.derive(),
|
||||
error_reason,
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -370,6 +400,9 @@ impl AccountsService for DefaultAccountsService {
|
||||
"provider credential refresh",
|
||||
)
|
||||
.await?;
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
let account = self.load_directory_item(&result.account_id, false).await?;
|
||||
Ok(AccountRefreshResult {
|
||||
@@ -404,7 +437,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, Utc::now());
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -438,7 +471,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, Utc::now())
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
@@ -449,12 +482,33 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_status: status,
|
||||
}))
|
||||
.collect(),
|
||||
Err(error) => vec![AccountConnectionTestEvent::Failed {
|
||||
message: error.client_message().to_owned(),
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
account_status: status,
|
||||
}],
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
.upstream_response()
|
||||
.map(gateway_core::engine::probe::AccountProbeUpstreamResponse::status);
|
||||
let upstream_content_type = error
|
||||
.upstream_response()
|
||||
.and_then(|response| response.content_type())
|
||||
.and_then(|value| std::str::from_utf8(value).ok())
|
||||
.map(ToOwned::to_owned);
|
||||
let upstream_body = error
|
||||
.upstream_response()
|
||||
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
|
||||
let message = upstream_body
|
||||
.as_deref()
|
||||
.filter(|body| !body.is_empty())
|
||||
.unwrap_or_else(|| error.client_message())
|
||||
.to_owned();
|
||||
vec![AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
provider_error_type: error.client_error_type().map(ToOwned::to_owned),
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
.flat_map(futures::stream::iter);
|
||||
@@ -462,28 +516,48 @@ impl AccountsService for DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
fn account_status(account: &AccountRecord, now: chrono::DateTime<Utc>) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else {
|
||||
match account.availability {
|
||||
crate::model::accounts::AccountAvailability::Banned => AccountStatus::Banned,
|
||||
crate::model::accounts::AccountAvailability::QuotaExhausted => {
|
||||
AccountStatus::QuotaExhausted
|
||||
}
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
| crate::model::accounts::AccountAvailability::Invalid => AccountStatus::Expired,
|
||||
_ if account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now) =>
|
||||
{
|
||||
AccountStatus::Expired
|
||||
}
|
||||
_ => AccountStatus::Active,
|
||||
}
|
||||
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造状态派生所需的信号快照。
|
||||
fn account_signals(
|
||||
account: &AccountRecord,
|
||||
quota: &ProviderQuota,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatusSignals {
|
||||
AccountStatusSignals {
|
||||
enabled: account.enabled,
|
||||
availability: account.availability,
|
||||
access_token_expired: account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now),
|
||||
quota_limit_reached: quota.limit_reached,
|
||||
rate_limited_until: quota.rate_limited_until.map(SystemTime::from),
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号运营分类(互斥展示状态)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
let quota = ProviderQuota {
|
||||
limit_reached: quota_limit_reached,
|
||||
..empty_quota()
|
||||
};
|
||||
account_signals(account, "a, now).derive()
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
@@ -496,7 +570,9 @@ fn quota_usage_window(
|
||||
account_id: &str,
|
||||
window: &ProviderQuotaWindow,
|
||||
) -> Option<AccountUsageWindowQuery> {
|
||||
if window.local_usage.is_some() {
|
||||
if window.local_usage.is_some()
|
||||
|| window.local_usage_attribution != QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let reset_at = window.reset_at?;
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
//! 备份配置、手动创建、下载与删除用例。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::routing::snapshot::SnapshotControl;
|
||||
use secrecy::ExposeSecret as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind, MutationContext,
|
||||
auth::{AdminAuditEvent, AuditActorKind},
|
||||
backup::{
|
||||
BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings,
|
||||
BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult,
|
||||
DownloadUrlResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
build_backup_seed, code,
|
||||
},
|
||||
};
|
||||
use crate::ports::{
|
||||
backup::{BackupObjectStorePort, BackupRepository},
|
||||
store::AuthStore,
|
||||
};
|
||||
|
||||
use super::map_store_error;
|
||||
|
||||
/// 默认下载地址有效期。
|
||||
const DOWNLOAD_TTL: Duration = Duration::from_secs(5 * 60);
|
||||
|
||||
/// API 消费的备份管理服务。
|
||||
#[async_trait]
|
||||
pub trait BackupService: Send + Sync {
|
||||
async fn load_settings(&self) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn update_storage(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupStorageCommand,
|
||||
) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn update_schedule(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
) -> Result<BackupSettings, AdminError>;
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<ConnectionTestResult, AdminError>;
|
||||
|
||||
async fn create_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecord, AdminError>;
|
||||
|
||||
async fn list_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> Result<BackupRecordPage, AdminError>;
|
||||
|
||||
async fn download_url(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<DownloadUrlResult, AdminError>;
|
||||
|
||||
async fn delete_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<BackupRecord, AdminError>;
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultBackupService {
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
}
|
||||
|
||||
impl DefaultBackupService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
repository: Arc<dyn BackupRepository>,
|
||||
object_store: Arc<dyn BackupObjectStorePort>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Self {
|
||||
Self {
|
||||
repository,
|
||||
object_store,
|
||||
auth,
|
||||
snapshot,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupService for DefaultBackupService {
|
||||
async fn load_settings(&self) -> Result<BackupSettings, AdminError> {
|
||||
self.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))
|
||||
}
|
||||
|
||||
async fn update_storage(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupStorageCommand,
|
||||
) -> Result<BackupSettings, AdminError> {
|
||||
validate_storage(&command)?;
|
||||
let (settings, revision) = self
|
||||
.repository
|
||||
.update_storage_settings(command, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup storage"))?;
|
||||
super::publish_committed(self.snapshot.as_ref(), revision).await?;
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
async fn update_schedule(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
) -> Result<BackupSettings, AdminError> {
|
||||
validate_schedule(&command)?;
|
||||
let next_run_at = if command.schedule_enabled {
|
||||
Some(
|
||||
crate::backup::policy::BackupSchedule::parse(
|
||||
&command.cron_expression,
|
||||
&command.schedule_timezone,
|
||||
)
|
||||
.map_err(map_backup_error)?
|
||||
.next_after(Utc::now())
|
||||
.ok_or_else(|| AdminError::bad_request("无法从计划推导下次执行时间"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
self.repository
|
||||
.update_schedule_settings(command, next_run_at, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup schedule"))
|
||||
}
|
||||
|
||||
async fn test_connection(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
) -> Result<ConnectionTestResult, AdminError> {
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Ok(ConnectionTestResult {
|
||||
ok: false,
|
||||
stage: "putObject",
|
||||
code: Some(code::INVALID_CONFIG),
|
||||
message: "备份存储配置不完整,无法执行连接测试".to_owned(),
|
||||
});
|
||||
};
|
||||
let result = self
|
||||
.object_store
|
||||
.test_connection(&config)
|
||||
.await
|
||||
.map_err(map_backup_error)?;
|
||||
if result.ok {
|
||||
let _ = self
|
||||
.repository
|
||||
.record_verification(config.storage_revision, Utc::now())
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
}
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.s3_connection_tested",
|
||||
"backup_settings",
|
||||
vec![],
|
||||
)
|
||||
.await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn create_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
expires_at: Option<DateTime<Utc>>,
|
||||
) -> Result<BackupRecord, AdminError> {
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
if !settings.storage_configured() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储配置不完整,请先保存存储配置",
|
||||
));
|
||||
}
|
||||
if !settings.storage_verified() {
|
||||
return Err(AdminError::bad_request(
|
||||
"备份存储尚未通过连接测试,请先测试存储配置",
|
||||
));
|
||||
}
|
||||
let prefix = settings.prefix.as_deref().unwrap_or_default();
|
||||
let seed = build_backup_seed(
|
||||
BackupTriggerKind::Manual,
|
||||
None,
|
||||
prefix,
|
||||
Utc::now(),
|
||||
expires_at,
|
||||
)
|
||||
.map_err(map_backup_error)?;
|
||||
let record = self
|
||||
.repository
|
||||
.insert_backup_record(seed)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?;
|
||||
self.append_audit(context, "backup.created", "backup", vec![record.id.clone()])
|
||||
.await?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn list_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> Result<BackupRecordPage, AdminError> {
|
||||
self.repository
|
||||
.list_backup_records(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup records"))
|
||||
}
|
||||
|
||||
async fn download_url(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<DownloadUrlResult, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if record.status != BackupStatus::Completed {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 的备份才能创建下载地址",
|
||||
));
|
||||
}
|
||||
let settings = self
|
||||
.repository
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup settings"))?;
|
||||
let Some(config) = BackupStorageConfig::from_settings(&settings) else {
|
||||
return Err(AdminError::bad_request("备份存储配置不完整"));
|
||||
};
|
||||
let file_name = crate::model::backup::build_download_file_name(backup_id);
|
||||
let url = self
|
||||
.object_store
|
||||
.presigned_download(&config, &record.object_key, &file_name, DOWNLOAD_TTL)
|
||||
.await
|
||||
.map_err(map_backup_error)?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.download_url_created",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
)
|
||||
.await?;
|
||||
Ok(DownloadUrlResult {
|
||||
url,
|
||||
file_name,
|
||||
expires_in: DOWNLOAD_TTL,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_backup(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
backup_id: &str,
|
||||
) -> Result<BackupRecord, AdminError> {
|
||||
let record = self.require_record(backup_id).await?;
|
||||
if !record.status.can_be_deleted() {
|
||||
return Err(AdminError::conflict(
|
||||
"只有 completed 或 failed 的备份才能删除",
|
||||
));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let deleting = self
|
||||
.repository
|
||||
.transition_to_deleting(backup_id, now)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.delete_requested",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
)
|
||||
.await?;
|
||||
Ok(deleting)
|
||||
}
|
||||
}
|
||||
|
||||
impl DefaultBackupService {
|
||||
async fn require_record(&self, backup_id: &str) -> Result<BackupRecord, AdminError> {
|
||||
self.repository
|
||||
.load_backup_record(backup_id)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::not_found("备份记录不存在"))
|
||||
}
|
||||
|
||||
async fn append_audit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
entity_ref: &str,
|
||||
changed_fields: Vec<String>,
|
||||
) -> Result<(), AdminError> {
|
||||
let actor_kind = match context.actor {
|
||||
crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession,
|
||||
crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey,
|
||||
crate::model::MutationActor::System => AuditActorKind::System,
|
||||
};
|
||||
let actor_ref = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => admin_user_id.clone(),
|
||||
crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(),
|
||||
crate::model::MutationActor::System => "system".to_owned(),
|
||||
};
|
||||
let actor_admin_user_id = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => {
|
||||
Some(admin_user_id.clone())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let event = AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
actor_kind,
|
||||
actor_admin_user_id,
|
||||
actor_ref,
|
||||
request_id: Some(context.request_id.clone()),
|
||||
action: action.to_owned(),
|
||||
entity_kind: "backup".to_owned(),
|
||||
entity_ref: entity_ref.to_owned(),
|
||||
config_revision: None,
|
||||
changed_fields,
|
||||
occurred_at: Utc::now(),
|
||||
};
|
||||
self.auth
|
||||
.append_audit_event(event)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup audit"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 校验存储配置;空串视为非法,Secret 缺省表示保留旧值。
|
||||
fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminError> {
|
||||
validate_endpoint(&command.endpoint)?;
|
||||
validate_nonempty("region", &command.region)?;
|
||||
validate_nonempty("bucket", &command.bucket)?;
|
||||
validate_nonempty("accessKeyId", &command.access_key_id)?;
|
||||
if let Some(secret) = command.secret_access_key.as_ref() {
|
||||
let value = secret.expose_secret();
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request(
|
||||
"secretAccessKey 不能为空字符串;缺省表示保留旧值",
|
||||
));
|
||||
}
|
||||
validate_length("secretAccessKey", value, 512)?;
|
||||
}
|
||||
crate::model::backup::normalize_prefix(&command.prefix).map_err(map_backup_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_endpoint(value: &str) -> Result<(), AdminError> {
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request("endpoint 不能为空"));
|
||||
}
|
||||
if !value.starts_with("https://") {
|
||||
return Err(AdminError::bad_request(
|
||||
"endpoint 必须使用 HTTPS;仅测试或显式开发配置允许 HTTP",
|
||||
));
|
||||
}
|
||||
if value.contains('@') || value.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(AdminError::bad_request("endpoint 格式非法"));
|
||||
}
|
||||
validate_length("endpoint", value, 2048)
|
||||
}
|
||||
|
||||
fn validate_nonempty(field: &str, value: &str) -> Result<(), AdminError> {
|
||||
if value.is_empty() {
|
||||
return Err(AdminError::bad_request(format!("{field} 不能为空")));
|
||||
}
|
||||
validate_length(field, value, 255)
|
||||
}
|
||||
|
||||
fn validate_length(field: &str, value: &str, max: usize) -> Result<(), AdminError> {
|
||||
if value.len() > max || value.bytes().any(|byte| byte.is_ascii_control()) {
|
||||
return Err(AdminError::bad_request(format!(
|
||||
"{field} 超出长度或包含非法字符"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 校验调度配置并解析 Cron 与时区。
|
||||
fn validate_schedule(command: &UpdateBackupScheduleCommand) -> Result<(), AdminError> {
|
||||
if command.cron_expression.trim().is_empty() {
|
||||
return Err(AdminError::bad_request("cronExpression 不能为空"));
|
||||
}
|
||||
crate::backup::policy::BackupSchedule::parse(
|
||||
&command.cron_expression,
|
||||
&command.schedule_timezone,
|
||||
)
|
||||
.map_err(map_backup_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 备份基础设施错误到既有 `AdminErrorKind` 的唯一边界映射。
|
||||
fn map_backup_error(error: BackupError) -> AdminError {
|
||||
match error.code() {
|
||||
code::INVALID_CONFIG | code::INVALID_CRON | code::INVALID_TIMEZONE => {
|
||||
AdminError::bad_request(error.message().to_owned())
|
||||
}
|
||||
code::RECORD_NOT_FOUND => AdminError::not_found(error.message().to_owned()),
|
||||
code::ACTIVE_TASK_CONFLICT | code::STATE_CONFLICT | code::STORAGE_IDENTITY_LOCKED => {
|
||||
AdminError::conflict(error.message().to_owned())
|
||||
}
|
||||
code::STORE_UNAVAILABLE => {
|
||||
AdminError::new(AdminErrorKind::Unavailable, error.message().to_owned())
|
||||
}
|
||||
_ => AdminError::bad_gateway(error.message().to_owned()),
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod openai;
|
||||
@@ -90,17 +91,10 @@ async fn pending_authorization(
|
||||
resource: &'static str,
|
||||
) -> Result<PendingAuthorizationMutation, AdminError> {
|
||||
let target = match &command.reauthorization {
|
||||
Some(target) => {
|
||||
let details =
|
||||
required_credential(accounts, provider_kind, &target.account_id, resource).await?;
|
||||
if details.credential.credential_revision != target.credential_revision {
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} reauthorization target is stale"
|
||||
)));
|
||||
}
|
||||
Some(account_id) => {
|
||||
required_credential(accounts, provider_kind, account_id, resource).await?;
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id: target.account_id.clone(),
|
||||
expected_credential_revision: target.credential_revision,
|
||||
account_id: account_id.clone(),
|
||||
}
|
||||
}
|
||||
None => AuthorizationMutationTarget::Create {
|
||||
@@ -142,7 +136,6 @@ fn validate_prepared_rotation(
|
||||
let facts = prepared.facts();
|
||||
if facts.account_id.as_str() != account.id.as_str()
|
||||
|| facts.provider_kind != account.provider_kind
|
||||
|| facts.expected_credential_revision != account.credential_revision
|
||||
{
|
||||
return Err(AdminError::conflict(format!(
|
||||
"{resource} prepared facts do not match the current credential"
|
||||
@@ -179,16 +172,11 @@ async fn validate_authorization_commit(
|
||||
PreparedAuthorizationCredential::Create(credential),
|
||||
) => credential.provider_kind == *provider_kind,
|
||||
(
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
},
|
||||
AuthorizationMutationTarget::Reauthorize { account_id },
|
||||
PreparedAuthorizationCredential::Reauthorize(credential),
|
||||
) => {
|
||||
let facts = credential.facts();
|
||||
facts.provider_kind == *provider_kind
|
||||
&& facts.account_id == *account_id
|
||||
&& facts.expected_credential_revision == *expected_credential_revision
|
||||
facts.provider_kind == *provider_kind && facts.account_id == *account_id
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
@@ -320,6 +308,9 @@ async fn set_credential_enabled(
|
||||
if !enabled {
|
||||
provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
Ok(CredentialMutationResult {
|
||||
config_revision: revision,
|
||||
account_id,
|
||||
@@ -352,6 +343,7 @@ async fn delete_credentials(
|
||||
for account_id in &account_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
provider.account_facts_changed(&account_ids).await;
|
||||
Ok(CredentialDeletionResult {
|
||||
config_revision: revision,
|
||||
account_ids,
|
||||
|
||||
@@ -112,15 +112,10 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let today =
|
||||
dashboard_period_metrics(&observation.trend, today_start, observation.range.end);
|
||||
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
|
||||
let total_billing_usd = observation
|
||||
.attempts
|
||||
.costs
|
||||
.iter()
|
||||
.find(|cost| cost.currency == "USD")
|
||||
.map(|cost| cost.amount.clone());
|
||||
let total_billing_usd = observation.totals.billing_usd.clone();
|
||||
let total_cached_token_rate = rate_or_zero(
|
||||
observation.requests.cached_tokens,
|
||||
observation.requests.input_tokens,
|
||||
observation.totals.cached_tokens,
|
||||
observation.totals.input_tokens,
|
||||
);
|
||||
let average_first_token_latency_ms = average(
|
||||
observation.requests.first_token_latency_sum_ms,
|
||||
@@ -247,6 +242,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
items: items
|
||||
.into_iter()
|
||||
.map(|item| DiagnosticsItem {
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
request_count: item.request_count,
|
||||
success_count: item.request_count.saturating_sub(item.failure_count),
|
||||
@@ -254,6 +250,7 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
error_rate: rate_or_zero(item.failure_count, item.request_count),
|
||||
request_share: rate_or_zero(item.request_count, total_requests),
|
||||
average_latency_ms: item.average_latency_ms,
|
||||
latency_p95_ms: item.latency_p95_ms,
|
||||
estimated_cost: usd_cost(&item.costs),
|
||||
attempt_count: item.attempt_count,
|
||||
total_tokens: item.total_tokens,
|
||||
@@ -428,6 +425,7 @@ fn recover_standard_costs(
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: fact.upstream_model_id,
|
||||
service_tier: fact.service_tier,
|
||||
input_tokens: fact.input_tokens,
|
||||
output_tokens: fact.output_tokens,
|
||||
cached_tokens: fact.cached_tokens,
|
||||
@@ -491,14 +489,14 @@ impl DefaultObservabilityService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 逐条尽力把 calculated 总额升级为完整分解;单条脏数据(非法 Provider kind
|
||||
/// 或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
/// 逐条尽力把可校验的总额升级为完整分解;单条脏数据(非法 Provider kind、
|
||||
/// 不支持的来源或费用规则失败)只保留该条已存的总额,不影响整页返回。
|
||||
fn enrich_billing(&self, records: &mut [crate::model::observability::UsageRecord]) {
|
||||
for record in records {
|
||||
let Some(UsageBilling::Total { source, total }) = record.billing.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
if source != "calculated" {
|
||||
if !matches!(source.as_str(), "calculated" | "provider_reported") {
|
||||
continue;
|
||||
}
|
||||
let (Some(provider), Some(upstream_model_id)) = (
|
||||
@@ -512,6 +510,7 @@ impl DefaultObservabilityService {
|
||||
};
|
||||
let input = ProviderBillingInput {
|
||||
upstream_model_id: upstream_model_id.clone(),
|
||||
service_tier: record.service_tier.clone(),
|
||||
input_tokens: record.input_tokens,
|
||||
output_tokens: record.output_tokens,
|
||||
cached_tokens: record.cached_tokens,
|
||||
|
||||
@@ -13,8 +13,7 @@ use crate::{
|
||||
CredentialDeletionResult, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutation,
|
||||
CredentialMutationResult, CredentialPage, ImportCredentials, PrepareCredentialImport,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCredential, ProviderQuotaRequest,
|
||||
RotateCredential, StartAuthorization,
|
||||
PrepareCredentialRotation, ProviderQuotaRequest, RotateCredential, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{provider::ProviderAdmin, store::AccountStore},
|
||||
@@ -85,25 +84,29 @@ impl DefaultOpenAiService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
let account_ids = account_ids.to_vec();
|
||||
let request_id = request_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,10 +154,12 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.commit_credential_import(CredentialImportCommit { prepared }, &context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "OpenAI credential import"))?;
|
||||
self.provider
|
||||
.account_facts_changed(&result.credential_ids)
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
|
||||
.await;
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -192,10 +197,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI authorization",
|
||||
)
|
||||
.await?;
|
||||
let creates_account = matches!(
|
||||
&prepared.credential,
|
||||
PreparedAuthorizationCredential::Create(_)
|
||||
);
|
||||
let result = commit_authorization(
|
||||
self.accounts.as_ref(),
|
||||
prepared,
|
||||
@@ -203,14 +204,14 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI authorization",
|
||||
)
|
||||
.await?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
if creates_account {
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
}
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -227,17 +228,11 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
if details.credential.credential_revision != command.expected_credential_revision {
|
||||
return Err(AdminError::conflict(
|
||||
"OpenAI credential rotation revision is stale",
|
||||
));
|
||||
}
|
||||
let account = details.credential;
|
||||
let prepared = self
|
||||
.provider
|
||||
.prepare_rotation(PrepareCredentialRotation {
|
||||
account: account.clone(),
|
||||
expected_credential_revision: command.expected_credential_revision,
|
||||
provider_material: command.provider_material,
|
||||
})
|
||||
.await
|
||||
@@ -250,6 +245,9 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
"OpenAI credential rotation",
|
||||
)
|
||||
.await?;
|
||||
self.provider
|
||||
.account_facts_changed(std::slice::from_ref(&result.account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
//! 备份模块测试:镜像 src/backup/ 的 policy 与 task。
|
||||
|
||||
mod policy;
|
||||
pub mod support;
|
||||
mod task;
|
||||
@@ -0,0 +1,171 @@
|
||||
//! 备份调度与保留策略(backup/policy.rs)的纯逻辑测试。
|
||||
|
||||
use chrono::{Duration, TimeZone as _, Utc};
|
||||
|
||||
use gateway_admin::{
|
||||
backup::policy::{BackupSchedule, RetentionReason, decide_retention},
|
||||
model::backup::{BackupRecord, BackupStatus, BackupTriggerKind},
|
||||
};
|
||||
|
||||
fn record(
|
||||
id: &str,
|
||||
trigger: BackupTriggerKind,
|
||||
status: BackupStatus,
|
||||
created_at: chrono::DateTime<Utc>,
|
||||
completed_at: Option<chrono::DateTime<Utc>>,
|
||||
) -> BackupRecord {
|
||||
BackupRecord {
|
||||
id: id.to_owned(),
|
||||
trigger_kind: trigger,
|
||||
status,
|
||||
scheduled_at: if trigger == BackupTriggerKind::Scheduled {
|
||||
Some(created_at)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
object_key: format!("k/{id}"),
|
||||
size_bytes: Some(1),
|
||||
sha256: Some("a".repeat(64)),
|
||||
attempt_count: 0,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
started_at: Some(created_at),
|
||||
completed_at,
|
||||
expires_at: None,
|
||||
created_at,
|
||||
updated_at: created_at,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_bad_cron_and_timezone() {
|
||||
assert!(BackupSchedule::parse("61 * * * *", "Asia/Shanghai").is_err());
|
||||
assert!(BackupSchedule::parse("0 2 * * *", "Not/AZone").is_err());
|
||||
assert!(BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_after_is_strictly_later() {
|
||||
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
|
||||
let at = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
|
||||
let next = schedule.next_after(at).unwrap();
|
||||
assert_eq!(next, Utc.with_ymd_and_hms(2026, 8, 2, 18, 0, 0).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn last_firing_is_at_or_before() {
|
||||
let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap();
|
||||
// 北京时间 02:00 = 前一日 18:00 UTC。
|
||||
let before = Utc.with_ymd_and_hms(2026, 8, 1, 17, 59, 0).unwrap();
|
||||
assert_eq!(
|
||||
schedule.last_firing_at_or_before(before).unwrap(),
|
||||
Utc.with_ymd_and_hms(2026, 7, 31, 18, 0, 0).unwrap()
|
||||
);
|
||||
let exact = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap();
|
||||
assert_eq!(schedule.last_firing_at_or_before(exact).unwrap(), exact);
|
||||
let after = Utc.with_ymd_and_hms(2026, 8, 1, 19, 0, 0).unwrap();
|
||||
assert_eq!(schedule.last_firing_at_or_before(after).unwrap(), exact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dst_spring_forward_skips_missing_hour() {
|
||||
let schedule = BackupSchedule::parse("30 2 * * *", "America/New_York").unwrap();
|
||||
let at = Utc.with_ymd_and_hms(2026, 3, 8, 6, 29, 0).unwrap(); // 01:29 EST
|
||||
let next = schedule.next_after(at).unwrap();
|
||||
assert_eq!(next, Utc.with_ymd_and_hms(2026, 3, 9, 6, 30, 0).unwrap());
|
||||
let at_after_spring = Utc.with_ymd_and_hms(2026, 3, 8, 10, 0, 0).unwrap(); // 06:00 EDT
|
||||
let last = schedule.last_firing_at_or_before(at_after_spring).unwrap();
|
||||
assert_eq!(last, Utc.with_ymd_and_hms(2026, 3, 7, 7, 30, 0).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_count_keeps_newest_n() {
|
||||
let now = Utc::now();
|
||||
let base = now - Duration::days(10);
|
||||
let records = vec![
|
||||
record(
|
||||
"newest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base + Duration::days(2),
|
||||
Some(base + Duration::days(2)),
|
||||
),
|
||||
record(
|
||||
"middle",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base + Duration::days(1),
|
||||
Some(base + Duration::days(1)),
|
||||
),
|
||||
record(
|
||||
"oldest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
base,
|
||||
Some(base),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(0, 2, now, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["oldest"]);
|
||||
assert_eq!(decisions[0].reason, RetentionReason::Count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_days_deletes_old_scheduled_but_keeps_newest() {
|
||||
let now = Utc::now();
|
||||
let records = vec![
|
||||
record(
|
||||
"recent",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(1),
|
||||
Some(now - Duration::days(1)),
|
||||
),
|
||||
record(
|
||||
"ancient",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(40),
|
||||
Some(now - Duration::days(40)),
|
||||
),
|
||||
];
|
||||
let decisions = decide_retention(30, 0, now, &records);
|
||||
let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect();
|
||||
assert_eq!(ids, vec!["ancient"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newest_scheduled_never_auto_deleted_even_when_ancient() {
|
||||
let now = Utc::now();
|
||||
let records = vec![record(
|
||||
"sole",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(400),
|
||||
Some(now - Duration::days(400)),
|
||||
)];
|
||||
assert!(decide_retention(30, 1, now, &records).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_thresholds_delete_nothing() {
|
||||
let now = Utc::now();
|
||||
let records = vec![
|
||||
record(
|
||||
"newest",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(2),
|
||||
Some(now - Duration::days(2)),
|
||||
),
|
||||
record(
|
||||
"old",
|
||||
BackupTriggerKind::Scheduled,
|
||||
BackupStatus::Completed,
|
||||
now - Duration::days(90),
|
||||
Some(now - Duration::days(90)),
|
||||
),
|
||||
];
|
||||
assert!(decide_retention(0, 0, now, &records).is_empty());
|
||||
}
|
||||
@@ -0,0 +1,664 @@
|
||||
//! 备份测试共享 fixture:内存 fake 端口与辅助函数。
|
||||
//!
|
||||
//! 用内存 fake 端口驱动编排层,不依赖 MinIO/真实 S3。
|
||||
//! s3.rs / pg_dump.rs 是 SDK 薄适配器,协议正确性交给 aws-sdk / pg_dump 保证。
|
||||
//! 供 backup::task 与 use_case::backup 的测试复用。
|
||||
|
||||
use std::sync::Mutex;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
MutationActor, MutationContext, Revision,
|
||||
auth::AdminAuditEvent,
|
||||
backup::{
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
|
||||
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
|
||||
UpdateBackupStorageCommand,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
backup::{
|
||||
BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpArtifact, DumpRequest,
|
||||
StagedArtifact, StatusTransitionUpdate, UploadObjectRequest,
|
||||
},
|
||||
store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult, AuthStore},
|
||||
},
|
||||
};
|
||||
use secrecy::SecretString;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// 假归档内容;dump 与上传校验共用同一字节与校验值。
|
||||
pub(crate) const DUMP_CONTENT: &[u8] = b"custom-format-archive-bytes-0123456789";
|
||||
|
||||
/// 计算一段内容的 SHA-256 十六进制。
|
||||
pub(crate) fn sha256_hex(content: &[u8]) -> String {
|
||||
hex::encode(Sha256::digest(content))
|
||||
}
|
||||
|
||||
/// 返回 32 位十六进制的合法备份记录 id。
|
||||
pub(crate) fn backup_id(suffix: &str) -> String {
|
||||
format!("backup_{}", sha256_hex(suffix.as_bytes()))
|
||||
}
|
||||
|
||||
/// 已配置且已通过连接测试的设置快照。
|
||||
pub(crate) fn configured_settings() -> BackupSettings {
|
||||
let now = Utc::now();
|
||||
BackupSettings {
|
||||
storage_revision: 1,
|
||||
endpoint: Some("https://s3.example.com".to_owned()),
|
||||
region: Some("auto".to_owned()),
|
||||
bucket: Some("backup-bucket".to_owned()),
|
||||
access_key_id: Some("ak".to_owned()),
|
||||
secret_access_key: Some(SecretString::from("sk-value")),
|
||||
prefix: Some("codex".to_owned()),
|
||||
force_path_style: false,
|
||||
schedule_enabled: false,
|
||||
cron_expression: None,
|
||||
schedule_timezone: None,
|
||||
retention_days: 0,
|
||||
retention_count: 0,
|
||||
next_run_at: None,
|
||||
last_verified_at: Some(now),
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 seed 构造 queued 记录。
|
||||
pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord {
|
||||
let now = Utc::now();
|
||||
BackupRecord {
|
||||
id: seed.id.clone(),
|
||||
trigger_kind: seed.trigger_kind,
|
||||
status: BackupStatus::Queued,
|
||||
scheduled_at: seed.scheduled_at,
|
||||
object_key: seed.object_key.clone(),
|
||||
size_bytes: None,
|
||||
sha256: None,
|
||||
attempt_count: 0,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
started_at: None,
|
||||
completed_at: None,
|
||||
expires_at: seed.expires_at,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
/// 内存版备份仓储。
|
||||
pub(crate) struct FakeBackupRepository {
|
||||
settings: Mutex<BackupSettings>,
|
||||
records: Mutex<Vec<BackupRecord>>,
|
||||
}
|
||||
|
||||
impl FakeBackupRepository {
|
||||
pub(crate) fn new(settings: BackupSettings) -> Self {
|
||||
Self {
|
||||
settings: Mutex::new(settings),
|
||||
records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 测试辅助:读取当前全部记录。
|
||||
pub(crate) fn all_records(&self) -> Vec<BackupRecord> {
|
||||
self.records.lock().expect("records").clone()
|
||||
}
|
||||
|
||||
/// 测试辅助:强制把一条记录标记为指定时间完成的 completed。
|
||||
pub(crate) fn set_completed(&self, id: &str, completed_at: DateTime<Utc>) {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
|
||||
record.status = BackupStatus::Completed;
|
||||
record.completed_at = Some(completed_at);
|
||||
record.updated_at = Utc::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// 测试辅助:强制把一条记录迁移到指定状态。
|
||||
pub(crate) fn force_status(&self, id: &str, status: BackupStatus) {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if let Some(record) = records.iter_mut().find(|record| record.id == id) {
|
||||
record.status = status;
|
||||
if matches!(status, BackupStatus::Completed | BackupStatus::Failed) {
|
||||
record.completed_at = Some(Utc::now());
|
||||
}
|
||||
record.updated_at = Utc::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupRepository for FakeBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
Ok(self.settings.lock().expect("settings").clone())
|
||||
}
|
||||
|
||||
async fn update_storage_settings(
|
||||
&self,
|
||||
command: UpdateBackupStorageCommand,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<(BackupSettings, Revision)> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.storage_revision += 1;
|
||||
settings.endpoint = Some(command.endpoint);
|
||||
settings.region = Some(command.region);
|
||||
settings.bucket = Some(command.bucket);
|
||||
settings.access_key_id = Some(command.access_key_id);
|
||||
if let Some(secret) = command.secret_access_key {
|
||||
settings.secret_access_key = Some(secret);
|
||||
}
|
||||
settings.prefix = Some(command.prefix);
|
||||
settings.force_path_style = command.force_path_style;
|
||||
settings.last_verified_at = None;
|
||||
settings.updated_at = Utc::now();
|
||||
let revision = Revision::new(settings.storage_revision).map_err(|_| {
|
||||
AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", "zero revision")
|
||||
})?;
|
||||
Ok((settings.clone(), revision))
|
||||
}
|
||||
|
||||
async fn update_schedule_settings(
|
||||
&self,
|
||||
command: UpdateBackupScheduleCommand,
|
||||
next_run_at: Option<DateTime<Utc>>,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupSettings> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.schedule_enabled = command.schedule_enabled;
|
||||
settings.cron_expression = Some(command.cron_expression);
|
||||
settings.schedule_timezone = Some(command.schedule_timezone);
|
||||
settings.retention_days = command.retention_days;
|
||||
settings.retention_count = command.retention_count;
|
||||
settings.next_run_at = next_run_at;
|
||||
settings.updated_at = Utc::now();
|
||||
Ok(settings.clone())
|
||||
}
|
||||
|
||||
async fn record_verification(
|
||||
&self,
|
||||
storage_revision: u64,
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
if settings.storage_revision != storage_revision {
|
||||
return Ok(false);
|
||||
}
|
||||
settings.last_verified_at = Some(at);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if records.iter().any(|record| record.status.is_active()) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Conflict,
|
||||
"backup record",
|
||||
"an active backup task already exists",
|
||||
));
|
||||
}
|
||||
let record = queued_record(&seed);
|
||||
records.push(record.clone());
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<bool> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if records.iter().any(|record| record.status.is_active())
|
||||
|| records.iter().any(|record| {
|
||||
record.trigger_kind == BackupTriggerKind::Scheduled
|
||||
&& record.scheduled_at == seed.scheduled_at
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
records.push(queued_record(&seed));
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn list_backup_records(
|
||||
&self,
|
||||
query: BackupRecordListQuery,
|
||||
) -> AdminStoreResult<BackupRecordPage> {
|
||||
let records = self.records.lock().expect("records");
|
||||
let mut items = records
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
query.status.is_none_or(|status| record.status == status)
|
||||
&& query
|
||||
.trigger
|
||||
.is_none_or(|trigger| record.trigger_kind == trigger)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
items.sort_by(|a, b| {
|
||||
b.created_at
|
||||
.cmp(&a.created_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
let total = items.len() as u64;
|
||||
let offset =
|
||||
(u64::from(query.page.saturating_sub(1)) * u64::from(query.page_size.get())) as usize;
|
||||
let limit = usize::from(query.page_size.get());
|
||||
items = items.into_iter().skip(offset).take(limit).collect();
|
||||
Ok(BackupRecordPage {
|
||||
items,
|
||||
total,
|
||||
page: query.page,
|
||||
page_size: query.page_size,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_backup_record(&self, id: &str) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.find(|record| record.id == id)
|
||||
.cloned())
|
||||
}
|
||||
|
||||
async fn list_intermediate_records(&self) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
matches!(
|
||||
record.status,
|
||||
BackupStatus::Dumping | BackupStatus::Uploading
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| record.status == BackupStatus::Deleting)
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let now = Utc::now();
|
||||
Ok(self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record
|
||||
.expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
&& matches!(
|
||||
record.status,
|
||||
BackupStatus::Completed | BackupStatus::Failed
|
||||
)
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn claim_next_queued(
|
||||
&self,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.status == BackupStatus::Queued)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = BackupStatus::Dumping;
|
||||
record.started_at = Some(now);
|
||||
record.attempt_count += 1;
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn transition_status(
|
||||
&self,
|
||||
id: &str,
|
||||
from: BackupStatus,
|
||||
to: BackupStatus,
|
||||
update: StatusTransitionUpdate,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.id == id && record.status == from)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = to;
|
||||
if let Some(size) = update.size_bytes {
|
||||
record.size_bytes = Some(size);
|
||||
}
|
||||
if let Some(sha256) = update.sha256 {
|
||||
record.sha256 = Some(sha256);
|
||||
}
|
||||
if let Some(code) = update.error_code {
|
||||
record.error_code = Some(code);
|
||||
}
|
||||
if let Some(message) = update.error_message {
|
||||
record.error_message = Some(message);
|
||||
}
|
||||
record.completed_at = update.completed_at.or_else(|| {
|
||||
matches!(to, BackupStatus::Completed | BackupStatus::Failed).then_some(now)
|
||||
});
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
.iter_mut()
|
||||
.find(|record| record.id == id && record.status.can_be_deleted())
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
record.status = BackupStatus::Deleting;
|
||||
record.updated_at = now;
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
async fn delete_record(&self, id: &str) -> AdminStoreResult<()> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
records.retain(|record| record.id != id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn advance_schedule_cursor(
|
||||
&self,
|
||||
next_run_at: DateTime<Utc>,
|
||||
_expected_cron: &str,
|
||||
_expected_timezone: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
let mut settings = self.settings.lock().expect("settings");
|
||||
settings.next_run_at = Some(next_run_at);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn list_scheduled_completed_desc(
|
||||
&self,
|
||||
_limit: u32,
|
||||
) -> AdminStoreResult<Vec<BackupRecord>> {
|
||||
let mut records = self
|
||||
.records
|
||||
.lock()
|
||||
.expect("records")
|
||||
.iter()
|
||||
.filter(|record| {
|
||||
record.trigger_kind == BackupTriggerKind::Scheduled
|
||||
&& record.status == BackupStatus::Completed
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
records.sort_by(|a, b| {
|
||||
b.completed_at
|
||||
.cmp(&a.completed_at)
|
||||
.then_with(|| b.id.cmp(&a.id))
|
||||
});
|
||||
Ok(records)
|
||||
}
|
||||
}
|
||||
|
||||
/// 假 `pg_dump` 端口:把固定内容写入临时文件作为归档;可注入失败。
|
||||
pub(crate) struct FakeDumpPort {
|
||||
dir: TempDir,
|
||||
fail_dump: bool,
|
||||
}
|
||||
|
||||
impl FakeDumpPort {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
dir: tempfile::tempdir().expect("temp dir"),
|
||||
fail_dump: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让下一次 `dump` 返回 `backup.pg_dump_failed`。
|
||||
pub(crate) fn fail_next_dump(mut self) -> Self {
|
||||
self.fail_dump = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn artifact(&self, backup_id: &str) -> DumpArtifact {
|
||||
let path = self.dir.path().join(format!("{backup_id}.dump"));
|
||||
std::fs::write(&path, DUMP_CONTENT).expect("write artifact");
|
||||
DumpArtifact {
|
||||
path,
|
||||
size_bytes: DUMP_CONTENT.len() as u64,
|
||||
sha256: sha256_hex(DUMP_CONTENT),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DatabaseDumpPort for FakeDumpPort {
|
||||
async fn dump(
|
||||
&self,
|
||||
request: DumpRequest,
|
||||
) -> Result<DumpArtifact, gateway_admin::model::backup::BackupError> {
|
||||
if self.fail_dump {
|
||||
return Err(gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::PG_DUMP_FAILED,
|
||||
"pg_dump failed".to_owned(),
|
||||
));
|
||||
}
|
||||
if request.cancellation.is_cancelled() {
|
||||
return Err(gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::CANCELLED,
|
||||
"cancelled".to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(self.artifact(&request.backup_id))
|
||||
}
|
||||
|
||||
async fn inspect_staging(
|
||||
&self,
|
||||
backup_id: &str,
|
||||
) -> Result<Option<StagedArtifact>, gateway_admin::model::backup::BackupError> {
|
||||
let path = self.dir.path().join(format!("{backup_id}.dump"));
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(StagedArtifact {
|
||||
path,
|
||||
size_bytes: DUMP_CONTENT.len() as u64,
|
||||
sha256: sha256_hex(DUMP_CONTENT),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn cleanup_staging(
|
||||
&self,
|
||||
_backup_id: &str,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让上传返回指定的脱敏 S3 错误。
|
||||
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
|
||||
self.upload_error = Some(BackupError::new(code, message.into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BackupObjectStorePort for FakeObjectStore {
|
||||
async fn test_connection(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
) -> Result<ConnectionTestResult, gateway_admin::model::backup::BackupError> {
|
||||
Ok(ConnectionTestResult {
|
||||
ok: true,
|
||||
stage: "deleteObject",
|
||||
code: None,
|
||||
message: "连接测试成功".to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn upload_file(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.upload_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let content = std::fs::read(&request.source).map_err(|_| {
|
||||
gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
|
||||
"read source".to_owned(),
|
||||
)
|
||||
})?;
|
||||
self.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(request.object_key, content);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn head_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<Option<BackupObjectMetadata>, gateway_admin::model::backup::BackupError> {
|
||||
let Some(content) = self.object(object_key) else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(BackupObjectMetadata::new(
|
||||
"backup-test".to_owned(),
|
||||
sha256_hex(&content),
|
||||
Utc::now(),
|
||||
content.len() as u64,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn delete_object(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
self.objects.lock().expect("objects").remove(object_key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn presigned_download(
|
||||
&self,
|
||||
_config: &BackupStorageConfig,
|
||||
object_key: &str,
|
||||
_file_name: &str,
|
||||
_ttl: std::time::Duration,
|
||||
) -> Result<String, gateway_admin::model::backup::BackupError> {
|
||||
Ok(format!("https://presigned.example.com/{object_key}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录审计事件的假 AuthStore。
|
||||
pub(crate) struct FakeAuthStore {
|
||||
audit: Mutex<Vec<AdminAuditEvent>>,
|
||||
}
|
||||
|
||||
impl FakeAuthStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
audit: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn audit_actions(&self) -> Vec<String> {
|
||||
self.audit
|
||||
.lock()
|
||||
.expect("audit")
|
||||
.iter()
|
||||
.map(|event| event.action.clone())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AuthStore for FakeAuthStore {
|
||||
async fn load_password_hash(&self, _admin_user_id: &str) -> AdminStoreResult<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn create_password_hash_if_absent(
|
||||
&self,
|
||||
_admin_user_id: &str,
|
||||
_password_hash: &str,
|
||||
) -> AdminStoreResult<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn load_admin_api_key(
|
||||
&self,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::settings::AdminApiKey>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn store_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
_session: &gateway_admin::model::auth::AdminSession,
|
||||
) -> AdminStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_session(
|
||||
&self,
|
||||
_session_id: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::auth::AdminSession>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> {
|
||||
self.audit.lock().expect("audit").push(event);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// 系统发起者上下文。
|
||||
pub(crate) fn system_context() -> MutationContext {
|
||||
MutationContext {
|
||||
actor: MutationActor::System,
|
||||
request_id: "backup-runtime-test".to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
//! BackupTask 执行链测试:驱动内存 fake 端口跑完整 dump/upload/verify/retention 流程。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::{
|
||||
backup::task::BackupTask,
|
||||
model::backup::{
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
|
||||
},
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
|
||||
use super::support::{
|
||||
DUMP_CONTENT, FakeBackupRepository, FakeDumpPort, FakeObjectStore, backup_id,
|
||||
configured_settings, sha256_hex, system_context,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_runs_full_dump_upload_verify_pipeline() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("manual"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/manual.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
assert_eq!(records.len(), 1);
|
||||
let finished = &records[0];
|
||||
assert_eq!(finished.status, BackupStatus::Completed);
|
||||
assert_eq!(finished.size_bytes, Some(DUMP_CONTENT.len() as u64));
|
||||
assert_eq!(
|
||||
finished.sha256.as_deref(),
|
||||
Some(sha256_hex(DUMP_CONTENT).as_str())
|
||||
);
|
||||
|
||||
// 对象已上传且内容与本地归档一致。
|
||||
let stored = object_store
|
||||
.object(&seed.object_key)
|
||||
.expect("uploaded object exists");
|
||||
assert_eq!(stored.as_slice(), DUMP_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new().fail_next_dump());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("fail"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/fail.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some("backup.pg_dump_failed"));
|
||||
assert!(failed.completed_at.is_some());
|
||||
// 归档未上传,对象不存在。
|
||||
assert!(object_store.object(&seed.object_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("recovered-upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
dump.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("stage dump");
|
||||
repository.force_status(&seed.id, BackupStatus::Dumping);
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&cancellation).await.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_cleans_expired_scheduled_backups() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
|
||||
// 造一条旧 completed 计划备份(超过 retention_days)与一条新备份。
|
||||
let old_seed = BackupRecordSeed {
|
||||
id: backup_id("old"),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(Utc::now() - Duration::days(40)),
|
||||
object_key: "codex/2026/06/01/old.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
let new_seed = BackupRecordSeed {
|
||||
id: backup_id("new"),
|
||||
trigger_kind: BackupTriggerKind::Scheduled,
|
||||
scheduled_at: Some(Utc::now() - Duration::days(1)),
|
||||
object_key: "codex/2026/07/01/new.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_scheduled_record(old_seed.clone())
|
||||
.await
|
||||
.expect("insert old");
|
||||
// 旧备份 40 天前完成(释放活跃名额),再插入新备份。
|
||||
let now = Utc::now();
|
||||
repository.set_completed(&backup_id("old"), now - Duration::days(40));
|
||||
repository
|
||||
.insert_scheduled_record(new_seed.clone())
|
||||
.await
|
||||
.expect("insert new");
|
||||
// 新备份 1 天前完成。
|
||||
repository.set_completed(&backup_id("new"), now - Duration::days(1));
|
||||
// 两个对象都已上传。
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(old_seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
object_store
|
||||
.objects
|
||||
.lock()
|
||||
.expect("objects")
|
||||
.insert(new_seed.object_key.clone(), DUMP_CONTENT.to_vec());
|
||||
|
||||
// 设置 retention_days = 30。
|
||||
repository
|
||||
.update_schedule_settings(
|
||||
UpdateBackupScheduleCommand {
|
||||
schedule_enabled: false,
|
||||
cron_expression: "0 2 * * *".to_owned(),
|
||||
schedule_timezone: "Asia/Shanghai".to_owned(),
|
||||
retention_days: 30,
|
||||
retention_count: 0,
|
||||
},
|
||||
None,
|
||||
&system_context(),
|
||||
)
|
||||
.await
|
||||
.expect("set retention");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone());
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
// 旧备份被硬删除,对象被删除;新备份保留。
|
||||
let records = repository.all_records();
|
||||
let ids: Vec<_> = records.iter().map(|record| record.id.as_str()).collect();
|
||||
assert_eq!(ids, vec![backup_id("new").as_str()]);
|
||||
assert!(object_store.object(&old_seed.object_key).is_none());
|
||||
assert!(object_store.object(&new_seed.object_key).is_some());
|
||||
}
|
||||
@@ -11,12 +11,15 @@ use gateway_admin::{
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::AdminApiKey,
|
||||
},
|
||||
ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore},
|
||||
};
|
||||
|
||||
mod backup;
|
||||
mod model;
|
||||
mod use_case;
|
||||
|
||||
#[test]
|
||||
@@ -125,6 +128,8 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
quota_window("shortTerm", Some(18_000), Some(30.0)),
|
||||
quota_window("shortTerm", Some(14_400), Some(45.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
@@ -141,9 +146,11 @@ fn quota_window(
|
||||
group: group.to_owned(),
|
||||
label: group.to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! 备份领域模型(model/backup.rs)的纯逻辑测试。
|
||||
|
||||
use chrono::{TimeZone as _, Utc};
|
||||
use secrecy::SecretString;
|
||||
|
||||
use gateway_admin::model::backup::{
|
||||
BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, build_object_key,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn status_machine_rejects_invalid_transitions() {
|
||||
assert!(BackupStatus::Queued.allows_transition_to(BackupStatus::Dumping));
|
||||
assert!(BackupStatus::Dumping.allows_transition_to(BackupStatus::Uploading));
|
||||
assert!(BackupStatus::Uploading.allows_transition_to(BackupStatus::Completed));
|
||||
assert!(BackupStatus::Completed.allows_transition_to(BackupStatus::Deleting));
|
||||
assert!(!BackupStatus::Queued.allows_transition_to(BackupStatus::Completed));
|
||||
assert!(!BackupStatus::Completed.allows_transition_to(BackupStatus::Dumping));
|
||||
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Failed));
|
||||
assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_classification() {
|
||||
assert!(BackupStatus::Queued.is_active());
|
||||
assert!(BackupStatus::Uploading.is_active());
|
||||
assert!(!BackupStatus::Completed.is_active());
|
||||
assert!(BackupStatus::Completed.can_be_deleted());
|
||||
assert!(BackupStatus::Failed.can_be_deleted());
|
||||
assert!(!BackupStatus::Queued.can_be_deleted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_key_shape() {
|
||||
let at = Utc.with_ymd_and_hms(2026, 8, 1, 12, 30, 0).unwrap();
|
||||
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
|
||||
assert_eq!(
|
||||
key,
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_key_rejects_dangerous_prefixes() {
|
||||
let at = Utc::now();
|
||||
assert!(build_object_key("/leading", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a/../b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a\\b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("a\u{0001}b", "backup_x", at).is_err());
|
||||
assert!(build_object_key("", "backup_x", at).is_err());
|
||||
assert!(build_object_key("trailing//", "backup_x", at).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trigger_and_status_roundtrip() {
|
||||
for status in [
|
||||
BackupStatus::Queued,
|
||||
BackupStatus::Dumping,
|
||||
BackupStatus::Uploading,
|
||||
BackupStatus::Completed,
|
||||
BackupStatus::Failed,
|
||||
BackupStatus::Deleting,
|
||||
] {
|
||||
assert_eq!(BackupStatus::parse(status.as_str()), Some(status));
|
||||
}
|
||||
assert_eq!(
|
||||
BackupTriggerKind::parse("manual"),
|
||||
Some(BackupTriggerKind::Manual)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupTriggerKind::parse("scheduled"),
|
||||
Some(BackupTriggerKind::Scheduled)
|
||||
);
|
||||
assert_eq!(BackupTriggerKind::parse("unknown"), None);
|
||||
assert_eq!(BackupStatus::parse("skipped"), None);
|
||||
assert_eq!(BackupStatus::parse("deleted"), None);
|
||||
assert_eq!(BackupStatus::parse("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_from_settings_redacts_secret_in_debug() {
|
||||
let mut settings = BackupSettings {
|
||||
storage_revision: 3,
|
||||
endpoint: Some("https://example.com".to_owned()),
|
||||
region: Some("auto".to_owned()),
|
||||
bucket: Some("b".to_owned()),
|
||||
access_key_id: Some("ak".to_owned()),
|
||||
secret_access_key: Some(SecretString::from("sk-secret")),
|
||||
prefix: Some("p".to_owned()),
|
||||
force_path_style: false,
|
||||
schedule_enabled: false,
|
||||
cron_expression: None,
|
||||
schedule_timezone: None,
|
||||
retention_days: 0,
|
||||
retention_count: 0,
|
||||
next_run_at: None,
|
||||
last_verified_at: None,
|
||||
updated_at: Utc::now(),
|
||||
};
|
||||
assert!(!format!("{:?}", settings).contains("sk-secret"));
|
||||
let config = BackupStorageConfig::from_settings(&settings).unwrap();
|
||||
assert!(!format!("{config:?}").contains("sk-secret"));
|
||||
settings.endpoint = None;
|
||||
assert!(BackupStorageConfig::from_settings(&settings).is_none());
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
//! 模型模块测试:镜像 src/model/ 中带逻辑的模块。
|
||||
|
||||
mod backup;
|
||||
@@ -6,7 +6,7 @@ use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
operation::{GenerateRequest, Operation, ProtocolPayload},
|
||||
@@ -33,7 +33,7 @@ use gateway_admin::{
|
||||
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -64,6 +64,7 @@ pub(super) struct FakeProviderAdmin {
|
||||
import_account_ids: Mutex<Vec<String>>,
|
||||
quota_requests: Mutex<Vec<ProviderQuotaRequest>>,
|
||||
quota: Mutex<ProviderQuota>,
|
||||
current_credential_revision: Mutex<Revision>,
|
||||
}
|
||||
|
||||
impl FakeProviderAdmin {
|
||||
@@ -79,6 +80,7 @@ impl FakeProviderAdmin {
|
||||
import_account_ids: Mutex::new(vec!["acct_prepared".to_owned()]),
|
||||
quota_requests: Mutex::new(Vec::new()),
|
||||
quota: Mutex::new(empty_quota()),
|
||||
current_credential_revision: Mutex::new(revision(1)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -122,6 +124,13 @@ impl FakeProviderAdmin {
|
||||
.expect("authorization retry") = true;
|
||||
}
|
||||
|
||||
pub(super) fn set_current_credential_revision(&self, revision: Revision) {
|
||||
*self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision") = revision;
|
||||
}
|
||||
|
||||
fn export_inputs(&self) -> Vec<ProviderExportCredentialInput> {
|
||||
self.export_inputs.lock().expect("export inputs").clone()
|
||||
}
|
||||
@@ -142,7 +151,11 @@ impl FakeProviderAdmin {
|
||||
PreparedCredentialRotationFacts {
|
||||
account_id: ProviderAccountId::new(account.id.clone()).expect("account ID"),
|
||||
provider_kind: account.provider_kind.clone(),
|
||||
expected_credential_revision: account.credential_revision,
|
||||
expected_credential_revision: *self
|
||||
.current_credential_revision
|
||||
.lock()
|
||||
.expect("current credential revision"),
|
||||
replacement_identity: None,
|
||||
name: account.name.clone(),
|
||||
email: account.email.clone(),
|
||||
plan_type: account.plan_type.clone(),
|
||||
@@ -168,6 +181,10 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
self.record("provider.account_unavailable");
|
||||
}
|
||||
|
||||
async fn account_facts_changed(&self, _: &[ProviderAccountId]) {
|
||||
self.record("provider.account_facts_changed");
|
||||
}
|
||||
|
||||
fn connection_test_operation(
|
||||
&self,
|
||||
model: &gateway_core::routing::UpstreamModelId,
|
||||
@@ -264,13 +281,9 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
AuthorizationMutationTarget::Create { name } => {
|
||||
PreparedAuthorizationCredential::Create(prepared_create(self.kind.clone(), name))
|
||||
}
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} => {
|
||||
AuthorizationMutationTarget::Reauthorize { account_id } => {
|
||||
let mut account = account_record(self.kind.as_str());
|
||||
account.id = account_id.as_str().to_owned();
|
||||
account.credential_revision = *expected_credential_revision;
|
||||
PreparedAuthorizationCredential::Reauthorize(self.prepared_rotation(&account))
|
||||
}
|
||||
};
|
||||
@@ -360,7 +373,8 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
|
||||
pub(super) struct FakeAccountStore {
|
||||
events: EventLog,
|
||||
account: AccountRecord,
|
||||
accounts: Mutex<Vec<AccountRecord>>,
|
||||
account_after_probe: Mutex<Option<AccountRecord>>,
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
@@ -374,7 +388,8 @@ impl FakeAccountStore {
|
||||
fn with_account(account: AccountRecord, events: EventLog) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
events,
|
||||
account,
|
||||
accounts: Mutex::new(vec![account]),
|
||||
account_after_probe: Mutex::new(None),
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
@@ -393,6 +408,17 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
|
||||
fn set_account_after_probe(&self, account: AccountRecord) {
|
||||
*self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe") = Some(account);
|
||||
}
|
||||
|
||||
fn record(&self, event: &'static str) {
|
||||
self.events.lock().expect("store events").push(event);
|
||||
}
|
||||
@@ -417,22 +443,41 @@ impl FakeAccountStore {
|
||||
#[async_trait]
|
||||
impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: vec![self.account.clone()],
|
||||
total: 1,
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
summary: AccountSummary {
|
||||
total: 1,
|
||||
active: 1,
|
||||
total: accounts.len() as u64,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
self.record("store.load_account");
|
||||
Ok((account_id == self.account.id).then(|| self.account.clone()))
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
.account_after_probe
|
||||
.lock()
|
||||
.expect("account after probe")
|
||||
.clone()
|
||||
.filter(|account| account.id == account_id)
|
||||
.or_else(|| {
|
||||
self.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -460,11 +505,12 @@ impl AccountStore for FakeAccountStore {
|
||||
_: CredentialListQuery,
|
||||
) -> AdminStoreResult<CredentialPage> {
|
||||
self.record("store.list_credentials");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
Ok(CredentialPage {
|
||||
config_revision: revision(1),
|
||||
items: (provider_kind == &self.account.provider_kind)
|
||||
.then(|| self.account.clone())
|
||||
items: accounts
|
||||
.into_iter()
|
||||
.filter(|account| &account.provider_kind == provider_kind)
|
||||
.collect(),
|
||||
next_cursor: None,
|
||||
})
|
||||
@@ -476,14 +522,19 @@ impl AccountStore for FakeAccountStore {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> AdminStoreResult<Option<CredentialDetails>> {
|
||||
self.record("store.credential_details");
|
||||
Ok(
|
||||
(provider_kind == &self.account.provider_kind
|
||||
&& account_id.as_str() == self.account.id)
|
||||
.then(|| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential: self.account.clone(),
|
||||
}),
|
||||
)
|
||||
let account = self
|
||||
.accounts
|
||||
.lock()
|
||||
.expect("accounts")
|
||||
.iter()
|
||||
.find(|account| {
|
||||
&account.provider_kind == provider_kind && account.id == account_id.as_str()
|
||||
})
|
||||
.cloned();
|
||||
Ok(account.map(|credential| CredentialDetails {
|
||||
config_revision: revision(1),
|
||||
credential,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn load_credentials_for_export(
|
||||
@@ -492,21 +543,26 @@ impl AccountStore for FakeAccountStore {
|
||||
account_ids: &[ProviderAccountId],
|
||||
) -> AdminStoreResult<Vec<ProviderExportCredentialInput>> {
|
||||
self.record("store.load_credentials_for_export");
|
||||
if provider_kind != &self.account.provider_kind
|
||||
|| account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() != self.account.id)
|
||||
{
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
if accounts.iter().any(|account| {
|
||||
&account.provider_kind != provider_kind
|
||||
|| !account_ids
|
||||
.iter()
|
||||
.any(|account_id| account_id.as_str() == account.id)
|
||||
}) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::NotFound,
|
||||
"test account",
|
||||
"credential not found",
|
||||
));
|
||||
}
|
||||
Ok(vec![ProviderExportCredentialInput {
|
||||
account: self.account.clone(),
|
||||
provider_material: document(),
|
||||
}])
|
||||
Ok(accounts
|
||||
.into_iter()
|
||||
.map(|account| ProviderExportCredentialInput {
|
||||
account,
|
||||
provider_material: document(),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn commit_credential_import(
|
||||
@@ -782,17 +838,77 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
&& error_type == "invalid_request_error"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_rate_limited_probe_preserves_normal_account_status() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
let probe = Arc::new(FreeModelQuotaProbe {
|
||||
store: Arc::clone(&store),
|
||||
});
|
||||
let services = accounts_service_with_probe(provider, store, probe).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_should_preserve_disabled_account_status() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.enabled = false;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
|
||||
let events = services
|
||||
.accounts()
|
||||
.test_connection(
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
gateway_core::routing::UpstreamModelId::new("grok-4.5").expect("model"),
|
||||
)
|
||||
.await
|
||||
.expect("connection test stream")
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_export_should_pass_store_loaded_timestamps_and_material_to_provider() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
let expected = store.account.clone();
|
||||
let expected = store.accounts.lock().expect("accounts")[0].clone();
|
||||
accounts_service(provider.clone(), store)
|
||||
.await
|
||||
.accounts()
|
||||
@@ -834,7 +950,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -844,6 +960,7 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
"provider.prepare_refresh",
|
||||
"store.commit_refresh",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"store.load_account",
|
||||
]
|
||||
);
|
||||
@@ -869,12 +986,229 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_dropping_page() {
|
||||
let events = events();
|
||||
let openai = FakeProviderAdmin::new("openai", events.clone());
|
||||
openai.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_bad_xai".to_owned();
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(openai)
|
||||
.provider(failing)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let page = services
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("one failing quota must not fail the directory page");
|
||||
|
||||
assert_eq!(page.items.len(), 2);
|
||||
let by_id = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|item| (item.account.id.as_str(), item))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
let healthy = by_id.get("acct_test").expect("healthy account item");
|
||||
assert_eq!(healthy.quota.windows.len(), 1);
|
||||
assert_eq!(healthy.quota.windows[0].key, "primary");
|
||||
let degraded = by_id.get("acct_bad_xai").expect("degraded account item");
|
||||
assert!(degraded.quota.windows.is_empty());
|
||||
assert!(degraded.quota.observed_at.is_none());
|
||||
assert!(degraded.quota.provider_data.is_none());
|
||||
assert_eq!(page.summary.total, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → quota_exhausted(配额耗尽);
|
||||
// rate_limited 只来自 ProviderQuota.rate_limited_until(429 Redis 冷却)。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: Some(Utc::now() + TimeDelta::minutes(5)),
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_quota_exhausted_when_only_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached、无 429 冷却 → quota_exhausted。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota exhausted account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -890,12 +1224,16 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
@@ -926,6 +1264,96 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
assert_eq!(usage.total_tokens, Some(4_330_000));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let reset_at = Utc::now() + TimeDelta::days(7);
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![
|
||||
ProviderQuotaWindow {
|
||||
key: "core-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "周额度".to_owned(),
|
||||
source: Some("codex".to_owned()),
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(1.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
ProviderQuotaWindow {
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "GPT-5.3-Codex-Spark · 周额度".to_owned(),
|
||||
source: Some("codex_bengalfox".to_owned()),
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(0.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
],
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
store.set_quota_window_usage(vec![
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "core-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 12_818_806),
|
||||
},
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 127_926),
|
||||
},
|
||||
]);
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota window usage");
|
||||
|
||||
let local_tokens = page.items[0]
|
||||
.quota
|
||||
.windows
|
||||
.iter()
|
||||
.map(|window| {
|
||||
(
|
||||
window.key.as_str(),
|
||||
window
|
||||
.local_usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.total_tokens),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
local_tokens,
|
||||
[
|
||||
("core-primary", Some(12_818_806)),
|
||||
("codex-bengalfox-primary", None),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
let events = events();
|
||||
@@ -1003,6 +1431,8 @@ fn empty_quota() -> ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
provider_data: None,
|
||||
}
|
||||
}
|
||||
@@ -1037,7 +1467,7 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
provider_kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
name: "test account".to_owned(),
|
||||
email: Some("test@example.invalid".to_owned()),
|
||||
upstream_user_id: "upstream-user".to_owned(),
|
||||
upstream_user_id: Some("upstream-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1047,9 +1477,8 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
last_error_message: None,
|
||||
quota_observed_at: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
@@ -1075,7 +1504,7 @@ fn prepared_create_with_id(
|
||||
provider_kind,
|
||||
name: name.to_owned(),
|
||||
email: Some("prepared@example.invalid".to_owned()),
|
||||
upstream_user_id: "prepared-user".to_owned(),
|
||||
upstream_user_id: Some("prepared-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1085,8 +1514,6 @@ fn prepared_create_with_id(
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_reason: None,
|
||||
cooldown_until: None,
|
||||
availability_observed_at: now,
|
||||
}
|
||||
}
|
||||
@@ -1128,7 +1555,7 @@ impl AccountProbe for SuccessfulAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Ok(AccountProbeResult {
|
||||
text: vec!["OK".to_owned()],
|
||||
@@ -1143,19 +1570,41 @@ impl AccountProbe for FailingAccountProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"included usage exhausted",
|
||||
Some("usage_exhausted".to_owned()),
|
||||
Some("invalid_request_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider error");
|
||||
);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
)
|
||||
.with_client_visible_upstream_error(detail))
|
||||
.with_client_visible_upstream_error(detail)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FreeModelQuotaProbe {
|
||||
store: Arc<FakeAccountStore>,
|
||||
}
|
||||
|
||||
impl AccountProbe for FreeModelQuotaProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
//! BackupService 用例测试:通过 AdminHarness 注入 fake 备份端口构造真实服务。
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_admin::model::backup::BackupStatus;
|
||||
use gateway_admin::ports::backup::BackupStorePorts;
|
||||
|
||||
use crate::backup::support::{
|
||||
FakeAuthStore, FakeBackupRepository, FakeDumpPort, FakeObjectStore, configured_settings,
|
||||
system_context,
|
||||
};
|
||||
|
||||
use super::AdminHarness;
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_service_creates_downloads_and_deletes_with_audit() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let auth = Arc::new(FakeAuthStore::new());
|
||||
|
||||
let services = AdminHarness::new()
|
||||
.auth(auth.clone())
|
||||
.backup(BackupStorePorts::new(
|
||||
repository.clone(),
|
||||
dump,
|
||||
object_store,
|
||||
))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// 创建手动备份 → queued + 审计。
|
||||
let created = services
|
||||
.backups()
|
||||
.create_backup(&system_context(), None)
|
||||
.await
|
||||
.expect("create backup");
|
||||
assert_eq!(created.status, BackupStatus::Queued);
|
||||
assert!(created.object_key.starts_with("codex/"));
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.created")
|
||||
);
|
||||
|
||||
// 下载地址要求 completed 记录。
|
||||
repository.force_status(&created.id, BackupStatus::Completed);
|
||||
let download = services
|
||||
.backups()
|
||||
.download_url(&system_context(), &created.id)
|
||||
.await
|
||||
.expect("download url");
|
||||
assert!(download.url.starts_with("https://presigned.example.com/"));
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.download_url_created")
|
||||
);
|
||||
|
||||
// 删除 → deleting + 审计。
|
||||
let deleting = services
|
||||
.backups()
|
||||
.delete_backup(&system_context(), &created.id)
|
||||
.await
|
||||
.expect("delete backup");
|
||||
assert_eq!(deleting.status, BackupStatus::Deleting);
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
.iter()
|
||||
.any(|action| action == "backup.delete_requested")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_service_rejects_download_of_non_completed_record() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store = Arc::new(FakeObjectStore::new());
|
||||
let auth = Arc::new(FakeAuthStore::new());
|
||||
|
||||
let services = AdminHarness::new()
|
||||
.auth(auth.clone())
|
||||
.backup(BackupStorePorts::new(
|
||||
repository.clone(),
|
||||
dump,
|
||||
object_store,
|
||||
))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let created = services
|
||||
.backups()
|
||||
.create_backup(&system_context(), None)
|
||||
.await
|
||||
.expect("create backup");
|
||||
|
||||
// queued 记录不能创建下载地址。
|
||||
let error = services
|
||||
.backups()
|
||||
.download_url(&system_context(), &created.id)
|
||||
.await
|
||||
.expect_err("non-completed record must be rejected");
|
||||
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
mod accounts;
|
||||
mod auth;
|
||||
mod backup;
|
||||
mod client_keys;
|
||||
mod observability;
|
||||
mod openai;
|
||||
@@ -46,6 +47,7 @@ use gateway_admin::{
|
||||
system::{SystemOperationAccepted, SystemUpdateDetail, SystemUpdateStatus, SystemVersion},
|
||||
},
|
||||
ports::{
|
||||
backup::BackupStorePorts,
|
||||
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
|
||||
store::{
|
||||
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
|
||||
@@ -60,7 +62,7 @@ use gateway_admin::{
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
policy::ClientApiKeyId,
|
||||
@@ -75,6 +77,7 @@ pub(super) struct AdminHarness {
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
backup: BackupStorePorts,
|
||||
providers: Vec<Arc<dyn ProviderAdmin>>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
system: Arc<dyn SystemOperations>,
|
||||
@@ -91,6 +94,7 @@ impl AdminHarness {
|
||||
client_keys: unavailable.clone(),
|
||||
observability: unavailable.clone(),
|
||||
settings: unavailable,
|
||||
backup: BackupStorePorts::disabled(),
|
||||
providers: vec![
|
||||
Arc::new(UnavailableProvider::new("openai")),
|
||||
Arc::new(UnavailableProvider::new("xai")),
|
||||
@@ -135,6 +139,11 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn backup(mut self, backup: BackupStorePorts) -> Self {
|
||||
self.backup = backup;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn provider(mut self, provider: Arc<dyn ProviderAdmin>) -> Self {
|
||||
self.providers
|
||||
.retain(|registered| registered.provider_kind() != provider.provider_kind());
|
||||
@@ -165,6 +174,7 @@ impl AdminHarness {
|
||||
self.client_keys,
|
||||
self.observability,
|
||||
self.settings,
|
||||
self.backup,
|
||||
),
|
||||
self.providers,
|
||||
Arc::new(NoopSnapshot),
|
||||
@@ -647,12 +657,13 @@ impl AccountProbe for UnavailableProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"test account probe is unavailable",
|
||||
))
|
||||
)
|
||||
.into())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,6 +249,7 @@ async fn observability_services_should_calculate_usage_insights_and_diagnostic_s
|
||||
bucket_start: quarter_hour_start(now),
|
||||
provider_kind: "openai".to_owned(),
|
||||
upstream_model_id: "gpt-5.5".to_owned(),
|
||||
service_tier: None,
|
||||
input_tokens: Some(800),
|
||||
output_tokens: Some(200),
|
||||
cached_tokens: Some(0),
|
||||
@@ -326,9 +327,19 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
let invalid_kind = "x".repeat(65);
|
||||
store.replace_usage_records(vec![
|
||||
calculated_total_record("request_invalid_kind", Some(&invalid_kind), now),
|
||||
calculated_total_record("request_unregistered_kind", Some("anthropic"), now),
|
||||
calculated_total_record("request_enriched", Some("openai"), now),
|
||||
total_record(
|
||||
"request_invalid_kind",
|
||||
Some(&invalid_kind),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record(
|
||||
"request_unregistered_kind",
|
||||
Some("anthropic"),
|
||||
"calculated",
|
||||
now,
|
||||
),
|
||||
total_record("request_enriched", Some("openai"), "calculated", now),
|
||||
]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
@@ -353,6 +364,30 @@ async fn usage_records_should_tolerate_records_that_fail_billing_enrichment() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_records_should_enrich_provider_reported_totals_when_pricing_matches() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_usage_records(vec![total_record(
|
||||
"request_provider_reported",
|
||||
Some("openai"),
|
||||
"provider_reported",
|
||||
now,
|
||||
)]);
|
||||
let services = observability_services_with_calculated_billing(store).await;
|
||||
|
||||
let page = services
|
||||
.observability()
|
||||
.usage_records(usage_query(now))
|
||||
.await
|
||||
.expect("usage records");
|
||||
|
||||
assert!(matches!(
|
||||
page.items[0].billing,
|
||||
Some(UsageBilling::Calculated(_))
|
||||
));
|
||||
}
|
||||
|
||||
struct FixtureObservabilityStore {
|
||||
trend: Mutex<Vec<RequestMetricPoint>>,
|
||||
overview: Mutex<UsageOverview>,
|
||||
@@ -414,6 +449,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
range,
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
account_usage: Vec::new(),
|
||||
@@ -566,9 +602,10 @@ fn usage_query(now: DateTime<Utc>) -> UsageQuery {
|
||||
}
|
||||
}
|
||||
|
||||
fn calculated_total_record(
|
||||
fn total_record(
|
||||
id: &str,
|
||||
provider_kind: Option<&str>,
|
||||
source: &str,
|
||||
now: DateTime<Utc>,
|
||||
) -> UsageRecord {
|
||||
UsageRecord {
|
||||
@@ -589,6 +626,7 @@ fn calculated_total_record(
|
||||
upstream_transport: None,
|
||||
http_version: None,
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
@@ -611,11 +649,11 @@ fn calculated_total_record(
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(1_000),
|
||||
cost_source: "calculated".to_owned(),
|
||||
cost_source: source.to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Total {
|
||||
source: "calculated".to_owned(),
|
||||
source: source.to_owned(),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: gateway_admin::model::observability::DecimalAmount::from_str("1.25")
|
||||
@@ -658,6 +696,7 @@ fn health_metric_point(bucket_start: DateTime<Utc>, metrics: RequestMetrics) ->
|
||||
|
||||
fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
DiagnosticObservation {
|
||||
key: name.to_owned(),
|
||||
name: name.to_owned(),
|
||||
request_count,
|
||||
success_count: request_count,
|
||||
@@ -665,6 +704,7 @@ fn diagnostic(name: &str, request_count: u64) -> DiagnosticObservation {
|
||||
attempt_count: request_count,
|
||||
total_tokens: request_count.saturating_mul(100),
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(200),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use gateway_admin::{
|
||||
AdminServices,
|
||||
model::provider_credentials::{
|
||||
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, CredentialMutation,
|
||||
ImportCredentials, ProviderQuotaRequest, ReauthorizationTarget, StartAuthorization,
|
||||
ImportCredentials, ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
@@ -30,7 +30,11 @@ async fn openai_enable_should_use_scoped_store_without_provider_mutation() {
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.credential_details", "store.set_enabled"]
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
@@ -54,6 +58,7 @@ async fn openai_disable_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -78,6 +83,7 @@ async fn openai_delete_should_commit_then_release_provider_resources() {
|
||||
"store.credential_details",
|
||||
"store.delete",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
@@ -119,11 +125,14 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
.await
|
||||
.expect("import credential");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.prepare_import",
|
||||
"store.commit_import",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
@@ -155,6 +164,8 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.await
|
||||
.expect("import credentials");
|
||||
|
||||
await_quota_requests(&provider, 2).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[
|
||||
@@ -196,6 +207,8 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.await
|
||||
.expect("committed import remains successful");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[ProviderAccountId::new("acct_prepared").expect("account ID")]
|
||||
@@ -229,12 +242,15 @@ async fn openai_authorization_create_should_observe_initial_quota() {
|
||||
.await
|
||||
.expect("complete authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"provider.start_authorization",
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
@@ -276,6 +292,8 @@ async fn openai_authorization_create_should_remain_successful_when_initial_quota
|
||||
.await
|
||||
.expect("quota observation must not fail authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.account_id.as_str(), "acct_prepared");
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
@@ -314,6 +332,8 @@ async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
.await
|
||||
.expect("released claim must permit retry");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -324,6 +344,7 @@ async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"authorization_guard.commit",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
@@ -351,7 +372,7 @@ async fn openai_import_provider_error_should_not_touch_store_transaction() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_through_commit() {
|
||||
async fn openai_reauthorization_should_commit_after_credential_revision_advances() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
@@ -361,10 +382,7 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.start_authorization(StartAuthorization {
|
||||
context: context("oauth-start-openai"),
|
||||
name: "reauthorize".to_owned(),
|
||||
reauthorization: Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
credential_revision: revision(1),
|
||||
}),
|
||||
reauthorization: Some(ProviderAccountId::new("acct_test").expect("account ID")),
|
||||
})
|
||||
.await
|
||||
.expect("start reauthorization");
|
||||
@@ -377,15 +395,14 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
);
|
||||
assert!(matches!(
|
||||
pending.target(),
|
||||
AuthorizationMutationTarget::Reauthorize {
|
||||
account_id,
|
||||
expected_credential_revision,
|
||||
} if account_id.as_str() == "acct_test"
|
||||
&& *expected_credential_revision == revision(1)
|
||||
AuthorizationMutationTarget::Reauthorize { account_id }
|
||||
if account_id.as_str() == "acct_test"
|
||||
));
|
||||
assert!(!format!("{pending:?}").contains("admin-test"));
|
||||
|
||||
services
|
||||
provider.set_current_credential_revision(revision(2));
|
||||
|
||||
let result = services
|
||||
.openai()
|
||||
.complete_authorization(CompleteAuthorization {
|
||||
context: context("oauth-complete-openai"),
|
||||
@@ -395,6 +412,10 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.credential_revision, Some(revision(3)));
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -403,8 +424,11 @@ async fn openai_reauthorization_should_restore_pending_envelope_and_hold_guard_t
|
||||
"provider.complete_authorization",
|
||||
"store.commit_authorization",
|
||||
"guard.finish",
|
||||
"provider.account_facts_changed",
|
||||
"provider.quota",
|
||||
]
|
||||
);
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
assert_eq!(store.audit_requests(), ["oauth-complete-openai"]);
|
||||
}
|
||||
|
||||
@@ -416,6 +440,16 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn await_quota_requests(provider: &FakeProviderAdmin, expected: usize) {
|
||||
for _ in 0..100 {
|
||||
if provider.quota_requests().len() == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(provider.quota_requests().len(), expected);
|
||||
}
|
||||
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-openai"),
|
||||
|
||||
@@ -29,6 +29,7 @@ async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-xai"]);
|
||||
|
||||
@@ -20,8 +20,9 @@ futures = "0.3.32"
|
||||
gateway-admin = { path = "../gateway-admin" }
|
||||
gateway-core = { path = "../gateway-core" }
|
||||
gateway-protocol = { path = "../gateway-protocol" }
|
||||
secrecy = "0.10"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "time"] }
|
||||
tower-http = { version = "0.7.0", features = ["cors", "fs", "request-id", "timeout", "trace"] }
|
||||
@@ -31,7 +32,9 @@ uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1.89"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] }
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio"] }
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "net", "rt-multi-thread"] }
|
||||
tokio-tungstenite = "0.29.0"
|
||||
tower = { version = "0.5.3", features = ["util"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -16,7 +16,7 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use gateway_admin::model::{
|
||||
AdminError as AdminServiceError, AdminErrorKind, PageSize, Revision,
|
||||
AdminError as AdminServiceError, PageSize,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
|
||||
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
|
||||
@@ -26,8 +26,8 @@ use gateway_admin::model::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialImportResult, CredentialMutation, CredentialMutationResult, ImportCredentials,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow,
|
||||
ReauthorizationTarget, RotateCredential, StartAuthorization,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow, RotateCredential,
|
||||
StartAuthorization,
|
||||
},
|
||||
};
|
||||
use gateway_core::{
|
||||
@@ -37,6 +37,7 @@ use gateway_core::{
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError,
|
||||
@@ -90,7 +91,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -126,17 +128,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -171,9 +162,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -193,11 +186,12 @@ pub struct AccountView {
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
pub token_refreshing: bool,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub availability: String,
|
||||
pub enabled: bool,
|
||||
pub credential_revision: u64,
|
||||
pub state_revision: Option<u64>,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
pub access_token_expires_at_display: Option<String>,
|
||||
pub refresh_token_expires_at: Option<String>,
|
||||
@@ -215,6 +209,9 @@ pub struct AccountView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -228,6 +225,7 @@ pub struct AccountQuotaWindowView {
|
||||
pub label_display: String,
|
||||
pub used_percent: Option<f64>,
|
||||
pub used_percent_display: String,
|
||||
pub limit_reached: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_usage: Option<serde_json::Value>,
|
||||
pub reset_at_display: String,
|
||||
@@ -527,19 +525,25 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code,
|
||||
provider_error_type,
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
"providerErrorCode": provider_error_code,
|
||||
"providerErrorType": provider_error_type,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status.as_str()
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -604,24 +608,16 @@ pub struct StartAccountAuthorizationRequest {
|
||||
pub provider: String,
|
||||
pub name: String,
|
||||
pub account_id: Option<String>,
|
||||
pub expected_credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl StartAccountAuthorizationRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_text(&self.name, MAX_NAME_BYTES, "name")?;
|
||||
match (
|
||||
self.account_id.as_deref(),
|
||||
self.expected_credential_revision,
|
||||
) {
|
||||
(None, None) => Ok(()),
|
||||
(Some(account_id), Some(credential_revision)) => {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
require_positive(credential_revision, "expectedCredentialRevision")
|
||||
}
|
||||
_ => Err(WireValidationError::new("reauthorization")),
|
||||
if let Some(account_id) = self.account_id.as_deref() {
|
||||
require_account_id(account_id, "accountId")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -630,15 +626,11 @@ impl StartAccountAuthorizationRequest {
|
||||
) -> Result<(AccountProvider, StartAuthorization), WireValidationError> {
|
||||
self.validate()?;
|
||||
let provider = AccountProvider::parse(&self.provider)?;
|
||||
let reauthorization = match (self.account_id, self.expected_credential_revision) {
|
||||
(None, None) => None,
|
||||
(Some(account_id), Some(credential_revision)) => Some(ReauthorizationTarget {
|
||||
account_id: ProviderAccountId::new(account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
credential_revision: revision(credential_revision, "expectedCredentialRevision")?,
|
||||
}),
|
||||
_ => return Err(WireValidationError::new("reauthorization")),
|
||||
};
|
||||
let reauthorization = self
|
||||
.account_id
|
||||
.map(ProviderAccountId::new)
|
||||
.transpose()
|
||||
.map_err(|_| WireValidationError::new("accountId"))?;
|
||||
Ok((
|
||||
provider,
|
||||
StartAuthorization {
|
||||
@@ -771,7 +763,6 @@ impl AccountDeletionRequest {
|
||||
pub struct RotateAccountRequest {
|
||||
pub provider: String,
|
||||
pub account_id: String,
|
||||
pub expected_credential_revision: u64,
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
}
|
||||
@@ -782,10 +773,6 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
require_positive(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?;
|
||||
validate_oauth_material(&self.access_token, self.refresh_token.as_deref())
|
||||
}
|
||||
|
||||
@@ -806,10 +793,6 @@ impl RotateAccountRequest {
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
expected_credential_revision: revision(
|
||||
self.expected_credential_revision,
|
||||
"expectedCredentialRevision",
|
||||
)?,
|
||||
provider_material: ProviderDocument::new(OpaqueProviderData::new(material)),
|
||||
})
|
||||
}
|
||||
@@ -858,15 +841,12 @@ impl From<AuthorizationStarted> for AccountAuthorizationData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountMutationData {
|
||||
pub account_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub credential_revision: Option<u64>,
|
||||
}
|
||||
|
||||
impl From<CredentialMutationResult> for AccountMutationData {
|
||||
fn from(result: CredentialMutationResult) -> Self {
|
||||
Self {
|
||||
account_id: result.account_id.to_string(),
|
||||
credential_revision: result.credential_revision.map(Revision::get),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1351,9 +1331,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1384,10 +1366,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
error_reason,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = status.as_str().to_owned();
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
@@ -1399,18 +1382,17 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
resource_ref: account.id,
|
||||
email: account.email,
|
||||
account_id: account.upstream_account_id,
|
||||
user_id: Some(account.upstream_user_id),
|
||||
user_id: account.upstream_user_id,
|
||||
label: None,
|
||||
plan_type: account.plan_type,
|
||||
authentication_kind: account.authentication_kind,
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
token_refreshing: false,
|
||||
error_reason: error_reason.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: account.last_error_message,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
enabled: account.enabled,
|
||||
credential_revision: account.credential_revision.get(),
|
||||
state_revision: None,
|
||||
access_token_expires_at: expires_at,
|
||||
access_token_expires_at_display: account
|
||||
.access_token_expires_at
|
||||
@@ -1438,9 +1420,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = quota.rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1453,9 +1438,11 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
group,
|
||||
label,
|
||||
source: _,
|
||||
local_usage_attribution: _,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
limit_reached,
|
||||
local_usage,
|
||||
provider_data: _,
|
||||
} = window;
|
||||
@@ -1467,6 +1454,7 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
used_percent,
|
||||
used_percent_display: used_percent
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
|
||||
limit_reached,
|
||||
local_usage: local_usage.as_ref().map(quota_local_usage),
|
||||
reset_at_display: reset_at.map_or_else(|| "—".to_owned(), |value| china_datetime(&value)),
|
||||
}
|
||||
@@ -1490,7 +1478,7 @@ fn quota_local_usage(usage: &AccountUsage) -> Value {
|
||||
"imageRequestCount": usage.image_request_count,
|
||||
"imageRequestFailedCount": usage.image_request_failed_count,
|
||||
"totalTokens": total_tokens,
|
||||
"totalTokensDisplay": format_tokens(total_tokens),
|
||||
"totalTokensDisplay": format_compact_number(total_tokens),
|
||||
"requestBuckets": usage.request_buckets.iter().map(|bucket| serde_json::json!({
|
||||
"bucketStart": bucket.bucket_start,
|
||||
"requestCount": bucket.request_count,
|
||||
@@ -1623,7 +1611,7 @@ fn account_currency_cost_view(cost: &AccountCost) -> CurrencyCostView {
|
||||
}
|
||||
|
||||
fn display_optional_tokens(value: Option<u64>) -> String {
|
||||
value.map_or_else(|| "—".to_owned(), format_tokens)
|
||||
value.map_or_else(|| "—".to_owned(), format_compact_number)
|
||||
}
|
||||
|
||||
fn empty_account_usage() -> AccountUsageView {
|
||||
@@ -1661,10 +1649,6 @@ fn empty_account_usage() -> AccountUsageView {
|
||||
}
|
||||
}
|
||||
|
||||
fn revision(value: u64, field: &'static str) -> Result<Revision, WireValidationError> {
|
||||
Revision::new(value).map_err(|_| WireValidationError::new(field))
|
||||
}
|
||||
|
||||
fn provider_document(
|
||||
value: Value,
|
||||
field: &'static str,
|
||||
@@ -1698,13 +1682,6 @@ fn require_text(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_positive(value: u64, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value == 0 || i64::try_from(value).is_err() {
|
||||
return Err(WireValidationError::new(field));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_oauth_material(
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
@@ -1742,47 +1719,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn format_number(value: u64) -> String {
|
||||
let text = value.to_string();
|
||||
let mut output = String::with_capacity(text.len() + text.len() / 3);
|
||||
for (index, character) in text.chars().rev().enumerate() {
|
||||
if index > 0 && index % 3 == 0 {
|
||||
output.push(',');
|
||||
}
|
||||
output.push(character);
|
||||
}
|
||||
output.chars().rev().collect()
|
||||
}
|
||||
|
||||
fn format_tokens(value: u64) -> String {
|
||||
if value < 1_000 {
|
||||
return format_number(value);
|
||||
}
|
||||
for (suffix, threshold) in [
|
||||
("P", 1_000_000_000_000_000_u64),
|
||||
("T", 1_000_000_000_000_u64),
|
||||
("B", 1_000_000_000_u64),
|
||||
("M", 1_000_000_u64),
|
||||
("K", 1_000_u64),
|
||||
] {
|
||||
if value >= threshold {
|
||||
let scaled = value as f64 / threshold as f64;
|
||||
return format!("{scaled:.1}{suffix}").replace(".0", "");
|
||||
}
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
@@ -1820,16 +1756,5 @@ fn map_wire_error(error: WireValidationError) -> AdminError {
|
||||
}
|
||||
|
||||
fn map_service_error(error: AdminServiceError) -> AdminError {
|
||||
match error.kind() {
|
||||
AdminErrorKind::Invalid => AdminError::bad_request(error.to_string()),
|
||||
AdminErrorKind::Unauthorized => AdminError::admin_session_required(),
|
||||
AdminErrorKind::NotFound => AdminError::not_found(error.to_string()),
|
||||
AdminErrorKind::Conflict => AdminError::conflict(error.to_string()),
|
||||
AdminErrorKind::RateLimited => AdminError::too_many_login_attempts(),
|
||||
AdminErrorKind::BadGateway => AdminError::bad_gateway(error.to_string()),
|
||||
AdminErrorKind::Unavailable => {
|
||||
AdminError::service_unavailable("Account directory unavailable")
|
||||
}
|
||||
AdminErrorKind::Internal => AdminError::internal(error.to_string()),
|
||||
}
|
||||
super::wire::map_admin_service_error(error, "Account directory unavailable")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
//! 备份设置、计划、手动创建、记录、下载与删除路由。
|
||||
//!
|
||||
//! 路径统一位于 `/api/admin/settings/backups/*`,内部由独立 `BackupService` 承担业务。
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::backup::{
|
||||
BackupRecord, BackupSettings, BackupStatus, BackupTriggerKind, ConnectionTestResult,
|
||||
UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
};
|
||||
use secrecy::{ExposeSecret as _, SecretString};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminPageData, AdminResponse, AdminSessionState,
|
||||
PageMeta, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
/// 备份设置视图;`secretAccessKey` 返回已保存的明文凭据,由前端掩码显示。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupSettingsView {
|
||||
pub storage_revision: u64,
|
||||
pub endpoint: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub bucket: Option<String>,
|
||||
pub access_key_id: Option<String>,
|
||||
pub secret_access_key: Option<String>,
|
||||
pub prefix: Option<String>,
|
||||
pub force_path_style: bool,
|
||||
pub verified: bool,
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: Option<String>,
|
||||
pub schedule_timezone: Option<String>,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
pub next_run_at: Option<DateTime<Utc>>,
|
||||
pub last_verified_at: Option<DateTime<Utc>>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl From<BackupSettings> for BackupSettingsView {
|
||||
fn from(settings: BackupSettings) -> Self {
|
||||
let verified = settings.storage_verified();
|
||||
Self {
|
||||
storage_revision: settings.storage_revision,
|
||||
endpoint: settings.endpoint,
|
||||
region: settings.region,
|
||||
bucket: settings.bucket,
|
||||
access_key_id: settings.access_key_id,
|
||||
secret_access_key: settings
|
||||
.secret_access_key
|
||||
.map(|secret| secret.expose_secret().to_owned()),
|
||||
prefix: settings.prefix,
|
||||
force_path_style: settings.force_path_style,
|
||||
verified,
|
||||
schedule_enabled: settings.schedule_enabled,
|
||||
cron_expression: settings.cron_expression,
|
||||
schedule_timezone: settings.schedule_timezone,
|
||||
retention_days: settings.retention_days,
|
||||
retention_count: settings.retention_count,
|
||||
next_run_at: settings.next_run_at,
|
||||
last_verified_at: settings.last_verified_at,
|
||||
updated_at: settings.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 更新 S3 存储配置请求;`secretAccessKey` 缺省表示保留旧值。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateBackupStorageRequest {
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub access_key_id: String,
|
||||
#[serde(default)]
|
||||
pub secret_access_key: Option<String>,
|
||||
pub prefix: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
/// 更新调度配置请求。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UpdateBackupScheduleRequest {
|
||||
pub schedule_enabled: bool,
|
||||
pub cron_expression: String,
|
||||
pub schedule_timezone: String,
|
||||
pub retention_days: u32,
|
||||
pub retention_count: u32,
|
||||
}
|
||||
|
||||
/// 备份记录 wire 视图。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupRecordView {
|
||||
pub id: String,
|
||||
pub trigger_kind: String,
|
||||
pub status: String,
|
||||
pub scheduled_at: Option<DateTime<Utc>>,
|
||||
pub object_key: String,
|
||||
pub size_bytes: Option<u64>,
|
||||
pub sha256: Option<String>,
|
||||
pub attempt_count: u32,
|
||||
pub error_code: Option<String>,
|
||||
pub error_message: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl From<BackupRecord> for BackupRecordView {
|
||||
fn from(record: BackupRecord) -> Self {
|
||||
Self {
|
||||
id: record.id,
|
||||
trigger_kind: record.trigger_kind.to_string(),
|
||||
status: record.status.to_string(),
|
||||
scheduled_at: record.scheduled_at,
|
||||
object_key: record.object_key,
|
||||
size_bytes: record.size_bytes,
|
||||
sha256: record.sha256,
|
||||
attempt_count: record.attempt_count,
|
||||
error_code: record.error_code,
|
||||
error_message: record.error_message,
|
||||
started_at: record.started_at,
|
||||
completed_at: record.completed_at,
|
||||
expires_at: record.expires_at,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 记录列表查询参数。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupRecordsQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub status: Option<String>,
|
||||
pub trigger: Option<String>,
|
||||
}
|
||||
|
||||
/// 下载地址请求。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct BackupIdRequest {
|
||||
pub backup_id: String,
|
||||
}
|
||||
|
||||
/// 下载地址响应。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DownloadUrlView {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub expires_in_seconds: u64,
|
||||
}
|
||||
|
||||
impl From<gateway_admin::model::backup::DownloadUrlResult> for DownloadUrlView {
|
||||
fn from(result: gateway_admin::model::backup::DownloadUrlResult) -> Self {
|
||||
Self {
|
||||
url: result.url,
|
||||
file_name: result.file_name,
|
||||
expires_in_seconds: result.expires_in.as_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 连接测试响应。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ConnectionTestView {
|
||||
pub ok: bool,
|
||||
pub stage: String,
|
||||
pub code: Option<String>,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl From<ConnectionTestResult> for ConnectionTestView {
|
||||
fn from(result: ConnectionTestResult) -> Self {
|
||||
Self {
|
||||
ok: result.ok,
|
||||
stage: result.stage.to_owned(),
|
||||
code: result.code.map(str::to_owned),
|
||||
message: result.message,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造固定 GET/POST 备份路由。
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/api/admin/settings/backups", get(backup_settings::<S>))
|
||||
.route(
|
||||
"/api/admin/settings/backups/storage/update",
|
||||
post(update_backup_storage::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/storage/test",
|
||||
post(test_backup_storage::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/schedule/update",
|
||||
post(update_backup_schedule::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/records",
|
||||
get(backup_records::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/create",
|
||||
post(create_backup::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/download-url",
|
||||
post(download_backup_url::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/settings/backups/delete",
|
||||
post(delete_backup::<S>),
|
||||
)
|
||||
}
|
||||
|
||||
async fn backup_settings<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.load_settings()
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_backup_storage<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<UpdateBackupStorageRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = UpdateBackupStorageCommand {
|
||||
endpoint: request.endpoint,
|
||||
region: request.region,
|
||||
bucket: request.bucket,
|
||||
access_key_id: request.access_key_id,
|
||||
secret_access_key: request.secret_access_key.map(SecretString::from),
|
||||
prefix: request.prefix,
|
||||
force_path_style: request.force_path_style,
|
||||
};
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.update_storage(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn test_backup_storage<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.test_connection(&auth.context().mutation_context())
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(ConnectionTestView::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn update_backup_schedule<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<UpdateBackupScheduleRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = UpdateBackupScheduleCommand {
|
||||
schedule_enabled: request.schedule_enabled,
|
||||
cron_expression: request.cron_expression,
|
||||
schedule_timezone: request.schedule_timezone,
|
||||
retention_days: request.retention_days,
|
||||
retention_count: request.retention_count,
|
||||
};
|
||||
let settings = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.update_schedule(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupSettingsView::from(settings)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn backup_records<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<BackupRecordsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let page_size = gateway_admin::model::PageSize::new(query.page_size.unwrap_or(20))
|
||||
.map_err(|_| AdminError::bad_request("Invalid pageSize"))?;
|
||||
let page = query.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(AdminError::bad_request("Invalid page"));
|
||||
}
|
||||
let status = match query.status.as_deref() {
|
||||
None => None,
|
||||
Some(value) => BackupStatus::parse(value)
|
||||
.map(Some)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid status"))?,
|
||||
};
|
||||
let trigger = match query.trigger.as_deref() {
|
||||
None => None,
|
||||
Some(value) => BackupTriggerKind::parse(value)
|
||||
.map(Some)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid trigger"))?,
|
||||
};
|
||||
let domain_query = gateway_admin::model::backup::BackupRecordListQuery {
|
||||
page,
|
||||
page_size,
|
||||
status,
|
||||
trigger,
|
||||
};
|
||||
let page_result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.list_records(domain_query)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let meta = PageMeta::new(
|
||||
page_result.page,
|
||||
u32::from(page_result.page_size.get()),
|
||||
page_result.total,
|
||||
page_result.total_pages(),
|
||||
);
|
||||
let items: Vec<BackupRecordView> = page_result
|
||||
.items
|
||||
.into_iter()
|
||||
.map(BackupRecordView::from)
|
||||
.collect();
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AdminPageData::new(items, meta)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn create_backup<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
body: Option<Json<CreateBackupRequest>>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let expires_at = body
|
||||
.and_then(|body| body.0.expires_in_days)
|
||||
.filter(|&days| days > 0)
|
||||
.map(|days| Utc::now() + chrono::Duration::days(i64::from(days)));
|
||||
let record = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.create_backup(&auth.context().mutation_context(), expires_at)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::ACCEPTED,
|
||||
AdminEnvelope::ok(BackupRecordView::from(record)),
|
||||
))
|
||||
}
|
||||
|
||||
/// 创建备份请求;`expiresInDays` 为 0 或缺省表示不设置过期时间。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CreateBackupRequest {
|
||||
pub expires_in_days: Option<u32>,
|
||||
}
|
||||
|
||||
async fn download_backup_url<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<BackupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let result = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.download_url(&auth.context().mutation_context(), &request.backup_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(DownloadUrlView::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_backup<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<BackupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let record = state
|
||||
.admin_services()
|
||||
.backups()
|
||||
.delete_backup(&auth.context().mutation_context(), &request.backup_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BackupRecordView::from(record)),
|
||||
))
|
||||
}
|
||||
|
||||
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
|
||||
map_admin_service_error(error, "Backup service unavailable")
|
||||
}
|
||||
@@ -9,8 +9,10 @@ use axum::{
|
||||
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backups;
|
||||
pub mod client_keys;
|
||||
pub mod observability;
|
||||
pub mod presenter;
|
||||
pub mod settings;
|
||||
pub mod system;
|
||||
pub mod wire;
|
||||
@@ -29,6 +31,7 @@ where
|
||||
Router::new()
|
||||
.merge(accounts::router::<S>())
|
||||
.merge(auth::router::<S>())
|
||||
.merge(backups::router::<S>())
|
||||
.merge(client_keys::router::<S>())
|
||||
.merge(observability::router::<S>())
|
||||
.merge(settings::router::<S>())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
//! Dashboard、用量与诊断查询的 wire 映射和固定路由。
|
||||
|
||||
use std::{collections::BTreeMap, num::NonZeroU32};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::get,
|
||||
};
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
mod presenter;
|
||||
mod query;
|
||||
mod routes;
|
||||
mod wire;
|
||||
|
||||
pub(crate) use presenter::*;
|
||||
pub use query::*;
|
||||
pub use routes::*;
|
||||
pub use wire::*;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,453 @@
|
||||
//! 查询 wire 类型、校验与领域命令映射。
|
||||
|
||||
use super::*;
|
||||
|
||||
use base64::Engine as _;
|
||||
|
||||
/// 观测列表默认页大小。
|
||||
pub const DEFAULT_PAGE_SIZE: u16 = 50;
|
||||
/// 观测列表允许的最大页大小。
|
||||
pub const MAX_PAGE_SIZE: u16 = 100;
|
||||
/// 游标编码在 HTTP 层解码前允许的最大长度。
|
||||
pub const MAX_CURSOR_BYTES: usize = 1024;
|
||||
|
||||
/// Dashboard 查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DashboardQuery {
|
||||
pub kind: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl DashboardQuery {
|
||||
/// 解析 dashboard 趋势类型。
|
||||
pub fn trend_kind(&self) -> Result<TrendKind, WireValidationError> {
|
||||
TrendKind::parse(self.kind.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 逻辑请求列表、汇总和洞察查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub outcome: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl UsageQuery {
|
||||
/// 校验分页字段,不改变仓储层的分页 owner。
|
||||
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let page = self.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(WireValidationError::new("page"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((page, page_size))
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界;编码和排序语义由应用层负责。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 详情查询参数。
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DetailQuery {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl DetailQuery {
|
||||
/// 校验详情 ID,错误不回显输入值。
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_text(&self.id, "id")
|
||||
}
|
||||
}
|
||||
|
||||
/// 诊断聚合查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct DiagnosticsQuery {
|
||||
pub dimension: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub search: Option<String>,
|
||||
}
|
||||
|
||||
impl DiagnosticsQuery {
|
||||
/// 解析诊断维度。
|
||||
pub fn dimension(&self) -> Result<DiagnosticDimension, WireValidationError> {
|
||||
DiagnosticDimension::parse(self.dimension.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// 运维错误查询参数。
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct OpsQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub cursor: Option<String>,
|
||||
pub kind: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub failure_class: Option<String>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
impl OpsQuery {
|
||||
/// 校验分页字段。
|
||||
pub fn validate_page(&self) -> Result<(u32, u16), WireValidationError> {
|
||||
let page = self.page.unwrap_or(1);
|
||||
if page == 0 {
|
||||
return Err(WireValidationError::new("page"));
|
||||
}
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
Ok((page, page_size))
|
||||
}
|
||||
|
||||
/// 校验游标的 wire 边界。
|
||||
pub fn validate_cursor(&self) -> Result<(), WireValidationError> {
|
||||
validate_cursor(self.cursor.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Dashboard 趋势类型。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TrendKind {
|
||||
Usage,
|
||||
Latency,
|
||||
Errors,
|
||||
}
|
||||
|
||||
impl TrendKind {
|
||||
/// 从 query 值解析趋势类型。
|
||||
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
|
||||
match trimmed(value) {
|
||||
None | Some("usage") => Ok(Self::Usage),
|
||||
Some("latency") => Ok(Self::Latency),
|
||||
Some("errors") => Ok(Self::Errors),
|
||||
Some(_) => Err(WireValidationError::new("kind")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 诊断聚合维度。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DiagnosticDimension {
|
||||
Model,
|
||||
Account,
|
||||
ApiKey,
|
||||
Provider,
|
||||
Transport,
|
||||
Failure,
|
||||
Status,
|
||||
}
|
||||
|
||||
impl DiagnosticDimension {
|
||||
/// 从 query 值解析诊断维度。
|
||||
pub fn parse(value: Option<&str>) -> Result<Self, WireValidationError> {
|
||||
match trimmed(value) {
|
||||
None | Some("model") => Ok(Self::Model),
|
||||
Some("account") => Ok(Self::Account),
|
||||
Some("apiKey" | "api_key") => Ok(Self::ApiKey),
|
||||
Some("provider") => Ok(Self::Provider),
|
||||
Some("transport") => Ok(Self::Transport),
|
||||
Some("failureClass" | "failure_class") => Ok(Self::Failure),
|
||||
Some("status") => Ok(Self::Status),
|
||||
Some(_) => Err(WireValidationError::new("dimension")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回终态响应中的稳定维度名称。
|
||||
#[must_use]
|
||||
pub const fn display_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Model => "model",
|
||||
Self::Account => "account",
|
||||
Self::ApiKey => "apiKey",
|
||||
Self::Provider => "provider",
|
||||
Self::Transport => "transport",
|
||||
Self::Failure => "failureClass",
|
||||
Self::Status => "status",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析 RFC3339 时间;错误不回显原始值。
|
||||
pub fn parse_datetime(value: Option<&str>) -> Result<Option<DateTime<Utc>>, WireValidationError> {
|
||||
let Some(value) = trimmed(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|value| Some(value.with_timezone(&Utc)))
|
||||
.map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
/// 解析 HTTP 状态码。
|
||||
pub fn parse_status(value: Option<i64>) -> Result<Option<u16>, WireValidationError> {
|
||||
value
|
||||
.map(|value| {
|
||||
u16::try_from(value)
|
||||
.ok()
|
||||
.filter(|value| (100..=599).contains(value))
|
||||
.ok_or_else(|| WireValidationError::new("statusCode"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// 解析尝试序号。
|
||||
pub fn parse_attempt_index(value: Option<i64>) -> Result<Option<u32>, WireValidationError> {
|
||||
value
|
||||
.map(|value| {
|
||||
u32::try_from(value)
|
||||
.ok()
|
||||
.filter(|value| *value > 0 && *value <= i32::MAX as u32)
|
||||
.ok_or_else(|| WireValidationError::new("attemptIndex"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn require_text(value: &str, field: &'static str) -> Result<(), WireValidationError> {
|
||||
if value.trim().is_empty() {
|
||||
Err(WireValidationError::new(field))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_cursor(value: Option<&str>) -> Result<(), WireValidationError> {
|
||||
if value.is_some_and(|value| value.is_empty() || value.len() > MAX_CURSOR_BYTES) {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn trimmed(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn domain_trend_kind(kind: TrendKind) -> domain::TrendKind {
|
||||
match kind {
|
||||
TrendKind::Usage => domain::TrendKind::Usage,
|
||||
TrendKind::Latency => domain::TrendKind::Latency,
|
||||
TrendKind::Errors => domain::TrendKind::Errors,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn domain_diagnostic_dimension(
|
||||
dimension: DiagnosticDimension,
|
||||
) -> domain::DiagnosticDimension {
|
||||
match dimension {
|
||||
DiagnosticDimension::Model => domain::DiagnosticDimension::Model,
|
||||
DiagnosticDimension::Account => domain::DiagnosticDimension::Account,
|
||||
DiagnosticDimension::ApiKey => domain::DiagnosticDimension::ApiKey,
|
||||
DiagnosticDimension::Provider => domain::DiagnosticDimension::Provider,
|
||||
DiagnosticDimension::Transport => domain::DiagnosticDimension::Transport,
|
||||
DiagnosticDimension::Failure => domain::DiagnosticDimension::Failure,
|
||||
DiagnosticDimension::Status => domain::DiagnosticDimension::Status,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_outcome(
|
||||
value: Option<String>,
|
||||
) -> Result<Option<domain::RequestOutcome>, WireValidationError> {
|
||||
non_empty(value)
|
||||
.map(|value| {
|
||||
domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn usage_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start = parse_datetime(start)?.unwrap_or(end - Duration::days(7));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_today_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start = parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, WireValidationError> {
|
||||
let outcome = query.outcome.clone().or_else(|| {
|
||||
non_empty(query.kind.clone()).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"running" | "succeeded" | "failed" | "cancelled" | "incomplete"
|
||||
)
|
||||
})
|
||||
});
|
||||
Ok(domain::UsageFilter {
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
model: non_empty(query.model.clone()),
|
||||
outcome: request_outcome(outcome)?,
|
||||
status_code: parse_status(query.status_code)?,
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
search: non_empty(query.search.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn usage_command(
|
||||
query: &UsageQuery,
|
||||
) -> Result<(domain::UsageQuery, u32, u16), WireValidationError> {
|
||||
let (page, page_size) = query.validate_page()?;
|
||||
query.validate_cursor()?;
|
||||
let page_number = NonZeroU32::new(page)
|
||||
.map(domain::PageNumber::new)
|
||||
.ok_or_else(|| WireValidationError::new("page"))?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
Ok((
|
||||
domain::UsageQuery {
|
||||
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
|
||||
filter: usage_filter(query)?,
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
page: page_number,
|
||||
page_size: page_size_value,
|
||||
},
|
||||
page,
|
||||
page_size,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn ops_command(
|
||||
query: &OpsQuery,
|
||||
) -> Result<(domain::OpsErrorQuery, u32, u16), WireValidationError> {
|
||||
let (page, page_size) = query.validate_page()?;
|
||||
query.validate_cursor()?;
|
||||
let page_number = NonZeroU32::new(page)
|
||||
.map(domain::PageNumber::new)
|
||||
.ok_or_else(|| WireValidationError::new("page"))?;
|
||||
let page_size_value =
|
||||
DomainPageSize::new(page_size).map_err(|_| WireValidationError::new("pageSize"))?;
|
||||
let status_code = parse_status(
|
||||
query
|
||||
.upstream_status_code
|
||||
.or(query.client_status_code)
|
||||
.or(query.status_code),
|
||||
)?;
|
||||
Ok((
|
||||
domain::OpsErrorQuery {
|
||||
range: usage_range(query.start_time.as_deref(), query.end_time.as_deref())?,
|
||||
filter: domain::OpsErrorFilter {
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()).or_else(|| non_empty(query.kind.clone())),
|
||||
model: non_empty(query.model.clone()),
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
failure_kind: non_empty(query.failure_class.clone()),
|
||||
status_code,
|
||||
search: non_empty(query.search.clone()),
|
||||
},
|
||||
cursor: decode_observability_cursor(query.cursor.as_deref())?,
|
||||
page: page_number,
|
||||
page_size: page_size_value,
|
||||
},
|
||||
page,
|
||||
page_size,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn decode_observability_cursor(
|
||||
value: Option<&str>,
|
||||
) -> Result<Option<domain::ObservabilityCursor>, WireValidationError> {
|
||||
value
|
||||
.map(|encoded| {
|
||||
if encoded.is_empty() || encoded.len() > MAX_CURSOR_BYTES {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
let bytes = URL_SAFE_NO_PAD
|
||||
.decode(encoded)
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
let wire: CursorWire =
|
||||
serde_json::from_slice(&bytes).map_err(|_| WireValidationError::new("cursor"))?;
|
||||
if wire.stable_id.trim().is_empty() {
|
||||
return Err(WireValidationError::new("cursor"));
|
||||
}
|
||||
Ok(domain::ObservabilityCursor {
|
||||
observed_at: wire.observed_at,
|
||||
stable_id: wire.stable_id,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn encode_observability_cursor(
|
||||
cursor: &domain::ObservabilityCursor,
|
||||
) -> Result<String, WireValidationError> {
|
||||
let bytes = serde_json::to_vec(&CursorWire {
|
||||
observed_at: cursor.observed_at,
|
||||
stable_id: cursor.stable_id.clone(),
|
||||
})
|
||||
.map_err(|_| WireValidationError::new("cursor"))?;
|
||||
Ok(URL_SAFE_NO_PAD.encode(bytes))
|
||||
}
|
||||
|
||||
pub(crate) fn non_empty(value: Option<String>) -> Option<String> {
|
||||
value
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
//! 固定 `/api/admin` 观测路由与 handler。
|
||||
|
||||
use super::*;
|
||||
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/api/admin/dashboard/summary", get(dashboard_summary::<S>))
|
||||
.route("/api/admin/dashboard/trend", get(dashboard_trend::<S>))
|
||||
.route("/api/admin/usage/records", get(usage_records::<S>))
|
||||
.route(
|
||||
"/api/admin/usage/records/detail",
|
||||
get(usage_record_detail::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/records/summary",
|
||||
get(usage_records_summary::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/insights/overview",
|
||||
get(usage_insights_overview::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/usage/insights/diagnostics",
|
||||
get(usage_insights_diagnostics::<S>),
|
||||
)
|
||||
.route("/api/admin/operations/errors", get(ops_errors::<S>))
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_summary<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DashboardQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
// 概览默认按中国时区当日统计,与单独趋势接口保持同一口径。
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.dashboard_summary(range, domain_trend_kind(kind))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(dashboard_view(result, kind)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn dashboard_trend<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DashboardQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.dashboard_trend(range, domain_trend_kind(kind))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(trend_view(result, kind)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_records<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (command, page, page_size) = usage_command(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_records(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = usage_page_view(result, page, page_size)
|
||||
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_record_detail<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DetailQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
query.validate().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_record_detail(query.id.trim())
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_detail_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_records_summary<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_summary_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_overview<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<UsageQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = usage_filter(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.usage_insights(range, filter)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(usage_insights_view(result)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn usage_insights_diagnostics<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<DiagnosticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let dimension = query.dimension().map_err(map_wire_error)?;
|
||||
let range = usage_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
model: non_empty(query.model),
|
||||
status_code: parse_status(query.status_code).map_err(map_wire_error)?,
|
||||
search: non_empty(query.search),
|
||||
..domain::UsageFilter::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.diagnostics(range, filter, domain_diagnostic_dimension(dimension))
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(diagnostics_view(result, dimension)),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn ops_errors<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<OpsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (command, page, page_size) = ops_command(&query).map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
.ops_errors(command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = ops_page_view(result, page, page_size)
|
||||
.map_err(|_| AdminError::internal("Failed to encode observability cursor"))?;
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
@@ -0,0 +1,712 @@
|
||||
//! 响应 DTO 与固定 wire 形状。
|
||||
|
||||
use super::*;
|
||||
|
||||
/// 观测列表游标的稳定 wire 形状。
|
||||
#[derive(Clone, Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CursorWire {
|
||||
pub observed_at: DateTime<Utc>,
|
||||
pub stable_id: String,
|
||||
}
|
||||
|
||||
/// 观测列表响应数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PageData<T> {
|
||||
pub items: Vec<T>,
|
||||
pub page: PageMeta,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
/// Token 详情展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TokenDetailsView {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub input_tokens_display: String,
|
||||
pub output_tokens_display: String,
|
||||
pub cached_tokens_display: String,
|
||||
pub cache_write_tokens_display: String,
|
||||
pub reasoning_tokens_display: String,
|
||||
pub image_input_tokens_display: String,
|
||||
pub image_output_tokens_display: String,
|
||||
pub total_tokens_display: String,
|
||||
}
|
||||
|
||||
/// 按货币展示的成本。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CostView {
|
||||
pub currency: String,
|
||||
pub estimated_amount: String,
|
||||
}
|
||||
|
||||
/// 成本覆盖状态计数。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CostCoverageView {
|
||||
pub known: u64,
|
||||
pub partial: u64,
|
||||
pub unknown: u64,
|
||||
pub not_billable: u64,
|
||||
}
|
||||
|
||||
/// Provider 受控价格规则生成的单次请求费用明细展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BillingView {
|
||||
pub input_amount_display: String,
|
||||
pub output_amount_display: String,
|
||||
pub cache_read_amount_display: String,
|
||||
pub cache_write_amount_display: String,
|
||||
pub standard_amount_display: String,
|
||||
pub total_amount_display: String,
|
||||
pub input_price_display: String,
|
||||
pub output_price_display: String,
|
||||
pub cache_read_price_display: String,
|
||||
pub cache_write_price_display: String,
|
||||
pub service_tier_display: String,
|
||||
pub multiplier_display: String,
|
||||
}
|
||||
|
||||
/// 单条逻辑请求展示。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordView {
|
||||
pub id: String,
|
||||
pub request_id: String,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub route: String,
|
||||
pub model: String,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub protocol: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub http_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub client_status_code: Option<i64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub websocket_pool: Option<WebSocketPoolMetadataView>,
|
||||
pub image_generation_requested: bool,
|
||||
pub image_generation_succeeded: Option<bool>,
|
||||
pub latency_details: UsageLatencyDetailsView,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub attempt_count: u64,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub cache_write_tokens: Option<u64>,
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
pub image_input_tokens: Option<u64>,
|
||||
pub image_output_tokens: Option<u64>,
|
||||
pub message: String,
|
||||
/// Provider 安全观测;Core 字段由顶层字段提供,不再复制进 metadata。
|
||||
pub metadata: BTreeMap<String, Value>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_at_display: String,
|
||||
pub client_ip: Option<String>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
pub compact: Option<bool>,
|
||||
pub request_kind: Option<String>,
|
||||
pub subagent_kind: Option<String>,
|
||||
pub token_details: TokenDetailsView,
|
||||
pub billing: Option<BillingView>,
|
||||
pub costs: Vec<CostView>,
|
||||
pub cost_coverage: CostCoverageView,
|
||||
pub first_token_latency_ms: Option<u64>,
|
||||
pub first_token_latency_ms_display: String,
|
||||
pub latency_ms_display: String,
|
||||
pub logical_outcome: String,
|
||||
}
|
||||
|
||||
/// WebSocket 池决策的稳定形状;与 Provider 选择逻辑无关。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct WebSocketPoolMetadataView {
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
/// 逻辑请求在上游和输出阶段测得的时延事实。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageLatencyDetailsView {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub transport_decision_wait_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ws_connect_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_headers_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_event_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_reasoning_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_text_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_token_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub openai_processing_ms: Option<u64>,
|
||||
}
|
||||
|
||||
/// 单次上游尝试展示。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageAttemptView {
|
||||
pub id: String,
|
||||
pub attempt_index: u32,
|
||||
pub trigger: String,
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub transport: String,
|
||||
pub send_state: String,
|
||||
pub outcome: String,
|
||||
pub downstream_committed: bool,
|
||||
pub status_code: Option<u16>,
|
||||
pub provider_error_code: Option<String>,
|
||||
pub failure_class: Option<String>,
|
||||
pub cost_estimate_status: String,
|
||||
pub estimated_cost_amount: Option<String>,
|
||||
pub estimated_cost_currency: Option<String>,
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cached_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub first_token_ms: Option<u64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub credential_name: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub account_email: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// 逻辑请求详情与其尝试列表。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageRecordDetailView {
|
||||
#[serde(flatten)]
|
||||
pub request: UsageRecordView,
|
||||
pub attempts: Vec<UsageAttemptView>,
|
||||
/// 尝试列表是否完整;best-effort 下恒为 false。
|
||||
pub attempts_complete: bool,
|
||||
}
|
||||
|
||||
/// Dashboard 趋势数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrendData {
|
||||
pub kind: TrendKind,
|
||||
pub points: Vec<TrendPointView>,
|
||||
pub summary: Vec<TrendSummaryView>,
|
||||
}
|
||||
|
||||
/// Dashboard 单个趋势桶。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrendPointView {
|
||||
pub time: String,
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub requests: String,
|
||||
pub requests_value: u64,
|
||||
pub input_tokens: String,
|
||||
pub input_tokens_value: u64,
|
||||
pub output_tokens: String,
|
||||
pub output_tokens_value: u64,
|
||||
pub cached_tokens: String,
|
||||
pub cached_tokens_value: u64,
|
||||
pub cache_hit_rate_value: f64,
|
||||
pub tokens_value: u64,
|
||||
pub errors: String,
|
||||
pub errors_value: u64,
|
||||
pub latency: String,
|
||||
pub latency_value: Option<u64>,
|
||||
pub max_latency: String,
|
||||
pub max_latency_value: Option<u64>,
|
||||
pub min_latency: String,
|
||||
pub min_latency_value: Option<u64>,
|
||||
pub success_rate: String,
|
||||
pub success_rate_value: Option<f64>,
|
||||
}
|
||||
|
||||
/// Dashboard 趋势摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct TrendSummaryView {
|
||||
pub label: String,
|
||||
pub value: String,
|
||||
pub ratio: Option<String>,
|
||||
}
|
||||
|
||||
/// Dashboard 卡片集合。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCardsView {
|
||||
pub credentials: DashboardCredentialsCardView,
|
||||
pub traffic: DashboardTrafficCardView,
|
||||
pub tokens: DashboardTokensCardView,
|
||||
pub cache: DashboardCacheCardView,
|
||||
}
|
||||
|
||||
/// Dashboard 上游凭据卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCredentialsCardView {
|
||||
pub total: String,
|
||||
pub total_value: u64,
|
||||
pub available: String,
|
||||
pub available_value: u64,
|
||||
pub unavailable: String,
|
||||
pub unavailable_value: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 流量卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardTrafficCardView {
|
||||
pub today_requests: String,
|
||||
pub today_requests_value: u64,
|
||||
pub yesterday_requests_value: u64,
|
||||
pub total_requests: String,
|
||||
}
|
||||
|
||||
/// Dashboard token 卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardTokensCardView {
|
||||
pub today_tokens: String,
|
||||
pub today_tokens_value: u64,
|
||||
pub yesterday_tokens_value: u64,
|
||||
pub total_tokens: String,
|
||||
pub total_billing_amount_usd: String,
|
||||
}
|
||||
|
||||
/// Dashboard 缓存卡片。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCacheCardView {
|
||||
pub today_hit_rate: String,
|
||||
pub today_hit_rate_value: Option<f64>,
|
||||
pub yesterday_hit_rate_value: Option<f64>,
|
||||
pub total_hit_rate: String,
|
||||
pub total_cached_tokens: String,
|
||||
pub average_first_token_latency_ms: String,
|
||||
}
|
||||
|
||||
/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardAccountUsageView {
|
||||
pub id: String,
|
||||
pub provider: String,
|
||||
pub authentication_kind: String,
|
||||
pub email: String,
|
||||
pub plan_type: Option<String>,
|
||||
pub tokens: String,
|
||||
pub request_count: u64,
|
||||
pub request_buckets: Vec<DashboardAccountRequestBucketView>,
|
||||
pub quota_used_percent: Option<f64>,
|
||||
pub last_used: String,
|
||||
}
|
||||
|
||||
/// Dashboard 账号单小时请求数。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardAccountRequestBucketView {
|
||||
pub bucket_start: DateTime<Utc>,
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// Provider 账号池的持久事实汇总。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
}
|
||||
|
||||
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardCapacityInfoView {
|
||||
pub max_concurrent_per_account: u64,
|
||||
pub total_slots: u64,
|
||||
pub used_slots: Option<u64>,
|
||||
pub available_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 逻辑请求指标展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RequestMetricsView {
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub reasoning_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 上游尝试指标展示。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AttemptMetricsView {
|
||||
pub attempt_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub rate_limited_count: u64,
|
||||
pub auth_failure_count: u64,
|
||||
pub provider5xx_count: u64,
|
||||
pub cost_coverage: CostCoverageView,
|
||||
pub costs: Vec<CostView>,
|
||||
}
|
||||
|
||||
/// 健康时间线单点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealthTimelinePointView {
|
||||
pub time: String,
|
||||
pub status: String,
|
||||
pub reliability_display: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 健康时间线。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HealthTimelineView {
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub reliability_display: String,
|
||||
pub status: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub points: Vec<HealthTimelinePointView>,
|
||||
}
|
||||
|
||||
/// Dashboard 展示的实际 Provider 上游请求身份。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireProfileView {
|
||||
pub provider: String,
|
||||
pub product: String,
|
||||
pub version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub build: Option<String>,
|
||||
pub target: DashboardWireTargetView,
|
||||
pub user_agent: String,
|
||||
pub attributes: Vec<DashboardWireAttributeView>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verified_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub release: Option<DashboardDesktopReleaseView>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireAttributeView {
|
||||
pub label: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireTargetView {
|
||||
pub os_type: String,
|
||||
pub os_version: String,
|
||||
pub arch: String,
|
||||
pub terminal: String,
|
||||
}
|
||||
|
||||
/// 发布检查与启动画像分离;未检查时使用明确的 `unchecked` 状态。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardDesktopReleaseView {
|
||||
pub status: DashboardDesktopReleaseStatusView,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub checked_at: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latest_version: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub latest_build: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Dashboard 发布检查的稳定 wire 状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum DashboardDesktopReleaseStatusView {
|
||||
Unchecked,
|
||||
Aligned,
|
||||
ReviewRequired,
|
||||
CheckFailed,
|
||||
}
|
||||
|
||||
impl From<domain::DesktopReleaseStatus> for DashboardDesktopReleaseStatusView {
|
||||
fn from(status: domain::DesktopReleaseStatus) -> Self {
|
||||
match status {
|
||||
domain::DesktopReleaseStatus::Unchecked => Self::Unchecked,
|
||||
domain::DesktopReleaseStatus::Current => Self::Aligned,
|
||||
domain::DesktopReleaseStatus::UpdateAvailable => Self::ReviewRequired,
|
||||
domain::DesktopReleaseStatus::Failed => Self::CheckFailed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dashboard 汇总响应数据。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardDataView {
|
||||
pub cards: DashboardCardsView,
|
||||
pub trend: TrendData,
|
||||
pub health_timeline: HealthTimelineView,
|
||||
pub wire_profiles: Vec<DashboardWireProfileView>,
|
||||
pub account_usage: Vec<DashboardAccountUsageView>,
|
||||
pub usage_records: Vec<UsageRecordView>,
|
||||
pub pool_summary: DashboardPoolSummaryView,
|
||||
pub capacity_info: DashboardCapacityInfoView,
|
||||
pub rotation_strategy: String,
|
||||
}
|
||||
|
||||
/// 用量汇总响应数据。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageSummaryView {
|
||||
pub total_requests: String,
|
||||
pub input_tokens: String,
|
||||
pub output_tokens: String,
|
||||
pub cached_tokens: String,
|
||||
pub cache_write_tokens: String,
|
||||
pub total_tokens: String,
|
||||
pub average_latency_ms: String,
|
||||
pub logical_requests: RequestMetricsView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
}
|
||||
|
||||
/// 洞察健康趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewHealthPointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub error_rate: f64,
|
||||
}
|
||||
|
||||
/// 洞察健康摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewHealthView {
|
||||
pub total_requests: u64,
|
||||
pub success_requests: u64,
|
||||
pub failed_requests: u64,
|
||||
pub cancelled_requests: u64,
|
||||
pub incomplete_requests: u64,
|
||||
pub caller_error_requests: u64,
|
||||
pub success_rate: f64,
|
||||
pub request_change_rate: Option<f64>,
|
||||
pub success_rate_change: Option<f64>,
|
||||
pub points: Vec<OverviewHealthPointView>,
|
||||
}
|
||||
|
||||
/// 洞察性能趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewPerformancePointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub latency_p50_ms: Option<f64>,
|
||||
pub latency_p95_ms: Option<f64>,
|
||||
pub latency_p99_ms: Option<f64>,
|
||||
pub first_token_p50_ms: Option<f64>,
|
||||
pub first_token_p95_ms: Option<f64>,
|
||||
pub first_token_p99_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// 洞察性能摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewPerformanceView {
|
||||
pub latency_p50_ms: Option<f64>,
|
||||
pub latency_p95_ms: Option<f64>,
|
||||
pub latency_p99_ms: Option<f64>,
|
||||
pub first_token_p50_ms: Option<f64>,
|
||||
pub first_token_p95_ms: Option<f64>,
|
||||
pub first_token_p99_ms: Option<f64>,
|
||||
pub latency_coverage: f64,
|
||||
pub first_token_coverage: f64,
|
||||
pub points: Vec<OverviewPerformancePointView>,
|
||||
}
|
||||
|
||||
/// 洞察成本趋势点。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewCostPointView {
|
||||
pub bucket: DateTime<Utc>,
|
||||
pub label: String,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub standard_cost: Option<String>,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// 洞察成本摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OverviewCostView {
|
||||
pub estimated_cost: Option<String>,
|
||||
pub standard_cost: Option<String>,
|
||||
pub cost_per_request: Option<String>,
|
||||
pub tokens_per_request: f64,
|
||||
pub cached_token_rate: f64,
|
||||
pub cache_hit_request_rate: Option<f64>,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub points: Vec<OverviewCostPointView>,
|
||||
pub costs: Vec<CostView>,
|
||||
pub coverage: CostCoverageView,
|
||||
}
|
||||
|
||||
/// Provider 维度洞察摘要。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ProviderOverviewView {
|
||||
pub provider: String,
|
||||
pub request_count: u64,
|
||||
pub attempt_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 用量洞察总响应。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UsageInsightsOverviewView {
|
||||
pub granularity: String,
|
||||
pub health: OverviewHealthView,
|
||||
pub performance: OverviewPerformanceView,
|
||||
pub cost: OverviewCostView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
pub providers: Vec<ProviderOverviewView>,
|
||||
}
|
||||
|
||||
/// 诊断聚合项目。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticItemView {
|
||||
pub key: String,
|
||||
pub name: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub error_count: u64,
|
||||
pub error_rate: f64,
|
||||
pub request_share: f64,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
pub latency_p95_ms: Option<u64>,
|
||||
pub estimated_cost: Option<String>,
|
||||
pub attempt_count: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// 诊断聚合响应。
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DiagnosticsView {
|
||||
pub dimension: String,
|
||||
pub items: Vec<DiagnosticItemView>,
|
||||
}
|
||||
|
||||
/// 运维错误项目。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OpsErrorView {
|
||||
pub id: String,
|
||||
pub request_id: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: String,
|
||||
pub model: Option<String>,
|
||||
pub client_status_code: Option<i64>,
|
||||
pub upstream_status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<u32>,
|
||||
pub failure_class: String,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub message: String,
|
||||
pub metadata: OpsErrorMetadataView,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_at_display: String,
|
||||
}
|
||||
|
||||
/// 运维错误安全元数据;只保留可查询的标识,不回显秘密材料。
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OpsErrorMetadataView {
|
||||
pub source: String,
|
||||
pub component: String,
|
||||
pub attempt_id: Option<String>,
|
||||
pub account_label: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
//! Admin API 的展示层辅助:数值格式化。
|
||||
//!
|
||||
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
|
||||
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
|
||||
|
||||
/// 千分位格式化。
|
||||
#[must_use]
|
||||
pub fn format_number(value: u64) -> String {
|
||||
let text = value.to_string();
|
||||
let mut output = String::with_capacity(text.len() + text.len() / 3);
|
||||
for (index, character) in text.chars().rev().enumerate() {
|
||||
if index > 0 && index % 3 == 0 {
|
||||
output.push(',');
|
||||
}
|
||||
output.push(character);
|
||||
}
|
||||
output.chars().rev().collect()
|
||||
}
|
||||
|
||||
/// 紧凑格式化:≥1000 时用 K/M/B/T/P 后缀,否则千分位。
|
||||
#[must_use]
|
||||
pub fn format_compact_number(value: u64) -> String {
|
||||
if value < 1_000 {
|
||||
return format_number(value);
|
||||
}
|
||||
for (suffix, threshold) in [
|
||||
("P", 1_000_000_000_000_000_u64),
|
||||
("T", 1_000_000_000_000_u64),
|
||||
("B", 1_000_000_000_u64),
|
||||
("M", 1_000_000_u64),
|
||||
("K", 1_000_u64),
|
||||
] {
|
||||
if value >= threshold {
|
||||
let scaled = value as f64 / threshold as f64;
|
||||
return format!("{scaled:.1}{suffix}").replace(".0", "");
|
||||
}
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
@@ -72,10 +72,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
if i64::try_from(self.request_interval_ms).is_err() {
|
||||
return Err(WireValidationError::new("requestIntervalMs"));
|
||||
}
|
||||
if !matches!(
|
||||
self.rotation_strategy.as_str(),
|
||||
"smart" | "quota_reset_priority" | "round_robin" | "sticky"
|
||||
) {
|
||||
if RotationStrategy::parse(&self.rotation_strategy).is_none() {
|
||||
return Err(WireValidationError::new("rotationStrategy"));
|
||||
}
|
||||
Ok(())
|
||||
@@ -91,7 +88,7 @@ impl UpdateRuntimeSettingsRequest {
|
||||
max_concurrent_per_account: u32::try_from(self.max_concurrent_per_account)
|
||||
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
|
||||
request_interval_ms: self.request_interval_ms,
|
||||
rotation_strategy: parse_rotation_strategy(&self.rotation_strategy)
|
||||
rotation_strategy: RotationStrategy::parse(&self.rotation_strategy)
|
||||
.ok_or_else(|| WireValidationError::new("rotationStrategy"))?,
|
||||
usage_retention_days: u32::try_from(self.usage_retention_days)
|
||||
.map_err(|_| WireValidationError::new("settingsUsageRetentionOverflow"))?,
|
||||
@@ -111,7 +108,7 @@ impl From<RuntimeSettings> for RuntimeSettingsView {
|
||||
refresh_concurrency: u64::from(settings.refresh_concurrency),
|
||||
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
|
||||
request_interval_ms: settings.request_interval_ms,
|
||||
rotation_strategy: rotation_strategy_name(settings.rotation_strategy).to_owned(),
|
||||
rotation_strategy: settings.rotation_strategy.as_str().to_owned(),
|
||||
usage_retention_days: u64::from(settings.usage_retention_days),
|
||||
ops_event_retention_days: u64::from(settings.ops_event_retention_days),
|
||||
audit_retention_days: u64::from(settings.audit_retention_days),
|
||||
@@ -319,25 +316,6 @@ fn wire_model_mappings(mappings: DomainModelMappings) -> ModelMappings {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_rotation_strategy(value: &str) -> Option<RotationStrategy> {
|
||||
match value {
|
||||
"smart" => Some(RotationStrategy::Smart),
|
||||
"quota_reset_priority" => Some(RotationStrategy::QuotaResetPriority),
|
||||
"round_robin" => Some(RotationStrategy::RoundRobin),
|
||||
"sticky" => Some(RotationStrategy::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const fn rotation_strategy_name(strategy: RotationStrategy) -> &'static str {
|
||||
match strategy {
|
||||
RotationStrategy::Smart => "smart",
|
||||
RotationStrategy::QuotaResetPriority => "quota_reset_priority",
|
||||
RotationStrategy::RoundRobin => "round_robin",
|
||||
RotationStrategy::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_model_name(value: &str, max_len: usize) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= max_len
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
use axum::http::{HeaderName, HeaderValue, Method, StatusCode};
|
||||
use axum::http::{HeaderName, HeaderValue, Method, Request, StatusCode};
|
||||
use axum::routing::get;
|
||||
use gateway_admin::AdminServices;
|
||||
use gateway_core::engine::execution::ExecutionService;
|
||||
@@ -166,10 +166,32 @@ pub fn initialize(
|
||||
Duration::from_secs(seconds),
|
||||
));
|
||||
}
|
||||
// Trace 必须在 SetRequestId 内侧,span 才能捕获本服务生成的 request_id;
|
||||
// 默认 DEBUG span 会被 info 日志过滤器丢弃,这里显式用 info_span。
|
||||
let trace_layer = TraceLayer::new_for_http().make_span_with({
|
||||
let request_id_header = request_id_header.clone();
|
||||
move |request: &Request<_>| {
|
||||
let request_id = request
|
||||
.headers()
|
||||
.get(&request_id_header)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default();
|
||||
tracing::info_span!(
|
||||
"request",
|
||||
request_id = %request_id,
|
||||
method = %request.method().as_str(),
|
||||
uri = %request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(axum::http::uri::PathAndQuery::as_str)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
});
|
||||
let router = router
|
||||
.layer(PropagateRequestIdLayer::new(request_id_header.clone()))
|
||||
.layer(trace_layer)
|
||||
.layer(SetRequestIdLayer::new(request_id_header, MakeRequestUuid))
|
||||
.layer(TraceLayer::new_for_http())
|
||||
.with_state(state);
|
||||
Ok(ApiBundle { router })
|
||||
}
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
use axum::{
|
||||
Json,
|
||||
http::StatusCode,
|
||||
body::Body,
|
||||
http::{HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::{EngineError, UpstreamSendState},
|
||||
error::{GatewayError, GatewayErrorKind, ProviderErrorKind},
|
||||
engine::EngineError,
|
||||
error::{GatewayError, GatewayErrorKind},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -71,19 +72,6 @@ pub fn model_not_found_response() -> (StatusCode, Json<Value>) {
|
||||
#[must_use]
|
||||
pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
match error {
|
||||
EngineError::Provider(error)
|
||||
if error.kind() == ProviderErrorKind::Unavailable
|
||||
&& error.send_state() == UpstreamSendState::NotSent =>
|
||||
{
|
||||
let gateway = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
);
|
||||
match error.client_visible_upstream_error() {
|
||||
Some(upstream) => gateway.with_client_visible_upstream_error(upstream.clone()),
|
||||
None => gateway,
|
||||
}
|
||||
}
|
||||
EngineError::Provider(error) => GatewayError::from_provider(error),
|
||||
EngineError::Cancelled => {
|
||||
GatewayError::new(GatewayErrorKind::Cancelled, "request was cancelled")
|
||||
@@ -121,6 +109,33 @@ pub fn gateway_error_response(error: &GatewayError) -> Response {
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// 在下游尚未提交时优先交付 Provider 的原始 HTTP 失败响应。
|
||||
pub fn engine_error_response(error: &EngineError) -> Response {
|
||||
if let EngineError::Provider(error) = error
|
||||
&& let Some(upstream) = error.client_visible_upstream_response()
|
||||
&& let Ok(status) = StatusCode::from_u16(upstream.status())
|
||||
{
|
||||
let mut response = Response::new(Body::from(upstream.body().clone()));
|
||||
*response.status_mut() = status;
|
||||
if let Some(content_type) = upstream.content_type()
|
||||
&& let Ok(content_type) = HeaderValue::from_bytes(content_type)
|
||||
{
|
||||
response.headers_mut().insert(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for header in upstream.headers() {
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
gateway_error_response(&gateway_error_from_engine(error))
|
||||
}
|
||||
|
||||
/// Gateway 错误稳定映射,供 HTTP、SSE 和 WebSocket 共用。
|
||||
#[must_use]
|
||||
pub const fn gateway_error_contract(
|
||||
@@ -157,6 +172,16 @@ pub const fn gateway_error_contract(
|
||||
"server_error",
|
||||
"no_available_provider",
|
||||
),
|
||||
GatewayErrorKind::AccountCapacityUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
),
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
),
|
||||
GatewayErrorKind::RateLimited => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"rate_limit_error",
|
||||
|
||||
@@ -152,9 +152,6 @@ pub enum ResponseEncodeError {
|
||||
/// 协议原生流缺少可返回的终态 response object。
|
||||
#[error("responses wire stream has no terminal response")]
|
||||
MissingWireTerminal,
|
||||
/// 协议原生事件携带了不一致的响应身份。
|
||||
#[error("responses wire stream changed its response identity")]
|
||||
WireIdentityChanged,
|
||||
/// 终态 response object 无法序列化为完整 HTTP JSON 响应。
|
||||
#[error("responses wire terminal serialization failed")]
|
||||
Serialization,
|
||||
@@ -167,7 +164,7 @@ impl ResponseEncodeError {
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_canonical_response",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
|
||||
@@ -8,7 +8,7 @@ use axum::{
|
||||
body::{Body, Bytes},
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, StatusCode,
|
||||
HeaderMap, HeaderName, HeaderValue, StatusCode,
|
||||
header::{CACHE_CONTROL, CONTENT_TYPE, USER_AGENT},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
@@ -25,8 +25,8 @@ use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{
|
||||
gateway_error_contract, gateway_error_from_engine, gateway_error_response,
|
||||
protocol_error_response, runtime_unavailable_response,
|
||||
engine_error_response, gateway_error_contract, gateway_error_from_engine,
|
||||
gateway_error_response, protocol_error_response, runtime_unavailable_response,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -180,7 +180,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -206,7 +206,7 @@ pub async fn collect_execution_response(session: Box<dyn ExecutionSession>) -> R
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
@@ -237,9 +237,7 @@ fn encode_collected_events(
|
||||
) -> Result<Vec<u8>, BufferedResponseEncodeError> {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
for event in events {
|
||||
encoder
|
||||
.push_sse(event)
|
||||
.map_err(BufferedResponseEncodeError::Canonical)?;
|
||||
encoder.push_sse(event);
|
||||
}
|
||||
let response = encoder
|
||||
.finish()
|
||||
@@ -253,23 +251,25 @@ fn json_body_response(encoded: Vec<u8>, response_headers: &[ProviderResponseHead
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_safe_response_headers(
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
let Some(name) = super::safe_response_header_name(header.name()) else {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_str(header.value().as_str()) else {
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::HeaderName::from_static(name), value);
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -295,7 +295,7 @@ pub async fn stream_execution_response(
|
||||
return response;
|
||||
}
|
||||
Err(error) => {
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
@@ -310,18 +310,7 @@ pub async fn stream_execution_response(
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let mut frames = Vec::new();
|
||||
for event in &first_events {
|
||||
match encoder.push_sse(event) {
|
||||
Ok(encoded) => frames.extend(encoded),
|
||||
Err(error) => {
|
||||
let response = protocol_error_response(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
);
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
frames.extend(encoder.push_sse(event));
|
||||
}
|
||||
if frames.is_empty() {
|
||||
let response = internal_gateway_response("gateway commit batch encoded no output");
|
||||
@@ -338,7 +327,7 @@ pub async fn stream_execution_response(
|
||||
.await
|
||||
{
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = gateway_error_response(&gateway_error_from_engine(&error));
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
let Some(session) = execution.into_session() else {
|
||||
@@ -371,7 +360,7 @@ fn event_stream_response(body: Body, response_headers: &[ProviderResponseHeader]
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CACHE_CONTROL, HeaderValue::from_static("no-cache"));
|
||||
apply_safe_response_headers(response, response_headers)
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn internal_gateway_response(message: &'static str) -> Response {
|
||||
@@ -403,15 +392,10 @@ impl PendingExecution {
|
||||
}
|
||||
|
||||
async fn record_response_status(&mut self, response: Response) -> Response {
|
||||
let Some(session) = self.session.as_mut() else {
|
||||
return internal_gateway_response("gateway response session is unavailable");
|
||||
};
|
||||
if session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return internal_gateway_response("gateway client status could not be persisted");
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
let _ = session
|
||||
.record_client_status(response.status().as_u16())
|
||||
.await;
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -499,10 +483,16 @@ impl ResponsesStreamState {
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| session.is_finalized());
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
));
|
||||
if self.execution_terminal {
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
} else {
|
||||
self.finish_with_gateway_error(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without finalizing the execution",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.execution_terminal = self
|
||||
@@ -521,18 +511,11 @@ impl ResponsesStreamState {
|
||||
}
|
||||
|
||||
async fn push_event(&mut self, event: ProviderEvent) {
|
||||
match self.encoder.push_sse(&event) {
|
||||
Ok(frames) => {
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.cancel_execution().await;
|
||||
self.finish_with_encode_error(error);
|
||||
}
|
||||
let frames = self.encoder.push_sse(&event);
|
||||
if self.encoder.is_completed() {
|
||||
self.finish_completed(frames).await;
|
||||
} else {
|
||||
self.pending.extend(frames);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -607,20 +590,6 @@ impl ResponsesStreamState {
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
|
||||
fn finish_with_encode_error(&mut self, error: ResponseEncodeError) {
|
||||
let protocol = error.protocol_body().error;
|
||||
self.pending
|
||||
.push_back(Bytes::from(response_failed_sse_event_with_id(
|
||||
self.encoder.response_id(),
|
||||
protocol.kind,
|
||||
protocol.code,
|
||||
&protocol.message,
|
||||
)));
|
||||
self.pending
|
||||
.push_back(Bytes::from_static(DONE_SSE_FRAME.as_bytes()));
|
||||
self.output_finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResponsesStreamState {
|
||||
|
||||
@@ -17,13 +17,63 @@ pub use response::OpenAiResponsesEncoder;
|
||||
pub(crate) use websocket::responses_websocket;
|
||||
pub use websocket::{ResponseCreateFrameError, decode_response_create_with_context};
|
||||
|
||||
pub(super) fn safe_response_header_name(name: &str) -> Option<&'static str> {
|
||||
match name {
|
||||
"x-request-id" => Some("x-request-id"),
|
||||
"openai-model" => Some("openai-model"),
|
||||
"x-models-etag" => Some("x-models-etag"),
|
||||
"x-reasoning-included" => Some("x-reasoning-included"),
|
||||
"openai-processing-ms" => Some("openai-processing-ms"),
|
||||
_ => None,
|
||||
use gateway_core::event::ProviderResponseHeader;
|
||||
|
||||
pub(super) fn response_connection_options(headers: &[ProviderResponseHeader]) -> Vec<String> {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|header| header.name().trim().eq_ignore_ascii_case("connection"))
|
||||
.filter_map(|header| std::str::from_utf8(header.value()).ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn response_header_is_forwardable(name: &str, connection_options: &[String]) -> bool {
|
||||
let name = name.trim().to_ascii_lowercase();
|
||||
if connection_options
|
||||
.iter()
|
||||
.any(|option| option.eq_ignore_ascii_case(&name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name.as_str(),
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "content-length"
|
||||
| "content-type"
|
||||
| "content-encoding"
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "www-authenticate"
|
||||
| "authentication-info"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "proxy-authentication-info"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "set-cookie"
|
||||
| "set-cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
| "x-codex-installation-id"
|
||||
| "x-codex-turn-state"
|
||||
| "x-codex-turn-metadata"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::{fmt, net::IpAddr};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, ProtocolPayload, ProviderSessionState};
|
||||
use gateway_protocol::openai::{
|
||||
X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER, X_OPENAI_MEMGEN_REQUEST_HEADER,
|
||||
@@ -14,6 +15,7 @@ use super::error::RequestDecodeError;
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const REVIEW_SUBAGENT: &str = "review";
|
||||
const OPENAI_SUBAGENT_KEY: &str = "x-openai-subagent";
|
||||
const PASSTHROUGH_HEADERS_CONTEXT_KEY: &str = "opaque_request_headers";
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct OpenAiRequestHeaders {
|
||||
@@ -34,6 +36,7 @@ pub struct OpenAiRequestHeaders {
|
||||
responses_lite: Option<String>,
|
||||
memgen_request: Option<String>,
|
||||
subagent: Option<String>,
|
||||
passthrough_headers: Vec<Value>,
|
||||
}
|
||||
|
||||
impl OpenAiRequestHeaders {
|
||||
@@ -57,12 +60,8 @@ impl OpenAiRequestHeaders {
|
||||
turn_id: header_string(headers, "x-codex-turn-id"),
|
||||
responses_lite: header_string(headers, X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER),
|
||||
memgen_request: header_string(headers, X_OPENAI_MEMGEN_REQUEST_HEADER),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"review" | "compact" | "memory_consolidation" | "collab_spawn"
|
||||
)
|
||||
}),
|
||||
subagent: header_string(headers, OPENAI_SUBAGENT_KEY),
|
||||
passthrough_headers: passthrough_headers(headers),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +107,12 @@ impl OpenAiRequestHeaders {
|
||||
insert_protocol_context(&mut context, "turn_id", self.turn_id.as_ref());
|
||||
insert_protocol_context(&mut context, "responses_lite", self.responses_lite.as_ref());
|
||||
insert_protocol_context(&mut context, "memgen_request", self.memgen_request.as_ref());
|
||||
if !self.passthrough_headers.is_empty() {
|
||||
context.insert(
|
||||
PASSTHROUGH_HEADERS_CONTEXT_KEY.to_owned(),
|
||||
Value::Array(self.passthrough_headers.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(use_websocket) = use_websocket {
|
||||
context.insert("use_websocket".to_owned(), Value::Bool(use_websocket));
|
||||
}
|
||||
@@ -322,13 +327,14 @@ pub(super) fn decode_request_object(
|
||||
review: bool,
|
||||
request_headers: &OpenAiRequestHeaders,
|
||||
) -> Result<DecodedResponsesRequest, RequestDecodeError> {
|
||||
// `use_websocket` 只影响本地 transport。无论值是否可识别,都不得进入上游 body。
|
||||
let use_websocket = object
|
||||
.remove("use_websocket")
|
||||
.and_then(|value| value.as_bool());
|
||||
// 仅消费已识别的本地 transport 开关;未知同名值保留给未来上游协议。
|
||||
let use_websocket = object.get("use_websocket").and_then(Value::as_bool);
|
||||
if use_websocket.is_some() {
|
||||
object.remove("use_websocket");
|
||||
}
|
||||
|
||||
let model = required_non_empty_string(&object, "model", "model")?.trim();
|
||||
if model.is_empty() {
|
||||
let model = required_non_empty_string(&object, "model", "model")?;
|
||||
if model.trim().is_empty() {
|
||||
return Err(RequestDecodeError::EmptyField {
|
||||
field: "model".to_owned(),
|
||||
});
|
||||
@@ -345,7 +351,6 @@ pub(super) fn decode_request_object(
|
||||
let continuation = object
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|response_id| !response_id.is_empty())
|
||||
.map(|response_id| ContinuationIntent::PreviousResponseId(response_id.to_owned()))
|
||||
.unwrap_or(ContinuationIntent::None);
|
||||
|
||||
@@ -376,26 +381,107 @@ fn inject_subagent_metadata(body: &mut Map<String, Value>, subagent: &str) {
|
||||
let metadata = body
|
||||
.entry("client_metadata".to_owned())
|
||||
.or_insert_with(|| Value::Object(Map::new()));
|
||||
if !metadata.is_object() {
|
||||
*metadata = Value::Object(Map::new());
|
||||
}
|
||||
if let Some(metadata) = metadata.as_object_mut() {
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
let Some(metadata) = metadata.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
metadata.insert(
|
||||
OPENAI_SUBAGENT_KEY.to_owned(),
|
||||
Value::String(subagent.to_owned()),
|
||||
);
|
||||
}
|
||||
|
||||
fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
fn passthrough_headers(headers: &HeaderMap) -> Vec<Value> {
|
||||
let connection_headers = headers
|
||||
.get_all("connection")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.flat_map(|value| value.split(','))
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(str::to_ascii_lowercase)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| passthrough_header_name(name.as_str(), &connection_headers))
|
||||
.map(|(name, value)| {
|
||||
Value::Array(vec![
|
||||
Value::String(name.as_str().to_owned()),
|
||||
Value::String(STANDARD.encode(value.as_bytes())),
|
||||
])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
if connection_headers
|
||||
.iter()
|
||||
.any(|connection_header| connection_header.eq_ignore_ascii_case(name))
|
||||
|| name.starts_with("sec-websocket-")
|
||||
|| name.starts_with("x-grok-")
|
||||
|| name.starts_with("x-xai-")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
!matches!(
|
||||
name,
|
||||
// 逐跳头、上游 authority 和重序列化后的实体长度由 transport 重建。
|
||||
"connection"
|
||||
| "keep-alive"
|
||||
| "proxy-connection"
|
||||
| "proxy-authenticate"
|
||||
| "proxy-authorization"
|
||||
| "te"
|
||||
| "trailer"
|
||||
| "transfer-encoding"
|
||||
| "upgrade"
|
||||
| "host"
|
||||
| "content-length"
|
||||
// 下游连接诊断事实只用于本地观测,不能伪装成 OpenAI 请求事实。
|
||||
| "forwarded"
|
||||
| "x-forwarded-for"
|
||||
| "x-forwarded-host"
|
||||
| "x-forwarded-proto"
|
||||
| "x-forwarded-port"
|
||||
| "x-real-ip"
|
||||
| "true-client-ip"
|
||||
| "cf-connecting-ip"
|
||||
| "x-request-id"
|
||||
// 下游鉴权和账号 cookie 绝不能成为上游账号身份。
|
||||
| "authorization"
|
||||
| "x-api-key"
|
||||
| "cookie"
|
||||
| "cookie2"
|
||||
| "chatgpt-account-id"
|
||||
| "chatgpt-organization-id"
|
||||
| "chatgpt-org-id"
|
||||
| "chatgpt-project-id"
|
||||
| "openai-organization"
|
||||
| "openai-project"
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
// installation ID 始终由当前 lease 重建。
|
||||
| "x-codex-installation-id"
|
||||
// 上游指纹必须由运行时画像统一生成,客户端 originator/User-Agent
|
||||
// 不能透传覆盖,避免不同下游客户端暴露不一致的设备指纹。
|
||||
| "originator"
|
||||
| "user-agent"
|
||||
// 设备 attestation/integrity 头只能由官方客户端或网关自身生成,
|
||||
// 客户端注入的 x-oai-attestation / X-OAI-IS 不得透传上游。
|
||||
| "x-oai-attestation"
|
||||
| "x-oai-is"
|
||||
| "x-oai-is-update"
|
||||
)
|
||||
}
|
||||
|
||||
fn required_non_empty_string<'a>(
|
||||
object: &'a Map<String, Value>,
|
||||
key: &str,
|
||||
|
||||
@@ -11,15 +11,14 @@ const OPENAI_PROTOCOL: &str = "openai";
|
||||
|
||||
/// OpenAI Responses wire 的转发器。
|
||||
///
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。这里仅保留同一
|
||||
/// 响应的身份和终态校验,不再从 canonical facts 重建客户端 JSON;因此
|
||||
/// OpenAI 上游的未知字段、字段顺序和事件语义不会被 API 层二次解释。
|
||||
/// OpenAI Provider 与 xAI adapter 都必须交付 OpenAI wire。wire 是客户端交付
|
||||
/// 与终态判断的事实来源;canonical facts 只在 wire 暂未携带 ID 时提供旁路观测,
|
||||
/// 不得反过来拒绝或改写上游事件。
|
||||
#[derive(Debug, Default)]
|
||||
pub struct OpenAiResponsesEncoder {
|
||||
response_id: Option<String>,
|
||||
wire_terminal: Option<Value>,
|
||||
wire_failure: bool,
|
||||
canonical_completed: bool,
|
||||
}
|
||||
|
||||
impl OpenAiResponsesEncoder {
|
||||
@@ -30,53 +29,41 @@ impl OpenAiResponsesEncoder {
|
||||
response_id: None,
|
||||
wire_terminal: None,
|
||||
wire_failure: false,
|
||||
canonical_completed: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 SSE frames。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Result<Vec<Bytes>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_sse(&mut self, event: &ProviderEvent) -> Vec<Bytes> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
self.observe_wire(wire);
|
||||
if let Some(raw_sse_frame) = wire.raw_sse_frame() {
|
||||
return Ok(vec![raw_sse_frame.clone()]);
|
||||
return vec![raw_sse_frame.clone()];
|
||||
}
|
||||
Ok(vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
vec![Bytes::from(encode_sse_event_with_metadata(
|
||||
wire.event_type().unwrap_or_default(),
|
||||
&wire.data().to_string(),
|
||||
wire.sse_id(),
|
||||
wire.sse_retry(),
|
||||
))])
|
||||
))]
|
||||
}
|
||||
|
||||
/// 消费一个 Provider event,并返回 WebSocket JSON messages。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 同一响应身份变化时返回错误。
|
||||
pub fn push_websocket(
|
||||
&mut self,
|
||||
event: &ProviderEvent,
|
||||
) -> Result<Vec<String>, ResponseEncodeError> {
|
||||
self.observe_identity(event)?;
|
||||
pub fn push_websocket(&mut self, event: &ProviderEvent) -> Vec<String> {
|
||||
self.observe_canonical_identity(event);
|
||||
let Some(wire) = openai_wire(event).filter(|wire| wire.has_json_data()) else {
|
||||
return Ok(Vec::new());
|
||||
return Vec::new();
|
||||
};
|
||||
self.observe_wire_terminal(wire);
|
||||
Ok(vec![wire.data().to_string()])
|
||||
self.observe_wire(wire);
|
||||
vec![wire.data().to_string()]
|
||||
}
|
||||
|
||||
/// 返回是否已经看到同一响应的 canonical 与 wire 终态。
|
||||
/// 返回是否已经看到客户端可见的 wire 终态。
|
||||
#[must_use]
|
||||
pub fn is_completed(&self) -> bool {
|
||||
self.wire_terminal.is_some() && self.canonical_completed
|
||||
self.wire_terminal.is_some()
|
||||
}
|
||||
|
||||
/// 返回是否已把 Provider 原生失败 event 交付给客户端。
|
||||
@@ -95,39 +82,31 @@ impl OpenAiResponsesEncoder {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 缺少 wire 或 canonical 终态时返回错误。
|
||||
/// 缺少可转换为非流式响应的 wire 终态时返回错误。
|
||||
pub fn finish(self) -> Result<Value, ResponseEncodeError> {
|
||||
if self.canonical_completed {
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
} else {
|
||||
Err(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
self.wire_terminal
|
||||
.ok_or(ResponseEncodeError::MissingWireTerminal)
|
||||
}
|
||||
|
||||
fn observe_identity(&mut self, event: &ProviderEvent) -> Result<(), ResponseEncodeError> {
|
||||
fn observe_canonical_identity(&mut self, event: &ProviderEvent) {
|
||||
for fact in event.canonical_facts() {
|
||||
let metadata = match fact {
|
||||
GatewayEvent::Started(metadata) => metadata,
|
||||
GatewayEvent::Completed(metadata) => {
|
||||
self.canonical_completed = true;
|
||||
metadata
|
||||
}
|
||||
GatewayEvent::Started(metadata) | GatewayEvent::Completed(metadata) => metadata,
|
||||
_ => continue,
|
||||
};
|
||||
if self
|
||||
.response_id
|
||||
.as_deref()
|
||||
.is_some_and(|current| current != metadata.response_id())
|
||||
{
|
||||
return Err(ResponseEncodeError::WireIdentityChanged);
|
||||
}
|
||||
self.response_id = Some(metadata.response_id().to_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_wire_terminal(&mut self, wire: &ProtocolWireEvent) {
|
||||
fn observe_wire(&mut self, wire: &ProtocolWireEvent) {
|
||||
if let Some(response_id) = wire
|
||||
.data()
|
||||
.pointer("/response/id")
|
||||
.or_else(|| wire.data().get("response_id"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
self.response_id = Some(response_id.to_owned());
|
||||
}
|
||||
let effective_type = wire
|
||||
.event_type()
|
||||
.or_else(|| wire.data().get("type").and_then(Value::as_str));
|
||||
|
||||
@@ -9,7 +9,7 @@ use axum::{
|
||||
connect_info::ConnectInfo,
|
||||
ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade, close_code},
|
||||
},
|
||||
http::{HeaderMap, StatusCode},
|
||||
http::{HeaderMap, HeaderName, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures::StreamExt;
|
||||
@@ -27,7 +27,6 @@ use crate::{
|
||||
openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{gateway_error_contract, gateway_error_from_engine, runtime_unavailable_response},
|
||||
router::MAX_CLIENT_REQUEST_BODY_BYTES,
|
||||
service::OpenAiService,
|
||||
},
|
||||
};
|
||||
@@ -102,8 +101,10 @@ impl ResponsesWebSocketAdapter {
|
||||
_connection_guard: connection_guard,
|
||||
};
|
||||
websocket
|
||||
.max_message_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
.max_frame_size(MAX_CLIENT_REQUEST_BODY_BYTES)
|
||||
// 覆盖 axum/tungstenite 的私有 64 MiB message 与 16 MiB frame 默认值。
|
||||
// Responses JSON 的协议可接受性由上游决定,代理不另设 wire 长度上限。
|
||||
.max_message_size(usize::MAX)
|
||||
.max_frame_size(usize::MAX)
|
||||
.on_upgrade(move |socket| async move {
|
||||
serve_responses_websocket(socket, session).await;
|
||||
})
|
||||
@@ -303,18 +304,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(messages) => first_messages.extend(messages),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
first_messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if first_messages.is_empty() {
|
||||
let error = GatewayError::new(
|
||||
@@ -336,9 +326,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) = commit_connection_replay(replay, &encoder, provider_state.take()) {
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_text(
|
||||
socket,
|
||||
response_metadata_event(&request_id, &response_headers),
|
||||
@@ -378,18 +366,7 @@ async fn forward_execution(
|
||||
if let Some(update) = event.take_session_update() {
|
||||
provider_state = Some(update);
|
||||
}
|
||||
match encoder.push_websocket(event) {
|
||||
Ok(encoded) => messages.extend(encoded),
|
||||
Err(error) => {
|
||||
return send_protocol_error(
|
||||
socket,
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error.protocol_body(),
|
||||
&request_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
messages.extend(encoder.push_websocket(event));
|
||||
}
|
||||
if encoder.is_completed() {
|
||||
if let Err(outcome) =
|
||||
@@ -397,11 +374,7 @@ async fn forward_execution(
|
||||
{
|
||||
return outcome;
|
||||
}
|
||||
if let Err(error) =
|
||||
commit_connection_replay(replay, &encoder, provider_state.take())
|
||||
{
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
if !send_messages(socket, messages).await {
|
||||
return ForwardOutcome::Disconnect;
|
||||
}
|
||||
@@ -413,9 +386,17 @@ async fn forward_execution(
|
||||
}
|
||||
}
|
||||
ActiveInput::Event(Ok(None)) => {
|
||||
if execution
|
||||
.session_mut()
|
||||
.is_some_and(|session| session.is_finalized())
|
||||
{
|
||||
commit_connection_replay(replay, &encoder, provider_state.take());
|
||||
execution.disarm();
|
||||
return ForwardOutcome::Continue;
|
||||
}
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway response ended without a terminal event",
|
||||
"gateway response ended without finalizing the execution",
|
||||
);
|
||||
return send_gateway_error(socket, &error, &request_id).await;
|
||||
}
|
||||
@@ -441,15 +422,10 @@ fn commit_connection_replay(
|
||||
replay: &mut ConnectionReplaySnapshot,
|
||||
encoder: &OpenAiResponsesEncoder,
|
||||
provider_state: Option<ProviderSessionState>,
|
||||
) -> Result<(), GatewayError> {
|
||||
let response_id = encoder.response_id().ok_or_else(|| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"gateway completed a response without an identity",
|
||||
)
|
||||
})?;
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
Ok(())
|
||||
) {
|
||||
if let Some(response_id) = encoder.response_id() {
|
||||
replay.commit(response_id.to_owned(), provider_state);
|
||||
}
|
||||
}
|
||||
|
||||
async fn confirm_completed_execution(
|
||||
@@ -595,21 +571,24 @@ fn response_metadata_event(
|
||||
request_id: &str,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> String {
|
||||
let mut headers = response_headers
|
||||
.iter()
|
||||
.filter_map(|header| {
|
||||
super::safe_response_header_name(header.name()).map(|name| {
|
||||
(
|
||||
name.to_owned(),
|
||||
Value::String(header.value().as_str().to_owned()),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Map<String, Value>>();
|
||||
headers.insert(
|
||||
"x-request-id".to_owned(),
|
||||
Value::String(request_id.to_owned()),
|
||||
);
|
||||
let connection_options = super::response_connection_options(response_headers);
|
||||
let mut headers = Map::new();
|
||||
for header in response_headers {
|
||||
if !super::response_header_is_forwardable(header.name(), &connection_options) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = std::str::from_utf8(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
// 官方 response.metadata 使用 string map,无法表达同名多值;后值保持既有覆盖语义。
|
||||
headers.insert(name.as_str().to_owned(), Value::String(value.to_owned()));
|
||||
}
|
||||
headers
|
||||
.entry("x-request-id".to_owned())
|
||||
.or_insert_with(|| Value::String(request_id.to_owned()));
|
||||
json!({
|
||||
"type": "response.metadata",
|
||||
"headers": headers,
|
||||
|
||||
@@ -13,9 +13,6 @@ use super::{
|
||||
|
||||
use crate::ApiState;
|
||||
|
||||
/// OpenAI 请求正文上限。
|
||||
pub const MAX_CLIENT_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// 构造 OpenAI 客户端协议路由。
|
||||
pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
@@ -26,5 +23,7 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
// Responses 正文属于 OpenAI/Codex 协议;代理不能用私有大小上限提前拒绝
|
||||
// 上游本可接受的未来 payload。
|
||||
.layer(DefaultBodyLimit::disable())
|
||||
}
|
||||
|
||||
@@ -81,16 +81,10 @@ impl OpenAiService {
|
||||
"requested model was not found",
|
||||
)
|
||||
})?;
|
||||
if !self.execution.contains_public_model(&client, &public_model) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let previous_response_id = match metadata.continuation() {
|
||||
ContinuationIntent::None => None,
|
||||
ContinuationIntent::PreviousResponseId(value) => {
|
||||
PreviousResponseId::new(value.clone()).ok()
|
||||
Some(PreviousResponseId::new(value.clone()))
|
||||
}
|
||||
};
|
||||
self.execution
|
||||
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -105,12 +118,15 @@ mod actions {
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
|
||||
provider_credentials::{CredentialDeletionResult, CredentialImportResult},
|
||||
provider_credentials::{
|
||||
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
|
||||
},
|
||||
};
|
||||
use gateway_api::admin::accounts::{
|
||||
AccountActionRequest, AccountConnectionTestEvent, AccountDeletionData,
|
||||
AccountDeletionRequest, AccountExportData, AccountExportQuery, AccountIdQuery,
|
||||
AccountImportData, AccountImportRequest, AccountRefreshRequest, AccountTestQuery,
|
||||
AccountImportData, AccountImportRequest, AccountMutationData, AccountRefreshRequest,
|
||||
AccountTestQuery, RotateAccountRequest, StartAccountAuthorizationRequest,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use serde_json::json;
|
||||
@@ -168,6 +184,58 @@ mod actions {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1"
|
||||
}))
|
||||
.expect("decode reauthorization");
|
||||
assert!(authorization.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<StartAccountAuthorizationRequest>(json!({
|
||||
"provider": "openai",
|
||||
"name": "reauthorize",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let rotation: RotateAccountRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"accessToken": "header.payload.signature",
|
||||
"refreshToken": "refresh-token"
|
||||
}))
|
||||
.expect("decode rotation");
|
||||
assert!(rotation.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<RotateAccountRequest>(json!({
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"expectedCredentialRevision": 1,
|
||||
"accessToken": "header.payload.signature"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_mutation_response_should_not_expose_internal_revision() {
|
||||
let response = AccountMutationData::from(CredentialMutationResult {
|
||||
config_revision: Revision::new(8).expect("config revision"),
|
||||
account_id: ProviderAccountId::new("acct_1").expect("account ID"),
|
||||
credential_revision: Some(Revision::new(9).expect("credential revision")),
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(response).expect("serialize credential mutation"),
|
||||
json!({ "accountId": "acct_1" })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_import_should_use_provider_and_opaque_data_fields() {
|
||||
let valid: AccountImportRequest = serde_json::from_value(json!({
|
||||
@@ -276,13 +344,16 @@ mod actions {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
account_status: AccountStatus::Normal,
|
||||
},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
provider_error_type: Some("invalid_request_error".to_owned()),
|
||||
account_status: AccountStatus::Active,
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -307,13 +378,16 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "normal" }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
"providerErrorCode": "usage_exhausted",
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"accountStatus": "active"
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "quota_exhausted"
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -23,9 +23,9 @@ use gateway_admin::{
|
||||
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
|
||||
},
|
||||
observability::{
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsErrorPage,
|
||||
OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter, UsageOverview,
|
||||
UsagePage, UsageQuery, UsageRecord,
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
|
||||
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageRecord,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -58,9 +58,8 @@ use gateway_api::admin::AdminSessionState;
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::ProviderAccountId,
|
||||
probe::{AccountProbe, AccountProbeRequest, AccountProbeResult},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::GatewayError,
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::{
|
||||
ConfigRevision, ProviderKind, PublicModelId, UpstreamModelId, snapshot::SnapshotControl,
|
||||
@@ -80,6 +79,11 @@ pub(super) struct AdminTestFixture {
|
||||
pub auth: Arc<MemoryAuthStore>,
|
||||
pub settings: Arc<MemorySettingsStore>,
|
||||
pub usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
pub usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -93,8 +97,18 @@ impl AdminTestFixture {
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
let client_keys = Arc::new(MemoryClientKeyStore);
|
||||
let usage_records = Arc::new(Mutex::new(Vec::new()));
|
||||
let usage_detail = Arc::new(Mutex::new(None));
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let dashboard_summary_range = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
@@ -102,6 +116,7 @@ impl AdminTestFixture {
|
||||
client_keys.clone(),
|
||||
unused,
|
||||
settings.clone(),
|
||||
gateway_admin::ports::backup::BackupStorePorts::disabled(),
|
||||
);
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new("openai")),
|
||||
@@ -126,6 +141,11 @@ impl AdminTestFixture {
|
||||
auth,
|
||||
settings,
|
||||
usage_records,
|
||||
usage_detail,
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
dashboard_summary_range,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,6 +413,11 @@ impl ClientKeyStore for MemoryClientKeyStore {
|
||||
|
||||
struct UnusedStore {
|
||||
usage_records: Arc<Mutex<Vec<UsageRecord>>>,
|
||||
usage_detail: Arc<Mutex<Option<UsageDetail>>>,
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -503,8 +528,16 @@ impl AccountStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
Err(unavailable("dashboard"))
|
||||
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
*self
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range") = Some(range);
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("dashboard"))
|
||||
}
|
||||
|
||||
async fn dashboard_trend(&self, _: TimeRange) -> AdminStoreResult<Vec<RequestMetricPoint>> {
|
||||
@@ -538,7 +571,11 @@ impl ObservabilityStore for UnusedStore {
|
||||
}
|
||||
|
||||
async fn usage_record_detail(&self, _: &str) -> AdminStoreResult<UsageDetail> {
|
||||
Err(unavailable("usage detail"))
|
||||
self.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.clone()
|
||||
.ok_or_else(|| unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
@@ -551,11 +588,16 @@ impl ObservabilityStore for UnusedStore {
|
||||
_: UsageFilter,
|
||||
_: DiagnosticDimension,
|
||||
) -> AdminStoreResult<Vec<DiagnosticObservation>> {
|
||||
Err(unavailable("usage diagnostics"))
|
||||
Ok(self.diagnostics.lock().expect("diagnostics").clone())
|
||||
}
|
||||
|
||||
async fn list_ops_errors(&self, _: OpsErrorQuery) -> AdminStoreResult<OpsErrorPage> {
|
||||
Err(unavailable("ops errors"))
|
||||
let items = self.ops_errors.lock().expect("ops errors").clone();
|
||||
Ok(OpsErrorPage {
|
||||
total: items.len() as u64,
|
||||
items,
|
||||
next_cursor: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,7 +717,7 @@ impl AccountProbe for NoopProbe {
|
||||
fn probe(
|
||||
&self,
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async { panic!("unused account probe") })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,456 +0,0 @@
|
||||
mod query {
|
||||
use gateway_api::admin::observability::{
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
|
||||
parse_attempt_index, parse_datetime, parse_status,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_bound_page_size_and_cursor() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"page": 2,
|
||||
"pageSize": 100,
|
||||
"cursor": "opaque"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(query.validate_page().unwrap(), (2, 100));
|
||||
assert!(query.validate_cursor().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_zero_page() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
|
||||
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
|
||||
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
|
||||
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
|
||||
assert_eq!(
|
||||
parse_attempt_index(Some(0)).unwrap_err().field(),
|
||||
"attemptIndex"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_datetime(Some("not-a-time")).unwrap_err().field(),
|
||||
"timeRange"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod response {
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::observability::DesktopReleaseStatus;
|
||||
use gateway_api::admin::observability::{
|
||||
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
|
||||
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
|
||||
DashboardWireProfileView, DashboardWireTargetView, PageData, PageMeta, TokenDetailsView,
|
||||
TrendData, TrendKind, TrendPointView, TrendSummaryView,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn usage_page_should_keep_terminal_camel_case_shape() {
|
||||
let data = PageData {
|
||||
items: vec![json!({"id": "request_1"})],
|
||||
page: PageMeta::new(1, 50, 1, 1),
|
||||
next_cursor: Some("cursor".to_owned()),
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["page"]["pageSize"], 50);
|
||||
assert_eq!(value["nextCursor"], "cursor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
|
||||
let value = serde_json::to_value(DashboardWireProfileView {
|
||||
provider: "xai".to_owned(),
|
||||
product: "Grok Build".to_owned(),
|
||||
version: "0.2.106".to_owned(),
|
||||
build: None,
|
||||
target: DashboardWireTargetView {
|
||||
os_type: "linux".to_owned(),
|
||||
os_version: "—".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "headless".to_owned(),
|
||||
},
|
||||
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
|
||||
attributes: vec![DashboardWireAttributeView {
|
||||
label: "客户端标识".to_owned(),
|
||||
value: "grok-shell".to_owned(),
|
||||
}],
|
||||
verified_at: None,
|
||||
release: None,
|
||||
})
|
||||
.expect("dashboard profile");
|
||||
|
||||
assert_eq!(value["provider"], "xai");
|
||||
assert_eq!(value["version"], "0.2.106");
|
||||
assert_eq!(value["attributes"][0]["label"], "客户端标识");
|
||||
assert!(value.get("release").is_none());
|
||||
assert!(value.get("verifiedAt").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_account_usage_should_keep_daily_request_timeline() {
|
||||
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let value = serde_json::to_value(DashboardAccountUsageView {
|
||||
id: "account_1".to_owned(),
|
||||
provider: "xai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
email: "account@example.com".to_owned(),
|
||||
plan_type: Some("free".to_owned()),
|
||||
tokens: "—".to_owned(),
|
||||
request_count: 3,
|
||||
request_buckets: vec![DashboardAccountRequestBucketView {
|
||||
bucket_start,
|
||||
request_count: 3,
|
||||
}],
|
||||
quota_used_percent: None,
|
||||
last_used: "刚刚".to_owned(),
|
||||
})
|
||||
.expect("dashboard account usage");
|
||||
|
||||
assert_eq!(value["requestCount"], 3);
|
||||
assert_eq!(
|
||||
value["requestBuckets"][0]["bucketStart"],
|
||||
"1970-01-01T00:00:00Z"
|
||||
);
|
||||
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
|
||||
let now = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let data = TrendData {
|
||||
kind: TrendKind::Usage,
|
||||
points: vec![TrendPointView {
|
||||
time: "08:00".to_owned(),
|
||||
bucket: now,
|
||||
label: "01-01 08:00".to_owned(),
|
||||
requests: "1".to_owned(),
|
||||
requests_value: 1,
|
||||
input_tokens: "2".to_owned(),
|
||||
input_tokens_value: 2,
|
||||
output_tokens: "3".to_owned(),
|
||||
output_tokens_value: 3,
|
||||
cached_tokens: "0".to_owned(),
|
||||
cached_tokens_value: 0,
|
||||
cache_hit_rate_value: 0.0,
|
||||
tokens_value: 5,
|
||||
errors: "0".to_owned(),
|
||||
errors_value: 0,
|
||||
latency: "1 ms".to_owned(),
|
||||
latency_value: Some(1),
|
||||
first_token_latency: "1 ms".to_owned(),
|
||||
first_token_latency_value: Some(1),
|
||||
max_latency: "1 ms".to_owned(),
|
||||
max_latency_value: Some(1),
|
||||
min_latency: "1 ms".to_owned(),
|
||||
min_latency_value: Some(1),
|
||||
success_rate: "100.0%".to_owned(),
|
||||
success_rate_value: Some(100.0),
|
||||
}],
|
||||
summary: vec![TrendSummaryView {
|
||||
label: "输入".to_owned(),
|
||||
value: "2".to_owned(),
|
||||
ratio: None,
|
||||
}],
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["kind"], "usage");
|
||||
assert_eq!(value["points"][0]["requestsValue"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
|
||||
let coverage = CostCoverageView {
|
||||
known: 1,
|
||||
partial: 0,
|
||||
unknown: 0,
|
||||
not_billable: 0,
|
||||
};
|
||||
let token_details = TokenDetailsView {
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(2),
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(3),
|
||||
input_tokens_display: "1".to_owned(),
|
||||
output_tokens_display: "2".to_owned(),
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
cache_write_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens_display: "-".to_owned(),
|
||||
total_tokens_display: "3".to_owned(),
|
||||
};
|
||||
let cursor = CursorWire {
|
||||
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
|
||||
stable_id: "request_1".to_owned(),
|
||||
};
|
||||
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
|
||||
assert!(value.to_string().contains("known"));
|
||||
assert!(!value.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_view_should_preserve_the_original_detail_contract() {
|
||||
let value = serde_json::to_value(BillingView {
|
||||
input_amount_display: "$0.03".to_owned(),
|
||||
output_amount_display: "$0.00".to_owned(),
|
||||
cache_read_amount_display: "$0.14".to_owned(),
|
||||
cache_write_amount_display: "$0.00".to_owned(),
|
||||
standard_amount_display: "$0.17".to_owned(),
|
||||
total_amount_display: "$0.17".to_owned(),
|
||||
input_price_display: "$10.0000 / 1M Token".to_owned(),
|
||||
output_price_display: "$60.0000 / 1M Token".to_owned(),
|
||||
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
|
||||
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
|
||||
service_tier_display: "Fast".to_owned(),
|
||||
multiplier_display: "1.00x".to_owned(),
|
||||
})
|
||||
.expect("billing view");
|
||||
|
||||
assert_eq!(value["inputAmountDisplay"], "$0.03");
|
||||
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
|
||||
assert_eq!(value["serviceTierDisplay"], "Fast");
|
||||
assert_eq!(value["multiplierDisplay"], "1.00x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
|
||||
for (domain, expected) in [
|
||||
(DesktopReleaseStatus::Unchecked, "unchecked"),
|
||||
(DesktopReleaseStatus::Current, "aligned"),
|
||||
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
|
||||
(DesktopReleaseStatus::Failed, "check_failed"),
|
||||
] {
|
||||
let status = DashboardDesktopReleaseStatusView::from(domain);
|
||||
assert_eq!(serde_json::to_value(status).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?outcome=provider_future_state")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_other_outcome")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageRecord {
|
||||
id: "request_endpoint".to_owned(),
|
||||
client_api_key_ref: "key_endpoint".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
provider_account_email: None,
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
"transport": "must-not-overwrite-core",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: Some("resp_usage_metadata".to_owned()),
|
||||
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(31),
|
||||
image_output_tokens: Some(9),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
first_event_ms: Some(17),
|
||||
first_reasoning_ms: Some(19),
|
||||
first_text_ms: Some(23),
|
||||
first_token_ms: Some(19),
|
||||
provider_processing_ms: Some(29),
|
||||
latency_ms: Some(31),
|
||||
client_ip: Some("127.0.0.1".to_owned()),
|
||||
user_agent: Some("usage-metadata-test".to_owned()),
|
||||
reasoning_effort: Some("max".to_owned()),
|
||||
reasoning_preset: Some("ultra".to_owned()),
|
||||
request_kind: Some("review".to_owned()),
|
||||
subagent_kind: Some("worker".to_owned()),
|
||||
compact: true,
|
||||
image_generation_requested: true,
|
||||
image_generation_succeeded: Some(true),
|
||||
started_at: Utc::now(),
|
||||
deadline_at: Utc::now(),
|
||||
completed_at: Some(Utc::now()),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_endpoint")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
"websocketPool": value["data"]["items"][0]["metadata"]["websocketPool"],
|
||||
"imageGenerationRequested": value["data"]["items"][0]["metadata"]["imageGenerationRequested"],
|
||||
"imageGenerationSucceeded": value["data"]["items"][0]["metadata"]["imageGenerationSucceeded"],
|
||||
"requestedModel": value["data"]["items"][0]["metadata"]["requestedModel"],
|
||||
"upstreamModel": value["data"]["items"][0]["metadata"]["upstreamModel"],
|
||||
"clientIp": value["data"]["items"][0]["metadata"]["clientIp"],
|
||||
"userAgent": value["data"]["items"][0]["metadata"]["userAgent"],
|
||||
"reasoningEffort": value["data"]["items"][0]["metadata"]["reasoningEffort"],
|
||||
"reasoningPreset": value["data"]["items"][0]["metadata"]["reasoningPreset"],
|
||||
"requestKind": value["data"]["items"][0]["metadata"]["requestKind"],
|
||||
"subagentKind": value["data"]["items"][0]["metadata"]["subagentKind"],
|
||||
"compact": value["data"]["items"][0]["metadata"]["compact"],
|
||||
"transport": value["data"]["items"][0]["metadata"]["transport"],
|
||||
"transportDecisionWaitMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["transportDecisionWaitMs"],
|
||||
"wsConnectMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["wsConnectMs"],
|
||||
"firstReasoningMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstReasoningMs"],
|
||||
"firstTextMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTextMs"],
|
||||
"firstTokenMs": value["data"]["items"][0]["metadata"]["latencyDetails"]["firstTokenMs"],
|
||||
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
|
||||
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
"websocketPool": {"kind": "reuse"},
|
||||
"imageGenerationRequested": true,
|
||||
"imageGenerationSucceeded": true,
|
||||
"requestedModel": "grok-4.5",
|
||||
"upstreamModel": "grok-4.5",
|
||||
"clientIp": "127.0.0.1",
|
||||
"userAgent": "usage-metadata-test",
|
||||
"reasoningEffort": "max",
|
||||
"reasoningPreset": "ultra",
|
||||
"requestKind": "review",
|
||||
"subagentKind": "worker",
|
||||
"compact": true,
|
||||
"transport": "http_sse",
|
||||
"transportDecisionWaitMs": 7,
|
||||
"wsConnectMs": 11,
|
||||
"firstReasoningMs": 19,
|
||||
"firstTextMs": 23,
|
||||
"firstTokenMs": 19,
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
//! Admin 观测 wire/路由的固定合同测试。
|
||||
|
||||
mod query;
|
||||
mod response;
|
||||
@@ -0,0 +1,64 @@
|
||||
//! 查询 wire 校验的固定合同测试。
|
||||
|
||||
use gateway_api::admin::observability::{
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
|
||||
parse_attempt_index, parse_datetime, parse_status,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "secret"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap_err().field(), "kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_bound_page_size_and_cursor() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"page": 2,
|
||||
"pageSize": 100,
|
||||
"cursor": "opaque"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(query.validate_page().unwrap(), (2, 100));
|
||||
assert!(query.validate_cursor().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_zero_page() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({"page": 0})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_page().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
serde_json::from_value(json!({"dimension": "failure_class"})).unwrap();
|
||||
assert_eq!(query.dimension().unwrap(), DiagnosticDimension::Failure);
|
||||
assert_eq!(DiagnosticDimension::Failure.display_name(), "failureClass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_query_parsers_should_reject_out_of_range_values_without_echoing_input() {
|
||||
assert_eq!(parse_status(Some(99)).unwrap_err().field(), "statusCode");
|
||||
assert_eq!(
|
||||
parse_attempt_index(Some(0)).unwrap_err().field(),
|
||||
"attemptIndex"
|
||||
);
|
||||
assert_eq!(
|
||||
parse_datetime(Some("not-a-time")).unwrap_err().field(),
|
||||
"timeRange"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,986 @@
|
||||
//! 响应 DTO 与固定 wire 形状的合同测试。
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::observability::DesktopReleaseStatus;
|
||||
use gateway_api::admin::PageMeta;
|
||||
use gateway_api::admin::observability::{
|
||||
BillingView, CostCoverageView, CursorWire, DashboardAccountRequestBucketView,
|
||||
DashboardAccountUsageView, DashboardDesktopReleaseStatusView, DashboardWireAttributeView,
|
||||
DashboardWireProfileView, DashboardWireTargetView, PageData, TokenDetailsView, TrendData,
|
||||
TrendKind, TrendPointView, TrendSummaryView,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn usage_page_should_keep_terminal_camel_case_shape() {
|
||||
let data = PageData {
|
||||
items: vec![json!({"id": "request_1"})],
|
||||
page: PageMeta::new(1, 50, 1, 1),
|
||||
next_cursor: Some("cursor".to_owned()),
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["page"]["pageSize"], 50);
|
||||
assert_eq!(value["nextCursor"], "cursor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_wire_profiles_should_keep_provider_specific_attributes() {
|
||||
let value = serde_json::to_value(DashboardWireProfileView {
|
||||
provider: "xai".to_owned(),
|
||||
product: "Grok Build".to_owned(),
|
||||
version: "0.2.106".to_owned(),
|
||||
build: None,
|
||||
target: DashboardWireTargetView {
|
||||
os_type: "linux".to_owned(),
|
||||
os_version: "—".to_owned(),
|
||||
arch: "x86_64".to_owned(),
|
||||
terminal: "headless".to_owned(),
|
||||
},
|
||||
user_agent: "grok-shell/0.2.106 (linux; x86_64)".to_owned(),
|
||||
attributes: vec![DashboardWireAttributeView {
|
||||
label: "客户端标识".to_owned(),
|
||||
value: "grok-shell".to_owned(),
|
||||
}],
|
||||
verified_at: None,
|
||||
release: None,
|
||||
})
|
||||
.expect("dashboard profile");
|
||||
|
||||
assert_eq!(value["provider"], "xai");
|
||||
assert_eq!(value["version"], "0.2.106");
|
||||
assert_eq!(value["attributes"][0]["label"], "客户端标识");
|
||||
assert!(value.get("release").is_none());
|
||||
assert!(value.get("verifiedAt").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dashboard_account_usage_should_keep_daily_request_timeline() {
|
||||
let bucket_start = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let value = serde_json::to_value(DashboardAccountUsageView {
|
||||
id: "account_1".to_owned(),
|
||||
provider: "xai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
email: "account@example.com".to_owned(),
|
||||
plan_type: Some("free".to_owned()),
|
||||
tokens: "—".to_owned(),
|
||||
request_count: 3,
|
||||
request_buckets: vec![DashboardAccountRequestBucketView {
|
||||
bucket_start,
|
||||
request_count: 3,
|
||||
}],
|
||||
quota_used_percent: None,
|
||||
last_used: "刚刚".to_owned(),
|
||||
})
|
||||
.expect("dashboard account usage");
|
||||
|
||||
assert_eq!(value["requestCount"], 3);
|
||||
assert_eq!(
|
||||
value["requestBuckets"][0]["bucketStart"],
|
||||
"1970-01-01T00:00:00Z"
|
||||
);
|
||||
assert_eq!(value["requestBuckets"][0]["requestCount"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trend_wire_should_serialize_kind_and_values_without_store_types() {
|
||||
let now = Utc.timestamp_opt(0, 0).single().unwrap();
|
||||
let data = TrendData {
|
||||
kind: TrendKind::Usage,
|
||||
points: vec![TrendPointView {
|
||||
time: "08:00".to_owned(),
|
||||
bucket: now,
|
||||
label: "01-01 08:00".to_owned(),
|
||||
requests: "1".to_owned(),
|
||||
requests_value: 1,
|
||||
input_tokens: "2".to_owned(),
|
||||
input_tokens_value: 2,
|
||||
output_tokens: "3".to_owned(),
|
||||
output_tokens_value: 3,
|
||||
cached_tokens: "0".to_owned(),
|
||||
cached_tokens_value: 0,
|
||||
cache_hit_rate_value: 0.0,
|
||||
tokens_value: 5,
|
||||
errors: "0".to_owned(),
|
||||
errors_value: 0,
|
||||
latency: "1 ms".to_owned(),
|
||||
latency_value: Some(1),
|
||||
max_latency: "1 ms".to_owned(),
|
||||
max_latency_value: Some(1),
|
||||
min_latency: "1 ms".to_owned(),
|
||||
min_latency_value: Some(1),
|
||||
success_rate: "100.0%".to_owned(),
|
||||
success_rate_value: Some(100.0),
|
||||
}],
|
||||
summary: vec![TrendSummaryView {
|
||||
label: "输入".to_owned(),
|
||||
value: "2".to_owned(),
|
||||
ratio: None,
|
||||
}],
|
||||
};
|
||||
let value = serde_json::to_value(data).unwrap();
|
||||
assert_eq!(value["kind"], "usage");
|
||||
assert_eq!(value["points"][0]["requestsValue"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sensitive_response_views_do_not_require_debug_or_add_secret_fields() {
|
||||
let coverage = CostCoverageView {
|
||||
known: 1,
|
||||
partial: 0,
|
||||
unknown: 0,
|
||||
not_billable: 0,
|
||||
};
|
||||
let token_details = TokenDetailsView {
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(2),
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
total_tokens: Some(3),
|
||||
input_tokens_display: "1".to_owned(),
|
||||
output_tokens_display: "2".to_owned(),
|
||||
cached_tokens_display: "-".to_owned(),
|
||||
cache_write_tokens_display: "-".to_owned(),
|
||||
reasoning_tokens_display: "-".to_owned(),
|
||||
image_input_tokens_display: "-".to_owned(),
|
||||
image_output_tokens_display: "-".to_owned(),
|
||||
total_tokens_display: "3".to_owned(),
|
||||
};
|
||||
let cursor = CursorWire {
|
||||
observed_at: Utc.timestamp_opt(0, 0).single().unwrap(),
|
||||
stable_id: "request_1".to_owned(),
|
||||
};
|
||||
let value = serde_json::to_value((&coverage, &token_details, &cursor)).unwrap();
|
||||
assert!(value.to_string().contains("known"));
|
||||
assert!(!value.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_view_should_preserve_the_original_detail_contract() {
|
||||
let value = serde_json::to_value(BillingView {
|
||||
input_amount_display: "$0.03".to_owned(),
|
||||
output_amount_display: "$0.00".to_owned(),
|
||||
cache_read_amount_display: "$0.14".to_owned(),
|
||||
cache_write_amount_display: "$0.00".to_owned(),
|
||||
standard_amount_display: "$0.17".to_owned(),
|
||||
total_amount_display: "$0.17".to_owned(),
|
||||
input_price_display: "$10.0000 / 1M Token".to_owned(),
|
||||
output_price_display: "$60.0000 / 1M Token".to_owned(),
|
||||
cache_read_price_display: "$1.0000 / 1M Token".to_owned(),
|
||||
cache_write_price_display: "$12.5000 / 1M Token".to_owned(),
|
||||
service_tier_display: "Fast".to_owned(),
|
||||
multiplier_display: "1.00x".to_owned(),
|
||||
})
|
||||
.expect("billing view");
|
||||
|
||||
assert_eq!(value["inputAmountDisplay"], "$0.03");
|
||||
assert_eq!(value["cacheReadPriceDisplay"], "$1.0000 / 1M Token");
|
||||
assert_eq!(value["serviceTierDisplay"], "Fast");
|
||||
assert_eq!(value["multiplierDisplay"], "1.00x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn desktop_release_status_should_preserve_the_existing_dashboard_wire_values() {
|
||||
for (domain, expected) in [
|
||||
(DesktopReleaseStatus::Unchecked, "unchecked"),
|
||||
(DesktopReleaseStatus::Current, "aligned"),
|
||||
(DesktopReleaseStatus::UpdateAvailable, "review_required"),
|
||||
(DesktopReleaseStatus::Failed, "check_failed"),
|
||||
] {
|
||||
let status = DashboardDesktopReleaseStatusView::from(domain);
|
||||
assert_eq!(serde_json::to_value(status).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
banned: 2,
|
||||
},
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_account_summary")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["credentials"]["totalValue"],
|
||||
&value["data"]["cards"]["credentials"]["availableValue"],
|
||||
&value["data"]["cards"]["credentials"]["unavailableValue"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!(807),
|
||||
&serde_json::json!(15),
|
||||
&serde_json::json!(792)
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, DashboardTotals, RequestMetrics,
|
||||
TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let range_requests = RequestMetrics {
|
||||
request_count: 2,
|
||||
input_tokens: 8,
|
||||
cached_tokens: 2,
|
||||
total_tokens: 12,
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: range_requests,
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: DashboardTotals {
|
||||
request_count: 42,
|
||||
input_tokens: 800,
|
||||
cached_tokens: 200,
|
||||
total_tokens: 840,
|
||||
billing_usd: None,
|
||||
},
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_lifetime_totals")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["traffic"]["totalRequests"],
|
||||
&value["data"]["cards"]["tokens"]["totalTokens"],
|
||||
&value["data"]["cards"]["cache"]["totalHitRate"],
|
||||
&value["data"]["cards"]["cache"]["totalCachedTokens"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("42"),
|
||||
&serde_json::json!("840"),
|
||||
&serde_json::json!("25.0%"),
|
||||
&serde_json::json!("200"),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_default_to_current_china_day() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
china_day_start,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_today_range")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let range = fixture
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range")
|
||||
.take()
|
||||
.expect("recorded dashboard summary range");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(range.start, china_day_start(range.end));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{UsageAttempt, UsageDetail};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
fixture
|
||||
.usage_detail
|
||||
.lock()
|
||||
.expect("usage detail")
|
||||
.replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
"req_detail",
|
||||
"acct_snap_a",
|
||||
"Snapshot Alpha",
|
||||
"alpha@example.invalid",
|
||||
"oauth",
|
||||
now,
|
||||
),
|
||||
attempts: vec![UsageAttempt {
|
||||
source: "ops_event".to_owned(),
|
||||
id: "ops_detail".to_owned(),
|
||||
attempt_index: 1,
|
||||
component: "routing".to_owned(),
|
||||
operation: "fallback".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_snap_b".to_owned()),
|
||||
provider_account_name: Some("Snapshot Beta".to_owned()),
|
||||
provider_account_email: Some("beta@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-b".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
upstream_send_state: Some("sent".to_owned()),
|
||||
outcome: gateway_admin::model::observability::RequestOutcome::Failed,
|
||||
downstream_committed: false,
|
||||
status_code: Some(429),
|
||||
provider_error_code: Some("rate_limit".to_owned()),
|
||||
failure_kind: Some("rate_limited".to_owned()),
|
||||
retry_after_ms: Some(1_000),
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(120),
|
||||
message: Some("limited".to_owned()),
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
total_tokens: None,
|
||||
cost_source: Some("unavailable".to_owned()),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
occurred_at: now,
|
||||
}],
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records/detail?id=req_detail")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_detail_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("usage detail request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage detail response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage detail body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage detail JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"accountId": value["data"]["accountId"],
|
||||
"accountName": value["data"]["accountName"],
|
||||
"accountEmail": value["data"]["accountEmail"],
|
||||
"authenticationKind": value["data"]["authenticationKind"],
|
||||
"attemptCredentialName": value["data"]["attempts"][0]["credentialName"],
|
||||
"attemptAccountId": value["data"]["attempts"][0]["accountId"],
|
||||
"attemptAccountName": value["data"]["attempts"][0]["accountName"],
|
||||
"attemptAccountEmail": value["data"]["attempts"][0]["accountEmail"],
|
||||
"attemptAuthenticationKind": value["data"]["attempts"][0]["authenticationKind"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"accountId": "acct_snap_a",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"attemptCredentialName": "Snapshot Beta",
|
||||
"attemptAccountId": "acct_snap_b",
|
||||
"attemptAccountName": "Snapshot Beta",
|
||||
"attemptAccountEmail": "beta@example.invalid",
|
||||
"attemptAuthenticationKind": "api_key",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_errors_should_keep_account_label_and_authentication_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::OpsError;
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.ops_errors
|
||||
.lock()
|
||||
.expect("ops errors")
|
||||
.push(OpsError {
|
||||
source: "model_request".to_owned(),
|
||||
event_id: "err_snapshot".to_owned(),
|
||||
request_id: Some("req_err".to_owned()),
|
||||
attempt_index: Some(1),
|
||||
client_api_key_ref: Some("key_err".to_owned()),
|
||||
component: "model_request".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: Some("/v1/responses".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some("acct_err".to_owned()),
|
||||
provider_account_name: None,
|
||||
provider_account_email: Some("err@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("api_key".to_owned()),
|
||||
upstream_model_id: Some("upstream-err".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
failure_kind: "upstream_error".to_owned(),
|
||||
client_status_code: Some(502),
|
||||
upstream_status_code: Some(502),
|
||||
provider_error_code: Some("upstream".to_owned()),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
latency_ms: Some(90),
|
||||
message: "snapshot error".to_owned(),
|
||||
occurrence_count: 1,
|
||||
occurred_at: Utc::now(),
|
||||
stable_sort_id: "model_request:req_err".to_owned(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/operations/errors")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_ops_errors_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("ops errors request"),
|
||||
)
|
||||
.await
|
||||
.expect("ops errors response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("ops errors body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("ops errors JSON");
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"provider": value["data"]["items"][0]["provider"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"kind": value["data"]["items"][0]["kind"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountLabel": value["data"]["items"][0]["metadata"]["accountLabel"],
|
||||
"clientStatusCode": value["data"]["items"][0]["clientStatusCode"],
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"provider": "openai",
|
||||
"authenticationKind": "api_key",
|
||||
"kind": "model_request",
|
||||
"accountId": "acct_err",
|
||||
"accountLabel": "err@example.invalid",
|
||||
"clientStatusCode": 502,
|
||||
"route": "/v1/responses",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_admin::model::observability::{CostCoverage, DiagnosticObservation};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
.diagnostics
|
||||
.lock()
|
||||
.expect("diagnostics")
|
||||
.push(DiagnosticObservation {
|
||||
key: "acct_diag".to_owned(),
|
||||
name: "diag@example.invalid".to_owned(),
|
||||
request_count: 2,
|
||||
success_count: 2,
|
||||
failure_count: 0,
|
||||
attempt_count: 2,
|
||||
total_tokens: 200,
|
||||
average_latency_ms: Some(100),
|
||||
latency_p95_ms: Some(3800),
|
||||
cost_coverage: CostCoverage::default(),
|
||||
costs: Vec::new(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/insights/diagnostics?dimension=account")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_diagnostics_snapshot")
|
||||
.body(Body::empty())
|
||||
.expect("diagnostics request"),
|
||||
)
|
||||
.await
|
||||
.expect("diagnostics response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["items"][0]["key"],
|
||||
&value["data"]["items"][0]["name"],
|
||||
&value["data"]["items"][0]["latencyP95Ms"],
|
||||
&value["data"]["dimension"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("acct_diag"),
|
||||
&serde_json::json!("diag@example.invalid"),
|
||||
&serde_json::json!(3800),
|
||||
&serde_json::json!("account"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
fn usage_record_with_account(
|
||||
id: &str,
|
||||
account_id: &str,
|
||||
name: &str,
|
||||
email: &str,
|
||||
authentication_kind: &str,
|
||||
started_at: chrono::DateTime<chrono::Utc>,
|
||||
) -> gateway_admin::model::observability::UsageRecord {
|
||||
use gateway_admin::model::observability::{RequestOutcome, UsageRecord};
|
||||
|
||||
UsageRecord {
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_detail".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "coding".to_owned(),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some(account_id.to_owned()),
|
||||
provider_account_name: Some(name.to_owned()),
|
||||
provider_account_email: Some(email.to_owned()),
|
||||
provider_account_authentication_kind: Some(authentication_kind.to_owned()),
|
||||
upstream_model_id: Some("upstream-model".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: None,
|
||||
service_tier: None,
|
||||
provider_metadata_json: None,
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: Some(started_at),
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: None,
|
||||
transport_decision_wait_ms: None,
|
||||
connect_ms: None,
|
||||
headers_ms: None,
|
||||
first_event_ms: None,
|
||||
first_reasoning_ms: None,
|
||||
first_text_ms: None,
|
||||
first_token_ms: None,
|
||||
provider_processing_ms: None,
|
||||
latency_ms: None,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
image_generation_requested: false,
|
||||
image_generation_succeeded: None,
|
||||
started_at,
|
||||
deadline_at: started_at + chrono::Duration::seconds(30),
|
||||
completed_at: Some(started_at),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?outcome=provider_future_state")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_other_outcome")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
use std::str::FromStr as _;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::observability::{
|
||||
CalculatedBillingBreakdown, CurrencyCost, DecimalAmount, RequestOutcome, UsageBilling,
|
||||
UsageRecord,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let usd = |amount: &str| CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: DecimalAmount::from_str(amount).expect("USD amount"),
|
||||
};
|
||||
fixture
|
||||
.usage_records
|
||||
.lock()
|
||||
.expect("usage records")
|
||||
.push(UsageRecord {
|
||||
id: "request_endpoint".to_owned(),
|
||||
client_api_key_ref: "key_endpoint".to_owned(),
|
||||
config_revision: 1,
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: Some("acct_snapshot".to_owned()),
|
||||
provider_account_name: Some("Snapshot Alpha".to_owned()),
|
||||
provider_account_email: Some("alpha@example.invalid".to_owned()),
|
||||
provider_account_authentication_kind: Some("oauth".to_owned()),
|
||||
upstream_model_id: Some("grok-4.5".to_owned()),
|
||||
upstream_transport: Some("http_sse".to_owned()),
|
||||
http_version: Some("h2".to_owned()),
|
||||
websocket_pool: Some("reuse".to_owned()),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
provider_metadata_json: Some(
|
||||
serde_json::json!({
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
"transport": "must-not-overwrite-core",
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
attempt_count: 1,
|
||||
upstream_send_state: "sent".to_owned(),
|
||||
downstream_committed_at: None,
|
||||
outcome: RequestOutcome::Succeeded,
|
||||
client_status_code: Some(200),
|
||||
upstream_status_code: Some(200),
|
||||
client_response_id: Some("resp_usage_metadata".to_owned()),
|
||||
upstream_request_id: Some("upstream_usage_metadata".to_owned()),
|
||||
upstream_response_id: None,
|
||||
error_kind: None,
|
||||
provider_error_code: None,
|
||||
error_message: None,
|
||||
retry_after_ms: None,
|
||||
input_tokens: Some(1),
|
||||
output_tokens: Some(1),
|
||||
cached_tokens: Some(0),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(31),
|
||||
image_output_tokens: Some(9),
|
||||
total_tokens: Some(2),
|
||||
cost_source: "unavailable".to_owned(),
|
||||
cost_amount: None,
|
||||
cost_currency: None,
|
||||
billing: Some(UsageBilling::Calculated(Box::new(
|
||||
CalculatedBillingBreakdown {
|
||||
input_amount: usd("0.03"),
|
||||
output_amount: usd("0.07"),
|
||||
cache_read_amount: usd("0.00"),
|
||||
cache_write_amount: usd("0.00"),
|
||||
standard_amount: usd("0.10"),
|
||||
total_amount: usd("0.10"),
|
||||
input_price_per_million: usd("10.0000"),
|
||||
output_price_per_million: usd("60.0000"),
|
||||
cache_read_price_per_million: usd("1.0000"),
|
||||
cache_write_price_per_million: usd("12.5000"),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
multiplier_percent: 100,
|
||||
},
|
||||
))),
|
||||
transport_decision_wait_ms: Some(7),
|
||||
connect_ms: Some(11),
|
||||
headers_ms: Some(13),
|
||||
first_event_ms: Some(17),
|
||||
first_reasoning_ms: Some(19),
|
||||
first_text_ms: Some(23),
|
||||
first_token_ms: Some(19),
|
||||
provider_processing_ms: Some(29),
|
||||
latency_ms: Some(31),
|
||||
client_ip: Some("127.0.0.1".to_owned()),
|
||||
user_agent: Some("usage-metadata-test".to_owned()),
|
||||
reasoning_effort: Some("max".to_owned()),
|
||||
reasoning_preset: Some("ultra".to_owned()),
|
||||
request_kind: Some("review".to_owned()),
|
||||
subagent_kind: Some("worker".to_owned()),
|
||||
compact: true,
|
||||
image_generation_requested: true,
|
||||
image_generation_succeeded: Some(true),
|
||||
started_at: Utc::now(),
|
||||
deadline_at: Utc::now(),
|
||||
completed_at: Some(Utc::now()),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_usage_endpoint")
|
||||
.body(Body::empty())
|
||||
.expect("usage request"),
|
||||
)
|
||||
.await
|
||||
.expect("usage response");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("usage response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("usage response JSON");
|
||||
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["inputPriceDisplay"],
|
||||
"$10 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["outputPriceDisplay"],
|
||||
"$60 / 1M Token"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][0]["billing"]["cacheWritePriceDisplay"],
|
||||
"$12.5 / 1M Token"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountName": value["data"]["items"][0]["accountName"],
|
||||
"accountEmail": value["data"]["items"][0]["accountEmail"],
|
||||
"authenticationKind": value["data"]["items"][0]["authenticationKind"],
|
||||
"imageInputTokens": value["data"]["items"][0]["tokenDetails"]["imageInputTokens"],
|
||||
"imageOutputTokens": value["data"]["items"][0]["tokenDetails"]["imageOutputTokens"],
|
||||
"websocketPool": value["data"]["items"][0]["websocketPool"],
|
||||
"imageGenerationRequested": value["data"]["items"][0]["imageGenerationRequested"],
|
||||
"imageGenerationSucceeded": value["data"]["items"][0]["imageGenerationSucceeded"],
|
||||
"requestedModel": value["data"]["items"][0]["requestedModel"],
|
||||
"upstreamModel": value["data"]["items"][0]["upstreamModel"],
|
||||
"clientIp": value["data"]["items"][0]["clientIp"],
|
||||
"userAgent": value["data"]["items"][0]["userAgent"],
|
||||
"reasoningEffort": value["data"]["items"][0]["reasoningEffort"],
|
||||
"reasoningPreset": value["data"]["items"][0]["reasoningPreset"],
|
||||
"requestKind": value["data"]["items"][0]["requestKind"],
|
||||
"subagentKind": value["data"]["items"][0]["subagentKind"],
|
||||
"compact": value["data"]["items"][0]["compact"],
|
||||
"transport": value["data"]["items"][0]["transport"],
|
||||
"transportDecisionWaitMs": value["data"]["items"][0]["latencyDetails"]["transportDecisionWaitMs"],
|
||||
"wsConnectMs": value["data"]["items"][0]["latencyDetails"]["wsConnectMs"],
|
||||
"firstReasoningMs": value["data"]["items"][0]["latencyDetails"]["firstReasoningMs"],
|
||||
"firstTextMs": value["data"]["items"][0]["latencyDetails"]["firstTextMs"],
|
||||
"firstTokenMs": value["data"]["items"][0]["latencyDetails"]["firstTokenMs"],
|
||||
"effectiveModel": value["data"]["items"][0]["metadata"]["effectiveModel"],
|
||||
"requestSummary": value["data"]["items"][0]["metadata"]["requestSummary"],
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"serviceTier": "priority",
|
||||
"accountId": "acct_snapshot",
|
||||
"accountName": "Snapshot Alpha",
|
||||
"accountEmail": "alpha@example.invalid",
|
||||
"authenticationKind": "oauth",
|
||||
"imageInputTokens": 31,
|
||||
"imageOutputTokens": 9,
|
||||
"websocketPool": {"kind": "reuse"},
|
||||
"imageGenerationRequested": true,
|
||||
"imageGenerationSucceeded": true,
|
||||
"requestedModel": "grok-4.5",
|
||||
"upstreamModel": "grok-4.5",
|
||||
"clientIp": "127.0.0.1",
|
||||
"userAgent": "usage-metadata-test",
|
||||
"reasoningEffort": "max",
|
||||
"reasoningPreset": "ultra",
|
||||
"requestKind": "review",
|
||||
"subagentKind": "worker",
|
||||
"compact": true,
|
||||
"transport": "http_sse",
|
||||
"transportDecisionWaitMs": 7,
|
||||
"wsConnectMs": 11,
|
||||
"firstReasoningMs": 19,
|
||||
"firstTextMs": 23,
|
||||
"firstTokenMs": 19,
|
||||
"effectiveModel": "grok-4.5",
|
||||
"requestSummary": {"inputItemsCount": 2},
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -62,6 +62,123 @@ fn settings_request_should_reject_unknown_rotation_strategy() {
|
||||
assert_eq!(request.validate().unwrap_err().field(), "rotationStrategy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{TimeZone, Utc};
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::engine::credential::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let settings = RuntimeSettings {
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: BTreeMap::from_iter([
|
||||
(
|
||||
PublicModelId::new("gpt-5.4").expect("public model"),
|
||||
UpstreamModelId::new("gpt-5.5").expect("upstream model"),
|
||||
),
|
||||
(
|
||||
PublicModelId::new("grok-latest").expect("public model"),
|
||||
UpstreamModelId::new("grok-4.5").expect("upstream model"),
|
||||
),
|
||||
]),
|
||||
refresh_margin_seconds: 1800,
|
||||
refresh_concurrency: 4,
|
||||
max_concurrent_per_account: 5,
|
||||
request_interval_ms: 25,
|
||||
rotation_strategy: RotationStrategy::RoundRobin,
|
||||
usage_retention_days: 32,
|
||||
ops_event_retention_days: 31,
|
||||
audit_retention_days: 91,
|
||||
updated_at: Utc
|
||||
.with_ymd_and_hms(2026, 8, 2, 10, 30, 0)
|
||||
.single()
|
||||
.expect("timestamp"),
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(RuntimeSettingsView::from(settings)).expect("serialize view");
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"modelMappings": {
|
||||
"gpt-5.4": "gpt-5.5",
|
||||
"grok-latest": "grok-4.5"
|
||||
},
|
||||
"refreshMarginSeconds": 1800,
|
||||
"refreshConcurrency": 4,
|
||||
"maxConcurrentPerAccount": 5,
|
||||
"requestIntervalMs": 25,
|
||||
"rotationStrategy": "round_robin",
|
||||
"usageRetentionDays": 32,
|
||||
"opsEventRetentionDays": 31,
|
||||
"auditRetentionDays": 91,
|
||||
"updatedAt": "2026-08-02T10:30:00Z"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::engine::credential::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(update_body()).expect("decode settings");
|
||||
request.validate().expect("fixture settings must validate");
|
||||
|
||||
let request_fields: BTreeSet<String> = update_body()
|
||||
.as_object()
|
||||
.expect("request body object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
.iter()
|
||||
.map(|(public, upstream)| {
|
||||
Ok((
|
||||
PublicModelId::new(public.clone())?,
|
||||
UpstreamModelId::new(upstream.clone())?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
|
||||
.expect("valid model mappings"),
|
||||
refresh_margin_seconds: request.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.max_concurrent_per_account).expect("u32"),
|
||||
request_interval_ms: request.request_interval_ms,
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
usage_retention_days: u32::try_from(request.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.ops_event_retention_days).expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.audit_retention_days).expect("u32"),
|
||||
updated_at: chrono::Utc::now(),
|
||||
};
|
||||
|
||||
let response_fields: BTreeSet<String> =
|
||||
serde_json::to_value(RuntimeSettingsView::from(settings))
|
||||
.expect("serialize view")
|
||||
.as_object()
|
||||
.expect("view object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut expected_fields = request_fields;
|
||||
expected_fields.insert("updatedAt".to_owned());
|
||||
|
||||
assert_eq!(response_fields, expected_fields);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
let mut body = update_body();
|
||||
|
||||
@@ -35,9 +35,15 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
let mut expected = vec![
|
||||
"src/admin/accounts.rs",
|
||||
"src/admin/auth.rs",
|
||||
"src/admin/backups.rs",
|
||||
"src/admin/client_keys.rs",
|
||||
"src/admin/mod.rs",
|
||||
"src/admin/observability.rs",
|
||||
"src/admin/observability/mod.rs",
|
||||
"src/admin/observability/presenter.rs",
|
||||
"src/admin/observability/query.rs",
|
||||
"src/admin/observability/routes.rs",
|
||||
"src/admin/observability/wire.rs",
|
||||
"src/admin/presenter.rs",
|
||||
"src/admin/settings.rs",
|
||||
"src/admin/system.rs",
|
||||
"src/admin/wire.rs",
|
||||
@@ -71,7 +77,9 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/admin/auth.rs",
|
||||
"tests/admin/client_keys.rs",
|
||||
"tests/admin/mod.rs",
|
||||
"tests/admin/observability.rs",
|
||||
"tests/admin/observability/mod.rs",
|
||||
"tests/admin/observability/query.rs",
|
||||
"tests/admin/observability/response.rs",
|
||||
"tests/admin/settings.rs",
|
||||
"tests/admin/system.rs",
|
||||
"tests/admin/wire.rs",
|
||||
|
||||
@@ -58,6 +58,30 @@ fn no_available_provider_error_should_map_to_service_unavailable() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_capacity_error_should_map_to_service_unavailable() {
|
||||
assert_eq!(
|
||||
gateway_error_contract(GatewayErrorKind::AccountCapacityUnavailable),
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"account_capacity_unavailable",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_infrastructure_error_should_map_to_service_unavailable() {
|
||||
assert_eq!(
|
||||
gateway_error_contract(GatewayErrorKind::ProviderInfrastructureUnavailable),
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"server_error",
|
||||
"provider_infrastructure_unavailable",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_error_should_map_to_openai_retryable_status() {
|
||||
assert_eq!(
|
||||
@@ -122,30 +146,32 @@ fn engine_provider_error_should_preserve_retry_classification() {
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
ProviderErrorKind::AccountCapacityUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request"
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
fn no_eligible_provider_account_should_preserve_safe_provider_detail() {
|
||||
let detail = ClientVisibleUpstreamError::new(
|
||||
"no account is eligible for the requested model",
|
||||
Some("no_eligible_account".to_owned()),
|
||||
Some("account_unavailable_error".to_owned()),
|
||||
)
|
||||
.expect("safe provider detail");
|
||||
);
|
||||
let error = EngineError::Provider(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::NotSent)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_client_visible_upstream_error(detail),
|
||||
);
|
||||
let gateway = gateway_error_from_engine(&error);
|
||||
|
||||
@@ -165,6 +191,22 @@ fn local_provider_capacity_exhaustion_should_preserve_safe_provider_detail() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_infrastructure_failure_should_have_a_distinct_client_contract() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable,
|
||||
UpstreamSendState::NotSent,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error),
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
@@ -182,16 +224,18 @@ fn no_eligible_provider_account_should_map_to_service_unavailable() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sent_provider_unavailability_should_remain_bad_gateway() {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
UpstreamSendState::Sent,
|
||||
));
|
||||
fn provider_unavailability_should_remain_bad_gateway_regardless_of_send_state() {
|
||||
for send_state in [UpstreamSendState::NotSent, UpstreamSendState::Sent] {
|
||||
let error = EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::Unavailable,
|
||||
send_state,
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error).kind(),
|
||||
GatewayErrorKind::UpstreamUnavailable
|
||||
);
|
||||
assert_eq!(
|
||||
gateway_error_from_engine(&error).kind(),
|
||||
GatewayErrorKind::UpstreamUnavailable
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -327,14 +371,11 @@ fn openai_error_response_should_preserve_only_safe_contract_fields() {
|
||||
async fn gateway_error_response_should_expose_only_structured_client_visible_upstream_fields() {
|
||||
let error = GatewayError::from_provider(
|
||||
&ProviderError::new(ProviderErrorKind::QuotaExhausted, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
),
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)),
|
||||
);
|
||||
|
||||
let response = gateway_error_response(&error);
|
||||
|
||||
@@ -8,7 +8,10 @@ use std::sync::{
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
extract::connect_info::ConnectInfo,
|
||||
http::{HeaderMap, Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, Request, StatusCode,
|
||||
header::{AUTHORIZATION, CONTENT_TYPE},
|
||||
},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
@@ -18,8 +21,8 @@ use gateway_core::engine::execution::{
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamError, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
SafeUpstreamValue,
|
||||
ClientVisibleUpstreamError, ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind,
|
||||
ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{
|
||||
ContentItem, ContentKind, GatewayEvent, ProtocolWireEvent, ProviderEvent,
|
||||
@@ -101,6 +104,8 @@ struct ContextCaptureExecution {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedClientContext {
|
||||
public_model: String,
|
||||
wire_model: Option<String>,
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
endpoint: String,
|
||||
@@ -109,6 +114,7 @@ struct CapturedClientContext {
|
||||
client_metadata: Option<Value>,
|
||||
protocol_context: Option<Value>,
|
||||
prompt_cache_key: Option<String>,
|
||||
previous_response_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ExecutionService for ContextCaptureExecution {
|
||||
@@ -124,11 +130,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
["model-a", "model-b"]
|
||||
.into_iter()
|
||||
.map(|model| PublicModelId::new(model).expect("catalog model"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
true
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
matches!(model.as_str(), "model-a" | "model-b")
|
||||
}
|
||||
|
||||
fn start(
|
||||
@@ -136,11 +145,16 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let public_model = request.public_model.as_str().to_owned();
|
||||
let operation_kind = request.operation.kind();
|
||||
let generation = match &request.operation {
|
||||
Operation::Generate(generation) => Some(generation),
|
||||
_ => None,
|
||||
};
|
||||
let wire_model = generation
|
||||
.and_then(|generation| generation.protocol_payload().body().get("model"))
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned);
|
||||
let (client_metadata, protocol_context, prompt_cache_key, input) =
|
||||
generation.map_or((None, None, None, None), |generation| {
|
||||
let payload = generation.protocol_payload();
|
||||
@@ -152,7 +166,14 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
payload.body().get("input").cloned(),
|
||||
)
|
||||
});
|
||||
let previous_response_id = request
|
||||
.metadata
|
||||
.previous_response_id
|
||||
.as_ref()
|
||||
.map(|value| value.as_str().to_owned());
|
||||
*self.observed.lock().expect("context capture lock") = Some(CapturedClientContext {
|
||||
public_model,
|
||||
wire_model,
|
||||
client_ip: request.metadata.client_ip,
|
||||
user_agent: request.metadata.user_agent,
|
||||
endpoint: request.metadata.endpoint,
|
||||
@@ -161,6 +182,7 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
client_metadata,
|
||||
protocol_context,
|
||||
prompt_cache_key,
|
||||
previous_response_id,
|
||||
});
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
@@ -503,14 +525,18 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
assert_eq!(
|
||||
captured_client_context(headers, peer).await,
|
||||
CapturedClientContext {
|
||||
public_model: "smart-code".to_owned(),
|
||||
wire_model: Some("smart-code".to_owned()),
|
||||
client_ip: Some("198.51.100.1".parse().expect("expected IP")),
|
||||
user_agent: Some("Codex-CLI/1.0".to_owned()),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
operation_kind: OperationKind::Generate,
|
||||
input: Some(json!("hello")),
|
||||
client_metadata: None,
|
||||
// 客户端 User-Agent 仅用于本地展示,不再透传给上游指纹上下文。
|
||||
protocol_context: None,
|
||||
prompt_cache_key: None,
|
||||
previous_response_id: None,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -533,14 +559,58 @@ async fn request_context_should_resolve_forwarded_precedence_and_peer_fallback()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_headers() {
|
||||
async fn stale_model_catalog_should_not_block_a_new_model_request() {
|
||||
let model = " gpt-future-codex ";
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({"model": model, "input": "hello"}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.public_model, model);
|
||||
assert_eq!(captured.wire_model.as_deref(), Some(model));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_pass_opaque_previous_response_id_to_core() {
|
||||
for response_id in [
|
||||
format!("resp_{}", "x".repeat(257)),
|
||||
"resp_control\0opaque".to_owned(),
|
||||
String::new(),
|
||||
] {
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}),
|
||||
HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
captured.previous_response_id.as_deref(),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_request_should_forward_codex_headers_without_projecting_xai_headers() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-codex-turn-state", "turn-state".parse().expect("header"));
|
||||
headers.insert("conversation-id", "conversation-1".parse().expect("header"));
|
||||
headers.insert("session-id", "session-1".parse().expect("header"));
|
||||
headers.insert("x-grok-turn-idx", "7".parse().expect("header"));
|
||||
headers.insert("x-openai-subagent", "compact".parse().expect("header"));
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-responses-lite",
|
||||
"true".parse().expect("header"),
|
||||
@@ -566,10 +636,33 @@ async fn http_request_should_forward_safe_codex_headers_without_projecting_xai_h
|
||||
.client_metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get("x-openai-subagent")),
|
||||
Some(&json!("compact"))
|
||||
Some(&json!("future_codex_mode"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subagent_header_should_not_replace_non_object_client_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
"x-openai-subagent",
|
||||
"future_codex_mode".parse().expect("header"),
|
||||
);
|
||||
|
||||
let captured = captured_http_request(
|
||||
"openai",
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "preserve metadata",
|
||||
"client_metadata": "opaque-client-value"
|
||||
}),
|
||||
headers,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(captured.client_metadata, Some(json!("opaque-client-value")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_private_headers_should_not_enter_openai_request_facts() {
|
||||
let peer = "192.0.2.10:443".parse().expect("peer address");
|
||||
@@ -632,6 +725,43 @@ async fn review_route_should_inject_review_subagent_and_record_its_endpoint() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_route_should_not_replace_non_object_client_metadata() {
|
||||
let observed = Arc::new(Mutex::new(None));
|
||||
let execution = Arc::new(ContextCaptureExecution {
|
||||
observed: Arc::clone(&observed),
|
||||
client: authenticated_client("sk_context_test"),
|
||||
});
|
||||
let request = Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_context_test")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "smart-code",
|
||||
"input": "review this",
|
||||
"client_metadata": ["opaque", "client", "value"]
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("review request");
|
||||
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(request)
|
||||
.await
|
||||
.expect("review response");
|
||||
let captured = observed
|
||||
.lock()
|
||||
.expect("review capture lock")
|
||||
.clone()
|
||||
.expect("captured review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(
|
||||
captured.client_metadata,
|
||||
Some(json!(["opaque", "client", "value"]))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -653,19 +783,22 @@ async fn streaming_encodes_first_frame_before_commit_and_http_delivery() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresentable_or_sensitive_ones()
|
||||
{
|
||||
let trace = Arc::new(Trace::default());
|
||||
let headers = vec![
|
||||
ProviderResponseHeader::new(
|
||||
"x-models-etag",
|
||||
SafeUpstreamValue::new("models-v2").expect("etag"),
|
||||
)
|
||||
.expect("safe header"),
|
||||
ProviderResponseHeader::new("x-models-etag", Bytes::from_static(b"models-v2")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
SafeUpstreamValue::new("should-not-cross-boundary").expect("header value"),
|
||||
)
|
||||
.expect("syntactically safe header"),
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-hop-secret")),
|
||||
ProviderResponseHeader::new("x-hop-secret", Bytes::from_static(b"hop-secret")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
];
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -685,7 +818,29 @@ async fn streaming_response_forwards_only_allowlisted_provider_headers() {
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("models-v2")
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-multi")
|
||||
.iter()
|
||||
.map(HeaderValue::as_bytes)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(HeaderValue::as_bytes),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-hop-secret").is_none());
|
||||
assert_eq!(
|
||||
response.headers().get(CONTENT_TYPE),
|
||||
Some(&HeaderValue::from_static("text/event-stream"))
|
||||
);
|
||||
std::mem::forget(response);
|
||||
}
|
||||
|
||||
@@ -801,7 +956,107 @@ async fn streaming_rate_limit_before_first_frame_should_persist_the_returned_429
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
async fn streaming_upstream_http_failure_before_first_frame_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(
|
||||
b"{\"error\":{\"message\":\"rate limited\",\"future_field\":{\"kept\":true}},\"top_level\":17}\x00",
|
||||
);
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
429,
|
||||
Some(b"application/problem+json; charset=utf-8".to_vec()),
|
||||
raw_body.clone(),
|
||||
)
|
||||
.with_headers(vec![
|
||||
ProviderResponseHeader::new("retry-after", Bytes::from_static(b"17")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req-failure")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-error", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
]),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![NextStep::Error(EngineError::Provider(error))],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"application/problem+json; charset=utf-8".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("retry-after")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"17".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-request-id")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"req-failure".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-future-error")
|
||||
.iter()
|
||||
.map(|value| value.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-future-bytes")
|
||||
.map(|value| value.as_bytes()),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![429]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_upstream_http_failure_should_preserve_raw_response() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_body = Bytes::from_static(b"not-json\xffstill-upstream");
|
||||
let error = ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_response(ClientVisibleUpstreamResponse::new(
|
||||
503,
|
||||
Some(b"application/octet-stream".to_vec()),
|
||||
raw_body.clone(),
|
||||
));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_collect_error(EngineError::Provider(error));
|
||||
|
||||
let response = collect_execution_response(Box::new(session)).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read raw error body"),
|
||||
raw_body
|
||||
);
|
||||
assert_eq!(trace.client_statuses(), vec![503]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_canonical_identity_change_does_not_interrupt_wire_delivery() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
@@ -811,7 +1066,7 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
mismatched_terminal_event(),
|
||||
CommitRequirement::AlreadyCommitted,
|
||||
)),
|
||||
NextStep::FinalizeCancelled,
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
@@ -820,12 +1075,16 @@ async fn streaming_postcommit_encode_failure_cancels_without_second_commit() {
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
|
||||
assert!(String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
assert!(body.contains("response.completed"));
|
||||
assert!(body.contains("resp_other"));
|
||||
assert!(!body.contains("response.failed"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.client_statuses(), vec![200]);
|
||||
assert!(trace.is_cancelled());
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(
|
||||
trace.snapshot(),
|
||||
vec!["next_event", "commit", "next_event", "cancel_finalize"]
|
||||
vec!["next_event", "commit", "next_event", "next_end"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -874,6 +1133,38 @@ async fn streaming_success_should_emit_terminal_event_and_done_marker() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_finalized_unknown_wire_should_end_cleanly_without_synthetic_failure() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw = Bytes::from_static(
|
||||
b"event: response.future_terminal\ndata: {\"type\":\"response.future_terminal\",\"opaque\":true}\n\n",
|
||||
);
|
||||
let event = ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_sse("openai", raw.clone()).expect("raw future event"),
|
||||
);
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(delivery_provider(
|
||||
event,
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)),
|
||||
NextStep::FinalizeSuccess,
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read transparent SSE body");
|
||||
|
||||
assert!(body.as_ref().starts_with(raw.as_ref()));
|
||||
assert!(body.as_ref().ends_with(b"data: [DONE]\n\n"));
|
||||
assert!(!String::from_utf8_lossy(&body).contains("response.failed"));
|
||||
assert!(!trace.is_cancelled());
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_end"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_completed_event_without_finalized_execution_should_fail_closed() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -942,14 +1233,11 @@ async fn streaming_empty_terminal_should_emit_failure_done_and_cancel_execution(
|
||||
async fn streaming_error_should_emit_client_visible_upstream_details() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let error = ProviderError::new(ProviderErrorKind::RateLimited, UpstreamSendState::Sent)
|
||||
.with_client_visible_upstream_error(
|
||||
ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
)
|
||||
.expect("safe structured upstream error"),
|
||||
);
|
||||
.with_client_visible_upstream_error(ClientVisibleUpstreamError::new(
|
||||
"Your Codex quota is exhausted",
|
||||
Some("quota_exhausted".to_owned()),
|
||||
Some("rate_limit_error".to_owned()),
|
||||
));
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
@@ -1055,6 +1343,86 @@ async fn streaming_upstream_wire_failure_should_not_be_rewritten_as_a_gateway_er
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atomic_uncommitted_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let raw_created = Bytes::from_static(
|
||||
b"event: response.created\r\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_atomic_api\",\"model\":\"public-model\",\"status\":\"in_progress\"}}\r\n\r\n",
|
||||
);
|
||||
let raw_failure = Bytes::from_static(
|
||||
b"event: response.failed\r\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_atomic_api\",\"status\":\"failed\",\"error\":{\"code\":\"rate_limit_exceeded\",\"message\":\"atomic upstream marker\"}}}\r\n\r\n",
|
||||
);
|
||||
let started_wire = ProviderEvent::canonical_with_wire(
|
||||
vec![started()],
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"model": "public-model",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
raw_created.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream started wire"),
|
||||
);
|
||||
let failed_wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json_with_raw_sse_metadata(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_atomic_api",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
raw_failure.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("valid upstream failure wire"),
|
||||
);
|
||||
let batch = CoordinatedEvent::try_batch(
|
||||
vec![started_wire, failed_wire],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic failure delivery");
|
||||
let session = FakeSession::streaming(
|
||||
Arc::clone(&trace),
|
||||
vec![
|
||||
NextStep::Event(batch),
|
||||
NextStep::Error(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
))),
|
||||
],
|
||||
);
|
||||
|
||||
let response = stream_execution_response(Box::new(session), None).await;
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read SSE body");
|
||||
let body = String::from_utf8(body.to_vec()).expect("SSE is UTF-8");
|
||||
|
||||
assert_eq!(body.matches("atomic upstream marker").count(), 1);
|
||||
assert_eq!(body.matches("event: response.failed").count(), 1);
|
||||
assert!(body.contains(std::str::from_utf8(&raw_created).expect("created frame")));
|
||||
assert!(body.contains(std::str::from_utf8(&raw_failure).expect("failure frame")));
|
||||
assert!(!body.contains("upstream capacity is temporarily unavailable"));
|
||||
assert!(body.ends_with("data: [DONE]\n\n"));
|
||||
assert_eq!(trace.snapshot(), vec!["next_event", "commit", "next_error"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_second_commit_request_should_fail_and_finalize_once() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
@@ -1132,13 +1500,10 @@ async fn buffered_response_commits_only_after_complete_json_is_encoded() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
async fn buffered_response_forwards_long_ordinary_provider_header_values() {
|
||||
let trace = Arc::new(Trace::default());
|
||||
let header = ProviderResponseHeader::new(
|
||||
"openai-model",
|
||||
SafeUpstreamValue::new("gpt-5.5-effective").expect("model header"),
|
||||
)
|
||||
.expect("safe header");
|
||||
let model = format!("gpt-{}", "x".repeat(512));
|
||||
let header = ProviderResponseHeader::new("openai-model", Bytes::from(model.clone()));
|
||||
let session = FakeSession::buffered(Arc::clone(&trace), vec![started(), completed()])
|
||||
.with_response_headers(vec![header]);
|
||||
|
||||
@@ -1149,7 +1514,7 @@ async fn buffered_response_forwards_allowlisted_provider_headers() {
|
||||
.headers()
|
||||
.get("openai-model")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("gpt-5.5-effective")
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ mod http;
|
||||
mod websocket;
|
||||
|
||||
use axum::http::{HeaderMap, HeaderValue};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_api::openai::responses::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, OpenAiResponsesEncoder,
|
||||
@@ -117,6 +118,28 @@ fn decoder_should_preserve_the_openai_body_and_only_derive_stable_routing_facts(
|
||||
assert!(decoded.metadata().store());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_client_model_values() {
|
||||
for model in [
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
"future\0model".to_owned(),
|
||||
" future-model ".to_owned(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": model.clone(),
|
||||
"input": "hello"
|
||||
}));
|
||||
|
||||
assert_eq!(decoded.metadata().public_model(), model);
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("model")
|
||||
.and_then(Value::as_str),
|
||||
Some(model.as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_compaction_trigger_for_the_openai_provider() {
|
||||
let decoded = generate_request(json!({
|
||||
@@ -149,6 +172,21 @@ fn decoder_should_keep_transport_override_out_of_the_openai_wire_body() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unrecognized_use_websocket_values() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "hello",
|
||||
"use_websocket": {"future": "transport-mode"}
|
||||
}));
|
||||
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded).get("use_websocket"),
|
||||
Some(&json!({"future": "transport-mode"}))
|
||||
);
|
||||
assert!(!openai_protocol_context(&decoded).contains_key("use_websocket"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -170,10 +208,155 @@ fn decoder_should_preserve_connection_metadata_outside_the_openai_wire_body() {
|
||||
("turn_state".to_owned(), json!("turn-state")),
|
||||
("turn_metadata".to_owned(), json!("{\"kind\":\"review\"}")),
|
||||
("conversation_id".to_owned(), json!("conversation")),
|
||||
(
|
||||
"opaque_request_headers".to_owned(),
|
||||
json!([
|
||||
["x-codex-turn-state", STANDARD.encode(b"turn-state")],
|
||||
[
|
||||
"x-codex-turn-metadata",
|
||||
STANDARD.encode(br#"{"kind":"review"}"#)
|
||||
],
|
||||
["conversation-id", STANDARD.encode(b"conversation")]
|
||||
]),
|
||||
),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_ordinary_request_headers_as_opaque_multivalues() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_static("future-ascii"),
|
||||
);
|
||||
headers.append(
|
||||
"x-openai-future-mode",
|
||||
HeaderValue::from_bytes(b"\x80\xff").expect("opaque header bytes"),
|
||||
);
|
||||
headers.append("version", HeaderValue::from_static("future-v1"));
|
||||
headers.append(
|
||||
"version",
|
||||
HeaderValue::from_bytes(b"future-\x80").expect("opaque known header bytes"),
|
||||
);
|
||||
headers.insert(
|
||||
"accept",
|
||||
HeaderValue::from_static("application/vnd.openai.responses+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"content-type",
|
||||
HeaderValue::from_static("application/vnd.openai.responses.request+json"),
|
||||
);
|
||||
headers.insert(
|
||||
"user-agent",
|
||||
HeaderValue::from_static("Codex future-client"),
|
||||
);
|
||||
headers.insert("originator", HeaderValue::from_static("future-codex"));
|
||||
headers.append(
|
||||
"openai-beta",
|
||||
HeaderValue::from_static("future_responses=v2"),
|
||||
);
|
||||
headers.append("openai-beta", HeaderValue::from_static("future_tools=v3"));
|
||||
headers.insert(
|
||||
"x-openai-internal-codex-residency",
|
||||
HeaderValue::from_static("future-region"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-turn-state",
|
||||
HeaderValue::from_static("same-account-turn"),
|
||||
);
|
||||
|
||||
headers.insert(
|
||||
"connection",
|
||||
HeaderValue::from_static("keep-alive, x-private-hop"),
|
||||
);
|
||||
headers.insert("x-private-hop", HeaderValue::from_static("drop-me"));
|
||||
headers.insert("host", HeaderValue::from_static("downstream.invalid"));
|
||||
headers.insert("content-length", HeaderValue::from_static("999"));
|
||||
headers.insert("authorization", HeaderValue::from_static("Bearer client"));
|
||||
headers.insert("x-api-key", HeaderValue::from_static("client-key"));
|
||||
headers.insert("cookie", HeaderValue::from_static("client=cookie"));
|
||||
headers.insert(
|
||||
"chatgpt-account-id",
|
||||
HeaderValue::from_static("client-account"),
|
||||
);
|
||||
headers.insert(
|
||||
"chatgpt-project-id",
|
||||
HeaderValue::from_static("client-project"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-codex-installation-id",
|
||||
HeaderValue::from_static("client-installation"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-oai-attestation",
|
||||
HeaderValue::from_static("client-attestation"),
|
||||
);
|
||||
headers.insert("x-oai-is", HeaderValue::from_static("client-is"));
|
||||
headers.insert(
|
||||
"x-oai-is-update",
|
||||
HeaderValue::from_static("client-is-update"),
|
||||
);
|
||||
|
||||
let decoded =
|
||||
decode_request_with_headers(br#"{"model":"smart-code","input":"hello"}"#, &headers)
|
||||
.expect("opaque request headers should not affect body decoding");
|
||||
let entries = openai_protocol_context(&decoded)
|
||||
.get("opaque_request_headers")
|
||||
.and_then(Value::as_array)
|
||||
.expect("opaque request header context");
|
||||
let values = |target: &str| {
|
||||
entries
|
||||
.iter()
|
||||
.filter_map(Value::as_array)
|
||||
.filter(|entry| entry.first().and_then(Value::as_str) == Some(target))
|
||||
.filter_map(|entry| entry.get(1).and_then(Value::as_str))
|
||||
.map(|encoded| STANDARD.decode(encoded).expect("valid base64 header value"))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
values("x-openai-future-mode"),
|
||||
vec![b"future-ascii".to_vec(), b"\x80\xff".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("version"),
|
||||
vec![b"future-v1".to_vec(), b"future-\x80".to_vec()]
|
||||
);
|
||||
assert_eq!(
|
||||
values("openai-beta"),
|
||||
vec![b"future_responses=v2".to_vec(), b"future_tools=v3".to_vec()]
|
||||
);
|
||||
for preserved in [
|
||||
"accept",
|
||||
"content-type",
|
||||
"x-openai-internal-codex-residency",
|
||||
"x-codex-turn-state",
|
||||
] {
|
||||
assert_eq!(values(preserved).len(), 1, "missing {preserved}");
|
||||
}
|
||||
for excluded in [
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"host",
|
||||
"content-length",
|
||||
"authorization",
|
||||
"x-api-key",
|
||||
"cookie",
|
||||
"chatgpt-account-id",
|
||||
"chatgpt-project-id",
|
||||
"x-codex-installation-id",
|
||||
// 上游指纹由运行时画像统一生成,客户端不得覆盖。
|
||||
"user-agent",
|
||||
"originator",
|
||||
"x-oai-attestation",
|
||||
"x-oai-is",
|
||||
"x-oai-is-update",
|
||||
] {
|
||||
assert!(values(excluded).is_empty(), "unexpected {excluded}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
let secret = "nested-private-value";
|
||||
@@ -191,25 +374,51 @@ fn decoder_should_preserve_unknown_nested_values_without_debug_disclosure() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_continuation_and_mark_the_routing_requirement() {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": "resp_private_continuation"
|
||||
}));
|
||||
fn decoder_should_preserve_unknown_arbitrary_precision_numbers() {
|
||||
let number = "12345678901234567890123456789012345678901234567890";
|
||||
let body = format!(r#"{{"model":"smart-code","input":"hello","future_number":{number}}}"#);
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == "resp_private_continuation"
|
||||
));
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
assert!(!format!("{decoded:?}").contains("resp_private_continuation"));
|
||||
let decoded = decode_request_with_headers(body.as_bytes(), &HeaderMap::new())
|
||||
.expect("opaque numeric field should decode");
|
||||
let encoded = serde_json::to_string(openai_wire_body(&decoded)).expect("encode request body");
|
||||
|
||||
assert!(encoded.contains(&format!(r#""future_number":{number}"#)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_preserve_opaque_continuation_and_mark_the_routing_requirement() {
|
||||
for response_id in [
|
||||
"resp_private_continuation".to_owned(),
|
||||
format!("resp_{}\0opaque", "x".repeat(257)),
|
||||
String::new(),
|
||||
] {
|
||||
let decoded = generate_request(json!({
|
||||
"model": "smart-code",
|
||||
"input": "continue",
|
||||
"previous_response_id": response_id.clone()
|
||||
}));
|
||||
|
||||
assert!(matches!(
|
||||
decoded.metadata().continuation(),
|
||||
ContinuationIntent::PreviousResponseId(value) if value == &response_id
|
||||
));
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str),
|
||||
Some(response_id.as_str())
|
||||
);
|
||||
assert!(
|
||||
decoded
|
||||
.operation()
|
||||
.capability_requirements()
|
||||
.features()
|
||||
.contains(&Feature::NativeContinuation)
|
||||
);
|
||||
if !response_id.is_empty() {
|
||||
assert!(!format!("{decoded:?}").contains(&response_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -316,7 +525,7 @@ fn transparent_encoder_should_forward_raw_sse_frames_unknown_events_and_terminal
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let frames = events
|
||||
.iter()
|
||||
.flat_map(|event| encoder.push_sse(event).expect("wire event should encode"))
|
||||
.flat_map(|event| encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let response = encoder.finish().expect("wire response should finish");
|
||||
let body = frames
|
||||
@@ -370,15 +579,11 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
|
||||
let sse = events
|
||||
.iter()
|
||||
.flat_map(|event| sse_encoder.push_sse(event).expect("SSE wire event"))
|
||||
.flat_map(|event| sse_encoder.push_sse(event))
|
||||
.collect::<Vec<_>>();
|
||||
let websocket = events
|
||||
.iter()
|
||||
.flat_map(|event| {
|
||||
websocket_encoder
|
||||
.push_websocket(event)
|
||||
.expect("WebSocket wire event")
|
||||
})
|
||||
.flat_map(|event| websocket_encoder.push_websocket(event))
|
||||
.collect::<Vec<_>>();
|
||||
let sse_text = String::from_utf8(
|
||||
sse.iter()
|
||||
@@ -400,8 +605,40 @@ fn transparent_encoder_should_use_identical_json_for_sse_and_websocket() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
fn transparent_encoder_should_follow_wire_when_canonical_identity_changes() {
|
||||
let mut encoder = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
))],
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "wire_resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started);
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "wire_resp_2"}}),
|
||||
);
|
||||
|
||||
let completed = encoder.push_sse(&changed_identity);
|
||||
|
||||
assert_eq!(completed.len(), 1);
|
||||
assert!(encoder.is_completed());
|
||||
assert_eq!(encoder.response_id(), Some("wire_resp_2"));
|
||||
assert_eq!(
|
||||
encoder
|
||||
.finish()
|
||||
.expect("wire terminal remains authoritative"),
|
||||
json!({"id": "wire_resp_2"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_encoder_should_only_require_wire_terminal_for_buffered_conversion() {
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
let started = openai_wire_event(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_1", "gpt-test",
|
||||
@@ -409,24 +646,8 @@ fn transparent_encoder_should_require_matching_canonical_and_wire_terminals() {
|
||||
"response.created",
|
||||
json!({"type": "response.created", "response": {"id": "resp_1"}}),
|
||||
);
|
||||
encoder.push_sse(&started).expect("started event");
|
||||
let changed_identity = openai_wire_event(
|
||||
vec![GatewayEvent::Completed(ResponseMeta::new(
|
||||
"resp_2", "gpt-test",
|
||||
))],
|
||||
"response.completed",
|
||||
json!({"type": "response.completed", "response": {"id": "resp_2"}}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
encoder
|
||||
.push_sse(&changed_identity)
|
||||
.expect_err("identity changes must fail"),
|
||||
ResponseEncodeError::WireIdentityChanged
|
||||
);
|
||||
|
||||
let mut missing_terminal = OpenAiResponsesEncoder::new();
|
||||
missing_terminal.push_sse(&started).expect("started event");
|
||||
missing_terminal.push_sse(&started);
|
||||
assert_eq!(
|
||||
missing_terminal
|
||||
.finish()
|
||||
|
||||
@@ -1,14 +1,35 @@
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{GatewayError, ProviderError, ProviderErrorKind};
|
||||
use gateway_core::event::{
|
||||
GatewayEvent, ProtocolWireEvent, ProviderEvent, ProviderResponseHeader, ResponseMeta,
|
||||
};
|
||||
use gateway_core::operation::Operation;
|
||||
use serde_json::json;
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use serde_json::{Value, json};
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Message as ClientMessage;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::decode_response_create;
|
||||
use crate::openai::{api_router, models::ModelsExecution};
|
||||
use crate::openai::{api_router, authenticated_client, models::ModelsExecution};
|
||||
|
||||
#[test]
|
||||
fn response_create_should_default_to_the_websocket_streaming_contract() {
|
||||
@@ -148,11 +169,403 @@ fn response_create_should_reject_non_boolean_stream_without_disclosing_body_valu
|
||||
assert!(!rendered.contains(opaque_stream_value));
|
||||
}
|
||||
|
||||
// `serve_responses_websocket`/`forward_execution` 只有拿到 `hyper::upgrade::OnUpgrade`
|
||||
// 的真实 HTTP/1.1 升级连接才会运行;本 crate 的依赖图按 axum `ws` feature 编译
|
||||
// hyper(无 `http1`),测试进程内不存在能完成升级的服务端,`hyper::upgrade` 也不
|
||||
// 提供公开构造器。集成测试因此止步于 upgrade 边界:会话循环内的事件转发、错误帧
|
||||
// 与断连取消仍未被覆盖,需要 dev 依赖开启 `http1` 或 socket 泛化后才能落地。
|
||||
#[derive(Default)]
|
||||
struct AtomicFailureTrace {
|
||||
next_calls: AtomicUsize,
|
||||
committed: AtomicBool,
|
||||
finalized: AtomicBool,
|
||||
}
|
||||
|
||||
struct AtomicFailureSession {
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionSession for AtomicFailureSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
match self.trace.next_calls.fetch_add(1, Ordering::AcqRel) {
|
||||
0 => Ok(Some(atomic_failure_batch())),
|
||||
1 => {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
Err(EngineError::Provider(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async { Err(EngineError::InvalidDeliveryState) })
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn commit_downstream(&mut self, _: Option<u16>) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.trace.committed.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.trace.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.trace.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AtomicFailureExecution {
|
||||
client: AuthenticatedClient,
|
||||
trace: Arc<AtomicFailureTrace>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
}
|
||||
|
||||
impl ExecutionService for AtomicFailureExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_ws_atomic")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
vec![PublicModelId::new("model-a").expect("public model")]
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
model.as_str() == "model-a"
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
Ok(StartedExecution {
|
||||
request_id: gateway_core::engine::ModelRequestId::new("req_ws_atomic")
|
||||
.expect("request id"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: request.metadata.stream,
|
||||
session: Box::new(AtomicFailureSession {
|
||||
trace: Arc::clone(&self.trace),
|
||||
response_headers: self.response_headers.clone(),
|
||||
}),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn atomic_failure_batch() -> CoordinatedEvent {
|
||||
let started = ProviderEvent::canonical_with_wire(
|
||||
vec![GatewayEvent::Started(ResponseMeta::new(
|
||||
"resp_ws_atomic",
|
||||
"model-a",
|
||||
))],
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.created".to_owned()),
|
||||
json!({
|
||||
"type": "response.created",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"model": "model-a",
|
||||
"status": "in_progress"
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("created wire"),
|
||||
);
|
||||
let failed = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": "resp_ws_atomic",
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "rate_limit_exceeded",
|
||||
"message": "websocket atomic upstream marker"
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("failed wire"),
|
||||
);
|
||||
CoordinatedEvent::try_batch(
|
||||
vec![started, failed],
|
||||
CommitRequirement::CommitBeforeDelivery,
|
||||
)
|
||||
.expect("atomic WebSocket batch")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_atomic_upstream_failure_batch_should_be_forwarded_once() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace: Arc::clone(&trace),
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut messages = Vec::<Value>::new();
|
||||
loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
let failed = value.get("type").and_then(Value::as_str) == Some("response.failed");
|
||||
messages.push(value);
|
||||
if failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
messages
|
||||
.iter()
|
||||
.filter(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let failed = messages
|
||||
.iter()
|
||||
.find(|message| message.get("type").and_then(Value::as_str) == Some("response.failed"))
|
||||
.expect("upstream failed event");
|
||||
assert_eq!(
|
||||
failed
|
||||
.pointer("/response/error/message")
|
||||
.and_then(Value::as_str),
|
||||
Some("websocket atomic upstream marker")
|
||||
);
|
||||
assert!(trace.committed.load(Ordering::Acquire));
|
||||
assert_eq!(trace.next_calls.load(Ordering::Acquire), 2);
|
||||
assert!(trace.finalized.load(Ordering::Acquire));
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_metadata_should_preserve_ordinary_headers_without_blocking_events() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: vec![
|
||||
ProviderResponseHeader::new("x-future-header", Bytes::from_static(b"future-value")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
ProviderResponseHeader::new(
|
||||
"authorization",
|
||||
Bytes::from_static(b"should-not-cross-boundary"),
|
||||
),
|
||||
ProviderResponseHeader::new("connection", Bytes::from_static(b"x-private-hop")),
|
||||
ProviderResponseHeader::new("x-private-hop", Bytes::from_static(b"private-hop")),
|
||||
ProviderResponseHeader::new("content-type", Bytes::from_static(b"application/private")),
|
||||
ProviderResponseHeader::new("x-request-id", Bytes::from_static(b"req_upstream")),
|
||||
],
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket
|
||||
.send(ClientMessage::Text(
|
||||
json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "hello"
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.expect("send response.create");
|
||||
|
||||
let mut metadata = None;
|
||||
let mut saw_failed = false;
|
||||
while metadata.is_none() || !saw_failed {
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
match value.get("type").and_then(Value::as_str) {
|
||||
Some("response.metadata") => metadata = Some(value),
|
||||
Some("response.failed") => saw_failed = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let headers = metadata
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("headers"))
|
||||
.and_then(Value::as_object)
|
||||
.expect("response metadata headers");
|
||||
assert_eq!(headers.get("x-future-header"), Some(&json!("future-value")));
|
||||
// 官方 metadata 是 string map,同名多值只能保持既有的后值覆盖语义。
|
||||
assert_eq!(headers.get("x-future-multi"), Some(&json!("second")));
|
||||
assert_eq!(headers.get("x-request-id"), Some(&json!("req_upstream")));
|
||||
for omitted in [
|
||||
"x-future-bytes",
|
||||
"bad\0name",
|
||||
"authorization",
|
||||
"connection",
|
||||
"x-private-hop",
|
||||
"content-type",
|
||||
] {
|
||||
assert!(
|
||||
!headers.contains_key(omitted),
|
||||
"unexpected metadata header: {omitted:?}"
|
||||
);
|
||||
}
|
||||
assert!(saw_failed);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
trace,
|
||||
response_headers: Vec::new(),
|
||||
});
|
||||
let app = api_router(execution).await;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind WebSocket test server");
|
||||
let address = listener.local_addr().expect("test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("serve WebSocket test router");
|
||||
});
|
||||
let mut request = format!("ws://{address}/v1/responses")
|
||||
.into_client_request()
|
||||
.expect("WebSocket request");
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
"Bearer sk_ws_atomic".parse().expect("authorization"),
|
||||
);
|
||||
let (mut socket, response) = connect_async(request).await.expect("upgrade WebSocket");
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
let payload = json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "x".repeat(16 * 1024 * 1024 + 1),
|
||||
})
|
||||
.to_string();
|
||||
assert!(payload.len() > 16 * 1024 * 1024);
|
||||
|
||||
socket
|
||||
.send(ClientMessage::Text(payload.into()))
|
||||
.await
|
||||
.expect("send response.create above the former frame limit");
|
||||
|
||||
let mut saw_upstream_event = false;
|
||||
for _ in 0..3 {
|
||||
let message = tokio::time::timeout(Duration::from_secs(10), socket.next())
|
||||
.await
|
||||
.expect("WebSocket response timeout")
|
||||
.expect("WebSocket remains open")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
continue;
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("JSON WebSocket event");
|
||||
if matches!(
|
||||
value.get("type").and_then(Value::as_str),
|
||||
Some("response.created" | "response.failed")
|
||||
) {
|
||||
saw_upstream_event = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(saw_upstream_event);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
const TEST_WEBSOCKET_KEY: &str = "AAAAAAAAAAAAAAAAAAAAAA==";
|
||||
|
||||
|
||||
@@ -1,57 +1,29 @@
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
extract::DefaultBodyLimit,
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode},
|
||||
routing::post,
|
||||
};
|
||||
use gateway_api::openai::router::MAX_CLIENT_REQUEST_BODY_BYTES;
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
fn body_limit_app() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
"/v1/responses",
|
||||
post(|body: Bytes| async move {
|
||||
if body.len() == MAX_CLIENT_REQUEST_BODY_BYTES {
|
||||
StatusCode::NO_CONTENT
|
||||
} else {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
}),
|
||||
)
|
||||
.layer(DefaultBodyLimit::max(MAX_CLIENT_REQUEST_BODY_BYTES))
|
||||
}
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_accept_exactly_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES]))
|
||||
.expect("build exact-limit request"),
|
||||
.body(Body::from(vec![b'a'; REMOVED_BODY_LIMIT_BYTES + 1]))
|
||||
.expect("build request above the removed limit"),
|
||||
)
|
||||
.await
|
||||
.expect("route exact-limit request");
|
||||
.expect("route request above the removed limit");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_limit_should_reject_one_byte_over_sixteen_mibibytes() {
|
||||
let response = body_limit_app()
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; MAX_CLIENT_REQUEST_BODY_BYTES + 1]))
|
||||
.expect("build over-limit request"),
|
||||
)
|
||||
.await
|
||||
.expect("route over-limit request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
|
||||
// The handler sees the body and rejects the missing credentials. A restored body limit
|
||||
// would return 413 before authentication runs.
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -17,8 +17,9 @@ bytes = "1.11.1"
|
||||
futures = "0.3.32"
|
||||
futures-timer = "3.0.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["preserve_order"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -39,6 +39,30 @@ impl Decimal {
|
||||
.filter(|value| *value <= MAX_SCALED_DECIMAL)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
/// 除以非零整数,保留最多十位小数。
|
||||
#[must_use]
|
||||
pub fn checked_div_u64(self, divisor: u64) -> Option<Self> {
|
||||
let divisor = u128::from(divisor);
|
||||
(divisor != 0)
|
||||
.then(|| self.0.checked_div(divisor))
|
||||
.flatten()
|
||||
.and_then(|value| Self::from_scaled(value).ok())
|
||||
}
|
||||
|
||||
/// 去尾零的 canonical 字符串,用于 wire 序列化。
|
||||
#[must_use]
|
||||
pub fn canonical(self) -> String {
|
||||
let integer = self.0 / DECIMAL_SCALE;
|
||||
let fraction = self.0 % DECIMAL_SCALE;
|
||||
if fraction == 0 {
|
||||
integer.to_string()
|
||||
} else {
|
||||
let fraction = format!("{fraction:010}");
|
||||
let trimmed = fraction.trim_end_matches('0');
|
||||
format!("{integer}.{trimmed}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Decimal {
|
||||
|
||||
@@ -8,24 +8,21 @@ use std::fmt;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::ProviderAccountId;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
use crate::routing::ProviderKind;
|
||||
|
||||
/// 客户端传入的 previous response ID。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
/// 客户端或 Provider 传递的 opaque response handle。
|
||||
///
|
||||
/// Codex 定义这个值的语义和格式。网关只将其作为亲和查找键或同账号上游
|
||||
/// continuation 的载体,不得按私有长度、字符集或前缀规则拒绝、归一化或改写。
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct PreviousResponseId(String);
|
||||
|
||||
impl PreviousResponseId {
|
||||
/// 校验并创建 previous response ID。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// ID 为空、过长或包含控制字符时返回错误。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, 256, false, None)?;
|
||||
Ok(Self(value))
|
||||
/// 按原样创建 opaque response handle。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -34,6 +31,12 @@ impl PreviousResponseId {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for PreviousResponseId {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PreviousResponseId(<redacted>)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 原生 response handle 的续接范围。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NativeContinuationScope {
|
||||
@@ -52,7 +55,7 @@ pub struct NativeContinuationPin {
|
||||
/// 客户端提交、用于查找可丢失会话亲和记录的 response ID。
|
||||
previous_response_id: PreviousResponseId,
|
||||
/// Provider 原生 response handle。
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
scope: NativeContinuationScope,
|
||||
@@ -63,7 +66,7 @@ impl NativeContinuationPin {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
previous_response_id: PreviousResponseId,
|
||||
upstream_response_id: SafeUpstreamValue,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
) -> Self {
|
||||
@@ -98,7 +101,7 @@ impl NativeContinuationPin {
|
||||
|
||||
/// 返回只允许发送给已冻结 Provider 目标的原生上游 handle。
|
||||
#[must_use]
|
||||
pub const fn upstream_response_id(&self) -> &SafeUpstreamValue {
|
||||
pub const fn upstream_response_id(&self) -> &PreviousResponseId {
|
||||
&self.upstream_response_id
|
||||
}
|
||||
|
||||
@@ -187,9 +190,26 @@ impl fmt::Debug for ContinuationBinding {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable")]
|
||||
pub struct NativeContinuationStoreError;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable: {detail}")]
|
||||
pub struct NativeContinuationStoreError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
impl NativeContinuationStoreError {
|
||||
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
|
||||
#[must_use]
|
||||
pub fn new(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn detail(&self) -> &str {
|
||||
&self.detail
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失的 previous-response 亲和存储端口。
|
||||
///
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! 唯一的账号重试、发送与下游 commit barrier owner。
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
@@ -17,7 +18,7 @@ use crate::engine::{
|
||||
ModelRequestTimings, NewModelRequest, ProviderAccountStateOwner, ProviderAttemptOutcome,
|
||||
RequestAttemptContext, UpstreamSendState,
|
||||
};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind};
|
||||
use crate::error::{GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError};
|
||||
use crate::event::{
|
||||
GatewayEvent, ProviderEvent, ProviderResponseHeader, ProviderResponseObservation,
|
||||
};
|
||||
@@ -164,8 +165,10 @@ where
|
||||
timings: ModelRequestTimings::default(),
|
||||
client_response_id: None,
|
||||
upstream_response_id: None,
|
||||
last_account_exhaustion: None,
|
||||
last_retryable_failure: None,
|
||||
last_retryable_failure_events: Vec::new(),
|
||||
provider_attempt_outcomes: Vec::new(),
|
||||
pending_terminal_failure: None,
|
||||
};
|
||||
|
||||
if session.cancellation.is_cancelled() {
|
||||
@@ -242,8 +245,18 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
timings: ModelRequestTimings,
|
||||
client_response_id: Option<String>,
|
||||
upstream_response_id: Option<String>,
|
||||
last_account_exhaustion: Option<ProviderError>,
|
||||
/// 最近一次为无感恢复而被丢弃的原始上游错误;只在后续空选路时成为终态。
|
||||
last_retryable_failure: Option<ProviderError>,
|
||||
/// 与 `last_retryable_failure` 同属一个 attempt 的原始失败批次。
|
||||
last_retryable_failure_events: Vec<ProviderEvent>,
|
||||
provider_attempt_outcomes: Vec<ProviderAttemptOutcome>,
|
||||
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误。
|
||||
pending_terminal_failure: Option<PendingTerminalFailure>,
|
||||
}
|
||||
|
||||
struct PendingTerminalFailure {
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
}
|
||||
|
||||
impl<S: ?Sized> ResponseExecutionSession<S>
|
||||
@@ -259,6 +272,11 @@ where
|
||||
if self.delivery_pending {
|
||||
return Err(EngineError::DownstreamCommitRequired);
|
||||
}
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
self.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(pending.error));
|
||||
}
|
||||
if self.finalized {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -278,6 +296,21 @@ where
|
||||
)));
|
||||
}
|
||||
PullOutcome::AttemptDiscarded => {}
|
||||
PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
} => {
|
||||
let requirement = if self.downstream_committed_at.is_some() {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
self.delivery_pending = true;
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
};
|
||||
self.pending_terminal_failure =
|
||||
Some(PendingTerminalFailure { error, send_state });
|
||||
return CoordinatedEvent::try_batch(events, requirement).map(Some);
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if self.downstream_committed_at.is_some() {
|
||||
self.finish_success().await?;
|
||||
@@ -306,6 +339,13 @@ where
|
||||
match self.pull().await? {
|
||||
PullOutcome::Event(event) => events.push(event),
|
||||
PullOutcome::AttemptDiscarded => events.clear(),
|
||||
PullOutcome::TerminalFailure {
|
||||
error, send_state, ..
|
||||
} => {
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(error));
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if events.is_empty() {
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
@@ -317,11 +357,11 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// 在协议 adapter 真正写出首字节前持久化下游不可撤回边界。
|
||||
/// 在协议 adapter 真正写出首字节前记录下游不可撤回边界。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 没有待提交结果、重复提交或 Store 失败时返回错误。
|
||||
/// 没有待提交结果或重复提交时返回错误;观测 Store 失败只记服务端告警。
|
||||
pub async fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
@@ -330,10 +370,18 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
let committed_at = SystemTime::now();
|
||||
self.engine
|
||||
.store()
|
||||
.mark_downstream_committed(&self.request_id, committed_at, client_status_code)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_downstream_committed",
|
||||
&self.request_id,
|
||||
self.engine.store().mark_downstream_committed(
|
||||
&self.request_id,
|
||||
committed_at,
|
||||
client_status_code,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.downstream_committed_at = Some(committed_at);
|
||||
self.client_status_code = client_status_code;
|
||||
self.delivery_pending = false;
|
||||
@@ -347,7 +395,7 @@ where
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 状态已经写入或 Store 无法写回时返回错误。
|
||||
/// 状态已经写入时返回错误;观测 Store 失败只记服务端告警。
|
||||
pub async fn record_client_status(
|
||||
&mut self,
|
||||
client_status_code: u16,
|
||||
@@ -356,10 +404,14 @@ where
|
||||
return Err(EngineError::InvalidDeliveryState);
|
||||
}
|
||||
if self.request_persisted {
|
||||
self.engine
|
||||
.store()
|
||||
.record_client_status(&self.request_id, client_status_code)
|
||||
.await?;
|
||||
best_effort_store_write(
|
||||
"record_client_status",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.record_client_status(&self.request_id, client_status_code),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.client_status_code = Some(client_status_code);
|
||||
Ok(())
|
||||
@@ -403,9 +455,8 @@ where
|
||||
}
|
||||
let account = current.metadata.provider_account_id()?.clone();
|
||||
let response_id = self.upstream_response_id.as_deref()?;
|
||||
let previous_response_id = PreviousResponseId::new(response_id.to_owned()).ok()?;
|
||||
let upstream_response_id =
|
||||
crate::error::SafeUpstreamValue::new(response_id.to_owned()).ok()?;
|
||||
let previous_response_id = PreviousResponseId::new(response_id.to_owned());
|
||||
let upstream_response_id = PreviousResponseId::new(response_id.to_owned());
|
||||
Some(
|
||||
NativeContinuationPin::new(
|
||||
previous_response_id,
|
||||
@@ -434,13 +485,20 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
self.delivery_pending = false;
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
return self
|
||||
.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await;
|
||||
}
|
||||
self.finish_interruption(EngineError::Cancelled).await
|
||||
}
|
||||
|
||||
async fn pull(&mut self) -> Result<PullOutcome, EngineError> {
|
||||
loop {
|
||||
if self.current.is_none() {
|
||||
self.prepare_attempt().await?;
|
||||
if self.current.is_none()
|
||||
&& let Some(outcome) = self.prepare_attempt().await?
|
||||
{
|
||||
return Ok(outcome);
|
||||
}
|
||||
|
||||
let boundary = {
|
||||
@@ -467,32 +525,17 @@ where
|
||||
return Err(EngineError::Deadline);
|
||||
}
|
||||
PollBoundary::Item(Some(Ok(mut event))) => {
|
||||
if let Some(observation) = event.take_observation()
|
||||
&& let Err(error) = self.observe_response(observation)
|
||||
{
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let mut identity_error = None;
|
||||
for fact in event.canonical_facts() {
|
||||
if let Err(error) = self.observe_response_identity(fact) {
|
||||
identity_error = Some(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(error) = identity_error {
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
continue;
|
||||
if let Some(observation) = event.take_observation() {
|
||||
self.observe_response(observation);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_event(fact).await?;
|
||||
self.observe_response_identity(fact);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_event(fact).await;
|
||||
}
|
||||
if event.wire_event().is_some() && !event.has_canonical_facts() {
|
||||
self.observe_wire_event().await?;
|
||||
self.observe_wire_event().await;
|
||||
}
|
||||
if !event.has_client_event() {
|
||||
continue;
|
||||
@@ -500,8 +543,13 @@ where
|
||||
return Ok(PullOutcome::Event(event));
|
||||
}
|
||||
PollBoundary::Item(Some(Err(error))) => {
|
||||
if self.handle_stream_error(error).await? {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
match self.handle_stream_error(error).await? {
|
||||
StreamErrorOutcome::AttemptDiscarded => {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
terminal @ StreamErrorOutcome::TerminalFailure { .. } => {
|
||||
return Ok(terminal.into_pull_outcome());
|
||||
}
|
||||
}
|
||||
}
|
||||
PollBoundary::Item(None) => {
|
||||
@@ -513,7 +561,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_attempt(&mut self) -> Result<(), EngineError> {
|
||||
async fn prepare_attempt(&mut self) -> Result<Option<PullOutcome>, EngineError> {
|
||||
if self.attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
@@ -612,23 +660,39 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if error.kind() == ProviderErrorKind::NoEligibleAccount
|
||||
&& let Some(exhaustion) = self.last_account_exhaustion.clone()
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::NoEligibleAccount
|
||||
) && let Some(last_failure) = self.last_retryable_failure.take()
|
||||
{
|
||||
let engine_error = provider_engine_error(&exhaustion);
|
||||
self.finish_provider_error(&exhaustion).await?;
|
||||
return Err(engine_error);
|
||||
let send_state = self.current_send_state();
|
||||
let mut events = std::mem::take(&mut self.last_retryable_failure_events);
|
||||
if !events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut events);
|
||||
return Ok(Some(PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error: last_failure,
|
||||
send_state,
|
||||
}));
|
||||
}
|
||||
self.finish_provider_error(&last_failure).await?;
|
||||
return Err(provider_engine_error(last_failure));
|
||||
}
|
||||
if !(matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::Unavailable | ProviderErrorKind::NoEligibleAccount
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::ProviderInfrastructureUnavailable
|
||||
) && error.send_state() == UpstreamSendState::NotSent)
|
||||
{
|
||||
self.record_provider_failure(candidate.provider().clone(), error.kind());
|
||||
}
|
||||
let engine_error = provider_engine_error(&error);
|
||||
self.finish_provider_error(&error).await?;
|
||||
return Err(engine_error);
|
||||
return Err(provider_engine_error(error));
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -717,17 +781,30 @@ where
|
||||
http_version: None,
|
||||
};
|
||||
if self.request_persisted {
|
||||
self.engine.store().record_attempt(attempt_record).await?;
|
||||
best_effort_store_write(
|
||||
"record_attempt",
|
||||
&self.request_id,
|
||||
self.engine.store().record_attempt(attempt_record),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
let request = self
|
||||
.pending_request
|
||||
.take()
|
||||
.as_ref()
|
||||
.ok_or(EngineError::InvalidDeliveryState)?;
|
||||
self.engine
|
||||
.store()
|
||||
.create_model_request_with_attempt(request, attempt_record)
|
||||
.await?;
|
||||
self.request_persisted = true;
|
||||
if best_effort_store_write(
|
||||
"create_model_request_with_attempt",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.create_model_request_with_attempt(request.clone(), attempt_record),
|
||||
)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
self.pending_request = None;
|
||||
self.request_persisted = true;
|
||||
}
|
||||
}
|
||||
self.attempts = next_attempt.get();
|
||||
self.current = Some(CurrentAttempt {
|
||||
@@ -739,10 +816,15 @@ where
|
||||
send_observed: false,
|
||||
response_observation: None,
|
||||
});
|
||||
Ok(())
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) -> Result<(), EngineError> {
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) {
|
||||
self.observe_event_facts(event);
|
||||
self.mark_send_observed().await;
|
||||
}
|
||||
|
||||
fn observe_event_facts(&mut self, event: &GatewayEvent) {
|
||||
let elapsed = elapsed_ms(self.timing_started_at);
|
||||
observe_event_timing(&mut self.timings, event, elapsed);
|
||||
if let GatewayEvent::Usage(observed) = event {
|
||||
@@ -756,46 +838,42 @@ where
|
||||
if let GatewayEvent::ProviderCost(observed) = event {
|
||||
self.cost = observed.into_estimate();
|
||||
}
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
}
|
||||
|
||||
async fn observe_wire_event(&mut self) {
|
||||
self.mark_send_observed().await;
|
||||
}
|
||||
|
||||
async fn mark_send_observed(&mut self) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn observe_wire_event(&mut self) -> Result<(), EngineError> {
|
||||
let current = self.current.as_mut().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if !current.send_observed {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, UpstreamSendState::Sent)
|
||||
.await?;
|
||||
current.send_observed = true;
|
||||
self.send_state_watermark = UpstreamSendState::Sent;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_response(
|
||||
&mut self,
|
||||
observation: ProviderResponseObservation,
|
||||
) -> Result<(), ProviderError> {
|
||||
let send_state = self.current_send_state();
|
||||
let current = self
|
||||
.current
|
||||
.as_mut()
|
||||
.ok_or_else(|| ProviderError::new(ProviderErrorKind::Protocol, send_state))?;
|
||||
fn observe_response(&mut self, observation: ProviderResponseObservation) {
|
||||
let Some(current) = self.current.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.is_some_and(|existing| existing.transport() != observation.transport())
|
||||
{
|
||||
return Err(ProviderError::new(ProviderErrorKind::Protocol, send_state));
|
||||
return;
|
||||
}
|
||||
let observed = observation.timings();
|
||||
if let Some(value) = observed.transport_decision_wait_ms {
|
||||
@@ -823,47 +901,27 @@ where
|
||||
self.timings.provider_processing_ms = Some(value);
|
||||
}
|
||||
current.response_observation = Some(observation);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe_response_identity(&mut self, event: &GatewayEvent) -> Result<(), ProviderError> {
|
||||
fn observe_response_identity(&mut self, event: &GatewayEvent) {
|
||||
let metadata = match event {
|
||||
GatewayEvent::Started(metadata) | GatewayEvent::Completed(metadata) => metadata,
|
||||
_ => return Ok(()),
|
||||
_ => return,
|
||||
};
|
||||
let response_id = crate::error::SafeUpstreamValue::new(metadata.response_id().to_owned())
|
||||
.map_err(|_| {
|
||||
ProviderError::new(ProviderErrorKind::Protocol, UpstreamSendState::Sent)
|
||||
})?;
|
||||
let response_id = response_id.as_str();
|
||||
if response_id.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if self
|
||||
.upstream_response_id
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != response_id)
|
||||
{
|
||||
return Err(ProviderError::new(
|
||||
ProviderErrorKind::Protocol,
|
||||
UpstreamSendState::Sent,
|
||||
));
|
||||
}
|
||||
let response_id = response_id.to_owned();
|
||||
let response_id = metadata.response_id().to_owned();
|
||||
self.client_response_id = Some(response_id.clone());
|
||||
self.upstream_response_id = Some(response_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 返回 `true` 表示调用方必须丢弃本 attempt 已收集的未提交事件。
|
||||
async fn handle_stream_error(&mut self, error: ProviderError) -> Result<bool, EngineError> {
|
||||
/// 失败可重试时要求丢弃本 attempt;预算耗尽时可返回最后一批原始失败事件。
|
||||
async fn handle_stream_error(
|
||||
&mut self,
|
||||
mut error: ProviderError,
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
|
||||
// 均只接触已剥离的稳定错误字段。
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::RateLimited | ProviderErrorKind::QuotaExhausted
|
||||
) {
|
||||
self.last_account_exhaustion = Some(error.clone());
|
||||
}
|
||||
self.record_provider_failure(current.metadata.provider().clone(), error.kind());
|
||||
// attempt_send_state 是本 attempt 自身的发送事实,驱动重试门;
|
||||
// 持久化与终态用请求级水位,二者不可混用(水位会把早先 attempt 的
|
||||
@@ -874,10 +932,16 @@ where
|
||||
error.send_state()
|
||||
};
|
||||
let send_state = self.raise_send_watermark(attempt_send_state);
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state)
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let provider_proved_replay_safe = provider_proved_replay_safe(&error);
|
||||
let continuation_retry = self.prepare_continuation_retry(
|
||||
¤t,
|
||||
@@ -885,6 +949,12 @@ where
|
||||
attempt_send_state,
|
||||
provider_proved_replay_safe,
|
||||
);
|
||||
let pre_delivery_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
&& !self.delivery_pending
|
||||
&& error.allows_pre_delivery_retry()
|
||||
&& self.attempts < self.plan.max_attempts().get();
|
||||
let ordinary_retry = self.account_selection.required_account().is_none()
|
||||
&& self.continuation_attempt == ContinuationAttempt::None
|
||||
&& self.downstream_committed_at.is_none()
|
||||
@@ -907,9 +977,13 @@ where
|
||||
.credential_recovery_attempted_accounts
|
||||
.contains(account)
|
||||
});
|
||||
let retryable = continuation_retry || same_account_retry || ordinary_retry;
|
||||
let retryable =
|
||||
continuation_retry || same_account_retry || ordinary_retry || pre_delivery_retry;
|
||||
|
||||
if retryable {
|
||||
// 普通 clone 只保留稳定事实;原始 wire/HTTP response 由 request-local
|
||||
// 所有权保留到下一次 attempt 成功,或最终空选路时返回客户端。
|
||||
let persistence_error = self.request_persisted.then(|| error.clone());
|
||||
if same_account_retry {
|
||||
if let Some(account) = current.metadata.provider_account_id() {
|
||||
self.credential_recovery_attempted_accounts
|
||||
@@ -923,37 +997,64 @@ where
|
||||
{
|
||||
self.excluded_accounts.insert(account.clone());
|
||||
}
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: current.metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: current.metadata.upstream_model().clone(),
|
||||
upstream_status_code: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code),
|
||||
upstream_request_id: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::request_id)
|
||||
.map(|value| value.as_str().to_owned()),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
})
|
||||
.await?;
|
||||
self.last_retryable_failure_events = atomic_client_events;
|
||||
self.last_retryable_failure = Some(error);
|
||||
if let Some(error) = persistence_error {
|
||||
best_effort_store_write(
|
||||
"record_intermediate_failure",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: current.metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: current.metadata.upstream_model().clone(),
|
||||
upstream_status_code: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code),
|
||||
upstream_request_id: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::request_id)
|
||||
.map(|value| value.as_str().to_owned()),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.reset_uncommitted_observations();
|
||||
return Ok(true);
|
||||
return Ok(StreamErrorOutcome::AttemptDiscarded);
|
||||
}
|
||||
|
||||
let engine_error = provider_engine_error(&error);
|
||||
self.current = Some(current);
|
||||
if !atomic_client_events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut atomic_client_events);
|
||||
return Ok(StreamErrorOutcome::TerminalFailure {
|
||||
events: atomic_client_events,
|
||||
error,
|
||||
send_state,
|
||||
});
|
||||
}
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
Err(engine_error)
|
||||
Err(provider_engine_error(error))
|
||||
}
|
||||
|
||||
fn observe_atomic_terminal_events(&mut self, events: &mut [ProviderEvent]) {
|
||||
for event in events {
|
||||
if let Some(observation) = event.take_observation() {
|
||||
self.observe_response(observation);
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observe_response_identity(fact);
|
||||
self.observe_event_facts(fact);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_continuation_retry(
|
||||
@@ -998,6 +1099,41 @@ where
|
||||
true
|
||||
}
|
||||
|
||||
/// 原生续写的原账号在本地调度阶段已不可用时,改用完整输入重放。
|
||||
///
|
||||
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
|
||||
/// 对应 Provider 的可携带会话状态时,才可安全移除 native handle 并换号。
|
||||
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt != ContinuationAttempt::Native
|
||||
|| !matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(pin) = self
|
||||
.continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if self
|
||||
.operation
|
||||
.provider_session_state(pin.provider().as_str())
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn reset_uncommitted_observations(&mut self) {
|
||||
self.usage = Usage::new();
|
||||
self.cost = CostEstimate::unavailable();
|
||||
@@ -1043,34 +1179,42 @@ where
|
||||
.and_then(ProviderResponseObservation::status_code);
|
||||
let (upstream_transport, http_version, websocket_pool) =
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: ExecutionOutcome::Succeeded,
|
||||
send_state: UpstreamSendState::Sent,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code,
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: self.upstream_response_id.clone(),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
provider_metadata_json,
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
retry_after_ms: None,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
})
|
||||
.await?;
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"finalize_model_request",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: ExecutionOutcome::Succeeded,
|
||||
send_state: UpstreamSendState::Sent,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code,
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: self.upstream_response_id.clone(),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
error: None,
|
||||
provider_error_code: None,
|
||||
retry_after_ms: None,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finalized = true;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1139,11 +1283,15 @@ where
|
||||
UpstreamSendState::Ambiguous
|
||||
};
|
||||
let send_state = self.raise_send_watermark(send_state);
|
||||
if self.attempts > 0 {
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state)
|
||||
.await?;
|
||||
if self.attempts > 0 && self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome,
|
||||
@@ -1193,36 +1341,44 @@ where
|
||||
.and_then(ProviderResponseObservation::status_code);
|
||||
let (upstream_transport, http_version, websocket_pool) =
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: finalization.outcome,
|
||||
send_state: finalization.send_state,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code: finalization.upstream_status_code.or(observed_status_code),
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: finalization
|
||||
.provider_response_id
|
||||
.or_else(|| self.upstream_response_id.clone()),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
provider_metadata_json,
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
})
|
||||
.await?;
|
||||
best_effort_store_write(
|
||||
"finalize_model_request",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.finalize_model_request(ModelRequestFinalization {
|
||||
request_id: self.request_id.clone(),
|
||||
outcome: finalization.outcome,
|
||||
send_state: finalization.send_state,
|
||||
attempt_count: self.attempts,
|
||||
downstream_committed_at: self.downstream_committed_at,
|
||||
client_status_code: self.client_status_code,
|
||||
upstream_status_code: finalization
|
||||
.upstream_status_code
|
||||
.or(observed_status_code),
|
||||
client_response_id: self.client_response_id.clone(),
|
||||
upstream_request_id,
|
||||
upstream_response_id: finalization
|
||||
.provider_response_id
|
||||
.or_else(|| self.upstream_response_id.clone()),
|
||||
upstream_transport,
|
||||
http_version,
|
||||
websocket_pool,
|
||||
service_tier,
|
||||
provider_metadata_json,
|
||||
error: Some(finalization.error),
|
||||
provider_error_code: finalization.provider_error_code,
|
||||
retry_after_ms: finalization.retry_after_ms,
|
||||
usage: self.usage.clone(),
|
||||
image_generation_succeeded: self.image_generation_succeeded(),
|
||||
cost: self.cost.clone(),
|
||||
timings: self.timings.clone(),
|
||||
completed_at,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
self.finalized = true;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1253,6 +1409,14 @@ where
|
||||
.map(|metadata| metadata.as_json().to_owned())
|
||||
}
|
||||
|
||||
fn current_service_tier(&self) -> Option<String> {
|
||||
self.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::service_tier)
|
||||
.map(str::to_owned)
|
||||
}
|
||||
|
||||
fn image_generation_succeeded(&self) -> Option<bool> {
|
||||
self.image_generation_requested
|
||||
.then(|| self.usage.image_output_tokens.unwrap_or_default() > 0)
|
||||
@@ -1302,12 +1466,62 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn best_effort_store_write<T>(
|
||||
operation: &'static str,
|
||||
request_id: &ModelRequestId,
|
||||
write: impl Future<Output = Result<T, StoreError>>,
|
||||
) -> Option<T> {
|
||||
match write.await {
|
||||
Ok(value) => Some(value),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
operation,
|
||||
request_id = request_id.as_str(),
|
||||
error_kind = ?error.kind(),
|
||||
"执行观测写入失败,数据面不受影响"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PullOutcome {
|
||||
Event(ProviderEvent),
|
||||
AttemptDiscarded,
|
||||
TerminalFailure {
|
||||
events: Vec<ProviderEvent>,
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
},
|
||||
End,
|
||||
}
|
||||
|
||||
enum StreamErrorOutcome {
|
||||
AttemptDiscarded,
|
||||
TerminalFailure {
|
||||
events: Vec<ProviderEvent>,
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
},
|
||||
}
|
||||
|
||||
impl StreamErrorOutcome {
|
||||
fn into_pull_outcome(self) -> PullOutcome {
|
||||
match self {
|
||||
Self::AttemptDiscarded => PullOutcome::AttemptDiscarded,
|
||||
Self::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
} => PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PollBoundary {
|
||||
Item(Option<Result<ProviderEvent, ProviderError>>),
|
||||
Cancelled,
|
||||
@@ -1380,11 +1594,11 @@ fn initial_continuation_attempt(
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_engine_error(error: &ProviderError) -> EngineError {
|
||||
fn provider_engine_error(error: ProviderError) -> EngineError {
|
||||
if error.kind() == ProviderErrorKind::Cancelled {
|
||||
EngineError::Cancelled
|
||||
} else {
|
||||
EngineError::Provider(error.clone())
|
||||
EngineError::Provider(error)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,46 @@ impl fmt::Display for ProviderAccountId {
|
||||
}
|
||||
}
|
||||
|
||||
/// credential 轮换时可选替换的上游账号身份。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderAccountIdentity {
|
||||
upstream_user_id: String,
|
||||
upstream_account_id: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderAccountIdentity {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderAccountIdentity")
|
||||
.field("upstream_user_id", &"<redacted>")
|
||||
.field(
|
||||
"upstream_account_id",
|
||||
&self.upstream_account_id.as_ref().map(|_| "<redacted>"),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAccountIdentity {
|
||||
#[must_use]
|
||||
pub const fn new(upstream_user_id: String, upstream_account_id: Option<String>) -> Self {
|
||||
Self {
|
||||
upstream_user_id,
|
||||
upstream_account_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_account_id(&self) -> Option<&str> {
|
||||
self.upstream_account_id.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// `provider_accounts.credential_revision` 的正数 CAS revision。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct CredentialRevision(NonZeroU64);
|
||||
@@ -148,11 +188,13 @@ impl fmt::Debug for OpaqueProviderData {
|
||||
}
|
||||
|
||||
/// 数据库中固定的账号运行状态。
|
||||
///
|
||||
/// 只承载终态或需要外部干预的状态;临时限流(429)不进入此状态,由
|
||||
/// Provider quota 数据驱动(`AccountQuotaSignals::limit_reached`)。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum AccountAvailability {
|
||||
Unknown,
|
||||
Ready,
|
||||
Cooldown,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Banned,
|
||||
@@ -166,7 +208,6 @@ impl AccountAvailability {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Ready => "ready",
|
||||
Self::Cooldown => "cooldown",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::Expired => "expired",
|
||||
Self::Banned => "banned",
|
||||
@@ -174,13 +215,26 @@ impl AccountAvailability {
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回账号进入不可恢复错误态、但 Provider 未给出更具体上游原因时的稳定兜底。
|
||||
///
|
||||
/// Provider 应优先持久化已脱敏的上游 message 或原因码;该值只避免错误态沿用
|
||||
/// 上一次无关的错误信息。
|
||||
#[must_use]
|
||||
pub const fn fallback_error_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Expired => Some("credential_expired"),
|
||||
Self::Banned => Some("account_banned"),
|
||||
Self::Invalid => Some("credential_invalid"),
|
||||
Self::Unknown | Self::Ready | Self::QuotaExhausted => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 解析数据库稳定值。
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"unknown" => Some(Self::Unknown),
|
||||
"ready" => Some(Self::Ready),
|
||||
"cooldown" => Some(Self::Cooldown),
|
||||
"quota_exhausted" => Some(Self::QuotaExhausted),
|
||||
"expired" => Some(Self::Expired),
|
||||
"banned" => Some(Self::Banned),
|
||||
@@ -197,14 +251,13 @@ pub struct ProviderAccount {
|
||||
provider: ProviderKind,
|
||||
name: String,
|
||||
email: Option<String>,
|
||||
upstream_user_id: String,
|
||||
upstream_user_id: Option<String>,
|
||||
upstream_account_id: Option<String>,
|
||||
plan_type: Option<String>,
|
||||
authentication_kind: String,
|
||||
revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
next_refresh_at: Option<SystemTime>,
|
||||
has_refresh_token: bool,
|
||||
@@ -217,7 +270,7 @@ impl ProviderAccount {
|
||||
id: ProviderAccountId,
|
||||
provider: ProviderKind,
|
||||
name: String,
|
||||
upstream_user_id: String,
|
||||
upstream_user_id: Option<String>,
|
||||
authentication_kind: String,
|
||||
revision: CredentialRevision,
|
||||
access_token_expires_at: Option<SystemTime>,
|
||||
@@ -234,7 +287,6 @@ impl ProviderAccount {
|
||||
revision,
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Unknown,
|
||||
cooldown_until: None,
|
||||
access_token_expires_at,
|
||||
next_refresh_at: None,
|
||||
has_refresh_token: false,
|
||||
@@ -259,14 +311,20 @@ impl ProviderAccount {
|
||||
mut self,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
cooldown_until: Option<SystemTime>,
|
||||
) -> Self {
|
||||
self.enabled = enabled;
|
||||
self.availability = availability;
|
||||
self.cooldown_until = cooldown_until;
|
||||
self.availability = if self.upstream_user_id.is_some() {
|
||||
availability
|
||||
} else {
|
||||
AccountAvailability::Unknown
|
||||
};
|
||||
self
|
||||
}
|
||||
|
||||
/// 设置 RT 存在性与失败后的最早重试时刻。
|
||||
///
|
||||
/// 正常 OAuth 预刷新由 worker 使用 AT 原始过期时间与当前运行时策略动态判断,
|
||||
/// 不应把提前量物化到 `next_refresh_at`。
|
||||
#[must_use]
|
||||
pub const fn with_refresh_schedule(
|
||||
mut self,
|
||||
@@ -304,8 +362,8 @@ impl ProviderAccount {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn upstream_user_id(&self) -> &str {
|
||||
&self.upstream_user_id
|
||||
pub fn upstream_user_id(&self) -> Option<&str> {
|
||||
self.upstream_user_id.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -333,16 +391,12 @@ impl ProviderAccount {
|
||||
self.enabled
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cooldown_until(&self) -> Option<SystemTime> {
|
||||
self.cooldown_until
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn access_token_expires_at(&self) -> Option<SystemTime> {
|
||||
self.access_token_expires_at
|
||||
}
|
||||
|
||||
/// 返回瞬态 OAuth 刷新失败后的最早重试时刻;正常账号为 `None`。
|
||||
#[must_use]
|
||||
pub const fn next_refresh_at(&self) -> Option<SystemTime> {
|
||||
self.next_refresh_at
|
||||
@@ -358,14 +412,14 @@ impl ProviderAccount {
|
||||
pub fn is_schedulable(&self, now: SystemTime) -> bool {
|
||||
let available = match self.availability {
|
||||
AccountAvailability::Ready => true,
|
||||
AccountAvailability::Cooldown => self.cooldown_until.is_some_and(|until| until <= now),
|
||||
AccountAvailability::Unknown
|
||||
| AccountAvailability::QuotaExhausted
|
||||
| AccountAvailability::Expired
|
||||
| AccountAvailability::Banned
|
||||
| AccountAvailability::Invalid => false,
|
||||
};
|
||||
self.enabled
|
||||
self.upstream_user_id.is_some()
|
||||
&& self.enabled
|
||||
&& available
|
||||
&& self
|
||||
.access_token_expires_at
|
||||
@@ -548,6 +602,8 @@ pub struct QuotaObservation {
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub quota: Option<OpaqueProviderData>,
|
||||
pub observed_at: Option<SystemTime>,
|
||||
/// 快照级限流事实(Provider 解析后物化);None 表示不更新物化列。
|
||||
pub limit_reached: Option<bool>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for QuotaObservation {
|
||||
@@ -574,9 +630,9 @@ pub struct AccountStateChange {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub expected_revision: CredentialRevision,
|
||||
pub availability: AccountAvailability,
|
||||
pub reason: Option<String>,
|
||||
pub cooldown_until: Option<SystemTime>,
|
||||
pub observed_at: SystemTime,
|
||||
/// 上游原因(错误原文 / 原因码);`Some` 写入持久化错误信息,`None` 不更新,恢复 `Ready` 时清空。
|
||||
pub message: Option<String>,
|
||||
}
|
||||
|
||||
/// `provider_accounts` 的数据库中立端口。
|
||||
@@ -602,6 +658,15 @@ pub trait ProviderAccountStore: Send + Sync {
|
||||
expected_revision: CredentialRevision,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
/// 读取账号当前 credential 及其 revision,不做任何版本比对。
|
||||
///
|
||||
/// 管理写入必须在临近 CAS 时用它取 fence,而不是携带调用方持有的旧 revision:
|
||||
/// 后台刷新随时会推进 revision,用陈旧快照会把正常的恢复操作误判为冲突。
|
||||
async fn load_current_credential(
|
||||
&self,
|
||||
account: &ProviderAccountId,
|
||||
) -> Result<LoadedCredential, StoreError>;
|
||||
|
||||
async fn compare_and_swap_credential(
|
||||
&self,
|
||||
update: CredentialCasUpdate,
|
||||
@@ -649,6 +714,17 @@ impl RotationStrategy {
|
||||
Self::Sticky => "sticky",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"smart" => Some(Self::Smart),
|
||||
"quota_reset_priority" => Some(Self::QuotaResetPriority),
|
||||
"round_robin" => Some(Self::RoundRobin),
|
||||
"sticky" => Some(Self::Sticky),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `runtime_settings` 冻结到一次请求计划的账号调度策略。
|
||||
@@ -696,6 +772,7 @@ pub struct AccountRuntimeSignals {
|
||||
pub last_started_at: Option<SystemTime>,
|
||||
pub quota_reset_at: Option<SystemTime>,
|
||||
pub quota_remaining_rank: Option<u64>,
|
||||
pub quota_limit_reached: bool,
|
||||
pub failure_rate_basis_points: Option<u16>,
|
||||
pub first_output_latency_ms: Option<u64>,
|
||||
}
|
||||
@@ -843,6 +920,7 @@ impl AccountRuntimeSignals {
|
||||
if let Some(quota) = quota {
|
||||
self.quota_reset_at = quota.reset_at;
|
||||
self.quota_remaining_rank = quota.remaining_rank;
|
||||
self.quota_limit_reached = quota.limit_reached;
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -864,14 +942,20 @@ impl AccountRuntimeSignals {
|
||||
pub struct AccountQuotaSignals {
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
}
|
||||
|
||||
impl AccountQuotaSignals {
|
||||
#[must_use]
|
||||
pub const fn new(reset_at: Option<SystemTime>, remaining_rank: Option<u64>) -> Self {
|
||||
pub const fn new(
|
||||
reset_at: Option<SystemTime>,
|
||||
remaining_rank: Option<u64>,
|
||||
limit_reached: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
reset_at,
|
||||
remaining_rank,
|
||||
limit_reached,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -884,6 +968,12 @@ impl AccountQuotaSignals {
|
||||
pub const fn remaining_rank(self) -> Option<u64> {
|
||||
self.remaining_rank
|
||||
}
|
||||
|
||||
/// 任一计量窗口已触顶(`limit_reached`/`used_percent >= 100`);调度据此排除候选。
|
||||
#[must_use]
|
||||
pub const fn limit_reached(self) -> bool {
|
||||
self.limit_reached
|
||||
}
|
||||
}
|
||||
|
||||
/// 账号持久事实与可重建运行信号的请求级组合。
|
||||
@@ -923,6 +1013,43 @@ impl AccountAvailabilityPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// 候选账号未进入调度池的约束。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountSchedulingBlocker {
|
||||
LocalAvailability,
|
||||
Excluded,
|
||||
ConcurrencyLimit,
|
||||
RequestInterval,
|
||||
}
|
||||
|
||||
/// 优先账号在本次选择中的处理结果。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PreferredAccountSelection {
|
||||
NotRequested,
|
||||
Hit,
|
||||
Missing,
|
||||
Blocked(AccountSchedulingBlocker),
|
||||
}
|
||||
|
||||
/// 一次完整账号选择;同时保留优先账号决策,供 Provider 记录调度遥测。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AccountSelection<'a> {
|
||||
candidate: &'a AccountCandidate,
|
||||
preferred: PreferredAccountSelection,
|
||||
}
|
||||
|
||||
impl<'a> AccountSelection<'a> {
|
||||
#[must_use]
|
||||
pub const fn candidate(self) -> &'a AccountCandidate {
|
||||
self.candidate
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn preferred(self) -> PreferredAccountSelection {
|
||||
self.preferred
|
||||
}
|
||||
}
|
||||
|
||||
/// 同一 target 内唯一的账号排序器。
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct AccountSelector;
|
||||
@@ -934,43 +1061,55 @@ impl AccountSelector {
|
||||
&self,
|
||||
candidates: &'a [AccountCandidate],
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<&'a AccountCandidate> {
|
||||
) -> Option<AccountSelection<'a>> {
|
||||
let preferred = if let Some(preferred) = context.preferred_account.as_ref() {
|
||||
match candidates
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
{
|
||||
Some(candidate) => match self.scheduling_blocker(candidate, context) {
|
||||
None => {
|
||||
return Some(AccountSelection {
|
||||
candidate,
|
||||
preferred: PreferredAccountSelection::Hit,
|
||||
});
|
||||
}
|
||||
Some(blocker) => PreferredAccountSelection::Blocked(blocker),
|
||||
},
|
||||
None => PreferredAccountSelection::Missing,
|
||||
}
|
||||
} else {
|
||||
PreferredAccountSelection::NotRequested
|
||||
};
|
||||
|
||||
let mut eligible = candidates
|
||||
.iter()
|
||||
.filter(|candidate| eligible(candidate, context))
|
||||
.filter(|candidate| self.scheduling_blocker(candidate, context).is_none())
|
||||
.collect::<Vec<_>>();
|
||||
if eligible.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(preferred) = context.preferred_account.as_ref()
|
||||
&& let Some(candidate) = eligible
|
||||
.iter()
|
||||
.find(|candidate| candidate.account.id() == preferred)
|
||||
&& (context.policy.strategy() == RotationStrategy::Sticky
|
||||
|| (context.policy.strategy() == RotationStrategy::Smart
|
||||
&& !smart_preference_should_escape(&candidate.signals)))
|
||||
{
|
||||
return Some(candidate);
|
||||
}
|
||||
|
||||
match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
}),
|
||||
let candidate = match context.policy.strategy() {
|
||||
RotationStrategy::QuotaResetPriority => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
(
|
||||
candidate.signals.quota_reset_at.is_none(),
|
||||
candidate.signals.quota_reset_at,
|
||||
candidate.signals.in_flight,
|
||||
candidate.signals.last_started_at,
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
RotationStrategy::RoundRobin => {
|
||||
eligible.sort_by_key(|candidate| candidate.account.id().clone());
|
||||
let index = context.round_robin_cursor as usize % eligible.len();
|
||||
return Some(eligible[index]);
|
||||
eligible.get(index).copied()?
|
||||
}
|
||||
RotationStrategy::Smart => {
|
||||
return select_smart_candidate(&eligible, context.round_robin_cursor);
|
||||
select_smart_candidate(&eligible, context.round_robin_cursor)?
|
||||
}
|
||||
RotationStrategy::Sticky => {
|
||||
eligible.sort_by_key(|candidate| {
|
||||
@@ -979,9 +1118,44 @@ impl AccountSelector {
|
||||
candidate.account.id().clone(),
|
||||
)
|
||||
});
|
||||
eligible.first().copied()?
|
||||
}
|
||||
};
|
||||
Some(AccountSelection {
|
||||
candidate,
|
||||
preferred,
|
||||
})
|
||||
}
|
||||
|
||||
fn scheduling_blocker(
|
||||
&self,
|
||||
candidate: &AccountCandidate,
|
||||
context: &AccountSelectionContext,
|
||||
) -> Option<AccountSchedulingBlocker> {
|
||||
if !context.availability.bypasses_local_availability()
|
||||
&& !candidate.account.is_schedulable(context.now)
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::LocalAvailability);
|
||||
}
|
||||
eligible.into_iter().next()
|
||||
if context.excluded_accounts.contains(candidate.account.id()) {
|
||||
return Some(AccountSchedulingBlocker::Excluded);
|
||||
}
|
||||
if candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get() {
|
||||
return Some(AccountSchedulingBlocker::ConcurrencyLimit);
|
||||
}
|
||||
if candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_some_and(|last_started| {
|
||||
!context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
{
|
||||
return Some(AccountSchedulingBlocker::RequestInterval);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -989,8 +1163,6 @@ const SMART_LOAD_WEIGHT: f64 = 1.0;
|
||||
const SMART_QUOTA_WEIGHT: f64 = 0.8;
|
||||
const SMART_FAILURE_WEIGHT: f64 = 1.0;
|
||||
const SMART_LATENCY_WEIGHT: f64 = 0.5;
|
||||
const SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS: u16 = 5_000;
|
||||
const SMART_PREFERENCE_LATENCY_ESCAPE_MS: u64 = 15_000;
|
||||
|
||||
struct SmartNormalization {
|
||||
max_in_flight: u32,
|
||||
@@ -1111,35 +1283,6 @@ fn higher_is_better(value: u64, minimum: u64, maximum: u64) -> f64 {
|
||||
(value.saturating_sub(minimum) as f64 / (maximum - minimum) as f64).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn smart_preference_should_escape(signals: &AccountRuntimeSignals) -> bool {
|
||||
signals
|
||||
.failure_rate_basis_points
|
||||
.is_some_and(|rate| rate > SMART_PREFERENCE_FAILURE_ESCAPE_BASIS_POINTS)
|
||||
|| signals
|
||||
.first_output_latency_ms
|
||||
.is_some_and(|latency| latency > SMART_PREFERENCE_LATENCY_ESCAPE_MS)
|
||||
}
|
||||
|
||||
fn eligible(candidate: &AccountCandidate, context: &AccountSelectionContext) -> bool {
|
||||
if (!context.availability.bypasses_local_availability()
|
||||
&& !candidate.account.is_schedulable(context.now))
|
||||
|| context.excluded_accounts.contains(candidate.account.id())
|
||||
|| candidate.signals.in_flight >= context.policy.max_concurrent_per_account().get()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
candidate
|
||||
.signals
|
||||
.last_started_at
|
||||
.is_none_or(|last_started| {
|
||||
context
|
||||
.now
|
||||
.duration_since(last_started)
|
||||
.is_ok_and(|elapsed| elapsed >= context.policy.request_interval())
|
||||
})
|
||||
}
|
||||
|
||||
/// Credential 值对象构造错误。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
|
||||
pub enum CredentialError {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::net::IpAddr;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
@@ -17,7 +18,10 @@ use crate::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
|
||||
};
|
||||
use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{AccountProbe, AccountProbeRequest, AccountProbeResult};
|
||||
use crate::engine::probe::{
|
||||
AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult,
|
||||
AccountProbeUpstreamResponse,
|
||||
};
|
||||
use crate::engine::provider::ProviderRegistry;
|
||||
use crate::engine::{
|
||||
AttemptCoordinator, AttemptRecord, CancellationToken, CoordinatedEvent, EngineError,
|
||||
@@ -36,7 +40,7 @@ use crate::routing::{
|
||||
};
|
||||
|
||||
const MODEL_REQUEST_DEADLINE: Duration = Duration::from_secs(10 * 60);
|
||||
const CONTINUATION_AFFINITY_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const COORDINATION_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClientTransport {
|
||||
@@ -170,6 +174,22 @@ pub enum ProviderCircuitDecision {
|
||||
#[error("provider circuit store is unavailable")]
|
||||
pub struct ProviderCircuitError;
|
||||
|
||||
/// Provider circuit 的可重建协调策略;由 Core 拥有并交给 Store adapter 执行。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ProviderCircuitPolicy {
|
||||
pub failure_threshold: NonZeroU32,
|
||||
pub open_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for ProviderCircuitPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
failure_threshold: NonZeroU32::new(3).unwrap_or(NonZeroU32::MIN),
|
||||
open_duration: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCircuitPort: Send + Sync {
|
||||
fn decision<'a>(
|
||||
&'a self,
|
||||
@@ -233,15 +253,6 @@ impl DefaultExecutionService {
|
||||
request.client.policy.authorize().map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
|
||||
})?;
|
||||
if !request.client.snapshot.contains_public_model_for_provider(
|
||||
&request.public_model,
|
||||
request.client.policy.provider_kind(),
|
||||
) {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
));
|
||||
}
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = started_at
|
||||
.checked_add(MODEL_REQUEST_DEADLINE)
|
||||
@@ -260,10 +271,20 @@ impl DefaultExecutionService {
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => result.ok().flatten(),
|
||||
result = resolve => match result {
|
||||
Ok(pin) => pin,
|
||||
Err(error) => {
|
||||
tracing::debug!(
|
||||
request_id = request_id.as_str(),
|
||||
%error,
|
||||
"Continuation affinity 查询失败,退化为外部续接"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
_ = timeout => None,
|
||||
};
|
||||
match pin {
|
||||
@@ -398,14 +419,33 @@ impl DefaultExecutionService {
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let blocked_providers = match self.circuits.decision(provider_kind).await.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"provider health state is temporarily unavailable",
|
||||
)
|
||||
})? {
|
||||
ProviderCircuitDecision::Allow => BTreeSet::new(),
|
||||
ProviderCircuitDecision::BlockedUntil(_) => BTreeSet::from([provider_kind.clone()]),
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
_ = timeout => None,
|
||||
};
|
||||
let blocked_providers = match decision {
|
||||
Some(Ok(ProviderCircuitDecision::Allow)) => BTreeSet::new(),
|
||||
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
|
||||
BTreeSet::from([provider_kind.clone()])
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
%error,
|
||||
"Provider circuit 读取失败,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
"Provider circuit 读取超时,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
};
|
||||
Ok(RoutingContext {
|
||||
provider_kind: Some(provider_kind.clone()),
|
||||
@@ -416,7 +456,7 @@ impl DefaultExecutionService {
|
||||
async fn probe_inner(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> Result<AccountProbeResult, GatewayError> {
|
||||
) -> Result<AccountProbeResult, AccountProbeError> {
|
||||
let AccountProbeRequest {
|
||||
account_id,
|
||||
provider_kind,
|
||||
@@ -525,26 +565,55 @@ impl DefaultExecutionService {
|
||||
})
|
||||
}
|
||||
|
||||
/// 探测失败先落库再转成客户端错误;转换会丢掉上游状态码与 provider code。
|
||||
/// 探测失败先记录脱敏分类事实,再把请求局部的原始上游响应交给认证管理端。
|
||||
async fn observe_probe_failure(
|
||||
&self,
|
||||
observed: &ProbeObservation,
|
||||
started_at: SystemTime,
|
||||
error: &EngineError,
|
||||
) -> GatewayError {
|
||||
) -> AccountProbeError {
|
||||
let upstream_response = match error {
|
||||
EngineError::Provider(provider_error) => provider_error
|
||||
.client_visible_upstream_response()
|
||||
.map(AccountProbeUpstreamResponse::from_client_response),
|
||||
_ => None,
|
||||
};
|
||||
if let EngineError::Provider(provider_error) = error {
|
||||
let _ = self
|
||||
let latency = started_at.elapsed().unwrap_or_default();
|
||||
let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
|
||||
tracing::warn!(
|
||||
target: "gateway_probe",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
upstream_model = observed.upstream_model.as_str(),
|
||||
failure_kind = provider_error.kind().as_str(),
|
||||
send_state = ?provider_error.send_state(),
|
||||
upstream_status = ?provider_error.upstream_status(),
|
||||
provider_error_code = ?provider_error.upstream_code().map(|code| code.as_str()),
|
||||
latency_ms,
|
||||
"账号连接测试失败"
|
||||
);
|
||||
if let Err(store_error) = self
|
||||
.observations
|
||||
.record_probe_failure(ProbeFailure {
|
||||
provider_kind: observed.provider_kind.clone(),
|
||||
account_id: observed.account_id.clone(),
|
||||
upstream_model_id: observed.upstream_model.clone(),
|
||||
error: provider_error.clone(),
|
||||
latency: started_at.elapsed().unwrap_or_default(),
|
||||
latency,
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
operation = "record_probe_failure",
|
||||
provider_kind = observed.provider_kind.as_str(),
|
||||
account_id = observed.account_id.as_str(),
|
||||
error_kind = ?store_error.kind(),
|
||||
"账号连接测试观测写入失败,测试结果不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
gateway_error_from_engine(error)
|
||||
AccountProbeError::new(gateway_error_from_engine(error), upstream_response)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -651,7 +720,7 @@ impl AccountProbe for DefaultExecutionService {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>> {
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move { self.probe_inner(request).await })
|
||||
}
|
||||
}
|
||||
@@ -664,10 +733,13 @@ struct AdmissionLease {
|
||||
|
||||
impl AdmissionLease {
|
||||
async fn release(self) {
|
||||
let _ = self
|
||||
if let Err(error) = self
|
||||
.port
|
||||
.release(&self.client_api_key_id, &self.model_request_id)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(%error, "Client admission 释放失败,依赖租约 TTL 收敛");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -742,13 +814,7 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
self.continuation_recorded = true;
|
||||
let record = self.continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(CONTINUATION_AFFINITY_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
_ = record => {},
|
||||
_ = timeout => {},
|
||||
}
|
||||
record_native_continuation(self.continuation.as_ref(), pin).await;
|
||||
}
|
||||
|
||||
async fn finalize_detached(&mut self) {
|
||||
@@ -756,8 +822,10 @@ impl DefaultExecutionSession {
|
||||
return;
|
||||
};
|
||||
core.cancel();
|
||||
if !core.is_finalized() {
|
||||
let _ = core.cancel_and_finalize().await;
|
||||
if !core.is_finalized()
|
||||
&& let Err(error) = core.cancel_and_finalize().await
|
||||
{
|
||||
tracing::warn!(%error, "Detached execution 终态收敛失败");
|
||||
}
|
||||
let pending = core
|
||||
.provider_attempt_outcomes()
|
||||
@@ -878,7 +946,43 @@ async fn publish_provider_attempt_outcomes(
|
||||
}
|
||||
Some(_) => continue,
|
||||
};
|
||||
let _ = result;
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(
|
||||
provider = outcome.provider_kind().as_str(),
|
||||
%error,
|
||||
"Provider circuit feedback 写入失败,数据面不受影响"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_native_continuation(
|
||||
continuation: &dyn NativeContinuationPort,
|
||||
pin: NativeContinuationPin,
|
||||
) {
|
||||
let provider = pin.provider().as_str().to_owned();
|
||||
let account = pin.account().as_str().to_owned();
|
||||
let record = continuation.record(pin).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(record, timeout);
|
||||
select_biased! {
|
||||
result = record => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
%error,
|
||||
"Continuation affinity 写入失败,后续请求将退化为外部续接"
|
||||
);
|
||||
}
|
||||
},
|
||||
_ = timeout => {
|
||||
tracing::warn!(
|
||||
provider = %provider,
|
||||
account = %account,
|
||||
"Continuation affinity 后台写入超时,已丢弃本次亲和记录"
|
||||
);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -615,6 +615,7 @@ pub struct ModelRequestFinalization {
|
||||
pub upstream_transport: Option<String>,
|
||||
pub http_version: Option<String>,
|
||||
pub websocket_pool: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
/// Provider 已筛选的专有观测 JSON;Core 不解释字段。
|
||||
pub provider_metadata_json: Option<String>,
|
||||
pub error: Option<GatewayError>,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! 管理端账号连通性测试的真实执行链端口。
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::error::GatewayError;
|
||||
use crate::error::{ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind};
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
use crate::{engine::credential::ProviderAccountId, operation::Operation};
|
||||
|
||||
@@ -19,9 +22,111 @@ pub struct AccountProbeResult {
|
||||
pub text: Vec<String>,
|
||||
}
|
||||
|
||||
/// 仅供当前管理端连接测试展示的原始上游失败响应。
|
||||
///
|
||||
/// 正文不进入 `Debug`、日志或持久化;Core 只在探测终态从原始 Provider 错误显式复制。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct AccountProbeUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl AccountProbeUpstreamResponse {
|
||||
pub(crate) fn from_client_response(response: &ClientVisibleUpstreamResponse) -> Self {
|
||||
Self {
|
||||
status: response.status(),
|
||||
content_type: response.content_type().map(<[u8]>::to_vec),
|
||||
body: response.body().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AccountProbeUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("AccountProbeUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 管理端连接测试的终态错误。
|
||||
///
|
||||
/// `gateway` 保留稳定分类,`upstream_response` 只面向本次认证管理请求展示源响应。
|
||||
#[derive(Debug)]
|
||||
pub struct AccountProbeError {
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
}
|
||||
|
||||
impl AccountProbeError {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
gateway: GatewayError,
|
||||
upstream_response: Option<AccountProbeUpstreamResponse>,
|
||||
) -> Self {
|
||||
Self {
|
||||
gateway,
|
||||
upstream_response,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> GatewayErrorKind {
|
||||
self.gateway.kind()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_message(&self) -> &str {
|
||||
self.gateway.client_message()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_type(&self) -> Option<&str> {
|
||||
self.gateway.client_error_type()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_error_code(&self) -> Option<&str> {
|
||||
self.gateway.client_error_code()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_response(&self) -> Option<&AccountProbeUpstreamResponse> {
|
||||
self.upstream_response.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GatewayError> for AccountProbeError {
|
||||
fn from(gateway: GatewayError) -> Self {
|
||||
Self::new(gateway, None)
|
||||
}
|
||||
}
|
||||
|
||||
pub trait AccountProbe: Send + Sync {
|
||||
fn probe(
|
||||
&self,
|
||||
request: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, GatewayError>>;
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>>;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::engine::credential::{
|
||||
};
|
||||
use crate::engine::{AttemptContext, UpstreamSendState};
|
||||
use crate::error::{
|
||||
IdentifierError, ProviderError, ProviderErrorKind, SafeUpstreamValue, validate_text,
|
||||
IdentifierError, OpaqueUpstreamValue, ProviderError, ProviderErrorKind, validate_text,
|
||||
};
|
||||
use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::operation::Operation;
|
||||
@@ -125,7 +125,7 @@ pub struct ProviderCallMetadata {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
resource: ProviderResource,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<OpaqueUpstreamValue>,
|
||||
transport: UpstreamTransport,
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 设置 adapter 已分类为非 bearer 的 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
@@ -186,7 +186,7 @@ impl ProviderCallMetadata {
|
||||
|
||||
/// 返回安全上游 request ID。
|
||||
#[must_use]
|
||||
pub const fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::engine::UpstreamSendState;
|
||||
use crate::event::{ProviderEvent, ProviderResponseHeader};
|
||||
|
||||
/// 应用层标识不满足约束。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Error)]
|
||||
@@ -127,8 +129,12 @@ pub enum ProviderErrorKind {
|
||||
RateLimited,
|
||||
/// Credential 配额耗尽。
|
||||
QuotaExhausted,
|
||||
/// 仍有符合条件的账号,但它们暂时没有可调度容量。
|
||||
AccountCapacityUnavailable,
|
||||
/// Provider 已确认当前请求无法选出可用账号。
|
||||
NoEligibleAccount,
|
||||
/// Provider 的账号存储、租约协调或本地凭据数据不可用。
|
||||
ProviderInfrastructureUnavailable,
|
||||
/// 请求超时。
|
||||
Timeout,
|
||||
/// 网络或 transport 失败。
|
||||
@@ -154,7 +160,9 @@ impl ProviderErrorKind {
|
||||
Self::PermissionDenied => "permission_denied",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::QuotaExhausted => "quota_exhausted",
|
||||
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
|
||||
Self::NoEligibleAccount => "no_eligible_account",
|
||||
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
|
||||
Self::Timeout => "timeout",
|
||||
Self::Transport => "transport",
|
||||
Self::Protocol => "protocol",
|
||||
@@ -174,22 +182,18 @@ pub enum ContinuationFailure {
|
||||
HistoryUnavailable,
|
||||
}
|
||||
|
||||
/// Adapter 已明确分类为非 bearer、可用于诊断的上游值。
|
||||
/// Adapter 已明确分类为非 bearer 的不透明上游值。
|
||||
///
|
||||
/// Core 不解释或校验内容,只通过自定义 [`Debug`] 避免它意外进入日志。协议 adapter
|
||||
/// 可在原客户端响应中读取原值。
|
||||
#[derive(Clone, PartialEq, Eq, Hash)]
|
||||
pub struct SafeUpstreamValue(String);
|
||||
pub struct OpaqueUpstreamValue(String);
|
||||
|
||||
impl SafeUpstreamValue {
|
||||
const MAX_BYTES: usize = 256;
|
||||
|
||||
/// 校验并创建诊断值。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 空值、超长值或控制字符会返回 [`IdentifierError`]。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
validate_text(&value, Self::MAX_BYTES, false, None)?;
|
||||
Ok(Self(value))
|
||||
impl OpaqueUpstreamValue {
|
||||
/// 原样保存上游值。
|
||||
#[must_use]
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
|
||||
/// 返回已经由 adapter 安全分类的原值。
|
||||
@@ -199,9 +203,9 @@ impl SafeUpstreamValue {
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for SafeUpstreamValue {
|
||||
impl fmt::Debug for OpaqueUpstreamValue {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("SafeUpstreamValue(<redacted-from-Debug>)")
|
||||
formatter.write_str("OpaqueUpstreamValue(<redacted-from-Debug>)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,31 +216,23 @@ impl fmt::Debug for SafeUpstreamValue {
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamError {
|
||||
message: String,
|
||||
code: Option<SafeUpstreamValue>,
|
||||
error_type: Option<SafeUpstreamValue>,
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
error_type: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamError {
|
||||
const MAX_MESSAGE_BYTES: usize = 8_192;
|
||||
|
||||
/// 从结构化上游 `message`、`code` 和 `type` 创建客户端可见错误。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// message 为空、过长或含控制字符时返回错误;可选 code/type 不满足安全
|
||||
/// 约束时会被省略,不会把整段上游 body 重新暴露。
|
||||
/// 原样保存结构化上游 `message`、`code` 和 `type`。
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
message: impl Into<String>,
|
||||
code: Option<String>,
|
||||
error_type: Option<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let message = message.into();
|
||||
validate_text(&message, Self::MAX_MESSAGE_BYTES, false, None)?;
|
||||
Ok(Self {
|
||||
message,
|
||||
code: code.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
error_type: error_type.and_then(|value| SafeUpstreamValue::new(value).ok()),
|
||||
})
|
||||
) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
code: code.map(OpaqueUpstreamValue::new),
|
||||
error_type: error_type.map(OpaqueUpstreamValue::new),
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 message。
|
||||
@@ -248,13 +244,13 @@ impl ClientVisibleUpstreamError {
|
||||
/// 返回原上游的结构化 code。
|
||||
#[must_use]
|
||||
pub fn code(&self) -> Option<&str> {
|
||||
self.code.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.code.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
|
||||
/// 返回原上游的结构化 type。
|
||||
#[must_use]
|
||||
pub fn error_type(&self) -> Option<&str> {
|
||||
self.error_type.as_ref().map(SafeUpstreamValue::as_str)
|
||||
self.error_type.as_ref().map(OpaqueUpstreamValue::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,28 +260,110 @@ impl fmt::Debug for ClientVisibleUpstreamError {
|
||||
}
|
||||
}
|
||||
|
||||
/// 只供当前客户端请求使用的原始上游 HTTP 失败响应。
|
||||
///
|
||||
/// 该值不属于稳定诊断事实,不能进入日志或持久化。它刻意不实现 [`Clone`];
|
||||
/// [`ProviderError`] 的普通 clone 会丢弃它,只有最终失败的原对象才能把响应交给
|
||||
/// 原客户端协议,或由认证管理端的账号连接测试显式复制到请求局部结果。
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct ClientVisibleUpstreamResponse {
|
||||
status: u16,
|
||||
content_type: Option<Vec<u8>>,
|
||||
headers: Vec<ProviderResponseHeader>,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
impl ClientVisibleUpstreamResponse {
|
||||
/// 保存 transport 实际收到的状态码、Content-Type 原值和正文。
|
||||
#[must_use]
|
||||
pub fn new(status: u16, content_type: Option<Vec<u8>>, body: Bytes) -> Self {
|
||||
Self {
|
||||
status,
|
||||
content_type,
|
||||
headers: Vec::new(),
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
/// 附加 Provider 已剔除账号绑定与 framing 信息的普通响应头。
|
||||
#[must_use]
|
||||
pub fn with_headers(mut self, headers: Vec<ProviderResponseHeader>) -> Self {
|
||||
self.headers = headers;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn status(&self) -> u16 {
|
||||
self.status
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn content_type(&self) -> Option<&[u8]> {
|
||||
self.content_type.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.headers
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClientVisibleUpstreamResponse {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ClientVisibleUpstreamResponse")
|
||||
.field("status", &self.status)
|
||||
.field(
|
||||
"content_type",
|
||||
&self.content_type.as_ref().map(|_| "<present>"),
|
||||
)
|
||||
.field("header_count", &self.headers.len())
|
||||
.field("body", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 单次 Provider 调用的稳定错误。
|
||||
///
|
||||
/// 该类型不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// 稳定诊断字段不接收原始响应正文,也不会在 `Debug` 或 `Display` 中打印上游
|
||||
/// code/request ID/response ID。Adapter 若捕获到可能含 secret 的上下文,只能调用
|
||||
/// [`ProviderError::redact_sensitive_context`] 丢弃正文并留下脱敏标记。
|
||||
#[derive(Clone)]
|
||||
///
|
||||
/// 唯一例外是 [`ProviderError::with_atomic_client_events`]:它只承载尚未交付客户端、
|
||||
/// 必须与本错误一起完成重试判断的协议事件。Core 会在任何持久化或日志记录前取走
|
||||
/// 这些事件,不能把它们当作诊断上下文使用。
|
||||
pub struct ProviderError {
|
||||
kind: ProviderErrorKind,
|
||||
send_state: UpstreamSendState,
|
||||
upstream_status: Option<u16>,
|
||||
upstream_code: Option<SafeUpstreamValue>,
|
||||
upstream_request_id: Option<SafeUpstreamValue>,
|
||||
upstream_response_id: Option<SafeUpstreamValue>,
|
||||
upstream_values: Option<Box<ProviderErrorUpstreamValues>>,
|
||||
retry_after: Option<Duration>,
|
||||
continuation_failure: Option<ContinuationFailure>,
|
||||
replay_safe: bool,
|
||||
pre_delivery_retry: bool,
|
||||
credential_recovery_required: bool,
|
||||
retry_same_account: bool,
|
||||
sensitive_context_redacted: bool,
|
||||
client_visible_upstream_error: Option<Box<ClientVisibleUpstreamError>>,
|
||||
client_visible_upstream_response: Option<Box<ClientVisibleUpstreamResponse>>,
|
||||
atomic_client_events: Option<Box<AtomicClientEvents>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ProviderErrorUpstreamValues {
|
||||
code: Option<OpaqueUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
response_id: Option<OpaqueUpstreamValue>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AtomicClientEvents(Vec<ProviderEvent>);
|
||||
|
||||
impl ProviderError {
|
||||
/// 创建 Provider 错误。
|
||||
#[must_use]
|
||||
@@ -294,16 +372,17 @@ impl ProviderError {
|
||||
kind,
|
||||
send_state,
|
||||
upstream_status: None,
|
||||
upstream_code: None,
|
||||
upstream_request_id: None,
|
||||
upstream_response_id: None,
|
||||
upstream_values: None,
|
||||
retry_after: None,
|
||||
continuation_failure: None,
|
||||
replay_safe: false,
|
||||
pre_delivery_retry: false,
|
||||
credential_recovery_required: false,
|
||||
retry_same_account: false,
|
||||
sensitive_context_redacted: false,
|
||||
client_visible_upstream_error: None,
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,25 +397,30 @@ impl ProviderError {
|
||||
|
||||
/// 附加 adapter 已分类为安全的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn with_upstream_code(mut self, code: SafeUpstreamValue) -> Self {
|
||||
self.upstream_code = Some(code);
|
||||
pub fn with_upstream_code(mut self, code: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().code = Some(code);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_request_id = Some(request_id);
|
||||
pub fn with_upstream_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 adapter 已分类为非 bearer 的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn with_upstream_response_id(mut self, response_id: SafeUpstreamValue) -> Self {
|
||||
self.upstream_response_id = Some(response_id);
|
||||
pub fn with_upstream_response_id(mut self, response_id: OpaqueUpstreamValue) -> Self {
|
||||
self.upstream_values_mut().response_id = Some(response_id);
|
||||
self
|
||||
}
|
||||
|
||||
fn upstream_values_mut(&mut self) -> &mut ProviderErrorUpstreamValues {
|
||||
self.upstream_values
|
||||
.get_or_insert_with(|| Box::new(ProviderErrorUpstreamValues::default()))
|
||||
}
|
||||
|
||||
/// 附加 Provider 建议的冷却时间。
|
||||
#[must_use]
|
||||
pub const fn with_retry_after(mut self, retry_after: Duration) -> Self {
|
||||
@@ -358,6 +442,17 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 允许 Core 仅在客户端尚未收到任何事件时执行一次受预算约束的换号恢复。
|
||||
///
|
||||
/// 该标记不证明上游未执行,也不等价于 [`Self::with_replay_safe`];它只表达
|
||||
/// Provider 选择了“客户端无感恢复优先”的传输策略。Core 在 continuation、
|
||||
/// 指定账号或下游已经进入提交状态时必须忽略它。
|
||||
#[must_use]
|
||||
pub const fn with_pre_delivery_retry(mut self) -> Self {
|
||||
self.pre_delivery_retry = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// 要求 Core 在账号凭据已恢复后仅对原账号重放一次。
|
||||
#[must_use]
|
||||
pub const fn with_same_account_retry(mut self) -> Self {
|
||||
@@ -379,6 +474,27 @@ impl ProviderError {
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加仅可返回给当前请求方的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn with_client_visible_upstream_response(
|
||||
mut self,
|
||||
response: ClientVisibleUpstreamResponse,
|
||||
) -> Self {
|
||||
self.client_visible_upstream_response = Some(Box::new(response));
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加必须先由 Core 判断重试、再决定丢弃或交付的客户端协议事件。
|
||||
///
|
||||
/// Provider 只能放入当前 attempt 尚未交付的事件;调用方应设置明确的有界缓冲,
|
||||
/// 并保证失败事件本身位于这批事件的末尾。
|
||||
#[must_use]
|
||||
pub fn with_atomic_client_events(mut self, events: Vec<ProviderEvent>) -> Self {
|
||||
self.atomic_client_events =
|
||||
(!events.is_empty()).then(|| Box::new(AtomicClientEvents(events)));
|
||||
self
|
||||
}
|
||||
|
||||
/// 丢弃敏感正文,只记录“发生过脱敏”这一事实。
|
||||
#[must_use]
|
||||
pub fn redact_sensitive_context(mut self, _sensitive_context: impl AsRef<str>) -> Self {
|
||||
@@ -406,20 +522,26 @@ impl ProviderError {
|
||||
|
||||
/// 返回安全分类的上游错误 code。
|
||||
#[must_use]
|
||||
pub fn upstream_code(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_code.as_ref()
|
||||
pub fn upstream_code(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.code.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 request ID。
|
||||
#[must_use]
|
||||
pub fn upstream_request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_request_id.as_ref()
|
||||
pub fn upstream_request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.request_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回安全分类的上游 response ID。
|
||||
#[must_use]
|
||||
pub fn upstream_response_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
self.upstream_response_id.as_ref()
|
||||
pub fn upstream_response_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.upstream_values
|
||||
.as_deref()
|
||||
.and_then(|values| values.response_id.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Provider 建议的冷却时间。
|
||||
@@ -440,6 +562,12 @@ impl ProviderError {
|
||||
self.replay_safe
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否允许在首个客户端事件前换号恢复传输失败。
|
||||
#[must_use]
|
||||
pub const fn allows_pre_delivery_retry(&self) -> bool {
|
||||
self.pre_delivery_retry
|
||||
}
|
||||
|
||||
/// 返回 Provider 是否已完成凭据恢复并要求原账号重放。
|
||||
#[must_use]
|
||||
pub const fn retries_same_account(&self) -> bool {
|
||||
@@ -462,6 +590,50 @@ impl ProviderError {
|
||||
pub fn client_visible_upstream_error(&self) -> Option<&ClientVisibleUpstreamError> {
|
||||
self.client_visible_upstream_error.as_deref()
|
||||
}
|
||||
|
||||
/// 返回只供当前协议响应使用的原始上游 HTTP 失败响应。
|
||||
#[must_use]
|
||||
pub fn client_visible_upstream_response(&self) -> Option<&ClientVisibleUpstreamResponse> {
|
||||
self.client_visible_upstream_response.as_deref()
|
||||
}
|
||||
|
||||
/// 取走只供本次重试/提交决策使用的原子客户端事件。
|
||||
///
|
||||
/// Core 必须在克隆错误、记录中间失败或构造终态前调用本方法,避免原始 wire
|
||||
/// 进入诊断或持久化对象。
|
||||
#[must_use]
|
||||
pub fn take_atomic_client_events(&mut self) -> Vec<ProviderEvent> {
|
||||
self.atomic_client_events
|
||||
.take()
|
||||
.map_or_else(Vec::new, |events| events.0)
|
||||
}
|
||||
|
||||
/// 返回是否携带尚未提交的原子客户端事件;不暴露事件内容。
|
||||
#[must_use]
|
||||
pub fn has_atomic_client_events(&self) -> bool {
|
||||
self.atomic_client_events.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for ProviderError {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
kind: self.kind,
|
||||
send_state: self.send_state,
|
||||
upstream_status: self.upstream_status,
|
||||
upstream_values: self.upstream_values.clone(),
|
||||
retry_after: self.retry_after,
|
||||
continuation_failure: self.continuation_failure,
|
||||
replay_safe: self.replay_safe,
|
||||
pre_delivery_retry: self.pre_delivery_retry,
|
||||
credential_recovery_required: self.credential_recovery_required,
|
||||
retry_same_account: self.retry_same_account,
|
||||
sensitive_context_redacted: self.sensitive_context_redacted,
|
||||
client_visible_upstream_error: self.client_visible_upstream_error.clone(),
|
||||
client_visible_upstream_response: None,
|
||||
atomic_client_events: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderError {
|
||||
@@ -473,25 +645,20 @@ impl fmt::Debug for ProviderError {
|
||||
.field("upstream_status", &self.upstream_status)
|
||||
.field(
|
||||
"upstream_code",
|
||||
&self.upstream_code.as_ref().map(|_| "<classified-safe>"),
|
||||
&self.upstream_code().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_request_id",
|
||||
&self
|
||||
.upstream_request_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_request_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field(
|
||||
"upstream_response_id",
|
||||
&self
|
||||
.upstream_response_id
|
||||
.as_ref()
|
||||
.map(|_| "<classified-safe>"),
|
||||
&self.upstream_response_id().map(|_| "<classified-safe>"),
|
||||
)
|
||||
.field("retry_after", &self.retry_after)
|
||||
.field("continuation_failure", &self.continuation_failure)
|
||||
.field("replay_safe", &self.replay_safe)
|
||||
.field("pre_delivery_retry", &self.pre_delivery_retry)
|
||||
.field(
|
||||
"credential_recovery_required",
|
||||
&self.credential_recovery_required,
|
||||
@@ -508,6 +675,20 @@ impl fmt::Debug for ProviderError {
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"client_visible_upstream_response",
|
||||
&self
|
||||
.client_visible_upstream_response
|
||||
.as_ref()
|
||||
.map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"atomic_client_events",
|
||||
&self
|
||||
.atomic_client_events
|
||||
.as_ref()
|
||||
.map_or(0, |events| events.0.len()),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -536,6 +717,10 @@ pub enum GatewayErrorKind {
|
||||
ModelNotFound,
|
||||
/// 当前没有可用 target。
|
||||
NoAvailableProvider,
|
||||
/// 符合条件的上游账号暂时没有调度容量。
|
||||
AccountCapacityUnavailable,
|
||||
/// Provider 的本地账号基础设施不可用。
|
||||
ProviderInfrastructureUnavailable,
|
||||
/// 上游限流。
|
||||
RateLimited,
|
||||
/// 上游暂不可用。
|
||||
@@ -559,6 +744,8 @@ impl GatewayErrorKind {
|
||||
Self::PolicyDenied => "policy_denied",
|
||||
Self::ModelNotFound => "model_not_found",
|
||||
Self::NoAvailableProvider => "no_available_provider",
|
||||
Self::AccountCapacityUnavailable => "account_capacity_unavailable",
|
||||
Self::ProviderInfrastructureUnavailable => "provider_infrastructure_unavailable",
|
||||
Self::RateLimited => "rate_limited",
|
||||
Self::UpstreamUnavailable => "upstream_unavailable",
|
||||
Self::Timeout => "timeout",
|
||||
@@ -607,10 +794,18 @@ impl GatewayError {
|
||||
GatewayErrorKind::RateLimited,
|
||||
"upstream capacity is temporarily unavailable",
|
||||
),
|
||||
ProviderErrorKind::AccountCapacityUnavailable => Self::new(
|
||||
GatewayErrorKind::AccountCapacityUnavailable,
|
||||
"all eligible upstream accounts are temporarily busy",
|
||||
),
|
||||
ProviderErrorKind::NoEligibleAccount => Self::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream provider is currently available for this request",
|
||||
),
|
||||
ProviderErrorKind::ProviderInfrastructureUnavailable => Self::new(
|
||||
GatewayErrorKind::ProviderInfrastructureUnavailable,
|
||||
"provider account infrastructure is temporarily unavailable",
|
||||
),
|
||||
ProviderErrorKind::Timeout => {
|
||||
Self::new(GatewayErrorKind::Timeout, "upstream request timed out")
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use thiserror::Error;
|
||||
|
||||
use crate::accounting::{CalculatedCost, ProviderReportedCost, Usage};
|
||||
use crate::engine::provider::UpstreamTransport;
|
||||
use crate::error::{IdentifierError, SafeUpstreamValue, validate_text};
|
||||
use crate::error::{IdentifierError, OpaqueUpstreamValue, validate_text};
|
||||
use crate::operation::ProviderSessionState;
|
||||
|
||||
/// 一次响应的稳定元数据。
|
||||
@@ -199,7 +199,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名或显式事件名为空、过长或含控制字符时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。上游事件名保持不透明。
|
||||
pub fn json(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -212,7 +212,7 @@ impl ProtocolWireEvent {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名、显式事件名或 SSE ID 不满足 wire 安全约束时返回错误。
|
||||
/// 协议名不满足内部路由标识约束时返回错误。SSE 元数据保持不透明。
|
||||
pub fn json_with_sse_metadata(
|
||||
protocol: impl Into<String>,
|
||||
event_type: Option<String>,
|
||||
@@ -222,15 +222,6 @@ impl ProtocolWireEvent {
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let protocol = protocol.into();
|
||||
validate_text(&protocol, 64, true, None)?;
|
||||
if let Some(event_type) = event_type.as_deref() {
|
||||
validate_text(event_type, 256, true, None)?;
|
||||
}
|
||||
if sse_id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id.contains(['\0', '\r', '\n']))
|
||||
{
|
||||
return Err(IdentifierError::ControlCharacter);
|
||||
}
|
||||
Ok(Self {
|
||||
protocol,
|
||||
event_type,
|
||||
@@ -440,24 +431,21 @@ impl fmt::Debug for ProviderResponseMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider 已筛选、可由协议 adapter 再次按白名单表达的响应头。
|
||||
/// Provider 已筛选、可由协议 adapter 尝试表达的不透明响应头。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResponseHeader {
|
||||
name: String,
|
||||
value: SafeUpstreamValue,
|
||||
value: Bytes,
|
||||
}
|
||||
|
||||
impl ProviderResponseHeader {
|
||||
/// 创建规范化的小写响应头;非 ASCII token 名称直接拒绝。
|
||||
/// 保存 Provider 交付的原始名称和值;HTTP 可表达性由最终 adapter 判断。
|
||||
#[must_use]
|
||||
pub fn new(name: impl Into<String>, value: SafeUpstreamValue) -> Option<Self> {
|
||||
let name = name.into().trim().to_ascii_lowercase();
|
||||
let valid = !name.is_empty()
|
||||
&& name.len() <= 64
|
||||
&& name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-');
|
||||
valid.then_some(Self { name, value })
|
||||
pub fn new(name: impl Into<String>, value: Bytes) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
value,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -466,7 +454,7 @@ impl ProviderResponseHeader {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn value(&self) -> &SafeUpstreamValue {
|
||||
pub const fn value(&self) -> &Bytes {
|
||||
&self.value
|
||||
}
|
||||
}
|
||||
@@ -475,7 +463,7 @@ impl fmt::Debug for ProviderResponseHeader {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderResponseHeader")
|
||||
.field("name", &self.name)
|
||||
.field("name", &"<redacted>")
|
||||
.field("value", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
@@ -488,7 +476,8 @@ pub struct ProviderResponseObservation {
|
||||
http_version: Option<UpstreamHttpVersion>,
|
||||
websocket_pool: Option<WebSocketPoolKind>,
|
||||
status_code: Option<u16>,
|
||||
request_id: Option<SafeUpstreamValue>,
|
||||
request_id: Option<OpaqueUpstreamValue>,
|
||||
service_tier: Option<String>,
|
||||
timings: ProviderResponseTimings,
|
||||
client_headers: Vec<ProviderResponseHeader>,
|
||||
provider_metadata: Option<ProviderResponseMetadata>,
|
||||
@@ -503,6 +492,7 @@ impl ProviderResponseObservation {
|
||||
websocket_pool: None,
|
||||
status_code: None,
|
||||
request_id: None,
|
||||
service_tier: None,
|
||||
timings: ProviderResponseTimings::default(),
|
||||
client_headers: Vec::new(),
|
||||
provider_metadata: None,
|
||||
@@ -528,11 +518,36 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_request_id(mut self, request_id: SafeUpstreamValue) -> Self {
|
||||
pub fn with_request_id(mut self, request_id: OpaqueUpstreamValue) -> Self {
|
||||
self.request_id = Some(request_id);
|
||||
self
|
||||
}
|
||||
|
||||
/// 附加 Provider 在响应中确认的服务档位。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 档位为空、超过 64 字节或包含控制字符时返回错误。
|
||||
pub fn try_with_service_tier(
|
||||
mut self,
|
||||
service_tier: impl Into<String>,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let service_tier = service_tier.into();
|
||||
validate_text(&service_tier, 64, false, None)?;
|
||||
self.service_tier = Some(service_tier);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// 附加合法服务档位;不适合持久化的上游观测值会被忽略。
|
||||
#[must_use]
|
||||
pub fn with_service_tier_if_valid(mut self, service_tier: impl Into<String>) -> Self {
|
||||
let service_tier = service_tier.into();
|
||||
if validate_text(&service_tier, 64, false, None).is_ok() {
|
||||
self.service_tier = Some(service_tier);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn with_timings(mut self, timings: ProviderResponseTimings) -> Self {
|
||||
self.timings = timings;
|
||||
@@ -573,10 +588,15 @@ impl ProviderResponseObservation {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> Option<&SafeUpstreamValue> {
|
||||
pub const fn request_id(&self) -> Option<&OpaqueUpstreamValue> {
|
||||
self.request_id.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn service_tier(&self) -> Option<&str> {
|
||||
self.service_tier.as_deref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn timings(&self) -> ProviderResponseTimings {
|
||||
self.timings
|
||||
|
||||
@@ -302,7 +302,7 @@ impl GenerateRequest {
|
||||
self.body()
|
||||
.get("previous_response_id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|response_id| !response_id.is_empty())
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// 返回客户端是否请求了图片生成工具。
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::engine::credential::{
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use crate::error::{IdentifierError, validate_text};
|
||||
use crate::routing::ProviderKind;
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
@@ -527,6 +527,80 @@ impl ProviderCooldown {
|
||||
}
|
||||
}
|
||||
|
||||
/// 可丢失 cooldown 的细粒度作用域。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum ProviderCooldownScope {
|
||||
/// 只阻止同一账号调用指定上游模型。
|
||||
UpstreamModel(UpstreamModelId),
|
||||
}
|
||||
|
||||
impl ProviderCooldownScope {
|
||||
#[must_use]
|
||||
pub const fn upstream_model(model: UpstreamModelId) -> Self {
|
||||
Self::UpstreamModel(model)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::UpstreamModel(_) => "model",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn value(&self) -> &str {
|
||||
match self {
|
||||
Self::UpstreamModel(model) => model.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 不进入账号持久状态的账号+作用域 cooldown。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderScopedCooldown {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
}
|
||||
|
||||
impl ProviderScopedCooldown {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
scope: ProviderCooldownScope,
|
||||
until: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
scope,
|
||||
until,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_id(&self) -> &ProviderAccountId {
|
||||
&self.account_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn credential_revision(&self) -> CredentialRevision {
|
||||
self.credential_revision
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn scope(&self) -> &ProviderCooldownScope {
|
||||
&self.scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn until(&self) -> SystemTime {
|
||||
self.until
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderCooldownPort: Send + Sync {
|
||||
fn put_if_later(
|
||||
&self,
|
||||
@@ -543,6 +617,31 @@ pub trait ProviderCooldownPort: Send + Sync {
|
||||
account_id: &'a ProviderAccountId,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn put_scoped_if_later(
|
||||
&self,
|
||||
cooldown: ProviderScopedCooldown,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>;
|
||||
|
||||
fn clear_scoped<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
scope: &'a ProviderCooldownScope,
|
||||
through_revision: CredentialRevision,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
|
||||
/// 删除账号时清除该账号全部 account/model scope cooldown key。
|
||||
/// 生命周期与 credential revision 无关;不要求调用方持有 revision。
|
||||
fn clear_all<'a>(
|
||||
&'a self,
|
||||
account_id: &'a ProviderAccountId,
|
||||
) -> BoxFuture<'a, Result<bool, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -575,33 +674,20 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按账号稳定扰动刷新提前量;相同账号与过期时间跨进程得到同一执行时刻。
|
||||
pub fn next_attempt_at(
|
||||
/// 判断 AT 是否已经进入当前配置的提前刷新窗口。
|
||||
///
|
||||
/// `margin` 是运行时策略,不投影为账号的持久化时间字段;已过期 AT 也需要
|
||||
/// 尝试用 RT 恢复,因此视为到期。
|
||||
#[must_use]
|
||||
pub fn is_refresh_due(
|
||||
self,
|
||||
account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
let base_margin_seconds = self.margin.as_secs();
|
||||
if base_margin_seconds == 0 {
|
||||
return Ok(observed_at);
|
||||
) -> bool {
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= self.margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
|
||||
let factor = stable_factor(account_id.as_str(), "normal", 850, 1_150);
|
||||
let jittered_seconds = base_margin_seconds
|
||||
.saturating_mul(u64::from(factor))
|
||||
.saturating_add(500)
|
||||
/ 1_000;
|
||||
if jittered_seconds >= remaining.as_secs() {
|
||||
return Ok(observed_at);
|
||||
}
|
||||
let lead = jittered_seconds.max(1);
|
||||
access_token_expires_at
|
||||
.checked_sub(Duration::from_secs(lead))
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -89,6 +89,18 @@ impl UpstreamModelId {
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
/// 从未命中显式映射的客户端模型名构造同名上游模型。
|
||||
///
|
||||
/// 目录和配置仍使用 [`Self::new`] 的内部标识约束;客户端 wire 只要求非空,
|
||||
/// 具体模型字符串是否可用由绑定 Provider 决定。
|
||||
pub fn from_client_wire(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
if value.is_empty() {
|
||||
return Err(IdentifierError::Empty);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
|
||||
@@ -244,13 +244,8 @@ async fn compile_runtime_snapshot(
|
||||
}
|
||||
|
||||
let model_mappings = facts.settings.model_mappings;
|
||||
let rotation_strategy = match facts.settings.rotation_strategy.as_str() {
|
||||
"smart" => RotationStrategy::Smart,
|
||||
"quota_reset_priority" => RotationStrategy::QuotaResetPriority,
|
||||
"round_robin" => RotationStrategy::RoundRobin,
|
||||
"sticky" => RotationStrategy::Sticky,
|
||||
_ => return Err(RuntimeSnapshotCompileError::InvalidData),
|
||||
};
|
||||
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
let selection_policy = AccountSelectionPolicy::new(
|
||||
rotation_strategy,
|
||||
NonZeroU32::new(facts.settings.max_concurrent_per_account)
|
||||
@@ -488,15 +483,11 @@ impl RuntimeSnapshot {
|
||||
|
||||
#[must_use]
|
||||
pub fn mapped_model(&self, requested: &str) -> String {
|
||||
let original = requested.trim();
|
||||
let original = requested;
|
||||
let mut current = original.to_owned();
|
||||
let mut seen = BTreeSet::new();
|
||||
for _ in 0..20 {
|
||||
let Some(target) = self
|
||||
.model_mappings
|
||||
.get(¤t)
|
||||
.map(|target| target.trim())
|
||||
else {
|
||||
let Some(target) = self.model_mappings.get(¤t).map(String::as_str) else {
|
||||
return current;
|
||||
};
|
||||
if !seen.insert(current.clone()) || seen.contains(target) {
|
||||
@@ -529,8 +520,14 @@ impl RuntimeSnapshot {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let upstream_model = UpstreamModelId::new(self.mapped_model(public_model.as_str()))
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let requested_model = public_model.as_str();
|
||||
let mapped_model = self.mapped_model(requested_model);
|
||||
let upstream_model = if self.model_mappings.contains_key(requested_model) {
|
||||
UpstreamModelId::new(mapped_model)
|
||||
} else {
|
||||
UpstreamModelId::from_client_wire(mapped_model)
|
||||
}
|
||||
.map_err(|_| RoutingError::InvalidIdentifier)?;
|
||||
let emulated_features = match self
|
||||
.provider_models
|
||||
.get(provider)
|
||||
|
||||
@@ -21,10 +21,11 @@ pub enum WorkerKind {
|
||||
StaleModelRequestRecovery,
|
||||
Retention,
|
||||
OpsFlush,
|
||||
Backup,
|
||||
}
|
||||
|
||||
impl WorkerKind {
|
||||
pub const ALL: [Self; 7] = [
|
||||
pub const ALL: [Self; 8] = [
|
||||
Self::OAuthRefresh,
|
||||
Self::QuotaCatalogHealth,
|
||||
Self::RuntimeSnapshotReconciliation,
|
||||
@@ -32,6 +33,7 @@ impl WorkerKind {
|
||||
Self::StaleModelRequestRecovery,
|
||||
Self::Retention,
|
||||
Self::OpsFlush,
|
||||
Self::Backup,
|
||||
];
|
||||
|
||||
#[must_use]
|
||||
@@ -44,6 +46,7 @@ impl WorkerKind {
|
||||
Self::StaleModelRequestRecovery => "stale_model_request_recovery",
|
||||
Self::Retention => "retention",
|
||||
Self::OpsFlush => "ops_flush",
|
||||
Self::Backup => "backup",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,22 @@ fn decimal_should_render_with_database_scale() {
|
||||
assert_eq!(decimal.to_string(), "12.3400000000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_canonical_should_strip_trailing_zeroes() {
|
||||
assert_eq!(Decimal::from_str("12.34").unwrap().canonical(), "12.34");
|
||||
assert_eq!(Decimal::from_str("12").unwrap().canonical(), "12");
|
||||
assert_eq!(Decimal::from_str("0").unwrap().canonical(), "0");
|
||||
assert_eq!(Decimal::from_str("1.0000000000").unwrap().canonical(), "1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decimal_div_u64_should_keep_up_to_ten_fraction_digits() {
|
||||
let decimal = Decimal::from_str("10").unwrap();
|
||||
let third = decimal.checked_div_u64(3).unwrap();
|
||||
assert_eq!(third.canonical(), "3.3333333333");
|
||||
assert_eq!(decimal.checked_div_u64(0), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_usd_ticks_should_map_exactly_to_decimal_scale() {
|
||||
let estimate = ProviderReportedCost::from_usd_ticks(12_345_678_901)
|
||||
|
||||
@@ -2,13 +2,12 @@ use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::error::SafeUpstreamValue;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
fn pin() -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new("response-private").expect("valid response"),
|
||||
SafeUpstreamValue::new("upstream-private").expect("valid upstream response"),
|
||||
PreviousResponseId::new("response-private"),
|
||||
PreviousResponseId::new("upstream-private"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
ProviderAccountId::new("acct_codex").expect("valid account"),
|
||||
)
|
||||
@@ -23,13 +22,20 @@ fn native_pin_debug_should_redact_previous_response_id() {
|
||||
|
||||
#[test]
|
||||
fn external_binding_debug_should_redact_previous_response_id() {
|
||||
let binding = ContinuationBinding::External(
|
||||
PreviousResponseId::new("external-private").expect("valid response"),
|
||||
);
|
||||
let binding = ContinuationBinding::External(PreviousResponseId::new("external-private"));
|
||||
|
||||
assert!(!format!("{binding:?}").contains("external-private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_response_id_preserves_any_codex_string_without_debug_disclosure() {
|
||||
let value = format!("resp_{}\0opaque", "x".repeat(257));
|
||||
let response_id = PreviousResponseId::new(value.clone());
|
||||
|
||||
assert_eq!(response_id.as_str(), value);
|
||||
assert!(!format!("{response_id:?}").contains(&value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_pin_should_reject_different_account() {
|
||||
assert!(!pin().matches(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user