mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
941c4d57fa | ||
|
|
cf046ad9a4 | ||
|
|
7699a954ac | ||
|
|
2c31708d1b | ||
|
|
80f338ce01 | ||
|
|
299088a6e7 | ||
|
|
a23bfca848 | ||
|
|
8951a64a05 | ||
|
|
eb20561893 | ||
|
|
a33637a99e | ||
|
|
97cd7a7097 | ||
|
|
b0fe132736 | ||
|
|
e2c75d1054 | ||
|
|
94f298927d | ||
|
|
3be827204f | ||
|
|
227a1174b9 | ||
|
|
c0defcb6df | ||
|
|
53739ebe5c | ||
|
|
7f420257a6 | ||
|
|
4f85209dc9 | ||
|
|
cd89e2e8fc | ||
|
|
0cb5a29699 | ||
|
|
61788804b4 | ||
|
|
2751ea6aa4 | ||
|
|
58fe66789c | ||
|
|
573b6b26db | ||
|
|
d425428ebe | ||
|
|
b26d3b386a | ||
|
|
cf0b14c5fd | ||
|
|
056561de41 | ||
|
|
f2215df72e | ||
|
|
72f8957163 | ||
|
|
73f0c15790 | ||
|
|
0558693949 | ||
|
|
2a9cb02cc9 | ||
|
|
5ce36931a8 | ||
|
|
0d826d9eba | ||
|
|
304242957a | ||
|
|
ef37ae8d9e | ||
|
|
fc13d4f2b6 | ||
|
|
78f9c0c800 | ||
|
|
a3bc7bc445 | ||
|
|
0132114285 | ||
|
|
5970efbb7c | ||
|
|
7bf28b571d | ||
|
|
bc3bd6bc43 | ||
|
|
9acfab0a86 | ||
|
|
3e5a84d2a0 | ||
|
|
92948f51a9 | ||
|
|
776637fccb | ||
|
|
82acb08d26 | ||
|
|
7744e1b0b4 | ||
|
|
0b8a1add18 | ||
|
|
2dd60c08b6 | ||
|
|
cce9880937 | ||
|
|
b8ea1cd076 | ||
|
|
4117840f75 | ||
|
|
f642764f34 | ||
|
|
482e192e2d | ||
|
|
ee51a07de4 | ||
|
|
bf23658aaf | ||
|
|
bc4728332e | ||
|
|
2cb5cda077 | ||
|
|
9d90dedb81 | ||
|
|
6074cff68c | ||
|
|
6ab8e7d0a6 | ||
|
|
17f9ef8915 | ||
|
|
15e0a28bd0 | ||
|
|
58bc438992 | ||
|
|
b04616f59f | ||
|
|
82ba4fcbba | ||
|
|
819f428ac3 | ||
|
|
35f28e8aab | ||
|
|
65645c7403 | ||
|
|
71159d5633 | ||
|
|
f2ebb09e34 | ||
|
|
07acc67ce6 | ||
|
|
01ad8bba8a | ||
|
|
986a128d15 | ||
|
|
6cad2bb6a1 | ||
|
|
2b8b3de0e4 | ||
|
|
4127886584 | ||
|
|
ee3e5db6c2 | ||
|
|
153cb67b9c | ||
|
|
f370d355b6 | ||
|
|
a31dad992c | ||
|
|
55a88f5f6b | ||
|
|
5d90a92582 | ||
|
|
93abe8efcd | ||
|
|
f4b75c42ed | ||
|
|
36ff038c78 | ||
|
|
545009590c | ||
|
|
5b7b9b21ee | ||
|
|
44a2a87b69 | ||
|
|
184e88083b | ||
|
|
53b40aea46 | ||
|
|
aa328b6dbf | ||
|
|
64231ff403 | ||
|
|
d7b80a2066 | ||
|
|
08d63ea3f3 | ||
|
|
ac9c4c85ad | ||
|
|
492c401599 | ||
|
|
a063258a40 | ||
|
|
dc590f2b5f | ||
|
|
ca52e2bfd9 | ||
|
|
43604e36dc | ||
|
|
a4a997e351 | ||
|
|
73ca463924 | ||
|
|
9eeec2c78c | ||
|
|
90890be2b4 | ||
|
|
2e4bab17a8 | ||
|
|
b052e79a16 | ||
|
|
9775451e83 | ||
|
|
c22a2ad144 | ||
|
|
7bd362d544 | ||
|
|
e7036a616f | ||
|
|
98652d4c68 | ||
|
|
d0dc7ef92b | ||
|
|
194fb20d5f | ||
|
|
30f528f084 | ||
|
|
28d024ceda | ||
|
|
6b9c98c59f | ||
|
|
1294e50b76 | ||
|
|
a0e540642b | ||
|
|
14d6a7de31 | ||
|
|
cd7b60a3ba | ||
|
|
ebbed22f36 | ||
|
|
f6e5d8f673 | ||
|
|
bf1c6211b8 | ||
|
|
d388fa344a | ||
|
|
eabb61b08c | ||
|
|
a5c3438927 | ||
|
|
b6948542bb | ||
|
|
0242bd3193 | ||
|
|
229ef48470 | ||
|
|
19c35ba901 | ||
|
|
3c06ae97e8 | ||
|
|
f34966e4c7 | ||
|
|
e9ebf45640 | ||
|
|
c00e5bd243 | ||
|
|
ab5f18a437 | ||
|
|
330b222359 | ||
|
|
d81cb2d4f0 | ||
|
|
8e2edb680d | ||
|
|
00879be47d | ||
|
|
34656a3f07 | ||
|
|
27cf037309 | ||
|
|
2198fa69ab |
@@ -21,6 +21,11 @@ updates:
|
||||
day: monday
|
||||
time: '09:10'
|
||||
timezone: Asia/Shanghai
|
||||
# 固定官方 Codex Desktop 的 TLS 指纹;升级前必须抓取握手并比对扩展顺序。
|
||||
ignore:
|
||||
- dependency-name: hyper-rustls
|
||||
- dependency-name: reqwest
|
||||
- dependency-name: rustls
|
||||
groups:
|
||||
minor-and-patch:
|
||||
update-types:
|
||||
|
||||
@@ -55,6 +55,9 @@ jobs:
|
||||
target: runtime
|
||||
load: true
|
||||
tags: ${{ env.CPR_IMAGE }}
|
||||
no-cache-filters: |
|
||||
runtime-base
|
||||
runtime
|
||||
cache-from: type=gha,scope=runtime-ci
|
||||
cache-to: type=gha,mode=max,scope=runtime-ci
|
||||
|
||||
@@ -76,7 +79,7 @@ jobs:
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
components: rustfmt,clippy
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
|
||||
- name: Detect affected checks
|
||||
id: filter
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
|
||||
@@ -375,7 +375,7 @@ jobs:
|
||||
PY
|
||||
|
||||
- name: Generate release asset provenance
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: dist/release/*
|
||||
|
||||
@@ -468,6 +468,9 @@ jobs:
|
||||
push: true
|
||||
build-args: |
|
||||
CPR_PREBUILT_DIR=dist/docker
|
||||
no-cache-filters: |
|
||||
runtime-base
|
||||
runtime-prebuilt
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }}
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.os }}-${{ matrix.arch }},mode=min
|
||||
provenance: mode=max
|
||||
@@ -581,7 +584,7 @@ jobs:
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate container provenance attestation
|
||||
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
subject-digest: ${{ steps.manifest.outputs.digest }}
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@1a3a6d54512beeaffd394f8d516ca16f2c506a20 # 1.97
|
||||
|
||||
- name: Cache Rust
|
||||
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: backend -> target
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
with:
|
||||
version: 11.7.0
|
||||
|
||||
|
||||
@@ -12,31 +12,35 @@
|
||||
[](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
|
||||
[](https://opensource.org/license/mit)
|
||||
|
||||
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [运维](#运维) · [部署](deploy/README.md) · [接口](docs/api.md) · [架构](docs/architecture.md)
|
||||
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [运行维护](#运行维护) · [部署](deploy/README.md) · [接口](docs/api.md) · [架构](docs/architecture.md)
|
||||
|
||||
</div>
|
||||
|
||||
Codex Proxy RS 在一个网关中管理 OpenAI 与 xAI 账号,为客户端提供统一鉴权、模型路由、额度状态、
|
||||
请求观测和管理控制面。
|
||||
|
||||
> [!NOTE]
|
||||
> 只支持 OpenAI Responses API,不提供 `/v1/chat/completions`。
|
||||
> 数据面支持 OpenAI Responses、Images 和模型目录协议,不提供 `/v1/chat/completions`。
|
||||
|
||||
## 能力
|
||||
## 功能
|
||||
|
||||
| 领域 | 实现 |
|
||||
| -------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| 协议 | OpenAI Responses JSON、SSE、WebSocket 与模型目录 |
|
||||
| Provider | 固定 OpenAI、xAI 两个 Provider;各自管理 OAuth、账号、额度与模型目录 |
|
||||
| 路由 | Client Key 绑定 Provider、全局模型映射、会话亲和与同 Provider 安全 fallback |
|
||||
| 透明边界 | OpenAI 请求保留未知字段和字段顺序,SSE 与 WebSocket 原样转发业务字节;xAI 在 Provider 内做协议转换 |
|
||||
| 延续 | OpenAI native/replay continuation;xAI 使用客户端提交的完整历史 |
|
||||
| 账号 | 导入、OAuth、credential 刷新、主动额度刷新,以及正常响应中的被动额度/状态观测 |
|
||||
| 管理 | Client Key、账号、模型目录、设置、观测、系统更新与 OAuth |
|
||||
| 计量 | 模型请求 Token、费用、延迟、账号与 Provider 归因,并记录响应确认的实际 `service_tier` |
|
||||
| 领域 | 当前能力 |
|
||||
| --- | --- |
|
||||
| 数据面 | Responses JSON、SSE、WebSocket、review 子代理、Images 生成/编辑与模型目录 |
|
||||
| Provider | 固定 `openai`、`xai` 两个 Provider,各自拥有 OAuth、credential、额度、目录与 transport |
|
||||
| 路由 | Client Key 限定账号分组;按模型能力编译 Provider 候选;支持全局精确模型映射、会话亲和和安全 fallback |
|
||||
| 透明边界 | OpenAI 保留未知请求字段及字段顺序,并原样转发 SSE、WebSocket 与 Images 业务字节;xAI 在 Provider 内转换协议 |
|
||||
| 账号 | OAuth、OpenAI AT/RT、账号文件与 Agent Identity 导入,credential/额度刷新、账号恢复及主动额度重置卡 |
|
||||
| 管理 | 账号与分组、Client Key、运行设置、用量/错误观测、S3/R2 备份、系统更新与回滚 |
|
||||
| 计量 | Token、图片、费用、延迟、账号与 Provider 归因,并记录上游响应确认的实际 `service_tier` |
|
||||
|
||||
架构边界、状态所有权和请求不变量见 [架构文档](docs/architecture.md)。
|
||||
|
||||
## 快速开始
|
||||
|
||||
需要 Docker Engine 与 Docker Compose Plugin。
|
||||
|
||||
### 1. 准备
|
||||
### 1. 准备配置和目录
|
||||
|
||||
```bash
|
||||
git clone https://github.com/zyycn/codex-proxy-rs.git
|
||||
@@ -56,16 +60,16 @@ openssl rand -hex 24
|
||||
openssl rand -hex 24
|
||||
```
|
||||
|
||||
把两个结果写入 `deploy/config.yaml`,并单独设置管理员初始密码:
|
||||
把两个结果分别写入 `deploy/config.yaml`,再设置管理员初始密码:
|
||||
|
||||
| 配置 | 约束 |
|
||||
| -------------------------- | ---------------------------- |
|
||||
| `store.database.password` | 48 位十六进制 |
|
||||
| `store.redis.password` | 48 位十六进制 |
|
||||
| `admin.default_password` | 至少 12 位,不能包含 `$` |
|
||||
| 配置 | 约束 |
|
||||
| --- | --- |
|
||||
| `store.database.password` | 48 位十六进制 |
|
||||
| `store.redis.password` | 48 位十六进制 |
|
||||
| `admin.default_password` | 至少 12 位,不能包含 `$` |
|
||||
|
||||
PostgreSQL、Redis 容器和应用通过同一份 `config.yaml` 使用这两个基础设施密码,不需要额外
|
||||
导出环境变量。管理员密码只在首次创建管理员时生效。
|
||||
PostgreSQL、Redis 和应用从同一份 `config.yaml` 读取基础设施密码,不需要额外导出环境变量。
|
||||
管理员密码只在首次创建管理员时生效。
|
||||
|
||||
Linux 需要允许容器组写入运行目录:
|
||||
|
||||
@@ -83,37 +87,36 @@ docker compose -f deploy/compose.yaml up -d --no-build
|
||||
curl -i http://127.0.0.1:8080/healthz
|
||||
```
|
||||
|
||||
`204 No Content` 表示应用、PostgreSQL 与 Redis 均可用。管理端地址:
|
||||
`204 No Content` 表示应用、PostgreSQL 与 Redis 均可用。管理端地址为
|
||||
`http://127.0.0.1:8080`。
|
||||
|
||||
### 3. 初始化
|
||||
### 3. 初始化网关
|
||||
|
||||
1. 使用 `admin@cpr.local` 与初始密码登录。
|
||||
2. 在账号页选择对应 Provider:OpenAI 支持 OAuth、账号文件与 Agent Identity;xAI 支持 OAuth
|
||||
和兼容账号文件导入。
|
||||
3. 按需配置客户端模型到上游模型的全局映射;未命中映射的模型名原样透传。
|
||||
4. 创建 `sk_...` Client Key,绑定 `openai` 或 `xai`,并设置速率与并发限制。
|
||||
2. 导入或授权 Provider 账号:OpenAI 支持 OAuth、逐行 AT/RT、OAuth JSON 与 Agent Identity;
|
||||
xAI 支持 OAuth 和账号 JSON。
|
||||
3. 创建账号分组并按需配置全局模型映射。未命中的模型名原样交给候选 Provider。
|
||||
4. 创建 `sk_...` Client Key,选择可用账号分组,并设置速率和并发限制;空分组范围表示全部账号。
|
||||
|
||||
OpenAI 账号导入和首次 OAuth 创建在 credential 提交后会立即尝试一次额度观测;额度接口临时失败不会
|
||||
回滚已提交的账号。Free、K12 等套餐共用同一套额度与状态转换逻辑,套餐字段只作为账号事实并隔离
|
||||
对应的模型目录 cache。credential 刷新、额度刷新和正常请求中的被动额度观测是三条独立链路,详见
|
||||
[账号接口](docs/api.md#5-账号)。
|
||||
OpenAI RT 导入会先向官方 token endpoint 换取 AT;仅 AT 的账号不具备自动续期能力,
|
||||
能否参与调度仍取决于 token 中可验证的身份与后续上游事实。账号导入和首次 OAuth 创建会尽力读取一次额度,
|
||||
额度上游暂时失败不会回滚已提交的账号。
|
||||
|
||||
OpenAI 账号的主动重置卡只在用户打开相应弹窗或手工刷新时查询。消费成功后管理端重新查询卡片与额度;
|
||||
系统不会通过修改本地 `reset_at` 伪造重置。完整账号合同见 [账号接口](docs/api.md#5-账号)。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> xAI 使用 OAuth session,不支持把 xAI API Key 作为上游 credential。
|
||||
|
||||
> [!NOTE]
|
||||
> `admin.default_password` 只在首次创建管理员时生效。
|
||||
|
||||
完整部署、权限和密码轮换规则见 [deploy/README.md](deploy/README.md)。
|
||||
完整的权限、密码轮换、备份和升级规则见 [部署文档](deploy/README.md)。
|
||||
|
||||
## 客户端接入
|
||||
|
||||
| 配置 | 值 |
|
||||
| -------- | ---------------------------------------- |
|
||||
| Base URL | `http://127.0.0.1:8080/v1` |
|
||||
| API Key | 管理端创建的 `sk_...` |
|
||||
| 鉴权 | `Authorization: Bearer <client-api-key>` |
|
||||
| 配置 | 值 |
|
||||
| --- | --- |
|
||||
| Base URL | `http://127.0.0.1:8080/v1` |
|
||||
| API Key | 管理端创建的 `sk_...` |
|
||||
| 鉴权 | `Authorization: Bearer <client-api-key>` |
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8080/v1/models \
|
||||
@@ -130,67 +133,70 @@ curl http://127.0.0.1:8080/v1/responses \
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>API 路由</summary>
|
||||
<summary>公开数据面路由</summary>
|
||||
|
||||
| 路由 | 用途 |
|
||||
| ---------------------------------- | ------------------------------ |
|
||||
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
|
||||
| `GET /v1/responses` | Responses WebSocket 升级 |
|
||||
| `POST /v1/responses/review` | review 子代理请求 |
|
||||
| `GET /v1/models` | 当前 Client Key 可用的模型列表 |
|
||||
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
|
||||
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
|
||||
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
|
||||
| 路由 | 用途 |
|
||||
| --- | --- |
|
||||
| `POST /v1/responses` | JSON 响应或 SSE Responses 流 |
|
||||
| `GET /v1/responses` | Responses WebSocket 升级 |
|
||||
| `POST /v1/responses/review` | review 子代理请求 |
|
||||
| `POST /v1/images/generations` | OpenAI Images 生成 JSON 透明转发 |
|
||||
| `POST /v1/images/edits` | OpenAI Images 编辑 JSON 透明转发 |
|
||||
| `GET /v1/models` | 当前 Client Key 账号范围内的模型列表 |
|
||||
| `GET /v1/models/catalog` | Codex 客户端展示用模型目录 |
|
||||
| `GET /v1/models/{model_id}/info` | Codex 客户端展示用模型详情 |
|
||||
| `GET /v1/models/{model_id}` | OpenAI 兼容模型详情 |
|
||||
|
||||
所有 `/v1/*` 路由都需要客户端 API Key。
|
||||
所有 `/v1/*` 路由都需要 Client Key。Images 是 OpenAI Provider 自有端点,不读取模型字段,也不参与
|
||||
文本模型映射。
|
||||
|
||||
</details>
|
||||
|
||||
完整的客户端与管理端路由、鉴权和 mutation 规则见 [接口文档](docs/api.md)。
|
||||
完整的客户端与管理端路由、鉴权、请求字段和 mutation 语义见 [接口文档](docs/api.md)。
|
||||
|
||||
## 运维
|
||||
## 运行维护
|
||||
|
||||
```bash
|
||||
# 实例升级
|
||||
# 同一大版本内升级发布镜像
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build
|
||||
|
||||
# 日志
|
||||
# 查看日志
|
||||
docker compose -f deploy/compose.yaml logs -f codex-proxy-rs
|
||||
|
||||
# 从源码构建
|
||||
# 从当前源码构建
|
||||
docker compose -f deploy/compose.yaml build codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d
|
||||
```
|
||||
|
||||
> [!IMPORTANT]
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点密钥和自更新状态。删除该目录会永久清除
|
||||
> 运行状态。
|
||||
> `.runtime/` 保存 PostgreSQL、Redis、日志、OpenAI 会话锚点和在线更新状态。删除该目录会永久清除
|
||||
> 本地运行状态。
|
||||
|
||||
> [!WARNING]
|
||||
> 不支持跨大版本在线升级。升级到新的大版本时,请使用全新的 `.runtime/` 数据目录重新部署,
|
||||
> 并重新导入或重新授权 Provider 账号与客户端 Key。
|
||||
> 不支持跨大版本在线升级。升级到新的大版本时请使用全新的 `.runtime/` 数据目录,并重新导入或授权
|
||||
> Provider 账号与 Client Key。
|
||||
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket
|
||||
upgrade 与真实客户端 IP;不要暴露 PostgreSQL 或 Redis。
|
||||
Compose 默认只绑定 `127.0.0.1`。公网接入应使用 HTTPS 反向代理,转发 WebSocket upgrade 与真实客户端
|
||||
IP;不要暴露 PostgreSQL 或 Redis。当前运行拓扑为单副本,不能通过复制应用容器扩容。
|
||||
|
||||
## 维护与发布
|
||||
|
||||
发布仓库版本必须从干净、已同步上游的发布分支运行:
|
||||
仓库版本只能从干净、已同步上游的发布分支发布:
|
||||
|
||||
```bash
|
||||
release/publish <version>
|
||||
```
|
||||
|
||||
脚本依赖已登录的 GitHub CLI(`gh auth login`)。它会更新 `release/version.yaml`、创建约定提交与带
|
||||
注释的 `v<version>` tag,原子推送分支和 tag,再从发布分支触发 Release 构建,使构建缓存可以跨版本
|
||||
复用。不要手工修改版本后单独打 tag。该脚本不会登录服务器,也不会改变任何运行实例;源码提交、
|
||||
仓库发版和实例升级是三个独立状态。
|
||||
脚本依赖已登录的 GitHub CLI(`gh auth login`),会更新 `release/version.yaml`、创建约定提交和带注释的
|
||||
`v<version>` tag,原子推送分支与 tag,再触发 Release 构建。源码提交、仓库发版和运行实例升级是三个
|
||||
独立状态;脚本不会登录服务器或改变任何运行实例。
|
||||
|
||||
## 文档
|
||||
|
||||
- [部署](deploy/README.md)
|
||||
- [部署与运维](deploy/README.md)
|
||||
- [客户端与管理端接口](docs/api.md)
|
||||
- [架构与数据边界](docs/architecture.md)
|
||||
- [系统架构](docs/architecture.md)
|
||||
- [数据库迁移规则](backend/migrations/README.md)
|
||||
- [Release](https://github.com/zyycn/codex-proxy-rs/releases)
|
||||
- [容器镜像](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
|
||||
|
||||
Generated
+52
-78
@@ -120,9 +120,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.91"
|
||||
version = "0.1.92"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -216,9 +216,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.140.0"
|
||||
version = "1.141.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9660cf991e512fbe6094f1041ff3d3282bc5aeeeb6184e8de437ec2de024a10"
|
||||
checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -451,9 +451,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.6.1"
|
||||
version = "1.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8"
|
||||
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -578,9 +578,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.0"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64-simd"
|
||||
@@ -801,9 +801,9 @@ checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.1"
|
||||
version = "0.18.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
|
||||
checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time",
|
||||
@@ -1246,7 +1246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1384,9 +1384,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
|
||||
checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -1399,9 +1399,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
|
||||
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
@@ -1409,15 +1409,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
|
||||
checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
@@ -1437,32 +1437,32 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
|
||||
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
|
||||
checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-timer"
|
||||
@@ -1472,9 +1472,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -1493,7 +1493,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"cron",
|
||||
@@ -1519,7 +1519,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -1686,9 +1686,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.15"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -2125,11 +2125,9 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"js-sys",
|
||||
"pem",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signature 2.2.0",
|
||||
"simple_asn1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -2280,16 +2278,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
@@ -2407,16 +2395,6 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3"
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "1.0.0"
|
||||
@@ -2597,12 +2575,13 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"cookie",
|
||||
"crypto_box",
|
||||
"ed25519-dalek",
|
||||
"flate2",
|
||||
"futures",
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
@@ -2610,7 +2589,6 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"hex",
|
||||
"hyper-rustls",
|
||||
"jsonwebtoken",
|
||||
"reqwest",
|
||||
"roxmltree",
|
||||
"rustls",
|
||||
@@ -2628,8 +2606,10 @@ dependencies = [
|
||||
"tokio-tungstenite 0.28.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tungstenite 0.27.0",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
"zstd",
|
||||
@@ -2641,7 +2621,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"base64 0.23.0",
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
@@ -2734,7 +2714,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3004,7 +2984,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3323,18 +3303,6 @@ version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||
|
||||
[[package]]
|
||||
name = "simple_asn1"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"thiserror",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.3"
|
||||
@@ -3660,23 +3628,23 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.19"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
|
||||
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.19"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
||||
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -4112,6 +4080,12 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urlencoding"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
|
||||
@@ -27,8 +27,9 @@ impl LoadableConfig for GatewayConfig {
|
||||
return Err(ConfigError::InvalidField("schema_version"));
|
||||
}
|
||||
self.host.resolve_and_validate(source_dir)?;
|
||||
let runtime_data_dir = self.host.runtime_data_dir().to_path_buf();
|
||||
self.store
|
||||
.resolve_and_validate(source_dir)
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("store"))?;
|
||||
self.admin
|
||||
.resolve_and_validate(source_dir)
|
||||
@@ -37,7 +38,7 @@ impl LoadableConfig for GatewayConfig {
|
||||
.resolve_and_validate(source_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("api"))?;
|
||||
self.openai
|
||||
.resolve_and_validate(source_dir)
|
||||
.resolve_and_validate(&runtime_data_dir)
|
||||
.map_err(|_| ConfigError::InvalidField("openai"))?;
|
||||
self.xai
|
||||
.resolve_and_validate(source_dir)
|
||||
|
||||
@@ -18,10 +18,18 @@ fn config_loader_should_load_complete_terminal_example() {
|
||||
fn config_loader_should_resolve_paths_relative_to_config_file() {
|
||||
let (config, _directory) = parse_config(&valid_config()).expect("resolved config");
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains(".runtime/data"));
|
||||
assert!(debug.contains(".runtime/logs"));
|
||||
assert!(debug.contains("frontend/dist"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_missing_runtime_data_dir() {
|
||||
let config = valid_config().replace(" runtime_data_dir: '../.runtime/data'\n", "");
|
||||
|
||||
assert!(parse_config(&config).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_inject_connection_passwords_into_urls() {
|
||||
parse_config(&valid_config()).expect("Store validates password injection into both URLs");
|
||||
@@ -134,8 +142,11 @@ fn config_loader_should_reject_removed_fingerprint_section() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_codex_core_version() {
|
||||
assert_rejected(valid_config().replace("codex_version: '0.146.0'", "codex_version: 'latest'"));
|
||||
fn config_loader_should_reject_invalid_codex_cli_version() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"codex_version: '0.147.0-alpha.6.6'",
|
||||
"codex_version: 'latest'",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -154,7 +165,7 @@ fn config_loader_should_reject_zero_server_port() {
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_desktop_profile_fields() {
|
||||
assert_rejected(valid_config().replace("desktop_build: '6119'", "desktop_build: 'build'"));
|
||||
assert_rejected(valid_config().replace("desktop_build: '6415'", "desktop_build: 'build'"));
|
||||
}
|
||||
|
||||
fn assert_rejected(config: String) {
|
||||
|
||||
@@ -119,11 +119,11 @@ pub struct RetentionDecision {
|
||||
|
||||
/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。
|
||||
///
|
||||
/// 规则(`docs/s3-backup-design-audit.md` §4.4):
|
||||
/// 自动保留规则:
|
||||
/// - `retentionDays = 0` 时不按天数清理;`retentionCount = 0` 时不按份数清理。
|
||||
/// - 自动计划备份满足任一启用阈值即可进入删除。
|
||||
/// - 无论阈值如何,至少保留最近一个成功的计划备份(倒序下标 0)。
|
||||
/// - 手动备份不受自动阈值影响;无 TTL,仅由管理员手动删除。
|
||||
/// - 手动备份不进入本规则;手工/计划记录各自的 `expires_at` 由到期扫描处理。
|
||||
#[must_use]
|
||||
pub fn decide_retention(
|
||||
retention_days: u32,
|
||||
|
||||
@@ -220,7 +220,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -229,9 +230,12 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -394,7 +398,8 @@ impl BackupTask {
|
||||
sha256: Some(artifact.sha256.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
self.repository
|
||||
let Some(uploading_record) = self
|
||||
.repository
|
||||
.transition_status(
|
||||
&record.id,
|
||||
BackupStatus::Dumping,
|
||||
@@ -403,10 +408,13 @@ impl BackupTask {
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.map_err(repo_error)?;
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.upload_and_verify(
|
||||
record,
|
||||
&uploading_record,
|
||||
&artifact.path,
|
||||
artifact.size_bytes,
|
||||
&artifact.sha256,
|
||||
@@ -448,8 +456,12 @@ impl BackupTask {
|
||||
.await
|
||||
{
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "上传到对象存储失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("上传到对象存储失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -475,19 +487,46 @@ impl BackupTask {
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(Some(remote)) => {
|
||||
let size_matches = remote.size_bytes == size_bytes;
|
||||
let sha256_matches = remote.sha256 == sha256;
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
expected_size_bytes = size_bytes,
|
||||
remote_size_bytes = remote.size_bytes,
|
||||
sha256_matches,
|
||||
"备份远端对象校验不一致"
|
||||
);
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"远端对象大小或校验值不一致",
|
||||
match (size_matches, sha256_matches) {
|
||||
(false, true) => "远端对象大小不一致",
|
||||
(true, false) => "远端对象 sha256 校验值不一致",
|
||||
(false, false) => "远端对象大小和 sha256 校验值均不一致",
|
||||
(true, true) => "远端对象校验状态异常",
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(None) => {
|
||||
warn!(backup_id = %record.id, "上传完成后未找到远端对象");
|
||||
self.fail_task(
|
||||
record,
|
||||
code::REMOTE_VERIFICATION_FAILED,
|
||||
"上传完成后未找到远端对象",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = store.delete_object(&config, &record.object_key).await;
|
||||
self.fail_task(record, error.code(), "远端对象校验失败")
|
||||
.await?;
|
||||
self.fail_task(
|
||||
record,
|
||||
error.code(),
|
||||
format!("远端对象校验失败:{}", error.message()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -500,10 +539,17 @@ impl BackupTask {
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<(), WorkerTaskError> {
|
||||
let message = message.into();
|
||||
warn!(
|
||||
backup_id = %record.id,
|
||||
error_code,
|
||||
error_message = %message,
|
||||
"备份任务失败"
|
||||
);
|
||||
let now = Utc::now();
|
||||
let update = StatusTransitionUpdate {
|
||||
error_code: Some(error_code.to_string()),
|
||||
error_message: Some(message.into()),
|
||||
error_message: Some(message),
|
||||
completed_at: Some(now),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -21,9 +21,9 @@ pub mod ports;
|
||||
mod use_case;
|
||||
|
||||
pub use use_case::{
|
||||
accounts::AccountsService, auth::AuthService, backup::BackupService,
|
||||
client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService,
|
||||
settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
account_groups::AccountGroupService, accounts::AccountsService, auth::AuthService,
|
||||
backup::BackupService, client_keys::ClientKeyService, observability::ObservabilityService,
|
||||
openai::OpenAiService, settings::SettingsService, system::SystemService, xai::XaiService,
|
||||
};
|
||||
|
||||
use model::{AdminError, AdminErrorKind};
|
||||
@@ -33,10 +33,10 @@ use ports::{
|
||||
system::SystemOperations,
|
||||
};
|
||||
use use_case::{
|
||||
accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService,
|
||||
client_keys::DefaultClientKeyService, observability::DefaultObservabilityService,
|
||||
openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService,
|
||||
xai::DefaultXaiService,
|
||||
account_groups::DefaultAccountGroupService, accounts::DefaultAccountsService,
|
||||
auth::DefaultAuthService, backup::DefaultBackupService, client_keys::DefaultClientKeyService,
|
||||
observability::DefaultObservabilityService, openai::DefaultOpenAiService,
|
||||
settings::DefaultSettingsService, system::DefaultSystemService, xai::DefaultXaiService,
|
||||
};
|
||||
|
||||
const OPENAI_PROVIDER_KIND: &str = "openai";
|
||||
@@ -147,6 +147,7 @@ pub enum AdminConfigError {
|
||||
pub struct AdminServices {
|
||||
auth: Arc<dyn AuthService>,
|
||||
accounts: Arc<dyn AccountsService>,
|
||||
account_groups: Arc<dyn AccountGroupService>,
|
||||
client_keys: Arc<dyn ClientKeyService>,
|
||||
observability: Arc<dyn ObservabilityService>,
|
||||
settings: Arc<dyn SettingsService>,
|
||||
@@ -167,6 +168,11 @@ impl AdminServices {
|
||||
self.accounts.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn account_groups(&self) -> &dyn AccountGroupService {
|
||||
self.account_groups.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn client_keys(&self) -> &dyn ClientKeyService {
|
||||
self.client_keys.as_ref()
|
||||
@@ -255,7 +261,6 @@ pub async fn initialize(
|
||||
|
||||
let accounts = Arc::new(DefaultAccountsService::new(
|
||||
store.accounts(),
|
||||
store.settings(),
|
||||
registry.clone(),
|
||||
snapshot.clone(),
|
||||
probe.clone(),
|
||||
@@ -275,6 +280,10 @@ pub async fn initialize(
|
||||
let services = AdminServices {
|
||||
auth,
|
||||
accounts,
|
||||
account_groups: Arc::new(DefaultAccountGroupService::new(
|
||||
store.account_groups(),
|
||||
snapshot.clone(),
|
||||
)),
|
||||
client_keys: Arc::new(DefaultClientKeyService::new(
|
||||
store.client_keys(),
|
||||
snapshot.clone(),
|
||||
@@ -338,7 +347,10 @@ fn map_provider_registry_error(error: ProviderAdminError) -> AdminError {
|
||||
}
|
||||
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::Unavailable | ProviderAdminErrorKind::CredentialRefreshRequired => {
|
||||
AdminErrorKind::Unavailable
|
||||
}
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
AdminError::new(kind, "Provider registry initialization failed")
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
//! Provider-neutral account group commands and projections.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::routing::AccountGroupId;
|
||||
|
||||
use super::{PageSize, Revision, observability::DecimalAmount};
|
||||
|
||||
/// Canonical `#RRGGBBAA` color persisted with an account group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupColor(String);
|
||||
|
||||
impl AccountGroupColor {
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
(value.len() == 9
|
||||
&& value.starts_with('#')
|
||||
&& value[1..].bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.then(|| Self(value.to_ascii_uppercase()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Group member availability at the current observation time.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountGroupAccountSummary {
|
||||
pub available: u64,
|
||||
pub limited: u64,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
/// Group scheduling slots derived from available accounts and runtime leases.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountGroupCapacity {
|
||||
pub used_slots: Option<u64>,
|
||||
pub total_slots: u64,
|
||||
}
|
||||
|
||||
/// Successful, downstream-committed USD request costs for group accounts.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupUsage {
|
||||
pub today_usd: DecimalAmount,
|
||||
pub total_usd: DecimalAmount,
|
||||
}
|
||||
|
||||
/// Lightweight group reference embedded in account and client-key views.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupRef {
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub color: AccountGroupColor,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
/// Account group list query.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupListQuery {
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub search: Option<String>,
|
||||
pub enabled: Option<bool>,
|
||||
}
|
||||
|
||||
/// Complete account group summary.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupRecord {
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub color: AccountGroupColor,
|
||||
pub enabled: bool,
|
||||
pub member_count: u64,
|
||||
pub provider_counts: BTreeMap<String, u64>,
|
||||
pub client_key_count: u64,
|
||||
pub account_summary: AccountGroupAccountSummary,
|
||||
pub capacity: AccountGroupCapacity,
|
||||
pub usage: AccountGroupUsage,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Paginated account groups.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupPage {
|
||||
pub config_revision: Revision,
|
||||
pub items: Vec<AccountGroupRecord>,
|
||||
pub total: u64,
|
||||
pub page: u32,
|
||||
pub page_size: u16,
|
||||
}
|
||||
|
||||
/// Create an account group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CreateAccountGroup {
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub color: AccountGroupColor,
|
||||
}
|
||||
|
||||
/// Store-ready create command with a generated stable ID.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NewAccountGroup {
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub color: AccountGroupColor,
|
||||
}
|
||||
|
||||
/// Update an account group's descriptive fields.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UpdateAccountGroup {
|
||||
pub id: AccountGroupId,
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
pub color: AccountGroupColor,
|
||||
}
|
||||
|
||||
/// Enable or disable an account group.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetAccountGroupEnabled {
|
||||
pub id: AccountGroupId,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
/// Delete an account group that is not referenced by a client key.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteAccountGroup {
|
||||
pub id: AccountGroupId,
|
||||
}
|
||||
|
||||
/// Result of any account-group mutation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountGroupMutation {
|
||||
pub config_revision: Revision,
|
||||
pub id: AccountGroupId,
|
||||
pub record: Option<AccountGroupRecord>,
|
||||
}
|
||||
@@ -7,22 +7,13 @@ use futures::Stream;
|
||||
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
use super::{PageSize, Revision, observability::TimeRange};
|
||||
use super::{PageSize, Revision, account_groups::AccountGroupRef, observability::TimeRange};
|
||||
|
||||
/// Provider 账号当前可用性;权威定义与字符串编解码在 gateway-core,此处仅复用。
|
||||
pub use gateway_core::engine::credential::AccountAvailability;
|
||||
|
||||
/// 管理页使用的归一化账号状态。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountStatus {
|
||||
Active,
|
||||
RateLimited,
|
||||
QuotaExhausted,
|
||||
Expired,
|
||||
Invalid,
|
||||
Disabled,
|
||||
Banned,
|
||||
}
|
||||
pub use gateway_core::engine::credential::{
|
||||
AccountConcurrencyLimit, AccountErrorReason, AccountStatus, AccountStatusFacts,
|
||||
AccountStatusProjection, AccountWeight, CredentialState, QuotaAccessState, QuotaEvidence,
|
||||
QuotaState, resolve_account_status,
|
||||
};
|
||||
|
||||
/// 账号列表排序字段。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -55,19 +46,28 @@ pub struct AccountListQuery {
|
||||
pub page: u32,
|
||||
pub page_size: PageSize,
|
||||
pub provider_kind: Option<ProviderKind>,
|
||||
pub group_filter: Option<AccountGroupFilter>,
|
||||
pub search: Option<String>,
|
||||
pub status: Option<AccountStatus>,
|
||||
pub sort: Option<AccountSort>,
|
||||
}
|
||||
|
||||
/// Optional account membership filter.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AccountGroupFilter {
|
||||
Group(gateway_core::routing::AccountGroupId),
|
||||
Ungrouped,
|
||||
}
|
||||
|
||||
/// 账号公共存储投影;Provider 专属字段不进入此结构。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountRecord {
|
||||
pub id: String,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub groups: Vec<AccountGroupRef>,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -76,9 +76,13 @@ pub struct AccountRecord {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub quota_observed_at: Option<DateTime<Utc>>,
|
||||
pub concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
pub weight: AccountWeight,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
pub quota: QuotaState,
|
||||
pub last_error_reason: Option<AccountErrorReason>,
|
||||
pub last_error_message: Option<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
@@ -163,27 +167,63 @@ pub struct AccountUsageWindowResult {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountPage {
|
||||
pub config_revision: Revision,
|
||||
pub items: Vec<AccountRecord>,
|
||||
pub items: Vec<AccountPageItem>,
|
||||
pub total: u64,
|
||||
pub summary: AccountSummary,
|
||||
}
|
||||
|
||||
/// 同一状态快照下的账号事实与唯一状态投影。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountPageItem {
|
||||
pub account: AccountRecord,
|
||||
pub projection: AccountStatusProjection,
|
||||
}
|
||||
|
||||
/// 统一账号目录的全局状态计数,不受当前筛选和分页影响。
|
||||
///
|
||||
/// 配额耗尽是可恢复状态,始终单列;`unavailable` 只包含过期、禁用和封禁账号。
|
||||
/// 计数与 [`AccountStatus`] 一一对应,由 store 按派生状态聚合。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct AccountSummary {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 账号启停写入命令。
|
||||
/// 账号可编辑事实的一次性替换命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SetAccountEnabled {
|
||||
pub struct UpdateAccount {
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
pub concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
pub weight: AccountWeight,
|
||||
pub group_ids: Vec<gateway_core::routing::AccountGroupId>,
|
||||
}
|
||||
|
||||
/// 账号更新结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountUpdateResult {
|
||||
pub config_revision: Revision,
|
||||
pub account_id: gateway_core::engine::credential::ProviderAccountId,
|
||||
}
|
||||
|
||||
/// 一批账号可编辑事实的一次性替换命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BatchUpdateAccounts {
|
||||
pub account_ids: Vec<String>,
|
||||
pub enabled: bool,
|
||||
pub concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
pub weight: AccountWeight,
|
||||
pub group_ids: Vec<gateway_core::routing::AccountGroupId>,
|
||||
}
|
||||
|
||||
/// 批量账号更新结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountsUpdateResult {
|
||||
pub config_revision: Revision,
|
||||
pub account_ids: Vec<gateway_core::engine::credential::ProviderAccountId>,
|
||||
}
|
||||
|
||||
/// 账号批量删除命令。
|
||||
@@ -207,9 +247,7 @@ pub enum AccountConnectionTestEvent {
|
||||
Content {
|
||||
text: String,
|
||||
},
|
||||
Completed {
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
Completed,
|
||||
Failed {
|
||||
message: String,
|
||||
provider_error_code: Option<String>,
|
||||
@@ -217,7 +255,6 @@ pub enum AccountConnectionTestEvent {
|
||||
upstream_status: Option<u16>,
|
||||
upstream_content_type: Option<String>,
|
||||
upstream_body: Option<String>,
|
||||
account_status: AccountStatus,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -288,7 +288,7 @@ pub struct BackupRecord {
|
||||
pub error_message: Option<String>,
|
||||
pub started_at: Option<DateTime<Utc>>,
|
||||
pub completed_at: Option<DateTime<Utc>>,
|
||||
/// 手动过期时间;创建时确定,到期自动清理。
|
||||
/// 创建时确定的过期时间;手工和计划备份均可设置,到期自动清理。
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
|
||||
@@ -6,10 +6,10 @@ use chrono::{DateTime, Utc};
|
||||
|
||||
use gateway_core::{
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::ProviderKind,
|
||||
routing::{AccountGroupId, ProviderKind},
|
||||
};
|
||||
|
||||
use super::{AdminModelError, Revision};
|
||||
use super::{AdminModelError, Revision, account_groups::AccountGroupRef};
|
||||
|
||||
/// Client Key 列表保持旧 HTTP 合同允许的完整非零 `u16` 页大小。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
@@ -88,7 +88,8 @@ pub struct ClientKeyRecord {
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub groups: Vec<AccountGroupRef>,
|
||||
pub provider_kinds: Vec<ProviderKind>,
|
||||
pub prefix: String,
|
||||
pub enabled: bool,
|
||||
pub limits: RateLimits,
|
||||
@@ -143,7 +144,7 @@ impl fmt::Debug for ClientKeySecret {
|
||||
pub struct CreateClientKey {
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub group_ids: Vec<AccountGroupId>,
|
||||
pub limits: RateLimits,
|
||||
}
|
||||
|
||||
@@ -153,7 +154,7 @@ pub struct NewClientKey {
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub group_ids: Vec<AccountGroupId>,
|
||||
pub limits: RateLimits,
|
||||
pub plaintext: String,
|
||||
}
|
||||
@@ -164,7 +165,7 @@ impl fmt::Debug for NewClientKey {
|
||||
.debug_struct("NewClientKey")
|
||||
.field("id", &self.id)
|
||||
.field("name", &self.name)
|
||||
.field("provider_kind", &self.provider_kind)
|
||||
.field("group_ids", &self.group_ids)
|
||||
.field("plaintext", &"[REDACTED]")
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
@@ -176,7 +177,7 @@ pub struct UpdateClientKey {
|
||||
pub id: ClientApiKeyId,
|
||||
pub name: String,
|
||||
pub label: Option<String>,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub group_ids: Vec<AccountGroupId>,
|
||||
pub limits: RateLimits,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::num::{NonZeroU16, NonZeroU64};
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
|
||||
@@ -439,25 +439,15 @@ pub struct RequestMetricPoint {
|
||||
pub costs: Vec<CurrencyCost>,
|
||||
}
|
||||
|
||||
/// 账号池容量统计。
|
||||
///
|
||||
/// 首页概览的 `unavailable` 表示不可参与调度,包含配额耗尽及过期、禁用和封禁账号。
|
||||
///
|
||||
/// `quota_exhausted`(402 确认耗尽)与 `rate_limited`(429 限流中)单列,
|
||||
/// 以便状态详情展示和诊断。
|
||||
/// 账号池的统一五态统计。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AccountPoolMetrics {
|
||||
pub total: u64,
|
||||
pub enabled: u64,
|
||||
pub unavailable: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 中一个账号的模型级用量事实。
|
||||
@@ -488,7 +478,7 @@ pub struct DashboardAccountRequestBucket {
|
||||
pub request_count: u64,
|
||||
}
|
||||
|
||||
/// Dashboard 中一个账号的完整用量与公共状态事实。
|
||||
/// Dashboard 中一个账号的完整用量事实。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardAccountUsage {
|
||||
pub account_id: String,
|
||||
@@ -497,8 +487,6 @@ pub struct DashboardAccountUsage {
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub enabled: bool,
|
||||
pub availability: String,
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub input_tokens: Option<u64>,
|
||||
@@ -525,18 +513,31 @@ pub struct DashboardAccountUsage {
|
||||
/// Dashboard 当前账号池的可重建运行时槽位事实。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct DashboardRuntimeSlots {
|
||||
/// 与持久账号池指标相同谓词下可参与调度的账号数。
|
||||
pub active_accounts: u64,
|
||||
/// 继承运行参数默认并发上限的可调度账号数。
|
||||
pub inherited_accounts: u64,
|
||||
/// 已配置账号级并发上限的可调度账号槽位总数。
|
||||
pub overridden_slots: u64,
|
||||
/// Redis 可用时的实时 in-flight 槽位总数;不可用时为 `None`。
|
||||
pub used_slots: Option<u64>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的公共观测事实。
|
||||
/// 仪表盘卡片脚注展示的全历史累计。
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct DashboardTotals {
|
||||
pub request_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub cached_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub billing_usd: Option<DecimalAmount>,
|
||||
}
|
||||
|
||||
/// 仪表盘所需的当前区间事实及全历史累计。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DashboardObservation {
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
pub totals: DashboardTotals,
|
||||
pub provider_accounts: AccountPoolMetrics,
|
||||
pub trend: Vec<RequestMetricPoint>,
|
||||
pub account_usage: Vec<DashboardAccountUsage>,
|
||||
@@ -549,11 +550,14 @@ pub struct UsageRecord {
|
||||
pub id: String,
|
||||
pub client_api_key_ref: String,
|
||||
pub config_revision: u64,
|
||||
pub routing_scope: String,
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub routing_group_names_snapshot: Vec<String>,
|
||||
pub protocol: String,
|
||||
pub operation: String,
|
||||
pub endpoint: String,
|
||||
pub client_transport: String,
|
||||
pub requested_model_id: String,
|
||||
pub requested_model_id: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
|
||||
@@ -8,10 +8,11 @@ use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId, ProviderAccountIdentity},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{
|
||||
AdminError, MutationActor, MutationContext, PageSize, Revision,
|
||||
accounts::{AccountAvailability, AccountRecord, AccountStatus, AccountSummary, AccountUsage},
|
||||
accounts::{AccountRecord, AccountSummary, AccountUsage, CredentialState},
|
||||
};
|
||||
|
||||
/// Provider-owned JSON;公共层只搬运且 Debug 不输出值。
|
||||
@@ -52,24 +53,24 @@ pub struct CredentialCursor {
|
||||
/// Provider credential 列表查询。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CredentialListQuery {
|
||||
pub availability: Option<CredentialAvailabilityFilter>,
|
||||
pub credential_state: Option<CredentialStateFilter>,
|
||||
pub enabled: Option<bool>,
|
||||
pub window: CredentialListWindow,
|
||||
}
|
||||
|
||||
/// Credential 可用性筛选;支持一个 wire 值对应多个持久状态。
|
||||
/// Credential 状态筛选。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CredentialAvailabilityFilter {
|
||||
Exact(AccountAvailability),
|
||||
AnyOf(Vec<AccountAvailability>),
|
||||
pub enum CredentialStateFilter {
|
||||
Exact(CredentialState),
|
||||
AnyOf(Vec<CredentialState>),
|
||||
}
|
||||
|
||||
impl CredentialAvailabilityFilter {
|
||||
impl CredentialStateFilter {
|
||||
#[must_use]
|
||||
pub fn matches(&self, availability: AccountAvailability) -> bool {
|
||||
pub fn matches(&self, credential_state: CredentialState) -> bool {
|
||||
match self {
|
||||
Self::Exact(expected) => *expected == availability,
|
||||
Self::AnyOf(expected) => expected.contains(&availability),
|
||||
Self::Exact(expected) => *expected == credential_state,
|
||||
Self::AnyOf(expected) => expected.contains(&credential_state),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -143,7 +144,7 @@ pub struct PreparedCredentialCreate {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub name: String,
|
||||
pub email: Option<String>,
|
||||
pub upstream_user_id: String,
|
||||
pub upstream_user_id: Option<String>,
|
||||
pub upstream_account_id: Option<String>,
|
||||
pub plan_type: Option<String>,
|
||||
pub authentication_kind: String,
|
||||
@@ -152,8 +153,8 @@ pub struct PreparedCredentialCreate {
|
||||
pub access_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub next_refresh_at: Option<DateTime<Utc>>,
|
||||
pub enabled: bool,
|
||||
pub availability: AccountAvailability,
|
||||
pub availability_observed_at: DateTime<Utc>,
|
||||
pub credential_state: CredentialState,
|
||||
pub credential_observed_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Provider 对一份导入文档的完整验证结果。
|
||||
@@ -561,13 +562,44 @@ impl fmt::Debug for RotateCredential {
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用账号用量能否可靠归属到一个 Provider quota 窗口。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum QuotaLocalUsageAttribution {
|
||||
/// 窗口覆盖账号的全部请求,可按账号与时间范围聚合。
|
||||
AccountWide,
|
||||
/// 窗口需要 Provider / 模型级归属,通用账号聚合不可用。
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
/// Provider quota bucket 中窗口的官方位置语义。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProviderQuotaWindowRole {
|
||||
Primary,
|
||||
Secondary,
|
||||
Monthly,
|
||||
}
|
||||
|
||||
impl ProviderQuotaWindowRole {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Primary => "primary",
|
||||
Self::Secondary => "secondary",
|
||||
Self::Monthly => "monthly",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个 Provider quota 窗口的公共投影。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderQuotaWindow {
|
||||
pub key: String,
|
||||
pub group: String,
|
||||
pub label: String,
|
||||
pub source: Option<String>,
|
||||
pub limit_id: Option<String>,
|
||||
pub limit_name: Option<String>,
|
||||
pub role: Option<ProviderQuotaWindowRole>,
|
||||
pub local_usage_attribution: QuotaLocalUsageAttribution,
|
||||
pub window_seconds: Option<u64>,
|
||||
pub used_percent: Option<f64>,
|
||||
pub reset_at: Option<DateTime<Utc>>,
|
||||
@@ -590,47 +622,153 @@ pub struct ProviderQuota {
|
||||
pub observed_at: Option<DateTime<Utc>>,
|
||||
pub refresh_token_expires_at: Option<DateTime<Utc>>,
|
||||
pub windows: Vec<ProviderQuotaWindow>,
|
||||
/// 快照级限流事实(顶层或任一窗口触顶);Admin 状态派生用,前端不再遍历窗口猜测。
|
||||
/// 展示用快照级触顶事实(顶层或任一窗口触顶);不参与账号五态派生。
|
||||
pub limit_reached: bool,
|
||||
pub provider_data: Option<ProviderDocument>,
|
||||
}
|
||||
|
||||
/// Provider 返回的一张安全主动额度重置卡。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResetCredit {
|
||||
pub id: String,
|
||||
pub status: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub expires_at: Option<DateTime<Utc>>,
|
||||
pub reset_type: Option<String>,
|
||||
}
|
||||
|
||||
/// Provider 主动额度重置卡列表。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResetCredits {
|
||||
pub available_count: u64,
|
||||
pub credits: Vec<ProviderResetCredit>,
|
||||
}
|
||||
|
||||
/// 一次主动额度重置卡消费命令。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ConsumeProviderResetCredit {
|
||||
pub account_id: ProviderAccountId,
|
||||
pub credit_id: Option<String>,
|
||||
pub redeem_request_id: Uuid,
|
||||
}
|
||||
|
||||
/// Provider 返回的主动额度重置卡消费结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResetCreditResult {
|
||||
pub code: String,
|
||||
pub credit: Option<ProviderResetCredit>,
|
||||
}
|
||||
|
||||
impl ProviderQuota {
|
||||
/// 返回 Dashboard 使用的代表性额度比例。
|
||||
/// 返回账号展开面板使用的当前额度窗口用量。
|
||||
///
|
||||
/// 优先使用短周期窗口,再依次使用周、月和其它窗口;同一优先级取较高的已用比例。
|
||||
/// 这里只解释跨 Provider 共享的窗口语义,绝不读取 Provider 私有 JSON。
|
||||
/// Provider 已提供账号级本地用量时,固定窗口和无固定重置点的滚动窗口
|
||||
/// 都可作为代表用量;优先级与 Dashboard 代表性额度窗口一致,
|
||||
/// 同一优先级保持 Provider 投影顺序。
|
||||
#[must_use]
|
||||
pub fn representative_used_percent(&self) -> Option<f64> {
|
||||
pub fn representative_window_usage(&self) -> Option<&AccountUsage> {
|
||||
self.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, window)| {
|
||||
window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
&& window.window_seconds.is_some()
|
||||
&& window.local_usage.is_some()
|
||||
})
|
||||
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
|
||||
.and_then(|(_, window)| window.local_usage.as_ref())
|
||||
}
|
||||
|
||||
/// 返回 Dashboard 使用的代表性额度比例。
|
||||
///
|
||||
/// 优先使用覆盖账号全部请求的窗口;同一归属范围内再依次使用短周期、周、月
|
||||
/// 和其它窗口,同一优先级取较高的已用比例。这里只解释跨 Provider 共享的窗口
|
||||
/// 语义,绝不读取 Provider 私有 JSON。
|
||||
#[must_use]
|
||||
pub fn representative_used_percent(&self) -> Option<f64> {
|
||||
if self.limit_reached {
|
||||
return Some(100.0);
|
||||
}
|
||||
self.representative_used_window()
|
||||
.map(|(_, used_percent)| used_percent)
|
||||
}
|
||||
|
||||
/// 将已确认的账号级额度耗尽事实投影到展示窗口。
|
||||
///
|
||||
/// Provider 已指出具体触顶窗口时只归一化这些窗口;否则归一化 Dashboard
|
||||
/// 同样会选择的代表窗口,避免把多个独立额度窗口全部伪造成已用尽。
|
||||
pub fn apply_limit_reached_display(&mut self) {
|
||||
if !self.limit_reached {
|
||||
return;
|
||||
}
|
||||
let reached_windows = self
|
||||
.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, window)| window.limit_reached.then_some(index))
|
||||
.collect::<Vec<_>>();
|
||||
if !reached_windows.is_empty() {
|
||||
for index in reached_windows {
|
||||
self.windows[index].used_percent = Some(100.0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let representative = self
|
||||
.representative_used_window()
|
||||
.map(|(index, _)| index)
|
||||
.or_else(|| {
|
||||
self.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.min_by_key(|(index, window)| (quota_usage_priority(window), *index))
|
||||
.map(|(index, _)| index)
|
||||
});
|
||||
if let Some(index) = representative {
|
||||
self.windows[index].used_percent = Some(100.0);
|
||||
self.windows[index].limit_reached = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn representative_used_window(&self) -> Option<(usize, f64)> {
|
||||
self.windows
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|window| {
|
||||
let (index, window) = window;
|
||||
let used_percent = window
|
||||
.used_percent
|
||||
.filter(|value| value.is_finite())
|
||||
.map(|value| value.clamp(0.0, 100.0))?;
|
||||
Some((quota_usage_priority(window), used_percent))
|
||||
Some((index, quota_usage_priority(window), used_percent))
|
||||
})
|
||||
.fold(None::<(u8, f64)>, |selected, candidate| match selected {
|
||||
Some(current)
|
||||
if current.0 < candidate.0
|
||||
|| (current.0 == candidate.0 && current.1 >= candidate.1) =>
|
||||
{
|
||||
.fold(
|
||||
None::<(usize, (u8, u8), f64)>,
|
||||
|selected, candidate| match selected {
|
||||
Some(current)
|
||||
}
|
||||
_ => Some(candidate),
|
||||
})
|
||||
.map(|(_, used_percent)| used_percent)
|
||||
if current.1 < candidate.1
|
||||
|| (current.1 == candidate.1 && current.2 >= candidate.2) =>
|
||||
{
|
||||
Some(current)
|
||||
}
|
||||
_ => Some(candidate),
|
||||
},
|
||||
)
|
||||
.map(|(index, _, used_percent)| (index, used_percent))
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_usage_priority(window: &ProviderQuotaWindow) -> u8 {
|
||||
match window.group.as_str() {
|
||||
fn quota_usage_priority(window: &ProviderQuotaWindow) -> (u8, u8) {
|
||||
let attribution = match window.local_usage_attribution {
|
||||
QuotaLocalUsageAttribution::AccountWide => 0,
|
||||
QuotaLocalUsageAttribution::Unavailable => 1,
|
||||
};
|
||||
let duration = match window.group.as_str() {
|
||||
"shortTerm" if window.window_seconds.is_some_and(is_week_window) => 1,
|
||||
"shortTerm" => 0,
|
||||
"monthly" => 2,
|
||||
_ => 3,
|
||||
}
|
||||
};
|
||||
(attribution, duration)
|
||||
}
|
||||
|
||||
fn is_week_window(seconds: u64) -> bool {
|
||||
@@ -687,7 +825,7 @@ impl fmt::Debug for ProviderExport {
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AccountDirectoryItem {
|
||||
pub account: AccountRecord,
|
||||
pub status: AccountStatus,
|
||||
pub projection: gateway_core::engine::credential::AccountStatusProjection,
|
||||
pub usage: Option<super::accounts::AccountUsage>,
|
||||
pub quota: ProviderQuota,
|
||||
}
|
||||
|
||||
@@ -13,11 +13,11 @@ use crate::model::observability::{
|
||||
CalculatedBillingBreakdown, DashboardWireProfile, ProviderBillingInput,
|
||||
};
|
||||
use crate::model::provider_credentials::{
|
||||
AuthorizationStarted, CompleteAuthorization, PendingAuthorizationMutation,
|
||||
PrepareCredentialImport, PrepareCredentialRefresh, PrepareCredentialRotation,
|
||||
PreparedAuthorizationCommit, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest,
|
||||
AuthorizationStarted, CompleteAuthorization, ConsumeProviderResetCredit,
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCommit, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, ProviderExport, ProviderExportCredentialInput, ProviderModels,
|
||||
ProviderQuota, ProviderQuotaRequest, ProviderResetCreditResult, ProviderResetCredits,
|
||||
};
|
||||
|
||||
/// Provider 管理失败的稳定分类。
|
||||
@@ -28,26 +28,56 @@ pub enum ProviderAdminErrorKind {
|
||||
NotFound,
|
||||
Conflict,
|
||||
Unavailable,
|
||||
CredentialRefreshRequired,
|
||||
BadGateway,
|
||||
Internal,
|
||||
}
|
||||
|
||||
/// 不泄漏 OAuth material 与 Provider 响应体的管理错误。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
/// 不携带 OAuth 请求材料的管理错误。
|
||||
///
|
||||
/// `message` 用于当前认证管理请求展示;上游没有标准 `error.message` 时,
|
||||
/// Provider 可以按其官方契约回退为完整的非成功响应正文。
|
||||
#[derive(Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("provider admin operation failed: {kind:?}")]
|
||||
pub struct ProviderAdminError {
|
||||
kind: ProviderAdminErrorKind,
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ProviderAdminError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderAdminError")
|
||||
.field("kind", &self.kind)
|
||||
.field("message", &self.message.as_ref().map(|_| "<redacted>"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProviderAdminError {
|
||||
#[must_use]
|
||||
pub const fn new(kind: ProviderAdminErrorKind) -> Self {
|
||||
Self { kind }
|
||||
Self {
|
||||
kind,
|
||||
message: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_message(mut self, message: impl Into<String>) -> Self {
|
||||
self.message = Some(message.into());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> ProviderAdminErrorKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn message(&self) -> Option<&str> {
|
||||
self.message.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// 一个具体 Provider 对管理控制面提供的解析、验证、上游交互与运行时资源回收能力。
|
||||
@@ -116,6 +146,22 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
request: ProviderQuotaRequest,
|
||||
) -> Result<ProviderQuota, ProviderAdminError>;
|
||||
|
||||
/// 查询 Provider 主动额度重置卡;不支持该能力的 Provider 使用默认拒绝。
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
_account_id: &ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
/// 消费 Provider 主动额度重置卡;不支持该能力的 Provider 使用默认拒绝。
|
||||
async fn consume_reset_credit(
|
||||
&self,
|
||||
_command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, ProviderAdminError> {
|
||||
Err(ProviderAdminError::new(ProviderAdminErrorKind::Unsupported))
|
||||
}
|
||||
|
||||
async fn models(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
|
||||
@@ -10,9 +10,14 @@ use chrono::{DateTime, Utc};
|
||||
use super::backup::BackupStorePorts;
|
||||
use crate::model::{
|
||||
MutationContext, Revision,
|
||||
account_groups::{
|
||||
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, DeleteAccountGroup,
|
||||
NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
|
||||
},
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
AccountListQuery, AccountPage, AccountPageItem, AccountUpdateResult, AccountUsage,
|
||||
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
client_keys::{
|
||||
@@ -83,7 +88,7 @@ pub type AdminStoreResult<T> = Result<T, AdminStoreError>;
|
||||
pub trait AccountStore: Send + Sync {
|
||||
async fn list_accounts(&self, query: AccountListQuery) -> AdminStoreResult<AccountPage>;
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>>;
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>>;
|
||||
|
||||
async fn load_account_usage(
|
||||
&self,
|
||||
@@ -138,11 +143,23 @@ pub trait AccountStore: Send + Sync {
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<CredentialMutationResult>;
|
||||
|
||||
async fn set_account_enabled(
|
||||
async fn update_account(
|
||||
&self,
|
||||
command: SetAccountEnabled,
|
||||
command: UpdateAccount,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Revision>;
|
||||
) -> AdminStoreResult<AccountUpdateResult>;
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &gateway_core::engine::credential::ProviderAccountId,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult>;
|
||||
|
||||
async fn batch_update_accounts(
|
||||
&self,
|
||||
command: BatchUpdateAccounts,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountsUpdateResult>;
|
||||
|
||||
async fn delete_accounts(
|
||||
&self,
|
||||
@@ -215,10 +232,48 @@ pub trait ClientKeyStore: Send + Sync {
|
||||
) -> AdminStoreResult<Revision>;
|
||||
}
|
||||
|
||||
/// Provider-neutral account group management transactions.
|
||||
#[async_trait]
|
||||
pub trait AccountGroupStore: Send + Sync {
|
||||
async fn list_account_groups(
|
||||
&self,
|
||||
query: AccountGroupListQuery,
|
||||
) -> AdminStoreResult<AccountGroupPage>;
|
||||
|
||||
async fn create_account_group(
|
||||
&self,
|
||||
command: NewAccountGroup,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
|
||||
async fn update_account_group(
|
||||
&self,
|
||||
command: UpdateAccountGroup,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
|
||||
async fn set_account_group_enabled(
|
||||
&self,
|
||||
command: SetAccountGroupEnabled,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
|
||||
async fn delete_account_group(
|
||||
&self,
|
||||
command: DeleteAccountGroup,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation>;
|
||||
}
|
||||
|
||||
/// 用量、趋势、诊断与运维错误的只读能力。
|
||||
#[async_trait]
|
||||
pub trait ObservabilityStore: Send + Sync {
|
||||
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation>;
|
||||
/// 返回历史统计区间和指定观测时刻下的实时账号状态。
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation>;
|
||||
|
||||
/// 返回 Dashboard 可选的实时槽位事实。
|
||||
///
|
||||
@@ -297,6 +352,7 @@ pub trait SettingsStore: Send + Sync {
|
||||
#[derive(Clone)]
|
||||
pub struct AdminStorePorts {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
account_groups: Arc<dyn AccountGroupStore>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
@@ -308,6 +364,7 @@ impl AdminStorePorts {
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
account_groups: Arc<dyn AccountGroupStore>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
@@ -316,6 +373,7 @@ impl AdminStorePorts {
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
account_groups,
|
||||
auth,
|
||||
client_keys,
|
||||
observability,
|
||||
@@ -329,6 +387,11 @@ impl AdminStorePorts {
|
||||
self.accounts.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn account_groups(&self) -> Arc<dyn AccountGroupStore> {
|
||||
self.account_groups.clone()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn auth(&self) -> Arc<dyn AuthStore> {
|
||||
self.auth.clone()
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
//! Account group management use cases.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use gateway_core::routing::{AccountGroupId, snapshot::SnapshotControl};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, MutationContext,
|
||||
account_groups::{
|
||||
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, CreateAccountGroup,
|
||||
DeleteAccountGroup, NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
|
||||
},
|
||||
},
|
||||
ports::store::AccountGroupStore,
|
||||
};
|
||||
|
||||
use super::{map_store_error, publish_committed};
|
||||
|
||||
/// API-facing account group management service.
|
||||
#[async_trait]
|
||||
pub trait AccountGroupService: Send + Sync {
|
||||
async fn list(&self, query: AccountGroupListQuery) -> Result<AccountGroupPage, AdminError>;
|
||||
async fn create(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: CreateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError>;
|
||||
async fn update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError>;
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: SetAccountGroupEnabled,
|
||||
) -> Result<AccountGroupMutation, AdminError>;
|
||||
async fn delete(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: DeleteAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError>;
|
||||
}
|
||||
|
||||
pub(crate) struct DefaultAccountGroupService {
|
||||
store: Arc<dyn AccountGroupStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
}
|
||||
|
||||
impl DefaultAccountGroupService {
|
||||
#[must_use]
|
||||
pub(crate) fn new(
|
||||
store: Arc<dyn AccountGroupStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
) -> Self {
|
||||
Self { store, snapshot }
|
||||
}
|
||||
|
||||
async fn publish(
|
||||
&self,
|
||||
result: Result<AccountGroupMutation, crate::ports::store::AdminStoreError>,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
let result = result.map_err(|error| map_store_error(error, "account group"))?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountGroupService for DefaultAccountGroupService {
|
||||
async fn list(&self, query: AccountGroupListQuery) -> Result<AccountGroupPage, AdminError> {
|
||||
self.store
|
||||
.list_account_groups(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account group"))
|
||||
}
|
||||
|
||||
async fn create(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: CreateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
let id = AccountGroupId::new(format!("grp_{}", Uuid::now_v7().simple()))
|
||||
.map_err(|_| AdminError::internal("Failed to create account group ID"))?;
|
||||
self.publish(
|
||||
self.store
|
||||
.create_account_group(
|
||||
NewAccountGroup {
|
||||
id,
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
color: command.color,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
self.publish(self.store.update_account_group(command, context).await)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn set_enabled(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: SetAccountGroupEnabled,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
self.publish(self.store.set_account_group_enabled(command, context).await)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: DeleteAccountGroup,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
self.publish(self.store.delete_account_group(command, context).await)
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -18,19 +18,18 @@ use crate::{
|
||||
AdminError, MutationContext,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent, AccountConnectionTestEventStream, AccountListQuery,
|
||||
AccountRecord, AccountStatus, AccountUsageWindowQuery,
|
||||
AccountPageItem, AccountUpdateResult, AccountUsageWindowQuery, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, UpdateAccount,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
PrepareCredentialRefresh, ProviderModels, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderQuotaWindow,
|
||||
ConsumeProviderResetCredit, PrepareCredentialRefresh, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow, ProviderResetCreditResult,
|
||||
ProviderResetCredits, QuotaLocalUsageAttribution,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
provider::ProviderAdminRegistry,
|
||||
store::{AccountStore, SettingsStore},
|
||||
},
|
||||
ports::{provider::ProviderAdminRegistry, store::AccountStore},
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -57,12 +56,50 @@ pub trait AccountsService: Send + Sync {
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError>;
|
||||
|
||||
async fn recover(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError>;
|
||||
|
||||
async fn update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError>;
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: BatchUpdateAccounts,
|
||||
) -> Result<AccountsUpdateResult, AdminError>;
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
refresh: bool,
|
||||
) -> Result<AccountDirectoryItem, AdminError>;
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
_context: &MutationContext,
|
||||
_account_id: ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, AdminError> {
|
||||
Err(AdminError::invalid(
|
||||
"Provider reset credits are not supported",
|
||||
))
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
&self,
|
||||
_context: &MutationContext,
|
||||
_command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, AdminError> {
|
||||
Err(AdminError::invalid(
|
||||
"Provider reset credits are not supported",
|
||||
))
|
||||
}
|
||||
|
||||
async fn models(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -78,34 +115,46 @@ pub trait AccountsService: Send + Sync {
|
||||
|
||||
pub(crate) struct DefaultAccountsService {
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
reset_credit_locks:
|
||||
Arc<futures::lock::Mutex<BTreeMap<ProviderAccountId, Arc<futures::lock::Mutex<()>>>>>,
|
||||
}
|
||||
|
||||
impl DefaultAccountsService {
|
||||
#[must_use]
|
||||
pub(crate) const fn new(
|
||||
pub(crate) fn new(
|
||||
accounts: Arc<dyn AccountStore>,
|
||||
settings: Arc<dyn SettingsStore>,
|
||||
providers: ProviderAdminRegistry,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
probe: Arc<dyn AccountProbe>,
|
||||
) -> Self {
|
||||
Self {
|
||||
accounts,
|
||||
settings,
|
||||
providers,
|
||||
snapshot,
|
||||
probe,
|
||||
reset_credit_locks: Arc::new(futures::lock::Mutex::new(BTreeMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
async fn reset_credit_lock(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Arc<futures::lock::Mutex<()>> {
|
||||
let mut locks = self.reset_credit_locks.lock().await;
|
||||
Arc::clone(
|
||||
locks
|
||||
.entry(account_id.clone())
|
||||
.or_insert_with(|| Arc::new(futures::lock::Mutex::new(()))),
|
||||
)
|
||||
}
|
||||
|
||||
async fn load_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<AccountRecord, AdminError> {
|
||||
) -> Result<AccountPageItem, AdminError> {
|
||||
self.accounts
|
||||
.load_account(account_id.as_str())
|
||||
.await
|
||||
@@ -118,32 +167,32 @@ impl DefaultAccountsService {
|
||||
account_id: &ProviderAccountId,
|
||||
) -> Result<
|
||||
(
|
||||
AccountRecord,
|
||||
AccountPageItem,
|
||||
Arc<dyn crate::ports::provider::ProviderAdmin>,
|
||||
),
|
||||
AdminError,
|
||||
> {
|
||||
let account = self.load_account(account_id).await?;
|
||||
let item = self.load_account(account_id).await?;
|
||||
let provider = self
|
||||
.providers
|
||||
.require(&account.provider_kind)
|
||||
.require(&item.account.provider_kind)
|
||||
.map_err(|error| map_provider_error(error, "provider account"))?;
|
||||
Ok((account, provider))
|
||||
Ok((item, provider))
|
||||
}
|
||||
|
||||
async fn attach_quota_local_usage(
|
||||
&self,
|
||||
accounts: &[AccountRecord],
|
||||
accounts: &[AccountPageItem],
|
||||
quotas: &mut [ProviderQuota],
|
||||
) -> Result<(), AdminError> {
|
||||
let windows = accounts
|
||||
.iter()
|
||||
.zip(quotas.iter())
|
||||
.flat_map(|(account, quota)| {
|
||||
.flat_map(|(item, quota)| {
|
||||
quota
|
||||
.windows
|
||||
.iter()
|
||||
.filter_map(|window| quota_usage_window(&account.id, window))
|
||||
.filter_map(|window| quota_usage_window(&item.account.id, window))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if windows.is_empty() {
|
||||
@@ -157,10 +206,12 @@ impl DefaultAccountsService {
|
||||
.into_iter()
|
||||
.map(|result| ((result.account_id, result.key), result.usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (account, quota) in accounts.iter().zip(quotas) {
|
||||
for (item, quota) in accounts.iter().zip(quotas) {
|
||||
for window in &mut quota.windows {
|
||||
if window.local_usage.is_none() {
|
||||
let key = (account.id.clone(), window.key.clone());
|
||||
if window.local_usage.is_none()
|
||||
&& window.local_usage_attribution == QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
let key = (item.account.id.clone(), window.key.clone());
|
||||
if let Some(usage) = usage_by_window.get(&key) {
|
||||
window.local_usage = Some(usage.clone());
|
||||
}
|
||||
@@ -175,33 +226,19 @@ impl DefaultAccountsService {
|
||||
account_id: &ProviderAccountId,
|
||||
refresh_quota: bool,
|
||||
) -> Result<AccountDirectoryItem, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(account_id).await?;
|
||||
let settings = self
|
||||
.settings
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "runtime settings"))?;
|
||||
let (stored, provider) = self.provider_for_account(account_id).await?;
|
||||
let account = &stored.account;
|
||||
let now = Utc::now();
|
||||
let retained_range = retained_usage_range(now, settings.usage_retention_days);
|
||||
let rolling_range = TimeRange {
|
||||
start: now - Duration::hours(24),
|
||||
end: now,
|
||||
};
|
||||
let ids = vec![account.id.clone()];
|
||||
let (usage, rolling_usage) = futures::try_join!(
|
||||
async {
|
||||
self.accounts
|
||||
.load_account_usage(retained_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account usage"))
|
||||
},
|
||||
async {
|
||||
self.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))
|
||||
},
|
||||
)?;
|
||||
let rolling_usage = self
|
||||
.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))?;
|
||||
let rolling_usage = rolling_usage.into_iter().next();
|
||||
let mut quota = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
@@ -212,14 +249,15 @@ impl DefaultAccountsService {
|
||||
.await
|
||||
.map_err(|error| map_provider_error(error, "provider quota"))?;
|
||||
self.attach_quota_local_usage(
|
||||
std::slice::from_ref(&account),
|
||||
std::slice::from_ref(&stored),
|
||||
std::slice::from_mut(&mut quota),
|
||||
)
|
||||
.await?;
|
||||
let usage = quota.representative_window_usage().cloned();
|
||||
Ok(AccountDirectoryItem {
|
||||
status: account_status(&account, quota.limit_reached, now),
|
||||
usage: usage.into_iter().next(),
|
||||
account,
|
||||
projection: stored.projection,
|
||||
usage,
|
||||
account: stored.account,
|
||||
quota,
|
||||
})
|
||||
}
|
||||
@@ -228,13 +266,12 @@ impl DefaultAccountsService {
|
||||
#[async_trait]
|
||||
impl AccountsService for DefaultAccountsService {
|
||||
async fn list(&self, query: AccountListQuery) -> Result<AccountDirectoryPage, AdminError> {
|
||||
let (page, settings) = futures::try_join!(
|
||||
self.accounts.list_accounts(query),
|
||||
self.settings.load_runtime_settings(),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "account directory"))?;
|
||||
let page = self
|
||||
.accounts
|
||||
.list_accounts(query)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "account directory"))?;
|
||||
let now = Utc::now();
|
||||
let retained_range = retained_usage_range(now, settings.usage_retention_days);
|
||||
let rolling_range = TimeRange {
|
||||
start: now - Duration::hours(24),
|
||||
end: now,
|
||||
@@ -242,22 +279,19 @@ impl AccountsService for DefaultAccountsService {
|
||||
let ids = page
|
||||
.items
|
||||
.iter()
|
||||
.map(|account| account.id.clone())
|
||||
.map(|item| item.account.id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let (usage, rolling_usage) = futures::try_join!(
|
||||
self.accounts.load_account_usage(retained_range, &ids),
|
||||
self.accounts.load_account_usage(rolling_range, &ids),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "account usage"))?;
|
||||
let usage = usage
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let rolling_usage = self
|
||||
.accounts
|
||||
.load_account_usage(rolling_range, &ids)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "rolling account usage"))?;
|
||||
let rolling_usage = rolling_usage
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|account| async {
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|item| async {
|
||||
let account = &item.account;
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
@@ -301,11 +335,14 @@ impl AccountsService for DefaultAccountsService {
|
||||
.items
|
||||
.into_iter()
|
||||
.zip(quotas)
|
||||
.map(|(account, quota)| AccountDirectoryItem {
|
||||
status: account_status(&account, quota.limit_reached, now),
|
||||
usage: usage.get(&account.id).cloned(),
|
||||
account,
|
||||
quota,
|
||||
.map(|(item, quota)| {
|
||||
let usage = quota.representative_window_usage().cloned();
|
||||
AccountDirectoryItem {
|
||||
usage,
|
||||
account: item.account,
|
||||
projection: item.projection,
|
||||
quota,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Ok(AccountDirectoryPage {
|
||||
@@ -331,7 +368,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
for account_id in account_ids {
|
||||
let account = self.load_account(&account_id).await?;
|
||||
grouped
|
||||
.entry(account.provider_kind)
|
||||
.entry(account.account.provider_kind)
|
||||
.or_default()
|
||||
.push(account_id);
|
||||
}
|
||||
@@ -374,7 +411,8 @@ impl AccountsService for DefaultAccountsService {
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let (stored, provider) = self.provider_for_account(&account_id).await?;
|
||||
let account = stored.account;
|
||||
let prepared = provider
|
||||
.prepare_refresh(PrepareCredentialRefresh {
|
||||
account: account.clone(),
|
||||
@@ -400,6 +438,98 @@ impl AccountsService for DefaultAccountsService {
|
||||
})
|
||||
}
|
||||
|
||||
async fn recover(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<AccountRefreshResult, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let result = self
|
||||
.accounts
|
||||
.recover_account(&account_id, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account recovery"))?;
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
let account = self.load_directory_item(&account_id, false).await?;
|
||||
Ok(AccountRefreshResult {
|
||||
config_revision: result.config_revision,
|
||||
account,
|
||||
})
|
||||
}
|
||||
|
||||
async fn update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: UpdateAccount,
|
||||
) -> Result<AccountUpdateResult, AdminError> {
|
||||
let account_id = ProviderAccountId::new(command.account_id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
let enabled = command.enabled;
|
||||
let result = self
|
||||
.accounts
|
||||
.update_account(command, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider account"))?;
|
||||
if !enabled {
|
||||
provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn batch_update(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: BatchUpdateAccounts,
|
||||
) -> Result<AccountsUpdateResult, AdminError> {
|
||||
let account_ids = command
|
||||
.account_ids
|
||||
.iter()
|
||||
.map(|id| {
|
||||
ProviderAccountId::new(id.clone())
|
||||
.map_err(|_| AdminError::invalid("Invalid provider account ID"))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut providers = BTreeMap::<
|
||||
ProviderKind,
|
||||
(
|
||||
Arc<dyn crate::ports::provider::ProviderAdmin>,
|
||||
Vec<ProviderAccountId>,
|
||||
),
|
||||
>::new();
|
||||
for account_id in &account_ids {
|
||||
let (item, provider) = self.provider_for_account(account_id).await?;
|
||||
providers
|
||||
.entry(item.account.provider_kind)
|
||||
.or_insert_with(|| (provider, Vec::new()))
|
||||
.1
|
||||
.push(account_id.clone());
|
||||
}
|
||||
let enabled = command.enabled;
|
||||
let result = self
|
||||
.accounts
|
||||
.batch_update_accounts(command, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "provider accounts"))?;
|
||||
for (provider, provider_ids) in providers.values() {
|
||||
if !enabled {
|
||||
for account_id in provider_ids {
|
||||
provider.account_unavailable(account_id).await;
|
||||
}
|
||||
}
|
||||
provider.account_facts_changed(provider_ids).await;
|
||||
}
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn quota(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -408,6 +538,57 @@ impl AccountsService for DefaultAccountsService {
|
||||
self.load_directory_item(account_id, refresh).await
|
||||
}
|
||||
|
||||
async fn reset_credits(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
account_id: ProviderAccountId,
|
||||
) -> Result<ProviderResetCredits, AdminError> {
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.reset_credits(&account_id).await {
|
||||
Ok(credits) => Ok(credits),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.reset_credits(&account_id)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset credits")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, AdminError> {
|
||||
let account_id = command.account_id.clone();
|
||||
// 覆盖 credential refresh + 同键重试的完整账号级临界区,避免 401 两次调用
|
||||
// 之间插入另一笔不可逆消费。
|
||||
let lock = self.reset_credit_lock(&account_id).await;
|
||||
let _guard = lock.lock().await;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
match provider.consume_reset_credit(command.clone()).await {
|
||||
Ok(result) => Ok(result),
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired =>
|
||||
{
|
||||
self.refresh(context, account_id.clone()).await?;
|
||||
let (_, provider) = self.provider_for_account(&account_id).await?;
|
||||
provider
|
||||
.consume_reset_credit(command)
|
||||
.await
|
||||
.map_err(map_reset_credits_error_after_refresh)
|
||||
}
|
||||
Err(error) => Err(map_provider_error(error, "provider reset-credit consume")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn models(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -425,8 +606,8 @@ impl AccountsService for DefaultAccountsService {
|
||||
account_id: ProviderAccountId,
|
||||
upstream_model: UpstreamModelId,
|
||||
) -> Result<AccountConnectionTestEventStream, AdminError> {
|
||||
let (account, provider) = self.provider_for_account(&account_id).await?;
|
||||
let initial_status = account_status(&account, false, Utc::now());
|
||||
let (stored, provider) = self.provider_for_account(&account_id).await?;
|
||||
let account = stored.account;
|
||||
let model = upstream_model.as_str().to_owned();
|
||||
let operation = provider
|
||||
.connection_test_operation(&upstream_model, CONNECTION_TEST_INPUT)
|
||||
@@ -443,8 +624,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
},
|
||||
];
|
||||
let probe = Arc::clone(&self.probe);
|
||||
let accounts = Arc::clone(&self.accounts);
|
||||
let stored_account_id = account.id.clone();
|
||||
let terminal = futures::stream::once(async move {
|
||||
let result = probe
|
||||
.probe(AccountProbeRequest {
|
||||
@@ -454,22 +633,12 @@ impl AccountsService for DefaultAccountsService {
|
||||
operation,
|
||||
})
|
||||
.await;
|
||||
let status = accounts
|
||||
.load_account(&stored_account_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map_or(initial_status, |account| {
|
||||
account_status(&account, false, Utc::now())
|
||||
});
|
||||
match result {
|
||||
Ok(result) => result
|
||||
.text
|
||||
.into_iter()
|
||||
.map(|text| AccountConnectionTestEvent::Content { text })
|
||||
.chain(std::iter::once(AccountConnectionTestEvent::Completed {
|
||||
account_status: status,
|
||||
}))
|
||||
.chain(std::iter::once(AccountConnectionTestEvent::Completed))
|
||||
.collect(),
|
||||
Err(error) => {
|
||||
let upstream_status = error
|
||||
@@ -483,11 +652,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
let upstream_body = error
|
||||
.upstream_response()
|
||||
.map(|response| String::from_utf8_lossy(response.body()).into_owned());
|
||||
let message = upstream_body
|
||||
.as_deref()
|
||||
.filter(|body| !body.is_empty())
|
||||
.unwrap_or_else(|| error.client_message())
|
||||
.to_owned();
|
||||
let message = error.client_message().to_owned();
|
||||
vec![AccountConnectionTestEvent::Failed {
|
||||
message,
|
||||
provider_error_code: error.client_error_code().map(ToOwned::to_owned),
|
||||
@@ -495,7 +660,6 @@ impl AccountsService for DefaultAccountsService {
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status: status,
|
||||
}]
|
||||
}
|
||||
}
|
||||
@@ -505,6 +669,19 @@ impl AccountsService for DefaultAccountsService {
|
||||
}
|
||||
}
|
||||
|
||||
fn map_reset_credits_error_after_refresh(
|
||||
error: crate::ports::provider::ProviderAdminError,
|
||||
) -> AdminError {
|
||||
if error.kind() == crate::ports::provider::ProviderAdminErrorKind::CredentialRefreshRequired {
|
||||
return AdminError::bad_gateway(
|
||||
error
|
||||
.message()
|
||||
.unwrap_or("OpenAI reset-credit upstream rejected the refreshed credential"),
|
||||
);
|
||||
}
|
||||
map_provider_error(error, "provider reset credits")
|
||||
}
|
||||
|
||||
/// 账号目录中单个账号 quota 读取失败时使用的空额度投影。
|
||||
fn empty_quota() -> ProviderQuota {
|
||||
ProviderQuota {
|
||||
@@ -516,54 +693,13 @@ fn empty_quota() -> ProviderQuota {
|
||||
}
|
||||
}
|
||||
|
||||
/// 互斥页面状态:availability 终态优先,quota 快照级 `limit_reached` 派生
|
||||
/// `rate_limited`,`Unknown` 保守映射 `invalid`(不可调度,不显示为 active)。
|
||||
fn account_status(
|
||||
account: &AccountRecord,
|
||||
quota_limit_reached: bool,
|
||||
now: chrono::DateTime<Utc>,
|
||||
) -> AccountStatus {
|
||||
if !account.enabled {
|
||||
AccountStatus::Disabled
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Banned {
|
||||
AccountStatus::Banned
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::QuotaExhausted {
|
||||
AccountStatus::QuotaExhausted
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Invalid
|
||||
) {
|
||||
AccountStatus::Invalid
|
||||
} else if matches!(
|
||||
account.availability,
|
||||
crate::model::accounts::AccountAvailability::Expired
|
||||
) || account
|
||||
.access_token_expires_at
|
||||
.is_some_and(|expires_at| expires_at <= now)
|
||||
{
|
||||
AccountStatus::Expired
|
||||
} else if quota_limit_reached {
|
||||
AccountStatus::RateLimited
|
||||
} else if account.availability == crate::model::accounts::AccountAvailability::Unknown {
|
||||
AccountStatus::Invalid
|
||||
} else {
|
||||
AccountStatus::Active
|
||||
}
|
||||
}
|
||||
|
||||
fn retained_usage_range(now: chrono::DateTime<Utc>, retention_days: u32) -> TimeRange {
|
||||
// 账号留存投影是内部查询,不受外部观测接口 366 天上限约束。
|
||||
TimeRange {
|
||||
start: now - Duration::days(i64::from(retention_days)),
|
||||
end: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_usage_window(
|
||||
account_id: &str,
|
||||
window: &ProviderQuotaWindow,
|
||||
) -> Option<AccountUsageWindowQuery> {
|
||||
if window.local_usage.is_some() {
|
||||
if window.local_usage.is_some()
|
||||
|| window.local_usage_attribution != QuotaLocalUsageAttribution::AccountWide
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let reset_at = window.reset_at?;
|
||||
|
||||
@@ -97,7 +97,7 @@ impl ClientKeyService for DefaultClientKeyService {
|
||||
id,
|
||||
name: command.name,
|
||||
label: command.label,
|
||||
provider_kind: command.provider_kind,
|
||||
group_ids: command.group_ids,
|
||||
limits: command.limits,
|
||||
plaintext: plaintext.clone(),
|
||||
},
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! 管理控制面的用例实现。
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backup;
|
||||
@@ -13,11 +14,11 @@ pub mod xai;
|
||||
use crate::{
|
||||
model::{
|
||||
AdminError, AdminErrorKind, MutationContext,
|
||||
accounts::{DeleteAccounts, SetAccountEnabled},
|
||||
accounts::DeleteAccounts,
|
||||
provider_credentials::{
|
||||
AuthorizationMutationTarget, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialDetails, CredentialMutation, CredentialMutationResult,
|
||||
CredentialRotationCommit, PendingAuthorizationMutation, PreparedAuthorizationCommit,
|
||||
CredentialDetails, CredentialMutationResult, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
StartAuthorization,
|
||||
},
|
||||
@@ -56,9 +57,15 @@ fn map_provider_error(
|
||||
ProviderAdminErrorKind::NotFound => AdminErrorKind::NotFound,
|
||||
ProviderAdminErrorKind::Conflict => AdminErrorKind::Conflict,
|
||||
ProviderAdminErrorKind::Unavailable => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::CredentialRefreshRequired => AdminErrorKind::Unavailable,
|
||||
ProviderAdminErrorKind::BadGateway => AdminErrorKind::BadGateway,
|
||||
ProviderAdminErrorKind::Internal => AdminErrorKind::Internal,
|
||||
};
|
||||
AdminError::new(kind, format!("{resource} operation failed"))
|
||||
let message = error
|
||||
.message()
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| format!("{resource} operation failed"));
|
||||
AdminError::new(kind, message)
|
||||
}
|
||||
|
||||
async fn publish_committed(
|
||||
@@ -280,44 +287,6 @@ async fn commit_credential_refresh(
|
||||
}
|
||||
}
|
||||
|
||||
async fn set_credential_enabled(
|
||||
accounts: &dyn AccountStore,
|
||||
provider: &dyn ProviderAdmin,
|
||||
command: CredentialMutation,
|
||||
enabled: bool,
|
||||
resource: &'static str,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
required_credential(
|
||||
accounts,
|
||||
provider.provider_kind(),
|
||||
&command.account_id,
|
||||
resource,
|
||||
)
|
||||
.await?;
|
||||
let account_id = command.account_id;
|
||||
let revision = accounts
|
||||
.set_account_enabled(
|
||||
SetAccountEnabled {
|
||||
account_id: account_id.as_str().to_owned(),
|
||||
enabled,
|
||||
},
|
||||
&command.context,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, resource))?;
|
||||
if !enabled {
|
||||
provider.account_unavailable(&account_id).await;
|
||||
}
|
||||
provider
|
||||
.account_facts_changed(std::slice::from_ref(&account_id))
|
||||
.await;
|
||||
Ok(CredentialMutationResult {
|
||||
config_revision: revision,
|
||||
account_id,
|
||||
credential_revision: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_credentials(
|
||||
accounts: &dyn AccountStore,
|
||||
provider: &dyn ProviderAdmin,
|
||||
|
||||
@@ -98,10 +98,11 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range: TimeRange,
|
||||
kind: TrendKind,
|
||||
) -> Result<DashboardResult, AdminError> {
|
||||
let observed_at = Utc::now();
|
||||
let (mut observation, settings, runtime_slots) = futures::try_join!(
|
||||
self.store.dashboard_summary(range),
|
||||
self.store.dashboard_summary(range, observed_at),
|
||||
self.settings.load_runtime_settings(),
|
||||
self.store.dashboard_runtime_slots(range.end),
|
||||
self.store.dashboard_runtime_slots(observed_at),
|
||||
)
|
||||
.map_err(|error| map_store_error(error, "dashboard"))?;
|
||||
self.enrich_billing(&mut observation.recent_requests);
|
||||
@@ -112,15 +113,10 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let today =
|
||||
dashboard_period_metrics(&observation.trend, today_start, observation.range.end);
|
||||
let yesterday = dashboard_period_metrics(&observation.trend, yesterday_start, today_start);
|
||||
let total_billing_usd = observation
|
||||
.attempts
|
||||
.costs
|
||||
.iter()
|
||||
.find(|cost| cost.currency == "USD")
|
||||
.map(|cost| cost.amount.clone());
|
||||
let total_billing_usd = observation.totals.billing_usd.clone();
|
||||
let total_cached_token_rate = rate_or_zero(
|
||||
observation.requests.cached_tokens,
|
||||
observation.requests.input_tokens,
|
||||
observation.totals.cached_tokens,
|
||||
observation.totals.input_tokens,
|
||||
);
|
||||
let average_first_token_latency_ms = average(
|
||||
observation.requests.first_token_latency_sum_ms,
|
||||
@@ -130,12 +126,20 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
let health_timeline = health_timeline_at(&observation.trend, Utc::now());
|
||||
let wire_profiles = self.providers.dashboard_wire_profiles();
|
||||
let max_concurrent_per_account = u64::from(settings.max_concurrent_per_account);
|
||||
let active_accounts = runtime_slots
|
||||
.as_ref()
|
||||
.map_or(observation.provider_accounts.active, |slots| {
|
||||
slots.active_accounts
|
||||
});
|
||||
let total_slots = active_accounts.saturating_mul(max_concurrent_per_account);
|
||||
let total_slots = runtime_slots.as_ref().map_or_else(
|
||||
|| {
|
||||
observation
|
||||
.provider_accounts
|
||||
.normal
|
||||
.saturating_mul(max_concurrent_per_account)
|
||||
},
|
||||
|slots| {
|
||||
slots
|
||||
.inherited_accounts
|
||||
.saturating_mul(max_concurrent_per_account)
|
||||
.saturating_add(slots.overridden_slots)
|
||||
},
|
||||
);
|
||||
let used_slots = runtime_slots.and_then(|slots| slots.used_slots);
|
||||
Ok(DashboardResult {
|
||||
capacity: DashboardCapacity {
|
||||
@@ -489,8 +493,7 @@ impl DefaultObservabilityService {
|
||||
});
|
||||
let quota_used = futures::future::join_all(quota_reads).await;
|
||||
for (account, used_percent) in accounts.iter_mut().zip(quota_used) {
|
||||
account.quota_used_percent = used_percent
|
||||
.or_else(|| (account.availability == "quota_exhausted").then_some(100.0));
|
||||
account.quota_used_percent = used_percent;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ use crate::{
|
||||
provider_credentials::{
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion,
|
||||
CredentialDeletionResult, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutation,
|
||||
CredentialMutationResult, CredentialPage, ImportCredentials, PrepareCredentialImport,
|
||||
PrepareCredentialRotation, ProviderQuotaRequest, RotateCredential, StartAuthorization,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
|
||||
ImportCredentials, PrepareCredentialImport, PrepareCredentialRotation,
|
||||
ProviderQuotaRequest, RotateCredential, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{provider::ProviderAdmin, store::AccountStore},
|
||||
@@ -22,8 +22,7 @@ use crate::{
|
||||
use super::{
|
||||
commit_authorization, commit_credential_rotation, delete_credentials, map_provider_error,
|
||||
map_store_error, pending_authorization, publish_committed, required_credential,
|
||||
set_credential_enabled, validate_authorization_commit, validate_prepared_import,
|
||||
validate_prepared_rotation,
|
||||
validate_authorization_commit, validate_prepared_import, validate_prepared_rotation,
|
||||
};
|
||||
|
||||
/// OpenAI 固定管理路由消费的服务。
|
||||
@@ -50,14 +49,6 @@ pub trait OpenAiService: Send + Sync {
|
||||
&self,
|
||||
command: RotateCredential,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn enable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn disable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn delete(
|
||||
&self,
|
||||
command: CredentialDeletion,
|
||||
@@ -84,25 +75,29 @@ impl DefaultOpenAiService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = self
|
||||
.provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
fn observe_initial_quotas(&self, account_ids: &[ProviderAccountId], request_id: &str) {
|
||||
let provider = Arc::clone(&self.provider);
|
||||
let account_ids = account_ids.to_vec();
|
||||
let request_id = request_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
for account_id in account_ids {
|
||||
if let Err(error) = provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id: account_id.clone(),
|
||||
refresh: true,
|
||||
rolling_usage: None,
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
request_id,
|
||||
account_id = %account_id.as_str(),
|
||||
quota_error = ?error.kind(),
|
||||
"OpenAI initial quota observation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,8 +150,7 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
.await;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
// 导入完成后立即做一次观察;失败不回滚已经提交的 credential。
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id)
|
||||
.await;
|
||||
self.observe_initial_quotas(&result.credential_ids, &context.request_id);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -208,8 +202,7 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
self.observe_initial_quotas(
|
||||
std::slice::from_ref(&result.account_id),
|
||||
&context.request_id,
|
||||
)
|
||||
.await;
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -250,38 +243,6 @@ impl OpenAiService for DefaultOpenAiService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn enable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let result = set_credential_enabled(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.as_ref(),
|
||||
command,
|
||||
true,
|
||||
"OpenAI credential",
|
||||
)
|
||||
.await?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn disable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let result = set_credential_enabled(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.as_ref(),
|
||||
command,
|
||||
false,
|
||||
"OpenAI credential",
|
||||
)
|
||||
.await?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
&self,
|
||||
command: CredentialDeletion,
|
||||
|
||||
@@ -11,8 +11,8 @@ use crate::{
|
||||
provider_credentials::{
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion,
|
||||
CredentialDeletionResult, CredentialImportCommit, CredentialImportResult,
|
||||
CredentialListQuery, CredentialMutation, CredentialMutationResult, CredentialPage,
|
||||
ImportCredentials, PrepareCredentialImport, ProviderQuotaRequest, StartAuthorization,
|
||||
CredentialListQuery, CredentialMutationResult, CredentialPage, ImportCredentials,
|
||||
PrepareCredentialImport, ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
},
|
||||
ports::{provider::ProviderAdmin, store::AccountStore},
|
||||
@@ -20,8 +20,8 @@ use crate::{
|
||||
|
||||
use super::{
|
||||
commit_authorization, delete_credentials, map_provider_error, map_store_error,
|
||||
pending_authorization, publish_committed, set_credential_enabled,
|
||||
validate_authorization_commit, validate_prepared_import,
|
||||
pending_authorization, publish_committed, validate_authorization_commit,
|
||||
validate_prepared_import,
|
||||
};
|
||||
|
||||
/// xAI 固定管理路由消费的服务。
|
||||
@@ -40,14 +40,6 @@ pub trait XaiService: Send + Sync {
|
||||
&self,
|
||||
command: CompleteAuthorization,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn enable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn disable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError>;
|
||||
async fn delete(
|
||||
&self,
|
||||
command: CredentialDeletion,
|
||||
@@ -166,38 +158,6 @@ impl XaiService for DefaultXaiService {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn enable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let result = set_credential_enabled(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.as_ref(),
|
||||
command,
|
||||
true,
|
||||
"xAI credential",
|
||||
)
|
||||
.await?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn disable(
|
||||
&self,
|
||||
command: CredentialMutation,
|
||||
) -> Result<CredentialMutationResult, AdminError> {
|
||||
let result = set_credential_enabled(
|
||||
self.accounts.as_ref(),
|
||||
self.provider.as_ref(),
|
||||
command,
|
||||
false,
|
||||
"xAI credential",
|
||||
)
|
||||
.await?;
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
&self,
|
||||
command: CredentialDeletion,
|
||||
|
||||
@@ -13,9 +13,10 @@ use gateway_admin::{
|
||||
MutationActor, MutationContext, Revision,
|
||||
auth::AdminAuditEvent,
|
||||
backup::{
|
||||
BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage,
|
||||
BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind,
|
||||
ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand,
|
||||
BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery,
|
||||
BackupRecordPage, BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig,
|
||||
BackupTriggerKind, ConnectionTestResult, UpdateBackupScheduleCommand,
|
||||
UpdateBackupStorageCommand,
|
||||
},
|
||||
},
|
||||
ports::{
|
||||
@@ -497,15 +498,23 @@ impl DatabaseDumpPort for FakeDumpPort {
|
||||
/// 内存版对象存储:把上传内容按 key 保存并支持 Head/Delete/预签名。
|
||||
pub(crate) struct FakeObjectStore {
|
||||
pub(crate) objects: Mutex<std::collections::HashMap<String, Vec<u8>>>,
|
||||
upload_error: Option<BackupError>,
|
||||
}
|
||||
|
||||
impl FakeObjectStore {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
objects: Mutex::new(std::collections::HashMap::new()),
|
||||
upload_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 让上传返回指定的脱敏 S3 错误。
|
||||
pub(crate) fn fail_upload(mut self, code: &'static str, message: impl Into<String>) -> Self {
|
||||
self.upload_error = Some(BackupError::new(code, message.into()));
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn object(&self, key: &str) -> Option<Vec<u8>> {
|
||||
self.objects.lock().expect("objects").get(key).cloned()
|
||||
}
|
||||
@@ -530,6 +539,9 @@ impl BackupObjectStorePort for FakeObjectStore {
|
||||
_config: &BackupStorageConfig,
|
||||
request: UploadObjectRequest,
|
||||
) -> Result<(), gateway_admin::model::backup::BackupError> {
|
||||
if let Some(error) = &self.upload_error {
|
||||
return Err(error.clone());
|
||||
}
|
||||
let content = std::fs::read(&request.source).map_err(|_| {
|
||||
gateway_admin::model::backup::BackupError::new(
|
||||
gateway_admin::model::backup::code::S3_UPLOAD_FAILED,
|
||||
|
||||
@@ -6,9 +6,9 @@ use chrono::{Duration, Utc};
|
||||
use gateway_admin::{
|
||||
backup::task::BackupTask,
|
||||
model::backup::{
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand,
|
||||
BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, code,
|
||||
},
|
||||
ports::backup::BackupRepository,
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::engine::CancellationToken;
|
||||
|
||||
@@ -89,6 +89,78 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
assert!(object_store.object(&seed.object_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&CancellationToken::new())
|
||||
.await
|
||||
.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let dump = Arc::new(FakeDumpPort::new());
|
||||
let object_store =
|
||||
Arc::new(FakeObjectStore::new().fail_upload(code::S3_UPLOAD_FAILED, "对象存储拒绝上传"));
|
||||
let seed = BackupRecordSeed {
|
||||
id: backup_id("recovered-upload-failure"),
|
||||
trigger_kind: BackupTriggerKind::Manual,
|
||||
scheduled_at: None,
|
||||
object_key: "codex/2026/08/01/recovered-upload-failure.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
dump.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
cancellation: cancellation.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("stage dump");
|
||||
repository.force_status(&seed.id, BackupStatus::Dumping);
|
||||
|
||||
let task = BackupTask::new(repository.clone(), dump, object_store);
|
||||
task.run_cycle(&cancellation).await.expect("run cycle");
|
||||
|
||||
let records = repository.all_records();
|
||||
let failed = &records[0];
|
||||
assert_eq!(failed.status, BackupStatus::Failed);
|
||||
assert_eq!(failed.error_code.as_deref(), Some(code::S3_UPLOAD_FAILED));
|
||||
assert_eq!(
|
||||
failed.error_message.as_deref(),
|
||||
Some("上传到对象存储失败:对象存储拒绝上传")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_cleans_expired_scheduled_backups() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
|
||||
@@ -6,11 +6,13 @@ use gateway_core::engine::credential::OpaqueProviderData;
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::AdminApiKey,
|
||||
},
|
||||
@@ -134,6 +136,115 @@ fn representative_quota_should_prefer_short_window_and_highest_usage() {
|
||||
assert_eq!(quota.representative_used_percent(), Some(45.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn representative_quota_should_prefer_account_wide_window_over_model_specific_window() {
|
||||
let mut model_specific = quota_window("shortTerm", Some(18_000), Some(64.0));
|
||||
model_specific.local_usage_attribution = QuotaLocalUsageAttribution::Unavailable;
|
||||
let quota = ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![
|
||||
model_specific,
|
||||
quota_window("shortTerm", Some(604_800), Some(5.0)),
|
||||
],
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
assert_eq!(quota.representative_used_percent(), Some(5.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn representative_window_usage_should_accept_account_wide_rolling_window_without_reset() {
|
||||
let usage = AccountUsage {
|
||||
account_id: "acct_xai".to_owned(),
|
||||
request_count: 5,
|
||||
success_count: 5,
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(20),
|
||||
cached_tokens: Some(80),
|
||||
cache_write_tokens: Some(0),
|
||||
reasoning_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: Some(120),
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at: Some(Utc::now()),
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
};
|
||||
let quota = ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "free-rolling-24h".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "日限额".to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(86_400),
|
||||
used_percent: None,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: Some(usage.clone()),
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
assert_eq!(quota.representative_window_usage(), Some(&usage));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_quota_should_project_full_usage_to_only_the_representative_window() {
|
||||
let mut quota = ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![
|
||||
quota_window("monthly", Some(2_592_000), Some(99.0)),
|
||||
quota_window("shortTerm", Some(604_800), Some(80.0)),
|
||||
quota_window("shortTerm", Some(18_000), Some(30.0)),
|
||||
quota_window("shortTerm", Some(14_400), Some(95.0)),
|
||||
],
|
||||
limit_reached: true,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
assert_eq!(quota.representative_used_percent(), Some(100.0));
|
||||
quota.apply_limit_reached_display();
|
||||
|
||||
assert_eq!(quota.windows[0].used_percent, Some(99.0));
|
||||
assert_eq!(quota.windows[1].used_percent, Some(80.0));
|
||||
assert_eq!(quota.windows[2].used_percent, Some(30.0));
|
||||
assert_eq!(quota.windows[3].used_percent, Some(100.0));
|
||||
assert!(quota.windows[3].limit_reached);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausted_quota_should_preserve_the_provider_identified_reached_window() {
|
||||
let mut reached = quota_window("monthly", Some(2_592_000), Some(98.0));
|
||||
reached.limit_reached = true;
|
||||
let mut quota = ProviderQuota {
|
||||
observed_at: None,
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![reached, quota_window("shortTerm", Some(18_000), Some(95.0))],
|
||||
limit_reached: true,
|
||||
provider_data: None,
|
||||
};
|
||||
|
||||
quota.apply_limit_reached_display();
|
||||
|
||||
assert_eq!(quota.windows[0].used_percent, Some(100.0));
|
||||
assert_eq!(quota.windows[1].used_percent, Some(95.0));
|
||||
assert!(!quota.windows[1].limit_reached);
|
||||
}
|
||||
|
||||
fn quota_window(
|
||||
group: &str,
|
||||
window_seconds: Option<u64>,
|
||||
@@ -143,7 +254,10 @@ fn quota_window(
|
||||
key: group.to_owned(),
|
||||
group: group.to_owned(),
|
||||
label: group.to_owned(),
|
||||
source: None,
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at: None,
|
||||
|
||||
@@ -5,7 +5,10 @@ use chrono::{TimeDelta, Utc};
|
||||
use futures::{StreamExt as _, future::BoxFuture};
|
||||
use gateway_core::{
|
||||
engine::{
|
||||
credential::{OpaqueProviderData, ProviderAccountId},
|
||||
credential::{
|
||||
AccountStatusFacts, CredentialState, OpaqueProviderData, ProviderAccountId,
|
||||
QuotaEvidence, QuotaState, resolve_account_status,
|
||||
},
|
||||
probe::{AccountProbe, AccountProbeError, AccountProbeRequest, AccountProbeResult},
|
||||
},
|
||||
error::{ClientVisibleUpstreamError, GatewayError, GatewayErrorKind},
|
||||
@@ -18,22 +21,25 @@ use gateway_admin::{
|
||||
model::{
|
||||
AdminError, MutationContext, Revision,
|
||||
accounts::{
|
||||
AccountAvailability, AccountConnectionTestEvent, AccountListQuery, AccountPage,
|
||||
AccountRecord, AccountSummary, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
AccountConnectionTestEvent, AccountListQuery, AccountPage, AccountPageItem,
|
||||
AccountRecord, AccountSummary, AccountUpdateResult, AccountUsage,
|
||||
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
|
||||
},
|
||||
observability::TimeRange,
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationCommitGuard, AuthorizationCredentialCommit,
|
||||
AuthorizationMutationTarget, AuthorizationStarted, CompleteAuthorization,
|
||||
CredentialCommitGuard, CredentialDetails, CredentialImportCommit,
|
||||
CredentialImportResult, CredentialListQuery, CredentialMutationResult, CredentialPage,
|
||||
CredentialRotationCommit, PendingAuthorizationMutation, PrepareCredentialImport,
|
||||
PrepareCredentialRefresh, PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
ConsumeProviderResetCredit, CredentialCommitGuard, CredentialDetails,
|
||||
CredentialImportCommit, CredentialImportResult, CredentialListQuery,
|
||||
CredentialMutationResult, CredentialPage, CredentialRotationCommit,
|
||||
PendingAuthorizationMutation, PrepareCredentialImport, PrepareCredentialRefresh,
|
||||
PrepareCredentialRotation, PreparedAuthorizationCommit,
|
||||
PreparedAuthorizationCredential, PreparedCredentialCreate, PreparedCredentialImport,
|
||||
PreparedCredentialRotation, PreparedCredentialRotationFacts, ProviderDocument,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModels, ProviderQuota,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderQuotaRequest, ProviderQuotaWindow, ProviderResetCreditResult,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -56,7 +62,7 @@ pub(super) type EventLog = Arc<Mutex<Vec<&'static str>>>;
|
||||
pub(super) struct FakeProviderAdmin {
|
||||
kind: ProviderKind,
|
||||
events: EventLog,
|
||||
failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
failure: Mutex<Option<ProviderAdminError>>,
|
||||
quota_failure: Mutex<Option<ProviderAdminErrorKind>>,
|
||||
pending: Arc<Mutex<Option<PendingAuthorizationMutation>>>,
|
||||
retry_authorization_after_abort: Mutex<bool>,
|
||||
@@ -65,6 +71,7 @@ pub(super) struct FakeProviderAdmin {
|
||||
quota_requests: Mutex<Vec<ProviderQuotaRequest>>,
|
||||
quota: Mutex<ProviderQuota>,
|
||||
current_credential_revision: Mutex<Revision>,
|
||||
reset_credit_commands: Mutex<Vec<ConsumeProviderResetCredit>>,
|
||||
}
|
||||
|
||||
impl FakeProviderAdmin {
|
||||
@@ -81,11 +88,17 @@ impl FakeProviderAdmin {
|
||||
quota_requests: Mutex::new(Vec::new()),
|
||||
quota: Mutex::new(empty_quota()),
|
||||
current_credential_revision: Mutex::new(revision(1)),
|
||||
reset_credit_commands: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn fail_next(&self, kind: ProviderAdminErrorKind) {
|
||||
*self.failure.lock().expect("provider failure") = Some(kind);
|
||||
*self.failure.lock().expect("provider failure") = Some(ProviderAdminError::new(kind));
|
||||
}
|
||||
|
||||
fn fail_next_with_message(&self, kind: ProviderAdminErrorKind, message: &str) {
|
||||
*self.failure.lock().expect("provider failure") =
|
||||
Some(ProviderAdminError::new(kind).with_message(message));
|
||||
}
|
||||
|
||||
pub(super) fn fail_next_quota(&self, kind: ProviderAdminErrorKind) {
|
||||
@@ -109,6 +122,13 @@ impl FakeProviderAdmin {
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn reset_credit_commands(&self) -> Vec<ConsumeProviderResetCredit> {
|
||||
self.reset_credit_commands
|
||||
.lock()
|
||||
.expect("provider reset-credit commands")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn set_quota(&self, quota: ProviderQuota) {
|
||||
*self.quota.lock().expect("provider quota") = quota;
|
||||
}
|
||||
@@ -141,7 +161,7 @@ impl FakeProviderAdmin {
|
||||
|
||||
fn require_available(&self) -> Result<(), ProviderAdminError> {
|
||||
match self.failure.lock().expect("provider failure").take() {
|
||||
Some(kind) => Err(ProviderAdminError::new(kind)),
|
||||
Some(error) => Err(error),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -340,6 +360,22 @@ impl ProviderAdmin for FakeProviderAdmin {
|
||||
Ok(self.quota.lock().expect("provider quota").clone())
|
||||
}
|
||||
|
||||
async fn consume_reset_credit(
|
||||
&self,
|
||||
command: ConsumeProviderResetCredit,
|
||||
) -> Result<ProviderResetCreditResult, ProviderAdminError> {
|
||||
self.record("provider.consume_reset_credit");
|
||||
self.reset_credit_commands
|
||||
.lock()
|
||||
.expect("provider reset-credit commands")
|
||||
.push(command);
|
||||
self.require_available()?;
|
||||
Ok(ProviderResetCreditResult {
|
||||
code: "reset".to_owned(),
|
||||
credit: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn models(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
@@ -378,6 +414,7 @@ pub(super) struct FakeAccountStore {
|
||||
fail_commit: Mutex<bool>,
|
||||
audit_requests: Mutex<Vec<String>>,
|
||||
quota_window_usage: Mutex<Vec<AccountUsageWindowResult>>,
|
||||
quota_window_queries: Mutex<Vec<AccountUsageWindowQuery>>,
|
||||
}
|
||||
|
||||
impl FakeAccountStore {
|
||||
@@ -393,6 +430,7 @@ impl FakeAccountStore {
|
||||
fail_commit: Mutex::new(false),
|
||||
audit_requests: Mutex::new(Vec::new()),
|
||||
quota_window_usage: Mutex::new(Vec::new()),
|
||||
quota_window_queries: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -408,6 +446,13 @@ impl FakeAccountStore {
|
||||
*self.quota_window_usage.lock().expect("quota window usage") = usage;
|
||||
}
|
||||
|
||||
fn quota_window_queries(&self) -> Vec<AccountUsageWindowQuery> {
|
||||
self.quota_window_queries
|
||||
.lock()
|
||||
.expect("quota window queries")
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub(super) fn set_accounts(&self, accounts: Vec<AccountRecord>) {
|
||||
*self.accounts.lock().expect("accounts") = accounts;
|
||||
}
|
||||
@@ -438,6 +483,22 @@ impl FakeAccountStore {
|
||||
.expect("audit requests")
|
||||
.push(context.request_id.clone());
|
||||
}
|
||||
|
||||
fn page_item(account: AccountRecord) -> AccountPageItem {
|
||||
let facts = AccountStatusFacts {
|
||||
enabled: account.enabled,
|
||||
credential_state: account.credential_state,
|
||||
access_token_expires_at: account.access_token_expires_at.map(Into::into),
|
||||
quota: account.quota,
|
||||
rate_limited_until: None,
|
||||
last_error_reason: account.last_error_reason,
|
||||
last_error_message: account.last_error_message.clone(),
|
||||
};
|
||||
AccountPageItem {
|
||||
account,
|
||||
projection: resolve_account_status(&facts, std::time::SystemTime::now()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -445,20 +506,23 @@ impl AccountStore for FakeAccountStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
self.record("store.list_accounts");
|
||||
let accounts = self.accounts.lock().expect("accounts").clone();
|
||||
let total = accounts.len() as u64;
|
||||
Ok(AccountPage {
|
||||
config_revision: revision(1),
|
||||
items: accounts.clone(),
|
||||
total: accounts.len() as u64,
|
||||
items: accounts.into_iter().map(Self::page_item).collect(),
|
||||
total,
|
||||
summary: AccountSummary {
|
||||
total: accounts.len() as u64,
|
||||
active: 1,
|
||||
total,
|
||||
normal: 1,
|
||||
quota_exhausted: 0,
|
||||
unavailable: 0,
|
||||
rate_limited: 0,
|
||||
disabled: 0,
|
||||
error: 0,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(&self, account_id: &str) -> AdminStoreResult<Option<AccountPageItem>> {
|
||||
self.record("store.load_account");
|
||||
// probe 后的账号状态覆盖只对同一 id 生效;其余按账号列表查询。
|
||||
let account = self
|
||||
@@ -475,7 +539,7 @@ impl AccountStore for FakeAccountStore {
|
||||
.find(|account| account.id == account_id)
|
||||
.cloned()
|
||||
});
|
||||
Ok(account)
|
||||
Ok(account.map(Self::page_item))
|
||||
}
|
||||
|
||||
async fn load_account_usage(
|
||||
@@ -488,8 +552,12 @@ impl AccountStore for FakeAccountStore {
|
||||
|
||||
async fn load_account_usage_by_windows(
|
||||
&self,
|
||||
_: &[AccountUsageWindowQuery],
|
||||
windows: &[AccountUsageWindowQuery],
|
||||
) -> AdminStoreResult<Vec<AccountUsageWindowResult>> {
|
||||
*self
|
||||
.quota_window_queries
|
||||
.lock()
|
||||
.expect("quota window queries") = windows.to_vec();
|
||||
Ok(self
|
||||
.quota_window_usage
|
||||
.lock()
|
||||
@@ -626,15 +694,61 @@ impl AccountStore for FakeAccountStore {
|
||||
Ok(rotation_result(command))
|
||||
}
|
||||
|
||||
async fn set_account_enabled(
|
||||
async fn update_account(
|
||||
&self,
|
||||
_: SetAccountEnabled,
|
||||
command: UpdateAccount,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<Revision> {
|
||||
self.record("store.set_enabled");
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
self.record("store.update_account");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
Ok(revision(2))
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: ProviderAccountId::new(command.account_id).expect("account ID"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
self.record("store.recover_account");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
let mut accounts = self.accounts.lock().expect("accounts");
|
||||
if let Some(account) = accounts
|
||||
.iter_mut()
|
||||
.find(|account| account.id == account_id.as_str())
|
||||
{
|
||||
account.enabled = true;
|
||||
account.credential_state = CredentialState::Ready;
|
||||
account.quota = QuotaState::allowed(std::time::SystemTime::now());
|
||||
account.last_error_reason = None;
|
||||
account.last_error_message = None;
|
||||
}
|
||||
Ok(AccountUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_id: account_id.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn batch_update_accounts(
|
||||
&self,
|
||||
command: BatchUpdateAccounts,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<AccountsUpdateResult> {
|
||||
self.record("store.batch_update_accounts");
|
||||
self.record_context(context);
|
||||
self.require_commit()?;
|
||||
Ok(AccountsUpdateResult {
|
||||
config_revision: revision(2),
|
||||
account_ids: command
|
||||
.account_ids
|
||||
.into_iter()
|
||||
.map(|id| ProviderAccountId::new(id).expect("account ID"))
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_accounts(
|
||||
@@ -813,7 +927,11 @@ fn provider_registry_should_reject_duplicate_kind() {
|
||||
async fn connection_test_should_probe_unavailable_account() {
|
||||
let provider = FakeProviderAdmin::new("xai", events());
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.quota = QuotaState::exhausted(
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
std::time::SystemTime::now(),
|
||||
None,
|
||||
);
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
let services =
|
||||
accounts_service_with_probe(provider, store, Arc::new(FailingAccountProbe)).await;
|
||||
@@ -835,7 +953,6 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
message,
|
||||
provider_error_code: Some(code),
|
||||
provider_error_type: Some(error_type),
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
..
|
||||
}) if message == "included usage exhausted"
|
||||
&& code == "usage_exhausted"
|
||||
@@ -844,7 +961,7 @@ async fn connection_test_should_probe_unavailable_account() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_test_rate_limited_probe_preserves_active_account_status() {
|
||||
async fn connection_test_rate_limited_probe_returns_provider_failure() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events);
|
||||
@@ -867,9 +984,10 @@ async fn connection_test_rate_limited_probe_preserves_active_account_status() {
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Active,
|
||||
message,
|
||||
..
|
||||
})
|
||||
if message == "xAI free model quota is exhausted"
|
||||
));
|
||||
}
|
||||
|
||||
@@ -895,10 +1013,7 @@ async fn connection_test_should_preserve_disabled_account_status() {
|
||||
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(AccountConnectionTestEvent::Failed {
|
||||
account_status: gateway_admin::model::accounts::AccountStatus::Disabled,
|
||||
..
|
||||
})
|
||||
Some(AccountConnectionTestEvent::Failed { .. })
|
||||
));
|
||||
}
|
||||
|
||||
@@ -947,8 +1062,8 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(result.config_revision, revision(2));
|
||||
assert_eq!(result.account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
result.account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
result.account.projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
@@ -965,6 +1080,175 @@ async fn accounts_refresh_should_keep_guard_through_store_commit() {
|
||||
assert_eq!(store.audit_requests(), ["refresh-request"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_recover_should_commit_facts_then_return_normal_account() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let mut account = account_record("openai");
|
||||
account.enabled = false;
|
||||
account.credential_state = CredentialState::Invalid;
|
||||
account.quota = QuotaState::exhausted(
|
||||
QuotaEvidence::UsageLimitReached,
|
||||
std::time::SystemTime::now(),
|
||||
None,
|
||||
);
|
||||
account.last_error_reason =
|
||||
Some(gateway_core::engine::credential::AccountErrorReason::CredentialInvalid);
|
||||
account.last_error_message = Some("invalid credential".to_owned());
|
||||
let store = FakeAccountStore::with_account(account, events.clone());
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
|
||||
let result = services
|
||||
.accounts()
|
||||
.recover(
|
||||
&context("recover-request"),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
.expect("recover account");
|
||||
|
||||
assert_eq!(result.config_revision, revision(2));
|
||||
assert!(result.account.account.enabled);
|
||||
assert_eq!(
|
||||
result.account.account.credential_state,
|
||||
CredentialState::Ready
|
||||
);
|
||||
assert_eq!(
|
||||
result.account.account.quota.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Allowed
|
||||
);
|
||||
assert_eq!(
|
||||
result.account.projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.load_account",
|
||||
"store.recover_account",
|
||||
"provider.account_facts_changed",
|
||||
"store.load_account",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["recover-request"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_update_should_commit_then_release_disabled_account_and_publish_facts() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = accounts_service(provider, store.clone()).await;
|
||||
|
||||
let result = services
|
||||
.accounts()
|
||||
.update(
|
||||
&context("update-request"),
|
||||
UpdateAccount {
|
||||
account_id: "acct_test".to_owned(),
|
||||
enabled: false,
|
||||
concurrency_limit: None,
|
||||
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
|
||||
group_ids: Vec::new(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("update account");
|
||||
|
||||
assert_eq!(result.config_revision, revision(2));
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.load_account",
|
||||
"store.update_account",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["update-request"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_update_should_not_notify_provider_when_store_commit_fails() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
store.fail_next_commit();
|
||||
let services = accounts_service(provider, store).await;
|
||||
|
||||
services
|
||||
.accounts()
|
||||
.update(
|
||||
&context("update-failure"),
|
||||
UpdateAccount {
|
||||
account_id: "acct_test".to_owned(),
|
||||
enabled: false,
|
||||
concurrency_limit: None,
|
||||
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
|
||||
group_ids: Vec::new(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("failed account update");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.load_account", "store.update_account"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
|
||||
let events = events();
|
||||
let openai = FakeProviderAdmin::new("openai", events.clone());
|
||||
let xai = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut openai_account = account_record("openai");
|
||||
openai_account.id = "acct_openai".to_owned();
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_xai".to_owned();
|
||||
store.set_accounts(vec![openai_account, xai_account]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore))
|
||||
.provider(openai)
|
||||
.provider(xai)
|
||||
.probe(Arc::new(SuccessfulAccountProbe))
|
||||
.build()
|
||||
.await;
|
||||
|
||||
let result = services
|
||||
.accounts()
|
||||
.batch_update(
|
||||
&context("batch-update-request"),
|
||||
BatchUpdateAccounts {
|
||||
account_ids: vec!["acct_openai".to_owned(), "acct_xai".to_owned()],
|
||||
enabled: false,
|
||||
concurrency_limit: None,
|
||||
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
|
||||
group_ids: Vec::new(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("batch update accounts");
|
||||
|
||||
assert_eq!(result.config_revision, revision(2));
|
||||
assert_eq!(result.account_ids.len(), 2);
|
||||
assert_eq!(store.audit_requests(), ["batch-update-request"]);
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.load_account",
|
||||
"store.load_account",
|
||||
"store.batch_update_accounts",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
@@ -976,6 +1260,7 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
@@ -984,12 +1269,12 @@ async fn accounts_list_should_return_complete_directory_semantics() {
|
||||
.expect("complete account directory");
|
||||
|
||||
assert_eq!(page.summary.total, 1);
|
||||
assert_eq!(page.summary.active, 1);
|
||||
assert_eq!(page.summary.normal, 1);
|
||||
let account = page.items.first().expect("account item");
|
||||
assert_eq!(account.account.provider_kind.as_str(), "openai");
|
||||
assert_eq!(
|
||||
account.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Active
|
||||
account.projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1004,7 +1289,10 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
@@ -1036,6 +1324,7 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
@@ -1060,10 +1349,14 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry() {
|
||||
async fn accounts_list_should_prefer_credential_error_over_quota_exhaustion() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::QuotaExhausted;
|
||||
account.quota = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
std::time::SystemTime::now(),
|
||||
None,
|
||||
);
|
||||
account.access_token_expires_at = Some(Utc::now() - TimeDelta::minutes(5));
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
@@ -1074,6 +1367,7 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
@@ -1082,17 +1376,17 @@ async fn accounts_list_should_prefer_quota_exhaustion_over_stale_token_expiry()
|
||||
.expect("usage-limited account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::QuotaExhausted,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
|
||||
// Unknown 不可调度,Admin 不得显示为 active。
|
||||
async fn accounts_list_should_map_unknown_credential_to_error_not_normal() {
|
||||
// Unknown 不可调度,Admin 不得显示为 normal。
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let mut account = account_record("openai");
|
||||
account.availability = AccountAvailability::Unknown;
|
||||
account.credential_state = CredentialState::Unknown;
|
||||
let store = FakeAccountStore::with_account(account, events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
@@ -1102,6 +1396,7 @@ async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
@@ -1110,14 +1405,13 @@ async fn accounts_list_should_map_unknown_availability_to_invalid_not_active() {
|
||||
.expect("unknown account directory");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::Invalid,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Error,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
// ready + 快照级 quota.limitReached → rate_limited。
|
||||
async fn accounts_list_should_not_derive_rate_limited_from_provider_quota_view() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
@@ -1126,7 +1420,10 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
@@ -1146,16 +1443,64 @@ async fn accounts_list_should_map_rate_limited_from_quota_limit_reached() {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("rate limited account directory");
|
||||
.expect("account directory with display quota cooldown");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").status,
|
||||
gateway_admin::model::accounts::AccountStatus::RateLimited,
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_not_derive_exhaustion_from_provider_quota_view() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![ProviderQuotaWindow {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(100.0),
|
||||
reset_at: Some(Utc::now() + TimeDelta::hours(1)),
|
||||
limit_reached: true,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
}],
|
||||
limit_reached: true,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("account directory with display quota limit");
|
||||
|
||||
assert_eq!(
|
||||
page.items.first().expect("account item").projection.status,
|
||||
gateway_admin::model::accounts::AccountStatus::Normal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1170,7 +1515,10 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
key: "primary".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "5小时限额".to_owned(),
|
||||
source: None,
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(5 * 60 * 60),
|
||||
used_percent: Some(97.0),
|
||||
reset_at: Some(reset_at),
|
||||
@@ -1188,13 +1536,14 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
usage: quota_local_usage("acct_test", 4_330_000),
|
||||
}]);
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
let page = accounts_service(provider, store.clone())
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
@@ -1202,11 +1551,124 @@ async fn accounts_list_should_attach_local_usage_to_quota_windows() {
|
||||
.await
|
||||
.expect("quota window usage");
|
||||
|
||||
let usage = page.items[0].quota.windows[0]
|
||||
let item = &page.items[0];
|
||||
let usage = item.quota.windows[0]
|
||||
.local_usage
|
||||
.as_ref()
|
||||
.expect("quota window local usage");
|
||||
assert_eq!(usage.total_tokens, Some(4_330_000));
|
||||
assert_eq!(
|
||||
item.usage.as_ref().and_then(|usage| usage.total_tokens),
|
||||
Some(4_330_000),
|
||||
);
|
||||
|
||||
let queries = store.quota_window_queries();
|
||||
assert_eq!(queries.len(), 1);
|
||||
assert_eq!(queries[0].account_id, "acct_test");
|
||||
assert_eq!(queries[0].key, "primary");
|
||||
assert_eq!(queries[0].range.start, reset_at - TimeDelta::hours(5));
|
||||
assert_eq!(queries[0].range.end, reset_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_list_should_not_attach_account_usage_to_model_specific_quota_windows() {
|
||||
let provider = FakeProviderAdmin::new("openai", events());
|
||||
let reset_at = Utc::now() + TimeDelta::days(7);
|
||||
provider.set_quota(ProviderQuota {
|
||||
observed_at: Some(Utc::now()),
|
||||
refresh_token_expires_at: None,
|
||||
windows: vec![
|
||||
ProviderQuotaWindow {
|
||||
key: "core-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "周额度".to_owned(),
|
||||
limit_id: Some("codex".to_owned()),
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(1.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
ProviderQuotaWindow {
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
group: "weekly".to_owned(),
|
||||
label: "周额度".to_owned(),
|
||||
limit_id: Some("codex_bengalfox".to_owned()),
|
||||
limit_name: Some("GPT-5.3-Codex-Spark".to_owned()),
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::Unavailable,
|
||||
window_seconds: Some(7 * 24 * 60 * 60),
|
||||
used_percent: Some(0.0),
|
||||
reset_at: Some(reset_at),
|
||||
limit_reached: false,
|
||||
local_usage: None,
|
||||
provider_data: None,
|
||||
},
|
||||
],
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
});
|
||||
let store = FakeAccountStore::new("openai", events());
|
||||
store.set_quota_window_usage(vec![
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "core-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 12_818_806),
|
||||
},
|
||||
AccountUsageWindowResult {
|
||||
account_id: "acct_test".to_owned(),
|
||||
key: "codex-bengalfox-primary".to_owned(),
|
||||
usage: quota_local_usage("acct_test", 127_926),
|
||||
},
|
||||
]);
|
||||
|
||||
let page = accounts_service(provider, store)
|
||||
.await
|
||||
.accounts()
|
||||
.list(AccountListQuery {
|
||||
page: 1,
|
||||
page_size: gateway_admin::model::PageSize::new(20).expect("page size"),
|
||||
provider_kind: None,
|
||||
group_filter: None,
|
||||
search: None,
|
||||
status: None,
|
||||
sort: None,
|
||||
})
|
||||
.await
|
||||
.expect("quota window usage");
|
||||
|
||||
let local_tokens = page.items[0]
|
||||
.quota
|
||||
.windows
|
||||
.iter()
|
||||
.map(|window| {
|
||||
(
|
||||
window.key.as_str(),
|
||||
window
|
||||
.local_usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.total_tokens),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
local_tokens,
|
||||
[
|
||||
("core-primary", Some(12_818_806)),
|
||||
("codex-bengalfox-primary", None),
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
page.items[0]
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.total_tokens),
|
||||
Some(12_818_806),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1232,6 +1694,59 @@ async fn accounts_refresh_provider_failure_should_not_call_store_commit() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reset_credit_oauth_refresh_should_reuse_the_exact_consume_command() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
provider.fail_next(ProviderAdminErrorKind::CredentialRefreshRequired);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = accounts_service(provider.clone(), store).await;
|
||||
let redeem_request_id =
|
||||
uuid::Uuid::parse_str("8fbf302d-11df-4bd5-82e4-08e4b3df7874").expect("UUID v4");
|
||||
let command = ConsumeProviderResetCredit {
|
||||
account_id: ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
credit_id: Some("credit_1".to_owned()),
|
||||
redeem_request_id,
|
||||
};
|
||||
|
||||
let result = services
|
||||
.accounts()
|
||||
.consume_reset_credit(&context("reset-credit-refresh"), command.clone())
|
||||
.await
|
||||
.expect("consume after OAuth refresh");
|
||||
|
||||
assert_eq!(result.code, "reset");
|
||||
assert_eq!(provider.reset_credit_commands(), [command.clone(), command]);
|
||||
assert!(recorded(&events).contains(&"provider.prepare_refresh"));
|
||||
assert!(recorded(&events).contains(&"store.commit_refresh"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_should_preserve_the_provider_failure_message() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let upstream_message = "Your refresh token has already been used.";
|
||||
provider.fail_next_with_message(ProviderAdminErrorKind::Conflict, upstream_message);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = accounts_service(provider, store).await;
|
||||
|
||||
let error = services
|
||||
.accounts()
|
||||
.refresh(
|
||||
&context("refresh-provider-message"),
|
||||
ProviderAccountId::new("acct_test").expect("account ID"),
|
||||
)
|
||||
.await
|
||||
.expect_err("Provider preparation must fail");
|
||||
|
||||
assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict);
|
||||
assert_eq!(error.to_string(), upstream_message);
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.load_account", "provider.prepare_refresh"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounts_refresh_store_failure_should_drop_guard_after_commit_attempt() {
|
||||
let events = events();
|
||||
@@ -1319,9 +1834,10 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
AccountRecord {
|
||||
id: "acct_test".to_owned(),
|
||||
provider_kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
groups: Vec::new(),
|
||||
name: "test account".to_owned(),
|
||||
email: Some("test@example.invalid".to_owned()),
|
||||
upstream_user_id: "upstream-user".to_owned(),
|
||||
upstream_user_id: Some("upstream-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1330,9 +1846,13 @@ pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
access_token_expires_at: Some(now + TimeDelta::hours(1)),
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
quota_observed_at: None,
|
||||
concurrency_limit: None,
|
||||
weight: gateway_core::engine::credential::AccountWeight::DEFAULT,
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: now,
|
||||
quota: QuotaState::allowed(now.into()),
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
@@ -1357,7 +1877,7 @@ fn prepared_create_with_id(
|
||||
provider_kind,
|
||||
name: name.to_owned(),
|
||||
email: Some("prepared@example.invalid".to_owned()),
|
||||
upstream_user_id: "prepared-user".to_owned(),
|
||||
upstream_user_id: Some("prepared-user".to_owned()),
|
||||
upstream_account_id: None,
|
||||
plan_type: Some("test".to_owned()),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
@@ -1366,8 +1886,8 @@ fn prepared_create_with_id(
|
||||
access_token_expires_at: Some(now + TimeDelta::hours(1)),
|
||||
next_refresh_at: Some(now + TimeDelta::minutes(30)),
|
||||
enabled: true,
|
||||
availability: AccountAvailability::Ready,
|
||||
availability_observed_at: now,
|
||||
credential_state: CredentialState::Ready,
|
||||
credential_observed_at: now,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1450,9 +1970,7 @@ impl AccountProbe for FreeModelQuotaProbe {
|
||||
_: AccountProbeRequest,
|
||||
) -> BoxFuture<'_, Result<AccountProbeResult, AccountProbeError>> {
|
||||
Box::pin(async move {
|
||||
let mut account = account_record("xai");
|
||||
account.availability = AccountAvailability::Ready;
|
||||
self.store.set_account_after_probe(account);
|
||||
self.store.set_account_after_probe(account_record("xai"));
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::RateLimited,
|
||||
"xAI free model quota is exhausted",
|
||||
|
||||
@@ -14,14 +14,20 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
AdminConfig, AdminServices, InitialAdminPassword,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
account_groups::{
|
||||
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, DeleteAccountGroup,
|
||||
NewAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
|
||||
},
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
AccountListQuery, AccountPage, AccountUpdateResult, AccountUsage,
|
||||
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
client_keys::{
|
||||
@@ -50,8 +56,8 @@ use gateway_admin::{
|
||||
backup::BackupStorePorts,
|
||||
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
|
||||
store::{
|
||||
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
|
||||
AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
|
||||
AccountGroupStore, AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts,
|
||||
AdminStoreResult, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
|
||||
},
|
||||
system::{
|
||||
SystemOperationError, SystemOperationErrorKind, SystemOperations,
|
||||
@@ -170,6 +176,7 @@ impl AdminHarness {
|
||||
},
|
||||
AdminStorePorts::new(
|
||||
self.accounts,
|
||||
Arc::new(UnavailableAccountGroupStore),
|
||||
self.auth,
|
||||
self.client_keys,
|
||||
self.observability,
|
||||
@@ -234,13 +241,60 @@ impl AuthStore for BootstrapAuthStore {
|
||||
|
||||
struct UnavailableStore;
|
||||
|
||||
struct UnavailableAccountGroupStore;
|
||||
|
||||
#[async_trait]
|
||||
impl AccountGroupStore for UnavailableAccountGroupStore {
|
||||
async fn list_account_groups(
|
||||
&self,
|
||||
_: AccountGroupListQuery,
|
||||
) -> AdminStoreResult<AccountGroupPage> {
|
||||
Err(unavailable("account groups"))
|
||||
}
|
||||
|
||||
async fn create_account_group(
|
||||
&self,
|
||||
_: NewAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
Err(unavailable("account group create"))
|
||||
}
|
||||
|
||||
async fn update_account_group(
|
||||
&self,
|
||||
_: UpdateAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
Err(unavailable("account group update"))
|
||||
}
|
||||
|
||||
async fn set_account_group_enabled(
|
||||
&self,
|
||||
_: SetAccountGroupEnabled,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
Err(unavailable("account group state"))
|
||||
}
|
||||
|
||||
async fn delete_account_group(
|
||||
&self,
|
||||
_: DeleteAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
Err(unavailable("account group delete"))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountStore for UnavailableStore {
|
||||
async fn list_accounts(&self, _: AccountListQuery) -> AdminStoreResult<AccountPage> {
|
||||
Err(unavailable("accounts"))
|
||||
}
|
||||
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(
|
||||
&self,
|
||||
_: &str,
|
||||
) -> AdminStoreResult<Option<gateway_admin::model::accounts::AccountPageItem>> {
|
||||
Err(unavailable("account"))
|
||||
}
|
||||
|
||||
@@ -315,14 +369,30 @@ impl AccountStore for UnavailableStore {
|
||||
Err(unavailable("credential refresh"))
|
||||
}
|
||||
|
||||
async fn set_account_enabled(
|
||||
async fn update_account(
|
||||
&self,
|
||||
_: SetAccountEnabled,
|
||||
_: UpdateAccount,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Revision> {
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
Err(unavailable("account recovery"))
|
||||
}
|
||||
|
||||
async fn batch_update_accounts(
|
||||
&self,
|
||||
_: BatchUpdateAccounts,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountsUpdateResult> {
|
||||
Err(unavailable("account batch update"))
|
||||
}
|
||||
|
||||
async fn delete_accounts(
|
||||
&self,
|
||||
_: DeleteAccounts,
|
||||
@@ -388,7 +458,11 @@ impl ClientKeyStore for UnavailableStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnavailableStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation> {
|
||||
Err(unavailable("dashboard"))
|
||||
}
|
||||
|
||||
|
||||
@@ -183,7 +183,8 @@ async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
|
||||
let now = Utc::now();
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(now)));
|
||||
store.replace_runtime_slots(Some(DashboardRuntimeSlots {
|
||||
active_accounts: 3,
|
||||
inherited_accounts: 2,
|
||||
overridden_slots: 5,
|
||||
used_slots: Some(2),
|
||||
}));
|
||||
let services = observability_services(store).await;
|
||||
@@ -195,9 +196,28 @@ async fn dashboard_summary_should_project_rebuildable_runtime_slots() {
|
||||
.expect("dashboard summary")
|
||||
.capacity;
|
||||
|
||||
assert_eq!(capacity.total_slots, 3);
|
||||
assert_eq!(capacity.total_slots, 7);
|
||||
assert_eq!(capacity.used_slots, Some(2));
|
||||
assert_eq!(capacity.available_slots, Some(1));
|
||||
assert_eq!(capacity.available_slots, Some(5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_share_one_current_observation_time_across_runtime_facts() {
|
||||
let historical_end = Utc::now() - Duration::hours(1);
|
||||
let store = Arc::new(FixtureObservabilityStore::new(observation_range(
|
||||
historical_end,
|
||||
)));
|
||||
let services = observability_services(store.clone()).await;
|
||||
|
||||
services
|
||||
.observability()
|
||||
.dashboard_summary(observation_range(historical_end), TrendKind::Usage)
|
||||
.await
|
||||
.expect("dashboard summary");
|
||||
|
||||
let (summary_observed_at, slots_observed_at) = store.observed_times();
|
||||
assert_eq!(summary_observed_at, slots_observed_at);
|
||||
assert!(summary_observed_at.is_some_and(|value| value > historical_end));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -394,6 +414,8 @@ struct FixtureObservabilityStore {
|
||||
calculated_billing_facts: Mutex<Vec<UsageCalculatedBillingFact>>,
|
||||
diagnostics: Mutex<Vec<DiagnosticObservation>>,
|
||||
runtime_slots: Mutex<Option<DashboardRuntimeSlots>>,
|
||||
summary_observed_at: Mutex<Option<DateTime<Utc>>>,
|
||||
slots_observed_at: Mutex<Option<DateTime<Utc>>>,
|
||||
usage_records: Mutex<Vec<UsageRecord>>,
|
||||
}
|
||||
|
||||
@@ -410,6 +432,8 @@ impl FixtureObservabilityStore {
|
||||
calculated_billing_facts: Mutex::new(Vec::new()),
|
||||
diagnostics: Mutex::new(Vec::new()),
|
||||
runtime_slots: Mutex::new(None),
|
||||
summary_observed_at: Mutex::new(None),
|
||||
slots_observed_at: Mutex::new(None),
|
||||
usage_records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -437,6 +461,16 @@ impl FixtureObservabilityStore {
|
||||
*self.runtime_slots.lock().expect("runtime slots") = runtime_slots;
|
||||
}
|
||||
|
||||
fn observed_times(&self) -> (Option<DateTime<Utc>>, Option<DateTime<Utc>>) {
|
||||
(
|
||||
*self
|
||||
.summary_observed_at
|
||||
.lock()
|
||||
.expect("summary observed at"),
|
||||
*self.slots_observed_at.lock().expect("slots observed at"),
|
||||
)
|
||||
}
|
||||
|
||||
fn replace_usage_records(&self, records: Vec<UsageRecord>) {
|
||||
*self.usage_records.lock().expect("usage records") = records;
|
||||
}
|
||||
@@ -444,11 +478,20 @@ impl FixtureObservabilityStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for FixtureObservabilityStore {
|
||||
async fn dashboard_summary(&self, range: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation> {
|
||||
*self
|
||||
.summary_observed_at
|
||||
.lock()
|
||||
.expect("summary observed at") = Some(observed_at);
|
||||
Ok(DashboardObservation {
|
||||
range,
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: self.trend.lock().expect("trend").clone(),
|
||||
account_usage: Vec::new(),
|
||||
@@ -458,8 +501,9 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
|
||||
async fn dashboard_runtime_slots(
|
||||
&self,
|
||||
_: DateTime<Utc>,
|
||||
observed_at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<DashboardRuntimeSlots>> {
|
||||
*self.slots_observed_at.lock().expect("slots observed at") = Some(observed_at);
|
||||
Ok(*self.runtime_slots.lock().expect("runtime slots"))
|
||||
}
|
||||
|
||||
@@ -611,11 +655,14 @@ fn total_record(
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_billing".to_owned(),
|
||||
config_revision: 1,
|
||||
routing_scope: "all".to_owned(),
|
||||
routing_group_refs: Vec::new(),
|
||||
routing_group_names_snapshot: Vec::new(),
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "gpt-5.5".to_owned(),
|
||||
requested_model_id: Some("gpt-5.5".to_owned()),
|
||||
provider_kind: provider_kind.map(str::to_owned),
|
||||
provider_account_ref: None,
|
||||
provider_account_name: None,
|
||||
|
||||
@@ -5,8 +5,8 @@ use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::provider_credentials::{
|
||||
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, CredentialMutation,
|
||||
ImportCredentials, ProviderQuotaRequest, StartAuthorization,
|
||||
AuthorizationMutationTarget, CompleteAuthorization, CredentialDeletion, ImportCredentials,
|
||||
ProviderQuotaRequest, StartAuthorization,
|
||||
},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
@@ -15,55 +15,6 @@ use super::accounts::{
|
||||
FakeAccountStore, FakeProviderAdmin, context, document, events, recorded, revision,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_enable_should_use_scoped_store_without_provider_mutation() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = service(provider, store.clone()).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.enable(mutation("acct_test"))
|
||||
.await
|
||||
.expect("enable credential");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_disable_should_commit_then_release_provider_resources() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let services = service(provider, store.clone()).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.disable(mutation("acct_test"))
|
||||
.await
|
||||
.expect("disable credential");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_delete_should_commit_then_release_provider_resources() {
|
||||
let events = events();
|
||||
@@ -89,26 +40,6 @@ async fn openai_delete_should_commit_then_release_provider_resources() {
|
||||
assert_eq!(store.audit_requests(), ["request-openai"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_disable_store_failure_should_not_release_provider_resources() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("openai", events.clone());
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
store.fail_next_commit();
|
||||
let services = service(provider, store).await;
|
||||
|
||||
services
|
||||
.openai()
|
||||
.disable(mutation("acct_test"))
|
||||
.await
|
||||
.expect_err("failed Store transaction");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
["store.credential_details", "store.set_enabled"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
let events = events();
|
||||
@@ -125,6 +56,8 @@ async fn openai_import_should_prepare_before_atomic_store_commit() {
|
||||
.await
|
||||
.expect("import credential");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -162,6 +95,8 @@ async fn openai_import_should_refresh_quota_for_every_imported_account() {
|
||||
.await
|
||||
.expect("import credentials");
|
||||
|
||||
await_quota_requests(&provider, 2).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[
|
||||
@@ -203,6 +138,8 @@ async fn openai_import_should_remain_successful_when_quota_refresh_fails() {
|
||||
.await
|
||||
.expect("committed import remains successful");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
result.credential_ids,
|
||||
[ProviderAccountId::new("acct_prepared").expect("account ID")]
|
||||
@@ -236,6 +173,8 @@ async fn openai_authorization_create_should_observe_initial_quota() {
|
||||
.await
|
||||
.expect("complete authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -284,6 +223,8 @@ async fn openai_authorization_create_should_remain_successful_when_initial_quota
|
||||
.await
|
||||
.expect("quota observation must not fail authorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.account_id.as_str(), "acct_prepared");
|
||||
assert_eq!(provider.quota_requests().len(), 1);
|
||||
}
|
||||
@@ -322,6 +263,8 @@ async fn openai_authorization_store_failure_should_release_claim_for_retry() {
|
||||
.await
|
||||
.expect("released claim must permit retry");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
@@ -400,6 +343,8 @@ async fn openai_reauthorization_should_commit_after_credential_revision_advances
|
||||
.await
|
||||
.expect("complete reauthorization");
|
||||
|
||||
await_quota_requests(&provider, 1).await;
|
||||
|
||||
assert_eq!(result.credential_revision, Some(revision(3)));
|
||||
|
||||
assert_eq!(
|
||||
@@ -426,11 +371,14 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
.await
|
||||
}
|
||||
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-openai"),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
async fn await_quota_requests(provider: &FakeProviderAdmin, expected: usize) {
|
||||
for _ in 0..100 {
|
||||
if provider.quota_requests().len() == expected {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
assert_eq!(provider.quota_requests().len(), expected);
|
||||
}
|
||||
|
||||
fn deletion(account_id: &str) -> CredentialDeletion {
|
||||
|
||||
@@ -4,37 +4,12 @@ use gateway_core::engine::credential::ProviderAccountId;
|
||||
|
||||
use gateway_admin::{
|
||||
AdminServices,
|
||||
model::provider_credentials::{CredentialMutation, ImportCredentials, ProviderQuotaRequest},
|
||||
model::provider_credentials::{ImportCredentials, ProviderQuotaRequest},
|
||||
ports::provider::ProviderAdminErrorKind,
|
||||
};
|
||||
|
||||
use super::accounts::{FakeAccountStore, FakeProviderAdmin, context, document, events, recorded};
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_disable_should_commit_then_notify_the_stateless_provider() {
|
||||
let events = events();
|
||||
let provider = FakeProviderAdmin::new("xai", events.clone());
|
||||
let store = FakeAccountStore::new("xai", events.clone());
|
||||
let services = service(provider.clone(), store.clone()).await;
|
||||
|
||||
services
|
||||
.xai()
|
||||
.disable(mutation("acct_test"))
|
||||
.await
|
||||
.expect("disable credential");
|
||||
|
||||
assert_eq!(
|
||||
recorded(&events),
|
||||
[
|
||||
"store.credential_details",
|
||||
"store.set_enabled",
|
||||
"provider.account_unavailable",
|
||||
"provider.account_facts_changed",
|
||||
]
|
||||
);
|
||||
assert_eq!(store.audit_requests(), ["request-xai"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xai_import_should_prepare_before_atomic_store_commit() {
|
||||
let events = events();
|
||||
@@ -132,10 +107,3 @@ async fn service(provider: Arc<FakeProviderAdmin>, store: Arc<FakeAccountStore>)
|
||||
.build()
|
||||
.await
|
||||
}
|
||||
|
||||
fn mutation(account_id: &str) -> CredentialMutation {
|
||||
CredentialMutation {
|
||||
context: context("request-xai"),
|
||||
account_id: ProviderAccountId::new(account_id).expect("account ID"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
//! Account group HTTP wire and fixed routes.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_admin::model::{
|
||||
PageSize,
|
||||
account_groups::{
|
||||
AccountGroupAccountSummary, AccountGroupCapacity, AccountGroupColor, AccountGroupListQuery,
|
||||
AccountGroupMutation, AccountGroupPage, AccountGroupRecord, AccountGroupUsage,
|
||||
CreateAccountGroup, DeleteAccountGroup, SetAccountGroupEnabled, UpdateAccountGroup,
|
||||
},
|
||||
};
|
||||
use gateway_core::routing::AccountGroupId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
};
|
||||
|
||||
const DEFAULT_PAGE_SIZE: u32 = 50;
|
||||
const MAX_PAGE_SIZE: u32 = 200;
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct ListAccountGroupsQuery {
|
||||
page: Option<u32>,
|
||||
page_size: Option<u32>,
|
||||
search: Option<String>,
|
||||
enabled: Option<bool>,
|
||||
}
|
||||
|
||||
impl ListAccountGroupsQuery {
|
||||
fn into_command(self) -> Result<AccountGroupListQuery, WireValidationError> {
|
||||
let page = self.page.unwrap_or(1);
|
||||
let page_size = self.page_size.unwrap_or(DEFAULT_PAGE_SIZE);
|
||||
if page == 0 || page_size == 0 || page_size > MAX_PAGE_SIZE {
|
||||
return Err(WireValidationError::new("page"));
|
||||
}
|
||||
let search = self
|
||||
.search
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty());
|
||||
if search
|
||||
.as_deref()
|
||||
.is_some_and(|value| value.len() > 256 || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err(WireValidationError::new("search"));
|
||||
}
|
||||
Ok(AccountGroupListQuery {
|
||||
page,
|
||||
page_size: PageSize::new(
|
||||
u16::try_from(page_size).map_err(|_| WireValidationError::new("pageSize"))?,
|
||||
)
|
||||
.map_err(|_| WireValidationError::new("pageSize"))?,
|
||||
search,
|
||||
enabled: self.enabled,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct CreateAccountGroupRequest {
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
color: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct UpdateAccountGroupRequest {
|
||||
id: String,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
color: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct AccountGroupIdRequest {
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupView {
|
||||
id: String,
|
||||
name: String,
|
||||
description: Option<String>,
|
||||
color: String,
|
||||
enabled: bool,
|
||||
member_count: u64,
|
||||
provider_counts: BTreeMap<String, u64>,
|
||||
client_key_count: u64,
|
||||
account_summary: AccountGroupAccountSummaryView,
|
||||
capacity: AccountGroupCapacityView,
|
||||
usage: AccountGroupUsageView,
|
||||
created_at: DateTime<Utc>,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupAccountSummaryView {
|
||||
available: u64,
|
||||
limited: u64,
|
||||
total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupCapacityView {
|
||||
used_slots: Option<u64>,
|
||||
total_slots: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupUsageView {
|
||||
today_usd: String,
|
||||
total_usd: String,
|
||||
}
|
||||
|
||||
impl From<AccountGroupRecord> for AccountGroupView {
|
||||
fn from(record: AccountGroupRecord) -> Self {
|
||||
Self {
|
||||
id: record.id.to_string(),
|
||||
name: record.name,
|
||||
description: record.description,
|
||||
color: record.color.as_str().to_owned(),
|
||||
enabled: record.enabled,
|
||||
member_count: record.member_count,
|
||||
provider_counts: record.provider_counts,
|
||||
client_key_count: record.client_key_count,
|
||||
account_summary: account_summary_view(record.account_summary),
|
||||
capacity: capacity_view(record.capacity),
|
||||
usage: usage_view(record.usage),
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn account_summary_view(summary: AccountGroupAccountSummary) -> AccountGroupAccountSummaryView {
|
||||
AccountGroupAccountSummaryView {
|
||||
available: summary.available,
|
||||
limited: summary.limited,
|
||||
total: summary.total,
|
||||
}
|
||||
}
|
||||
|
||||
fn capacity_view(capacity: AccountGroupCapacity) -> AccountGroupCapacityView {
|
||||
AccountGroupCapacityView {
|
||||
used_slots: capacity.used_slots,
|
||||
total_slots: capacity.total_slots,
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_view(usage: AccountGroupUsage) -> AccountGroupUsageView {
|
||||
AccountGroupUsageView {
|
||||
today_usd: usage.today_usd.to_string(),
|
||||
total_usd: usage.total_usd.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupPageData {
|
||||
items: Vec<AccountGroupView>,
|
||||
page: PageMeta,
|
||||
config_revision: u64,
|
||||
}
|
||||
|
||||
impl From<AccountGroupPage> for AccountGroupPageData {
|
||||
fn from(page: AccountGroupPage) -> Self {
|
||||
let total_pages = if page.total == 0 {
|
||||
0
|
||||
} else {
|
||||
page.total.div_ceil(u64::from(page.page_size))
|
||||
};
|
||||
Self {
|
||||
items: page.items.into_iter().map(Into::into).collect(),
|
||||
page: PageMeta::new(
|
||||
page.page,
|
||||
u32::from(page.page_size),
|
||||
page.total,
|
||||
u32::try_from(total_pages).unwrap_or(u32::MAX),
|
||||
),
|
||||
config_revision: page.config_revision.get(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct AccountGroupMutationData {
|
||||
id: String,
|
||||
record: Option<AccountGroupView>,
|
||||
config_revision: u64,
|
||||
}
|
||||
|
||||
impl From<AccountGroupMutation> for AccountGroupMutationData {
|
||||
fn from(mutation: AccountGroupMutation) -> Self {
|
||||
Self {
|
||||
id: mutation.id.to_string(),
|
||||
record: mutation.record.map(Into::into),
|
||||
config_revision: mutation.config_revision.get(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct all fixed account-group management routes.
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.route("/api/admin/account-groups", get(list::<S>))
|
||||
.route("/api/admin/account-groups/create", post(create::<S>))
|
||||
.route("/api/admin/account-groups/update", post(update::<S>))
|
||||
.route("/api/admin/account-groups/enable", post(enable::<S>))
|
||||
.route("/api/admin/account-groups/disable", post(disable::<S>))
|
||||
.route("/api/admin/account-groups/delete", post(delete::<S>))
|
||||
}
|
||||
|
||||
async fn list<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<ListAccountGroupsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let result = state
|
||||
.admin_services()
|
||||
.account_groups()
|
||||
.list(query.into_command().map_err(map_wire_error)?)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountGroupPageData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn create<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<CreateAccountGroupRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
validate_group_fields(&request.name, request.description.as_deref())?;
|
||||
mutation_response(
|
||||
StatusCode::CREATED,
|
||||
state
|
||||
.admin_services()
|
||||
.account_groups()
|
||||
.create(
|
||||
&auth.context().mutation_context(),
|
||||
CreateAccountGroup {
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
color: group_color(&request.color)?,
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn update<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<UpdateAccountGroupRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
validate_group_fields(&request.name, request.description.as_deref())?;
|
||||
mutation_response(
|
||||
StatusCode::OK,
|
||||
state
|
||||
.admin_services()
|
||||
.account_groups()
|
||||
.update(
|
||||
&auth.context().mutation_context(),
|
||||
UpdateAccountGroup {
|
||||
id: group_id(request.id)?,
|
||||
name: request.name,
|
||||
description: request.description,
|
||||
color: group_color(&request.color)?,
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn enable<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountGroupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
set_enabled(auth, state, request, true).await
|
||||
}
|
||||
|
||||
async fn disable<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountGroupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
set_enabled(auth, state, request, false).await
|
||||
}
|
||||
|
||||
async fn set_enabled<S>(
|
||||
auth: AdminAuth,
|
||||
state: S,
|
||||
request: AccountGroupIdRequest,
|
||||
enabled: bool,
|
||||
) -> Result<AdminResponse<AdminEnvelope<AccountGroupMutationData>>, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
mutation_response(
|
||||
StatusCode::OK,
|
||||
state
|
||||
.admin_services()
|
||||
.account_groups()
|
||||
.set_enabled(
|
||||
&auth.context().mutation_context(),
|
||||
SetAccountGroupEnabled {
|
||||
id: group_id(request.id)?,
|
||||
enabled,
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn delete<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountGroupIdRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
mutation_response(
|
||||
StatusCode::OK,
|
||||
state
|
||||
.admin_services()
|
||||
.account_groups()
|
||||
.delete(
|
||||
&auth.context().mutation_context(),
|
||||
DeleteAccountGroup {
|
||||
id: group_id(request.id)?,
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
|
||||
fn mutation_response(
|
||||
status: StatusCode,
|
||||
result: Result<AccountGroupMutation, gateway_admin::model::AdminError>,
|
||||
) -> Result<AdminResponse<AdminEnvelope<AccountGroupMutationData>>, AdminError> {
|
||||
let result = result.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
status,
|
||||
AdminEnvelope::ok(AccountGroupMutationData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
fn group_id(value: String) -> Result<AccountGroupId, AdminError> {
|
||||
AccountGroupId::new(value).map_err(|_| AdminError::bad_request("Invalid account group ID"))
|
||||
}
|
||||
|
||||
fn group_color(value: &str) -> Result<AccountGroupColor, AdminError> {
|
||||
AccountGroupColor::parse(value)
|
||||
.ok_or_else(|| AdminError::bad_request("Invalid account group color"))
|
||||
}
|
||||
|
||||
fn validate_group_fields(name: &str, description: Option<&str>) -> Result<(), AdminError> {
|
||||
if name.trim() != name
|
||||
|| name.is_empty()
|
||||
|| name.chars().count() > 100
|
||||
|| name.chars().any(char::is_control)
|
||||
|| description
|
||||
.is_some_and(|value| value.len() > 4096 || value.chars().any(char::is_control))
|
||||
{
|
||||
return Err(AdminError::bad_request("Invalid account group request"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn map_wire_error(_: WireValidationError) -> AdminError {
|
||||
AdminError::bad_request("Invalid account group query")
|
||||
}
|
||||
|
||||
fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError {
|
||||
map_admin_service_error(error, "Account group repository unavailable")
|
||||
}
|
||||
@@ -18,26 +18,29 @@ use futures::{Stream, StreamExt as _};
|
||||
use gateway_admin::model::{
|
||||
AdminError as AdminServiceError, PageSize,
|
||||
accounts::{
|
||||
AccountConnectionTestEvent as DomainConnectionTestEvent, AccountCost, AccountListQuery,
|
||||
AccountModelUsage, AccountSort, AccountSortField, AccountStatus as DomainAccountStatus,
|
||||
AccountUsage, SortDirection,
|
||||
AccountConcurrencyLimit, AccountConnectionTestEvent as DomainConnectionTestEvent,
|
||||
AccountCost, AccountGroupFilter, AccountListQuery, AccountModelUsage, AccountSort,
|
||||
AccountSortField, AccountStatus as DomainAccountStatus, AccountUpdateResult, AccountUsage,
|
||||
AccountWeight, AccountsUpdateResult, BatchUpdateAccounts, SortDirection, UpdateAccount,
|
||||
},
|
||||
provider_credentials::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountRefreshResult,
|
||||
AuthorizationStarted, CompleteAuthorization, CredentialDeletion, CredentialDeletionResult,
|
||||
CredentialImportResult, CredentialMutation, CredentialMutationResult, ImportCredentials,
|
||||
ProviderDocument, ProviderModels, ProviderQuota, ProviderQuotaWindow, RotateCredential,
|
||||
StartAuthorization,
|
||||
AuthorizationStarted, CompleteAuthorization, ConsumeProviderResetCredit,
|
||||
CredentialDeletion, CredentialDeletionResult, CredentialImportResult, CredentialMutation,
|
||||
CredentialMutationResult, ImportCredentials, ProviderDocument, ProviderModels,
|
||||
ProviderQuota, ProviderQuotaWindow, ProviderResetCredit, ProviderResetCreditResult,
|
||||
ProviderResetCredits, RotateCredential, StartAuthorization,
|
||||
},
|
||||
};
|
||||
use gateway_core::{
|
||||
engine::credential::{OpaqueProviderData, ProviderAccountId},
|
||||
routing::{ProviderKind, UpstreamModelId},
|
||||
routing::{AccountGroupId, ProviderKind, UpstreamModelId},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use uuid::{Uuid, Version};
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError,
|
||||
@@ -51,8 +54,10 @@ const MAX_NAME_BYTES: usize = 512;
|
||||
const MAX_IMPORT_DATA_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_ACCESS_TOKEN_BYTES: usize = 16 * 1024;
|
||||
const MAX_REFRESH_TOKEN_BYTES: usize = 64 * 1024;
|
||||
const MAX_ID_TOKEN_BYTES: usize = 16 * 1024;
|
||||
const MAX_CALLBACK_URL_BYTES: usize = 64 * 1024;
|
||||
const MAX_ACCOUNT_DELETE_BATCH: usize = 200;
|
||||
const MAX_ACCOUNT_GROUP_BATCH: usize = 1000;
|
||||
|
||||
/// 账号列表查询参数。
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
@@ -61,12 +66,74 @@ pub struct ListQuery {
|
||||
pub page: Option<u32>,
|
||||
pub page_size: Option<u32>,
|
||||
pub provider: Option<String>,
|
||||
pub group_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub sort_by: Option<String>,
|
||||
pub sort_direction: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct BatchUpdateAccountsRequest {
|
||||
pub account_ids: Vec<String>,
|
||||
pub enabled: bool,
|
||||
#[serde(deserialize_with = "deserialize_required_nullable")]
|
||||
pub concurrency_limit: Option<u64>,
|
||||
pub weight: u64,
|
||||
pub group_ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct BatchUpdatedAccountsData {
|
||||
account_ids: Vec<String>,
|
||||
config_revision: u64,
|
||||
}
|
||||
|
||||
impl From<AccountsUpdateResult> for BatchUpdatedAccountsData {
|
||||
fn from(result: AccountsUpdateResult) -> Self {
|
||||
Self {
|
||||
account_ids: result
|
||||
.account_ids
|
||||
.into_iter()
|
||||
.map(|id| id.to_string())
|
||||
.collect(),
|
||||
config_revision: result.config_revision.get(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BatchUpdateAccountsRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
if self.account_ids.is_empty()
|
||||
|| self.account_ids.len() > MAX_ACCOUNT_GROUP_BATCH
|
||||
|| self
|
||||
.account_ids
|
||||
.iter()
|
||||
.any(|id| require_account_id(id, "accountIds").is_err())
|
||||
|| self.account_ids.iter().collect::<BTreeSet<_>>().len() != self.account_ids.len()
|
||||
{
|
||||
return Err(WireValidationError::new("accountIds"));
|
||||
}
|
||||
validate_wire_group_ids(&self.group_ids)?;
|
||||
parse_concurrency_limit(self.concurrency_limit)?;
|
||||
parse_account_weight(self.weight)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<BatchUpdateAccounts, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(BatchUpdateAccounts {
|
||||
account_ids: self.account_ids,
|
||||
enabled: self.enabled,
|
||||
concurrency_limit: parse_concurrency_limit(self.concurrency_limit)?,
|
||||
weight: parse_account_weight(self.weight)?,
|
||||
group_ids: validate_wire_group_ids(&self.group_ids)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ListQuery {
|
||||
/// 解析并校验全部 wire 字段,生成 Admin 查询命令。
|
||||
pub fn validate(self) -> Result<AccountListQuery, WireValidationError> {
|
||||
@@ -79,6 +146,14 @@ impl ListQuery {
|
||||
return Err(WireValidationError::new("pageSize"));
|
||||
}
|
||||
let provider_kind = parse_provider(self.provider.as_deref().unwrap_or("all"))?;
|
||||
let group_filter = match self.group_id.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some("ungrouped") => Some(AccountGroupFilter::Ungrouped),
|
||||
Some(value) => Some(AccountGroupFilter::Group(
|
||||
AccountGroupId::new(value.to_owned())
|
||||
.map_err(|_| WireValidationError::new("groupId"))?,
|
||||
)),
|
||||
};
|
||||
let search = self
|
||||
.search
|
||||
.map(|value| value.trim().to_owned())
|
||||
@@ -91,7 +166,8 @@ impl ListQuery {
|
||||
let status = match self.status.as_deref().map(str::trim) {
|
||||
None | Some("") => None,
|
||||
Some(value) => Some(
|
||||
parse_account_status(value).ok_or_else(|| WireValidationError::new("status"))?,
|
||||
DomainAccountStatus::parse(&value.to_ascii_lowercase())
|
||||
.ok_or_else(|| WireValidationError::new("status"))?,
|
||||
),
|
||||
};
|
||||
let sort = match (self.sort_by.as_deref(), self.sort_direction.as_deref()) {
|
||||
@@ -110,6 +186,7 @@ impl ListQuery {
|
||||
)
|
||||
.map_err(|_| WireValidationError::new("pageSize"))?,
|
||||
provider_kind,
|
||||
group_filter,
|
||||
search,
|
||||
status,
|
||||
sort,
|
||||
@@ -127,17 +204,6 @@ fn parse_provider(value: &str) -> Result<Option<ProviderKind>, WireValidationErr
|
||||
.map_err(|_| WireValidationError::new("provider"))
|
||||
}
|
||||
|
||||
fn parse_account_status(value: &str) -> Option<DomainAccountStatus> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"active" => Some(DomainAccountStatus::Active),
|
||||
"expired" => Some(DomainAccountStatus::Expired),
|
||||
"quota_exhausted" => Some(DomainAccountStatus::QuotaExhausted),
|
||||
"disabled" => Some(DomainAccountStatus::Disabled),
|
||||
"banned" => Some(DomainAccountStatus::Banned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sort_field(value: &str) -> Option<AccountSortField> {
|
||||
match value.trim() {
|
||||
"email" => Some(AccountSortField::Email),
|
||||
@@ -172,9 +238,11 @@ pub struct AccountPageData {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub unavailable: u64,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 一条安全账号视图。
|
||||
@@ -184,6 +252,7 @@ pub struct AccountView {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub provider: String,
|
||||
pub groups: Vec<AccountGroupRefView>,
|
||||
pub resource_ref: String,
|
||||
pub email: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
@@ -193,9 +262,13 @@ pub struct AccountView {
|
||||
pub authentication_kind: String,
|
||||
pub has_refresh_token: bool,
|
||||
pub status: String,
|
||||
pub display_status: String,
|
||||
pub availability: String,
|
||||
/// `status == "error"` 时的具体原因;其余状态为 `null`。
|
||||
pub error_reason: Option<String>,
|
||||
/// 最近一次失败的上游错误描述;仅错误状态存在。
|
||||
pub error_message: Option<String>,
|
||||
pub enabled: bool,
|
||||
pub concurrency_limit: Option<u32>,
|
||||
pub weight: u16,
|
||||
pub access_token_expires_at: Option<String>,
|
||||
pub access_token_expires_at_display: Option<String>,
|
||||
pub refresh_token_expires_at: Option<String>,
|
||||
@@ -208,12 +281,23 @@ pub struct AccountView {
|
||||
pub usage: AccountUsageView,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountGroupRefView {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub color: String,
|
||||
pub enabled: bool,
|
||||
}
|
||||
|
||||
/// Provider quota 安全视图。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 429 临时限流(Redis 冷却)到期时间展示;非限流中为 `null`。
|
||||
pub rate_limited_until: Option<String>,
|
||||
pub windows: Vec<AccountQuotaWindowView>,
|
||||
}
|
||||
|
||||
@@ -223,8 +307,12 @@ pub struct AccountQuotaView {
|
||||
pub struct AccountQuotaWindowView {
|
||||
pub key: String,
|
||||
pub group: String,
|
||||
pub limit_id: Option<String>,
|
||||
pub limit_name: Option<String>,
|
||||
pub role: Option<String>,
|
||||
pub window_seconds: Option<u64>,
|
||||
pub label_display: String,
|
||||
pub window_label_display: String,
|
||||
pub used_percent: Option<f64>,
|
||||
pub used_percent_display: String,
|
||||
pub limit_reached: bool,
|
||||
@@ -390,6 +478,48 @@ impl AccountActionRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 主动额度重置卡消费请求。幂等键由 UI 生成并在不确定重试时复用,与官方一致。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountResetCreditConsumeRequest {
|
||||
pub account_id: String,
|
||||
#[serde(default)]
|
||||
pub credit_id: Option<String>,
|
||||
pub redeem_request_id: String,
|
||||
}
|
||||
|
||||
impl AccountResetCreditConsumeRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
if self.credit_id.as_deref().is_some_and(|value| {
|
||||
let value = value.trim();
|
||||
value.is_empty() || value.len() > 512 || value.chars().any(char::is_control)
|
||||
}) {
|
||||
return Err(WireValidationError::new("creditId"));
|
||||
}
|
||||
let value = self.redeem_request_id.trim();
|
||||
let redeem_request_id =
|
||||
Uuid::parse_str(value).map_err(|_| WireValidationError::new("redeemRequestId"))?;
|
||||
if redeem_request_id.get_version() != Some(Version::Random)
|
||||
|| redeem_request_id.hyphenated().to_string() != value
|
||||
{
|
||||
return Err(WireValidationError::new("redeemRequestId"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(self) -> Result<ConsumeProviderResetCredit, WireValidationError> {
|
||||
self.validate()?;
|
||||
Ok(ConsumeProviderResetCredit {
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
credit_id: self.credit_id.map(|value| value.trim().to_owned()),
|
||||
redeem_request_id: Uuid::parse_str(&self.redeem_request_id)
|
||||
.map_err(|_| WireValidationError::new("redeemRequestId"))?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 手工 OAuth 刷新会变更持久 credential。
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -457,6 +587,67 @@ pub struct AccountQuotaData {
|
||||
pub account: AccountView,
|
||||
}
|
||||
|
||||
/// 主动额度重置卡列表响应。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountResetCreditsData {
|
||||
pub available_count: u64,
|
||||
pub credits: Vec<AccountResetCreditView>,
|
||||
}
|
||||
|
||||
/// 一张安全主动额度重置卡视图。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountResetCreditView {
|
||||
pub id: String,
|
||||
pub status: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub expires_at: Option<String>,
|
||||
pub reset_type: Option<String>,
|
||||
}
|
||||
|
||||
/// 主动额度重置卡消费响应。
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountResetCreditResultData {
|
||||
pub code: String,
|
||||
pub credit: Option<AccountResetCreditView>,
|
||||
}
|
||||
|
||||
impl From<ProviderResetCredit> for AccountResetCreditView {
|
||||
fn from(credit: ProviderResetCredit) -> Self {
|
||||
Self {
|
||||
id: credit.id,
|
||||
status: credit.status,
|
||||
title: credit.title,
|
||||
expires_at: credit.expires_at.map(|value| value.to_rfc3339()),
|
||||
reset_type: credit.reset_type,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ProviderResetCredits> for AccountResetCreditsData {
|
||||
fn from(credits: ProviderResetCredits) -> Self {
|
||||
Self {
|
||||
available_count: credits.available_count,
|
||||
credits: credits
|
||||
.credits
|
||||
.into_iter()
|
||||
.map(AccountResetCreditView::from)
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ProviderResetCreditResult> for AccountResetCreditResultData {
|
||||
fn from(result: ProviderResetCreditResult) -> Self {
|
||||
Self {
|
||||
code: result.code,
|
||||
credit: result.credit.map(AccountResetCreditView::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provider-owned 明文导出文档;Debug 永远不输出内部 JSON。
|
||||
#[derive(Serialize)]
|
||||
#[serde(transparent)]
|
||||
@@ -524,10 +715,9 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
DomainConnectionTestEvent::Content { text } => {
|
||||
serde_json::json!({ "type": "content", "text": text })
|
||||
}
|
||||
DomainConnectionTestEvent::Completed { account_status } => serde_json::json!({
|
||||
DomainConnectionTestEvent::Completed => serde_json::json!({
|
||||
"type": "test_complete",
|
||||
"success": true,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"success": true
|
||||
}),
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message,
|
||||
@@ -536,7 +726,6 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
upstream_status,
|
||||
upstream_content_type,
|
||||
upstream_body,
|
||||
account_status,
|
||||
} => serde_json::json!({
|
||||
"type": "error",
|
||||
"error": message,
|
||||
@@ -544,8 +733,7 @@ impl From<DomainConnectionTestEvent> for AccountConnectionTestEvent {
|
||||
"providerErrorType": provider_error_type,
|
||||
"upstreamStatus": upstream_status,
|
||||
"upstreamContentType": upstream_content_type,
|
||||
"upstreamBody": upstream_body,
|
||||
"accountStatus": account_status_name(account_status)
|
||||
"upstreamBody": upstream_body
|
||||
}),
|
||||
};
|
||||
Self { data }
|
||||
@@ -688,31 +876,49 @@ impl CompleteAccountAuthorizationRequest {
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct AccountMutationRequest {
|
||||
pub provider: String,
|
||||
pub struct UpdateAccountRequest {
|
||||
pub account_id: String,
|
||||
pub enabled: bool,
|
||||
#[serde(deserialize_with = "deserialize_required_nullable")]
|
||||
pub concurrency_limit: Option<u64>,
|
||||
pub weight: u64,
|
||||
pub group_ids: Vec<String>,
|
||||
}
|
||||
|
||||
impl AccountMutationRequest {
|
||||
impl UpdateAccountRequest {
|
||||
pub fn validate(&self) -> Result<(), WireValidationError> {
|
||||
AccountProvider::parse(&self.provider)?;
|
||||
require_account_id(&self.account_id, "accountId")
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
parse_concurrency_limit(self.concurrency_limit)?;
|
||||
parse_account_weight(self.weight)?;
|
||||
validate_wire_group_ids(&self.group_ids)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
self,
|
||||
context: gateway_admin::model::MutationContext,
|
||||
) -> Result<(AccountProvider, CredentialMutation), WireValidationError> {
|
||||
fn into_command(self) -> Result<UpdateAccount, WireValidationError> {
|
||||
self.validate()?;
|
||||
let provider = AccountProvider::parse(&self.provider)?;
|
||||
Ok((
|
||||
provider,
|
||||
CredentialMutation {
|
||||
context,
|
||||
account_id: ProviderAccountId::new(self.account_id)
|
||||
.map_err(|_| WireValidationError::new("accountId"))?,
|
||||
},
|
||||
))
|
||||
Ok(UpdateAccount {
|
||||
account_id: self.account_id,
|
||||
enabled: self.enabled,
|
||||
concurrency_limit: parse_concurrency_limit(self.concurrency_limit)?,
|
||||
weight: parse_account_weight(self.weight)?,
|
||||
group_ids: validate_wire_group_ids(&self.group_ids)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct UpdatedAccountData {
|
||||
pub account_id: String,
|
||||
pub config_revision: u64,
|
||||
}
|
||||
|
||||
impl From<AccountUpdateResult> for UpdatedAccountData {
|
||||
fn from(result: AccountUpdateResult) -> Self {
|
||||
Self {
|
||||
account_id: result.account_id.to_string(),
|
||||
config_revision: result.config_revision.get(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -767,6 +973,7 @@ pub struct RotateAccountRequest {
|
||||
pub account_id: String,
|
||||
pub access_token: String,
|
||||
pub refresh_token: Option<String>,
|
||||
pub id_token: Option<String>,
|
||||
}
|
||||
|
||||
impl RotateAccountRequest {
|
||||
@@ -775,7 +982,11 @@ impl RotateAccountRequest {
|
||||
return Err(WireValidationError::new("provider"));
|
||||
}
|
||||
require_account_id(&self.account_id, "accountId")?;
|
||||
validate_oauth_material(&self.access_token, self.refresh_token.as_deref())
|
||||
validate_oauth_material(
|
||||
&self.access_token,
|
||||
self.refresh_token.as_deref(),
|
||||
self.id_token.as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
fn into_command(
|
||||
@@ -789,6 +1000,10 @@ impl RotateAccountRequest {
|
||||
"refresh_token".to_owned(),
|
||||
self.refresh_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
material.insert(
|
||||
"id_token".to_owned(),
|
||||
self.id_token.map_or(Value::Null, Value::String),
|
||||
);
|
||||
Ok(RotateCredential {
|
||||
mutation: CredentialMutation {
|
||||
context,
|
||||
@@ -807,8 +1022,8 @@ pub struct AccountImportData {
|
||||
pub account_ids: Vec<String>,
|
||||
}
|
||||
|
||||
impl From<CredentialImportResult> for AccountImportData {
|
||||
fn from(result: CredentialImportResult) -> Self {
|
||||
impl AccountImportData {
|
||||
pub fn from_result(result: CredentialImportResult) -> Self {
|
||||
let account_ids = result
|
||||
.credential_ids
|
||||
.into_iter()
|
||||
@@ -885,6 +1100,46 @@ fn require_account_id(value: &str, field: &'static str) -> Result<(), WireValida
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_concurrency_limit(
|
||||
value: Option<u64>,
|
||||
) -> Result<Option<AccountConcurrencyLimit>, WireValidationError> {
|
||||
value
|
||||
.map(|value| {
|
||||
u32::try_from(value)
|
||||
.ok()
|
||||
.and_then(AccountConcurrencyLimit::new)
|
||||
.ok_or_else(|| WireValidationError::new("concurrencyLimit"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn deserialize_required_nullable<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Option::<u64>::deserialize(deserializer)
|
||||
}
|
||||
|
||||
fn parse_account_weight(value: u64) -> Result<AccountWeight, WireValidationError> {
|
||||
u16::try_from(value)
|
||||
.ok()
|
||||
.and_then(AccountWeight::new)
|
||||
.ok_or_else(|| WireValidationError::new("weight"))
|
||||
}
|
||||
|
||||
fn validate_wire_group_ids(values: &[String]) -> Result<Vec<AccountGroupId>, WireValidationError> {
|
||||
if values.len() > MAX_ACCOUNT_GROUP_BATCH
|
||||
|| values.iter().collect::<BTreeSet<_>>().len() != values.len()
|
||||
{
|
||||
return Err(WireValidationError::new("groupIds"));
|
||||
}
|
||||
values
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|value| AccountGroupId::new(value).map_err(|_| WireValidationError::new("groupIds")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 构造统一账号管理路由。
|
||||
pub fn router<S>() -> Router<S>
|
||||
where
|
||||
@@ -896,11 +1151,19 @@ where
|
||||
.route("/api/admin/accounts/export", get(export_accounts::<S>))
|
||||
.route("/api/admin/accounts/import", post(import_accounts::<S>))
|
||||
.route("/api/admin/accounts/refresh", post(refresh_account::<S>))
|
||||
.route("/api/admin/accounts/recover", post(recover_account::<S>))
|
||||
.route("/api/admin/accounts/rotate", post(rotate_account::<S>))
|
||||
.route("/api/admin/accounts/enable", post(enable_account::<S>))
|
||||
.route("/api/admin/accounts/disable", post(disable_account::<S>))
|
||||
.route("/api/admin/accounts/update", post(update_account::<S>))
|
||||
.route("/api/admin/accounts/delete", post(delete_accounts::<S>))
|
||||
.route(
|
||||
"/api/admin/accounts/batch-update",
|
||||
post(batch_update_accounts::<S>),
|
||||
)
|
||||
.route("/api/admin/accounts/quota", get(account_quota::<S>))
|
||||
.route(
|
||||
"/api/admin/accounts/reset-credits",
|
||||
get(account_reset_credits::<S>).post(consume_account_reset_credit::<S>),
|
||||
)
|
||||
.route(
|
||||
"/api/admin/accounts/quota/refresh",
|
||||
post(refresh_account_quota::<S>),
|
||||
@@ -924,6 +1187,27 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
async fn batch_update_accounts<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<BatchUpdateAccountsRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = request.into_command().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.batch_update(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(BatchUpdatedAccountsData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_accounts<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
@@ -1013,7 +1297,7 @@ where
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::CREATED,
|
||||
AdminEnvelope::ok(AccountImportData::from(result)),
|
||||
AdminEnvelope::ok(AccountImportData::from_result(result)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1108,64 +1392,24 @@ where
|
||||
))
|
||||
}
|
||||
|
||||
async fn enable_account<S>(
|
||||
async fn update_account<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountMutationRequest>,
|
||||
Json(request): Json<UpdateAccountRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
mutate_account(auth, state, request, AccountMutationAction::Enable).await
|
||||
}
|
||||
|
||||
async fn disable_account<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountMutationRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
mutate_account(auth, state, request, AccountMutationAction::Disable).await
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum AccountMutationAction {
|
||||
Enable,
|
||||
Disable,
|
||||
}
|
||||
|
||||
async fn mutate_account<S>(
|
||||
auth: AdminAuth,
|
||||
state: S,
|
||||
request: AccountMutationRequest,
|
||||
action: AccountMutationAction,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let (provider, command) = request
|
||||
.into_command(auth.context().mutation_context())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = match (provider, action) {
|
||||
(AccountProvider::OpenAi, AccountMutationAction::Enable) => {
|
||||
state.admin_services().openai().enable(command).await
|
||||
}
|
||||
(AccountProvider::OpenAi, AccountMutationAction::Disable) => {
|
||||
state.admin_services().openai().disable(command).await
|
||||
}
|
||||
(AccountProvider::Xai, AccountMutationAction::Enable) => {
|
||||
state.admin_services().xai().enable(command).await
|
||||
}
|
||||
(AccountProvider::Xai, AccountMutationAction::Disable) => {
|
||||
state.admin_services().xai().disable(command).await
|
||||
}
|
||||
}
|
||||
.map_err(map_service_error)?;
|
||||
let command = request.into_command().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.update(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountMutationData::from(result)),
|
||||
AdminEnvelope::ok(UpdatedAccountData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1210,6 +1454,25 @@ where
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
async fn recover_account<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountActionRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let account_id = request.into_id().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.recover(&auth.context().mutation_context(), account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let data = account_refresh_data(result, Utc::now());
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
async fn account_quota<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
@@ -1252,6 +1515,48 @@ where
|
||||
Ok(AdminResponse::new(StatusCode::OK, AdminEnvelope::ok(data)))
|
||||
}
|
||||
|
||||
async fn account_reset_credits<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Query(query): Query<AccountIdQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let account_id = query.into_id().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.reset_credits(&auth.context().mutation_context(), account_id)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountResetCreditsData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn consume_account_reset_credit<S>(
|
||||
auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
Json(request): Json<AccountResetCreditConsumeRequest>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let command = request.into_command().map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
.accounts()
|
||||
.consume_reset_credit(&auth.context().mutation_context(), command)
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
Ok(AdminResponse::new(
|
||||
StatusCode::OK,
|
||||
AdminEnvelope::ok(AccountResetCreditResultData::from(result)),
|
||||
))
|
||||
}
|
||||
|
||||
async fn account_models<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
@@ -1333,9 +1638,11 @@ fn account_page_data(
|
||||
page: PageMeta::new(page, u32::from(page_size), result.total, total_pages),
|
||||
summary: AccountSummaryView {
|
||||
total: result.summary.total,
|
||||
active: result.summary.active,
|
||||
normal: result.summary.normal,
|
||||
quota_exhausted: result.summary.quota_exhausted,
|
||||
unavailable: result.summary.unavailable,
|
||||
rate_limited: result.summary.rate_limited,
|
||||
disabled: result.summary.disabled,
|
||||
error: result.summary.error,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1365,31 +1672,46 @@ fn account_models_data(result: ProviderModels) -> AccountModelsData {
|
||||
fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
let AccountDirectoryItem {
|
||||
account,
|
||||
status,
|
||||
projection,
|
||||
usage,
|
||||
quota,
|
||||
} = item;
|
||||
let status = account_status_name(status).to_owned();
|
||||
let status = projection.status.as_str().to_owned();
|
||||
let rate_limited_until = projection.rate_limited_until.map(DateTime::<Utc>::from);
|
||||
let expires_at = account.access_token_expires_at.as_ref().map(china_rfc3339);
|
||||
let added_at = china_rfc3339(&account.created_at);
|
||||
let updated_at = china_rfc3339(&account.updated_at);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, now);
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, rate_limited_until, now);
|
||||
AccountView {
|
||||
id: account.id.clone(),
|
||||
name: account.name,
|
||||
provider: account.provider_kind.to_string(),
|
||||
groups: account
|
||||
.groups
|
||||
.into_iter()
|
||||
.map(|group| AccountGroupRefView {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
})
|
||||
.collect(),
|
||||
resource_ref: account.id,
|
||||
email: account.email,
|
||||
account_id: account.upstream_account_id,
|
||||
user_id: Some(account.upstream_user_id),
|
||||
user_id: account.upstream_user_id,
|
||||
label: None,
|
||||
plan_type: account.plan_type,
|
||||
authentication_kind: account.authentication_kind,
|
||||
has_refresh_token: account.has_refresh_token,
|
||||
status: status.clone(),
|
||||
display_status: status,
|
||||
availability: account.availability.as_str().to_owned(),
|
||||
status,
|
||||
error_reason: projection
|
||||
.error_reason
|
||||
.map(|reason| reason.as_str().to_owned()),
|
||||
error_message: projection.error_message,
|
||||
enabled: account.enabled,
|
||||
concurrency_limit: account.concurrency_limit.map(|limit| limit.get()),
|
||||
weight: account.weight.get(),
|
||||
access_token_expires_at: expires_at,
|
||||
access_token_expires_at_display: account
|
||||
.access_token_expires_at
|
||||
@@ -1407,9 +1729,11 @@ fn account_view(item: AccountDirectoryItem, now: DateTime<Utc>) -> AccountView {
|
||||
}
|
||||
|
||||
fn account_quota_view(
|
||||
quota: ProviderQuota,
|
||||
mut quota: ProviderQuota,
|
||||
rate_limited_until: Option<DateTime<Utc>>,
|
||||
now: DateTime<Utc>,
|
||||
) -> (AccountQuotaView, Option<String>) {
|
||||
quota.apply_limit_reached_display();
|
||||
let refresh_token_expires_at = quota
|
||||
.refresh_token_expires_at
|
||||
.map(|value| china_rfc3339(&value));
|
||||
@@ -1417,10 +1741,12 @@ fn account_quota_view(
|
||||
.observed_at
|
||||
.map_or_else(|| "—".to_owned(), |value| relative_time(value, now));
|
||||
let windows = quota.windows.into_iter().map(quota_window_view).collect();
|
||||
let rate_limited_until = rate_limited_until.map(|until| china_datetime(&until));
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
windows,
|
||||
},
|
||||
refresh_token_expires_at,
|
||||
@@ -1432,7 +1758,10 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
key,
|
||||
group,
|
||||
label,
|
||||
source: _,
|
||||
limit_id,
|
||||
limit_name,
|
||||
role,
|
||||
local_usage_attribution: _,
|
||||
window_seconds,
|
||||
used_percent,
|
||||
reset_at,
|
||||
@@ -1440,10 +1769,17 @@ fn quota_window_view(window: ProviderQuotaWindow) -> AccountQuotaWindowView {
|
||||
local_usage,
|
||||
provider_data: _,
|
||||
} = window;
|
||||
let label_display = limit_name
|
||||
.as_ref()
|
||||
.map_or_else(|| label.clone(), |name| format!("{name} · {label}"));
|
||||
AccountQuotaWindowView {
|
||||
label_display: label,
|
||||
label_display,
|
||||
window_label_display: label,
|
||||
key,
|
||||
group,
|
||||
limit_id,
|
||||
limit_name,
|
||||
role: role.map(|value| value.as_str().to_owned()),
|
||||
window_seconds,
|
||||
used_percent,
|
||||
used_percent_display: used_percent
|
||||
@@ -1584,7 +1920,7 @@ fn account_model_usage_view(usage: AccountModelUsage, now: DateTime<Utc>) -> Mod
|
||||
billing_amount_usd: usd.map(|cost| cost.amount.as_str().to_owned()),
|
||||
billing_amount_usd_display: usd.map_or_else(
|
||||
|| "—".to_owned(),
|
||||
|cost| format!("${}", cost.amount.as_str()),
|
||||
|cost| format_decimal_currency(cost.amount.as_str(), "USD"),
|
||||
),
|
||||
cost_estimate_status: cost_estimate_status.to_owned(),
|
||||
known_cost_count: known_count,
|
||||
@@ -1679,6 +2015,7 @@ fn require_text(
|
||||
fn validate_oauth_material(
|
||||
access_token: &str,
|
||||
refresh_token: Option<&str>,
|
||||
id_token: Option<&str>,
|
||||
) -> Result<(), WireValidationError> {
|
||||
if access_token.len() > MAX_ACCESS_TOKEN_BYTES
|
||||
|| !valid_visible_ascii(access_token)
|
||||
@@ -1693,6 +2030,13 @@ fn validate_oauth_material(
|
||||
}) {
|
||||
return Err(WireValidationError::new("refreshToken"));
|
||||
}
|
||||
if id_token.is_some_and(|token| {
|
||||
token.len() > MAX_ID_TOKEN_BYTES
|
||||
|| !valid_visible_ascii(token)
|
||||
|| !valid_compact_jwt_shape(token)
|
||||
}) {
|
||||
return Err(WireValidationError::new("idToken"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1713,18 +2057,6 @@ fn provider_document_value(document: ProviderDocument) -> Value {
|
||||
Value::Object(document.into_provider_data().into_inner())
|
||||
}
|
||||
|
||||
fn account_status_name(status: DomainAccountStatus) -> &'static str {
|
||||
match status {
|
||||
DomainAccountStatus::Active => "active",
|
||||
DomainAccountStatus::RateLimited => "rate_limited",
|
||||
DomainAccountStatus::QuotaExhausted => "quota_exhausted",
|
||||
DomainAccountStatus::Expired => "expired",
|
||||
DomainAccountStatus::Invalid => "invalid",
|
||||
DomainAccountStatus::Disabled => "disabled",
|
||||
DomainAccountStatus::Banned => "banned",
|
||||
}
|
||||
}
|
||||
|
||||
fn relative_time(value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
|
||||
@@ -12,7 +12,7 @@ use gateway_admin::model::client_keys::{
|
||||
};
|
||||
use gateway_core::{
|
||||
policy::{ClientApiKeyId, RateLimits},
|
||||
routing::ProviderKind,
|
||||
routing::AccountGroupId,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -137,7 +137,7 @@ impl ClientKeySort {
|
||||
pub struct CreateClientKeyRequest {
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
group_ids: Vec<String>,
|
||||
max_concurrency: u64,
|
||||
requests_per_minute: u64,
|
||||
}
|
||||
@@ -147,14 +147,13 @@ impl CreateClientKeyRequest {
|
||||
pub fn into_command(self) -> Result<CreateClientKey, WireValidationError> {
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
let group_ids = validate_group_ids(self.group_ids)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(CreateClientKey {
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
provider_kind: ProviderKind::new(self.provider_kind)
|
||||
.map_err(|_| WireValidationError::new("providerKind"))?,
|
||||
group_ids,
|
||||
limits: RateLimits {
|
||||
max_concurrency: self.max_concurrency,
|
||||
requests_per_minute: self.requests_per_minute,
|
||||
@@ -170,7 +169,7 @@ pub struct UpdateClientKeyRequest {
|
||||
id: String,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
group_ids: Vec<String>,
|
||||
max_concurrency: u64,
|
||||
requests_per_minute: u64,
|
||||
}
|
||||
@@ -181,15 +180,14 @@ impl UpdateClientKeyRequest {
|
||||
validate_required_text(&self.id, "id")?;
|
||||
validate_required_text(&self.name, "name")?;
|
||||
validate_optional_text(self.label.as_deref(), "label")?;
|
||||
validate_provider_kind(&self.provider_kind)?;
|
||||
let group_ids = validate_group_ids(self.group_ids)?;
|
||||
validate_limit(self.max_concurrency, "maxConcurrency")?;
|
||||
validate_limit(self.requests_per_minute, "requestsPerMinute")?;
|
||||
Ok(UpdateClientKey {
|
||||
id: client_key_id(self.id, "clientKeyMutationNotFound")?,
|
||||
name: self.name,
|
||||
label: self.label,
|
||||
provider_kind: ProviderKind::new(self.provider_kind)
|
||||
.map_err(|_| WireValidationError::new("providerKind"))?,
|
||||
group_ids,
|
||||
limits: RateLimits {
|
||||
max_concurrency: self.max_concurrency,
|
||||
requests_per_minute: self.requests_per_minute,
|
||||
@@ -242,7 +240,9 @@ pub struct ClientKeyView {
|
||||
id: String,
|
||||
name: String,
|
||||
label: Option<String>,
|
||||
provider_kind: String,
|
||||
routing_scope: &'static str,
|
||||
groups: Vec<ClientKeyGroupView>,
|
||||
provider_kinds: Vec<String>,
|
||||
prefix: String,
|
||||
enabled: bool,
|
||||
max_concurrency: u64,
|
||||
@@ -252,13 +252,42 @@ pub struct ClientKeyView {
|
||||
last_used_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ClientKeyGroupView {
|
||||
id: String,
|
||||
name: String,
|
||||
color: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl From<ClientKeyRecord> for ClientKeyView {
|
||||
fn from(record: ClientKeyRecord) -> Self {
|
||||
let routing_scope = if record.groups.is_empty() {
|
||||
"all"
|
||||
} else {
|
||||
"groups"
|
||||
};
|
||||
Self {
|
||||
id: record.id.to_string(),
|
||||
name: record.name,
|
||||
label: record.label,
|
||||
provider_kind: record.provider_kind.to_string(),
|
||||
routing_scope,
|
||||
groups: record
|
||||
.groups
|
||||
.into_iter()
|
||||
.map(|group| ClientKeyGroupView {
|
||||
id: group.id.to_string(),
|
||||
name: group.name,
|
||||
color: group.color.as_str().to_owned(),
|
||||
enabled: group.enabled,
|
||||
})
|
||||
.collect(),
|
||||
provider_kinds: record
|
||||
.provider_kinds
|
||||
.into_iter()
|
||||
.map(|provider| provider.to_string())
|
||||
.collect(),
|
||||
prefix: record.prefix,
|
||||
enabled: record.enabled,
|
||||
max_concurrency: record.limits.max_concurrency,
|
||||
@@ -572,21 +601,20 @@ fn validate_required_text(value: &str, field: &'static str) -> Result<(), WireVa
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_provider_kind(value: &str) -> Result<(), WireValidationError> {
|
||||
let valid = !value.is_empty()
|
||||
&& value.len() <= 64
|
||||
&& value.bytes().enumerate().all(|(index, byte)| {
|
||||
byte.is_ascii_lowercase()
|
||||
|| byte.is_ascii_digit()
|
||||
|| (byte == b'-' && index > 0 && index + 1 < value.len())
|
||||
})
|
||||
&& !value.starts_with('-')
|
||||
&& !value.ends_with('-');
|
||||
if valid {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(WireValidationError::new("providerKind"))
|
||||
fn validate_group_ids(values: Vec<String>) -> Result<Vec<AccountGroupId>, WireValidationError> {
|
||||
if values.len() > 1000
|
||||
|| values
|
||||
.iter()
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
.len()
|
||||
!= values.len()
|
||||
{
|
||||
return Err(WireValidationError::new("groupIds"));
|
||||
}
|
||||
values
|
||||
.into_iter()
|
||||
.map(|value| AccountGroupId::new(value).map_err(|_| WireValidationError::new("groupIds")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn validate_optional_text(
|
||||
|
||||
@@ -7,6 +7,7 @@ use axum::{
|
||||
response::Response,
|
||||
};
|
||||
|
||||
pub mod account_groups;
|
||||
pub mod accounts;
|
||||
pub mod auth;
|
||||
pub mod backups;
|
||||
@@ -29,6 +30,7 @@ where
|
||||
S: AdminSessionState + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Router::new()
|
||||
.merge(account_groups::router::<S>())
|
||||
.merge(accounts::router::<S>())
|
||||
.merge(auth::router::<S>())
|
||||
.merge(backups::router::<S>())
|
||||
|
||||
@@ -15,7 +15,7 @@ use gateway_admin::model::{PageSize as DomainPageSize, observability as domain};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::presenter::{format_compact_number, format_number};
|
||||
use super::presenter::{format_compact_number, format_decimal_currency, format_number};
|
||||
use super::{
|
||||
AdminAuth, AdminEnvelope, AdminError, AdminResponse, AdminSessionState, PageMeta,
|
||||
WireValidationError, wire::map_admin_service_error,
|
||||
|
||||
@@ -127,20 +127,6 @@ pub(crate) fn token_details(record: &domain::UsageRecord) -> TokenDetailsView {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn format_decimal_currency(amount: &str, currency: &str) -> String {
|
||||
if currency == "USD" {
|
||||
let value = amount.parse::<f64>().map_or(0.0, |value| value);
|
||||
let precision = if value != 0.0 && value.abs() < 0.01 {
|
||||
4
|
||||
} else {
|
||||
2
|
||||
};
|
||||
format!("${value:.precision$}")
|
||||
} else {
|
||||
format!("{currency} {amount}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn format_money(cost: &domain::CurrencyCost) -> String {
|
||||
format_decimal_currency(cost.amount.as_str(), &cost.currency)
|
||||
}
|
||||
@@ -236,7 +222,7 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
|
||||
let model = record
|
||||
.upstream_model_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| record.requested_model_id.clone());
|
||||
.or_else(|| record.requested_model_id.clone());
|
||||
let transport = record
|
||||
.upstream_transport
|
||||
.clone()
|
||||
@@ -246,6 +232,9 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
|
||||
id: record.id.clone(),
|
||||
request_id: record.id,
|
||||
client_api_key_id: Some(record.client_api_key_ref),
|
||||
routing_scope: record.routing_scope,
|
||||
routing_group_refs: record.routing_group_refs,
|
||||
routing_group_names_snapshot: record.routing_group_names_snapshot,
|
||||
kind: record.operation,
|
||||
provider: record.provider_kind,
|
||||
authentication_kind: record.provider_account_authentication_kind,
|
||||
@@ -254,7 +243,7 @@ pub(crate) fn usage_record_view(record: domain::UsageRecord) -> UsageRecordView
|
||||
account_name: record.provider_account_name,
|
||||
route: record.endpoint,
|
||||
model,
|
||||
requested_model: Some(record.requested_model_id),
|
||||
requested_model: record.requested_model_id,
|
||||
upstream_model: record.upstream_model_id,
|
||||
service_tier: record.service_tier,
|
||||
status_code,
|
||||
@@ -362,9 +351,7 @@ pub(crate) fn usage_attempt_view(attempt: domain::UsageAttempt) -> UsageAttemptV
|
||||
provider: attempt
|
||||
.provider_kind
|
||||
.unwrap_or_else(|| "unknown".to_owned()),
|
||||
model: attempt
|
||||
.upstream_model_id
|
||||
.unwrap_or_else(|| "unknown".to_owned()),
|
||||
model: attempt.upstream_model_id,
|
||||
transport: attempt
|
||||
.upstream_transport
|
||||
.unwrap_or_else(|| "unknown".to_owned()),
|
||||
@@ -709,7 +696,7 @@ pub(crate) fn dashboard_view(
|
||||
} = result;
|
||||
let domain::DashboardObservation {
|
||||
range,
|
||||
requests,
|
||||
totals,
|
||||
provider_accounts,
|
||||
account_usage,
|
||||
recent_requests,
|
||||
@@ -742,27 +729,31 @@ pub(crate) fn dashboard_view(
|
||||
last_used: relative_time(credential.last_used_at, range.end),
|
||||
});
|
||||
}
|
||||
let unavailable_accounts = provider_accounts
|
||||
.total
|
||||
.saturating_sub(provider_accounts.normal);
|
||||
|
||||
DashboardDataView {
|
||||
cards: DashboardCardsView {
|
||||
credentials: DashboardCredentialsCardView {
|
||||
total: format_compact_number(provider_accounts.total),
|
||||
total: provider_accounts.total.to_string(),
|
||||
total_value: provider_accounts.total,
|
||||
available: format_compact_number(provider_accounts.active),
|
||||
available_value: provider_accounts.active,
|
||||
unavailable: format_compact_number(provider_accounts.unavailable),
|
||||
unavailable_value: provider_accounts.unavailable,
|
||||
available: provider_accounts.normal.to_string(),
|
||||
available_value: provider_accounts.normal,
|
||||
unavailable: unavailable_accounts.to_string(),
|
||||
unavailable_value: unavailable_accounts,
|
||||
},
|
||||
traffic: DashboardTrafficCardView {
|
||||
today_requests: format_compact_number(today.request_count),
|
||||
today_requests_value: today.request_count,
|
||||
yesterday_requests_value: yesterday.request_count,
|
||||
total_requests: format_compact_number(requests.request_count),
|
||||
total_requests: format_compact_number(totals.request_count),
|
||||
},
|
||||
tokens: DashboardTokensCardView {
|
||||
today_tokens: format_compact_number(today.total_tokens),
|
||||
today_tokens_value: today.total_tokens,
|
||||
yesterday_tokens_value: yesterday.total_tokens,
|
||||
total_tokens: format_compact_number(requests.total_tokens),
|
||||
total_tokens: format_compact_number(totals.total_tokens),
|
||||
total_billing_amount_usd: total_billing_usd
|
||||
.as_ref()
|
||||
.map_or_else(|| "—".to_owned(), |amount| format!("${}", amount.as_str())),
|
||||
@@ -772,7 +763,7 @@ pub(crate) fn dashboard_view(
|
||||
today_hit_rate_value: today.observed_cached_token_rate,
|
||||
yesterday_hit_rate_value: yesterday.observed_cached_token_rate,
|
||||
total_hit_rate: display_rate(total_cached_token_rate),
|
||||
total_cached_tokens: format_compact_number(requests.cached_tokens),
|
||||
total_cached_tokens: format_compact_number(totals.cached_tokens),
|
||||
average_first_token_latency_ms: display_duration(average_first_token_latency_ms),
|
||||
},
|
||||
},
|
||||
@@ -783,14 +774,11 @@ pub(crate) fn dashboard_view(
|
||||
usage_records: recent_requests.into_iter().map(usage_record_view).collect(),
|
||||
pool_summary: DashboardPoolSummaryView {
|
||||
total: provider_accounts.total,
|
||||
active: provider_accounts.active,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
expired: provider_accounts.expired,
|
||||
invalid: provider_accounts.invalid,
|
||||
normal: provider_accounts.normal,
|
||||
quota_exhausted: provider_accounts.quota_exhausted,
|
||||
refreshing: provider_accounts.refreshing,
|
||||
rate_limited: provider_accounts.rate_limited,
|
||||
disabled: provider_accounts.disabled,
|
||||
banned: provider_accounts.banned,
|
||||
error: provider_accounts.error,
|
||||
},
|
||||
capacity_info: DashboardCapacityInfoView {
|
||||
max_concurrent_per_account: capacity.max_concurrent_per_account,
|
||||
|
||||
@@ -310,16 +310,6 @@ pub(crate) fn usage_range(
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
) -> Result<domain::TimeRange, WireValidationError> {
|
||||
let end = parse_datetime(end)?.unwrap_or_else(Utc::now);
|
||||
let start =
|
||||
parse_datetime(start)?.unwrap_or_else(|| domain::china_day_start(end) - Duration::days(1));
|
||||
domain::TimeRange::new(start, end).map_err(|_| WireValidationError::new("timeRange"))
|
||||
}
|
||||
|
||||
pub(crate) fn dashboard_today_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
|
||||
@@ -38,7 +38,8 @@ where
|
||||
S: AdminSessionState + Send + Sync,
|
||||
{
|
||||
let kind = query.trend_kind().map_err(map_wire_error)?;
|
||||
let range = dashboard_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
// 概览默认按中国时区当日统计,与单独趋势接口保持同一口径。
|
||||
let range = dashboard_today_range(query.start_time.as_deref(), query.end_time.as_deref())
|
||||
.map_err(map_wire_error)?;
|
||||
let result = state
|
||||
.admin_services()
|
||||
|
||||
@@ -84,6 +84,9 @@ pub struct UsageRecordView {
|
||||
pub id: String,
|
||||
pub request_id: String,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub routing_scope: String,
|
||||
pub routing_group_refs: Vec<String>,
|
||||
pub routing_group_names_snapshot: Vec<String>,
|
||||
pub kind: String,
|
||||
pub provider: Option<String>,
|
||||
pub authentication_kind: Option<String>,
|
||||
@@ -91,7 +94,7 @@ pub struct UsageRecordView {
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub route: String,
|
||||
pub model: String,
|
||||
pub model: Option<String>,
|
||||
pub requested_model: Option<String>,
|
||||
pub upstream_model: Option<String>,
|
||||
pub service_tier: Option<String>,
|
||||
@@ -180,7 +183,7 @@ pub struct UsageAttemptView {
|
||||
pub attempt_index: u32,
|
||||
pub trigger: String,
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub model: Option<String>,
|
||||
pub transport: String,
|
||||
pub send_state: String,
|
||||
pub outcome: String,
|
||||
@@ -347,14 +350,11 @@ pub struct DashboardAccountRequestBucketView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardPoolSummaryView {
|
||||
pub total: u64,
|
||||
pub active: u64,
|
||||
pub rate_limited: u64,
|
||||
pub expired: u64,
|
||||
pub invalid: u64,
|
||||
pub normal: u64,
|
||||
pub quota_exhausted: u64,
|
||||
pub refreshing: Option<u64>,
|
||||
pub rate_limited: u64,
|
||||
pub disabled: u64,
|
||||
pub banned: u64,
|
||||
pub error: u64,
|
||||
}
|
||||
|
||||
/// 同 target 账号调度容量;Redis 未提供聚合事实时运行中槽位保持空值。
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
//! Admin API 的展示层辅助:数值格式化。
|
||||
//!
|
||||
//! 按 docs/project-redundancy-boundary-audit.md BE-02 的目标,display 字段最终
|
||||
//! 迁移到前端;过渡期统一由本模块提供,避免 accounts/observability 各自维护。
|
||||
//! 当前 wire 仍保留 display 字段;统一由本模块提供,避免 accounts/observability
|
||||
//! 各自维护格式化规则。
|
||||
|
||||
use gateway_core::accounting::Decimal;
|
||||
|
||||
/// 千分位格式化。
|
||||
#[must_use]
|
||||
@@ -37,3 +39,33 @@ pub fn format_compact_number(value: u64) -> String {
|
||||
}
|
||||
format_number(value)
|
||||
}
|
||||
|
||||
/// 货币展示格式化。USD 常规金额保留两位,小于 1 美元时最多保留四位。
|
||||
#[must_use]
|
||||
pub fn format_decimal_currency(amount: &str, currency: &str) -> String {
|
||||
if currency != "USD" {
|
||||
return format!("{currency} {amount}");
|
||||
}
|
||||
|
||||
const DECIMAL_SCALE: u128 = 10_000_000_000;
|
||||
let scaled = amount.parse::<Decimal>().unwrap_or_default().scaled();
|
||||
let precision = if scaled != 0 && scaled < DECIMAL_SCALE {
|
||||
4_u32
|
||||
} else {
|
||||
2_u32
|
||||
};
|
||||
let rounding_unit = 10_u128.pow(10 - precision);
|
||||
let rounded = (scaled + rounding_unit / 2) / rounding_unit;
|
||||
let display_scale = 10_u128.pow(precision);
|
||||
let whole = rounded / display_scale;
|
||||
let fraction = rounded % display_scale;
|
||||
let mut display = format!("{whole}.{fraction:0width$}", width = precision as usize);
|
||||
while display.ends_with('0')
|
||||
&& display
|
||||
.split_once('.')
|
||||
.is_some_and(|(_, fraction)| fraction.len() > 2)
|
||||
{
|
||||
display.pop();
|
||||
}
|
||||
format!("${display}")
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! `/healthz` 对 Core、Store 与 Host worker 健康事实的聚合。
|
||||
//! `/healthz` 对 Core、Store 与 Host 关键 worker 健康事实的聚合。
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -6,8 +6,10 @@ use std::time::Duration;
|
||||
use axum::{extract::State, http::StatusCode};
|
||||
use futures::future::join_all;
|
||||
use gateway_core::health::{
|
||||
HealthProbe, HealthState, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState,
|
||||
HealthProbe, HealthState, WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource,
|
||||
WorkerRuntimeState,
|
||||
};
|
||||
use gateway_core::task::WorkerKind;
|
||||
|
||||
use crate::ApiState;
|
||||
|
||||
@@ -32,7 +34,13 @@ impl HealthStatus {
|
||||
}
|
||||
|
||||
pub(crate) async fn healthy(&self) -> bool {
|
||||
if !self.workers.snapshot().iter().all(worker_is_healthy) {
|
||||
if !self
|
||||
.workers
|
||||
.snapshot()
|
||||
.iter()
|
||||
.filter(|worker| worker_affects_healthz(worker))
|
||||
.all(worker_is_healthy)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let checks = self.probes.iter().map(|probe| probe.check());
|
||||
@@ -54,6 +62,16 @@ pub(crate) async fn healthz(State(state): State<ApiState>) -> StatusCode {
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_affects_healthz(worker: &WorkerHealthSnapshot) -> bool {
|
||||
let WorkerHealthKey::Task(id) = &worker.key else {
|
||||
return false;
|
||||
};
|
||||
!matches!(
|
||||
id.kind(),
|
||||
WorkerKind::OAuthRefresh | WorkerKind::QuotaCatalogHealth
|
||||
)
|
||||
}
|
||||
|
||||
fn worker_is_healthy(worker: &WorkerHealthSnapshot) -> bool {
|
||||
match worker.state {
|
||||
WorkerRuntimeState::Disabled | WorkerRuntimeState::Standby => true,
|
||||
|
||||
@@ -89,6 +89,7 @@ pub fn gateway_error_from_engine(error: &EngineError) -> GatewayError {
|
||||
| EngineError::ProviderMetadataMismatch
|
||||
| EngineError::ContinuationPinMismatch
|
||||
| EngineError::RequiredAccountMismatch
|
||||
| EngineError::AccountOutsideClientScope
|
||||
| EngineError::DownstreamCommitRequired
|
||||
| EngineError::NoActiveAttempt
|
||||
| EngineError::InvalidDeliveryState => {
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
//! Codex Images 非流式 HTTP adapter。
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
extract::{Extension, State, connect_info::ConnectInfo},
|
||||
http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header::CONTENT_TYPE},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use gateway_core::engine::execution::StartedExecution;
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::operation::{ImageRequest, ImageRequestKind, Operation, RawJsonPayload};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, authentication_error_response},
|
||||
error::{engine_error_response, gateway_error_response, protocol_error_response},
|
||||
responses::{PendingExecution, ProtocolError, ProtocolErrorBody, request_client_context},
|
||||
};
|
||||
|
||||
const OPENAI_PROTOCOL: &str = "openai";
|
||||
const IMAGE_TURN_ID_CONTEXT_KEY: &str = "image_turn_id";
|
||||
|
||||
/// `POST /v1/images/generations`。
|
||||
pub(crate) async fn image_generations(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_image_request(
|
||||
state,
|
||||
connect_info,
|
||||
headers,
|
||||
body,
|
||||
ImageRequestKind::Generation,
|
||||
"/v1/images/generations",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `POST /v1/images/edits`。
|
||||
pub(crate) async fn image_edits(
|
||||
State(state): State<ApiState>,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
) -> Response {
|
||||
handle_image_request(
|
||||
state,
|
||||
connect_info,
|
||||
headers,
|
||||
body,
|
||||
ImageRequestKind::Edit,
|
||||
"/v1/images/edits",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn handle_image_request(
|
||||
state: ApiState,
|
||||
connect_info: Option<Extension<ConnectInfo<SocketAddr>>>,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
kind: ImageRequestKind,
|
||||
endpoint: &'static str,
|
||||
) -> Response {
|
||||
let service = state.openai();
|
||||
let client = match authenticate_client(service, &headers) {
|
||||
Ok(client) => client,
|
||||
Err(error) => return authentication_error_response(error),
|
||||
};
|
||||
let (client_ip, user_agent) = request_client_context(
|
||||
&headers,
|
||||
connect_info.map(|Extension(ConnectInfo(address))| address),
|
||||
);
|
||||
let operation = match image_operation(body, &headers, kind) {
|
||||
Ok(operation) => operation,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
let started = match service
|
||||
.start_image(client, operation, client_ip, user_agent, endpoint)
|
||||
.await
|
||||
{
|
||||
Ok(started) => started,
|
||||
Err(error) => return gateway_error_response(&error),
|
||||
};
|
||||
collect_image_response(started).await
|
||||
}
|
||||
|
||||
fn image_operation(
|
||||
body: Bytes,
|
||||
headers: &HeaderMap,
|
||||
kind: ImageRequestKind,
|
||||
) -> Result<Operation, GatewayError> {
|
||||
let mut context = Map::new();
|
||||
if let Some(turn_id) = headers
|
||||
.get("x-codex-image-turn-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
{
|
||||
context.insert(
|
||||
IMAGE_TURN_ID_CONTEXT_KEY.to_owned(),
|
||||
Value::String(turn_id.to_owned()),
|
||||
);
|
||||
}
|
||||
let payload = RawJsonPayload::new(OPENAI_PROTOCOL, body)
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI protocol identifier is invalid",
|
||||
)
|
||||
})?
|
||||
.with_context(context);
|
||||
Ok(Operation::GenerateImage(ImageRequest::from_raw_json(
|
||||
kind, payload,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn collect_image_response(started: StartedExecution) -> Response {
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
let events = match session.collect_uncommitted().await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
};
|
||||
let body = match raw_json_body(events) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
let response = invalid_upstream_response();
|
||||
let response = execution.record_response_status(response).await;
|
||||
execution.cancel_and_finalize().await;
|
||||
return response;
|
||||
}
|
||||
};
|
||||
let status = session
|
||||
.response_status_code()
|
||||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.filter(StatusCode::is_success)
|
||||
.unwrap_or(StatusCode::OK);
|
||||
let response_headers = session.response_headers().to_vec();
|
||||
let response = json_body_response(body, status, &response_headers);
|
||||
let Some(session) = execution.session_mut() else {
|
||||
return invalid_upstream_response();
|
||||
};
|
||||
if let Err(error) = session.commit_downstream(Some(status.as_u16())).await {
|
||||
execution.cancel_and_finalize().await;
|
||||
let response = engine_error_response(&error);
|
||||
return execution.record_response_status(response).await;
|
||||
}
|
||||
if !session.is_finalized() {
|
||||
execution.cancel_and_finalize().await;
|
||||
return invalid_upstream_response();
|
||||
}
|
||||
execution.disarm();
|
||||
response
|
||||
}
|
||||
|
||||
fn raw_json_body(events: Vec<ProviderEvent>) -> Option<Bytes> {
|
||||
let mut body = None;
|
||||
for event in events {
|
||||
let (_, wire) = event.into_parts();
|
||||
let Some(wire) = wire.filter(|wire| wire.protocol() == "openai") else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = wire.into_raw_json_body() else {
|
||||
continue;
|
||||
};
|
||||
if body.replace(raw).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn json_body_response(
|
||||
body: Bytes,
|
||||
status: StatusCode,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
apply_response_headers(response, response_headers)
|
||||
}
|
||||
|
||||
fn apply_response_headers(
|
||||
mut response: Response,
|
||||
response_headers: &[ProviderResponseHeader],
|
||||
) -> Response {
|
||||
let connection_options =
|
||||
crate::openai::responses::response_connection_options(response_headers);
|
||||
for header in response_headers {
|
||||
if !crate::openai::responses::response_header_is_forwardable(
|
||||
header.name(),
|
||||
&connection_options,
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let Ok(name) = HeaderName::from_bytes(header.name().as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_bytes(header.value()) else {
|
||||
continue;
|
||||
};
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
fn invalid_upstream_response() -> Response {
|
||||
protocol_error_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
ProtocolErrorBody {
|
||||
error: ProtocolError {
|
||||
kind: "server_error",
|
||||
code: "invalid_upstream_response",
|
||||
message: "The gateway could not forward the upstream image response.".to_owned(),
|
||||
param: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! OpenAI/Codex Images JSON 透明 adapter。
|
||||
|
||||
mod http;
|
||||
|
||||
pub(crate) use http::{image_edits, image_generations};
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub mod auth;
|
||||
pub mod error;
|
||||
pub mod images;
|
||||
pub mod models;
|
||||
pub mod responses;
|
||||
pub mod router;
|
||||
|
||||
@@ -123,7 +123,7 @@ async fn handle_responses(
|
||||
}
|
||||
|
||||
/// 从 socket 与标准转发头提取旧 Usage 页面使用的诊断事实。
|
||||
pub(super) fn request_client_context(
|
||||
pub(in crate::openai) fn request_client_context(
|
||||
headers: &HeaderMap,
|
||||
peer_address: Option<SocketAddr>,
|
||||
) -> (Option<IpAddr>, Option<String>) {
|
||||
@@ -367,22 +367,24 @@ fn internal_gateway_response(message: &'static str) -> Response {
|
||||
gateway_error_response(&GatewayError::new(GatewayErrorKind::Internal, message))
|
||||
}
|
||||
|
||||
pub(super) struct PendingExecution {
|
||||
pub(in crate::openai) struct PendingExecution {
|
||||
session: Option<Box<dyn ExecutionSession>>,
|
||||
}
|
||||
|
||||
impl PendingExecution {
|
||||
pub(super) fn new(session: Box<dyn ExecutionSession>) -> Self {
|
||||
pub(in crate::openai) fn new(session: Box<dyn ExecutionSession>) -> Self {
|
||||
Self {
|
||||
session: Some(session),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn session_mut(&mut self) -> Option<&mut (dyn ExecutionSession + 'static)> {
|
||||
pub(in crate::openai) fn session_mut(
|
||||
&mut self,
|
||||
) -> Option<&mut (dyn ExecutionSession + 'static)> {
|
||||
self.session.as_deref_mut()
|
||||
}
|
||||
|
||||
async fn cancel_and_finalize(&mut self) {
|
||||
pub(in crate::openai) async fn cancel_and_finalize(&mut self) {
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
session.cancel();
|
||||
if !session.is_finalized() {
|
||||
@@ -391,7 +393,10 @@ impl PendingExecution {
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_response_status(&mut self, response: Response) -> Response {
|
||||
pub(in crate::openai) async fn record_response_status(
|
||||
&mut self,
|
||||
response: Response,
|
||||
) -> Response {
|
||||
if let Some(session) = self.session.as_mut() {
|
||||
let _ = session
|
||||
.record_client_status(response.status().as_u16())
|
||||
@@ -400,7 +405,7 @@ impl PendingExecution {
|
||||
response
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) {
|
||||
pub(in crate::openai) fn disarm(&mut self) {
|
||||
self.session = None;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,8 @@ mod request;
|
||||
mod response;
|
||||
mod websocket;
|
||||
|
||||
pub use error::{ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub use error::{ProtocolError, ProtocolErrorBody, RequestDecodeError, ResponseEncodeError};
|
||||
pub(super) use http::{PendingExecution, request_client_context};
|
||||
pub use http::{collect_execution_response, stream_execution_response};
|
||||
pub(crate) use http::{responses, review_responses};
|
||||
pub use request::{
|
||||
@@ -73,7 +74,6 @@ pub(super) fn response_header_is_forwardable(name: &str, connection_options: &[S
|
||||
| "x-openai-organization"
|
||||
| "x-openai-project"
|
||||
| "x-codex-installation-id"
|
||||
| "x-codex-turn-state"
|
||||
| "x-codex-turn-metadata"
|
||||
)
|
||||
}
|
||||
|
||||
@@ -165,7 +165,6 @@ impl fmt::Debug for ContinuationIntent {
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ResponsesRequestMetadata {
|
||||
requested_model: String,
|
||||
public_model: String,
|
||||
stream: bool,
|
||||
store: bool,
|
||||
continuation: ContinuationIntent,
|
||||
@@ -180,12 +179,6 @@ impl ResponsesRequestMetadata {
|
||||
&self.requested_model
|
||||
}
|
||||
|
||||
/// 返回协议层公开模型候选。
|
||||
#[must_use]
|
||||
pub fn public_model(&self) -> &str {
|
||||
&self.public_model
|
||||
}
|
||||
|
||||
/// 返回客户端是否请求 SSE。
|
||||
#[must_use]
|
||||
pub const fn stream(&self) -> bool {
|
||||
@@ -222,7 +215,6 @@ impl fmt::Debug for ResponsesRequestMetadata {
|
||||
formatter
|
||||
.debug_struct("ResponsesRequestMetadata")
|
||||
.field("requested_model", &self.requested_model)
|
||||
.field("public_model", &self.public_model)
|
||||
.field("stream", &self.stream)
|
||||
.field("store", &self.store)
|
||||
.field("continuation", &self.continuation)
|
||||
@@ -366,8 +358,7 @@ pub(super) fn decode_request_object(
|
||||
Ok(DecodedResponsesRequest {
|
||||
operation,
|
||||
metadata: ResponsesRequestMetadata {
|
||||
requested_model: model.clone(),
|
||||
public_model: model,
|
||||
requested_model: model,
|
||||
stream,
|
||||
store,
|
||||
continuation,
|
||||
|
||||
@@ -7,6 +7,7 @@ use axum::{
|
||||
};
|
||||
|
||||
use super::{
|
||||
images::{image_edits, image_generations},
|
||||
models::{model_catalog, model_detail, model_info, models},
|
||||
responses::{responses, responses_websocket, review_responses},
|
||||
};
|
||||
@@ -16,6 +17,8 @@ use crate::ApiState;
|
||||
/// 构造 OpenAI 客户端协议路由。
|
||||
pub(crate) fn router() -> Router<ApiState> {
|
||||
Router::new()
|
||||
.route("/v1/images/generations", post(image_generations))
|
||||
.route("/v1/images/edits", post(image_edits))
|
||||
.route("/v1/responses", get(responses_websocket).post(responses))
|
||||
.route("/v1/responses/review", post(review_responses))
|
||||
.route("/v1/models", get(models))
|
||||
@@ -23,7 +26,7 @@ pub(crate) fn router() -> Router<ApiState> {
|
||||
.route("/v1/models/{model_id}/info", get(model_info))
|
||||
// 官方 OpenAI 模型详情合同使用 path ID;它不属于 Admin API 约束。
|
||||
.route("/v1/models/{model_id}", get(model_detail))
|
||||
// Responses 正文属于 OpenAI/Codex 协议;代理不能用私有大小上限提前拒绝
|
||||
// OpenAI 数据面正文属于客户端/上游协议;代理不能用私有大小上限提前拒绝
|
||||
// 上游本可接受的未来 payload。
|
||||
.layer(DefaultBodyLimit::disable())
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
//! OpenAI wire adapter 到 Core 执行用例的唯一映射。
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_core::engine::continuation::PreviousResponseId;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionRequestMetadata,
|
||||
ExecutionService, StartExecution, StartedExecution,
|
||||
ExecutionService, StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLifecycle};
|
||||
@@ -74,13 +75,13 @@ impl OpenAiService {
|
||||
endpoint: &'static str,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
let (operation, metadata) = request.into_parts();
|
||||
let public_model = PublicModelId::from_client_wire(metadata.public_model().to_owned())
|
||||
let public_model = PublicModelId::from_client_wire(metadata.requested_model().to_owned())
|
||||
.map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
)
|
||||
})?;
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::ModelNotFound,
|
||||
"requested model was not found",
|
||||
)
|
||||
})?;
|
||||
let previous_response_id = match metadata.continuation() {
|
||||
ContinuationIntent::None => None,
|
||||
ContinuationIntent::PreviousResponseId(value) => {
|
||||
@@ -105,6 +106,38 @@ impl OpenAiService {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_image(
|
||||
&self,
|
||||
client: AuthenticatedClient,
|
||||
operation: gateway_core::operation::Operation,
|
||||
client_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
endpoint: &'static str,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
let provider = gateway_core::routing::ProviderKind::new("openai").map_err(|_| {
|
||||
GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"OpenAI provider identifier is invalid",
|
||||
)
|
||||
})?;
|
||||
self.execution
|
||||
.start_provider_endpoint(StartProviderExecution {
|
||||
client,
|
||||
provider,
|
||||
operation,
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip,
|
||||
user_agent,
|
||||
previous_response_id: None,
|
||||
},
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn try_register_connection(
|
||||
&self,
|
||||
) -> Result<Box<dyn ConnectionGuard>, ConnectionDraining> {
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, to_bytes},
|
||||
http::{Method, Request, StatusCode, header},
|
||||
};
|
||||
use gateway_api::admin::account_groups;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use super::{AdminTestFixture, AdminTestState, PRIMARY_GROUP_ID};
|
||||
|
||||
const SESSION_COOKIE: &str = "cpr_admin_session=valid-session";
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_route_should_keep_camel_case_group_and_page_wire() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::GET,
|
||||
"/api/admin/account-groups?page=1&pageSize=1&search=alpha&enabled=true",
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let value = response_json(response).await;
|
||||
assert_eq!(value["data"]["page"]["page"], 1);
|
||||
assert_eq!(value["data"]["page"]["pageSize"], 1);
|
||||
assert_eq!(value["data"]["page"]["total"], 1);
|
||||
assert_eq!(value["data"]["page"]["totalPages"], 1);
|
||||
assert_eq!(value["data"]["configRevision"], 7);
|
||||
assert_eq!(value["data"]["items"][0]["id"], PRIMARY_GROUP_ID);
|
||||
assert_eq!(value["data"]["items"][0]["memberCount"], 2);
|
||||
assert_eq!(value["data"]["items"][0]["providerCounts"]["openai"], 1);
|
||||
assert_eq!(value["data"]["items"][0]["clientKeyCount"], 2);
|
||||
assert_eq!(value["data"]["items"][0]["accountSummary"]["available"], 1);
|
||||
assert_eq!(value["data"]["items"][0]["accountSummary"]["limited"], 1);
|
||||
assert_eq!(value["data"]["items"][0]["accountSummary"]["total"], 2);
|
||||
assert_eq!(value["data"]["items"][0]["capacity"]["usedSlots"], 0);
|
||||
assert_eq!(value["data"]["items"][0]["capacity"]["totalSlots"], 1);
|
||||
assert_eq!(value["data"]["items"][0]["usage"]["todayUsd"], "1.25");
|
||||
assert_eq!(value["data"]["items"][0]["usage"]["totalUsd"], "5.5");
|
||||
assert!(value["data"]["items"][0].get("member_count").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_route_should_create_an_empty_group() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/create",
|
||||
Some(json!({
|
||||
"name": "Gamma routing",
|
||||
"description": "New traffic pool",
|
||||
"color": "#a855f780"
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let value = response_json(response).await;
|
||||
assert!(
|
||||
value["data"]["id"]
|
||||
.as_str()
|
||||
.is_some_and(|id| id.starts_with("grp_") && id.len() == 36)
|
||||
);
|
||||
assert_eq!(value["data"]["record"]["name"], "Gamma routing");
|
||||
assert_eq!(value["data"]["record"]["memberCount"], 0);
|
||||
assert_eq!(value["data"]["record"]["color"], "#A855F780");
|
||||
assert_eq!(value["data"]["configRevision"], 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_route_should_replace_group_fields() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/update",
|
||||
Some(json!({
|
||||
"id": PRIMARY_GROUP_ID,
|
||||
"name": "Renamed routing",
|
||||
"description": null,
|
||||
"color": "#F43F5ECC"
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let value = response_json(response).await;
|
||||
assert_eq!(value["data"]["record"]["name"], "Renamed routing");
|
||||
assert!(value["data"]["record"]["description"].is_null());
|
||||
assert_eq!(value["data"]["record"]["color"], "#F43F5ECC");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enable_and_disable_routes_should_set_explicit_group_state() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let disabled = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/disable",
|
||||
Some(json!({ "id": PRIMARY_GROUP_ID })),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
let enabled = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/enable",
|
||||
Some(json!({ "id": PRIMARY_GROUP_ID })),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(disabled.status(), StatusCode::OK);
|
||||
assert!(
|
||||
!response_json(disabled).await["data"]["record"]["enabled"]
|
||||
.as_bool()
|
||||
.expect("disabled flag")
|
||||
);
|
||||
assert_eq!(enabled.status(), StatusCode::OK);
|
||||
assert!(
|
||||
response_json(enabled).await["data"]["record"]["enabled"]
|
||||
.as_bool()
|
||||
.expect("enabled flag")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_route_should_return_deleted_id_without_a_live_record() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/delete",
|
||||
Some(json!({ "id": PRIMARY_GROUP_ID })),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let value = response_json(response).await;
|
||||
assert_eq!(value["data"]["id"], PRIMARY_GROUP_ID);
|
||||
assert!(value["data"]["record"].is_null());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_route_should_reject_unknown_and_invalid_pagination_fields() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
for uri in [
|
||||
"/api/admin/account-groups?other=true",
|
||||
"/api/admin/account-groups?page=0",
|
||||
"/api/admin/account-groups?pageSize=0",
|
||||
"/api/admin/account-groups?pageSize=201",
|
||||
] {
|
||||
let response = request(router(&fixture), Method::GET, uri, None, true).await;
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutation_route_should_reject_unknown_json_fields() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/create",
|
||||
Some(json!({
|
||||
"name": "Unknown field",
|
||||
"description": null,
|
||||
"color": "#2563EBFF",
|
||||
"providerKind": "openai"
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_route_should_reject_six_digit_color_without_alpha() {
|
||||
let fixture = authenticated_fixture().await;
|
||||
let response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/create",
|
||||
Some(json!({
|
||||
"name": "Opaque legacy color",
|
||||
"description": null,
|
||||
"color": "#2563EB"
|
||||
})),
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_group_routes_should_require_admin_authentication() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
let get_response = request(
|
||||
router(&fixture),
|
||||
Method::GET,
|
||||
"/api/admin/account-groups",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
let post_response = request(
|
||||
router(&fixture),
|
||||
Method::POST,
|
||||
"/api/admin/account-groups/create",
|
||||
Some(json!({ "name": "Unauthorized", "description": null, "color": "#2563EBFF" })),
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(get_response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(post_response.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
async fn authenticated_fixture() -> AdminTestFixture {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
fixture
|
||||
}
|
||||
|
||||
fn router(fixture: &AdminTestFixture) -> Router {
|
||||
account_groups::router::<AdminTestState>().with_state(fixture.state())
|
||||
}
|
||||
|
||||
async fn request(
|
||||
router: Router,
|
||||
method: Method,
|
||||
uri: &str,
|
||||
body: Option<Value>,
|
||||
authenticated: bool,
|
||||
) -> axum::response::Response {
|
||||
let mut builder = Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("x-request-id", "req_account_groups");
|
||||
if authenticated {
|
||||
builder = builder.header(header::COOKIE, SESSION_COOKIE);
|
||||
}
|
||||
let body = match body {
|
||||
Some(value) => {
|
||||
builder = builder.header(header::CONTENT_TYPE, "application/json");
|
||||
Body::from(serde_json::to_vec(&value).expect("serialize request body"))
|
||||
}
|
||||
None => Body::empty(),
|
||||
};
|
||||
router
|
||||
.oneshot(builder.body(body).expect("account group request"))
|
||||
.await
|
||||
.expect("account group response")
|
||||
}
|
||||
|
||||
async fn response_json(response: axum::response::Response) -> Value {
|
||||
let body = to_bytes(response.into_body(), 64 * 1024)
|
||||
.await
|
||||
.expect("account group response body");
|
||||
serde_json::from_slice(&body).expect("account group response JSON")
|
||||
}
|
||||
@@ -10,7 +10,7 @@ mod query {
|
||||
"pageSize": 20,
|
||||
"provider": "xai",
|
||||
"search": " operator ",
|
||||
"status": "active",
|
||||
"status": "normal",
|
||||
"sortBy": "lastUsedAt",
|
||||
"sortDirection": "desc"
|
||||
}))
|
||||
@@ -23,7 +23,7 @@ mod query {
|
||||
Some(ref provider) if provider.as_str() == "xai"
|
||||
));
|
||||
assert_eq!(query.search.as_deref(), Some("operator"));
|
||||
assert_eq!(query.status, Some(AccountStatus::Active));
|
||||
assert_eq!(query.status, Some(AccountStatus::Normal));
|
||||
assert_eq!(
|
||||
query.sort.expect("sort").field,
|
||||
AccountSortField::LastUsedAt
|
||||
@@ -34,6 +34,19 @@ mod query {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_parse_rate_limited_status() {
|
||||
let query: ListQuery = serde_json::from_value(json!({
|
||||
"status": "rate_limited"
|
||||
}))
|
||||
.expect("deserialize account query");
|
||||
|
||||
assert_eq!(
|
||||
query.validate().expect("validate account query").status,
|
||||
Some(AccountStatus::RateLimited)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unbounded_page_size() {
|
||||
let query: ListQuery =
|
||||
@@ -57,8 +70,138 @@ mod query {
|
||||
}
|
||||
}
|
||||
|
||||
mod batch_update {
|
||||
use gateway_api::admin::accounts::{BatchUpdateAccountsRequest, UpdateAccountRequest};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn batch_update_should_accept_complete_atomic_payload() {
|
||||
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
|
||||
"accountIds": ["acct_openai", "acct_xai"],
|
||||
"enabled": false,
|
||||
"concurrencyLimit": null,
|
||||
"weight": 1,
|
||||
"groupIds": ["grp_00000000000000000000000000000001"]
|
||||
}))
|
||||
.expect("deserialize batch update");
|
||||
|
||||
request.validate().expect("validate batch update");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_update_should_reject_duplicate_or_invalid_account_ids() {
|
||||
for account_ids in [
|
||||
json!(["acct_same", "acct_same"]),
|
||||
json!(["not-an-account"]),
|
||||
json!([]),
|
||||
] {
|
||||
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
|
||||
"accountIds": account_ids,
|
||||
"enabled": true,
|
||||
"concurrencyLimit": 8,
|
||||
"weight": 100,
|
||||
"groupIds": []
|
||||
}))
|
||||
.expect("deserialize invalid batch update");
|
||||
assert_eq!(
|
||||
request.validate().expect_err("reject account IDs").field(),
|
||||
"accountIds"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_update_should_require_enabled_and_reject_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"],
|
||||
"concurrencyLimit": null,
|
||||
"weight": 1,
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"],
|
||||
"enabled": true,
|
||||
"concurrencyLimit": null,
|
||||
"weight": 1,
|
||||
"groupIds": [],
|
||||
"legacy": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_update_should_reject_invalid_scheduling_bounds_and_missing_fields() {
|
||||
for (concurrency_limit, weight, field) in [
|
||||
(json!(0), json!(1), "concurrencyLimit"),
|
||||
(json!(4294967296_u64), json!(1), "concurrencyLimit"),
|
||||
(json!(null), json!(0), "weight"),
|
||||
(json!(null), json!(101), "weight"),
|
||||
] {
|
||||
let request: BatchUpdateAccountsRequest = serde_json::from_value(json!({
|
||||
"accountIds": ["acct_test"],
|
||||
"enabled": true,
|
||||
"concurrencyLimit": concurrency_limit,
|
||||
"weight": weight,
|
||||
"groupIds": []
|
||||
}))
|
||||
.expect("deserialize invalid scheduling");
|
||||
assert_eq!(
|
||||
request.validate().expect_err("reject scheduling").field(),
|
||||
field
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"],
|
||||
"enabled": true,
|
||||
"weight": 1,
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"],
|
||||
"enabled": true,
|
||||
"concurrencyLimit": null,
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_update_should_require_the_same_complete_scheduling_contract() {
|
||||
let request: UpdateAccountRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_test",
|
||||
"enabled": true,
|
||||
"concurrencyLimit": 4294967295_u64,
|
||||
"weight": 100,
|
||||
"groupIds": []
|
||||
}))
|
||||
.expect("deserialize single update");
|
||||
request.validate().expect("validate single update");
|
||||
|
||||
assert!(
|
||||
serde_json::from_value::<UpdateAccountRequest>(json!({
|
||||
"accountId": "acct_test",
|
||||
"enabled": true,
|
||||
"weight": 1,
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod response {
|
||||
use gateway_api::admin::accounts::AccountUsageView;
|
||||
use gateway_api::admin::presenter::format_decimal_currency;
|
||||
|
||||
#[test]
|
||||
fn account_usage_view_should_keep_unobserved_numbers_null() {
|
||||
@@ -99,12 +242,22 @@ mod response {
|
||||
assert!(value["totalTokens"].is_null());
|
||||
assert_eq!(value["createdTokensDisplay"], "-");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_usage_currency_should_limit_usd_to_four_fraction_digits() {
|
||||
assert_eq!(format_decimal_currency("0.1204956", "USD"), "$0.1205");
|
||||
assert_eq!(format_decimal_currency("0.99996", "USD"), "$1.00");
|
||||
assert_eq!(format_decimal_currency("0.0300", "USD"), "$0.03");
|
||||
assert_eq!(format_decimal_currency("12.3456", "USD"), "$12.35");
|
||||
assert_eq!(format_decimal_currency("0.1204956", "CNY"), "CNY 0.1204956");
|
||||
}
|
||||
}
|
||||
|
||||
mod actions {
|
||||
use base64::Engine as _;
|
||||
use gateway_admin::model::{
|
||||
Revision,
|
||||
accounts::{AccountConnectionTestEvent as DomainConnectionTestEvent, AccountStatus},
|
||||
accounts::AccountConnectionTestEvent as DomainConnectionTestEvent,
|
||||
provider_credentials::{
|
||||
CredentialDeletionResult, CredentialImportResult, CredentialMutationResult,
|
||||
},
|
||||
@@ -113,7 +266,8 @@ mod actions {
|
||||
AccountActionRequest, AccountConnectionTestEvent, AccountDeletionData,
|
||||
AccountDeletionRequest, AccountExportData, AccountExportQuery, AccountIdQuery,
|
||||
AccountImportData, AccountImportRequest, AccountMutationData, AccountRefreshRequest,
|
||||
AccountTestQuery, RotateAccountRequest, StartAccountAuthorizationRequest,
|
||||
AccountResetCreditConsumeRequest, AccountTestQuery, CompleteAccountAuthorizationRequest,
|
||||
RotateAccountRequest, StartAccountAuthorizationRequest,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use serde_json::json;
|
||||
@@ -171,6 +325,44 @@ mod actions {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_credit_consume_should_require_canonical_v4_idempotency_key() {
|
||||
let valid: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1",
|
||||
"creditId": "credit_1",
|
||||
"redeemRequestId": "8fbf302d-11df-4bd5-82e4-08e4b3df7874"
|
||||
}))
|
||||
.expect("decode reset-credit consume");
|
||||
valid.validate().expect("validate reset-credit consume");
|
||||
|
||||
for (redeem_request_id, field) in [
|
||||
("019c0000-0000-7000-8000-000000000000", "redeemRequestId"),
|
||||
("8FBF302D-11DF-4BD5-82E4-08E4B3DF7874", "redeemRequestId"),
|
||||
("invalid", "redeemRequestId"),
|
||||
] {
|
||||
let request: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1",
|
||||
"redeemRequestId": redeem_request_id
|
||||
}))
|
||||
.expect("decode invalid reset-credit consume");
|
||||
assert_eq!(request.validate().expect_err("reject UUID").field(), field);
|
||||
}
|
||||
|
||||
let invalid_credit: AccountResetCreditConsumeRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1",
|
||||
"creditId": " ",
|
||||
"redeemRequestId": "8fbf302d-11df-4bd5-82e4-08e4b3df7874"
|
||||
}))
|
||||
.expect("decode invalid credit");
|
||||
assert_eq!(
|
||||
invalid_credit
|
||||
.validate()
|
||||
.expect_err("reject credit")
|
||||
.field(),
|
||||
"creditId"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_recovery_requests_should_not_accept_client_revision_fences() {
|
||||
let authorization: StartAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
@@ -194,7 +386,8 @@ mod actions {
|
||||
"provider": "openai",
|
||||
"accountId": "acct_1",
|
||||
"accessToken": "header.payload.signature",
|
||||
"refreshToken": "refresh-token"
|
||||
"refreshToken": "refresh-token",
|
||||
"idToken": "id-header.id-payload.id-signature"
|
||||
}))
|
||||
.expect("decode rotation");
|
||||
assert!(rotation.validate().is_ok());
|
||||
@@ -256,6 +449,35 @@ mod actions {
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"data": {},
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_complete_should_not_accept_group_assignment() {
|
||||
let flow_id = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7_u8; 32]);
|
||||
let request: CompleteAccountAuthorizationRequest = serde_json::from_value(json!({
|
||||
"provider": "openai",
|
||||
"flowId": flow_id,
|
||||
"callbackUrl": "http://localhost/callback?code=ok"
|
||||
}))
|
||||
.expect("decode OAuth completion");
|
||||
assert!(request.validate().is_ok());
|
||||
assert!(
|
||||
serde_json::from_value::<CompleteAccountAuthorizationRequest>(json!({
|
||||
"provider": "xai",
|
||||
"flowId": "flow_test",
|
||||
"callbackUrl": "http://localhost/callback?code=ok",
|
||||
"groupIds": []
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -292,7 +514,7 @@ mod actions {
|
||||
|
||||
#[test]
|
||||
fn account_import_response_should_emit_account_ids() {
|
||||
let response = AccountImportData::from(CredentialImportResult {
|
||||
let response = AccountImportData::from_result(CredentialImportResult {
|
||||
config_revision: Revision::new(8).expect("revision"),
|
||||
credential_ids: vec![ProviderAccountId::new("acct_imported").expect("account ID")],
|
||||
});
|
||||
@@ -330,9 +552,7 @@ mod actions {
|
||||
DomainConnectionTestEvent::Content {
|
||||
text: "OK".to_owned(),
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {
|
||||
account_status: AccountStatus::Active,
|
||||
},
|
||||
DomainConnectionTestEvent::Completed {},
|
||||
DomainConnectionTestEvent::Failed {
|
||||
message: "upstream unavailable".to_owned(),
|
||||
provider_error_code: Some("usage_exhausted".to_owned()),
|
||||
@@ -340,7 +560,6 @@ mod actions {
|
||||
upstream_status: Some(429),
|
||||
upstream_content_type: Some("application/json".to_owned()),
|
||||
upstream_body: Some(r#"{"error":{"type":"usage_limit_reached"}}"#.to_owned()),
|
||||
account_status: AccountStatus::QuotaExhausted,
|
||||
},
|
||||
]
|
||||
.map(|event| AccountConnectionTestEvent::from(event).data);
|
||||
@@ -365,7 +584,7 @@ mod actions {
|
||||
}
|
||||
}),
|
||||
json!({ "type": "content", "text": "OK" }),
|
||||
json!({ "type": "test_complete", "success": true, "accountStatus": "active" }),
|
||||
json!({ "type": "test_complete", "success": true }),
|
||||
json!({
|
||||
"type": "error",
|
||||
"error": "upstream unavailable",
|
||||
@@ -373,8 +592,7 @@ mod actions {
|
||||
"providerErrorType": "invalid_request_error",
|
||||
"upstreamStatus": 429,
|
||||
"upstreamContentType": "application/json",
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#,
|
||||
"accountStatus": "quota_exhausted"
|
||||
"upstreamBody": r#"{"error":{"type":"usage_limit_reached"}}"#
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
@@ -105,7 +105,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
let valid = serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"name": "terminal key",
|
||||
"label": "production",
|
||||
"providerKind": "openai",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 2,
|
||||
"requestsPerMinute": 60
|
||||
}))
|
||||
@@ -118,7 +118,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
(
|
||||
json!({
|
||||
"name": " ",
|
||||
"providerKind": "openai",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0
|
||||
}),
|
||||
@@ -127,7 +127,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
(
|
||||
json!({
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": u64::MAX,
|
||||
"requestsPerMinute": 0
|
||||
}),
|
||||
@@ -150,7 +150,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
"id": "key_1",
|
||||
"expectedConfigRevision": 1,
|
||||
"name": "key",
|
||||
"providerKind": "openai",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0,
|
||||
"tokensPerMinute": 0
|
||||
@@ -161,7 +161,7 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"name": "terminal key",
|
||||
"providerKind": "openai",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 2,
|
||||
"requestsPerMinute": 60
|
||||
}))
|
||||
@@ -210,7 +210,10 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
id: gateway_core::policy::ClientApiKeyId::new("key_visible").expect("Client Key ID"),
|
||||
name: "visible".to_owned(),
|
||||
label: None,
|
||||
provider_kind: gateway_core::routing::ProviderKind::new("openai").expect("Provider kind"),
|
||||
groups: Vec::new(),
|
||||
provider_kinds: vec![
|
||||
gateway_core::routing::ProviderKind::new("openai").expect("Provider kind"),
|
||||
],
|
||||
prefix: "sk_visible12".to_owned(),
|
||||
enabled: true,
|
||||
limits: gateway_core::policy::RateLimits {
|
||||
@@ -226,7 +229,11 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
assert!(list.get("configRevision").is_none());
|
||||
assert_eq!(list["total"], 1);
|
||||
assert_eq!(list["items"][0]["id"], "key_visible");
|
||||
assert_eq!(list["items"][0]["providerKind"], "openai");
|
||||
assert_eq!(
|
||||
list["items"][0]["providerKinds"],
|
||||
serde_json::json!(["openai"])
|
||||
);
|
||||
assert_eq!(list["items"][0]["routingScope"], "all");
|
||||
assert_eq!(list["items"][0]["maxConcurrency"], 2);
|
||||
assert_eq!(list["items"][0]["requestsPerMinute"], 60);
|
||||
assert!(list["items"][0].get("tokensPerMinute").is_none());
|
||||
|
||||
@@ -7,15 +7,22 @@ use std::{
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_admin::{
|
||||
AdminConfig, AdminServices, InitialAdminPassword,
|
||||
model::{
|
||||
MutationContext, Revision,
|
||||
account_groups::{
|
||||
AccountGroupAccountSummary, AccountGroupCapacity, AccountGroupColor,
|
||||
AccountGroupListQuery, AccountGroupMutation, AccountGroupPage, AccountGroupRecord,
|
||||
AccountGroupUsage, DeleteAccountGroup, NewAccountGroup, SetAccountGroupEnabled,
|
||||
UpdateAccountGroup,
|
||||
},
|
||||
accounts::{
|
||||
AccountListQuery, AccountPage, AccountRecord, AccountUsage, AccountUsageWindowQuery,
|
||||
AccountUsageWindowResult, DeleteAccounts, SetAccountEnabled,
|
||||
AccountListQuery, AccountPage, AccountPageItem, AccountUpdateResult, AccountUsage,
|
||||
AccountUsageWindowQuery, AccountUsageWindowResult, AccountsUpdateResult,
|
||||
BatchUpdateAccounts, DeleteAccounts, UpdateAccount,
|
||||
},
|
||||
auth::{AdminAuditEvent, AdminSession},
|
||||
client_keys::{
|
||||
@@ -23,9 +30,9 @@ use gateway_admin::{
|
||||
NewClientKey, SetClientKeyEnabled, UpdateClientKey,
|
||||
},
|
||||
observability::{
|
||||
DashboardObservation, DiagnosticDimension, DiagnosticObservation, OpsError,
|
||||
OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail, UsageFilter,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageRecord,
|
||||
DashboardObservation, DecimalAmount, DiagnosticDimension, DiagnosticObservation,
|
||||
OpsError, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange, UsageDetail,
|
||||
UsageFilter, UsageOverview, UsagePage, UsageQuery, UsageRecord,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -45,8 +52,8 @@ use gateway_admin::{
|
||||
ports::{
|
||||
provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind},
|
||||
store::{
|
||||
AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult,
|
||||
AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
|
||||
AccountGroupStore, AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts,
|
||||
AdminStoreResult, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore,
|
||||
},
|
||||
system::{
|
||||
SystemOperationError, SystemOperationErrorKind, SystemOperations,
|
||||
@@ -66,6 +73,7 @@ use gateway_core::{
|
||||
},
|
||||
};
|
||||
|
||||
mod account_groups;
|
||||
mod accounts;
|
||||
mod auth;
|
||||
mod client_keys;
|
||||
@@ -83,6 +91,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
pub ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
impl AdminTestFixture {
|
||||
@@ -95,20 +104,24 @@ impl AdminTestFixture {
|
||||
let auth = Arc::new(MemoryAuthStore::new(api_key.clone()));
|
||||
let settings = Arc::new(MemorySettingsStore::new(api_key));
|
||||
let client_keys = Arc::new(MemoryClientKeyStore);
|
||||
let account_groups = Arc::new(MemoryAccountGroupStore::new());
|
||||
let usage_records = Arc::new(Mutex::new(Vec::new()));
|
||||
let usage_detail = Arc::new(Mutex::new(None));
|
||||
let diagnostics = Arc::new(Mutex::new(Vec::new()));
|
||||
let ops_errors = Arc::new(Mutex::new(Vec::new()));
|
||||
let dashboard_observation = Arc::new(Mutex::new(None));
|
||||
let dashboard_summary_range = Arc::new(Mutex::new(None));
|
||||
let unused = Arc::new(UnusedStore {
|
||||
usage_records: Arc::clone(&usage_records),
|
||||
usage_detail: Arc::clone(&usage_detail),
|
||||
diagnostics: Arc::clone(&diagnostics),
|
||||
ops_errors: Arc::clone(&ops_errors),
|
||||
dashboard_observation: Arc::clone(&dashboard_observation),
|
||||
dashboard_summary_range: Arc::clone(&dashboard_summary_range),
|
||||
});
|
||||
let stores = AdminStorePorts::new(
|
||||
unused.clone(),
|
||||
account_groups.clone(),
|
||||
auth.clone(),
|
||||
client_keys.clone(),
|
||||
unused,
|
||||
@@ -142,6 +155,7 @@ impl AdminTestFixture {
|
||||
diagnostics,
|
||||
ops_errors,
|
||||
dashboard_observation,
|
||||
dashboard_summary_range,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -341,6 +355,204 @@ impl SettingsStore for MemorySettingsStore {
|
||||
|
||||
pub(super) struct MemoryClientKeyStore;
|
||||
|
||||
pub(super) const PRIMARY_GROUP_ID: &str = "grp_11111111111111111111111111111111";
|
||||
pub(super) const SECONDARY_GROUP_ID: &str = "grp_22222222222222222222222222222222";
|
||||
|
||||
struct MemoryAccountGroupState {
|
||||
revision: Revision,
|
||||
groups: BTreeMap<gateway_core::routing::AccountGroupId, AccountGroupRecord>,
|
||||
}
|
||||
|
||||
pub(super) struct MemoryAccountGroupStore {
|
||||
state: Mutex<MemoryAccountGroupState>,
|
||||
}
|
||||
|
||||
impl MemoryAccountGroupStore {
|
||||
fn new() -> Self {
|
||||
let now = Utc::now();
|
||||
let primary_id = group_id(PRIMARY_GROUP_ID);
|
||||
let secondary_id = group_id(SECONDARY_GROUP_ID);
|
||||
let groups = BTreeMap::from([
|
||||
(
|
||||
primary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
id: primary_id,
|
||||
name: "Alpha routing".to_owned(),
|
||||
description: Some("Primary traffic".to_owned()),
|
||||
color: group_color("#2563ebff"),
|
||||
enabled: true,
|
||||
member_count: 2,
|
||||
provider_counts: BTreeMap::from([
|
||||
("openai".to_owned(), 1),
|
||||
("xai".to_owned(), 1),
|
||||
]),
|
||||
client_key_count: 2,
|
||||
account_summary: account_summary(1, 1, 2),
|
||||
capacity: capacity(Some(0), 1),
|
||||
usage: usage("1.25", "5.5"),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
),
|
||||
(
|
||||
secondary_id.clone(),
|
||||
AccountGroupRecord {
|
||||
id: secondary_id,
|
||||
name: "Beta routing".to_owned(),
|
||||
description: None,
|
||||
color: group_color("#64748B80"),
|
||||
enabled: false,
|
||||
member_count: 0,
|
||||
provider_counts: BTreeMap::new(),
|
||||
client_key_count: 0,
|
||||
account_summary: account_summary(0, 0, 0),
|
||||
capacity: capacity(Some(0), 0),
|
||||
usage: usage("0", "0"),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
),
|
||||
]);
|
||||
Self {
|
||||
state: Mutex::new(MemoryAccountGroupState {
|
||||
revision: Revision::new(7).expect("revision"),
|
||||
groups,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl AccountGroupStore for MemoryAccountGroupStore {
|
||||
async fn list_account_groups(
|
||||
&self,
|
||||
query: AccountGroupListQuery,
|
||||
) -> AdminStoreResult<AccountGroupPage> {
|
||||
let state = self.state.lock().expect("account groups");
|
||||
let search = query.search.as_ref().map(|value| value.to_lowercase());
|
||||
let mut matching = state
|
||||
.groups
|
||||
.values()
|
||||
.filter(|record| {
|
||||
query
|
||||
.enabled
|
||||
.is_none_or(|enabled| record.enabled == enabled)
|
||||
})
|
||||
.filter(|record| {
|
||||
search.as_ref().is_none_or(|search| {
|
||||
record.name.to_lowercase().contains(search)
|
||||
|| record
|
||||
.description
|
||||
.as_ref()
|
||||
.is_some_and(|description| description.to_lowercase().contains(search))
|
||||
})
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
matching.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
let total = matching.len() as u64;
|
||||
let page_size = usize::from(query.page_size.get());
|
||||
let offset = usize::try_from(query.page.saturating_sub(1))
|
||||
.unwrap_or(usize::MAX)
|
||||
.saturating_mul(page_size);
|
||||
let items = matching.into_iter().skip(offset).take(page_size).collect();
|
||||
Ok(AccountGroupPage {
|
||||
config_revision: state.revision,
|
||||
items,
|
||||
total,
|
||||
page: query.page,
|
||||
page_size: query.page_size.get(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn create_account_group(
|
||||
&self,
|
||||
command: NewAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
let now = Utc::now();
|
||||
let record = AccountGroupRecord {
|
||||
id: command.id.clone(),
|
||||
name: command.name,
|
||||
description: command.description,
|
||||
color: command.color,
|
||||
enabled: true,
|
||||
member_count: 0,
|
||||
provider_counts: BTreeMap::new(),
|
||||
client_key_count: 0,
|
||||
account_summary: account_summary(0, 0, 0),
|
||||
capacity: capacity(Some(0), 0),
|
||||
usage: usage("0", "0"),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
state.groups.insert(command.id.clone(), record);
|
||||
mutation(&mut state, command.id, true)
|
||||
}
|
||||
|
||||
async fn update_account_group(
|
||||
&self,
|
||||
command: UpdateAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
let record = state
|
||||
.groups
|
||||
.get_mut(&command.id)
|
||||
.ok_or_else(|| not_found("account group"))?;
|
||||
record.name = command.name;
|
||||
record.description = command.description;
|
||||
record.color = command.color;
|
||||
record.updated_at = Utc::now();
|
||||
mutation(&mut state, command.id, true)
|
||||
}
|
||||
|
||||
async fn set_account_group_enabled(
|
||||
&self,
|
||||
command: SetAccountGroupEnabled,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
let record = state
|
||||
.groups
|
||||
.get_mut(&command.id)
|
||||
.ok_or_else(|| not_found("account group"))?;
|
||||
record.enabled = command.enabled;
|
||||
record.updated_at = Utc::now();
|
||||
mutation(&mut state, command.id, true)
|
||||
}
|
||||
|
||||
async fn delete_account_group(
|
||||
&self,
|
||||
command: DeleteAccountGroup,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
let mut state = self.state.lock().expect("account groups");
|
||||
if state.groups.remove(&command.id).is_none() {
|
||||
return Err(not_found("account group"));
|
||||
}
|
||||
mutation(&mut state, command.id, false)
|
||||
}
|
||||
}
|
||||
|
||||
fn group_id(value: &str) -> gateway_core::routing::AccountGroupId {
|
||||
gateway_core::routing::AccountGroupId::new(value).expect("account group ID")
|
||||
}
|
||||
|
||||
fn mutation(
|
||||
state: &mut MemoryAccountGroupState,
|
||||
id: gateway_core::routing::AccountGroupId,
|
||||
include_record: bool,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
state.revision = next_revision(state.revision);
|
||||
Ok(AccountGroupMutation {
|
||||
config_revision: state.revision,
|
||||
record: include_record.then(|| state.groups.get(&id).expect("group record").clone()),
|
||||
id,
|
||||
})
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ClientKeyStore for MemoryClientKeyStore {
|
||||
async fn list_client_keys(&self, _: ClientKeyListQuery) -> AdminStoreResult<ClientKeyPage> {
|
||||
@@ -362,7 +574,8 @@ impl ClientKeyStore for MemoryClientKeyStore {
|
||||
id: id.clone(),
|
||||
name: "revealed".to_owned(),
|
||||
label: None,
|
||||
provider_kind: ProviderKind::new("openai").expect("provider kind"),
|
||||
groups: Vec::new(),
|
||||
provider_kinds: vec![ProviderKind::new("openai").expect("provider kind")],
|
||||
prefix: "sk_aaaaaaaaa".to_owned(),
|
||||
enabled: true,
|
||||
limits: RateLimits::unlimited(),
|
||||
@@ -413,6 +626,7 @@ struct UnusedStore {
|
||||
diagnostics: Arc<Mutex<Vec<DiagnosticObservation>>>,
|
||||
ops_errors: Arc<Mutex<Vec<OpsError>>>,
|
||||
dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -421,7 +635,7 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("account list"))
|
||||
}
|
||||
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountRecord>> {
|
||||
async fn load_account(&self, _: &str) -> AdminStoreResult<Option<AccountPageItem>> {
|
||||
Err(unavailable("account"))
|
||||
}
|
||||
|
||||
@@ -496,14 +710,30 @@ impl AccountStore for UnusedStore {
|
||||
Err(unavailable("credential refresh"))
|
||||
}
|
||||
|
||||
async fn set_account_enabled(
|
||||
async fn update_account(
|
||||
&self,
|
||||
_: SetAccountEnabled,
|
||||
_: UpdateAccount,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<Revision> {
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
Err(unavailable("account enabled"))
|
||||
}
|
||||
|
||||
async fn recover_account(
|
||||
&self,
|
||||
_: &ProviderAccountId,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountUpdateResult> {
|
||||
Err(unavailable("account recovery"))
|
||||
}
|
||||
|
||||
async fn batch_update_accounts(
|
||||
&self,
|
||||
_: BatchUpdateAccounts,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountsUpdateResult> {
|
||||
Err(unavailable("account batch update"))
|
||||
}
|
||||
|
||||
async fn delete_accounts(
|
||||
&self,
|
||||
_: DeleteAccounts,
|
||||
@@ -523,7 +753,15 @@ impl AccountStore for UnusedStore {
|
||||
|
||||
#[async_trait]
|
||||
impl ObservabilityStore for UnusedStore {
|
||||
async fn dashboard_summary(&self, _: TimeRange) -> AdminStoreResult<DashboardObservation> {
|
||||
async fn dashboard_summary(
|
||||
&self,
|
||||
range: TimeRange,
|
||||
_: DateTime<Utc>,
|
||||
) -> AdminStoreResult<DashboardObservation> {
|
||||
*self
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range") = Some(range);
|
||||
self.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation")
|
||||
@@ -779,6 +1017,32 @@ fn next_revision(revision: Revision) -> Revision {
|
||||
Revision::new(revision.get().saturating_add(1)).expect("next revision")
|
||||
}
|
||||
|
||||
fn account_summary(available: u64, limited: u64, total: u64) -> AccountGroupAccountSummary {
|
||||
AccountGroupAccountSummary {
|
||||
available,
|
||||
limited,
|
||||
total,
|
||||
}
|
||||
}
|
||||
|
||||
fn group_color(value: &str) -> AccountGroupColor {
|
||||
AccountGroupColor::parse(value).expect("group color")
|
||||
}
|
||||
|
||||
fn capacity(used_slots: Option<u64>, total_slots: u64) -> AccountGroupCapacity {
|
||||
AccountGroupCapacity {
|
||||
used_slots,
|
||||
total_slots,
|
||||
}
|
||||
}
|
||||
|
||||
fn usage(today_usd: &str, total_usd: &str) -> AccountGroupUsage {
|
||||
AccountGroupUsage {
|
||||
today_usd: today_usd.parse::<DecimalAmount>().expect("today cost"),
|
||||
total_usd: total_usd.parse::<DecimalAmount>().expect("total cost"),
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable(resource: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
@@ -787,6 +1051,10 @@ fn unavailable(resource: &'static str) -> AdminStoreError {
|
||||
)
|
||||
}
|
||||
|
||||
fn not_found(resource: &'static str) -> AdminStoreError {
|
||||
AdminStoreError::new(AdminStoreErrorKind::NotFound, resource, "not found")
|
||||
}
|
||||
|
||||
fn unsupported_provider() -> ProviderAdminError {
|
||||
ProviderAdminError::new(ProviderAdminErrorKind::Unsupported)
|
||||
}
|
||||
|
||||
@@ -219,18 +219,14 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics {
|
||||
total: 807,
|
||||
enabled: 804,
|
||||
unavailable: 792,
|
||||
active: 15,
|
||||
total: 1002,
|
||||
normal: 943,
|
||||
quota_exhausted: 52,
|
||||
rate_limited: 0,
|
||||
expired: 2,
|
||||
invalid: 0,
|
||||
quota_exhausted: 785,
|
||||
refreshing: None,
|
||||
disabled: 3,
|
||||
banned: 2,
|
||||
error: 4,
|
||||
},
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
@@ -263,11 +259,160 @@ async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headli
|
||||
&value["data"]["cards"]["credentials"]["unavailableValue"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!(807),
|
||||
&serde_json::json!(15),
|
||||
&serde_json::json!(792)
|
||||
&serde_json::json!(1002),
|
||||
&serde_json::json!(943),
|
||||
&serde_json::json!(59)
|
||||
),
|
||||
);
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["credentials"]["total"],
|
||||
&value["data"]["cards"]["credentials"]["available"],
|
||||
&value["data"]["cards"]["credentials"]["unavailable"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("1002"),
|
||||
&serde_json::json!("943"),
|
||||
&serde_json::json!("59")
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_use_lifetime_totals_for_card_footers() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, DashboardTotals, RequestMetrics,
|
||||
TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
let range_requests = RequestMetrics {
|
||||
request_count: 2,
|
||||
input_tokens: 8,
|
||||
cached_tokens: 2,
|
||||
total_tokens: 12,
|
||||
..RequestMetrics::default()
|
||||
};
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: range_requests,
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: DashboardTotals {
|
||||
request_count: 42,
|
||||
input_tokens: 800,
|
||||
cached_tokens: 200,
|
||||
total_tokens: 840,
|
||||
billing_usd: None,
|
||||
},
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_lifetime_totals")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.expect("dashboard response body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("dashboard response JSON");
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
&value["data"]["cards"]["traffic"]["totalRequests"],
|
||||
&value["data"]["cards"]["tokens"]["totalTokens"],
|
||||
&value["data"]["cards"]["cache"]["totalHitRate"],
|
||||
&value["data"]["cards"]["cache"]["totalCachedTokens"],
|
||||
),
|
||||
(
|
||||
&serde_json::json!("42"),
|
||||
&serde_json::json!("840"),
|
||||
&serde_json::json!("25.0%"),
|
||||
&serde_json::json!("200"),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_default_to_current_china_day() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{Duration, Utc};
|
||||
use gateway_admin::model::observability::{
|
||||
AccountPoolMetrics, AttemptMetrics, DashboardObservation, RequestMetrics, TimeRange,
|
||||
china_day_start,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture
|
||||
.dashboard_observation
|
||||
.lock()
|
||||
.expect("dashboard observation") = Some(DashboardObservation {
|
||||
range: TimeRange::new(now - Duration::hours(1), now).expect("dashboard range"),
|
||||
requests: RequestMetrics::default(),
|
||||
attempts: AttemptMetrics::default(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: AccountPoolMetrics::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: Vec::new(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/dashboard/summary")
|
||||
.header(header::COOKIE, "cpr_admin_session=valid-session")
|
||||
.header("x-request-id", "req_dashboard_today_range")
|
||||
.body(Body::empty())
|
||||
.expect("dashboard request"),
|
||||
)
|
||||
.await
|
||||
.expect("dashboard response");
|
||||
let range = fixture
|
||||
.dashboard_summary_range
|
||||
.lock()
|
||||
.expect("dashboard summary range")
|
||||
.take()
|
||||
.expect("recorded dashboard summary range");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(range.start, china_day_start(range.end));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -541,11 +686,14 @@ fn usage_record_with_account(
|
||||
id: id.to_owned(),
|
||||
client_api_key_ref: "key_detail".to_owned(),
|
||||
config_revision: 1,
|
||||
routing_scope: "all".to_owned(),
|
||||
routing_group_refs: Vec::new(),
|
||||
routing_group_names_snapshot: Vec::new(),
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "responses".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "coding".to_owned(),
|
||||
requested_model_id: Some("coding".to_owned()),
|
||||
provider_kind: Some("openai".to_owned()),
|
||||
provider_account_ref: Some(account_id.to_owned()),
|
||||
provider_account_name: Some(name.to_owned()),
|
||||
@@ -667,11 +815,14 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
id: "request_endpoint".to_owned(),
|
||||
client_api_key_ref: "key_endpoint".to_owned(),
|
||||
config_revision: 1,
|
||||
routing_scope: "groups".to_owned(),
|
||||
routing_group_refs: vec!["grp_usage".to_owned()],
|
||||
routing_group_names_snapshot: vec!["Usage group".to_owned()],
|
||||
protocol: "openai".to_owned(),
|
||||
operation: "generate".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model_id: "grok-4.5".to_owned(),
|
||||
requested_model_id: Some("grok-4.5".to_owned()),
|
||||
provider_kind: Some("xai".to_owned()),
|
||||
provider_account_ref: Some("acct_snapshot".to_owned()),
|
||||
provider_account_name: Some("Snapshot Alpha".to_owned()),
|
||||
@@ -786,6 +937,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"route": value["data"]["items"][0]["route"],
|
||||
"routingScope": value["data"]["items"][0]["routingScope"],
|
||||
"routingGroupRefs": value["data"]["items"][0]["routingGroupRefs"],
|
||||
"routingGroupNamesSnapshot": value["data"]["items"][0]["routingGroupNamesSnapshot"],
|
||||
"serviceTier": value["data"]["items"][0]["serviceTier"],
|
||||
"accountId": value["data"]["items"][0]["accountId"],
|
||||
"accountName": value["data"]["items"][0]["accountName"],
|
||||
@@ -816,6 +970,9 @@ async fn usage_route_should_expose_image_and_websocket_facts() {
|
||||
}),
|
||||
serde_json::json!({
|
||||
"route": "/v1/responses",
|
||||
"routingScope": "groups",
|
||||
"routingGroupRefs": ["grp_usage"],
|
||||
"routingGroupNamesSnapshot": ["Usage group"],
|
||||
"serviceTier": "priority",
|
||||
"accountId": "acct_snapshot",
|
||||
"accountName": "Snapshot Alpha",
|
||||
|
||||
@@ -33,6 +33,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
let mut actual = rust_files(&root.join("src"));
|
||||
actual.sort();
|
||||
let mut expected = vec![
|
||||
"src/admin/account_groups.rs",
|
||||
"src/admin/accounts.rs",
|
||||
"src/admin/auth.rs",
|
||||
"src/admin/backups.rs",
|
||||
@@ -51,6 +52,8 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/lib.rs",
|
||||
"src/openai/auth.rs",
|
||||
"src/openai/error.rs",
|
||||
"src/openai/images/http.rs",
|
||||
"src/openai/images/mod.rs",
|
||||
"src/openai/mod.rs",
|
||||
"src/openai/models.rs",
|
||||
"src/openai/responses/error.rs",
|
||||
@@ -73,6 +76,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
let mut actual = rust_files(&root.join("tests"));
|
||||
actual.sort();
|
||||
let mut expected = vec![
|
||||
"tests/admin/account_groups.rs",
|
||||
"tests/admin/accounts.rs",
|
||||
"tests/admin/auth.rs",
|
||||
"tests/admin/client_keys.rs",
|
||||
@@ -88,6 +92,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/main.rs",
|
||||
"tests/openai/auth.rs",
|
||||
"tests/openai/error.rs",
|
||||
"tests/openai/images.rs",
|
||||
"tests/openai/mod.rs",
|
||||
"tests/openai/models.rs",
|
||||
"tests/openai/responses/http.rs",
|
||||
|
||||
@@ -8,10 +8,12 @@ use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, StartExecution,
|
||||
StartedExecution,
|
||||
StartProviderExecution, StartedExecution,
|
||||
},
|
||||
error::GatewayError,
|
||||
health::{WorkerHealthKey, WorkerHealthSnapshot, WorkerHealthSource, WorkerRuntimeState},
|
||||
routing::PublicModelId,
|
||||
task::{WorkerId, WorkerKind},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
@@ -31,6 +33,75 @@ async fn healthz_should_return_no_content_when_all_inputs_are_healthy() {
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthz_should_ignore_provider_maintenance_worker_failure() {
|
||||
let worker_health = StaticWorkerHealth(vec![worker_snapshot(
|
||||
WorkerKind::QuotaCatalogHealth,
|
||||
WorkerRuntimeState::BackingOff,
|
||||
)]);
|
||||
let response = crate::openai::api_router_with_worker_health(
|
||||
Arc::new(UnusedExecution),
|
||||
Arc::new(worker_health),
|
||||
)
|
||||
.await
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/healthz")
|
||||
.body(Body::empty())
|
||||
.expect("health request"),
|
||||
)
|
||||
.await
|
||||
.expect("health response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthz_should_reject_critical_worker_failure() {
|
||||
let worker_health = StaticWorkerHealth(vec![worker_snapshot(
|
||||
WorkerKind::RuntimeSnapshotReconciliation,
|
||||
WorkerRuntimeState::BackingOff,
|
||||
)]);
|
||||
let response = crate::openai::api_router_with_worker_health(
|
||||
Arc::new(UnusedExecution),
|
||||
Arc::new(worker_health),
|
||||
)
|
||||
.await
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/healthz")
|
||||
.body(Body::empty())
|
||||
.expect("health request"),
|
||||
)
|
||||
.await
|
||||
.expect("health response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct StaticWorkerHealth(Vec<WorkerHealthSnapshot>);
|
||||
|
||||
impl WorkerHealthSource for StaticWorkerHealth {
|
||||
fn snapshot(&self) -> Vec<WorkerHealthSnapshot> {
|
||||
self.0.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn worker_snapshot(kind: WorkerKind, state: WorkerRuntimeState) -> WorkerHealthSnapshot {
|
||||
let id = WorkerId::try_new(kind, "health-test").expect("worker ID");
|
||||
WorkerHealthSnapshot {
|
||||
key: WorkerHealthKey::Task(id),
|
||||
state,
|
||||
consecutive_failures: 1,
|
||||
completed_cycles: 0,
|
||||
last_fencing_token: None,
|
||||
last_success_at: None,
|
||||
last_failure_at: None,
|
||||
last_error: Some("test worker failure".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
struct UnusedExecution;
|
||||
|
||||
impl ExecutionService for UnusedExecution {
|
||||
@@ -49,4 +120,11 @@ impl ExecutionService for UnusedExecution {
|
||||
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async { unreachable!("health check does not execute requests") })
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
_: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async { unreachable!("health check does not execute provider endpoints") })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::UpstreamSendState;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ClientTransport, ExecutionService,
|
||||
ExecutionSession, StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CoordinatedEvent, EngineError, ModelRequestId};
|
||||
use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayError, GatewayErrorKind, ProviderError, ProviderErrorKind,
|
||||
};
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::operation::{ImageRequestKind, Operation};
|
||||
use gateway_core::routing::PublicModelId;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
const IMAGE_RESPONSE: &[u8] =
|
||||
br#"{ "created": 1787212800, "data": [{"b64_json":"AAEC"}], "future": 9007199254740993 }"#;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
struct CapturedImageRequest {
|
||||
provider: String,
|
||||
endpoint: String,
|
||||
transport: ClientTransport,
|
||||
kind: ImageRequestKind,
|
||||
body: Bytes,
|
||||
context: Value,
|
||||
}
|
||||
|
||||
struct ImageExecution {
|
||||
client: AuthenticatedClient,
|
||||
captured: Arc<Mutex<Vec<CapturedImageRequest>>>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
fail_with_upstream_response: bool,
|
||||
}
|
||||
|
||||
impl ImageExecution {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_images_test"),
|
||||
captured: Arc::new(Mutex::new(Vec::new())),
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
fail_with_upstream_response: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn failing() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
client: authenticated_client("sk_images_test"),
|
||||
captured: Arc::new(Mutex::new(Vec::new())),
|
||||
committed_statuses: Arc::new(Mutex::new(Vec::new())),
|
||||
fail_with_upstream_response: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn captured(&self) -> Vec<CapturedImageRequest> {
|
||||
self.captured.lock().expect("capture lock").clone()
|
||||
}
|
||||
|
||||
fn committed_statuses(&self) -> Vec<u16> {
|
||||
self.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionService for ImageExecution {
|
||||
fn authenticate(
|
||||
&self,
|
||||
plaintext: &str,
|
||||
) -> Result<AuthenticatedClient, ClientAuthenticationError> {
|
||||
(plaintext == "sk_images_test")
|
||||
.then(|| self.client.clone())
|
||||
.ok_or(ClientAuthenticationError::InvalidKey)
|
||||
}
|
||||
|
||||
fn public_models(&self, _: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, _: &AuthenticatedClient, _: &PublicModelId) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn start(&self, _: StartExecution) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"images test must use provider endpoint execution",
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move {
|
||||
let Operation::GenerateImage(image) = &request.operation else {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"images test received a non-image operation",
|
||||
));
|
||||
};
|
||||
let payload = image.payload();
|
||||
self.captured
|
||||
.lock()
|
||||
.expect("capture lock")
|
||||
.push(CapturedImageRequest {
|
||||
provider: request.provider.as_str().to_owned(),
|
||||
endpoint: request.metadata.endpoint,
|
||||
transport: request.metadata.transport,
|
||||
kind: image.kind(),
|
||||
body: payload.body().clone(),
|
||||
context: Value::Object(payload.context().clone()),
|
||||
});
|
||||
Ok(StartedExecution {
|
||||
request_id: ModelRequestId::new("req_images_test").expect("request ID"),
|
||||
created_at: SystemTime::now(),
|
||||
stream: false,
|
||||
session: Box::new(ImageSession {
|
||||
response: Some(Bytes::from_static(IMAGE_RESPONSE)),
|
||||
response_headers: vec![ProviderResponseHeader::new(
|
||||
"x-image-rate-limit",
|
||||
Bytes::from_static(b"42"),
|
||||
)],
|
||||
committed_statuses: Arc::clone(&self.committed_statuses),
|
||||
finalized: Arc::new(AtomicBool::new(false)),
|
||||
fail_with_upstream_response: self.fail_with_upstream_response,
|
||||
}),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ImageSession {
|
||||
response: Option<Bytes>,
|
||||
response_headers: Vec<ProviderResponseHeader>,
|
||||
committed_statuses: Arc<Mutex<Vec<u16>>>,
|
||||
finalized: Arc<AtomicBool>,
|
||||
fail_with_upstream_response: bool,
|
||||
}
|
||||
|
||||
impl ExecutionSession for ImageSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async { unreachable!("buffered image delivery does not stream") })
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async move {
|
||||
if self.fail_with_upstream_response {
|
||||
return Err(EngineError::Provider(
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::InvalidRequest,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(StatusCode::UNPROCESSABLE_ENTITY.as_u16())
|
||||
.with_client_visible_upstream_response(
|
||||
ClientVisibleUpstreamResponse::new(
|
||||
StatusCode::UNPROCESSABLE_ENTITY.as_u16(),
|
||||
Some(b"application/problem+json".to_vec()),
|
||||
Bytes::from_static(
|
||||
br#"{ "error":{"message":"future image validation"}, "future":9007199254740993 }"#,
|
||||
),
|
||||
)
|
||||
.with_headers(vec![ProviderResponseHeader::new(
|
||||
"x-future-image-error",
|
||||
Bytes::from_static(b"preserved"),
|
||||
)]),
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(vec![ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_json(
|
||||
"openai",
|
||||
self.response.take().expect("single image response"),
|
||||
)
|
||||
.expect("OpenAI protocol"),
|
||||
)])
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.response_headers
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
Some(StatusCode::CREATED.as_u16())
|
||||
}
|
||||
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.committed_statuses
|
||||
.lock()
|
||||
.expect("committed status lock")
|
||||
.push(client_status_code.expect("image response status"));
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_routes_should_not_decode_bodies_and_should_preserve_both_directions() {
|
||||
let execution = ImageExecution::new();
|
||||
let router = api_router(execution.clone()).await;
|
||||
let cases = [
|
||||
(
|
||||
"/v1/images/generations",
|
||||
ImageRequestKind::Generation,
|
||||
br#"{ "model":"gpt-image-future", "prompt":"a lighthouse", "future":9007199254740993 }"#.as_slice(),
|
||||
),
|
||||
(
|
||||
"/v1/images/edits",
|
||||
ImageRequestKind::Edit,
|
||||
br#"{"model":"gpt-image-2","images":[{"image_url":"data:image/png;base64,AAEC"}],"prompt":"add fog","prompt":"duplicate stays raw"}"#.as_slice(),
|
||||
),
|
||||
];
|
||||
|
||||
for (path, _, body) in &cases {
|
||||
let response = router
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::post(*path)
|
||||
.header(AUTHORIZATION, "Bearer sk_images_test")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-codex-image-turn-id", "turn_image_route")
|
||||
.body(Body::from(body.to_vec()))
|
||||
.expect("image request"),
|
||||
)
|
||||
.await
|
||||
.expect("image response");
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
assert_eq!(
|
||||
response.headers().get("x-image-rate-limit"),
|
||||
Some(&"42".parse().expect("header value"))
|
||||
);
|
||||
let response_body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read image response");
|
||||
assert_eq!(response_body.as_ref(), IMAGE_RESPONSE);
|
||||
}
|
||||
|
||||
let captured = execution.captured();
|
||||
assert_eq!(captured.len(), cases.len());
|
||||
for (captured, (endpoint, kind, body)) in captured.iter().zip(cases.iter()) {
|
||||
assert_eq!(captured.provider, "openai");
|
||||
assert_eq!(captured.endpoint, *endpoint);
|
||||
assert_eq!(captured.transport, ClientTransport::HttpJson);
|
||||
assert_eq!(captured.kind, *kind);
|
||||
assert_eq!(captured.body.as_ref(), *body);
|
||||
assert_eq!(
|
||||
captured.context,
|
||||
json!({"image_turn_id": "turn_image_route"})
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
execution.committed_statuses(),
|
||||
vec![StatusCode::CREATED.as_u16(); cases.len()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_route_should_return_the_exact_upstream_error_response() {
|
||||
let execution = ImageExecution::failing();
|
||||
let response = api_router(execution)
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/images/edits")
|
||||
.header(AUTHORIZATION, "Bearer sk_images_test")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
br#"{ "model":"gpt-image-2", "future_invalid":9007199254740993 }"#.to_vec(),
|
||||
))
|
||||
.expect("image request"),
|
||||
)
|
||||
.await
|
||||
.expect("image response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(
|
||||
response.headers().get("content-type"),
|
||||
Some(&"application/problem+json".parse().expect("content type"))
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers().get("x-future-image-error"),
|
||||
Some(&"preserved".parse().expect("header value"))
|
||||
);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("read image response");
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
br#"{ "error":{"message":"future image validation"}, "future":9007199254740993 }"#
|
||||
);
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
mod auth;
|
||||
mod error;
|
||||
mod images;
|
||||
mod models;
|
||||
mod responses;
|
||||
mod router;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -18,7 +19,9 @@ use gateway_core::engine::admission::{
|
||||
use gateway_core::engine::continuation::{
|
||||
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::{AccountSelectionPolicy, RotationStrategy};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientApiKeyUsageSink, DefaultExecutionService, ExecutionService,
|
||||
ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
@@ -34,17 +37,37 @@ use gateway_core::lifecycle::{ConnectionDraining, ConnectionGuard, ConnectionLif
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ProviderKind,
|
||||
ProviderModel, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
|
||||
pub(super) async fn api_router(execution: Arc<dyn ExecutionService>) -> axum::Router {
|
||||
api_router_with_origins(execution, Vec::new()).await
|
||||
}
|
||||
|
||||
pub(super) async fn api_router_with_worker_health(
|
||||
execution: Arc<dyn ExecutionService>,
|
||||
worker_health: Arc<dyn WorkerHealthSource>,
|
||||
) -> axum::Router {
|
||||
api_router_with_origins_and_worker_health(execution, Vec::new(), worker_health).await
|
||||
}
|
||||
|
||||
pub(super) async fn api_router_with_origins(
|
||||
execution: Arc<dyn ExecutionService>,
|
||||
cors_allowed_origins: Vec<String>,
|
||||
) -> axum::Router {
|
||||
api_router_with_origins_and_worker_health(
|
||||
execution,
|
||||
cors_allowed_origins,
|
||||
Arc::new(EmptyWorkerHealth),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn api_router_with_origins_and_worker_health(
|
||||
execution: Arc<dyn ExecutionService>,
|
||||
cors_allowed_origins: Vec<String>,
|
||||
worker_health: Arc<dyn WorkerHealthSource>,
|
||||
) -> axum::Router {
|
||||
let admin = crate::admin::AdminTestFixture::new().await;
|
||||
gateway_api::initialize(
|
||||
@@ -57,7 +80,7 @@ pub(super) async fn api_router_with_origins(
|
||||
execution,
|
||||
admin.services,
|
||||
Vec::new(),
|
||||
Arc::new(EmptyWorkerHealth),
|
||||
worker_health,
|
||||
Arc::new(TestLifecycle::default()),
|
||||
)
|
||||
.expect("API bundle")
|
||||
@@ -94,6 +117,10 @@ impl ClientApiKeyUsageSink for IgnoredClientApiKeyUsage {
|
||||
|
||||
fn snapshot(plaintext: &str, provider_name: &str) -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new(provider_name).expect("provider");
|
||||
let account_directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
|
||||
ProviderAccountId::new("acct_api_test").expect("account ID"),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
|
||||
)])));
|
||||
let capabilities = ModelCapabilities::new(
|
||||
BTreeSet::from([gateway_core::operation::OperationKind::Generate]),
|
||||
Some(16_000),
|
||||
@@ -119,12 +146,16 @@ fn snapshot(plaintext: &str, provider_name: &str) -> RuntimeSnapshot {
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_api_test").expect("key ID"),
|
||||
PlaintextClientApiKey::new(plaintext).expect("plaintext key"),
|
||||
provider,
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
)
|
||||
.expect("runtime snapshot")
|
||||
.with_account_directory(account_directory)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -260,6 +291,7 @@ struct UnusedContinuation;
|
||||
impl NativeContinuationPort for UnusedContinuation {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientApiKeyId,
|
||||
_: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
Box::pin(async { unreachable!("authentication fixture does not execute") })
|
||||
|
||||
@@ -7,7 +7,7 @@ use axum::{
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, StartExecution,
|
||||
StartedExecution,
|
||||
StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::routing::{
|
||||
@@ -99,6 +99,18 @@ impl ExecutionService for ModelsExecution {
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
_: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"models test must not start a provider endpoint",
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn authorized_request(path: &str) -> Request<Body> {
|
||||
|
||||
@@ -17,7 +17,7 @@ use bytes::Bytes;
|
||||
use futures::future::{BoxFuture, pending};
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartedExecution,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{
|
||||
@@ -190,6 +190,18 @@ impl ExecutionService for ContextCaptureExecution {
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
_: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async {
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"responses context test must not start a provider endpoint",
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FakeSession {
|
||||
@@ -788,6 +800,10 @@ async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresenta
|
||||
let trace = Arc::new(Trace::default());
|
||||
let headers = vec![
|
||||
ProviderResponseHeader::new("x-models-etag", Bytes::from_static(b"models-v2")),
|
||||
ProviderResponseHeader::new(
|
||||
"x-codex-turn-state",
|
||||
Bytes::from_static(b"turn-state-from-upstream"),
|
||||
),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
@@ -834,6 +850,13 @@ async fn streaming_response_should_append_ordinary_headers_and_skip_unrepresenta
|
||||
.map(HeaderValue::as_bytes),
|
||||
Some(b"\xffopaque".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-codex-turn-state")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("turn-state-from-upstream")
|
||||
);
|
||||
assert!(response.headers().get("authorization").is_none());
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-hop-secret").is_none());
|
||||
|
||||
@@ -130,7 +130,7 @@ fn decoder_should_preserve_opaque_client_model_values() {
|
||||
"input": "hello"
|
||||
}));
|
||||
|
||||
assert_eq!(decoded.metadata().public_model(), model);
|
||||
assert_eq!(decoded.metadata().requested_model(), model);
|
||||
assert_eq!(
|
||||
openai_wire_body(&decoded)
|
||||
.get("model")
|
||||
@@ -451,15 +451,23 @@ fn decoder_should_not_reject_large_bodies_using_catalog_context_limits() {
|
||||
.to_string();
|
||||
let decoded = decode_request(body.as_bytes()).expect("large request should decode");
|
||||
|
||||
let plan = super::snapshot("sk_context_test", "openai")
|
||||
let snapshot = super::snapshot("sk_context_test", "openai");
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new(decoded.metadata().public_model()).expect("public model"),
|
||||
&PublicModelId::new(decoded.metadata().requested_model()).expect("public model"),
|
||||
decoded.operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext::default(),
|
||||
)
|
||||
.expect("large body must not be locally context-gated");
|
||||
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "model-a");
|
||||
assert_eq!(
|
||||
plan.candidates()[0]
|
||||
.upstream_model()
|
||||
.expect("model route candidate")
|
||||
.as_str(),
|
||||
"model-a"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -13,7 +13,7 @@ use futures::{SinkExt, StreamExt, future::BoxFuture};
|
||||
use gateway_api::openai::responses::ResponseCreateFrameError;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionService, ExecutionSession,
|
||||
StartExecution, StartedExecution,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CommitRequirement, CoordinatedEvent, EngineError, UpstreamSendState};
|
||||
use gateway_core::error::{GatewayError, ProviderError, ProviderErrorKind};
|
||||
@@ -271,6 +271,13 @@ impl ExecutionService for AtomicFailureExecution {
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
_: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async { unreachable!("WebSocket test does not execute provider endpoints") })
|
||||
}
|
||||
}
|
||||
|
||||
fn atomic_failure_batch() -> CoordinatedEvent {
|
||||
@@ -411,6 +418,10 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
|
||||
ProviderResponseHeader::new("x-future-header", Bytes::from_static(b"future-value")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"first")),
|
||||
ProviderResponseHeader::new("x-future-multi", Bytes::from_static(b"second")),
|
||||
ProviderResponseHeader::new(
|
||||
"x-codex-turn-state",
|
||||
Bytes::from_static(b"turn-state-from-upstream"),
|
||||
),
|
||||
ProviderResponseHeader::new("x-future-bytes", Bytes::from_static(b"\xffopaque")),
|
||||
ProviderResponseHeader::new("bad\0name", Bytes::from_static(b"unrepresentable")),
|
||||
ProviderResponseHeader::new(
|
||||
@@ -483,6 +494,10 @@ async fn websocket_response_metadata_should_preserve_ordinary_headers_without_bl
|
||||
// 官方 metadata 是 string map,同名多值只能保持既有的后值覆盖语义。
|
||||
assert_eq!(headers.get("x-future-multi"), Some(&json!("second")));
|
||||
assert_eq!(headers.get("x-request-id"), Some(&json!("req_upstream")));
|
||||
assert_eq!(
|
||||
headers.get("x-codex-turn-state"),
|
||||
Some(&json!("turn-state-from-upstream"))
|
||||
);
|
||||
for omitted in [
|
||||
"x-future-bytes",
|
||||
"bad\0name",
|
||||
|
||||
@@ -9,6 +9,7 @@ use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::ProviderAccountId;
|
||||
use crate::operation::ProviderSessionState;
|
||||
use crate::policy::ClientApiKeyId;
|
||||
use crate::routing::ProviderKind;
|
||||
|
||||
/// 客户端或 Provider 传递的 opaque response handle。
|
||||
@@ -48,14 +49,15 @@ pub enum NativeContinuationScope {
|
||||
|
||||
/// 从可丢失会话亲和存储恢复的 native previous-response pin。
|
||||
///
|
||||
/// 该值不依赖调用方 API Key;它只阻止 native handle 与不透明 Provider state
|
||||
/// 被发送到错误的 Provider 或账号。
|
||||
/// 该值同时绑定调用方 API Key、Provider 与账号,阻止 native handle 和不透明
|
||||
/// Provider state 被跨租户或跨上游目标复用。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct NativeContinuationPin {
|
||||
/// 客户端提交、用于查找可丢失会话亲和记录的 response ID。
|
||||
previous_response_id: PreviousResponseId,
|
||||
/// Provider 原生 response handle。
|
||||
upstream_response_id: PreviousResponseId,
|
||||
client_api_key_id: ClientApiKeyId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
scope: NativeContinuationScope,
|
||||
@@ -67,12 +69,14 @@ impl NativeContinuationPin {
|
||||
pub const fn new(
|
||||
previous_response_id: PreviousResponseId,
|
||||
upstream_response_id: PreviousResponseId,
|
||||
client_api_key_id: ClientApiKeyId,
|
||||
provider: ProviderKind,
|
||||
account: ProviderAccountId,
|
||||
) -> Self {
|
||||
Self {
|
||||
previous_response_id,
|
||||
upstream_response_id,
|
||||
client_api_key_id,
|
||||
provider,
|
||||
account,
|
||||
scope: NativeContinuationScope::ConnectionLocal,
|
||||
@@ -105,6 +109,16 @@ impl NativeContinuationPin {
|
||||
&self.upstream_response_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn client_api_key_id(&self) -> &ClientApiKeyId {
|
||||
&self.client_api_key_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn matches_client(&self, client_api_key_id: &ClientApiKeyId) -> bool {
|
||||
self.client_api_key_id == *client_api_key_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider(&self) -> &ProviderKind {
|
||||
&self.provider
|
||||
@@ -139,6 +153,7 @@ impl fmt::Debug for NativeContinuationPin {
|
||||
.debug_struct("NativeContinuationPin")
|
||||
.field("previous_response_id", &"<redacted>")
|
||||
.field("upstream_response_id", &"<redacted>")
|
||||
.field("client_api_key_id", &self.client_api_key_id)
|
||||
.field("provider", &self.provider)
|
||||
.field("account", &self.account)
|
||||
.field("scope", &self.scope)
|
||||
@@ -190,21 +205,57 @@ impl fmt::Debug for ContinuationBinding {
|
||||
}
|
||||
}
|
||||
|
||||
/// Native continuation lookup 的稳定失败分类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum NativeContinuationStoreErrorKind {
|
||||
/// Redis 等可丢失协调存储暂不可用;调用方可以按外部 continuation fail-open。
|
||||
Unavailable,
|
||||
/// 存储中的记录无法还原为可信 pin;不得将原始 handle 继续透传上游。
|
||||
InvalidData,
|
||||
/// 已存在的记录属于另一个客户端 API Key;必须 fail closed。
|
||||
OwnershipMismatch,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("native continuation store is unavailable: {detail}")]
|
||||
#[error("native continuation store failed: {kind:?}: {detail}")]
|
||||
pub struct NativeContinuationStoreError {
|
||||
kind: NativeContinuationStoreErrorKind,
|
||||
detail: String,
|
||||
}
|
||||
|
||||
impl NativeContinuationStoreError {
|
||||
/// 构造带可排查细节的亲和存储错误;`detail` 不应包含 response handle。
|
||||
/// 构造可 fail-open 的协调存储不可用错误;`detail` 不应包含 response handle。
|
||||
#[must_use]
|
||||
pub fn new(detail: impl Into<String>) -> Self {
|
||||
pub fn unavailable(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: NativeContinuationStoreErrorKind::Unavailable,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造必须 fail closed 的无效记录错误;`detail` 不应包含 response handle。
|
||||
#[must_use]
|
||||
pub fn invalid_data(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: NativeContinuationStoreErrorKind::InvalidData,
|
||||
detail: detail.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 构造不泄露记录归属细节的跨调用方错误。
|
||||
#[must_use]
|
||||
pub fn ownership_mismatch() -> Self {
|
||||
Self {
|
||||
kind: NativeContinuationStoreErrorKind::OwnershipMismatch,
|
||||
detail: "continuation belongs to a different client API key".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> NativeContinuationStoreErrorKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn detail(&self) -> &str {
|
||||
&self.detail
|
||||
@@ -213,11 +264,12 @@ impl NativeContinuationStoreError {
|
||||
|
||||
/// 可丢失的 previous-response 亲和存储端口。
|
||||
///
|
||||
/// 亲和记录不按调用方 API Key 隔离:与旧版一样,response ID 是会话级 opaque
|
||||
/// handle;Redis 不可用或超时必须由调用方 fail-open。
|
||||
/// 亲和记录按调用方 API Key 隔离;Redis 不可用或超时必须由调用方 fail-open,
|
||||
/// 但命中其他 Key 的记录必须 fail closed。
|
||||
pub trait NativeContinuationPort: Send + Sync {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
client_api_key_id: &'a ClientApiKeyId,
|
||||
previous_response_id: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>>;
|
||||
|
||||
|
||||
@@ -46,6 +46,10 @@ impl AccountSelection {
|
||||
Self::Diagnostic(account) => Some(account),
|
||||
}
|
||||
}
|
||||
|
||||
const fn is_diagnostic(&self) -> bool {
|
||||
matches!(self, Self::Diagnostic(_))
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: ?Sized> AttemptCoordinator<S>
|
||||
@@ -126,6 +130,15 @@ where
|
||||
.map(ProviderAccountStateOwner::from_continuation);
|
||||
let continuation_attempt =
|
||||
initial_continuation_attempt(&operation, &plan, continuation.as_ref());
|
||||
let candidate_index = continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
.map_or(Ok(0), |pin| {
|
||||
plan.candidates()
|
||||
.iter()
|
||||
.position(|candidate| candidate.provider() == pin.provider())
|
||||
.ok_or(EngineError::ContinuationPinMismatch)
|
||||
})?;
|
||||
let image_generation_requested = operation.image_generation_requested();
|
||||
let mut session = ResponseExecutionSession {
|
||||
engine: Arc::clone(&self.engine),
|
||||
@@ -149,6 +162,7 @@ where
|
||||
account_state_owner,
|
||||
cancellation,
|
||||
attempts: 0,
|
||||
candidate_index,
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
credential_recovery_attempted_accounts: BTreeSet::new(),
|
||||
recovery_account: None,
|
||||
@@ -225,6 +239,7 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
account_state_owner: Option<ProviderAccountStateOwner>,
|
||||
cancellation: CancellationToken,
|
||||
attempts: u32,
|
||||
candidate_index: usize,
|
||||
excluded_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
credential_recovery_attempted_accounts: BTreeSet<crate::engine::credential::ProviderAccountId>,
|
||||
/// 凭据恢复后的一次性同账号钉选;只约束紧随其后的 replay attempt,
|
||||
@@ -440,6 +455,15 @@ where
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// 返回最终选中 attempt 观察到的上游 HTTP 状态码。
|
||||
#[must_use]
|
||||
pub fn response_status_code(&self) -> Option<u16> {
|
||||
self.current
|
||||
.as_ref()
|
||||
.and_then(|current| current.response_observation.as_ref())
|
||||
.and_then(ProviderResponseObservation::status_code)
|
||||
}
|
||||
|
||||
/// 将已完成响应的账号事实与 Provider 私有状态封装为可丢失的亲和记录。
|
||||
///
|
||||
/// Core 不读取 `state` 内容;Provider 将在后续同账号 continuation 时自行解释。
|
||||
@@ -461,6 +485,7 @@ where
|
||||
NativeContinuationPin::new(
|
||||
previous_response_id,
|
||||
upstream_response_id,
|
||||
self.client_api_key_ref.clone(),
|
||||
provider,
|
||||
account,
|
||||
)
|
||||
@@ -565,7 +590,7 @@ where
|
||||
if self.attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
let Some(candidate) = self.plan.candidates().first().cloned() else {
|
||||
let Some(candidate) = self.plan.candidates().get(self.candidate_index).cloned() else {
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no upstream Provider is available",
|
||||
@@ -606,7 +631,8 @@ where
|
||||
self.excluded_accounts.clone(),
|
||||
pinned_account.clone(),
|
||||
self.account_state_owner.clone(),
|
||||
),
|
||||
)
|
||||
.with_account_scope(Arc::clone(self.plan.account_scope())),
|
||||
}
|
||||
.with_credential_recovery_attempted(pinned_account.as_ref().is_some_and(|account| {
|
||||
self.credential_recovery_attempted_accounts
|
||||
@@ -660,6 +686,23 @@ where
|
||||
ProviderBoundary::Result(result) => match *result {
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
if self.prepare_unavailable_native_continuation_replay(&error) {
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::ProviderInfrastructureUnavailable
|
||||
) && error.send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
self.continuation_attempt,
|
||||
ContinuationAttempt::None | ContinuationAttempt::ReplayAny
|
||||
)
|
||||
&& self.advance_provider_candidate()
|
||||
{
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -713,6 +756,27 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
if !self.account_selection.is_diagnostic()
|
||||
&& metadata
|
||||
.provider_account_id()
|
||||
.is_none_or(|account| !candidate.account_scope().allows(account))
|
||||
{
|
||||
let error = GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"provider selected an account outside the frozen client scope",
|
||||
);
|
||||
self.finish_failure(FailureFinalization {
|
||||
outcome: ExecutionOutcome::Failed,
|
||||
send_state: self.current_send_state(),
|
||||
error,
|
||||
upstream_status_code: None,
|
||||
provider_error_code: None,
|
||||
retry_after_ms: None,
|
||||
provider_response_id: None,
|
||||
})
|
||||
.await?;
|
||||
return Err(EngineError::AccountOutsideClientScope);
|
||||
}
|
||||
if pinned_account
|
||||
.as_ref()
|
||||
.is_some_and(|required| metadata.provider_account_id() != Some(required))
|
||||
@@ -773,7 +837,7 @@ where
|
||||
provider_kind: metadata.provider().clone(),
|
||||
provider_account_id: metadata.provider_account_id().cloned(),
|
||||
provider_account_ref: metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: metadata.upstream_model().clone(),
|
||||
upstream_model_id: metadata.upstream_model().cloned(),
|
||||
upstream_transport: metadata.transport().as_str().to_owned(),
|
||||
http_version: None,
|
||||
};
|
||||
@@ -816,6 +880,17 @@ where
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn advance_provider_candidate(&mut self) -> bool {
|
||||
let Some(next) = self.candidate_index.checked_add(1) else {
|
||||
return false;
|
||||
};
|
||||
if next >= self.plan.candidates().len() {
|
||||
return false;
|
||||
}
|
||||
self.candidate_index = next;
|
||||
true
|
||||
}
|
||||
|
||||
async fn observe_event(&mut self, event: &GatewayEvent) {
|
||||
self.observe_event_facts(event);
|
||||
self.mark_send_observed().await;
|
||||
@@ -1008,7 +1083,7 @@ where
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: current.metadata.provider_account_id().cloned(),
|
||||
upstream_model_id: current.metadata.upstream_model().clone(),
|
||||
upstream_model_id: current.metadata.upstream_model().cloned(),
|
||||
upstream_status_code: current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
@@ -1096,6 +1171,42 @@ where
|
||||
true
|
||||
}
|
||||
|
||||
/// 原生续写的原账号在本地调度阶段已不可用时,交给 Provider 执行跨账号恢复。
|
||||
///
|
||||
/// 这不是强制 Smart:下一次 attempt 仍使用请求配置的调度策略。只有网关持有
|
||||
/// 对应 Provider 的会话状态时才进入恢复;是否保留 native handle、执行 probe
|
||||
/// 或使用完整 transcript,由 Provider 自己的协议边界决定。
|
||||
fn prepare_unavailable_native_continuation_replay(&mut self, error: &ProviderError) -> bool {
|
||||
if self.account_selection.required_account().is_some()
|
||||
|| self.continuation_attempt != ContinuationAttempt::Native
|
||||
|| !matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(pin) = self
|
||||
.continuation
|
||||
.as_ref()
|
||||
.and_then(ContinuationBinding::pinned)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if self
|
||||
.operation
|
||||
.provider_session_state(pin.provider().as_str())
|
||||
.is_none()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
self.continuation_attempt = ContinuationAttempt::ReplayAny;
|
||||
self.excluded_accounts.insert(pin.account().clone());
|
||||
true
|
||||
}
|
||||
|
||||
fn reset_uncommitted_observations(&mut self) {
|
||||
self.usage = Usage::new();
|
||||
self.cost = CostEstimate::unavailable();
|
||||
@@ -1380,8 +1491,13 @@ where
|
||||
}
|
||||
|
||||
fn image_generation_succeeded(&self) -> Option<bool> {
|
||||
self.image_generation_requested
|
||||
.then(|| self.usage.image_output_tokens.unwrap_or_default() > 0)
|
||||
self.image_generation_requested.then(|| {
|
||||
if matches!(self.operation, Operation::GenerateImage(_)) {
|
||||
self.upstream_complete
|
||||
} else {
|
||||
self.usage.image_output_tokens.unwrap_or_default() > 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn current_send_state(&self) -> UpstreamSendState {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,7 +15,8 @@ use crate::engine::admission::{
|
||||
ClientAdmissionDecision, ClientAdmissionPort, ClientAdmissionRejection, ClientAdmissionRequest,
|
||||
};
|
||||
use crate::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort, PreviousResponseId,
|
||||
ContinuationBinding, NativeContinuationPin, NativeContinuationPort,
|
||||
NativeContinuationStoreErrorKind, PreviousResponseId,
|
||||
};
|
||||
use crate::engine::coordinator::ResponseExecutionSession;
|
||||
use crate::engine::probe::{
|
||||
@@ -117,6 +118,35 @@ pub struct StartExecution {
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
/// 启动一个由协议 adapter 明确绑定到 Provider 自有端点的请求。
|
||||
pub struct StartProviderExecution {
|
||||
pub client: AuthenticatedClient,
|
||||
pub provider: ProviderKind,
|
||||
pub operation: Operation,
|
||||
pub metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
enum ExecutionTarget {
|
||||
Model(PublicModelId),
|
||||
ProviderEndpoint(ProviderKind),
|
||||
}
|
||||
|
||||
impl ExecutionTarget {
|
||||
fn into_public_model(self) -> Option<PublicModelId> {
|
||||
match self {
|
||||
Self::Model(model) => Some(model),
|
||||
Self::ProviderEndpoint(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingStartExecution {
|
||||
client: AuthenticatedClient,
|
||||
target: ExecutionTarget,
|
||||
operation: Operation,
|
||||
metadata: ExecutionRequestMetadata,
|
||||
}
|
||||
|
||||
pub struct StartedExecution {
|
||||
pub request_id: ModelRequestId,
|
||||
pub created_at: SystemTime,
|
||||
@@ -128,6 +158,9 @@ pub trait ExecutionSession: Send {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>>;
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>>;
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader];
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
None
|
||||
}
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
@@ -155,6 +188,10 @@ pub trait ExecutionService: Send + Sync {
|
||||
&self,
|
||||
request: StartExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>>;
|
||||
}
|
||||
|
||||
/// 成功认证后的 API Key 使用事实接收器。
|
||||
@@ -246,9 +283,44 @@ impl DefaultExecutionService {
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_inner(
|
||||
async fn start_inner(&self, request: StartExecution) -> Result<StartedExecution, GatewayError> {
|
||||
let StartExecution {
|
||||
client,
|
||||
public_model,
|
||||
operation,
|
||||
metadata,
|
||||
} = request;
|
||||
self.start_inner_with_target(PendingStartExecution {
|
||||
client,
|
||||
target: ExecutionTarget::Model(public_model),
|
||||
operation,
|
||||
metadata,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn start_provider_endpoint_inner(
|
||||
&self,
|
||||
mut request: StartExecution,
|
||||
request: StartProviderExecution,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
let StartProviderExecution {
|
||||
client,
|
||||
provider,
|
||||
operation,
|
||||
metadata,
|
||||
} = request;
|
||||
self.start_inner_with_target(PendingStartExecution {
|
||||
client,
|
||||
target: ExecutionTarget::ProviderEndpoint(provider),
|
||||
operation,
|
||||
metadata,
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn start_inner_with_target(
|
||||
&self,
|
||||
mut request: PendingStartExecution,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
request.client.policy.authorize().map_err(|_| {
|
||||
GatewayError::new(GatewayErrorKind::PolicyDenied, "client API key is disabled")
|
||||
@@ -261,22 +333,40 @@ impl DefaultExecutionService {
|
||||
})?;
|
||||
let request_id = new_request_id()?;
|
||||
let routing_context = self
|
||||
.route_context(request.client.policy.provider_kind())
|
||||
.route_context(request.client.policy.account_scope().provider_kinds())
|
||||
.await?;
|
||||
let plan = request
|
||||
.client
|
||||
.snapshot
|
||||
.plan(&request.public_model, &request.operation, &routing_context)
|
||||
.map_err(map_routing_error)?;
|
||||
let account_scope = Arc::clone(request.client.policy.account_scope());
|
||||
let plan = match &request.target {
|
||||
ExecutionTarget::ProviderEndpoint(provider) => {
|
||||
request.client.snapshot.plan_provider_endpoint(
|
||||
provider,
|
||||
&request.operation,
|
||||
account_scope,
|
||||
&routing_context,
|
||||
)
|
||||
}
|
||||
ExecutionTarget::Model(public_model) => request.client.snapshot.plan(
|
||||
public_model,
|
||||
&request.operation,
|
||||
account_scope,
|
||||
&routing_context,
|
||||
),
|
||||
}
|
||||
.map_err(map_routing_error)?;
|
||||
let continuation = match request.metadata.previous_response_id.as_ref() {
|
||||
Some(previous) => {
|
||||
let resolve = self.continuation.resolve(previous).fuse();
|
||||
let resolve = self
|
||||
.continuation
|
||||
.resolve(request.client.policy.key_id(), previous)
|
||||
.fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(resolve, timeout);
|
||||
let pin = select_biased! {
|
||||
result = resolve => match result {
|
||||
Ok(pin) => pin,
|
||||
Err(error) => {
|
||||
Err(error)
|
||||
if error.kind() == NativeContinuationStoreErrorKind::Unavailable =>
|
||||
{
|
||||
tracing::debug!(
|
||||
request_id = request_id.as_str(),
|
||||
%error,
|
||||
@@ -284,10 +374,53 @@ impl DefaultExecutionService {
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(error)
|
||||
if error.kind()
|
||||
== NativeContinuationStoreErrorKind::OwnershipMismatch =>
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"continuation does not belong to this client API key",
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
request_id = request_id.as_str(),
|
||||
%error,
|
||||
"Continuation affinity 记录无效,已拒绝续接"
|
||||
);
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"continuation state is invalid",
|
||||
));
|
||||
}
|
||||
},
|
||||
_ = timeout => None,
|
||||
};
|
||||
match pin {
|
||||
Some(pin) if !pin.matches_client(request.client.policy.key_id()) => {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"continuation does not belong to this client API key",
|
||||
));
|
||||
}
|
||||
Some(pin) if !request.client.policy.account_scope().allows(pin.account()) => {
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::PolicyDenied,
|
||||
"continuation account is outside the client account scope",
|
||||
));
|
||||
}
|
||||
Some(pin)
|
||||
if !plan
|
||||
.candidates()
|
||||
.iter()
|
||||
.any(|candidate| candidate.provider() == pin.provider()) =>
|
||||
{
|
||||
return Err(GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"continuation provider is not available",
|
||||
));
|
||||
}
|
||||
Some(pin) => {
|
||||
attach_continuation_session_state(&mut request.operation, &pin);
|
||||
ContinuationBinding::Pinned(pin)
|
||||
@@ -321,18 +454,24 @@ impl DefaultExecutionService {
|
||||
|
||||
async fn start_without_continuation(
|
||||
&self,
|
||||
request: StartExecution,
|
||||
request: PendingStartExecution,
|
||||
request_id: ModelRequestId,
|
||||
started_at: SystemTime,
|
||||
deadline_at: SystemTime,
|
||||
plan: crate::routing::RoutingPlan,
|
||||
continuation: Option<ContinuationBinding>,
|
||||
) -> Result<StartedExecution, GatewayError> {
|
||||
let PendingStartExecution {
|
||||
client,
|
||||
target,
|
||||
operation,
|
||||
metadata,
|
||||
} = request;
|
||||
let admission_request = ClientAdmissionRequest {
|
||||
model_request_id: request_id.clone(),
|
||||
client_api_key_id: request.client.policy.key_id().clone(),
|
||||
client_api_key_id: client.policy.key_id().clone(),
|
||||
lease_ttl: MODEL_REQUEST_DEADLINE,
|
||||
limits: request.client.policy.limits(),
|
||||
limits: client.policy.limits(),
|
||||
};
|
||||
match self
|
||||
.admissions
|
||||
@@ -357,30 +496,35 @@ impl DefaultExecutionService {
|
||||
}
|
||||
let admission = AdmissionLease {
|
||||
port: Arc::clone(&self.admissions),
|
||||
client_api_key_id: request.client.policy.key_id().clone(),
|
||||
client_api_key_id: client.policy.key_id().clone(),
|
||||
model_request_id: request_id.clone(),
|
||||
};
|
||||
let observation = self
|
||||
.providers
|
||||
.request_observation(request.client.policy.provider_kind(), &request.operation);
|
||||
let observation = plan
|
||||
.candidates()
|
||||
.first()
|
||||
.map_or_else(Default::default, |candidate| {
|
||||
self.providers
|
||||
.request_observation(candidate.provider(), &operation)
|
||||
});
|
||||
let new_request = NewModelRequest {
|
||||
id: request_id.clone(),
|
||||
client_api_key_id: Some(request.client.policy.key_id().clone()),
|
||||
client_api_key_ref: request.client.policy.key_id().clone(),
|
||||
client_api_key_id: Some(client.policy.key_id().clone()),
|
||||
client_api_key_ref: client.policy.key_id().clone(),
|
||||
config_revision: plan.config_revision(),
|
||||
protocol: request.metadata.protocol,
|
||||
operation: request.operation.kind(),
|
||||
endpoint: request.metadata.endpoint,
|
||||
client_transport: request.metadata.transport.as_str().to_owned(),
|
||||
requested_model: request.public_model,
|
||||
client_ip: request.metadata.client_ip,
|
||||
user_agent: request.metadata.user_agent,
|
||||
routing: client.policy.account_scope().routing_snapshot(),
|
||||
protocol: metadata.protocol,
|
||||
operation: operation.kind(),
|
||||
endpoint: metadata.endpoint,
|
||||
client_transport: metadata.transport.as_str().to_owned(),
|
||||
requested_model: target.into_public_model(),
|
||||
client_ip: metadata.client_ip,
|
||||
user_agent: metadata.user_agent,
|
||||
reasoning_effort: observation.reasoning_effort,
|
||||
reasoning_preset: observation.reasoning_preset,
|
||||
request_kind: observation.request_kind,
|
||||
subagent_kind: observation.subagent_kind,
|
||||
compact: observation.compact,
|
||||
image_generation_requested: request.operation.image_generation_requested(),
|
||||
image_generation_requested: operation.image_generation_requested(),
|
||||
started_at,
|
||||
deadline_at,
|
||||
};
|
||||
@@ -388,7 +532,7 @@ impl DefaultExecutionService {
|
||||
.coordinator
|
||||
.start(
|
||||
new_request,
|
||||
request.operation,
|
||||
operation,
|
||||
plan,
|
||||
None,
|
||||
continuation,
|
||||
@@ -405,7 +549,7 @@ impl DefaultExecutionService {
|
||||
Ok(StartedExecution {
|
||||
request_id,
|
||||
created_at: started_at,
|
||||
stream: request.metadata.stream,
|
||||
stream: metadata.stream,
|
||||
session: Box::new(DefaultExecutionSession::new(
|
||||
core,
|
||||
admission,
|
||||
@@ -417,38 +561,42 @@ impl DefaultExecutionService {
|
||||
|
||||
async fn route_context(
|
||||
&self,
|
||||
provider_kind: &ProviderKind,
|
||||
provider_kinds: &BTreeSet<ProviderKind>,
|
||||
) -> Result<RoutingContext, GatewayError> {
|
||||
let decision = self.circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
_ = timeout => None,
|
||||
};
|
||||
let blocked_providers = match decision {
|
||||
Some(Ok(ProviderCircuitDecision::Allow)) => BTreeSet::new(),
|
||||
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
|
||||
BTreeSet::from([provider_kind.clone()])
|
||||
let decisions = futures::future::join_all(provider_kinds.iter().map(|provider_kind| {
|
||||
let circuits = Arc::clone(&self.circuits);
|
||||
async move {
|
||||
let decision = circuits.decision(provider_kind).fuse();
|
||||
let timeout = Delay::new(COORDINATION_TIMEOUT).fuse();
|
||||
pin_mut!(decision, timeout);
|
||||
let decision = select_biased! {
|
||||
result = decision => Some(result),
|
||||
_ = timeout => None,
|
||||
};
|
||||
(provider_kind, decision)
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
tracing::warn!(
|
||||
}))
|
||||
.await;
|
||||
let mut blocked_providers = BTreeSet::new();
|
||||
for (provider_kind, decision) in decisions {
|
||||
match decision {
|
||||
Some(Ok(ProviderCircuitDecision::BlockedUntil(_))) => {
|
||||
blocked_providers.insert(provider_kind.clone());
|
||||
}
|
||||
Some(Err(error)) => tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
%error,
|
||||
"Provider circuit 读取失败,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
),
|
||||
None => tracing::warn!(
|
||||
provider = provider_kind.as_str(),
|
||||
"Provider circuit 读取超时,按可重建协调状态 fail-open"
|
||||
);
|
||||
BTreeSet::new()
|
||||
),
|
||||
Some(Ok(ProviderCircuitDecision::Allow)) => {}
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(RoutingContext {
|
||||
provider_kind: Some(provider_kind.clone()),
|
||||
required_provider: None,
|
||||
blocked_providers,
|
||||
})
|
||||
}
|
||||
@@ -479,11 +627,16 @@ impl DefaultExecutionService {
|
||||
GatewayError::new(GatewayErrorKind::Unsupported, "requested model is invalid")
|
||||
})?;
|
||||
let routing_context = RoutingContext {
|
||||
provider_kind: Some(provider_kind),
|
||||
required_provider: Some(provider_kind),
|
||||
..RoutingContext::default()
|
||||
};
|
||||
let plan = snapshot
|
||||
.plan(&public_model, &operation, &routing_context)
|
||||
.plan(
|
||||
&public_model,
|
||||
&operation,
|
||||
snapshot.all_account_scope(),
|
||||
&routing_context,
|
||||
)
|
||||
.map_err(map_routing_error)?;
|
||||
let started_at = SystemTime::now();
|
||||
let deadline_at = started_at
|
||||
@@ -499,11 +652,12 @@ impl DefaultExecutionService {
|
||||
client_api_key_id: None,
|
||||
client_api_key_ref: actor,
|
||||
config_revision: plan.config_revision(),
|
||||
routing: crate::routing::AccountRoutingSnapshot::all(),
|
||||
protocol: "admin_connection_test".to_owned(),
|
||||
operation: operation.kind(),
|
||||
endpoint: "/api/admin/accounts/connection-test".to_owned(),
|
||||
client_transport: ClientTransport::InternalProbe.as_str().to_owned(),
|
||||
requested_model: public_model,
|
||||
requested_model: Some(public_model),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
@@ -693,19 +847,19 @@ impl ExecutionService for DefaultExecutionService {
|
||||
fn public_models(&self, client: &AuthenticatedClient) -> Vec<PublicModelId> {
|
||||
client
|
||||
.snapshot
|
||||
.public_models_for_provider(client.policy.provider_kind())
|
||||
.public_models_for_scope(client.policy.account_scope())
|
||||
}
|
||||
|
||||
fn public_model_profiles(&self, client: &AuthenticatedClient) -> Vec<PublicModelProfile> {
|
||||
client
|
||||
.snapshot
|
||||
.public_model_profiles_for_provider(client.policy.provider_kind())
|
||||
.public_model_profiles_for_scope(client.policy.account_scope())
|
||||
}
|
||||
|
||||
fn contains_public_model(&self, client: &AuthenticatedClient, model: &PublicModelId) -> bool {
|
||||
client
|
||||
.snapshot
|
||||
.contains_public_model_for_provider(model, client.policy.provider_kind())
|
||||
.contains_public_model_for_scope(model, client.policy.account_scope())
|
||||
}
|
||||
|
||||
fn start(
|
||||
@@ -714,6 +868,13 @@ impl ExecutionService for DefaultExecutionService {
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move { self.start_inner(request).await })
|
||||
}
|
||||
|
||||
fn start_provider_endpoint(
|
||||
&self,
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
Box::pin(async move { self.start_provider_endpoint_inner(request).await })
|
||||
}
|
||||
}
|
||||
|
||||
impl AccountProbe for DefaultExecutionService {
|
||||
@@ -879,6 +1040,12 @@ impl ExecutionSession for DefaultExecutionSession {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
self.core
|
||||
.as_ref()
|
||||
.and_then(ResponseExecutionSession::response_status_code)
|
||||
}
|
||||
|
||||
fn commit_downstream(
|
||||
&mut self,
|
||||
client_status_code: Option<u16>,
|
||||
@@ -1006,7 +1173,9 @@ fn new_request_id() -> Result<ModelRequestId, GatewayError> {
|
||||
|
||||
fn map_routing_error(error: crate::error::RoutingError) -> GatewayError {
|
||||
match error {
|
||||
crate::error::RoutingError::NoCapableProvider { .. } => GatewayError::new(
|
||||
crate::error::RoutingError::NoCapableProvider { .. }
|
||||
| crate::error::RoutingError::NoCapableProviderEndpoint { .. }
|
||||
| crate::error::RoutingError::EmptyAccountScope => GatewayError::new(
|
||||
GatewayErrorKind::NoAvailableProvider,
|
||||
"no provider can execute this request",
|
||||
),
|
||||
|
||||
@@ -233,6 +233,7 @@ pub struct AccountAttemptContext {
|
||||
state_owner: Option<ProviderAccountStateOwner>,
|
||||
credential_recovery_attempted: bool,
|
||||
diagnostic_required_account: bool,
|
||||
account_scope: Option<Arc<crate::routing::FrozenAccountScope>>,
|
||||
}
|
||||
|
||||
impl AccountAttemptContext {
|
||||
@@ -248,6 +249,7 @@ impl AccountAttemptContext {
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: false,
|
||||
account_scope: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,9 +268,17 @@ impl AccountAttemptContext {
|
||||
state_owner,
|
||||
credential_recovery_attempted: false,
|
||||
diagnostic_required_account: true,
|
||||
account_scope: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 附着普通请求认证时冻结的账号范围。
|
||||
#[must_use]
|
||||
pub fn with_account_scope(mut self, scope: Arc<crate::routing::FrozenAccountScope>) -> Self {
|
||||
self.account_scope = Some(scope);
|
||||
self
|
||||
}
|
||||
|
||||
/// 标记本请求已对即将选择的固定账号执行过一次凭据恢复。
|
||||
#[must_use]
|
||||
pub const fn with_credential_recovery_attempted(mut self, attempted: bool) -> Self {
|
||||
@@ -301,6 +311,11 @@ impl AccountAttemptContext {
|
||||
pub const fn is_diagnostic_required_account(&self) -> bool {
|
||||
self.diagnostic_required_account
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_scope(&self) -> Option<&Arc<crate::routing::FrozenAccountScope>> {
|
||||
self.account_scope.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求中 Provider 账号绑定状态的唯一归属。
|
||||
@@ -347,9 +362,14 @@ pub enum ContinuationAttempt {
|
||||
None,
|
||||
/// 使用 Store 解析出的原生 handle 与账号绑定。
|
||||
Native,
|
||||
/// 在原账号上用完整连接内 transcript 重放。
|
||||
/// 在原账号上执行 Provider 定义的恢复。
|
||||
///
|
||||
/// 只有持有可携带 transcript 的 Provider 才能清除 native handle 后重放;
|
||||
/// 只持有 opaque state 的 Provider 必须保留 continuation 依赖或返回客户端错误。
|
||||
ReplayOwner,
|
||||
/// transcript 已可携带,允许选择其他账号。
|
||||
/// 允许选择其他账号并执行 Provider 定义的恢复。
|
||||
///
|
||||
/// 该枚举本身不证明客户端 input 完整,也不授权 Provider 清除 previous-response。
|
||||
ReplayAny,
|
||||
}
|
||||
|
||||
@@ -506,6 +526,12 @@ impl AttemptContext {
|
||||
self.account.is_diagnostic_required_account()
|
||||
}
|
||||
|
||||
/// 普通请求认证时冻结的账号范围;管理端诊断为 `None`。
|
||||
#[must_use]
|
||||
pub const fn account_scope(&self) -> Option<&Arc<crate::routing::FrozenAccountScope>> {
|
||||
self.account.account_scope()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn continuation(&self) -> Option<&ContinuationBinding> {
|
||||
self.continuation.as_ref()
|
||||
@@ -529,11 +555,12 @@ pub struct NewModelRequest {
|
||||
pub client_api_key_id: Option<ClientApiKeyId>,
|
||||
pub client_api_key_ref: ClientApiKeyId,
|
||||
pub config_revision: ConfigRevision,
|
||||
pub routing: crate::routing::AccountRoutingSnapshot,
|
||||
pub protocol: String,
|
||||
pub operation: OperationKind,
|
||||
pub endpoint: String,
|
||||
pub client_transport: String,
|
||||
pub requested_model: PublicModelId,
|
||||
pub requested_model: Option<PublicModelId>,
|
||||
pub client_ip: Option<IpAddr>,
|
||||
pub user_agent: Option<String>,
|
||||
pub reasoning_effort: Option<String>,
|
||||
@@ -555,7 +582,7 @@ pub struct AttemptRecord {
|
||||
pub provider_kind: ProviderKind,
|
||||
pub provider_account_id: Option<ProviderAccountId>,
|
||||
pub provider_account_ref: Option<ProviderAccountId>,
|
||||
pub upstream_model_id: UpstreamModelId,
|
||||
pub upstream_model_id: Option<UpstreamModelId>,
|
||||
pub upstream_transport: String,
|
||||
pub http_version: Option<String>,
|
||||
}
|
||||
@@ -568,7 +595,7 @@ pub struct IntermediateFailure {
|
||||
pub trigger: AttemptTrigger,
|
||||
pub provider_kind: ProviderKind,
|
||||
pub account_id: Option<ProviderAccountId>,
|
||||
pub upstream_model_id: UpstreamModelId,
|
||||
pub upstream_model_id: Option<UpstreamModelId>,
|
||||
pub upstream_status_code: Option<u16>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub error: ProviderError,
|
||||
@@ -749,6 +776,8 @@ pub enum EngineError {
|
||||
ContinuationPinMismatch,
|
||||
#[error("provider did not use the account required by this execution")]
|
||||
RequiredAccountMismatch,
|
||||
#[error("provider selected an account outside the frozen client scope")]
|
||||
AccountOutsideClientScope,
|
||||
#[error("provider execution failed")]
|
||||
Provider(ProviderError),
|
||||
#[error("request was cancelled")]
|
||||
|
||||
@@ -123,7 +123,7 @@ impl UpstreamTransport {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCallMetadata {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
upstream_model: Option<UpstreamModelId>,
|
||||
resource: ProviderResource,
|
||||
upstream_request_id: Option<OpaqueUpstreamValue>,
|
||||
transport: UpstreamTransport,
|
||||
@@ -140,7 +140,23 @@ impl ProviderCallMetadata {
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
upstream_model,
|
||||
upstream_model: Some(upstream_model),
|
||||
resource,
|
||||
upstream_request_id: None,
|
||||
transport,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建一次不声明模型的 Provider 原生端点调用事实。
|
||||
#[must_use]
|
||||
pub const fn for_provider_endpoint(
|
||||
provider: ProviderKind,
|
||||
resource: ProviderResource,
|
||||
transport: UpstreamTransport,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
upstream_model: None,
|
||||
resource,
|
||||
upstream_request_id: None,
|
||||
transport,
|
||||
@@ -160,10 +176,10 @@ impl ProviderCallMetadata {
|
||||
&self.provider
|
||||
}
|
||||
|
||||
/// 返回实际模型;必须与冻结 target 一致。
|
||||
/// 返回实际模型;Provider 原生端点没有模型事实。
|
||||
#[must_use]
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
pub const fn upstream_model(&self) -> Option<&UpstreamModelId> {
|
||||
self.upstream_model.as_ref()
|
||||
}
|
||||
|
||||
/// 返回匿名资源;账号调用返回 `None`。
|
||||
@@ -199,7 +215,8 @@ impl ProviderCallMetadata {
|
||||
/// 确认 metadata 没有替换请求计划中冻结的 Provider 候选。
|
||||
#[must_use]
|
||||
pub fn confirms(&self, candidate: &ProviderCandidate) -> bool {
|
||||
candidate.provider() == &self.provider && candidate.upstream_model() == &self.upstream_model
|
||||
candidate.provider() == &self.provider
|
||||
&& candidate.upstream_model() == self.upstream_model.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,7 +534,7 @@ impl fmt::Debug for ProviderRequest {
|
||||
.debug_struct("ProviderRequest")
|
||||
.field("operation", &self.operation)
|
||||
.field("provider", self.candidate.provider())
|
||||
.field("upstream_model", self.candidate.upstream_model())
|
||||
.field("upstream_model", &self.candidate.upstream_model())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@ pub enum IdentifierError {
|
||||
/// 规定前缀。
|
||||
expected: &'static str,
|
||||
},
|
||||
/// 标识不满足该领域值对象的固定格式。
|
||||
#[error("identifier has an invalid format")]
|
||||
InvalidFormat,
|
||||
}
|
||||
|
||||
/// Operation 构造或校验失败。
|
||||
@@ -67,6 +70,12 @@ pub enum RoutingError {
|
||||
/// 配置 revision 必须为正数。
|
||||
#[error("config revision must be greater than zero")]
|
||||
InvalidRevision,
|
||||
/// Restricted scope 必须保留至少一个持久化分组 binding。
|
||||
#[error("account routing scope is invalid")]
|
||||
InvalidAccountScope,
|
||||
/// Key 的账号范围中当前没有任何可参与路由的账号。
|
||||
#[error("account routing scope is empty")]
|
||||
EmptyAccountScope,
|
||||
/// 快照中存在重复实体。
|
||||
#[error("duplicate {entity} `{id}`")]
|
||||
DuplicateEntity {
|
||||
@@ -89,6 +98,12 @@ pub enum RoutingError {
|
||||
/// 客户端提交的模型名称。
|
||||
model: String,
|
||||
},
|
||||
/// 固定 Provider 的原生端点当前不可执行。
|
||||
#[error("provider endpoint `{provider}` is unavailable")]
|
||||
NoCapableProviderEndpoint {
|
||||
/// adapter 已绑定的 Provider。
|
||||
provider: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// 调用方策略不满足约束或拒绝请求。
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::operation::ProviderSessionState;
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ResponseMeta {
|
||||
response_id: String,
|
||||
model: String,
|
||||
model: Option<String>,
|
||||
finish_reason: Option<FinishReason>,
|
||||
}
|
||||
|
||||
@@ -26,7 +26,17 @@ impl ResponseMeta {
|
||||
pub fn new(response_id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
response_id: response_id.into(),
|
||||
model: model.into(),
|
||||
model: Some(model.into()),
|
||||
finish_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建不声明模型的 Provider 原生端点响应元数据。
|
||||
#[must_use]
|
||||
pub fn for_provider_endpoint(response_id: impl Into<String>) -> Self {
|
||||
Self {
|
||||
response_id: response_id.into(),
|
||||
model: None,
|
||||
finish_reason: None,
|
||||
}
|
||||
}
|
||||
@@ -46,8 +56,8 @@ impl ResponseMeta {
|
||||
|
||||
/// 返回对客户端公开的模型名。
|
||||
#[must_use]
|
||||
pub fn model(&self) -> &str {
|
||||
&self.model
|
||||
pub fn model(&self) -> Option<&str> {
|
||||
self.model.as_deref()
|
||||
}
|
||||
|
||||
/// 返回规范化终止原因。
|
||||
@@ -190,6 +200,7 @@ pub struct ProtocolWireEvent {
|
||||
data: Value,
|
||||
has_json_data: bool,
|
||||
raw_sse_frame: Option<Bytes>,
|
||||
raw_json_body: Option<Bytes>,
|
||||
sse_id: Option<String>,
|
||||
sse_retry: Option<u64>,
|
||||
}
|
||||
@@ -228,6 +239,7 @@ impl ProtocolWireEvent {
|
||||
data,
|
||||
has_json_data: true,
|
||||
raw_sse_frame: None,
|
||||
raw_json_body: None,
|
||||
sse_id,
|
||||
sse_retry,
|
||||
})
|
||||
@@ -274,6 +286,32 @@ impl ProtocolWireEvent {
|
||||
data: Value::Null,
|
||||
has_json_data: false,
|
||||
raw_sse_frame: Some(raw_sse_frame),
|
||||
raw_json_body: None,
|
||||
sse_id: None,
|
||||
sse_retry: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// 创建未经改写的完整 JSON 响应正文。
|
||||
///
|
||||
/// 该载荷用于非流式协议端点;Core 不解析或重编码其中的值。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名不满足内部路由标识约束时返回错误。
|
||||
pub fn raw_json(
|
||||
protocol: impl Into<String>,
|
||||
raw_json_body: Bytes,
|
||||
) -> Result<Self, IdentifierError> {
|
||||
let protocol = protocol.into();
|
||||
validate_text(&protocol, 64, true, None)?;
|
||||
Ok(Self {
|
||||
protocol,
|
||||
event_type: None,
|
||||
data: Value::Null,
|
||||
has_json_data: false,
|
||||
raw_sse_frame: None,
|
||||
raw_json_body: Some(raw_json_body),
|
||||
sse_id: None,
|
||||
sse_retry: None,
|
||||
})
|
||||
@@ -321,6 +359,18 @@ impl ProtocolWireEvent {
|
||||
self.raw_sse_frame.as_ref()
|
||||
}
|
||||
|
||||
/// 返回可直接写给 JSON 客户端的未经改写响应正文。
|
||||
#[must_use]
|
||||
pub const fn raw_json_body(&self) -> Option<&Bytes> {
|
||||
self.raw_json_body.as_ref()
|
||||
}
|
||||
|
||||
/// 拆出未经改写的完整 JSON 响应正文。
|
||||
#[must_use]
|
||||
pub fn into_raw_json_body(self) -> Option<Bytes> {
|
||||
self.raw_json_body
|
||||
}
|
||||
|
||||
/// 拆出协议原生 JSON 数据。
|
||||
#[must_use]
|
||||
pub fn into_data(self) -> Value {
|
||||
@@ -336,6 +386,7 @@ impl fmt::Debug for ProtocolWireEvent {
|
||||
.field("event_type", &self.event_type)
|
||||
.field("has_json_data", &self.has_json_data)
|
||||
.field("has_raw_sse_frame", &self.raw_sse_frame.is_some())
|
||||
.field("has_raw_json_body", &self.raw_json_body.is_some())
|
||||
.field("has_sse_id", &self.sse_id.is_some())
|
||||
.field("sse_retry", &self.sse_retry)
|
||||
.field("data", &"<not included in Debug>")
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
//! 协议无关的业务 operation。
|
||||
//!
|
||||
//! 这里只保留网关需要解释、路由或结算的稳定语义。协议 adapter 无法写入
|
||||
//! wire body 的连接级事实随 [`ProtocolPayload`] 不透明传递,由对应 Provider
|
||||
//! 自己解释。
|
||||
//! wire body 与连接级事实随 [`ProtocolPayload`] 或 [`RawJsonPayload`] 不透明传递,
|
||||
//! 由对应 Provider 自己解释。
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
@@ -239,6 +240,72 @@ impl fmt::Debug for ProtocolPayload {
|
||||
}
|
||||
}
|
||||
|
||||
/// 客户端协议交给 Provider 的未经重编码 JSON 正文。
|
||||
///
|
||||
/// 协议 adapter 只确定路由端点;Core 保留原始字节与非 wire 上下文,
|
||||
/// 不验证、解析或改写正文。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct RawJsonPayload {
|
||||
protocol: String,
|
||||
body: Bytes,
|
||||
context: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl RawJsonPayload {
|
||||
/// 创建未经重编码的协议 JSON 正文。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// 协议名称为空、过长或含控制字符时返回错误。
|
||||
pub fn new(protocol: impl Into<String>, body: Bytes) -> Result<Self, OperationError> {
|
||||
let protocol = protocol.into();
|
||||
validate_text(&protocol, 64, true, None).map_err(|_| OperationError::EmptyField {
|
||||
field: "raw_json_payload protocol",
|
||||
})?;
|
||||
Ok(Self {
|
||||
protocol,
|
||||
body,
|
||||
context: Map::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 返回协议名称。
|
||||
#[must_use]
|
||||
pub fn protocol(&self) -> &str {
|
||||
&self.protocol
|
||||
}
|
||||
|
||||
/// 返回 adapter 收到的原始 JSON 字节。
|
||||
#[must_use]
|
||||
pub const fn body(&self) -> &Bytes {
|
||||
&self.body
|
||||
}
|
||||
|
||||
/// 附着不写入 wire body 的协议连接上下文。
|
||||
#[must_use]
|
||||
pub fn with_context(mut self, context: Map<String, Value>) -> Self {
|
||||
self.context = context;
|
||||
self
|
||||
}
|
||||
|
||||
/// 返回仅供对应 Provider 解释的非 wire 上下文。
|
||||
#[must_use]
|
||||
pub const fn context(&self) -> &Map<String, Value> {
|
||||
&self.context
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RawJsonPayload {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RawJsonPayload")
|
||||
.field("protocol", &self.protocol)
|
||||
.field("body", &"<not included in Debug>")
|
||||
.field("context", &"<not included in Debug>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// 通用生成请求。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct GenerateRequest {
|
||||
@@ -501,32 +568,39 @@ impl fmt::Debug for RerankRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// 图像生成请求。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
/// 图像 API 的稳定端点语义。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ImageRequestKind {
|
||||
/// 从文本生成图像。
|
||||
Generation,
|
||||
/// 使用输入图像执行编辑。
|
||||
Edit,
|
||||
}
|
||||
|
||||
/// 图像生成或编辑请求。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct ImageRequest {
|
||||
prompt: String,
|
||||
count: u32,
|
||||
kind: ImageRequestKind,
|
||||
payload: RawJsonPayload,
|
||||
}
|
||||
|
||||
impl ImageRequest {
|
||||
/// 创建图像生成请求。
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Prompt 为空时返回错误。
|
||||
pub fn new(prompt: impl Into<String>) -> Result<Self, OperationError> {
|
||||
let prompt = prompt.into();
|
||||
if prompt.is_empty() {
|
||||
return Err(OperationError::EmptyField { field: "prompt" });
|
||||
}
|
||||
Ok(Self { prompt, count: 1 })
|
||||
/// 创建携带原始协议 JSON 正文的图像请求。
|
||||
#[must_use]
|
||||
pub const fn from_raw_json(kind: ImageRequestKind, payload: RawJsonPayload) -> Self {
|
||||
Self { kind, payload }
|
||||
}
|
||||
|
||||
/// 设置图像数量。
|
||||
/// 返回图像 API 端点语义。
|
||||
#[must_use]
|
||||
pub const fn with_count(mut self, count: u32) -> Self {
|
||||
self.count = count;
|
||||
self
|
||||
pub const fn kind(&self) -> ImageRequestKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// 返回未经重编码的协议 JSON 正文。
|
||||
#[must_use]
|
||||
pub const fn payload(&self) -> &RawJsonPayload {
|
||||
&self.payload
|
||||
}
|
||||
}
|
||||
|
||||
@@ -534,8 +608,8 @@ impl fmt::Debug for ImageRequest {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ImageRequest")
|
||||
.field("prompt", &"<not included in Debug>")
|
||||
.field("count", &self.count)
|
||||
.field("kind", &self.kind)
|
||||
.field("payload", &"<not included in Debug>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -649,7 +723,8 @@ impl Operation {
|
||||
pub fn image_generation_requested(&self) -> bool {
|
||||
match self {
|
||||
Self::Generate(request) => request.image_generation_requested(),
|
||||
Self::Embed(_) | Self::Rerank(_) | Self::GenerateImage(_) | Self::Speech(_) => false,
|
||||
Self::GenerateImage(_) => true,
|
||||
Self::Embed(_) | Self::Rerank(_) | Self::Speech(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
//! 下游 Client API Key 的准入策略。
|
||||
//!
|
||||
//! Client API Key 绑定一个 Provider 平台;模型名称不参与改写或 allowlist 判断。
|
||||
//! Client API Key 冻结账号分组权限;模型名称不参与权限判断。
|
||||
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::error::{IdentifierError, PolicyError, validate_text};
|
||||
use crate::routing::ProviderKind;
|
||||
use crate::routing::FrozenAccountScope;
|
||||
|
||||
/// `client_api_keys.id` 的核心值对象。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
@@ -88,7 +89,7 @@ impl RateLimits {
|
||||
pub struct ClientPolicy {
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
provider_kind: ProviderKind,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
enabled: bool,
|
||||
limits: RateLimits,
|
||||
}
|
||||
@@ -98,14 +99,14 @@ impl ClientPolicy {
|
||||
pub const fn new(
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
provider_kind: ProviderKind,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
enabled: bool,
|
||||
limits: RateLimits,
|
||||
) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
plaintext_key,
|
||||
provider_kind,
|
||||
account_scope,
|
||||
enabled,
|
||||
limits,
|
||||
}
|
||||
@@ -122,8 +123,8 @@ impl ClientPolicy {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
|
||||
&self.account_scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
||||
@@ -9,10 +9,11 @@ use std::time::{Duration, SystemTime};
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::engine::credential::{
|
||||
AccountAvailability, AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision,
|
||||
AccountFeedbackStats, AccountRuntimeSignals, CredentialRevision, CredentialState,
|
||||
OpaqueProviderData, ProviderAccountId, ProviderAccountStore,
|
||||
};
|
||||
use crate::error::{IdentifierError, validate_text};
|
||||
use crate::policy::ClientApiKeyId;
|
||||
use crate::routing::{ProviderKind, UpstreamModelId};
|
||||
|
||||
const MAX_PENDING_FLOW_TTL: Duration = Duration::from_secs(30 * 60);
|
||||
@@ -170,6 +171,7 @@ pub enum ProviderLeaseRequest {
|
||||
pub trait ProviderLeasePort: Send + Sync {
|
||||
fn load_state<'a>(
|
||||
&'a self,
|
||||
client_api_key_id: &'a ClientApiKeyId,
|
||||
provider_kind: &'a ProviderKind,
|
||||
accounts: &'a [ProviderAccountId],
|
||||
) -> BoxFuture<'a, Result<ProviderSchedulingState, ProviderStoreError>>;
|
||||
@@ -405,13 +407,91 @@ pub trait ProviderCatalogCachePort: Send + Sync {
|
||||
) -> BoxFuture<'a, Result<Option<OpaqueProviderData>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Provider 从官方制品核验出的可重建请求画像。
|
||||
///
|
||||
/// Core 只用单调制品序号约束覆盖顺序;具体版本字段由对应 Provider 放在
|
||||
/// `profile` 中解释。每个 Provider 在 Store 中只保留一份最新画像。
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub struct ProviderArtifactProfile {
|
||||
provider_kind: ProviderKind,
|
||||
artifact_sequence: u64,
|
||||
verified_at: SystemTime,
|
||||
profile: OpaqueProviderData,
|
||||
}
|
||||
|
||||
impl ProviderArtifactProfile {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
provider_kind: ProviderKind,
|
||||
artifact_sequence: u64,
|
||||
verified_at: SystemTime,
|
||||
profile: OpaqueProviderData,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
artifact_sequence,
|
||||
verified_at,
|
||||
profile,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn artifact_sequence(&self) -> u64 {
|
||||
self.artifact_sequence
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn verified_at(&self) -> SystemTime {
|
||||
self.verified_at
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn profile(&self) -> &OpaqueProviderData {
|
||||
&self.profile
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ProviderArtifactProfile {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ProviderArtifactProfile")
|
||||
.field("provider_kind", &self.provider_kind)
|
||||
.field("artifact_sequence", &self.artifact_sequence)
|
||||
.field("verified_at", &self.verified_at)
|
||||
.field("profile", &"[PROVIDER-OWNED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait ProviderArtifactProfileCachePort: Send + Sync {
|
||||
/// 覆盖同一 Provider 的固定 cache key。
|
||||
///
|
||||
/// 返回 `false` 表示 Store 已持有更高的制品序号;相同序号但内容不同必须返回
|
||||
/// [`ProviderStoreErrorKind::Conflict`],不能静默改写已核验画像。
|
||||
fn replace_if_newer(
|
||||
&self,
|
||||
profile: ProviderArtifactProfile,
|
||||
ttl: Duration,
|
||||
) -> BoxFuture<'_, Result<bool, ProviderStoreError>>;
|
||||
|
||||
fn read<'a>(
|
||||
&'a self,
|
||||
provider_kind: &'a ProviderKind,
|
||||
) -> BoxFuture<'a, Result<Option<ProviderArtifactProfile>, ProviderStoreError>>;
|
||||
}
|
||||
|
||||
/// Redis 中可重建的账号状态投影。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCredentialState {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
}
|
||||
|
||||
@@ -421,14 +501,14 @@ impl ProviderCredentialState {
|
||||
account_id: ProviderAccountId,
|
||||
credential_revision: CredentialRevision,
|
||||
enabled: bool,
|
||||
availability: AccountAvailability,
|
||||
credential_state: CredentialState,
|
||||
observed_at: SystemTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
credential_revision,
|
||||
enabled,
|
||||
availability,
|
||||
credential_state,
|
||||
observed_at,
|
||||
}
|
||||
}
|
||||
@@ -449,8 +529,8 @@ impl ProviderCredentialState {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn availability(&self) -> AccountAvailability {
|
||||
self.availability
|
||||
pub const fn credential_state(&self) -> CredentialState {
|
||||
self.credential_state
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
@@ -674,22 +754,20 @@ impl ProviderRefreshPolicy {
|
||||
self.concurrency
|
||||
}
|
||||
|
||||
/// 按配置的精确提前量计算刷新时刻。
|
||||
pub fn next_attempt_at(
|
||||
/// 判断 AT 是否已经进入当前配置的提前刷新窗口。
|
||||
///
|
||||
/// `margin` 是运行时策略,不投影为账号的持久化时间字段;已过期 AT 也需要
|
||||
/// 尝试用 RT 恢复,因此视为到期。
|
||||
#[must_use]
|
||||
pub fn is_refresh_due(
|
||||
self,
|
||||
_account_id: &ProviderAccountId,
|
||||
access_token_expires_at: SystemTime,
|
||||
observed_at: SystemTime,
|
||||
) -> Result<SystemTime, ProviderStoreError> {
|
||||
let remaining = access_token_expires_at
|
||||
.duration_since(observed_at)
|
||||
.map_err(|_| invalid_refresh_policy("schedule expired access token"))?;
|
||||
if self.margin >= remaining {
|
||||
return Ok(observed_at);
|
||||
) -> bool {
|
||||
match access_token_expires_at.duration_since(observed_at) {
|
||||
Ok(remaining) => remaining <= self.margin,
|
||||
Err(_) => true,
|
||||
}
|
||||
access_token_expires_at
|
||||
.checked_sub(self.margin)
|
||||
.ok_or_else(|| invalid_refresh_policy("schedule access token refresh"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -946,6 +1024,7 @@ pub struct ProviderStorePorts {
|
||||
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
|
||||
account_feedback: Arc<AccountFeedbackStats>,
|
||||
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
|
||||
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -962,6 +1041,7 @@ impl ProviderStorePorts {
|
||||
session_affinity: Arc<dyn ProviderSessionAffinityPort>,
|
||||
session_exclusions: Arc<dyn ProviderSessionExclusionPort>,
|
||||
catalog_cache: Arc<dyn ProviderCatalogCachePort>,
|
||||
artifact_profiles: Arc<dyn ProviderArtifactProfileCachePort>,
|
||||
credential_state: Arc<dyn ProviderCredentialStatePort>,
|
||||
cooldowns: Arc<dyn ProviderCooldownPort>,
|
||||
runtime_policy: Arc<dyn ProviderRuntimePolicyPort>,
|
||||
@@ -974,6 +1054,7 @@ impl ProviderStorePorts {
|
||||
session_exclusions,
|
||||
account_feedback: Arc::new(AccountFeedbackStats::default()),
|
||||
catalog_cache,
|
||||
artifact_profiles,
|
||||
credential_state,
|
||||
cooldowns,
|
||||
runtime_policy,
|
||||
@@ -1011,6 +1092,11 @@ impl ProviderStorePorts {
|
||||
Arc::clone(&self.catalog_cache)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn artifact_profiles(&self) -> Arc<dyn ProviderArtifactProfileCachePort> {
|
||||
Arc::clone(&self.artifact_profiles)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn credential_state(&self) -> Arc<dyn ProviderCredentialStatePort> {
|
||||
Arc::clone(&self.credential_state)
|
||||
|
||||
@@ -9,12 +9,293 @@ use std::fmt;
|
||||
use std::num::{NonZeroU32, NonZeroU64};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::engine::credential::AccountSelectionPolicy;
|
||||
use crate::engine::credential::{AccountSelectionPolicy, ProviderAccountId};
|
||||
use crate::error::{IdentifierError, RoutingError, validate_text};
|
||||
use crate::operation::{CapabilityRequirements, Feature, OperationKind};
|
||||
|
||||
const MAX_REQUEST_ATTEMPTS: u32 = 32;
|
||||
|
||||
/// `account_groups.id` 的核心值对象。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct AccountGroupId(String);
|
||||
|
||||
impl AccountGroupId {
|
||||
/// 校验并创建账号分组 ID。
|
||||
pub fn new(value: impl Into<String>) -> Result<Self, IdentifierError> {
|
||||
let value = value.into();
|
||||
let Some(suffix) = value.strip_prefix("grp_") else {
|
||||
return Err(IdentifierError::MissingPrefix { expected: "grp_" });
|
||||
};
|
||||
if suffix.len() != 32
|
||||
|| !suffix
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err(IdentifierError::InvalidFormat);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for AccountGroupId {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// 快照中一个账号的 Provider 与分组归属。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RuntimeAccount {
|
||||
provider_kind: ProviderKind,
|
||||
group_ids: Arc<BTreeSet<AccountGroupId>>,
|
||||
}
|
||||
|
||||
impl RuntimeAccount {
|
||||
#[must_use]
|
||||
pub fn new(provider_kind: ProviderKind, group_ids: BTreeSet<AccountGroupId>) -> Self {
|
||||
Self {
|
||||
provider_kind,
|
||||
group_ids: Arc::new(group_ids),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn provider_kind(&self) -> &ProviderKind {
|
||||
&self.provider_kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn group_ids(&self) -> &BTreeSet<AccountGroupId> {
|
||||
&self.group_ids
|
||||
}
|
||||
}
|
||||
|
||||
/// 全快照共享的账号、Provider 与分组反向索引。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RuntimeAccountDirectory {
|
||||
accounts: BTreeMap<ProviderAccountId, RuntimeAccount>,
|
||||
providers_with_accounts: BTreeSet<ProviderKind>,
|
||||
providers_by_group: BTreeMap<AccountGroupId, BTreeSet<ProviderKind>>,
|
||||
}
|
||||
|
||||
impl RuntimeAccountDirectory {
|
||||
#[must_use]
|
||||
pub fn new(accounts: BTreeMap<ProviderAccountId, RuntimeAccount>) -> Self {
|
||||
let mut providers_with_accounts = BTreeSet::new();
|
||||
let mut providers_by_group = BTreeMap::<AccountGroupId, BTreeSet<ProviderKind>>::new();
|
||||
for account in accounts.values() {
|
||||
providers_with_accounts.insert(account.provider_kind.clone());
|
||||
for group_id in account.group_ids.iter() {
|
||||
providers_by_group
|
||||
.entry(group_id.clone())
|
||||
.or_default()
|
||||
.insert(account.provider_kind.clone());
|
||||
}
|
||||
}
|
||||
Self {
|
||||
accounts,
|
||||
providers_with_accounts,
|
||||
providers_by_group,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn account(&self, account_id: &ProviderAccountId) -> Option<&RuntimeAccount> {
|
||||
self.accounts.get(account_id)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn providers_with_accounts(&self) -> &BTreeSet<ProviderKind> {
|
||||
&self.providers_with_accounts
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn providers_for_groups<'a>(
|
||||
&self,
|
||||
group_ids: impl IntoIterator<Item = &'a AccountGroupId>,
|
||||
) -> BTreeSet<ProviderKind> {
|
||||
group_ids
|
||||
.into_iter()
|
||||
.filter_map(|group_id| self.providers_by_group.get(group_id))
|
||||
.flatten()
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// 历史请求保存的账号范围种类。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AccountRoutingScopeKind {
|
||||
All,
|
||||
Groups,
|
||||
}
|
||||
|
||||
impl AccountRoutingScopeKind {
|
||||
#[must_use]
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::All => "all",
|
||||
Self::Groups => "groups",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求开始时冻结的分组 ID 与名称。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RoutingGroupSnapshot {
|
||||
id: AccountGroupId,
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl RoutingGroupSnapshot {
|
||||
#[must_use]
|
||||
pub fn new(id: AccountGroupId, name: String) -> Self {
|
||||
Self { id, name }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn id(&self) -> &AccountGroupId {
|
||||
&self.id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求历史所需的完整、稳定账号范围快照。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AccountRoutingSnapshot {
|
||||
kind: AccountRoutingScopeKind,
|
||||
groups: Arc<[RoutingGroupSnapshot]>,
|
||||
}
|
||||
|
||||
impl AccountRoutingSnapshot {
|
||||
#[must_use]
|
||||
pub fn all() -> Self {
|
||||
Self {
|
||||
kind: AccountRoutingScopeKind::All,
|
||||
groups: Arc::from([]),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn groups(groups: Vec<RoutingGroupSnapshot>) -> Self {
|
||||
Self {
|
||||
kind: AccountRoutingScopeKind::Groups,
|
||||
groups: Arc::from(groups),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> AccountRoutingScopeKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn groups_snapshot(&self) -> &[RoutingGroupSnapshot] {
|
||||
&self.groups
|
||||
}
|
||||
}
|
||||
|
||||
/// Key 持久 binding 编译出的账号权限。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ClientRoutingScope {
|
||||
AllAccounts,
|
||||
Restricted {
|
||||
bound_groups: Arc<[RoutingGroupSnapshot]>,
|
||||
enabled_group_ids: Arc<BTreeSet<AccountGroupId>>,
|
||||
provider_kinds: Arc<BTreeSet<ProviderKind>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ClientRoutingScope {
|
||||
#[must_use]
|
||||
pub fn all_accounts() -> Self {
|
||||
Self::AllAccounts
|
||||
}
|
||||
|
||||
pub fn restricted(
|
||||
bound_groups: Vec<RoutingGroupSnapshot>,
|
||||
enabled_group_ids: BTreeSet<AccountGroupId>,
|
||||
provider_kinds: BTreeSet<ProviderKind>,
|
||||
) -> Result<Self, RoutingError> {
|
||||
if bound_groups.is_empty() {
|
||||
return Err(RoutingError::InvalidAccountScope);
|
||||
}
|
||||
Ok(Self::Restricted {
|
||||
bound_groups: Arc::from(bound_groups),
|
||||
enabled_group_ids: Arc::new(enabled_group_ids),
|
||||
provider_kinds: Arc::new(provider_kinds),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次认证随 RuntimeSnapshot 冻结的账号目录与 Key 权限。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FrozenAccountScope {
|
||||
directory: Arc<RuntimeAccountDirectory>,
|
||||
client_scope: ClientRoutingScope,
|
||||
}
|
||||
|
||||
impl FrozenAccountScope {
|
||||
#[must_use]
|
||||
pub const fn new(
|
||||
directory: Arc<RuntimeAccountDirectory>,
|
||||
client_scope: ClientRoutingScope,
|
||||
) -> Self {
|
||||
Self {
|
||||
directory,
|
||||
client_scope,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn allows(&self, account_id: &ProviderAccountId) -> bool {
|
||||
let Some(account) = self.directory.account(account_id) else {
|
||||
return false;
|
||||
};
|
||||
match &self.client_scope {
|
||||
ClientRoutingScope::AllAccounts => true,
|
||||
ClientRoutingScope::Restricted {
|
||||
enabled_group_ids, ..
|
||||
} => account
|
||||
.group_ids()
|
||||
.iter()
|
||||
.any(|group_id| enabled_group_ids.contains(group_id)),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn provider_kinds(&self) -> &BTreeSet<ProviderKind> {
|
||||
match &self.client_scope {
|
||||
ClientRoutingScope::AllAccounts => self.directory.providers_with_accounts(),
|
||||
ClientRoutingScope::Restricted { provider_kinds, .. } => provider_kinds,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn routing_snapshot(&self) -> AccountRoutingSnapshot {
|
||||
match &self.client_scope {
|
||||
ClientRoutingScope::AllAccounts => AccountRoutingSnapshot::all(),
|
||||
ClientRoutingScope::Restricted { bound_groups, .. } => {
|
||||
AccountRoutingSnapshot::groups(bound_groups.to_vec())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn directory(&self) -> &Arc<RuntimeAccountDirectory> {
|
||||
&self.directory
|
||||
}
|
||||
}
|
||||
|
||||
/// 编译进二进制的 Provider adapter slug。
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct ProviderKind(String);
|
||||
@@ -494,17 +775,18 @@ impl ProviderModel {
|
||||
/// 本次请求选择 Provider 时使用的动态过滤事实。
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RoutingContext {
|
||||
/// 已认证 Client API Key 绑定的平台;模型名称不参与平台猜测。
|
||||
pub provider_kind: Option<ProviderKind>,
|
||||
/// 管理端 connection test 显式限制的 Provider;普通请求留空。
|
||||
pub required_provider: Option<ProviderKind>,
|
||||
pub blocked_providers: BTreeSet<ProviderKind>,
|
||||
}
|
||||
|
||||
/// 已绑定 Provider 与真实上游模型的请求候选。
|
||||
/// 已绑定 Provider 的请求候选;模型端点携带真实上游模型,原生端点不虚构模型。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProviderCandidate {
|
||||
provider: ProviderKind,
|
||||
upstream_model: UpstreamModelId,
|
||||
upstream_model: Option<UpstreamModelId>,
|
||||
emulated_features: BTreeSet<Feature>,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
}
|
||||
|
||||
impl ProviderCandidate {
|
||||
@@ -514,14 +796,19 @@ impl ProviderCandidate {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn upstream_model(&self) -> &UpstreamModelId {
|
||||
&self.upstream_model
|
||||
pub const fn upstream_model(&self) -> Option<&UpstreamModelId> {
|
||||
self.upstream_model.as_ref()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn emulated_features(&self) -> &BTreeSet<Feature> {
|
||||
&self.emulated_features
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
|
||||
&self.account_scope
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次请求冻结的 Provider 尝试顺序。
|
||||
@@ -529,9 +816,9 @@ impl ProviderCandidate {
|
||||
pub struct RoutingPlan {
|
||||
config_revision: ConfigRevision,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
public_model: PublicModelId,
|
||||
operation: OperationKind,
|
||||
max_attempts: NonZeroU32,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
candidates: Arc<[ProviderCandidate]>,
|
||||
}
|
||||
|
||||
@@ -546,11 +833,6 @@ impl RoutingPlan {
|
||||
self.account_selection_policy
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn public_model(&self) -> &PublicModelId {
|
||||
&self.public_model
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn operation(&self) -> OperationKind {
|
||||
self.operation
|
||||
@@ -561,6 +843,11 @@ impl RoutingPlan {
|
||||
self.max_attempts
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn account_scope(&self) -> &Arc<FrozenAccountScope> {
|
||||
&self.account_scope
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn candidates(&self) -> &[ProviderCandidate] {
|
||||
&self.candidates
|
||||
|
||||
@@ -11,7 +11,7 @@ use futures::{FutureExt as _, Stream, StreamExt as _, pin_mut, select_biased};
|
||||
use futures_timer::Delay;
|
||||
|
||||
use crate::engine::CancellationToken;
|
||||
use crate::engine::credential::{AccountSelectionPolicy, RotationStrategy};
|
||||
use crate::engine::credential::{AccountSelectionPolicy, ProviderAccountId, RotationStrategy};
|
||||
use crate::engine::provider::{ProviderCatalogGeneration, ProviderRegistry};
|
||||
use crate::error::RoutingError;
|
||||
use crate::health::{HealthProbe, HealthState};
|
||||
@@ -24,8 +24,9 @@ use crate::task::{
|
||||
};
|
||||
|
||||
use super::{
|
||||
ConfigRevision, ModelCapabilities, ProviderCandidate, ProviderKind, ProviderModel,
|
||||
PublicModelId, RoutingContext, RoutingPlan, UpstreamModelId,
|
||||
AccountGroupId, ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities,
|
||||
ProviderCandidate, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
|
||||
RoutingGroupSnapshot, RoutingPlan, RuntimeAccount, RuntimeAccountDirectory, UpstreamModelId,
|
||||
};
|
||||
|
||||
const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(5);
|
||||
@@ -66,7 +67,7 @@ impl SnapshotSettingsFacts {
|
||||
pub struct SnapshotClientPolicyFacts {
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
provider_kind: String,
|
||||
group_ids: Vec<AccountGroupId>,
|
||||
limits: RateLimits,
|
||||
}
|
||||
|
||||
@@ -75,18 +76,67 @@ impl SnapshotClientPolicyFacts {
|
||||
pub fn new(
|
||||
key_id: ClientApiKeyId,
|
||||
plaintext_key: PlaintextClientApiKey,
|
||||
provider_kind: impl Into<String>,
|
||||
group_ids: Vec<AccountGroupId>,
|
||||
limits: RateLimits,
|
||||
) -> Self {
|
||||
Self {
|
||||
key_id,
|
||||
plaintext_key,
|
||||
provider_kind: provider_kind.into(),
|
||||
group_ids,
|
||||
limits,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Store 读取到的账号分组事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotAccountGroupFacts {
|
||||
id: AccountGroupId,
|
||||
name: String,
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl SnapshotAccountGroupFacts {
|
||||
#[must_use]
|
||||
pub fn new(id: AccountGroupId, name: String, enabled: bool) -> Self {
|
||||
Self { id, name, enabled }
|
||||
}
|
||||
}
|
||||
|
||||
/// Store 读取到的账号及其固有 Provider 事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotProviderAccountFacts {
|
||||
account_id: ProviderAccountId,
|
||||
provider_kind: String,
|
||||
}
|
||||
|
||||
impl SnapshotProviderAccountFacts {
|
||||
#[must_use]
|
||||
pub fn new(account_id: ProviderAccountId, provider_kind: impl Into<String>) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
provider_kind: provider_kind.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Store 读取到的一条分组成员关系。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotAccountGroupMemberFacts {
|
||||
group_id: AccountGroupId,
|
||||
account_id: ProviderAccountId,
|
||||
}
|
||||
|
||||
impl SnapshotAccountGroupMemberFacts {
|
||||
#[must_use]
|
||||
pub const fn new(group_id: AccountGroupId, account_id: ProviderAccountId) -> Self {
|
||||
Self {
|
||||
group_id,
|
||||
account_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 一次一致性读取产生的全部 RuntimeSnapshot 持久事实。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotFacts {
|
||||
@@ -94,6 +144,9 @@ pub struct SnapshotFacts {
|
||||
observed_current_revision: ConfigRevision,
|
||||
settings: SnapshotSettingsFacts,
|
||||
client_policies: Vec<SnapshotClientPolicyFacts>,
|
||||
account_groups: Vec<SnapshotAccountGroupFacts>,
|
||||
provider_accounts: Vec<SnapshotProviderAccountFacts>,
|
||||
group_memberships: Vec<SnapshotAccountGroupMemberFacts>,
|
||||
}
|
||||
|
||||
impl SnapshotFacts {
|
||||
@@ -103,12 +156,18 @@ impl SnapshotFacts {
|
||||
observed_current_revision: ConfigRevision,
|
||||
settings: SnapshotSettingsFacts,
|
||||
client_policies: Vec<SnapshotClientPolicyFacts>,
|
||||
account_groups: Vec<SnapshotAccountGroupFacts>,
|
||||
provider_accounts: Vec<SnapshotProviderAccountFacts>,
|
||||
group_memberships: Vec<SnapshotAccountGroupMemberFacts>,
|
||||
) -> Self {
|
||||
Self {
|
||||
config_revision,
|
||||
observed_current_revision,
|
||||
settings,
|
||||
client_policies,
|
||||
account_groups,
|
||||
provider_accounts,
|
||||
group_memberships,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,13 +282,16 @@ async fn compile_runtime_snapshot(
|
||||
providers: &ProviderRegistry,
|
||||
) -> Result<RuntimeSnapshot, RuntimeSnapshotCompileError> {
|
||||
let provider_kinds = providers.provider_kinds().cloned().collect::<Vec<_>>();
|
||||
let registered_providers = provider_kinds.iter().cloned().collect::<BTreeSet<_>>();
|
||||
|
||||
// 实时目录只提供公开模型与能力提示;查询失败时保留 Provider 透传语义。
|
||||
// 目录查询失败表示未知;查询成功后,即使为空,也必须与“已知缺少模型”区分。
|
||||
let mut provider_models = Vec::new();
|
||||
let mut known_provider_catalogs = BTreeSet::new();
|
||||
for provider in &provider_kinds {
|
||||
let Ok(models) = providers.query_model_capabilities(provider).await else {
|
||||
continue;
|
||||
};
|
||||
known_provider_catalogs.insert(provider.clone());
|
||||
provider_models.extend(models.into_iter().map(|model| {
|
||||
let compiled = ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -243,6 +305,56 @@ async fn compile_runtime_snapshot(
|
||||
}));
|
||||
}
|
||||
|
||||
let mut groups = BTreeMap::new();
|
||||
for group in facts.account_groups {
|
||||
if group.name.trim() != group.name
|
||||
|| group.name.is_empty()
|
||||
|| group.name.chars().count() > 100
|
||||
|| group.name.chars().any(char::is_control)
|
||||
|| groups.insert(group.id.clone(), group).is_some()
|
||||
{
|
||||
return Err(RuntimeSnapshotCompileError::InvalidData);
|
||||
}
|
||||
}
|
||||
|
||||
let mut account_groups = BTreeMap::<ProviderAccountId, BTreeSet<AccountGroupId>>::new();
|
||||
let mut accounts = BTreeMap::new();
|
||||
for account in facts.provider_accounts {
|
||||
let provider_kind = ProviderKind::new(account.provider_kind)
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?;
|
||||
if !registered_providers.contains(&provider_kind)
|
||||
|| accounts
|
||||
.insert(
|
||||
account.account_id.clone(),
|
||||
RuntimeAccount::new(provider_kind, BTreeSet::new()),
|
||||
)
|
||||
.is_some()
|
||||
{
|
||||
return Err(RuntimeSnapshotCompileError::InvalidData);
|
||||
}
|
||||
account_groups.insert(account.account_id, BTreeSet::new());
|
||||
}
|
||||
let mut memberships = BTreeSet::new();
|
||||
for membership in facts.group_memberships {
|
||||
if !groups.contains_key(&membership.group_id)
|
||||
|| !accounts.contains_key(&membership.account_id)
|
||||
|| !memberships.insert((membership.group_id.clone(), membership.account_id.clone()))
|
||||
{
|
||||
return Err(RuntimeSnapshotCompileError::InvalidData);
|
||||
}
|
||||
account_groups
|
||||
.get_mut(&membership.account_id)
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?
|
||||
.insert(membership.group_id);
|
||||
}
|
||||
for (account_id, group_ids) in account_groups {
|
||||
let account = accounts
|
||||
.get_mut(&account_id)
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
*account = RuntimeAccount::new(account.provider_kind().clone(), group_ids);
|
||||
}
|
||||
let account_directory = Arc::new(RuntimeAccountDirectory::new(accounts));
|
||||
|
||||
let model_mappings = facts.settings.model_mappings;
|
||||
let rotation_strategy = RotationStrategy::parse(facts.settings.rotation_strategy.as_str())
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
@@ -252,21 +364,48 @@ async fn compile_runtime_snapshot(
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?,
|
||||
Duration::from_millis(facts.settings.request_interval_ms),
|
||||
);
|
||||
let client_policies = facts
|
||||
.client_policies
|
||||
.into_iter()
|
||||
.map(|policy| {
|
||||
let provider = ProviderKind::new(policy.provider_kind)
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?;
|
||||
Ok(ClientPolicy::new(
|
||||
policy.key_id,
|
||||
policy.plaintext_key,
|
||||
provider,
|
||||
true,
|
||||
policy.limits,
|
||||
))
|
||||
})
|
||||
.collect::<Result<Vec<_>, RuntimeSnapshotCompileError>>()?;
|
||||
let mut client_policies = Vec::with_capacity(facts.client_policies.len());
|
||||
for policy in facts.client_policies {
|
||||
let account_scope = if policy.group_ids.is_empty() {
|
||||
FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)
|
||||
} else {
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut bound_groups = Vec::with_capacity(policy.group_ids.len());
|
||||
let mut enabled_group_ids = BTreeSet::new();
|
||||
for group_id in policy.group_ids {
|
||||
if !seen.insert(group_id.clone()) {
|
||||
return Err(RuntimeSnapshotCompileError::InvalidData);
|
||||
}
|
||||
let group = groups
|
||||
.get(&group_id)
|
||||
.ok_or(RuntimeSnapshotCompileError::InvalidData)?;
|
||||
bound_groups.push(RoutingGroupSnapshot::new(
|
||||
group.id.clone(),
|
||||
group.name.clone(),
|
||||
));
|
||||
if group.enabled {
|
||||
enabled_group_ids.insert(group_id);
|
||||
}
|
||||
}
|
||||
bound_groups.sort_by(|left, right| left.id().cmp(right.id()));
|
||||
let provider_kinds = account_directory.providers_for_groups(&enabled_group_ids);
|
||||
FrozenAccountScope::new(
|
||||
Arc::clone(&account_directory),
|
||||
ClientRoutingScope::restricted(bound_groups, enabled_group_ids, provider_kinds)
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)?,
|
||||
)
|
||||
};
|
||||
client_policies.push(ClientPolicy::new(
|
||||
policy.key_id,
|
||||
policy.plaintext_key,
|
||||
Arc::new(account_scope),
|
||||
true,
|
||||
policy.limits,
|
||||
));
|
||||
}
|
||||
|
||||
RuntimeSnapshot::new(
|
||||
facts.config_revision,
|
||||
@@ -276,7 +415,12 @@ async fn compile_runtime_snapshot(
|
||||
client_policies,
|
||||
)
|
||||
.map_err(|_| RuntimeSnapshotCompileError::InvalidData)
|
||||
.map(|snapshot| snapshot.with_model_mappings(model_mappings))
|
||||
.map(|snapshot| {
|
||||
snapshot
|
||||
.with_model_mappings(model_mappings)
|
||||
.with_account_directory(account_directory)
|
||||
.with_known_provider_catalogs(known_provider_catalogs)
|
||||
})
|
||||
}
|
||||
|
||||
/// 数据面使用的不可变配置快照。
|
||||
@@ -290,6 +434,8 @@ pub struct RuntimeSnapshot {
|
||||
Arc<BTreeMap<ProviderKind, BTreeMap<UpstreamModelId, super::ModelPresentation>>>,
|
||||
model_mappings: Arc<BTreeMap<String, String>>,
|
||||
provider_catalog_generations: Arc<BTreeMap<ProviderKind, ProviderCatalogGeneration>>,
|
||||
known_provider_catalogs: Arc<BTreeSet<ProviderKind>>,
|
||||
account_directory: Arc<RuntimeAccountDirectory>,
|
||||
client_policies: Arc<BTreeMap<ClientApiKeyId, ClientPolicy>>,
|
||||
}
|
||||
|
||||
@@ -312,6 +458,7 @@ impl RuntimeSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
let mut known_provider_catalogs = BTreeSet::new();
|
||||
let mut model_map =
|
||||
BTreeMap::<ProviderKind, BTreeMap<UpstreamModelId, ModelCapabilities>>::new();
|
||||
let mut presentation_map =
|
||||
@@ -323,6 +470,7 @@ impl RuntimeSnapshot {
|
||||
capabilities,
|
||||
presentation,
|
||||
} = model;
|
||||
known_provider_catalogs.insert(provider.clone());
|
||||
if !provider_set.contains(&provider) {
|
||||
return Err(RoutingError::NotFound {
|
||||
entity: "provider",
|
||||
@@ -367,6 +515,8 @@ impl RuntimeSnapshot {
|
||||
provider_model_presentations: Arc::new(presentation_map),
|
||||
model_mappings: Arc::new(BTreeMap::new()),
|
||||
provider_catalog_generations: Arc::new(BTreeMap::new()),
|
||||
known_provider_catalogs: Arc::new(known_provider_catalogs),
|
||||
account_directory: Arc::new(RuntimeAccountDirectory::default()),
|
||||
client_policies: Arc::new(client_policy_map),
|
||||
})
|
||||
}
|
||||
@@ -377,6 +527,26 @@ impl RuntimeSnapshot {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_account_directory(mut self, directory: Arc<RuntimeAccountDirectory>) -> Self {
|
||||
self.account_directory = directory;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn with_known_provider_catalogs(mut self, providers: BTreeSet<ProviderKind>) -> Self {
|
||||
self.known_provider_catalogs = Arc::new(providers);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn all_account_scope(&self) -> Arc<FrozenAccountScope> {
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::clone(&self.account_directory),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn with_provider_catalog_generations(
|
||||
mut self,
|
||||
@@ -459,8 +629,39 @@ impl RuntimeSnapshot {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 已取得目录时按目录与全局别名判断模型可用性;目录暂不可用时沿用旧版
|
||||
/// fail-open 语义,把具体 wire 可接受性留给上游。
|
||||
/// 合并冻结账号范围内实际存在 Provider 的公开模型。
|
||||
#[must_use]
|
||||
pub fn public_models_for_scope(&self, scope: &FrozenAccountScope) -> Vec<PublicModelId> {
|
||||
scope
|
||||
.provider_kinds()
|
||||
.iter()
|
||||
.flat_map(|provider| self.public_models_for_provider(provider))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 合并冻结账号范围内实际存在 Provider 的公开模型画像。
|
||||
#[must_use]
|
||||
pub fn public_model_profiles_for_scope(
|
||||
&self,
|
||||
scope: &FrozenAccountScope,
|
||||
) -> Vec<super::PublicModelProfile> {
|
||||
let mut profiles = BTreeMap::new();
|
||||
for provider in scope.provider_kinds() {
|
||||
for profile in self.public_model_profiles_for_provider(provider) {
|
||||
profiles
|
||||
.entry(profile.model().clone())
|
||||
.or_insert_with(|| profile.presentation().clone());
|
||||
}
|
||||
}
|
||||
profiles
|
||||
.into_iter()
|
||||
.map(|(model, presentation)| super::PublicModelProfile::new(model, presentation))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 已取得目录时以映射后的上游模型为准;目录不可用时由 Provider 在发送前判定。
|
||||
#[must_use]
|
||||
pub fn contains_public_model_for_provider(
|
||||
&self,
|
||||
@@ -470,17 +671,25 @@ impl RuntimeSnapshot {
|
||||
if !self.providers.contains(provider) {
|
||||
return false;
|
||||
}
|
||||
if self.model_mappings.contains_key(public_model.as_str()) {
|
||||
return true;
|
||||
}
|
||||
let upstream_model = self.mapped_model(public_model.as_str());
|
||||
match self.provider_models.get(provider) {
|
||||
Some(models) if !models.is_empty() => models
|
||||
.keys()
|
||||
.any(|model| model.as_str() == public_model.as_str()),
|
||||
Some(_) | None => true,
|
||||
Some(models) => models.keys().any(|model| model.as_str() == upstream_model),
|
||||
None => !self.known_provider_catalogs.contains(provider),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn contains_public_model_for_scope(
|
||||
&self,
|
||||
public_model: &PublicModelId,
|
||||
scope: &FrozenAccountScope,
|
||||
) -> bool {
|
||||
scope
|
||||
.provider_kinds()
|
||||
.iter()
|
||||
.any(|provider| self.contains_public_model_for_provider(public_model, provider))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn mapped_model(&self, requested: &str) -> String {
|
||||
let original = requested;
|
||||
@@ -506,14 +715,26 @@ impl RuntimeSnapshot {
|
||||
&self,
|
||||
public_model: &PublicModelId,
|
||||
operation: &Operation,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
context: &RoutingContext,
|
||||
) -> Result<RoutingPlan, RoutingError> {
|
||||
let requirements = operation.capability_requirements();
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
for provider in self.providers.iter() {
|
||||
if context.required_provider.is_none() && account_scope.provider_kinds().is_empty() {
|
||||
return Err(RoutingError::EmptyAccountScope);
|
||||
}
|
||||
|
||||
let providers = context.required_provider.as_ref().map_or_else(
|
||||
|| account_scope.provider_kinds().clone(),
|
||||
|provider| BTreeSet::from([provider.clone()]),
|
||||
);
|
||||
for provider in &providers {
|
||||
if !self.providers.contains(provider) {
|
||||
continue;
|
||||
}
|
||||
if context
|
||||
.provider_kind
|
||||
.required_provider
|
||||
.as_ref()
|
||||
.is_some_and(|expected| expected != provider)
|
||||
|| context.blocked_providers.contains(provider)
|
||||
@@ -539,12 +760,14 @@ impl RuntimeSnapshot {
|
||||
};
|
||||
emulated
|
||||
}
|
||||
None if self.known_provider_catalogs.contains(provider) => continue,
|
||||
None => BTreeSet::new(),
|
||||
};
|
||||
candidates.push(ProviderCandidate {
|
||||
provider: provider.clone(),
|
||||
upstream_model,
|
||||
upstream_model: Some(upstream_model),
|
||||
emulated_features,
|
||||
account_scope: Arc::clone(&account_scope),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -557,13 +780,54 @@ impl RuntimeSnapshot {
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
public_model: public_model.clone(),
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
.expect("constant request attempt limit is non-zero"),
|
||||
account_scope,
|
||||
candidates: Arc::from(candidates),
|
||||
})
|
||||
}
|
||||
|
||||
/// 为 Provider 自有、且不属于文本模型目录的端点冻结请求计划。
|
||||
///
|
||||
/// 端点 adapter 已经确定 Provider,因此这里只执行账号范围、circuit 和注册
|
||||
/// 状态检查;不会读取业务正文、构造模型或查询 Provider 文本模型目录。
|
||||
pub fn plan_provider_endpoint(
|
||||
&self,
|
||||
provider: &ProviderKind,
|
||||
operation: &Operation,
|
||||
account_scope: Arc<FrozenAccountScope>,
|
||||
context: &RoutingContext,
|
||||
) -> Result<RoutingPlan, RoutingError> {
|
||||
let allowed = !account_scope.provider_kinds().is_empty()
|
||||
&& account_scope.provider_kinds().contains(provider)
|
||||
&& self.providers.contains(provider)
|
||||
&& context
|
||||
.required_provider
|
||||
.as_ref()
|
||||
.is_none_or(|required| required == provider)
|
||||
&& !context.blocked_providers.contains(provider);
|
||||
if !allowed {
|
||||
return Err(RoutingError::NoCapableProviderEndpoint {
|
||||
provider: provider.as_str().to_owned(),
|
||||
});
|
||||
}
|
||||
let candidate = ProviderCandidate {
|
||||
provider: provider.clone(),
|
||||
upstream_model: None,
|
||||
emulated_features: BTreeSet::new(),
|
||||
account_scope: Arc::clone(&account_scope),
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
config_revision: self.revision,
|
||||
account_selection_policy: self.account_selection_policy,
|
||||
operation: operation.kind(),
|
||||
max_attempts: NonZeroU32::new(super::MAX_REQUEST_ATTEMPTS)
|
||||
.expect("constant request attempt limit is non-zero"),
|
||||
account_scope,
|
||||
candidates: Arc::from([candidate]),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// 请求级冻结失败;此状态必须 fail closed。
|
||||
|
||||
@@ -2,12 +2,14 @@ use gateway_core::engine::continuation::{
|
||||
ContinuationBinding, NativeContinuationPin, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::ProviderAccountId;
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
fn pin() -> NativeContinuationPin {
|
||||
NativeContinuationPin::new(
|
||||
PreviousResponseId::new("response-private"),
|
||||
PreviousResponseId::new("upstream-private"),
|
||||
ClientApiKeyId::new("key_client").expect("valid client key"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
ProviderAccountId::new("acct_codex").expect("valid account"),
|
||||
)
|
||||
@@ -43,3 +45,8 @@ fn native_pin_should_reject_different_account() {
|
||||
&ProviderAccountId::new("acct_other").expect("valid account"),
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_pin_should_reject_different_client_api_key() {
|
||||
assert!(!pin().matches_client(&ClientApiKeyId::new("key_other").expect("valid client key")));
|
||||
}
|
||||
|
||||
@@ -43,8 +43,9 @@ use gateway_core::operation::{
|
||||
};
|
||||
use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
|
||||
RoutingPlan, RuntimeSnapshot, UpstreamModelId,
|
||||
AccountRoutingSnapshot, ClientRoutingScope, ConfigRevision, FrozenAccountScope,
|
||||
ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext, RoutingPlan,
|
||||
RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot, UpstreamModelId,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
@@ -316,7 +317,10 @@ impl Provider for ScriptedProvider {
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
candidate.upstream_model().clone(),
|
||||
candidate
|
||||
.upstream_model()
|
||||
.cloned()
|
||||
.expect("model route candidate"),
|
||||
ProviderResource::Account {
|
||||
id: ProviderAccountId::new(account_id).expect("account id"),
|
||||
revision: CredentialRevision::new(1).expect("revision"),
|
||||
@@ -336,7 +340,10 @@ impl Provider for ScriptedProvider {
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
candidate.upstream_model().clone(),
|
||||
candidate
|
||||
.upstream_model()
|
||||
.cloned()
|
||||
.expect("model route candidate"),
|
||||
ProviderResource::Account {
|
||||
id: ProviderAccountId::new(account_id).expect("account id"),
|
||||
revision: CredentialRevision::new(1).expect("revision"),
|
||||
@@ -356,7 +363,10 @@ impl Provider for ScriptedProvider {
|
||||
let candidate = request.candidate();
|
||||
let metadata = ProviderCallMetadata::new(
|
||||
candidate.provider().clone(),
|
||||
candidate.upstream_model().clone(),
|
||||
candidate
|
||||
.upstream_model()
|
||||
.cloned()
|
||||
.expect("model route candidate"),
|
||||
ProviderResource::Account {
|
||||
id: ProviderAccountId::new(account_id).expect("account id"),
|
||||
revision: CredentialRevision::new(1).expect("revision"),
|
||||
@@ -492,6 +502,31 @@ fn bare_atomic_response_failed(response_id: &str) -> ProviderError {
|
||||
.with_atomic_client_events(vec![failed])
|
||||
}
|
||||
|
||||
fn atomic_history_unavailable(response_id: &str) -> ProviderError {
|
||||
let failed = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json(
|
||||
"openai",
|
||||
Some("response.failed".to_owned()),
|
||||
json!({
|
||||
"type": "response.failed",
|
||||
"response": {
|
||||
"id": response_id,
|
||||
"status": "failed",
|
||||
"error": {
|
||||
"code": "previous_response_not_found",
|
||||
"message": "Previous response was not found. Retrying the full request."
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.expect("history unavailable wire"),
|
||||
);
|
||||
ProviderError::new(ProviderErrorKind::InvalidRequest, UpstreamSendState::Sent)
|
||||
.with_status(400)
|
||||
.with_continuation_failure(ContinuationFailure::HistoryUnavailable)
|
||||
.with_atomic_client_events(vec![failed])
|
||||
}
|
||||
|
||||
fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, ProviderError>> {
|
||||
let mut usage = Usage::new();
|
||||
usage.input_tokens = Some(12);
|
||||
@@ -513,17 +548,52 @@ fn image_stream(image_output_tokens: Option<u64>) -> Vec<Result<GatewayEvent, Pr
|
||||
}
|
||||
|
||||
fn plan(operation: &Operation) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
plan_with_policy(
|
||||
operation,
|
||||
AccountSelectionPolicy::new(
|
||||
RotationStrategy::Smart,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn plan_with_policy(
|
||||
operation: &Operation,
|
||||
account_selection_policy: AccountSelectionPolicy,
|
||||
) -> RoutingPlan {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-5").expect("public model");
|
||||
let capabilities = ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation();
|
||||
let directory = Arc::new(RuntimeAccountDirectory::new(
|
||||
[
|
||||
"acct_failed",
|
||||
"acct_first",
|
||||
"acct_image",
|
||||
"acct_observation_mismatch",
|
||||
"acct_observed",
|
||||
"acct_one",
|
||||
"acct_only",
|
||||
"acct_other",
|
||||
"acct_required",
|
||||
"acct_second",
|
||||
"acct_tool",
|
||||
"acct_two",
|
||||
"acct_wrong",
|
||||
]
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
(
|
||||
ProviderAccountId::new(id).expect("account"),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
));
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("config revision"),
|
||||
account_selection_policy,
|
||||
vec![provider.clone()],
|
||||
vec![ProviderModel::new(
|
||||
provider.clone(),
|
||||
@@ -533,15 +603,21 @@ fn plan(operation: &Operation) -> RoutingPlan {
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.plan(
|
||||
&public_model,
|
||||
operation,
|
||||
&RoutingContext {
|
||||
provider_kind: Some(provider),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("routing plan")
|
||||
.with_account_directory(Arc::clone(&directory));
|
||||
snapshot
|
||||
.plan(
|
||||
&public_model,
|
||||
operation,
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
directory,
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)),
|
||||
&RoutingContext {
|
||||
required_provider: Some(provider),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("routing plan")
|
||||
}
|
||||
|
||||
fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest {
|
||||
@@ -551,11 +627,12 @@ fn model_request(operation: &Operation, deadline: SystemTime) -> NewModelRequest
|
||||
client_api_key_id: Some(client_key.clone()),
|
||||
client_api_key_ref: client_key,
|
||||
config_revision: ConfigRevision::new(1).expect("config revision"),
|
||||
routing: AccountRoutingSnapshot::all(),
|
||||
protocol: "openai".to_owned(),
|
||||
operation: operation.kind(),
|
||||
endpoint: "responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model: PublicModelId::new("gpt-5").expect("model"),
|
||||
requested_model: Some(PublicModelId::new("gpt-5").expect("model")),
|
||||
client_ip: Some("127.0.0.1".parse().expect("client IP")),
|
||||
user_agent: Some("gateway-core-test".to_owned()),
|
||||
reasoning_effort: Some("medium".to_owned()),
|
||||
@@ -1344,6 +1421,7 @@ fn authenticated_native_continuation_reaches_every_attempt_context() {
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1401,6 +1479,7 @@ fn streaming_native_continuation_delivers_started_event_before_later_output() {
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1436,7 +1515,7 @@ fn streaming_native_continuation_delivers_started_event_before_later_output() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
fn native_continuation_uses_provider_defined_owner_then_cross_account_recovery() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
@@ -1477,6 +1556,7 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
@@ -1490,7 +1570,7 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("portable replay succeeds");
|
||||
block_on(session.collect_uncommitted()).expect("provider-defined recovery succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
@@ -1514,6 +1594,141 @@ fn native_continuation_replays_owner_before_safely_switching_account() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_native_continuation_uses_provider_recovery_with_the_configured_account_policy() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let policy = AccountSelectionPolicy::new(
|
||||
RotationStrategy::RoundRobin,
|
||||
NonZeroU32::new(2).expect("account concurrency"),
|
||||
Duration::from_millis(50),
|
||||
);
|
||||
let route_plan = plan_with_policy(&operation, policy);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let original = ProviderAccountId::new("acct_one").expect("account");
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
original.clone(),
|
||||
);
|
||||
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
block_on(session.collect_uncommitted()).expect("provider-defined recovery succeeds");
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit response");
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].continuation_attempt(),
|
||||
ContinuationAttempt::ReplayAny
|
||||
);
|
||||
assert_eq!(
|
||||
contexts[1].account_selection_policy().strategy(),
|
||||
RotationStrategy::RoundRobin
|
||||
);
|
||||
assert!(contexts[1].excluded_accounts().contains(&original));
|
||||
let state = store.state.lock().expect("store lock");
|
||||
assert_eq!(state.attempts.len(), 1);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuation_history_failure_without_replay_proof_is_delivered_without_hidden_retry() {
|
||||
let Operation::Generate(generate) = generate_operation() else {
|
||||
panic!("generate operation");
|
||||
};
|
||||
let operation = Operation::Generate(generate.with_provider_session_state(
|
||||
ProviderSessionState::new("openai", Map::new()).expect("provider session state"),
|
||||
));
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
Script::ObservedStream {
|
||||
account_id: "acct_one",
|
||||
items: vec![Err(atomic_history_unavailable("response-history-missing"))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_two",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-response"),
|
||||
PreviousResponseId::new("upstream-response"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_one").expect("account"),
|
||||
);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
Some(ContinuationBinding::Pinned(continuation)),
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.expect("start execution");
|
||||
|
||||
let terminal = block_on(session.next_event())
|
||||
.expect("history failure remains deliverable")
|
||||
.expect("terminal history failure batch");
|
||||
assert_eq!(
|
||||
terminal.commit_requirement(),
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
);
|
||||
let events = terminal.into_provider_events();
|
||||
let failure = events
|
||||
.iter()
|
||||
.filter_map(ProviderEvent::wire_event)
|
||||
.find(|wire| wire.event_type() == Some("response.failed"))
|
||||
.expect("raw response.failed event");
|
||||
assert_eq!(
|
||||
failure
|
||||
.data()
|
||||
.pointer("/response/error/code")
|
||||
.and_then(Value::as_str),
|
||||
Some("previous_response_not_found")
|
||||
);
|
||||
block_on(session.commit_downstream(Some(200))).expect("commit terminal history failure");
|
||||
assert!(matches!(
|
||||
block_on(session.next_event()),
|
||||
Err(EngineError::Provider(_))
|
||||
));
|
||||
|
||||
let contexts = provider.contexts.lock().expect("contexts lock");
|
||||
assert_eq!(contexts.len(), 1);
|
||||
assert_eq!(
|
||||
contexts[0].continuation_attempt(),
|
||||
ContinuationAttempt::Native
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_account_reaches_provider_and_matching_metadata_succeeds() {
|
||||
let operation = operation(RetrySafety::Idempotent);
|
||||
@@ -2424,6 +2639,7 @@ fn native_continuation_explicit_429_is_not_retried() {
|
||||
let continuation = NativeContinuationPin::new(
|
||||
PreviousResponseId::new("previous-secret-id"),
|
||||
PreviousResponseId::new("provider-native-id"),
|
||||
ClientApiKeyId::new("key_client_1").expect("client key"),
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
ProviderAccountId::new("acct_first").expect("account"),
|
||||
);
|
||||
|
||||
@@ -1,30 +1,39 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use gateway_core::engine::credential::{
|
||||
AccountAttemptFeedback, AccountAvailability, AccountAvailabilityPolicy, AccountCandidate,
|
||||
AccountAttemptFeedback, AccountCandidate, AccountConcurrencyLimit, AccountEligibilityPolicy,
|
||||
AccountFeedbackStats, AccountQuotaSignals, AccountRuntimeSignals, AccountSchedulingBlocker,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, CredentialCasUpdate,
|
||||
CredentialRevision, OpaqueProviderData, PlaintextCredential, PreferredAccountSelection,
|
||||
ProviderAccount, ProviderAccountId, ProviderAccountIdentity, ProviderAccountUpdate,
|
||||
RotationStrategy,
|
||||
AccountSelectionContext, AccountSelectionPolicy, AccountSelector, AccountStatus, AccountWeight,
|
||||
CredentialCasUpdate, CredentialRevision, CredentialState, OpaqueProviderData,
|
||||
PlaintextCredential, PreferredAccountSelection, ProviderAccount, ProviderAccountId,
|
||||
ProviderAccountIdentity, ProviderAccountUpdate, QuotaEvidence, QuotaState, RotationStrategy,
|
||||
};
|
||||
use gateway_core::routing::{
|
||||
ClientRoutingScope, FrozenAccountScope, ProviderKind, RuntimeAccount, RuntimeAccountDirectory,
|
||||
};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
|
||||
fn account(id: &str) -> ProviderAccount {
|
||||
ProviderAccount::new(
|
||||
ProviderAccountId::new(id).expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
id.to_owned(),
|
||||
format!("user-{id}"),
|
||||
Some(format!("user-{id}")),
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_runtime_state(true, AccountAvailability::Ready)
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandidate {
|
||||
@@ -35,13 +44,50 @@ fn candidate(id: &str, in_flight: u32, remaining: Option<u64>) -> AccountCandida
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: remaining,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_with_concurrency(id: &str, in_flight: u32, concurrency: u32) -> AccountCandidate {
|
||||
let mut candidate = candidate(id, in_flight, Some(100));
|
||||
candidate.account = candidate.account.with_scheduling(
|
||||
Some(AccountConcurrencyLimit::new(concurrency).expect("concurrency override")),
|
||||
AccountWeight::DEFAULT,
|
||||
);
|
||||
candidate
|
||||
}
|
||||
|
||||
fn weighted_candidate(id: &str, weight: u16, in_flight: u32) -> AccountCandidate {
|
||||
let mut candidate = candidate(id, in_flight, None);
|
||||
candidate.account = candidate.account.with_scheduling(
|
||||
None,
|
||||
AccountWeight::new(weight).expect("valid account weight"),
|
||||
);
|
||||
candidate
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_scheduling_should_default_and_resolve_concurrency_override() {
|
||||
let default = NonZeroU32::new(12).expect("default concurrency");
|
||||
let account = account("acct_scheduling_default");
|
||||
assert_eq!(account.concurrency_limit(), None);
|
||||
assert_eq!(account.weight(), AccountWeight::DEFAULT);
|
||||
assert_eq!(account.effective_concurrency(default), default);
|
||||
|
||||
let limit = AccountConcurrencyLimit::new(7).expect("concurrency override");
|
||||
let weight = AccountWeight::new(80).expect("weight");
|
||||
let account = account.with_scheduling(Some(limit), weight);
|
||||
assert_eq!(account.concurrency_limit(), Some(limit));
|
||||
assert_eq!(account.weight(), weight);
|
||||
assert_eq!(account.effective_concurrency(default).get(), 7);
|
||||
assert!(AccountConcurrencyLimit::new(0).is_none());
|
||||
assert!(AccountWeight::new(0).is_none());
|
||||
assert!(AccountWeight::new(101).is_none());
|
||||
}
|
||||
|
||||
fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
AccountSelectionContext {
|
||||
policy: AccountSelectionPolicy::new(
|
||||
@@ -53,7 +99,8 @@ fn context(strategy: RotationStrategy) -> AccountSelectionContext {
|
||||
excluded_accounts: BTreeSet::new(),
|
||||
preferred_account: None,
|
||||
round_robin_cursor: 0,
|
||||
availability: AccountAvailabilityPolicy::Enforce,
|
||||
eligibility: AccountEligibilityPolicy::Enforce,
|
||||
account_scope: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,43 +148,99 @@ fn provider_account_identity_debug_should_redact_upstream_ids() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn availability_should_round_trip_all_database_values() {
|
||||
fn account_fact_values_should_round_trip() {
|
||||
assert_eq!(
|
||||
AccountAvailability::parse(AccountAvailability::QuotaExhausted.as_str()),
|
||||
Some(AccountAvailability::QuotaExhausted)
|
||||
CredentialState::parse(CredentialState::Banned.as_str()),
|
||||
Some(CredentialState::Banned)
|
||||
);
|
||||
assert_eq!(
|
||||
QuotaEvidence::parse("provider_denied"),
|
||||
Some(QuotaEvidence::ProviderDenied)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
fn exhausted_quota_refresh_is_due_at_reset_or_provider_fallback() {
|
||||
let now = SystemTime::now();
|
||||
let observed_at = now - Duration::from_secs(60);
|
||||
let before_reset = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
observed_at,
|
||||
Some(now + Duration::from_secs(1)),
|
||||
);
|
||||
let after_reset = QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
observed_at,
|
||||
Some(now - Duration::from_secs(1)),
|
||||
);
|
||||
let without_reset = QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None);
|
||||
|
||||
assert!(!before_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(after_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(without_reset.exhaustion_refresh_due(now, Duration::from_secs(30)));
|
||||
assert!(!without_reset.exhaustion_refresh_due(now, Duration::from_secs(120)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inconclusive_quota_observation_cannot_erase_confirmed_access() {
|
||||
let now = SystemTime::now();
|
||||
let exhausted = QuotaState::exhausted(QuotaEvidence::ProviderDenied, now, None);
|
||||
|
||||
assert_eq!(
|
||||
exhausted
|
||||
.merge_observation(QuotaState::observed_unknown(now + Duration::from_secs(1)))
|
||||
.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Exhausted
|
||||
);
|
||||
assert_eq!(
|
||||
exhausted
|
||||
.merge_observation(QuotaState::allowed(now + Duration::from_secs(1)))
|
||||
.access(),
|
||||
gateway_core::engine::credential::QuotaAccessState::Allowed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostic_selection_bypasses_all_local_account_eligibility() {
|
||||
let observed_at = SystemTime::now();
|
||||
let exhausted = AccountCandidate {
|
||||
account: account("acct_exhausted")
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted),
|
||||
account: account("acct_exhausted").with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let disabled = AccountCandidate {
|
||||
account: account("acct_disabled")
|
||||
.with_runtime_state(false, AccountAvailability::QuotaExhausted),
|
||||
account: account("acct_disabled").with_account_facts(
|
||||
false,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, observed_at, None),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
signals: AccountRuntimeSignals {
|
||||
in_flight: 0,
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
},
|
||||
};
|
||||
let mut context = context(RotationStrategy::Sticky);
|
||||
context.availability = AccountAvailabilityPolicy::BypassForDiagnostic;
|
||||
context.eligibility = AccountEligibilityPolicy::BypassForDiagnostic;
|
||||
|
||||
assert_eq!(
|
||||
AccountSelector
|
||||
@@ -154,10 +257,91 @@ fn diagnostic_selection_bypasses_all_local_account_availability() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account = account("acct_disabled").with_runtime_state(false, AccountAvailability::Ready);
|
||||
fn selector_should_reject_a_preferred_account_outside_the_frozen_client_scope() {
|
||||
let allowed = candidate("acct_allowed", 0, None);
|
||||
let outside = candidate("acct_outside", 0, None);
|
||||
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
|
||||
allowed.account.id().clone(),
|
||||
RuntimeAccount::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
BTreeSet::new(),
|
||||
),
|
||||
)])));
|
||||
let mut context = context(RotationStrategy::Sticky);
|
||||
context.preferred_account = Some(outside.account.id().clone());
|
||||
context.account_scope = Some(Arc::new(FrozenAccountScope::new(
|
||||
directory,
|
||||
ClientRoutingScope::all_accounts(),
|
||||
)));
|
||||
|
||||
assert!(!account.is_schedulable(SystemTime::now()));
|
||||
let candidates = [outside, allowed];
|
||||
let selection = AccountSelector
|
||||
.select(&candidates, &context)
|
||||
.expect("allowed account should remain eligible");
|
||||
assert_eq!(selection.candidate().account.id().as_str(), "acct_allowed");
|
||||
assert_eq!(
|
||||
selection.preferred(),
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::OutsideClientScope)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_account_should_not_be_schedulable() {
|
||||
let account = account("acct_disabled").with_account_facts(
|
||||
false,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
account.status_projection(SystemTime::now(), None).status,
|
||||
AccountStatus::Disabled
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rate_limited_projection_carries_only_the_active_cooldown_deadline() {
|
||||
let now = SystemTime::now();
|
||||
let active_until = now + Duration::from_secs(60);
|
||||
let account = account("acct_rate_limited");
|
||||
|
||||
let active = account.status_projection(now, Some(active_until));
|
||||
assert_eq!(active.status, AccountStatus::RateLimited);
|
||||
assert_eq!(active.rate_limited_until, Some(active_until));
|
||||
|
||||
let elapsed = account.status_projection(now, Some(now - Duration::from_secs(1)));
|
||||
assert_eq!(elapsed.status, AccountStatus::Normal);
|
||||
assert_eq!(elapsed.rate_limited_until, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_without_upstream_identity_should_stay_unknown_and_not_schedulable() {
|
||||
let account = ProviderAccount::new(
|
||||
ProviderAccountId::new("acct_pending_identity").expect("valid account"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
"pending identity".to_owned(),
|
||||
None,
|
||||
"oauth".to_owned(),
|
||||
CredentialRevision::new(1).expect("valid revision"),
|
||||
Some(SystemTime::now() + Duration::from_secs(3600)),
|
||||
)
|
||||
.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
account.credential_state(),
|
||||
account.status_projection(SystemTime::now(), None).status
|
||||
),
|
||||
(CredentialState::Unknown, AccountStatus::Error)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -173,6 +357,84 @@ fn smart_selector_should_prefer_lower_inflight_count() {
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_idle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_prefer_lower_capacity_utilization_over_lower_inflight_count() {
|
||||
let candidates = [
|
||||
candidate_with_concurrency("acct_smaller", 1, 2),
|
||||
candidate_with_concurrency("acct_larger", 2, 10),
|
||||
];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_larger");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_should_prioritize_the_highest_weight_for_every_rotation_strategy() {
|
||||
let candidates = vec![
|
||||
weighted_candidate("acct_low_weight", 1, 0),
|
||||
weighted_candidate("acct_high_weight", 100, 2),
|
||||
];
|
||||
|
||||
for strategy in [
|
||||
RotationStrategy::Smart,
|
||||
RotationStrategy::QuotaResetPriority,
|
||||
RotationStrategy::RoundRobin,
|
||||
RotationStrategy::Sticky,
|
||||
] {
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(strategy))
|
||||
.expect("highest weight account should be eligible");
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_high_weight",
|
||||
"strategy {strategy:?} must select inside the highest weight tier"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_should_not_let_soft_affinity_bypass_a_higher_weight_account() {
|
||||
let preferred = weighted_candidate("acct_preferred", 1, 0);
|
||||
let preferred_id = preferred.account.id().clone();
|
||||
let higher = weighted_candidate("acct_higher", 10, 0);
|
||||
let mut selection = context(RotationStrategy::Smart);
|
||||
selection.preferred_account = Some(preferred_id);
|
||||
let candidates = [preferred, higher];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &selection)
|
||||
.expect("higher weight account should be eligible");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_higher");
|
||||
assert_eq!(
|
||||
selected.preferred(),
|
||||
PreferredAccountSelection::Blocked(AccountSchedulingBlocker::LowerWeight)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_should_fall_back_when_a_higher_weight_account_reaches_its_override() {
|
||||
let mut saturated = weighted_candidate("acct_high_weight", 100, 1);
|
||||
saturated.account = saturated.account.with_scheduling(
|
||||
Some(AccountConcurrencyLimit::new(1).expect("concurrency override")),
|
||||
AccountWeight::new(100).expect("weight"),
|
||||
);
|
||||
let fallback = weighted_candidate("acct_low_weight", 1, 0);
|
||||
let candidates = [saturated, fallback];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("lower weight account should be used after saturation");
|
||||
|
||||
assert_eq!(
|
||||
selected.candidate().account.id().as_str(),
|
||||
"acct_low_weight"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_balance_signals_instead_of_using_lexicographic_load() {
|
||||
let mut healthy = candidate("acct_healthy", 1, Some(100));
|
||||
@@ -286,15 +548,11 @@ fn provider_quota_overlay_should_preserve_store_concurrency_facts() {
|
||||
last_started_at: Some(last_started_at),
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: None,
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: None,
|
||||
first_output_latency_ms: None,
|
||||
}
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(
|
||||
Some(reset_at),
|
||||
Some(75),
|
||||
false,
|
||||
)));
|
||||
.with_provider_quota(Some(AccountQuotaSignals::new(Some(reset_at), Some(75))));
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
@@ -387,6 +645,22 @@ fn quota_reset_selector_should_prefer_known_earliest_window() {
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_earlier");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_reset_selector_should_use_capacity_utilization_as_load_tiebreaker() {
|
||||
let reset_at = SystemTime::now() + Duration::from_secs(60);
|
||||
let mut smaller = candidate_with_concurrency("acct_smaller", 1, 2);
|
||||
smaller.signals.quota_reset_at = Some(reset_at);
|
||||
let mut larger = candidate_with_concurrency("acct_larger", 2, 10);
|
||||
larger.signals.quota_reset_at = Some(reset_at);
|
||||
let candidates = [smaller, larger];
|
||||
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::QuotaResetPriority))
|
||||
.expect("candidate available");
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_larger");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smart_selector_should_prefer_known_quota_over_unknown_after_load_is_equal() {
|
||||
let candidates = vec![
|
||||
@@ -611,21 +885,31 @@ fn credential_cas_should_reject_refresh_schedule_without_refresh_token() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
// availability 终态(Banned/Expired/Invalid)始终排除,
|
||||
// 不受 quota 信号或 skip_exhausted 放宽影响。
|
||||
fn enforced_eligibility_excludes_credential_errors_despite_quota_signals() {
|
||||
let mut banned = candidate("acct_banned", 0, Some(100));
|
||||
banned.account = banned
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Banned);
|
||||
banned.account = banned.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Banned,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let mut expired = candidate("acct_expired", 0, Some(100));
|
||||
expired.account = expired
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Expired);
|
||||
expired.account = expired.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Expired,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let mut invalid = candidate("acct_invalid", 0, Some(100));
|
||||
invalid.account = invalid
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::Invalid);
|
||||
invalid.account = invalid.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Invalid,
|
||||
QuotaState::unknown(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![banned, expired, invalid, healthy];
|
||||
|
||||
@@ -637,14 +921,15 @@ fn enforced_availability_excludes_terminal_states_despite_quota_signals() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection() {
|
||||
// QuotaExhausted 是可恢复额度耗尽,skip_exhausted=false 的
|
||||
// 放宽投影(provider 层)允许其参与调度;core 的 Enforce 策略本身
|
||||
// 只排除终态。这里验证 Enforce 下 QuotaExhausted 仍被排除(严格模式)。
|
||||
fn enforced_eligibility_excludes_authoritative_quota_exhaustion() {
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted
|
||||
.account
|
||||
.with_runtime_state(true, AccountAvailability::QuotaExhausted);
|
||||
exhausted.account = exhausted.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
let candidates = vec![exhausted, healthy];
|
||||
|
||||
@@ -654,3 +939,31 @@ fn enforced_availability_allows_quota_exhausted_with_skip_exhausted_projection()
|
||||
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elapsed_quota_reset_does_not_fabricate_recovery() {
|
||||
let now = SystemTime::now();
|
||||
let mut exhausted = candidate("acct_exhausted", 0, Some(100));
|
||||
exhausted.account = exhausted.account.with_account_facts(
|
||||
true,
|
||||
CredentialState::Ready,
|
||||
QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
now - Duration::from_secs(2),
|
||||
Some(now - Duration::from_secs(1)),
|
||||
),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let healthy = candidate("acct_healthy", 0, Some(100));
|
||||
|
||||
assert_eq!(
|
||||
exhausted.account.status_projection(now, None).status,
|
||||
AccountStatus::QuotaExhausted
|
||||
);
|
||||
let candidates = [exhausted, healthy];
|
||||
let selected = AccountSelector
|
||||
.select(&candidates, &context(RotationStrategy::Smart))
|
||||
.expect("healthy candidate available");
|
||||
assert_eq!(selected.candidate().account.id().as_str(), "acct_healthy");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
@@ -18,17 +18,17 @@ use gateway_core::engine::continuation::{
|
||||
NativeContinuationPin, NativeContinuationPort, NativeContinuationStoreError, PreviousResponseId,
|
||||
};
|
||||
use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
|
||||
AccountSelectionPolicy, CredentialRevision, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::engine::execution::{
|
||||
ClientApiKeyUsageSink, ClientTransport, DefaultExecutionService, ExecutionRequestMetadata,
|
||||
ExecutionService, ProviderCircuitDecision, ProviderCircuitError, ProviderCircuitPort,
|
||||
StartExecution, provider_failure_affects_circuit,
|
||||
StartExecution, StartProviderExecution, provider_failure_affects_circuit,
|
||||
};
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCatalogGeneration, ProviderModelCapabilities, ProviderRegistry,
|
||||
ProviderRequest, ProviderStream,
|
||||
Provider, ProviderCallMetadata, ProviderCatalogGeneration, ProviderModelCapabilities,
|
||||
ProviderRegistry, ProviderRequest, ProviderResource, ProviderStream, UpstreamTransport,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
@@ -38,11 +38,15 @@ use gateway_core::error::{
|
||||
ClientVisibleUpstreamResponse, GatewayErrorKind, ProviderError, ProviderErrorKind, StoreError,
|
||||
StoreErrorKind,
|
||||
};
|
||||
use gateway_core::operation::{GenerateRequest, Operation, OperationKind, ProtocolPayload};
|
||||
use gateway_core::operation::{
|
||||
GenerateRequest, ImageRequest, ImageRequestKind, Operation, OperationKind, ProtocolPayload,
|
||||
RawJsonPayload,
|
||||
};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::RuntimeSnapshotHandle;
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RuntimeSnapshot,
|
||||
ClientRoutingScope, ConfigRevision, FrozenAccountScope, ModelCapabilities, ProviderKind,
|
||||
ProviderModel, PublicModelId, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
use serde_json::json;
|
||||
@@ -192,6 +196,50 @@ impl Provider for FailingProvider {
|
||||
}
|
||||
}
|
||||
|
||||
struct ColdFailingProvider;
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for ColdFailingProvider {
|
||||
fn name(&self) -> &'static str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn catalog_generation(&self) -> ProviderCatalogGeneration {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
request: ProviderRequest,
|
||||
_: AttemptContext,
|
||||
) -> Result<ProviderStream, ProviderError> {
|
||||
let metadata = ProviderCallMetadata::for_provider_endpoint(
|
||||
request.candidate().provider().clone(),
|
||||
ProviderResource::Account {
|
||||
id: ProviderAccountId::new("acct_usage").expect("account"),
|
||||
revision: CredentialRevision::new(1).expect("revision"),
|
||||
},
|
||||
UpstreamTransport::new("http_json").expect("transport"),
|
||||
);
|
||||
Ok(ProviderStream::new(
|
||||
metadata,
|
||||
futures::stream::once(async {
|
||||
Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
))
|
||||
}),
|
||||
(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_authentication_should_record_client_key_usage() {
|
||||
let usage = Arc::new(RecordingClientApiKeyUsage::default());
|
||||
@@ -212,6 +260,60 @@ fn successful_authentication_should_record_client_key_usage() {
|
||||
assert_eq!(usage.recorded(), vec!["key_usage_test".to_owned()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_endpoint_should_persist_its_real_v1_endpoint() {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(client_snapshot()),
|
||||
store.clone(),
|
||||
ProviderRegistry::new([Arc::new(ColdFailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry"),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_usage_test")
|
||||
.expect("authenticated client");
|
||||
let operation = Operation::GenerateImage(ImageRequest::from_raw_json(
|
||||
ImageRequestKind::Generation,
|
||||
RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from_static(br#"{"model":"gpt-image-2","prompt":"hello"}"#),
|
||||
)
|
||||
.expect("image payload"),
|
||||
));
|
||||
|
||||
let mut started = block_on(service.start_provider_endpoint(StartProviderExecution {
|
||||
client,
|
||||
provider: ProviderKind::new("openai").expect("provider"),
|
||||
operation,
|
||||
metadata: ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/images/generations".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("provider endpoint request should start without a text catalog entry");
|
||||
|
||||
let error = block_on(started.session.collect_uncommitted())
|
||||
.expect_err("the cold provider stops execution after persistence");
|
||||
assert_eq!(
|
||||
store.model_request_endpoints(),
|
||||
vec!["/v1/images/generations".to_owned()],
|
||||
"execution error: {error:?}"
|
||||
);
|
||||
assert_eq!(store.requested_models(), vec![None]);
|
||||
let upstream_models = store.upstream_models();
|
||||
assert!(!upstream_models.is_empty());
|
||||
assert!(upstream_models.iter().all(Option::is_none));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn circuit_store_failure_should_fail_open_during_request_start() {
|
||||
let service = DefaultExecutionService::new(
|
||||
@@ -283,7 +385,7 @@ fn slow_circuit_store_should_time_out_and_fail_open_during_request_start() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
|
||||
fn known_catalog_should_reject_a_model_that_the_provider_did_not_publish() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
@@ -298,7 +400,7 @@ fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
|
||||
.expect("authenticated client");
|
||||
let model = "gpt-future-not-in-catalog";
|
||||
|
||||
let started = block_on(service.start(StartExecution {
|
||||
let result = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::from_client_wire(model).expect("client model"),
|
||||
operation: start_operation_for_model(model),
|
||||
@@ -311,10 +413,80 @@ fn stale_nonempty_catalog_should_not_block_a_new_model_during_request_start() {
|
||||
user_agent: None,
|
||||
previous_response_id: None,
|
||||
},
|
||||
}))
|
||||
.expect("stale model catalog must not reject the request");
|
||||
}));
|
||||
let Err(error) = result else {
|
||||
panic!("a known provider catalog must be authoritative for model availability");
|
||||
};
|
||||
|
||||
assert!(!started.session.is_finalized());
|
||||
assert_eq!(error.kind(), GatewayErrorKind::NoAvailableProvider);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuation_owned_by_another_client_api_key_should_fail_closed() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(RejectedContinuation::OwnershipMismatch),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
|
||||
let result = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: execution_metadata_with_continuation(),
|
||||
}));
|
||||
let Err(error) = result else {
|
||||
panic!("cross-client continuation must be rejected");
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::PolicyDenied);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_continuation_record_should_not_be_forwarded_as_an_external_handle() {
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(start_snapshot()),
|
||||
Arc::new(TrackingExecutionStore::default()),
|
||||
ProviderRegistry::default(),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(RejectedContinuation::InvalidData),
|
||||
Arc::new(RecordingClientApiKeyUsage::default()),
|
||||
);
|
||||
let client = service
|
||||
.authenticate("sk_start_test")
|
||||
.expect("authenticated client");
|
||||
|
||||
let result = block_on(service.start(StartExecution {
|
||||
client,
|
||||
public_model: PublicModelId::new("gpt-start").expect("public model"),
|
||||
operation: start_operation(),
|
||||
metadata: execution_metadata_with_continuation(),
|
||||
}));
|
||||
let Err(error) = result else {
|
||||
panic!("invalid continuation state must be rejected");
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), GatewayErrorKind::Internal);
|
||||
}
|
||||
|
||||
fn execution_metadata_with_continuation() -> ExecutionRequestMetadata {
|
||||
ExecutionRequestMetadata {
|
||||
protocol: "openai".to_owned(),
|
||||
endpoint: "/v1/responses".to_owned(),
|
||||
transport: ClientTransport::HttpJson,
|
||||
stream: false,
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
previous_response_id: Some(PreviousResponseId::new("response-private")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -341,6 +513,9 @@ impl ClientApiKeyUsageSink for RecordingClientApiKeyUsage {
|
||||
struct TrackingExecutionStore {
|
||||
touched: AtomicBool,
|
||||
probe_failures: Mutex<Vec<String>>,
|
||||
model_request_endpoints: Mutex<Vec<String>>,
|
||||
requested_models: Mutex<Vec<Option<String>>>,
|
||||
upstream_models: Mutex<Vec<Option<String>>>,
|
||||
fail_probe_observation: AtomicBool,
|
||||
}
|
||||
|
||||
@@ -355,17 +530,60 @@ impl TrackingExecutionStore {
|
||||
.expect("probe failures lock")
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn model_request_endpoints(&self) -> Vec<String> {
|
||||
self.model_request_endpoints
|
||||
.lock()
|
||||
.expect("model request endpoints lock")
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn requested_models(&self) -> Vec<Option<String>> {
|
||||
self.requested_models
|
||||
.lock()
|
||||
.expect("requested models lock")
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn upstream_models(&self) -> Vec<Option<String>> {
|
||||
self.upstream_models
|
||||
.lock()
|
||||
.expect("upstream models lock")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ExecutionStore for TrackingExecutionStore {
|
||||
async fn create_model_request(&self, _: NewModelRequest) -> Result<(), StoreError> {
|
||||
async fn create_model_request(&self, request: NewModelRequest) -> Result<(), StoreError> {
|
||||
self.touch();
|
||||
self.requested_models
|
||||
.lock()
|
||||
.expect("requested models lock")
|
||||
.push(
|
||||
request
|
||||
.requested_model
|
||||
.as_ref()
|
||||
.map(|model| model.as_str().to_owned()),
|
||||
);
|
||||
self.model_request_endpoints
|
||||
.lock()
|
||||
.expect("model request endpoints lock")
|
||||
.push(request.endpoint);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_attempt(&self, _: AttemptRecord) -> Result<(), StoreError> {
|
||||
async fn record_attempt(&self, attempt: AttemptRecord) -> Result<(), StoreError> {
|
||||
self.touch();
|
||||
self.upstream_models
|
||||
.lock()
|
||||
.expect("upstream models lock")
|
||||
.push(
|
||||
attempt
|
||||
.upstream_model_id
|
||||
.as_ref()
|
||||
.map(|model| model.as_str().to_owned()),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -526,6 +744,7 @@ struct UnusedContinuation;
|
||||
impl NativeContinuationPort for UnusedContinuation {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientApiKeyId,
|
||||
_: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
Box::pin(async { Ok(None) })
|
||||
@@ -539,6 +758,43 @@ impl NativeContinuationPort for UnusedContinuation {
|
||||
}
|
||||
}
|
||||
|
||||
enum RejectedContinuation {
|
||||
OwnershipMismatch,
|
||||
InvalidData,
|
||||
}
|
||||
|
||||
impl NativeContinuationPort for RejectedContinuation {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
_: &'a ClientApiKeyId,
|
||||
_: &'a PreviousResponseId,
|
||||
) -> BoxFuture<'a, Result<Option<NativeContinuationPin>, NativeContinuationStoreError>> {
|
||||
Box::pin(async move {
|
||||
Err(match self {
|
||||
Self::OwnershipMismatch => NativeContinuationStoreError::ownership_mismatch(),
|
||||
Self::InvalidData => NativeContinuationStoreError::invalid_data("invalid record"),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn record<'a>(
|
||||
&'a self,
|
||||
_: NativeContinuationPin,
|
||||
) -> BoxFuture<'a, Result<(), NativeContinuationStoreError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
}
|
||||
|
||||
fn account_scope(provider: &ProviderKind, account_id: &str) -> Arc<FrozenAccountScope> {
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
|
||||
ProviderAccountId::new(account_id).expect("account ID"),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::new()),
|
||||
)]))),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
))
|
||||
}
|
||||
|
||||
fn probe_snapshot() -> RuntimeSnapshot {
|
||||
let provider = ProviderKind::new("openai").expect("provider kind");
|
||||
let capabilities =
|
||||
@@ -575,7 +831,7 @@ fn client_snapshot() -> RuntimeSnapshot {
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_usage_test").expect("client API key ID"),
|
||||
PlaintextClientApiKey::new("sk_usage_test").expect("plaintext client API key"),
|
||||
provider,
|
||||
account_scope(&provider, "acct_usage"),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
@@ -603,7 +859,7 @@ fn start_snapshot() -> RuntimeSnapshot {
|
||||
vec![ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_start_test").expect("client API key ID"),
|
||||
PlaintextClientApiKey::new("sk_start_test").expect("plaintext client API key"),
|
||||
provider,
|
||||
account_scope(&provider, "acct_start"),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
|
||||
@@ -77,6 +77,17 @@ fn protocol_wire_event_should_preserve_opaque_sse_metadata() {
|
||||
assert_eq!(wire.sse_id(), Some(sse_id.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_wire_event_should_preserve_raw_json_body_without_parsing_it() {
|
||||
let raw = Bytes::from_static(
|
||||
br#"{ "created": 1, "data": [{"b64_json":"AAAA"}], "future": 9007199254740993 }"#,
|
||||
);
|
||||
let wire = ProtocolWireEvent::raw_json("openai", raw.clone()).expect("raw JSON event");
|
||||
|
||||
assert_eq!(wire.raw_json_body(), Some(&raw));
|
||||
assert_eq!(wire.into_raw_json_body(), Some(raw));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validator_should_accept_started_content_delta_completed_sequence() {
|
||||
let mut validator = EventSequenceValidator::new();
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use bytes::Bytes;
|
||||
use gateway_core::error::OperationError;
|
||||
use gateway_core::operation::{
|
||||
EmbedRequest, Feature, GenerateRequest, ImageRequest, Operation, OperationKind,
|
||||
ProtocolPayload, ProviderSessionState, RerankRequest, RetrySafety, SpeechRequest,
|
||||
EmbedRequest, Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
|
||||
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, RerankRequest,
|
||||
RetrySafety, SpeechRequest,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
@@ -12,6 +14,17 @@ fn generate(body: Value) -> GenerateRequest {
|
||||
)
|
||||
}
|
||||
|
||||
fn image(kind: ImageRequestKind, body: Value) -> ImageRequest {
|
||||
ImageRequest::from_raw_json(
|
||||
kind,
|
||||
RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from(serde_json::to_vec(&body).expect("image JSON")),
|
||||
)
|
||||
.expect("OpenAI payload"),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generate_request_should_keep_client_body_opaque_and_redacted() {
|
||||
let secret = "private prompt body";
|
||||
@@ -113,10 +126,6 @@ fn non_generate_requests_should_reject_missing_required_fields() {
|
||||
RerankRequest::new("", vec!["document".to_owned()]).expect_err("query is required"),
|
||||
OperationError::EmptyField { field: "query" }
|
||||
);
|
||||
assert_eq!(
|
||||
ImageRequest::new("").expect_err("prompt is required"),
|
||||
OperationError::EmptyField { field: "prompt" }
|
||||
);
|
||||
assert_eq!(
|
||||
SpeechRequest::new("hello", "").expect_err("voice is required"),
|
||||
OperationError::EmptyField { field: "voice" }
|
||||
@@ -132,7 +141,10 @@ fn operation_kind_and_retry_safety_should_remain_stable() {
|
||||
let rerank = Operation::Rerank(
|
||||
RerankRequest::new("query", vec!["document".to_owned()]).expect("rerank request is valid"),
|
||||
);
|
||||
let image = Operation::GenerateImage(ImageRequest::new("draw").expect("image request"));
|
||||
let image = Operation::GenerateImage(image(
|
||||
ImageRequestKind::Generation,
|
||||
json!({"model": "gpt-image-2", "prompt": "draw"}),
|
||||
));
|
||||
let speech = Operation::Speech(SpeechRequest::new("hello", "alloy").expect("speech request"));
|
||||
|
||||
assert_eq!(generate.kind(), OperationKind::Generate);
|
||||
@@ -140,5 +152,35 @@ fn operation_kind_and_retry_safety_should_remain_stable() {
|
||||
assert_eq!(embed.retry_safety(), RetrySafety::Idempotent);
|
||||
assert_eq!(rerank.retry_safety(), RetrySafety::Idempotent);
|
||||
assert_eq!(image.kind(), OperationKind::GenerateImage);
|
||||
assert!(image.image_generation_requested());
|
||||
assert_eq!(speech.kind(), OperationKind::Speech);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_request_should_preserve_generation_and_edit_payloads_opaque() {
|
||||
let generation_body = json!({
|
||||
"model": "gpt-image-2",
|
||||
"prompt": "draw",
|
||||
"future_official_field": {"keep": true},
|
||||
});
|
||||
let edit_body = json!({
|
||||
"model": "gpt-image-2",
|
||||
"images": [{"image_url": "data:image/png;base64,AAAA"}],
|
||||
"prompt": "edit",
|
||||
"future_official_field": [1, 2, 3],
|
||||
});
|
||||
let generation = image(ImageRequestKind::Generation, generation_body.clone());
|
||||
let edit = image(ImageRequestKind::Edit, edit_body.clone());
|
||||
|
||||
assert_eq!(generation.kind(), ImageRequestKind::Generation);
|
||||
assert_eq!(edit.kind(), ImageRequestKind::Edit);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<Value>(generation.payload().body()).expect("generation JSON"),
|
||||
generation_body
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<Value>(edit.payload().body()).expect("edit JSON"),
|
||||
edit_body
|
||||
);
|
||||
assert!(!format!("{edit:?}").contains("data:image/png"));
|
||||
}
|
||||
|
||||
@@ -1,16 +1,25 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::ProviderKind;
|
||||
use gateway_core::routing::{ClientRoutingScope, FrozenAccountScope, RuntimeAccountDirectory};
|
||||
|
||||
fn plaintext(value: &str) -> PlaintextClientApiKey {
|
||||
PlaintextClientApiKey::new(value).expect("valid plaintext client key")
|
||||
}
|
||||
|
||||
fn account_scope() -> Arc<FrozenAccountScope> {
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
Arc::new(RuntimeAccountDirectory::default()),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_client_key_should_be_denied() {
|
||||
let policy = ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_disabled").expect("valid key ID"),
|
||||
plaintext("sk_disabled_secret"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
account_scope(),
|
||||
false,
|
||||
RateLimits::unlimited(),
|
||||
);
|
||||
@@ -23,7 +32,7 @@ fn enabled_client_key_should_be_authorized() {
|
||||
let policy = ClientPolicy::new(
|
||||
ClientApiKeyId::new("key_enabled").expect("valid key ID"),
|
||||
plaintext("sk_enabled_secret"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
account_scope(),
|
||||
true,
|
||||
RateLimits::unlimited(),
|
||||
);
|
||||
|
||||
@@ -57,8 +57,7 @@ fn refresh_policy_requires_a_positive_margin() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let account = ProviderAccountId::new("acct_stable").expect("valid account");
|
||||
fn refresh_policy_should_mark_tokens_due_at_the_exact_configured_margin() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
NonZeroU32::new(2).expect("positive concurrency"),
|
||||
@@ -67,15 +66,12 @@ fn refresh_schedule_should_use_the_exact_configured_margin() {
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
let expires_at = observed_at + Duration::from_secs(7_200);
|
||||
|
||||
let first = policy
|
||||
.next_attempt_at(&account, expires_at, observed_at)
|
||||
.expect("schedule refresh");
|
||||
assert_eq!(first, expires_at - Duration::from_secs(3_600));
|
||||
assert!(!policy.is_refresh_due(expires_at, observed_at));
|
||||
assert!(policy.is_refresh_due(observed_at + Duration::from_secs(3_600), observed_at));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
|
||||
let account = ProviderAccountId::new("acct_boundary").expect("valid account");
|
||||
fn refresh_policy_should_mark_expired_tokens_due() {
|
||||
let policy = ProviderRefreshPolicy::try_new(
|
||||
Duration::from_secs(3_600),
|
||||
NonZeroU32::new(1).expect("positive concurrency"),
|
||||
@@ -83,12 +79,7 @@ fn refresh_schedule_uses_observed_time_when_token_is_inside_safe_window() {
|
||||
.expect("valid policy");
|
||||
let observed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(10_000);
|
||||
|
||||
assert_eq!(
|
||||
policy
|
||||
.next_attempt_at(&account, observed_at + Duration::from_secs(1), observed_at,)
|
||||
.expect("schedule immediate refresh"),
|
||||
observed_at
|
||||
);
|
||||
assert!(policy.is_refresh_due(observed_at - Duration::from_secs(1), observed_at));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -101,7 +92,7 @@ fn scheduling_state_preserves_provider_neutral_signals() {
|
||||
last_started_at: None,
|
||||
quota_reset_at: None,
|
||||
quota_remaining_rank: Some(7),
|
||||
quota_limit_reached: false,
|
||||
rate_limited_until: None,
|
||||
failure_rate_basis_points: Some(125),
|
||||
first_output_latency_ms: Some(250),
|
||||
},
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use gateway_core::engine::credential::{AccountSelectionPolicy, RotationStrategy};
|
||||
use bytes::Bytes;
|
||||
use gateway_core::engine::credential::{
|
||||
AccountSelectionPolicy, ProviderAccountId, RotationStrategy,
|
||||
};
|
||||
use gateway_core::operation::{
|
||||
CapabilityRequirements, Feature, GenerateRequest, Operation, OperationKind, ProtocolPayload,
|
||||
CapabilityRequirements, Feature, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
|
||||
OperationKind, ProtocolPayload, RawJsonPayload,
|
||||
};
|
||||
use gateway_core::policy::{ClientApiKeyId, ClientPolicy, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::{
|
||||
ConfigRevision, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId, RoutingContext,
|
||||
RuntimeSnapshot, UpstreamModelId,
|
||||
AccountGroupId, AccountRoutingScopeKind, ClientRoutingScope, ConfigRevision,
|
||||
FrozenAccountScope, ModelCapabilities, ProviderKind, ProviderModel, PublicModelId,
|
||||
RoutingContext, RoutingGroupSnapshot, RuntimeAccount, RuntimeAccountDirectory, RuntimeSnapshot,
|
||||
UpstreamModelId,
|
||||
};
|
||||
|
||||
mod snapshot;
|
||||
@@ -45,11 +52,49 @@ fn operation() -> Operation {
|
||||
))
|
||||
}
|
||||
|
||||
fn image_operation() -> Operation {
|
||||
let body = serde_json::json!({
|
||||
"model": "gpt-image-2",
|
||||
"prompt": "draw",
|
||||
});
|
||||
Operation::GenerateImage(ImageRequest::from_raw_json(
|
||||
ImageRequestKind::Generation,
|
||||
RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from(serde_json::to_vec(&body).expect("image JSON")),
|
||||
)
|
||||
.expect("OpenAI payload"),
|
||||
))
|
||||
}
|
||||
|
||||
fn account_directory() -> Arc<RuntimeAccountDirectory> {
|
||||
Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([
|
||||
(
|
||||
ProviderAccountId::new("acct_openai").expect("account"),
|
||||
RuntimeAccount::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
BTreeSet::new(),
|
||||
),
|
||||
),
|
||||
(
|
||||
ProviderAccountId::new("acct_xai").expect("account"),
|
||||
RuntimeAccount::new(ProviderKind::new("xai").expect("provider"), BTreeSet::new()),
|
||||
),
|
||||
])))
|
||||
}
|
||||
|
||||
fn account_scope() -> Arc<FrozenAccountScope> {
|
||||
Arc::new(FrozenAccountScope::new(
|
||||
account_directory(),
|
||||
ClientRoutingScope::all_accounts(),
|
||||
))
|
||||
}
|
||||
|
||||
fn client_policy(id: &str, plaintext: &str, enabled: bool) -> ClientPolicy {
|
||||
ClientPolicy::new(
|
||||
ClientApiKeyId::new(id).expect("client key ID"),
|
||||
PlaintextClientApiKey::new(plaintext).expect("plaintext client key"),
|
||||
ProviderKind::new("openai").expect("valid provider"),
|
||||
account_scope(),
|
||||
enabled,
|
||||
RateLimits::unlimited(),
|
||||
)
|
||||
@@ -70,6 +115,7 @@ fn snapshot() -> RuntimeSnapshot {
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.with_account_directory(account_directory())
|
||||
.with_model_mappings(BTreeMap::from([
|
||||
("gpt-5.4".to_owned(), "gpt-5.5".to_owned()),
|
||||
("grok-latest".to_owned(), "grok-4.5".to_owned()),
|
||||
@@ -81,6 +127,92 @@ fn config_revision_should_reject_zero() {
|
||||
assert!(ConfigRevision::new(0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_group_id_should_match_the_database_contract_exactly() {
|
||||
let valid = "grp_0123456789abcdef0123456789abcdef";
|
||||
assert_eq!(
|
||||
AccountGroupId::new(valid).expect("valid group ID").as_str(),
|
||||
valid
|
||||
);
|
||||
for invalid in [
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
"grp_0123456789abcdef0123456789abcde",
|
||||
"grp_0123456789abcdef0123456789abcdef0",
|
||||
"grp_0123456789abcdef0123456789abcdeF",
|
||||
"grp_0123456789abcdef0123456789abcdeg",
|
||||
] {
|
||||
assert!(
|
||||
AccountGroupId::new(invalid).is_err(),
|
||||
"invalid group ID was accepted: {invalid}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricted_scope_should_use_enabled_group_union_and_preserve_all_bindings_for_history() {
|
||||
let enabled =
|
||||
AccountGroupId::new("grp_11111111111111111111111111111111").expect("enabled group");
|
||||
let disabled =
|
||||
AccountGroupId::new("grp_22222222222222222222222222222222").expect("disabled group");
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let allowed_account = ProviderAccountId::new("acct_allowed").expect("account");
|
||||
let disabled_only_account = ProviderAccountId::new("acct_disabled").expect("account");
|
||||
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([
|
||||
(
|
||||
allowed_account.clone(),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::from([enabled.clone()])),
|
||||
),
|
||||
(
|
||||
disabled_only_account.clone(),
|
||||
RuntimeAccount::new(provider.clone(), BTreeSet::from([disabled.clone()])),
|
||||
),
|
||||
])));
|
||||
let scope = FrozenAccountScope::new(
|
||||
directory,
|
||||
ClientRoutingScope::restricted(
|
||||
vec![
|
||||
RoutingGroupSnapshot::new(enabled.clone(), "Enabled".to_owned()),
|
||||
RoutingGroupSnapshot::new(disabled, "Disabled".to_owned()),
|
||||
],
|
||||
BTreeSet::from([enabled]),
|
||||
BTreeSet::from([provider]),
|
||||
)
|
||||
.expect("restricted scope"),
|
||||
);
|
||||
|
||||
assert!(scope.allows(&allowed_account));
|
||||
assert!(!scope.allows(&disabled_only_account));
|
||||
let history = scope.routing_snapshot();
|
||||
assert_eq!(history.kind(), AccountRoutingScopeKind::Groups);
|
||||
assert_eq!(history.groups_snapshot().len(), 2);
|
||||
assert_eq!(history.groups_snapshot()[1].name(), "Disabled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricted_scope_with_no_enabled_group_should_fail_closed() {
|
||||
let group = AccountGroupId::new("grp_33333333333333333333333333333333").expect("group");
|
||||
let account = ProviderAccountId::new("acct_group_member").expect("account");
|
||||
let directory = Arc::new(RuntimeAccountDirectory::new(BTreeMap::from([(
|
||||
account.clone(),
|
||||
RuntimeAccount::new(
|
||||
ProviderKind::new("openai").expect("provider"),
|
||||
BTreeSet::from([group.clone()]),
|
||||
),
|
||||
)])));
|
||||
let scope = FrozenAccountScope::new(
|
||||
directory,
|
||||
ClientRoutingScope::restricted(
|
||||
vec![RoutingGroupSnapshot::new(group, "Disabled".to_owned())],
|
||||
BTreeSet::new(),
|
||||
BTreeSet::new(),
|
||||
)
|
||||
.expect("restricted scope"),
|
||||
);
|
||||
|
||||
assert!(scope.provider_kinds().is_empty());
|
||||
assert!(!scope.allows(&account));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_should_publish_only_enabled_plaintext_client_policies() {
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
@@ -98,7 +230,12 @@ fn snapshot_should_publish_only_enabled_plaintext_client_policies() {
|
||||
let policies = snapshot.client_policies().collect::<Vec<_>>();
|
||||
assert_eq!(policies.len(), 1);
|
||||
assert_eq!(policies[0].key_id().as_str(), "key_enabled");
|
||||
assert_eq!(policies[0].provider_kind().as_str(), "openai");
|
||||
assert!(
|
||||
policies[0]
|
||||
.account_scope()
|
||||
.provider_kinds()
|
||||
.contains(&ProviderKind::new("openai").expect("provider"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -137,8 +274,9 @@ fn selected_provider_should_use_global_model_mapping() {
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-5.4").expect("model"),
|
||||
&operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
required_provider: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
@@ -146,7 +284,13 @@ fn selected_provider_should_use_global_model_mapping() {
|
||||
|
||||
assert_eq!(plan.candidates().len(), 1);
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "openai");
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "gpt-5.5");
|
||||
assert_eq!(
|
||||
plan.candidates()[0]
|
||||
.upstream_model()
|
||||
.expect("model route candidate")
|
||||
.as_str(),
|
||||
"gpt-5.5"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -170,26 +314,82 @@ fn cyclic_model_mapping_should_fall_back_to_the_original_name() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_mapping_should_apply_to_every_provider() {
|
||||
fn known_provider_catalog_missing_mapped_model_should_be_filtered() {
|
||||
let snapshot = snapshot();
|
||||
let plan = snapshot
|
||||
let error = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-5.4").expect("model"),
|
||||
&operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("xai").expect("provider")),
|
||||
required_provider: Some(ProviderKind::new("xai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("transparent plan");
|
||||
.expect_err("known xAI catalog does not contain the mapped OpenAI model");
|
||||
|
||||
assert_eq!(plan.candidates()[0].provider().as_str(), "xai");
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), "gpt-5.5");
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_core::error::RoutingError::NoCapableProvider { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_endpoint_plan_should_not_consult_or_publish_the_text_model_catalog() {
|
||||
let snapshot = snapshot();
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let public_model = PublicModelId::new("gpt-image-2").expect("model");
|
||||
|
||||
assert!(!snapshot.contains_public_model_for_provider(&public_model, &provider));
|
||||
let plan = snapshot
|
||||
.plan_provider_endpoint(
|
||||
&provider,
|
||||
&image_operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext::default(),
|
||||
)
|
||||
.expect("provider-owned endpoint remains routable");
|
||||
|
||||
assert_eq!(plan.operation(), OperationKind::GenerateImage);
|
||||
assert_eq!(plan.candidates().len(), 1);
|
||||
assert_eq!(plan.candidates()[0].provider(), &provider);
|
||||
assert_eq!(plan.candidates()[0].upstream_model(), None);
|
||||
assert!(!snapshot.contains_public_model_for_provider(&public_model, &provider));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_endpoint_plan_should_still_respect_circuit_filtering() {
|
||||
let snapshot = snapshot();
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let error = snapshot
|
||||
.plan_provider_endpoint(
|
||||
&provider,
|
||||
&image_operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
blocked_providers: BTreeSet::from([provider.clone()]),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect_err("blocked provider has no endpoint candidate");
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
gateway_core::error::RoutingError::NoCapableProviderEndpoint { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unmapped_model_should_pass_through_unchanged() {
|
||||
let snapshot = snapshot();
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).expect("revision"),
|
||||
scheduling(),
|
||||
vec![ProviderKind::new("openai").expect("provider")],
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
.expect("snapshot")
|
||||
.with_account_directory(account_directory());
|
||||
for requested in [
|
||||
"future-openai-model".to_owned(),
|
||||
format!("future-{}", "x".repeat(512)),
|
||||
@@ -200,14 +400,21 @@ fn unmapped_model_should_pass_through_unchanged() {
|
||||
.plan(
|
||||
&PublicModelId::from_client_wire(requested.clone()).expect("client model"),
|
||||
&operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
required_provider: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
.expect("unknown model remains transparent");
|
||||
|
||||
assert_eq!(plan.candidates()[0].upstream_model().as_str(), requested);
|
||||
assert_eq!(
|
||||
plan.candidates()[0]
|
||||
.upstream_model()
|
||||
.expect("model route candidate")
|
||||
.as_str(),
|
||||
requested
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,8 +436,9 @@ fn blocked_provider_should_be_filtered() {
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-5.5").expect("model"),
|
||||
&operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
required_provider: Some(ProviderKind::new("openai").expect("provider")),
|
||||
blocked_providers: BTreeSet::from([ProviderKind::new("openai").expect("provider")]),
|
||||
},
|
||||
)
|
||||
@@ -262,8 +470,9 @@ fn known_unsupported_operation_should_not_be_bypassed() {
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-known-unsupported").expect("model"),
|
||||
&operation(),
|
||||
snapshot.all_account_scope(),
|
||||
&RoutingContext {
|
||||
provider_kind: Some(ProviderKind::new("openai").expect("provider")),
|
||||
required_provider: Some(ProviderKind::new("openai").expect("provider")),
|
||||
..RoutingContext::default()
|
||||
},
|
||||
)
|
||||
@@ -316,3 +525,20 @@ fn unknown_model_should_be_hidden_when_provider_catalog_is_available() {
|
||||
&ProviderKind::new("openai").expect("provider"),
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alias_should_only_be_available_from_a_provider_with_its_mapped_model() {
|
||||
let snapshot = snapshot();
|
||||
let alias = PublicModelId::new("grok-latest").expect("model");
|
||||
|
||||
assert!(!snapshot.contains_public_model_for_provider(
|
||||
&alias,
|
||||
&ProviderKind::new("openai").expect("provider"),
|
||||
));
|
||||
assert!(
|
||||
snapshot.contains_public_model_for_provider(
|
||||
&alias,
|
||||
&ProviderKind::new("xai").expect("provider"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ use gateway_core::operation::OperationKind;
|
||||
use gateway_core::policy::{ClientApiKeyId, PlaintextClientApiKey, RateLimits};
|
||||
use gateway_core::routing::snapshot::{
|
||||
RuntimeSnapshotCompileError, RuntimeSnapshotCompiler, RuntimeSnapshotHandle,
|
||||
RuntimeSnapshotPublisher, SnapshotClientPolicyFacts, SnapshotControl, SnapshotFacts,
|
||||
RuntimeSnapshotPublisher, SnapshotAccountGroupFacts, SnapshotAccountGroupMemberFacts,
|
||||
SnapshotClientPolicyFacts, SnapshotControl, SnapshotFacts, SnapshotProviderAccountFacts,
|
||||
SnapshotRevisionStream, SnapshotSettingsFacts, SnapshotStoreError, SnapshotStorePort,
|
||||
SnapshotSubscriptionError, SnapshotSubscriptionPort, runtime_revision_needs_refresh,
|
||||
};
|
||||
@@ -348,9 +349,12 @@ fn facts(config_revision: u64, observed_current_revision: u64) -> SnapshotFacts
|
||||
vec![SnapshotClientPolicyFacts::new(
|
||||
ClientApiKeyId::new("key_one").expect("key ID"),
|
||||
PlaintextClientApiKey::new("sk_test").expect("plaintext key"),
|
||||
"alpha",
|
||||
Vec::new(),
|
||||
RateLimits::unlimited(),
|
||||
)],
|
||||
Vec::<SnapshotAccountGroupFacts>::new(),
|
||||
Vec::<SnapshotProviderAccountFacts>::new(),
|
||||
Vec::<SnapshotAccountGroupMemberFacts>::new(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ pub fn load_config<T: LoadableConfig>() -> Result<T, ConfigError> {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HostConfig {
|
||||
pub listen: ListenConfig,
|
||||
pub runtime_data_dir: PathBuf,
|
||||
pub logging: LoggingConfig,
|
||||
#[serde(default)]
|
||||
pub system_update: SystemUpdateConfig,
|
||||
@@ -76,11 +77,21 @@ impl HostConfig {
|
||||
"host.worker_shutdown_timeout_seconds",
|
||||
));
|
||||
}
|
||||
if self.runtime_data_dir.as_os_str().is_empty() {
|
||||
return Err(ConfigError::InvalidField("host.runtime_data_dir"));
|
||||
}
|
||||
resolve_relative_path(source_dir, &mut self.runtime_data_dir);
|
||||
self.logging.resolve_and_validate(source_dir)?;
|
||||
self.system_update.resolve_and_validate(source_dir)?;
|
||||
self.system_update
|
||||
.resolve_and_validate(source_dir, &self.runtime_data_dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn runtime_data_dir(&self) -> &Path {
|
||||
&self.runtime_data_dir
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn drain_timeout(&self) -> Duration {
|
||||
Duration::from_secs(self.drain_timeout_seconds)
|
||||
|
||||
@@ -13,6 +13,9 @@ use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::Subscriber
|
||||
use crate::config::LoggingConfig;
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
|
||||
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
@@ -34,7 +37,8 @@ pub enum LogError {
|
||||
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = EnvFilter::try_new(&directive).map_err(|_| LogError::InvalidFilter)?;
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
@@ -47,25 +51,37 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
guards.push(guard);
|
||||
writer
|
||||
});
|
||||
let file_writer = if config.file.enabled {
|
||||
let (file_writer, recovery_file_writer) = if config.file.enabled {
|
||||
let maximum_bytes = config
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
let file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (writer, guard) = NonBlockingBuilder::default()
|
||||
let recovery_file_writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
maximum_bytes,
|
||||
config.file.retention_days,
|
||||
config.file.max_files,
|
||||
)?;
|
||||
let (file_writer, file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-file")
|
||||
.finish(writer);
|
||||
guards.push(guard);
|
||||
Some(writer)
|
||||
.finish(file_writer);
|
||||
let (recovery_file_writer, recovery_file_guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-oauth-recovery-file")
|
||||
.finish(recovery_file_writer);
|
||||
guards.push(file_guard);
|
||||
guards.push(recovery_file_guard);
|
||||
(Some(file_writer), Some(recovery_file_writer))
|
||||
} else {
|
||||
None
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
@@ -89,14 +105,43 @@ pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError>
|
||||
.with_span_list(true)
|
||||
.with_filter(file_filter)
|
||||
});
|
||||
let recovery_file_layer = recovery_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard { _writers: guards })
|
||||
}
|
||||
|
||||
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
directive.to_owned()
|
||||
@@ -107,6 +152,7 @@ fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> Str
|
||||
|
||||
struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -119,17 +165,20 @@ struct RotatingLogWriter {
|
||||
impl RotatingLogWriter {
|
||||
fn open(
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = Utc::now().date_naive();
|
||||
cleanup_log_files(&directory, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) = writable_log_segment(&directory, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, date, segment)?;
|
||||
cleanup_log_files(&directory, file_prefix, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) =
|
||||
writable_log_segment(&directory, file_prefix, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, file_prefix, date, segment)?;
|
||||
Ok(Self {
|
||||
directory,
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
maximum_files,
|
||||
@@ -157,10 +206,11 @@ impl RotatingLogWriter {
|
||||
} else {
|
||||
self.segment = self.segment.saturating_add(1);
|
||||
}
|
||||
self.file = open_log_segment(&self.directory, self.date, self.segment)?;
|
||||
self.file = open_log_segment(&self.directory, self.file_prefix, self.date, self.segment)?;
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
cleanup_log_files(
|
||||
&self.directory,
|
||||
self.file_prefix,
|
||||
self.date,
|
||||
self.retention_days,
|
||||
self.maximum_files,
|
||||
@@ -192,10 +242,11 @@ struct ManagedLogFile {
|
||||
|
||||
fn writable_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
maximum_bytes: u64,
|
||||
) -> io::Result<(usize, u64)> {
|
||||
let latest = managed_log_files(directory)?
|
||||
let latest = managed_log_files(directory, file_prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| entry.segment);
|
||||
@@ -210,26 +261,31 @@ fn writable_log_segment(
|
||||
}
|
||||
}
|
||||
|
||||
fn open_log_segment(directory: &Path, date: chrono::NaiveDate, segment: usize) -> io::Result<File> {
|
||||
fn open_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(directory.join(log_file_name(date, segment)))
|
||||
.open(directory.join(log_file_name(file_prefix, date, segment)))
|
||||
}
|
||||
|
||||
fn log_file_name(date: chrono::NaiveDate, segment: usize) -> String {
|
||||
fn log_file_name(file_prefix: &str, date: chrono::NaiveDate, segment: usize) -> String {
|
||||
if segment == 0 {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.log")
|
||||
format!("{file_prefix}.{date}.log")
|
||||
} else {
|
||||
format!("{LOG_FILE_PREFIX}.{date}.{segment}.log")
|
||||
format!("{file_prefix}.{date}.{segment}.log")
|
||||
}
|
||||
}
|
||||
|
||||
fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
fn managed_log_files(directory: &Path, file_prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
|
||||
Ok(fs::read_dir(directory)?
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref()).map(
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref(), file_prefix).map(
|
||||
|(date, segment)| ManagedLogFile {
|
||||
date,
|
||||
segment,
|
||||
@@ -240,9 +296,9 @@ fn managed_log_files(directory: &Path) -> io::Result<Vec<ManagedLogFile>> {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
fn parse_log_file_name(name: &str, file_prefix: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
let body = name
|
||||
.strip_prefix(&format!("{LOG_FILE_PREFIX}."))?
|
||||
.strip_prefix(&format!("{file_prefix}."))?
|
||||
.strip_suffix(".log")?;
|
||||
let (date, segment) = body.split_once('.').map_or((body, 0), |(date, segment)| {
|
||||
(date, segment.parse().ok().unwrap_or(usize::MAX))
|
||||
@@ -257,6 +313,7 @@ fn parse_log_file_name(name: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
today: chrono::NaiveDate,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
@@ -266,7 +323,7 @@ fn cleanup_log_files(
|
||||
.checked_sub_signed(chrono::Duration::days(retention_offset))
|
||||
.unwrap_or(chrono::NaiveDate::MIN);
|
||||
let mut retained = Vec::new();
|
||||
for entry in managed_log_files(directory)? {
|
||||
for entry in managed_log_files(directory, file_prefix)? {
|
||||
if entry.date < cutoff {
|
||||
fs::remove_file(entry.path)?;
|
||||
} else {
|
||||
|
||||
@@ -81,10 +81,10 @@ impl Default for SystemUpdateConfig {
|
||||
});
|
||||
let update_state_file = environment_value("CPR_UPDATE_STATE_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("/app/.runtime/data/update-state.json"));
|
||||
.unwrap_or_else(|| PathBuf::from("update-state.json"));
|
||||
let update_lock_file = environment_value("CPR_UPDATE_LOCK_FILE")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| update_state_file.with_extension("lock"));
|
||||
.unwrap_or_else(|| update_state_file.with_file_name("update.lock"));
|
||||
let update_temp_dir = environment_value("CPR_UPDATE_TEMP_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| state::default_temp_dir(&update_state_file));
|
||||
@@ -120,21 +120,28 @@ impl Default for SystemUpdateConfig {
|
||||
}
|
||||
|
||||
impl SystemUpdateConfig {
|
||||
pub(crate) fn resolve_and_validate(&mut self, source_dir: &Path) -> Result<(), ConfigError> {
|
||||
for path in [
|
||||
self.executable_path.as_mut(),
|
||||
Some(&mut self.web_dist_dir),
|
||||
Some(&mut self.update_state_file),
|
||||
Some(&mut self.update_lock_file),
|
||||
Some(&mut self.update_temp_dir),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
pub(crate) fn resolve_and_validate(
|
||||
&mut self,
|
||||
source_dir: &Path,
|
||||
runtime_data_dir: &Path,
|
||||
) -> Result<(), ConfigError> {
|
||||
for path in [self.executable_path.as_mut(), Some(&mut self.web_dist_dir)]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if path.is_relative() {
|
||||
*path = source_dir.join(&*path);
|
||||
}
|
||||
}
|
||||
for path in [
|
||||
&mut self.update_state_file,
|
||||
&mut self.update_lock_file,
|
||||
&mut self.update_temp_dir,
|
||||
] {
|
||||
if path.is_relative() {
|
||||
*path = runtime_data_dir.join(&*path);
|
||||
}
|
||||
}
|
||||
if self.version.trim().is_empty() {
|
||||
return Err(ConfigError::InvalidField("host.system_update.version"));
|
||||
}
|
||||
|
||||
@@ -34,6 +34,40 @@ fn host_config_resolves_only_host_owned_relative_paths() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_resolves_runtime_data_dir_relative_to_configuration() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
config.runtime_data_dir(),
|
||||
PathBuf::from("/srv/gateway/runtime-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_derives_update_paths_from_runtime_data_dir() {
|
||||
let mut config = valid_config();
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.expect("valid host config");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
config.system_update.update_state_file,
|
||||
config.system_update.update_lock_file,
|
||||
config.system_update.update_temp_dir,
|
||||
),
|
||||
(
|
||||
PathBuf::from("/srv/gateway/runtime-data/update-state.json"),
|
||||
PathBuf::from("/srv/gateway/runtime-data/update.lock"),
|
||||
PathBuf::from("/srv/gateway/runtime-data/update-tmp"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_config_rejects_zero_drain_window() {
|
||||
let mut config = valid_config();
|
||||
@@ -48,9 +82,9 @@ fn host_config_rejects_zero_drain_window() {
|
||||
|
||||
fn valid_config() -> HostConfig {
|
||||
let system_update = SystemUpdateConfig {
|
||||
update_state_file: PathBuf::from(".runtime/update-state.json"),
|
||||
update_lock_file: PathBuf::from(".runtime/update-state.lock"),
|
||||
update_temp_dir: PathBuf::from(".runtime/update-tmp"),
|
||||
update_state_file: PathBuf::from("update-state.json"),
|
||||
update_lock_file: PathBuf::from("update.lock"),
|
||||
update_temp_dir: PathBuf::from("update-tmp"),
|
||||
..SystemUpdateConfig::default()
|
||||
};
|
||||
HostConfig {
|
||||
@@ -58,6 +92,7 @@ fn valid_config() -> HostConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
},
|
||||
runtime_data_dir: PathBuf::from("runtime-data"),
|
||||
logging: LoggingConfig {
|
||||
level: "info".to_owned(),
|
||||
stdout: true,
|
||||
|
||||
@@ -1,8 +1,21 @@
|
||||
use std::path::PathBuf;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use gateway_host::config::{FileLoggingConfig, HostConfig, ListenConfig, LoggingConfig};
|
||||
use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
|
||||
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
|
||||
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_MARKER: &str = "oauth-recovery-file-filter-test";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
|
||||
#[test]
|
||||
fn logging_requires_at_least_one_sink() {
|
||||
let mut config = HostConfig {
|
||||
@@ -10,6 +23,7 @@ fn logging_requires_at_least_one_sink() {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
},
|
||||
runtime_data_dir: PathBuf::from("/tmp/runtime-data"),
|
||||
logging: LoggingConfig {
|
||||
level: "info".to_owned(),
|
||||
stdout: false,
|
||||
@@ -32,3 +46,116 @@ fn logging_requires_at_least_one_sink() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
write_oauth_recovery_log(PathBuf::from(
|
||||
env::var_os(LOG_DIRECTORY_ENV).expect("child log directory"),
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
let directory = tempfile::tempdir().expect("create log directory");
|
||||
let output = Command::new(env::current_exe().expect("current test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_file_logging_is_separate_and_overrides_global_log_level",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env(
|
||||
"RUST_LOG",
|
||||
"off,logging_test_application=info,oauth_recovery=off",
|
||||
)
|
||||
.output()
|
||||
.expect("run logging child process");
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"logging child process failed:\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
|
||||
let application_log = read_log_file_set(directory.path(), APPLICATION_LOG_FILE_PREFIX);
|
||||
assert!(application_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(application_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!application_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(!application_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
|
||||
let recovery_log = read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX);
|
||||
assert!(recovery_log.contains(&json_target_field(OAUTH_RECOVERY_LOG_TARGET)));
|
||||
assert!(recovery_log.contains(OAUTH_RECOVERY_LOG_MARKER));
|
||||
assert!(!recovery_log.contains(&json_target_field(APPLICATION_LOG_TARGET)));
|
||||
assert!(!recovery_log.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(recovery_log.contains(&format!(r#""provider":"{OAUTH_RECOVERY_PROVIDER}""#)));
|
||||
}
|
||||
|
||||
fn write_oauth_recovery_log(directory: PathBuf) {
|
||||
let config = HostConfig {
|
||||
listen: ListenConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
},
|
||||
runtime_data_dir: PathBuf::from("/tmp/runtime-data"),
|
||||
logging: LoggingConfig {
|
||||
level: "off".to_owned(),
|
||||
stdout: false,
|
||||
file: FileLoggingConfig {
|
||||
enabled: true,
|
||||
directory,
|
||||
retention_days: 1,
|
||||
max_file_size_mb: 1,
|
||||
max_files: 1,
|
||||
},
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create logging runtime");
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
|
||||
tracing::info!(
|
||||
target: APPLICATION_LOG_TARGET,
|
||||
marker = APPLICATION_LOG_MARKER,
|
||||
"application file test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
let mut files = fs::read_dir(directory)
|
||||
.expect("read log directory")
|
||||
.map(|entry| entry.expect("read log entry").path())
|
||||
.filter(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
files.sort();
|
||||
|
||||
assert!(!files.is_empty(), "expected log file set {file_prefix}");
|
||||
files
|
||||
.into_iter()
|
||||
.map(|path| fs::read_to_string(path).expect("read log file"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn json_target_field(target: &str) -> String {
|
||||
format!(r#""target":"{target}""#)
|
||||
}
|
||||
|
||||
@@ -208,11 +208,24 @@ pub struct RateLimitWindow {
|
||||
pub reset_at: Option<i64>,
|
||||
}
|
||||
|
||||
/// 限流项键的来源。
|
||||
///
|
||||
/// HTTP headers 和事件里的 `metered_limit_name` 提供稳定 `limit_id`;
|
||||
/// WebSocket `additional_rate_limits` 在部分协议版本里只以可读名称为 map key,
|
||||
/// 此时必须由持久化层用既有名称元数据解析,不能把名称误当成稳定 ID。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RateLimitKeySource {
|
||||
LimitId,
|
||||
LimitName,
|
||||
}
|
||||
|
||||
/// 单个计量项的限流信息。
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RateLimitDetails {
|
||||
/// 稳定的计量项 ID。
|
||||
pub limit_id: String,
|
||||
/// [`limit_id`](Self::limit_id) 是稳定 ID,还是名称生成的临时解析键。
|
||||
pub key_source: RateLimitKeySource,
|
||||
/// 上游提供的可读名称。
|
||||
pub limit_name: Option<String>,
|
||||
/// 当前请求是否被允许。
|
||||
@@ -315,26 +328,66 @@ pub fn is_rate_limit_header_name(name: &str) -> bool {
|
||||
|| rate_limit_id_from_header_name(&normalized).is_some()
|
||||
}
|
||||
|
||||
/// 判断响应头是否为仅供代理本地观测的 Codex 额度信号。
|
||||
///
|
||||
/// 这类头会让下游官方 Codex 客户端显示账户额度提示,因此不能穿过账号隔离边界。
|
||||
/// 通用限流头(例如 `retry-after`)和非额度的 Codex 头仍允许下发。
|
||||
pub fn is_codex_quota_header_name(name: &str) -> bool {
|
||||
let normalized = name.trim().to_ascii_lowercase();
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"x-codex-credits-has-credits"
|
||||
| "x-codex-credits-unlimited"
|
||||
| "x-codex-credits-balance"
|
||||
| "x-codex-active-limit"
|
||||
| "x-codex-plan-type"
|
||||
| "x-codex-promo-message"
|
||||
| "x-codex-rate-limit-reached-type"
|
||||
| "x-codex-primary-over-secondary-limit-percent"
|
||||
) || rate_limit_id_from_header_name(&normalized)
|
||||
.is_some_and(|limit_id| limit_id == "codex" || limit_id.starts_with("codex_"))
|
||||
}
|
||||
|
||||
/// 从内部 `codex.rate_limits` 事件中解析限流信息。
|
||||
pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
if value.get("type").and_then(Value::as_str) != Some("codex.rate_limits") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let limit_id = rate_limit_name(value)
|
||||
.map(normalize_limit_id)
|
||||
let explicit_limit_id = top_level_rate_limit_id(value).map(normalize_limit_id);
|
||||
let explicit_limit_name = non_empty_string(value, "limit_name");
|
||||
let default_limit_id = explicit_limit_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "codex".to_string());
|
||||
let limit_name = value
|
||||
.get("metered_limit_name")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|_| value.get("limit_name"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(ToString::to_string);
|
||||
let details = value
|
||||
.get("rate_limits")
|
||||
.and_then(|details| parse_details_from_object(details, &limit_id, limit_name));
|
||||
let default_details = value.get("rate_limits").and_then(|details| {
|
||||
parse_details_from_object(
|
||||
details,
|
||||
&default_limit_id,
|
||||
explicit_limit_name,
|
||||
RateLimitKeySource::LimitId,
|
||||
)
|
||||
});
|
||||
let mut limits = parse_additional_rate_limits(value);
|
||||
let matching_additional_ids = default_details
|
||||
.as_ref()
|
||||
.map(|default| {
|
||||
limits
|
||||
.iter()
|
||||
.filter(|(_, additional)| same_rate_limit_facts(default, additional))
|
||||
.map(|(limit_id, _)| limit_id.clone())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
// 当前 WebSocket 协议会把活动具名桶同时镜像到顶层 `rate_limits`,但不附
|
||||
// `metered_limit_name`。只要 additional 中存在相同事实,顶层就不是 core 桶。
|
||||
let default_is_additional_alias =
|
||||
explicit_limit_id.is_none() && !matching_additional_ids.is_empty();
|
||||
if let Some(details) = default_details
|
||||
&& !default_is_additional_alias
|
||||
{
|
||||
limits.insert(default_limit_id.clone(), details);
|
||||
}
|
||||
let credits = value.get("credits").and_then(parse_credits_from_object);
|
||||
let plan_type = value
|
||||
.get("plan_type")
|
||||
@@ -343,17 +396,18 @@ pub fn parse_rate_limits_event(value: &Value) -> Option<ParsedRateLimits> {
|
||||
.filter(|plan| !plan.is_empty())
|
||||
.map(ToString::to_string);
|
||||
|
||||
if details.is_none() && credits.is_none() && plan_type.is_none() {
|
||||
if limits.is_empty() && credits.is_none() && plan_type.is_none() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut limits = BTreeMap::new();
|
||||
if let Some(details) = details {
|
||||
limits.insert(limit_id.clone(), details);
|
||||
}
|
||||
let active_limit = explicit_limit_id.or_else(|| match matching_additional_ids.as_slice() {
|
||||
[limit_id] => Some(limit_id.clone()),
|
||||
[] if !limits.is_empty() => Some(default_limit_id),
|
||||
_ => None,
|
||||
});
|
||||
Some(ParsedRateLimits {
|
||||
limits,
|
||||
active_limit: Some(limit_id),
|
||||
active_limit,
|
||||
credits,
|
||||
plan_type,
|
||||
promo_message: value
|
||||
@@ -577,6 +631,7 @@ fn parse_details_from_lookup(
|
||||
}
|
||||
Some(RateLimitDetails {
|
||||
limit_id: limit_id.to_string(),
|
||||
key_source: RateLimitKeySource::LimitId,
|
||||
limit_name,
|
||||
allowed,
|
||||
limit_reached,
|
||||
@@ -667,6 +722,7 @@ fn parse_details_from_object(
|
||||
value: &Value,
|
||||
limit_id: &str,
|
||||
limit_name: Option<String>,
|
||||
key_source: RateLimitKeySource,
|
||||
) -> Option<RateLimitDetails> {
|
||||
let primary = value.get("primary").and_then(parse_window_from_object);
|
||||
let secondary = value.get("secondary").and_then(parse_window_from_object);
|
||||
@@ -677,6 +733,7 @@ fn parse_details_from_object(
|
||||
}
|
||||
Some(RateLimitDetails {
|
||||
limit_id: limit_id.to_string(),
|
||||
key_source,
|
||||
limit_name,
|
||||
allowed,
|
||||
limit_reached,
|
||||
@@ -732,11 +789,59 @@ fn value_as_positive_i64(value: &Value) -> Option<i64> {
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
fn rate_limit_name(value: &Value) -> Option<&str> {
|
||||
fn rate_limit_id(value: &Value) -> Option<&str> {
|
||||
value
|
||||
.get("metered_limit_name")
|
||||
.or_else(|| value.get("limit_name"))
|
||||
.or_else(|| value.get("limit_id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn top_level_rate_limit_id(value: &Value) -> Option<&str> {
|
||||
rate_limit_id(value).or_else(|| {
|
||||
value
|
||||
.get("limit_name")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
fn non_empty_string(value: &Value, field: &str) -> Option<String> {
|
||||
value
|
||||
.get(field)
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToString::to_string)
|
||||
}
|
||||
|
||||
fn parse_additional_rate_limits(value: &Value) -> BTreeMap<String, RateLimitDetails> {
|
||||
value
|
||||
.get("additional_rate_limits")
|
||||
.and_then(Value::as_object)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|(map_key, value)| {
|
||||
let explicit_limit_id = rate_limit_id(value).map(normalize_limit_id);
|
||||
let (limit_id, key_source) = explicit_limit_id.map_or_else(
|
||||
|| (normalize_limit_id(map_key), RateLimitKeySource::LimitName),
|
||||
|limit_id| (limit_id, RateLimitKeySource::LimitId),
|
||||
);
|
||||
let limit_name = non_empty_string(value, "limit_name")
|
||||
.or_else(|| Some(map_key.trim().to_owned()).filter(|name| !name.is_empty()));
|
||||
parse_details_from_object(value, &limit_id, limit_name, key_source)
|
||||
.map(|details| (limit_id, details))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn same_rate_limit_facts(left: &RateLimitDetails, right: &RateLimitDetails) -> bool {
|
||||
left.allowed == right.allowed
|
||||
&& left.limit_reached == right.limit_reached
|
||||
&& left.primary == right.primary
|
||||
&& left.secondary == right.secondary
|
||||
}
|
||||
|
||||
fn normalize_limit_id(value: impl AsRef<str>) -> String {
|
||||
|
||||
@@ -70,7 +70,7 @@ pub const MAX_SSE_EVENT_BUFFER_BYTES: usize = 64 * 1024 * 1024;
|
||||
pub const DONE_SSE_FRAME: &str = "data: [DONE]\n\n";
|
||||
|
||||
/// 用于处理任意分块边界的增量 SSE 解码器。
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug)]
|
||||
pub struct SseEventDecoder {
|
||||
pending: Vec<u8>,
|
||||
/// `pending` 中已确认不含帧分隔符的前缀长度。新数据到达时从该边界
|
||||
@@ -79,6 +79,19 @@ pub struct SseEventDecoder {
|
||||
/// 超大未完成帧已开始直接交付;在遇到帧分隔符前只保留跨 chunk 扫描尾部,
|
||||
/// 不再尝试解析该帧。
|
||||
opaque_frame: bool,
|
||||
/// 只有整个 SSE 流的第一条物理行允许携带 UTF-8 BOM。
|
||||
stream_start: bool,
|
||||
}
|
||||
|
||||
impl Default for SseEventDecoder {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pending: Vec::new(),
|
||||
scanned: 0,
|
||||
opaque_frame: false,
|
||||
stream_start: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SseEventDecoder {
|
||||
@@ -115,7 +128,8 @@ impl SseEventDecoder {
|
||||
while let Some(end) = self.next_frame_end(consumed) {
|
||||
let frame = std::str::from_utf8(&self.pending[consumed..end])
|
||||
.map_err(|error| SseError::ParseError(error.to_string()))?;
|
||||
events.extend(parse_sse_events(frame)?);
|
||||
events.extend(parse_sse_events_inner(frame, self.stream_start)?);
|
||||
self.stream_start = false;
|
||||
consumed = end;
|
||||
}
|
||||
|
||||
@@ -149,9 +163,10 @@ impl SseEventDecoder {
|
||||
} else {
|
||||
std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.and_then(|frame| parse_sse_events_inner(frame, self.stream_start).ok())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
self.stream_start = false;
|
||||
frames.push(SseFrame::new(raw, events));
|
||||
consumed = end;
|
||||
}
|
||||
@@ -173,7 +188,9 @@ impl SseEventDecoder {
|
||||
let pending = std::mem::take(&mut self.pending);
|
||||
let frame = std::str::from_utf8(&pending)
|
||||
.map_err(|error| SseError::ParseError(error.to_string()))?;
|
||||
parse_sse_events(frame)
|
||||
let events = parse_sse_events_inner(frame, self.stream_start)?;
|
||||
self.stream_start = false;
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
/// 在流结束时返回未带空行分隔符的最后一帧及其旁路解析结果。
|
||||
@@ -191,9 +208,10 @@ impl SseEventDecoder {
|
||||
} else {
|
||||
std::str::from_utf8(&raw)
|
||||
.ok()
|
||||
.and_then(|frame| parse_sse_events(frame).ok())
|
||||
.and_then(|frame| parse_sse_events_inner(frame, self.stream_start).ok())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
self.stream_start = false;
|
||||
self.opaque_frame = false;
|
||||
vec![SseFrame::new(raw, events)]
|
||||
}
|
||||
@@ -207,6 +225,7 @@ impl SseEventDecoder {
|
||||
let remaining = self.pending.split_off(end);
|
||||
let raw = std::mem::replace(&mut self.pending, remaining);
|
||||
frames.push(SseFrame::new(raw, Vec::new()));
|
||||
self.stream_start = false;
|
||||
self.opaque_frame = false;
|
||||
self.scanned = 0;
|
||||
true
|
||||
@@ -226,6 +245,7 @@ impl SseEventDecoder {
|
||||
let raw = std::mem::replace(&mut self.pending, tail);
|
||||
self.scanned = 0;
|
||||
frames.push(SseFrame::new(raw, Vec::new()));
|
||||
self.stream_start = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,12 +371,21 @@ impl EventBuilder {
|
||||
|
||||
/// 解析 SSE 事件流。
|
||||
pub fn parse_sse_events(input: &str) -> Result<Vec<SseEvent>, SseError> {
|
||||
parse_sse_events_inner(input, true)
|
||||
}
|
||||
|
||||
fn parse_sse_events_inner(input: &str, strip_leading_bom: bool) -> Result<Vec<SseEvent>, SseError> {
|
||||
let mut events = Vec::new();
|
||||
let mut builder = EventBuilder::default();
|
||||
let mut saw_sse_syntax = false;
|
||||
let mut event_buffer_bytes = 0usize;
|
||||
|
||||
for raw_line in input.lines() {
|
||||
for (index, raw_line) in input.lines().enumerate() {
|
||||
let raw_line = if strip_leading_bom && index == 0 {
|
||||
raw_line.strip_prefix('\u{feff}').unwrap_or(raw_line)
|
||||
} else {
|
||||
raw_line
|
||||
};
|
||||
let line = raw_line.strip_suffix('\r').unwrap_or(raw_line);
|
||||
if line.is_empty() {
|
||||
event_buffer_bytes = 0;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user