mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 08:05:32 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76291f448c | ||
|
|
c10efea12c | ||
|
|
806f7ad9ee | ||
|
|
632edee7dc | ||
|
|
22518d064c | ||
|
|
64c1ec2afd | ||
|
|
a3f0c1b028 | ||
|
|
605354c861 | ||
|
|
60e6bc0125 | ||
|
|
bfea3e1d6f | ||
|
|
6b4b8faaf4 | ||
|
|
f195de38f5 | ||
|
|
07bdd425b6 | ||
|
|
d48982eb04 | ||
|
|
39b35d46a0 | ||
|
|
8fc8c6228d | ||
|
|
7da79b82fd | ||
|
|
a23afbf2f5 | ||
|
|
b4a5210e44 | ||
|
|
f7f59103a3 | ||
|
|
5b77a6b8de | ||
|
|
3f95f8b0c4 | ||
|
|
28fa655c26 | ||
|
|
7a60bc6ded | ||
|
|
a5b767e69b | ||
|
|
78bc57c5cf | ||
|
|
ef11b72d8a | ||
|
|
52fb1e7180 | ||
|
|
7a7a7ed6ad | ||
|
|
b8479b59db | ||
|
|
2480f07f2d | ||
|
|
b2dc9f493d | ||
|
|
d92d310e45 | ||
|
|
3caf83131a | ||
|
|
af35cde374 | ||
|
|
5c115fb38f | ||
|
|
ed558e5396 | ||
|
|
929f781a63 | ||
|
|
e0326ecba4 | ||
|
|
6057088300 | ||
|
|
e3a49a6217 | ||
|
|
b32f87fa8a | ||
|
|
91e8105e5d | ||
|
|
c4d9576639 | ||
|
|
015cd60be1 |
@@ -40,7 +40,9 @@ Issue 可记录复现、排查和变更经过;涉及常规文档修改时执
|
||||
|
||||
## 回复口吻
|
||||
|
||||
草拟和发布 Issue 回复时,执行 [GitHub 回复口吻](../../../CONTRIBUTING.md#github-回复口吻)
|
||||
草拟、发布或修改 Issue 回复时,按 [GitHub 回复口吻](../../../CONTRIBUTING.md#github-回复口吻)将核实结果写成面向提问者的交流答复,具体口吻规则集中维护在该节
|
||||
|
||||
发送或更新前从提问者视角通读:是否接住了他的目标和困难,建议是否便于尝试,补充材料的用途是否清楚,澄清或拒绝是否耐心解释了原因,能提供的替代办法是否说清。发现生硬时调整整句措辞与前后衔接,不能仅加缓和词;保留已核实的结论
|
||||
|
||||
## 发布与收尾
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
## 清单与处理器
|
||||
|
||||
- `publisher` 与机器短名 `name` 派生插件 ID,`displayName` 用于展示;复制示例后同时调整扩展项 ID 与注册结果
|
||||
- `contributes` 每种能力最多一项,普通贡献项可省略派生 ID、能力版本和固定阶段;中间件必须声明版本 `4` 并显式选择阶段,上游适配器必须声明版本 `2`。自定义完整扩展项 ID 属于自己的插件命名空间,运行注册与规范化清单一致
|
||||
- `contributes` 每种能力最多一项,派生 ID 和固定阶段可省略;能力版本与中间件阶段按[扩展项简写](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)声明。自定义完整扩展项 ID 属于自己的插件命名空间,运行注册与规范化清单一致
|
||||
- 作者清单用 `Manifest::from_author_slice` 规范化,不手写 `package`、固定阶段或重复注册;CLI 生成平台、协议与资源摘要,版本范围不能使用全版本通配
|
||||
- 安装后宿主会按 `configurationSchema` 准备默认配置,配置完整即可启用;默认绑定的空范围不限制请求。需要业务参数时声明真实必填项,通过 `secretFields` 声明敏感字段,不依赖用户再走一遍自建安装向导
|
||||
- Rust 插件只依赖公开 `gateway-plugin-sdk`;需要异步会话辅助时开启 `io`,使用 `PluginSession` 管理握手、回调关联、流控、取消和关闭
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
调用入口前重新核对当前脚本的副作用、目标版本和待推送提交,保留无关工作区内容,不自动清理或暂存用户文件。
|
||||
先编辑 `release/notes.md`,保留为未提交改动(已暂存或未暂存均可);其他已跟踪文件的改动应先单独提交。
|
||||
脚本会更新 `release/version.yaml`,将它与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
|
||||
脚本会更新 `release/version.yaml`,按[接口弃用规则](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)校验并维护 `plugin-api-deprecations.json`,将这些差异与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
|
||||
不要提前单独提交说明或把版本文件改成目标版本;脚本要求说明相对 HEAD 有改动,并自行更新版本号。
|
||||
本任务需要的提交按[项目约定](../../../../CONTRIBUTING.md#项目约定)保留英文 Conventional Commits 与相应署名。
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
release/publish "$release_version"
|
||||
```
|
||||
|
||||
该命令会创建包含版本号和发布说明的单次提交与 annotated tag,原子推送分支和 tag,再从长期分支手动触发 `release.yml`。
|
||||
该命令会创建上述版本提交与 annotated tag,原子推送分支和 tag,再从长期分支手动触发 `release.yml`。
|
||||
它没有只读预演模式;不能为了检查或演练而执行。发布工作流读取 tag 中的说明,修改 tag 之后的本地文件不会改变该次构建。
|
||||
|
||||
只调整已发布页面的标题或正文时,编辑对应 GitHub Release 的 `name` 或 `body` 并回读核对;
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
- 只翻译已有准确说明时保持事实与范围,不混入其他变更
|
||||
- 常规文档仍只描述当前状态;必要修订执行[文档检查](../../cpr-dev-guide/references/documentation.md),不复制发布说明中的版本变化列表
|
||||
|
||||
实际发布由 `release/publish` 提交说明与版本文件,准备说明时不要提前单独提交或把版本文件改成目标版本
|
||||
实际发布由 `release/publish` 提交说明、版本文件及[插件接口弃用清单](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)的必要更新,准备说明时不要提前单独提交或把版本文件改成目标版本
|
||||
用户只要求准备时保留本地改动,纯说明修改无需重建前后端
|
||||
|
||||
## 发行准备检查
|
||||
|
||||
@@ -18,7 +18,7 @@ runs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
@@ -38,7 +38,7 @@ runs:
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: inputs.upload-artifact == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist
|
||||
|
||||
@@ -242,7 +242,7 @@ runs:
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -156,6 +156,20 @@ jobs:
|
||||
echo "CPR_TEST_REDIS_URL=redis://:${redis_password}@127.0.0.1:6379"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Expose matching PostgreSQL backup and restore tools
|
||||
shell: bash
|
||||
env:
|
||||
CPR_TEST_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
client_dir="$RUNNER_TEMP/pg-client"
|
||||
mkdir -p "$client_dir"
|
||||
for tool in pg_dump psql; do
|
||||
printf '#!/bin/sh\nexec docker exec -i --env PGPASSWORD "%s" %s "$@"\n' "$CPR_TEST_POSTGRES_CONTAINER" "$tool" > "$client_dir/$tool"
|
||||
chmod +x "$client_dir/$tool"
|
||||
done
|
||||
echo "$client_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Rust tests
|
||||
working-directory: backend
|
||||
run: |
|
||||
|
||||
@@ -22,6 +22,9 @@ jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
backend: ${{ steps.filter.outputs.backend }}
|
||||
frontend: ${{ steps.filter.outputs.frontend }}
|
||||
@@ -119,7 +122,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
- name: Setup Rust
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload sealed plugin assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -219,19 +219,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/frontend
|
||||
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: codex-proxy-rs-*
|
||||
path: dist/binaries
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
subject-path: dist/release/*
|
||||
|
||||
- name: Upload release assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
@@ -283,19 +283,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/docker/web/dist
|
||||
|
||||
- name: Download Docker binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -362,7 +362,7 @@ jobs:
|
||||
printf '%s\n' "$IMAGE_DIGEST" > "dist/digests/linux-${IMAGE_ARCH}.txt"
|
||||
|
||||
- name: Upload verified image digest
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: image-digest-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/digests/*.txt
|
||||
@@ -390,13 +390,13 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download release assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
|
||||
- name: Download verified image digests
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: image-digest-*
|
||||
path: dist/digests
|
||||
|
||||
@@ -33,14 +33,23 @@ jobs:
|
||||
...params,
|
||||
per_page: 100,
|
||||
});
|
||||
// 按 PR 作者识别官方机器人,维护者重开或编辑 PR 时仍使用同一判断
|
||||
const isDependabot = pr.user.login === 'dependabot[bot]'
|
||||
&& pr.user.type === 'Bot'
|
||||
&& pr.head.repo?.full_name === `${owner}/${repo}`;
|
||||
const isDependencyUpdate = file => isDependabot
|
||||
&& file.status === 'modified'
|
||||
&& (/^\.github\/workflows\/[^/]+\.ya?ml$/.test(file.filename)
|
||||
|| /^\.github\/actions\/.+\/(?:action\.ya?ml|Dockerfile)$/.test(file.filename));
|
||||
const isProtectedRule = path => typeof path === 'string'
|
||||
&& (path === 'AGENTS.md' || path.endsWith('/AGENTS.md')
|
||||
|| path === 'CONTRIBUTING.md'
|
||||
|| path === '.agents/skills' || path.startsWith('.agents/skills/')
|
||||
|| path === '.github' || path.startsWith('.github/'));
|
||||
|
||||
// 同时检查重命名前后的路径,避免通过移走规则文件绕过检查。
|
||||
if (!files.some(file => isProtectedRule(file.filename) || isProtectedRule(file.previous_filename))) {
|
||||
// 依赖更新只豁免已有文件的修改,重命名前后的受保护路径仍参与检查
|
||||
if (!files.some(file => !isDependencyUpdate(file)
|
||||
&& (isProtectedRule(file.filename) || isProtectedRule(file.previous_filename)))) {
|
||||
core.info('No protected contribution rule changes detected.');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
|
||||
+14
-5
@@ -21,13 +21,17 @@
|
||||
|
||||
### GitHub 回复口吻
|
||||
|
||||
AI 草拟或发布 Issue、PR 讨论及审查意见的回复时:
|
||||
AI 草拟、发布或修改 Issue、PR 讨论及审查意见的回复时,以一起排查问题、讨论方案的口吻交流:
|
||||
|
||||
- 用第一人称“我”,不用“我们”“本助手”代称自己
|
||||
- 围绕问题、事实和结论展开,能自然省略主语时省略;需要说明个人判断或行动归属时用“我”,避免反复以“我”起句或强调自己,不用“我们”“本助手”代称自己
|
||||
- 普通回复直接回应问题,不加问候语或称呼;向对方提问时称呼“佬”,同一条回复不重复称呼
|
||||
- 客气、委婉地回应具体问题,补充信息或提出不同意见时说明缘由,避免命令式、责备式表达
|
||||
- 先接住对方想完成的事和遇到的困难,再给出核实结果与建议;回复让对方感到问题被理解、有下一步可做,避免一上来归类、纠正或划定责任
|
||||
- 建议用商量的口吻,说明尝试某个操作能帮助判断什么,给对方留出选择空间;“可以试试”“方便的话”等表达随语境使用,整句仍需自然,不靠堆叠缓和词修饰命令
|
||||
- 回应已有能力、暂不支持或未采纳的建议时,结合场景说明可用入口或具体原因,有可行替代办法时一并说明,再交代必要边界;避免只回复“已有功能”“重复问题”“不在范围内”等结论
|
||||
- 需要补充材料或配合操作时,先利用已有信息,只询问仍缺少的部分,说明用途和最小范围;例如“佬,方便再观察几次吗?如果还出现,可以附一下对应请求的诊断包,便于对照终态用量”
|
||||
- 澄清误解或提出不同意见时,从观察和依据说起,照顾对方的使用环境与理解路径,避免责备、反问或居高临下的判断;已确认的事实保持明确,尚未确认的原因保留余地
|
||||
- 使用自然短句,简单回复用短段,并列信息较多时再分点;少用句号,避免客服套话、机械模板和空泛保证
|
||||
- 只把实际完成的排查、复现、修改和验证写成“我已……”;未采纳的意见说明依据,未确认、待处理或未验证的内容如实交代,说清结论、限制和下一步
|
||||
- 排查、复现、修改和验证的完成状态与实际一致;未采纳的意见说明依据,未确认、待处理或未验证的内容如实交代,说清结论、限制和下一步
|
||||
|
||||
## 问题与方案依据
|
||||
|
||||
@@ -85,7 +89,9 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名:
|
||||
- `.agents/skills/` 中的所有文件,包括技能入口、引用文档、配置和脚本
|
||||
- `.github/` 中的所有文件,包括工作流、Actions、Issue/PR 模板和依赖更新配置
|
||||
|
||||
调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
官方 `dependabot[bot]` 从本仓库分支发起的依赖更新可修改 `.github/workflows/` 中已有的 YAML 工作流,以及 `.github/actions/` 下已有的 `action.yml`、`action.yaml` 和 `Dockerfile`,继续执行正常审查与 CI;其他受保护文件以及新增、删除、重命名不在例外范围内
|
||||
|
||||
除此之外,调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
|
||||
源码变更遵守现有架构边界与迁移冻结规则,由对应 CI 检查执行;不额外设立整块业务目录的 PR 禁改清单。构建、测试和安全扫描复用现有工作流
|
||||
|
||||
@@ -143,6 +149,9 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名:
|
||||
| [插件使用](docs/plugins.md) | 如何安装、授权、配置和使用插件 | 管理员操作与影响,不展开 SDK 调用或宿主事务实现 |
|
||||
| [插件 SDK](backend/crates/gateway-plugin/sdk/README.md) / [CLI](backend/apps/plugin-cli/README.md) | 如何编写和打包插件 | 作者合同、权限边界、示例与打包验证 |
|
||||
| [数据库迁移](backend/migrations/README.md) | 如何维护 schema 和准备测试库 | 冻结规则、升级约束和测试环境,不罗列迁移历史 |
|
||||
| [共用组件](modules/ui/README.md)及其组件参考 | 如何安装、使用和开发独立 UI 库 | 组件公开 API、主题能力与库内开发;宿主页面约定归管理端主题,跨仓库联调归开发文档 |
|
||||
| [官方插件示例](modules/plugins/README.md)及其示例说明 | 如何构建、使用和维护示例插件 | 示例自身的功能、配置与验证;通用协议归 SDK,宿主安装操作归插件使用文档 |
|
||||
| [AGENTS.md](AGENTS.md) 与[项目技能](.agents/skills/cpr-dev-guide/SKILL.md) | AI 如何选择并执行任务流程 | 任务匹配、执行步骤和规范入口,不另建模块合同、版本表或协作标准 |
|
||||
| 本文 | 如何协作、审查和交付 | 流程、判断标准与验证要求,任务入口由 [开发技能](.agents/skills/cpr-dev-guide/SKILL.md) 串联 |
|
||||
|
||||
- 同一事实只在所属文档完整说明,其他位置按读者任务保留必要摘要和链接
|
||||
|
||||
@@ -80,7 +80,7 @@ INSTALL_DIR="$HOME/services/codex-proxy-rs" bash install.sh
|
||||
|
||||
部署完成后,打开 `http://127.0.0.1:8080`,使用 `admin@cpr.local` 和管理员密码登录
|
||||
|
||||
API Key 持有者可在登录页切换身份,查看自己的用量与额度,并通过「密钥配置」复制 Codex 配置或导入 CCSwitch
|
||||
API Key 持有者可在登录页切换身份,查看自己的用量与额度,并通过「密钥配置」复制 Codex 配置或导入 CC Switch
|
||||
|
||||
默认地址只能在服务器本机访问,从其他设备使用时,需要配置
|
||||
[HTTPS 反向代理](deploy/README.md#公网访问)
|
||||
@@ -93,7 +93,7 @@ API Key 持有者可在登录页切换身份,查看自己的用量与额度,
|
||||
|
||||
## 客户端接入
|
||||
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入,合并配置后重启 Codex
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,合并配置后重启 Codex;也可通过「导入到」把服务连接与密钥添加到 CC Switch
|
||||
|
||||
完整步骤、生图配置与排障见[客户端配置](deploy/README.md#客户端配置)
|
||||
|
||||
@@ -118,7 +118,8 @@ curl http://127.0.0.1:8080/v1/models \
|
||||
| 部署与使用 | [部署、备份与恢复](deploy/README.md) · [客户端接入与生图](deploy/README.md#客户端配置) |
|
||||
| 接口集成 | [API 参考](docs/api.md) · [模型定价](docs/api.md#模型定价) |
|
||||
| 使用插件 | [安装、配置与使用](docs/plugins.md) |
|
||||
| 开发插件 | [SDK 与合同](backend/crates/gateway-plugin/sdk/README.md) · [打包工具](backend/apps/plugin-cli/README.md) |
|
||||
| 开发插件 | [SDK 与合同](backend/crates/gateway-plugin/sdk/README.md) · [打包工具](backend/apps/plugin-cli/README.md) · [官方示例](modules/plugins/README.md) |
|
||||
| 开发共用组件 | [组件库说明](modules/ui/README.md) · [组件参考](modules/ui/playground/components/index.md) |
|
||||
| 开发宿主 | [开发与源码联调](docs/development.md) · [系统架构](docs/architecture.md) · [管理端主题](docs/theme.md) · [数据库迁移](backend/migrations/README.md) |
|
||||
| 参与协作 | [贡献、审查与验证](CONTRIBUTING.md) |
|
||||
|
||||
|
||||
Generated
+17
-12
@@ -791,9 +791,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chacha20"
|
||||
version = "0.10.1"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
|
||||
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
@@ -1189,9 +1189,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ctutils"
|
||||
version = "0.4.2"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e"
|
||||
checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
@@ -1759,6 +1759,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
"zip",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
@@ -1903,6 +1904,7 @@ dependencies = [
|
||||
name = "gateway-store"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
@@ -2251,8 +2253,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.20"
|
||||
source = "git+https://github.com/hyperium/hyper-util?rev=191b1acf0a6b18e807e4abf5a6d4a7c47979ecd6#191b1acf0a6b18e807e4abf5a6d4a7c47979ecd6"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"bytes",
|
||||
@@ -2260,6 +2263,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"hyper",
|
||||
"ipnet",
|
||||
"libc",
|
||||
@@ -3832,9 +3836,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
version = "0.103.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -5222,9 +5226,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.5"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
|
||||
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
@@ -5470,13 +5474,14 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
||||
|
||||
[[package]]
|
||||
name = "wnaf"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
|
||||
checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa"
|
||||
dependencies = [
|
||||
"ff",
|
||||
"group",
|
||||
"hybrid-array",
|
||||
"primefield",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -57,7 +57,5 @@ codegen-units = 1
|
||||
panic = "abort"
|
||||
|
||||
[patch.crates-io]
|
||||
# Upstream SOCKS IPv6 fix; remove once a release includes hyperium/hyper-util#302.
|
||||
hyper-util = { git = "https://github.com/hyperium/hyper-util", rev = "191b1acf0a6b18e807e4abf5a6d4a7c47979ecd6" }
|
||||
# 固定上游 BEGIN 取消回滚修复,避免未结束事务复用;见 transact-rs/sqlx#4394
|
||||
sqlx = { git = "https://github.com/transact-rs/sqlx", rev = "f1e94ec34dd5f11ad6488a3a8ac7988439a3f757" }
|
||||
|
||||
@@ -96,6 +96,10 @@ async fn execute(
|
||||
Some(plugin_runtime.policy_registry()),
|
||||
Some(plugin_runtime.execution_registry()),
|
||||
);
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_snapshots(&core.snapshots())
|
||||
);
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
or_shutdown!(plugin_runtime, management.bind(&plugin_runtime));
|
||||
let core = or_shutdown!(plugin_runtime, core.activate().await);
|
||||
|
||||
@@ -31,6 +31,10 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
Some(plugin_runtime.execution_registry()),
|
||||
Some(plugin_runtime.frontend_authentication_registry()),
|
||||
);
|
||||
or_shutdown!(
|
||||
plugin_runtime,
|
||||
plugin_runtime.bind_snapshots(&core.snapshots())
|
||||
);
|
||||
let management = ManagementPorts::new(&store, providers.admin.clone(), core.snapshot_control());
|
||||
or_shutdown!(plugin_runtime, management.bind(&plugin_runtime));
|
||||
let mut core = or_shutdown!(plugin_runtime, core.activate().await);
|
||||
|
||||
@@ -26,10 +26,15 @@ cpr-plugin package \
|
||||
--output-dir /path/to/output
|
||||
```
|
||||
|
||||
资源映射相对于作者清单所在目录解析。输出包括完整插件 ID、版本和目标 triple 命名的 `.tar.gz` 及 `.sha256`;归档根目录直接包含 `plugin.json`、可执行文件和已声明资源,不包含开发依赖
|
||||
`--resource-map web=web/dist` 将包内 `web/` 前缀映射到作者清单所在目录下的 `web/dist/`。
|
||||
可重复传入不同前缀,源目录必须位于项目内;没有对应资源时省略该参数,未使用的映射会被拒绝。
|
||||
未映射的资源按清单中的相对路径读取,只收集 `resources` 声明的文件
|
||||
|
||||
作者清单的省略规则、能力版本和挂载阶段见 [SDK 扩展项声明](../../crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)。
|
||||
CLI 与 SDK 的 `Manifest::from_author_slice`、`PluginBuilder::from_json` 共用规范化入口,归档只保留补全字段后的 `plugin.json`
|
||||
输出为 `<插件 ID>-<版本>-<目标 triple>.tar.gz` 及同名 `.tar.gz.sha256`;归档根目录直接包含 `plugin.json`、可执行文件和声明资源。
|
||||
输入清单及生成后的清单均不得超过 64 KiB,压缩包上限为 32 MiB,展开归档上限为 128 MiB
|
||||
|
||||
作者清单不填写 `package`;字段省略规则、能力版本和挂载阶段见 [SDK 扩展项声明](../../crates/gateway-plugin/sdk/docs/manifest.md#扩展项简写)。
|
||||
CLI 生成规范化后的 `plugin.json`,补入协议版本、目标平台和文件摘要。打包成功只证明输入符合包合同,实际启动与能力仍需在目标宿主验证
|
||||
|
||||
支持 `x86_64-unknown-linux-gnu`、`aarch64-unknown-linux-gnu` 和 `aarch64-apple-darwin`。
|
||||
完整开发示例在独立仓库 `codex-proxy-plugins` 的 `examples/workbench`,可通过[子模块](../../../docs/development.md#源码联调)检出;本工具不负责源码构建、上传或启用插件
|
||||
|
||||
@@ -11,7 +11,7 @@ use chrono::{DateTime, Utc};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
task::{DaemonTask, WorkerDaemonContext, WorkerTaskError},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -75,18 +75,20 @@ impl BackupTask {
|
||||
}
|
||||
|
||||
impl DaemonTask for BackupTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move { self.run_daemon(cancellation).await })
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move { self.run_daemon(context).await })
|
||||
}
|
||||
}
|
||||
|
||||
impl BackupTask {
|
||||
async fn run_daemon(&self, cancellation: CancellationToken) -> Result<(), WorkerTaskError> {
|
||||
async fn run_daemon(&self, context: WorkerDaemonContext) -> Result<(), WorkerTaskError> {
|
||||
let cancellation = context.cancellation();
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_cycle(&cancellation).await?;
|
||||
self.run_cycle(cancellation).await?;
|
||||
context.succeeded();
|
||||
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Ok(()),
|
||||
@@ -647,7 +649,7 @@ impl BackupTask {
|
||||
for record in expired {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&record.id, now)
|
||||
.transition_to_deleting(&record.id, now, None)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
@@ -671,7 +673,7 @@ impl BackupTask {
|
||||
for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) {
|
||||
let Some(record) = self
|
||||
.repository
|
||||
.transition_to_deleting(&decision.record_id, now)
|
||||
.transition_to_deleting(&decision.record_id, now, None)
|
||||
.await
|
||||
.map_err(repo_error)?
|
||||
else {
|
||||
@@ -706,6 +708,10 @@ impl BackupTask {
|
||||
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
|
||||
match error.code() {
|
||||
code::CANCELLED => (code::CANCELLED, "备份已取消"),
|
||||
code::ANALYTICS_NOT_READY => (
|
||||
code::ANALYTICS_NOT_READY,
|
||||
"用量统计正在重建,请在统计就绪后重试备份",
|
||||
),
|
||||
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
|
||||
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
|
||||
}
|
||||
|
||||
@@ -123,6 +123,7 @@ pub struct AccountRecord {
|
||||
pub concurrency_limit: Option<AccountConcurrencyLimit>,
|
||||
pub weight: AccountWeight,
|
||||
pub model_access: gateway_core::account::AccountModelAccess,
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
pub outbound_proxy: Option<gateway_core::account::OutboundProxy>,
|
||||
pub outbound_proxy_enabled: bool,
|
||||
pub credential_state: CredentialState,
|
||||
|
||||
@@ -19,6 +19,10 @@ pub enum MutationAuditOperation {
|
||||
ProviderAccountUpdate,
|
||||
BackupStorageUpdate,
|
||||
BackupScheduleUpdate,
|
||||
BackupCreate,
|
||||
BackupDelete,
|
||||
BackupConnectionTest,
|
||||
BackupDownloadUrl,
|
||||
ProviderAccountAdaptConcurrency,
|
||||
ProviderAccountRecover,
|
||||
ProviderAccountBatchUpdate,
|
||||
@@ -76,6 +80,10 @@ impl MutationAuditOperation {
|
||||
Self::ProviderAccountUpdate => ("update", "provider_account"),
|
||||
Self::BackupStorageUpdate => ("backup.s3_config_updated", "backup_settings"),
|
||||
Self::BackupScheduleUpdate => ("backup.schedule_updated", "backup_settings"),
|
||||
Self::BackupCreate => ("backup.created", "backup"),
|
||||
Self::BackupDelete => ("backup.delete_requested", "backup"),
|
||||
Self::BackupConnectionTest => ("backup.s3_connection_tested", "backup_settings"),
|
||||
Self::BackupDownloadUrl => ("backup.download_url_created", "backup"),
|
||||
Self::ProviderAccountAdaptConcurrency => ("adapt_concurrency", "provider_account"),
|
||||
Self::ProviderAccountRecover => ("recover", "provider_account"),
|
||||
Self::ProviderAccountBatchUpdate => ("batch_update", "provider_account"),
|
||||
|
||||
@@ -15,6 +15,8 @@ pub const STATE_CONFLICT: &str = "backup.state_conflict";
|
||||
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
|
||||
/// `pg_dump` 非零退出或失败
|
||||
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
|
||||
/// 用量汇总尚未完整发布
|
||||
pub const ANALYTICS_NOT_READY: &str = "backup.analytics_not_ready";
|
||||
/// 暂存磁盘空间不足
|
||||
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
|
||||
/// S3 认证失败
|
||||
|
||||
@@ -499,7 +499,7 @@ impl BackupError {
|
||||
/// 构造稳定错误码常量
|
||||
pub mod code;
|
||||
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.sql.gz`
|
||||
///
|
||||
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
|
||||
/// 输出统一使用 `/` 分隔且不含空段
|
||||
@@ -510,7 +510,7 @@ pub fn build_object_key(
|
||||
) -> Result<String, BackupError> {
|
||||
let prefix = normalize_prefix(prefix)?;
|
||||
Ok(format!(
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.sql.gz",
|
||||
at.format("%Y"),
|
||||
at.format("%m"),
|
||||
at.format("%d"),
|
||||
@@ -519,10 +519,10 @@ pub fn build_object_key(
|
||||
))
|
||||
}
|
||||
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.sql.gz`,与对象 key 的文件名一致
|
||||
#[must_use]
|
||||
pub fn build_download_file_name(backup_id: &str) -> String {
|
||||
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
|
||||
format!("codex-proxy-rs_{}.sql.gz", short_backup_id(backup_id))
|
||||
}
|
||||
|
||||
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex
|
||||
|
||||
@@ -157,12 +157,15 @@ impl OtherRequestOutcome {
|
||||
/// 用量记录过滤条件
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
|
||||
pub struct UsageFilter {
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub provider_account_name: Option<String>,
|
||||
pub client_api_key_ref: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub provider_account_ref: Option<String>,
|
||||
pub operation: Option<String>,
|
||||
pub provider_kind: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub model_contains: Option<String>,
|
||||
pub outcome: Option<RequestOutcome>,
|
||||
pub status_code: Option<u16>,
|
||||
pub transport: Option<String>,
|
||||
@@ -506,6 +509,46 @@ pub struct RequestMetrics {
|
||||
pub cache_hit_request_count: u64,
|
||||
}
|
||||
|
||||
/// 摘要实际消费的请求计数与延迟样本,不承诺分位数或分布
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct RequestSummaryMetrics {
|
||||
pub request_count: u64,
|
||||
pub success_count: u64,
|
||||
pub failure_count: u64,
|
||||
pub cancelled_count: u64,
|
||||
pub incomplete_count: u64,
|
||||
pub caller_error_count: u64,
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cached_input_tokens: u64,
|
||||
pub cache_write_input_tokens: u64,
|
||||
pub reasoning_output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
pub latency_sum_ms: u64,
|
||||
pub latency_count: u64,
|
||||
}
|
||||
|
||||
impl From<&RequestMetrics> for RequestSummaryMetrics {
|
||||
fn from(metrics: &RequestMetrics) -> Self {
|
||||
Self {
|
||||
request_count: metrics.request_count,
|
||||
success_count: metrics.success_count,
|
||||
failure_count: metrics.failure_count,
|
||||
cancelled_count: metrics.cancelled_count,
|
||||
incomplete_count: metrics.incomplete_count,
|
||||
caller_error_count: metrics.caller_error_count,
|
||||
input_tokens: metrics.input_tokens,
|
||||
output_tokens: metrics.output_tokens,
|
||||
cached_input_tokens: metrics.cached_input_tokens,
|
||||
cache_write_input_tokens: metrics.cache_write_input_tokens,
|
||||
reasoning_output_tokens: metrics.reasoning_output_tokens,
|
||||
total_tokens: metrics.total_tokens,
|
||||
latency_sum_ms: metrics.latency_sum_ms,
|
||||
latency_count: metrics.latency_count,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 上游 attempt 聚合指标
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct AttemptMetrics {
|
||||
@@ -893,10 +936,19 @@ pub struct UsageOverview {
|
||||
pub providers: Vec<ProviderObservation>,
|
||||
}
|
||||
|
||||
/// 用量摘要所需的窄聚合事实
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageSummaryObservation {
|
||||
pub analytics: AnalyticsMetadata,
|
||||
pub range: TimeRange,
|
||||
pub requests: RequestSummaryMetrics,
|
||||
pub attempts: AttemptMetrics,
|
||||
}
|
||||
|
||||
/// 用量摘要的用例结果
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct UsageSummary {
|
||||
pub overview: UsageOverview,
|
||||
pub observation: UsageSummaryObservation,
|
||||
pub average_latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
@@ -1138,10 +1190,10 @@ pub struct DashboardWireProfile {
|
||||
pub release: Option<DashboardDesktopRelease>,
|
||||
}
|
||||
|
||||
/// Provider 画像的差异字段;展示层只显示标签和值,不解释 Provider 语义
|
||||
/// Provider 画像的差异字段,以稳定标识传递语义,名称由前端本地化
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DashboardWireAttribute {
|
||||
pub label: String,
|
||||
pub key: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
|
||||
@@ -820,6 +820,13 @@ pub struct ProviderQuotaRequest {
|
||||
pub rolling_usage: Option<AccountUsage>,
|
||||
}
|
||||
|
||||
/// 当前目录页已读取的账号事实;批量投影不触发上游刷新
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ProviderQuotaReadRequest<'a> {
|
||||
pub account: &'a AccountRecord,
|
||||
pub rolling_usage: Option<&'a AccountUsage>,
|
||||
}
|
||||
|
||||
/// Provider 已解析的 quota 结果及其不透明差异字段
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProviderQuota {
|
||||
|
||||
@@ -66,8 +66,12 @@ pub trait BackupRepository: Send + Sync {
|
||||
at: DateTime<Utc>,
|
||||
) -> AdminStoreResult<bool>;
|
||||
|
||||
/// 插入 queued 任务;存在活跃任务时返回 `backup.active_task_conflict`
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord>;
|
||||
/// 同事务插入人工 queued 任务与审计;存在活跃任务时返回 `backup.active_task_conflict`
|
||||
async fn insert_backup_record(
|
||||
&self,
|
||||
seed: BackupRecordSeed,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupRecord>;
|
||||
|
||||
/// 核对启停、Cron、时区及旧游标后,同事务推进游标并插入 scheduled 任务
|
||||
/// 配置已变化时不写入;同一时间点或活跃冲突时只推进游标,返回 `false`
|
||||
@@ -111,11 +115,12 @@ pub trait BackupRepository: Send + Sync {
|
||||
now: DateTime<Utc>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 迁移到删除流程(completed/failed → deleting);供管理员与保留策略调用
|
||||
/// 迁移到删除流程(completed/failed → deleting);人工操作同事务记录审计,保留策略传 None
|
||||
async fn transition_to_deleting(
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
context: Option<&MutationContext>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>>;
|
||||
|
||||
/// 远端删除成功后硬删除记录行
|
||||
@@ -313,6 +318,7 @@ impl BackupRepository for UnavailableBackupRepository {
|
||||
async fn insert_backup_record(
|
||||
&self,
|
||||
_seed: BackupRecordSeed,
|
||||
_context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupRecord> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
@@ -363,6 +369,7 @@ impl BackupRepository for UnavailableBackupRepository {
|
||||
&self,
|
||||
_id: &str,
|
||||
_now: DateTime<Utc>,
|
||||
_context: Option<&MutationContext>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
Err(disabled_store())
|
||||
}
|
||||
|
||||
@@ -18,8 +18,9 @@ use crate::model::provider_credentials::{
|
||||
PrepareCredentialImport, PrepareCredentialRefresh, PrepareCredentialRotation,
|
||||
PreparedAuthorizationCommit, PreparedCredentialImport, PreparedCredentialRotation,
|
||||
ProviderExport, ProviderExportCredentialInput, ProviderModelCatalogDocument, ProviderModels,
|
||||
ProviderProfileAvatar, ProviderProfileStatistics, ProviderQuota, ProviderQuotaRequest,
|
||||
ProviderResetCreditResult, ProviderResetCredits, ProviderSubscription, explicit_plan_type,
|
||||
ProviderProfileAvatar, ProviderProfileStatistics, ProviderQuota, ProviderQuotaReadRequest,
|
||||
ProviderQuotaRequest, ProviderResetCreditResult, ProviderResetCredits, ProviderSubscription,
|
||||
explicit_plan_type,
|
||||
};
|
||||
use crate::model::{
|
||||
provider_credentials::{ProviderDocument, ProviderQuotaWindow},
|
||||
@@ -289,6 +290,25 @@ pub trait ProviderAdmin: Send + Sync {
|
||||
request: ProviderQuotaRequest,
|
||||
) -> Result<ProviderQuota, ProviderAdminError>;
|
||||
|
||||
/// 只读投影当前页额度,返回值与输入逐项对应,单个失败不影响其他账号
|
||||
/// 没有批量存储能力的 Provider 继续使用原有单账号语义
|
||||
async fn read_quotas(
|
||||
&self,
|
||||
requests: &[ProviderQuotaReadRequest<'_>],
|
||||
) -> Vec<Result<ProviderQuota, ProviderAdminError>> {
|
||||
futures::future::join_all(requests.iter().map(|request| async {
|
||||
let account_id = ProviderAccountId::new(request.account.id.clone())
|
||||
.map_err(|_| ProviderAdminError::new(ProviderAdminErrorKind::Invalid))?;
|
||||
self.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: request.rolling_usage.cloned(),
|
||||
})
|
||||
.await
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
/// 历史观测的协议字段仅由具体 Provider 解释;不支持时保留累计估算
|
||||
fn quota_forecast_observation(
|
||||
&self,
|
||||
|
||||
@@ -30,7 +30,7 @@ use crate::model::{
|
||||
DashboardObservation, DashboardQuery, DiagnosticDimension, DiagnosticsObservation,
|
||||
Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint, TimeRange,
|
||||
UsageBillingProjection, UsageCalculatedBillingFact, UsageDetail, UsageFilter,
|
||||
UsageInsightsObservation, UsageOverview, UsagePage, UsageQuery,
|
||||
UsageInsightsObservation, UsagePage, UsageQuery, UsageSummaryObservation,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, CredentialDetails, CredentialImportCommit, CredentialImportResult,
|
||||
@@ -485,7 +485,7 @@ pub trait ObservabilityStore: Send + Sync {
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview>;
|
||||
) -> AdminStoreResult<UsageSummaryObservation>;
|
||||
|
||||
async fn usage_diagnostics(
|
||||
&self,
|
||||
|
||||
@@ -114,10 +114,14 @@ impl DefaultAccountGroupService {
|
||||
result: Result<AccountGroupMutation, crate::ports::store::AdminStoreError>,
|
||||
) -> Result<AccountGroupMutation, AdminError> {
|
||||
let mut result = result.map_err(|error| map_store_error(error, "account group"))?;
|
||||
if let Some(record) = result.record.as_mut() {
|
||||
self.enrich_records(std::slice::from_mut(record)).await?;
|
||||
}
|
||||
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
|
||||
if let Some(record) = result.record.as_mut()
|
||||
&& let Err(error) = self.enrich_records(std::slice::from_mut(record)).await
|
||||
{
|
||||
tracing::warn!(group_id = %result.id.as_str(), error = %error, "committed account group runtime projection is unavailable");
|
||||
// record 可缺省;不要用未完成的容量投影冒充有效展示数据。
|
||||
result.record = None;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,8 +30,9 @@ use crate::{
|
||||
AccountDirectoryItem, AccountDirectoryPage, AccountExportBundle, AccountPersonalInfo,
|
||||
AccountRefreshResult, AccountUsagePeriod, ConsumeProviderResetCredit,
|
||||
PrepareCredentialRefresh, ProviderModelCatalogDocument, ProviderModels,
|
||||
ProviderProfileAvatar, ProviderQuota, ProviderQuotaRequest, ProviderQuotaWindow,
|
||||
ProviderResetCreditResult, ProviderResetCredits, QuotaLocalUsageAttribution,
|
||||
ProviderProfileAvatar, ProviderQuota, ProviderQuotaReadRequest, ProviderQuotaRequest,
|
||||
ProviderQuotaWindow, ProviderResetCreditResult, ProviderResetCredits,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
quota_forecast::{
|
||||
AccountQuotaForecastReport, account_quota_forecasts, quota_forecast_source_window,
|
||||
@@ -236,6 +237,78 @@ impl DefaultAccountsService {
|
||||
)
|
||||
}
|
||||
|
||||
async fn read_quotas(
|
||||
providers: &ProviderAdminRegistry,
|
||||
items: &[AccountPageItem],
|
||||
rolling_usage: &BTreeMap<String, AccountUsage>,
|
||||
) -> Vec<ProviderQuota> {
|
||||
let mut groups = BTreeMap::<_, Vec<usize>>::new();
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
groups
|
||||
.entry(&item.account.provider_kind)
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
let batches = groups.into_iter().map(|(kind, indices)| async move {
|
||||
let provider = match providers.require(kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
provider = %kind,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let requests = indices
|
||||
.iter()
|
||||
.map(|&index| {
|
||||
let account = &items[index].account;
|
||||
ProviderQuotaReadRequest {
|
||||
account,
|
||||
rolling_usage: rolling_usage.get(&account.id),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let results = provider.read_quotas(&requests).await;
|
||||
if results.len() != indices.len() {
|
||||
tracing::warn!(
|
||||
provider = %kind,
|
||||
expected = indices.len(),
|
||||
actual = results.len(),
|
||||
"account directory quota batch has invalid length; showing empty quota"
|
||||
);
|
||||
return Vec::new();
|
||||
}
|
||||
indices
|
||||
.into_iter()
|
||||
.zip(results)
|
||||
.map(|(index, result)| {
|
||||
let quota = result.unwrap_or_else(|error| {
|
||||
tracing::warn!(
|
||||
account_id = %items[index].account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
empty_quota()
|
||||
});
|
||||
(index, quota)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
// Provider 可独立完成,回填原页顺序并保留未注册或失败账号的位置
|
||||
let mut quotas = vec![empty_quota(); items.len()];
|
||||
for (index, quota) in futures::future::join_all(batches)
|
||||
.await
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
quotas[index] = quota;
|
||||
}
|
||||
quotas
|
||||
}
|
||||
|
||||
async fn load_account(
|
||||
&self,
|
||||
account_id: &ProviderAccountId,
|
||||
@@ -487,45 +560,7 @@ impl AccountsService for DefaultAccountsService {
|
||||
.into_iter()
|
||||
.map(|usage| (usage.account_id.clone(), usage))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut quotas = futures::future::join_all(page.items.iter().map(|item| async {
|
||||
let account = &item.account;
|
||||
let account_id = ProviderAccountId::new(account.id.clone())
|
||||
.map_err(|_| AdminError::invalid("Provider 账号 ID 不合法"))?;
|
||||
// 单个账号的 quota 投影失败(Provider 未注册或 quota 读取失败)不拖垮整页:
|
||||
// 该账号降级为空额度投影,其余账号与页面状态照常返回
|
||||
let provider = match providers.require(&account.provider_kind) {
|
||||
Ok(provider) => provider,
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory provider is not registered; showing empty quota"
|
||||
);
|
||||
return Ok(empty_quota());
|
||||
}
|
||||
};
|
||||
match provider
|
||||
.quota(ProviderQuotaRequest {
|
||||
account_id,
|
||||
refresh: false,
|
||||
rolling_usage: rolling_usage.get(&account.id).cloned(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(quota) => Ok(quota),
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
account_id = %account.id,
|
||||
error = %error,
|
||||
"account directory quota projection failed; showing empty quota"
|
||||
);
|
||||
Ok(empty_quota())
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>, AdminError>>()?;
|
||||
let mut quotas = Self::read_quotas(providers, &page.items, &rolling_usage).await;
|
||||
self.attach_quota_local_usage(&page.items, &mut quotas)
|
||||
.await?;
|
||||
let mut api_key_usage = self.load_api_key_usage(&page.items).await?;
|
||||
|
||||
@@ -7,11 +7,10 @@ use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use gateway_core::runtime::SnapshotControl;
|
||||
use secrecy::ExposeSecret as _;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind, MutationContext,
|
||||
auth::{AdminAuditEvent, AuditActorKind},
|
||||
audit::{MutationAuditIntent, MutationAuditOperation},
|
||||
backup::{
|
||||
BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings,
|
||||
BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult,
|
||||
@@ -180,8 +179,8 @@ impl BackupService for DefaultBackupService {
|
||||
}
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.s3_connection_tested",
|
||||
"backup_settings",
|
||||
MutationAuditOperation::BackupConnectionTest,
|
||||
"1",
|
||||
vec![],
|
||||
)
|
||||
.await?;
|
||||
@@ -219,11 +218,9 @@ impl BackupService for DefaultBackupService {
|
||||
.map_err(map_backup_error)?;
|
||||
let record = self
|
||||
.repository
|
||||
.insert_backup_record(seed)
|
||||
.insert_backup_record(seed, context)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?;
|
||||
self.append_audit(context, "backup.created", "backup", vec![record.id.clone()])
|
||||
.await?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -264,9 +261,9 @@ impl BackupService for DefaultBackupService {
|
||||
.map_err(map_backup_error)?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.download_url_created",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
MutationAuditOperation::BackupDownloadUrl,
|
||||
backup_id,
|
||||
vec![],
|
||||
)
|
||||
.await?;
|
||||
Ok(DownloadUrlResult {
|
||||
@@ -290,17 +287,10 @@ impl BackupService for DefaultBackupService {
|
||||
let now = Utc::now();
|
||||
let deleting = self
|
||||
.repository
|
||||
.transition_to_deleting(backup_id, now)
|
||||
.transition_to_deleting(backup_id, now, Some(context))
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "backup record"))?
|
||||
.ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?;
|
||||
self.append_audit(
|
||||
context,
|
||||
"backup.delete_requested",
|
||||
"backup",
|
||||
vec![backup_id.to_owned()],
|
||||
)
|
||||
.await?;
|
||||
Ok(deleting)
|
||||
}
|
||||
}
|
||||
@@ -317,41 +307,15 @@ impl DefaultBackupService {
|
||||
async fn append_audit(
|
||||
&self,
|
||||
context: &MutationContext,
|
||||
action: &str,
|
||||
operation: MutationAuditOperation,
|
||||
entity_ref: &str,
|
||||
changed_fields: Vec<String>,
|
||||
) -> Result<(), AdminError> {
|
||||
let actor_kind = match context.actor {
|
||||
crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession,
|
||||
crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey,
|
||||
crate::model::MutationActor::System => AuditActorKind::System,
|
||||
};
|
||||
let actor_ref = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => {
|
||||
crate::model::auth::admin_session_actor_ref(admin_user_id)
|
||||
}
|
||||
crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(),
|
||||
crate::model::MutationActor::System => "system".to_owned(),
|
||||
};
|
||||
let actor_admin_user_id = match &context.actor {
|
||||
crate::model::MutationActor::AdminSession { admin_user_id } => {
|
||||
Some(admin_user_id.clone())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let event = AdminAuditEvent {
|
||||
id: format!("audit_{}", Uuid::now_v7().simple()),
|
||||
actor_kind,
|
||||
actor_admin_user_id,
|
||||
actor_ref,
|
||||
request_id: Some(context.request_id.clone()),
|
||||
action: action.to_owned(),
|
||||
entity_kind: "backup".to_owned(),
|
||||
entity_ref: entity_ref.to_owned(),
|
||||
config_revision: None,
|
||||
changed_fields,
|
||||
occurred_at: Utc::now(),
|
||||
};
|
||||
let event = MutationAuditIntent {
|
||||
operation,
|
||||
entity_ref,
|
||||
}
|
||||
.event(context, changed_fields);
|
||||
self.auth
|
||||
.append_audit_event(event)
|
||||
.await
|
||||
|
||||
@@ -9,10 +9,7 @@ use std::{
|
||||
|
||||
use chrono::Utc;
|
||||
use futures::{FutureExt as _, StreamExt as _, future::BoxFuture, stream::FuturesUnordered};
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
task::{DaemonTask, WorkerTaskError},
|
||||
};
|
||||
use gateway_core::task::{DaemonTask, WorkerDaemonContext, WorkerTaskError};
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -332,8 +329,9 @@ impl ImportTasksService for DefaultImportTasksService {
|
||||
pub(crate) struct ImportTaskWorker(pub Arc<DefaultImportTasksService>);
|
||||
|
||||
impl DaemonTask for ImportTaskWorker {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let cancellation = context.cancellation();
|
||||
let mut active = FuturesUnordered::new();
|
||||
// 固定一个取消 future,避免循环反复注册中立 CancellationToken 的 waiter
|
||||
let cancelled = cancellation.cancelled();
|
||||
@@ -354,6 +352,7 @@ impl DaemonTask for ImportTaskWorker {
|
||||
};
|
||||
active.push(self.0.execute(id, index, input));
|
||||
}
|
||||
context.succeeded();
|
||||
tokio::select! {
|
||||
() = &mut cancelled => {},
|
||||
_ = active.next(), if !active.is_empty() => {},
|
||||
|
||||
@@ -106,7 +106,7 @@ impl DefaultKeyUsageService {
|
||||
fn usage_filter(id: &ClientApiKeyId, model: Option<String>) -> UsageFilter {
|
||||
UsageFilter {
|
||||
client_api_key_ref: Some(id.as_str().to_owned()),
|
||||
model,
|
||||
model_contains: model,
|
||||
..UsageFilter::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
//! 集中归纳已采集证据的缺口,不从缺失事件推断业务结果
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::model::diagnostics::DiagnosticBundle;
|
||||
|
||||
use super::gap;
|
||||
|
||||
pub(super) fn trace_gaps(bundle: &mut DiagnosticBundle, section: &str, trace: &Value) {
|
||||
let trace_section = format!("{section}.trace");
|
||||
for (field, reason) in [
|
||||
("droppedEvents", "events_evicted"),
|
||||
("droppedEvidenceEvents", "critical_evidence_evicted"),
|
||||
("stageCountOverflow", "stage_coverage_incomplete"),
|
||||
] {
|
||||
if trace[field].as_u64().unwrap_or(0) > 0 {
|
||||
gap(bundle, &trace_section, reason);
|
||||
}
|
||||
}
|
||||
if !trace["stageCounts"].is_object() {
|
||||
gap(bundle, &trace_section, "stage_coverage_not_recorded");
|
||||
}
|
||||
let events = trace["events"].as_array();
|
||||
if !events.is_some_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
event["stage"] == "request.routed" && event["data"]["accountPolicy"].is_object()
|
||||
})
|
||||
}) {
|
||||
gap(bundle, section, "incident_policy_not_recorded");
|
||||
}
|
||||
for (index, event) in events.into_iter().flatten().enumerate() {
|
||||
let event_section = format!("{trace_section}.events[{index}]");
|
||||
let data = &event["data"];
|
||||
for (missing, reason) in [
|
||||
(
|
||||
event["stage"] == "provider.response.parse_gap",
|
||||
"response_parse_failed",
|
||||
),
|
||||
(data["jsonValid"] == false, "json_parse_failed"),
|
||||
(data["truncated"] == true, "event_data_truncated"),
|
||||
(
|
||||
data["input"]["omittedItems"].as_u64().unwrap_or(0) > 0,
|
||||
"input_prefix_sample_limited",
|
||||
),
|
||||
(
|
||||
data["correlation"].as_object().is_some_and(|headers| {
|
||||
headers
|
||||
.values()
|
||||
.any(|header| header["omittedValues"].as_u64().unwrap_or(0) > 0)
|
||||
}),
|
||||
"correlation_header_sample_limited",
|
||||
),
|
||||
] {
|
||||
if missing {
|
||||
gap(bundle, &event_section, reason);
|
||||
}
|
||||
}
|
||||
if event["stage"] == "capture.gap" || data["truncated"] == true {
|
||||
gap(bundle, &trace_section, "capture_gap_or_truncation");
|
||||
}
|
||||
if event["stage"] == "retry.decided" && data["phase"].is_null() {
|
||||
gap(bundle, &trace_section, "legacy_retry_facts_not_recorded");
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-32
@@ -7,6 +7,8 @@ use std::{
|
||||
|
||||
use tokio::time::{Instant, timeout_at};
|
||||
|
||||
mod availability;
|
||||
|
||||
use super::{DefaultObservabilityService, map_store_error};
|
||||
use crate::model::{
|
||||
AdminError, AdminErrorKind,
|
||||
@@ -150,38 +152,7 @@ impl DefaultObservabilityService {
|
||||
&format!("{section}.trace"),
|
||||
"not_recorded_or_retained",
|
||||
),
|
||||
Some(trace) => {
|
||||
if trace["droppedEvents"].as_u64().unwrap_or(0) > 0 {
|
||||
gap(&mut bundle, &format!("{section}.trace"), "events_evicted");
|
||||
}
|
||||
let events = trace["events"].as_array();
|
||||
if !events.is_some_and(|events| {
|
||||
events.iter().any(|event| {
|
||||
event["stage"] == "request.routed"
|
||||
&& event["data"]["accountPolicy"].is_object()
|
||||
})
|
||||
}) {
|
||||
gap(&mut bundle, §ion, "incident_policy_not_recorded");
|
||||
}
|
||||
for event in events.into_iter().flatten() {
|
||||
if event["stage"] == "capture.gap" || event["data"]["truncated"] == true
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"capture_gap_or_truncation",
|
||||
);
|
||||
}
|
||||
if event["stage"] == "retry.decided" && event["data"]["phase"].is_null()
|
||||
{
|
||||
gap(
|
||||
&mut bundle,
|
||||
&format!("{section}.trace"),
|
||||
"legacy_retry_facts_not_recorded",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(trace) => availability::trace_gaps(&mut bundle, §ion, trace),
|
||||
}
|
||||
gap(
|
||||
&mut bundle,
|
||||
@@ -267,17 +267,17 @@ impl ObservabilityService for DefaultObservabilityService {
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> Result<UsageSummary, AdminError> {
|
||||
let overview = self
|
||||
let observation = self
|
||||
.store
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.map_err(|error| map_store_error(error, "usage summary"))?;
|
||||
let average_latency_ms = average(
|
||||
overview.requests.latency_sum_ms,
|
||||
overview.requests.latency_count,
|
||||
observation.requests.latency_sum_ms,
|
||||
observation.requests.latency_count,
|
||||
);
|
||||
Ok(UsageSummary {
|
||||
overview,
|
||||
observation,
|
||||
average_latency_ms,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl AnalyticsValue for DashboardObservation {
|
||||
&mut self.analytics
|
||||
}
|
||||
}
|
||||
impl AnalyticsValue for UsageOverview {
|
||||
impl AnalyticsValue for UsageSummaryObservation {
|
||||
fn metadata(&mut self) -> &mut AnalyticsMetadata {
|
||||
&mut self.analytics
|
||||
}
|
||||
@@ -177,7 +177,7 @@ type UsageKey = (TimeRange, UsageFilter);
|
||||
pub(crate) struct CachedObservabilityStore {
|
||||
inner: Arc<dyn ObservabilityStore>,
|
||||
dashboard: Cache<DashboardQuery, DashboardObservation>,
|
||||
summaries: Cache<UsageKey, UsageOverview>,
|
||||
summaries: Cache<UsageKey, UsageSummaryObservation>,
|
||||
insights: Cache<(UsageKey, Granularity), UsageInsightsObservation>,
|
||||
diagnostics: Cache<(UsageKey, DiagnosticDimension, u16), DiagnosticsObservation>,
|
||||
}
|
||||
@@ -254,7 +254,7 @@ impl ObservabilityStore for CachedObservabilityStore {
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
) -> AdminStoreResult<UsageSummaryObservation> {
|
||||
self.summaries
|
||||
.get((range, filter.clone()), self.inner.as_ref(), TTL, || {
|
||||
self.inner.usage_summary(range, filter)
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
//! 供 backup::task 与 use_case::backup 的测试复用
|
||||
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -91,7 +92,9 @@ pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord {
|
||||
|
||||
/// 内存版备份仓储
|
||||
pub(crate) struct FakeBackupRepository {
|
||||
audits: Mutex<Vec<AdminAuditEvent>>,
|
||||
settings: Mutex<BackupSettings>,
|
||||
settings_unavailable: AtomicBool,
|
||||
records: Mutex<Vec<BackupRecord>>,
|
||||
}
|
||||
|
||||
@@ -99,10 +102,21 @@ impl FakeBackupRepository {
|
||||
pub(crate) fn new(settings: BackupSettings) -> Self {
|
||||
Self {
|
||||
settings: Mutex::new(settings),
|
||||
audits: Mutex::new(Vec::new()),
|
||||
settings_unavailable: AtomicBool::new(false),
|
||||
records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn audit_events(&self) -> Vec<AdminAuditEvent> {
|
||||
self.audits.lock().expect("audits").clone()
|
||||
}
|
||||
|
||||
pub(crate) fn set_settings_unavailable(&self, unavailable: bool) {
|
||||
self.settings_unavailable
|
||||
.store(unavailable, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// 测试辅助:读取当前全部记录
|
||||
pub(crate) fn all_records(&self) -> Vec<BackupRecord> {
|
||||
self.records.lock().expect("records").clone()
|
||||
@@ -134,6 +148,13 @@ impl FakeBackupRepository {
|
||||
#[async_trait]
|
||||
impl BackupRepository for FakeBackupRepository {
|
||||
async fn load_settings(&self) -> AdminStoreResult<BackupSettings> {
|
||||
if self.settings_unavailable.load(Ordering::SeqCst) {
|
||||
return Err(AdminStoreError::new(
|
||||
AdminStoreErrorKind::Unavailable,
|
||||
"backup",
|
||||
"test store unavailable",
|
||||
));
|
||||
}
|
||||
Ok(self.settings.lock().expect("settings").clone())
|
||||
}
|
||||
|
||||
@@ -192,7 +213,11 @@ impl BackupRepository for FakeBackupRepository {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult<BackupRecord> {
|
||||
async fn insert_backup_record(
|
||||
&self,
|
||||
seed: BackupRecordSeed,
|
||||
context: &MutationContext,
|
||||
) -> AdminStoreResult<BackupRecord> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
if records.iter().any(|record| record.status.is_active()) {
|
||||
return Err(AdminStoreError::new(
|
||||
@@ -203,6 +228,13 @@ impl BackupRepository for FakeBackupRepository {
|
||||
}
|
||||
let record = queued_record(&seed);
|
||||
records.push(record.clone());
|
||||
self.audits.lock().expect("audits").push(
|
||||
gateway_admin::model::audit::MutationAuditIntent {
|
||||
operation: gateway_admin::model::audit::MutationAuditOperation::BackupCreate,
|
||||
entity_ref: &record.id,
|
||||
}
|
||||
.event(context, vec!["status".to_owned()]),
|
||||
);
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -386,6 +418,7 @@ impl BackupRepository for FakeBackupRepository {
|
||||
&self,
|
||||
id: &str,
|
||||
now: DateTime<Utc>,
|
||||
context: Option<&MutationContext>,
|
||||
) -> AdminStoreResult<Option<BackupRecord>> {
|
||||
let mut records = self.records.lock().expect("records");
|
||||
let Some(record) = records
|
||||
@@ -396,6 +429,15 @@ impl BackupRepository for FakeBackupRepository {
|
||||
};
|
||||
record.status = BackupStatus::Deleting;
|
||||
record.updated_at = now;
|
||||
if let Some(context) = context {
|
||||
self.audits.lock().expect("audits").push(
|
||||
gateway_admin::model::audit::MutationAuditIntent {
|
||||
operation: gateway_admin::model::audit::MutationAuditOperation::BackupDelete,
|
||||
entity_ref: id,
|
||||
}
|
||||
.event(context, vec!["status".to_owned()]),
|
||||
);
|
||||
}
|
||||
Ok(Some(record.clone()))
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use gateway_admin::{
|
||||
ports::backup::{BackupRepository, DatabaseDumpPort, DumpRequest},
|
||||
};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::task::{DaemonTask, WorkerDaemonContext};
|
||||
|
||||
use super::support::{
|
||||
DUMP_CONTENT, FakeBackupRepository, FakeDumpPort, FakeObjectStore, backup_id,
|
||||
@@ -31,7 +32,7 @@ async fn backup_task_runs_full_dump_upload_verify_pipeline() {
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
@@ -72,7 +73,7 @@ async fn backup_task_marks_failed_when_dump_port_fails() {
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
@@ -104,7 +105,7 @@ async fn backup_task_marks_failed_with_the_upload_error() {
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
|
||||
@@ -137,7 +138,7 @@ async fn backup_task_recovery_marks_failed_with_the_upload_error() {
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(seed.clone())
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert queued");
|
||||
let cancellation = CancellationToken::new();
|
||||
@@ -184,14 +185,14 @@ async fn retention_cleans_expired_scheduled_backups() {
|
||||
expires_at: None,
|
||||
};
|
||||
repository
|
||||
.insert_backup_record(old_seed.clone())
|
||||
.insert_backup_record(old_seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert old");
|
||||
// 旧备份 40 天前完成(释放活跃名额),再插入新备份
|
||||
let now = Utc::now();
|
||||
repository.set_completed(&backup_id("old"), now - Duration::days(40));
|
||||
repository
|
||||
.insert_backup_record(new_seed.clone())
|
||||
.insert_backup_record(new_seed.clone(), &system_context())
|
||||
.await
|
||||
.expect("insert new");
|
||||
// 新备份 1 天前完成
|
||||
@@ -278,7 +279,10 @@ async fn completed_recovery_removes_staging_for_dumping_and_uploading_tasks() {
|
||||
object_key: "recovery.dump".to_owned(),
|
||||
expires_at: None,
|
||||
};
|
||||
repository.insert_backup_record(seed.clone()).await.unwrap();
|
||||
repository
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.unwrap();
|
||||
let artifact = dump
|
||||
.dump(DumpRequest {
|
||||
backup_id: seed.id.clone(),
|
||||
@@ -343,7 +347,10 @@ async fn manual_backup_preserves_primary_and_final_delete_failures() {
|
||||
object_key: "codex/cleanup-failure.dump".into(),
|
||||
expires_at: expiring.then(|| Utc::now() + Duration::hours(1)),
|
||||
};
|
||||
repository.insert_backup_record(seed.clone()).await.unwrap();
|
||||
repository
|
||||
.insert_backup_record(seed.clone(), &system_context())
|
||||
.await
|
||||
.unwrap();
|
||||
let task = BackupTask::new(repository.clone(), dump.clone(), store.clone());
|
||||
task.run_cycle(&CancellationToken::new()).await.unwrap();
|
||||
let records = repository.all_records();
|
||||
@@ -379,3 +386,40 @@ async fn manual_backup_preserves_primary_and_final_delete_failures() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn daemon_reports_success_only_after_a_complete_cycle_and_recovers_after_store_failure() {
|
||||
let repository = Arc::new(FakeBackupRepository::new(configured_settings()));
|
||||
let task = Arc::new(BackupTask::new(
|
||||
repository.clone(),
|
||||
Arc::new(FakeDumpPort::new()),
|
||||
Arc::new(FakeObjectStore::new()),
|
||||
));
|
||||
let context = WorkerDaemonContext::new(CancellationToken::new());
|
||||
repository.set_settings_unavailable(true);
|
||||
let error = task.run(context.clone()).await.unwrap_err();
|
||||
assert_eq!(error.as_safe_str(), "backup store failed");
|
||||
assert_eq!(context.completed_cycles(), 0);
|
||||
|
||||
repository.set_settings_unavailable(false);
|
||||
let recovered = WorkerDaemonContext::new(CancellationToken::new());
|
||||
let worker = tokio::spawn({
|
||||
let task = task.clone();
|
||||
let context = recovered.clone();
|
||||
async move { task.run(context).await }
|
||||
});
|
||||
recovered.progress_since(0).await;
|
||||
assert_eq!(recovered.completed_cycles(), 1);
|
||||
repository.set_settings_unavailable(true);
|
||||
tokio::time::advance(std::time::Duration::from_secs(30)).await;
|
||||
assert_eq!(
|
||||
worker.await.unwrap().unwrap_err().as_safe_str(),
|
||||
"backup store failed"
|
||||
);
|
||||
assert_eq!(recovered.completed_cycles(), 1);
|
||||
|
||||
let cancelled = WorkerDaemonContext::new(CancellationToken::new());
|
||||
cancelled.cancellation().cancel();
|
||||
task.run(cancelled.clone()).await.unwrap();
|
||||
assert_eq!(cancelled.completed_cycles(), 0);
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ fn object_key_shape() {
|
||||
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
|
||||
assert_eq!(
|
||||
key,
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.sql.gz"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -149,6 +149,9 @@ async fn group_options_query_should_preserve_store_failures() {
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeGroupStore {
|
||||
projection_failure: bool,
|
||||
missing_record: bool,
|
||||
published_revision: Option<Arc<std::sync::atomic::AtomicU64>>,
|
||||
requested_groups: Mutex<Vec<String>>,
|
||||
members: Option<Vec<AccountGroupMemberFact>>,
|
||||
}
|
||||
@@ -223,6 +226,16 @@ impl AccountGroupStore for FakeGroupStore {
|
||||
&self,
|
||||
group_ids: &[AccountGroupId],
|
||||
) -> AdminStoreResult<Vec<AccountGroupMemberFact>> {
|
||||
if self.projection_failure {
|
||||
assert_eq!(
|
||||
self.published_revision
|
||||
.as_ref()
|
||||
.expect("publication counter")
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
2
|
||||
);
|
||||
return Err(unused());
|
||||
}
|
||||
*self.requested_groups.lock().expect("requested groups") = group_ids
|
||||
.iter()
|
||||
.map(|group_id| group_id.as_str().to_owned())
|
||||
@@ -251,10 +264,16 @@ impl AccountGroupStore for FakeGroupStore {
|
||||
|
||||
async fn set_account_group_enabled(
|
||||
&self,
|
||||
_: SetAccountGroupEnabled,
|
||||
command: SetAccountGroupEnabled,
|
||||
_: &MutationContext,
|
||||
) -> AdminStoreResult<AccountGroupMutation> {
|
||||
Err(unused())
|
||||
let mut record = group_record();
|
||||
record.enabled = command.enabled;
|
||||
Ok(AccountGroupMutation {
|
||||
config_revision: Revision::new(2).expect("committed revision"),
|
||||
id: record.id.clone(),
|
||||
record: (!self.missing_record).then_some(record),
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_account_group(
|
||||
@@ -431,3 +450,50 @@ async fn invalid_group_fields_are_rejected_before_store_mutations() {
|
||||
assert_eq!(created.kind(), AdminErrorKind::Invalid);
|
||||
assert_eq!(updated.kind(), AdminErrorKind::Invalid);
|
||||
}
|
||||
|
||||
struct RecordingPublication(Arc<std::sync::atomic::AtomicU64>);
|
||||
|
||||
impl gateway_core::runtime::SnapshotControl for RecordingPublication {
|
||||
fn publish_committed(
|
||||
&self,
|
||||
revision: gateway_core::routing::ConfigRevision,
|
||||
) -> futures::future::BoxFuture<'_, ()> {
|
||||
Box::pin(async move {
|
||||
self.0
|
||||
.store(revision.get(), std::sync::atomic::Ordering::SeqCst);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_group_mutations_publish_even_when_the_display_projection_is_unavailable() {
|
||||
for missing_record in [false, true] {
|
||||
let published = Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let groups = Arc::new(FakeGroupStore {
|
||||
projection_failure: true,
|
||||
missing_record,
|
||||
published_revision: Some(published.clone()),
|
||||
..Default::default()
|
||||
});
|
||||
let services = AdminHarness::new()
|
||||
.account_groups(groups)
|
||||
.snapshot(Arc::new(RecordingPublication(published.clone())))
|
||||
.build()
|
||||
.await;
|
||||
let mutation = services
|
||||
.account_groups()
|
||||
.set_enabled(
|
||||
&crate::backup::support::system_context(),
|
||||
SetAccountGroupEnabled {
|
||||
id: group_id(),
|
||||
enabled: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("committed mutation remains successful");
|
||||
assert_eq!(mutation.config_revision.get(), 2);
|
||||
assert_eq!(mutation.id, group_id());
|
||||
assert!(mutation.record.is_none());
|
||||
assert_eq!(published.load(std::sync::atomic::Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2098,11 +2098,23 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
provider_data: None,
|
||||
});
|
||||
let failing = FakeProviderAdmin::new("xai", events.clone());
|
||||
let mut xai_quota = failing.quota.lock().expect("provider quota").clone();
|
||||
xai_quota.observed_at = Some(Utc::now());
|
||||
failing.set_quota(xai_quota);
|
||||
failing.fail_next_quota(ProviderAdminErrorKind::Invalid);
|
||||
let store = FakeAccountStore::new("openai", events.clone());
|
||||
let mut xai_account = account_record("xai");
|
||||
xai_account.id = "acct_bad_xai".to_owned();
|
||||
store.set_accounts(vec![account_record("openai"), xai_account]);
|
||||
let mut healthy_xai = xai_account.clone();
|
||||
healthy_xai.id = "acct_healthy_xai".to_owned();
|
||||
let mut second_openai = account_record("openai");
|
||||
second_openai.id = "acct_second_openai".to_owned();
|
||||
store.set_accounts(vec![
|
||||
account_record("openai"),
|
||||
xai_account,
|
||||
second_openai,
|
||||
healthy_xai,
|
||||
]);
|
||||
let services = super::AdminHarness::new()
|
||||
.accounts(store.clone())
|
||||
.settings(Arc::new(StaticSettingsStore::default()))
|
||||
@@ -2126,7 +2138,18 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
.await
|
||||
.expect("one failing quota must not fail the directory page");
|
||||
|
||||
assert_eq!(page.items.len(), 2);
|
||||
assert_eq!(
|
||||
page.items
|
||||
.iter()
|
||||
.map(|item| item.account.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"acct_test",
|
||||
"acct_bad_xai",
|
||||
"acct_second_openai",
|
||||
"acct_healthy_xai"
|
||||
]
|
||||
);
|
||||
let by_id = page
|
||||
.items
|
||||
.iter()
|
||||
@@ -2139,7 +2162,9 @@ async fn accounts_list_should_degrade_quota_failure_to_empty_window_without_drop
|
||||
assert!(degraded.quota.windows.is_empty());
|
||||
assert!(degraded.quota.observed_at.is_none());
|
||||
assert!(degraded.quota.provider_data.is_none());
|
||||
assert_eq!(page.summary.total, 2);
|
||||
assert!(by_id["acct_healthy_xai"].quota.observed_at.is_some());
|
||||
assert_eq!(by_id["acct_second_openai"].quota.windows.len(), 1);
|
||||
assert_eq!(page.summary.total, 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3194,6 +3219,7 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
|
||||
pub(super) fn account_record(kind: &str) -> AccountRecord {
|
||||
let now = Utc::now();
|
||||
AccountRecord {
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
notes: None,
|
||||
model_access: Default::default(),
|
||||
|
||||
@@ -38,9 +38,12 @@ async fn backup_service_creates_downloads_and_deletes_with_audit() {
|
||||
assert_eq!(created.status, BackupStatus::Queued);
|
||||
assert!(created.object_key.starts_with("codex/"));
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
repository
|
||||
.audit_events()
|
||||
.iter()
|
||||
.any(|action| action == "backup.created")
|
||||
.any(|event| event.action == "backup.created"
|
||||
&& event.entity_ref == created.id
|
||||
&& event.request_id.as_deref() == Some(system_context().request_id.as_str()))
|
||||
);
|
||||
|
||||
// 下载地址要求 completed 记录
|
||||
@@ -65,9 +68,12 @@ async fn backup_service_creates_downloads_and_deletes_with_audit() {
|
||||
.expect("delete backup");
|
||||
assert_eq!(deleting.status, BackupStatus::Deleting);
|
||||
assert!(
|
||||
auth.audit_actions()
|
||||
repository
|
||||
.audit_events()
|
||||
.iter()
|
||||
.any(|action| action == "backup.delete_requested")
|
||||
.any(|event| event.action == "backup.delete_requested"
|
||||
&& event.entity_ref == created.id
|
||||
&& event.changed_fields == ["status"])
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ use gateway_admin::{
|
||||
use gateway_core::{
|
||||
lifecycle::CancellationToken,
|
||||
routing::ProviderKind,
|
||||
task::{WorkerContribution, WorkerKind, WorkerRunnable},
|
||||
task::{WorkerContribution, WorkerDaemonContext, WorkerKind, WorkerRunnable},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -75,7 +75,9 @@ fn start(bundle: &mut AdminBundle) -> (CancellationToken, tokio::task::JoinHandl
|
||||
let cancellation = CancellationToken::new();
|
||||
let token = cancellation.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
task.run(token).await.expect("import daemon exits cleanly");
|
||||
task.run(WorkerDaemonContext::new(token))
|
||||
.await
|
||||
.expect("import daemon exits cleanly");
|
||||
});
|
||||
(cancellation, handle)
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ use gateway_admin::{
|
||||
DashboardDesktopRelease, DashboardObservation, DashboardQuery, DashboardWireAttribute,
|
||||
DashboardWireProfile, DashboardWireTarget, DesktopReleaseStatus, DiagnosticDimension,
|
||||
DiagnosticsObservation, Granularity, OpsErrorPage, OpsErrorQuery, RequestMetricPoint,
|
||||
TimeRange, UsageDetail, UsageFilter, UsageOverview, UsagePage, UsageQuery,
|
||||
TimeRange, UsageDetail, UsageFilter, UsagePage, UsageQuery, UsageSummaryObservation,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -99,6 +99,7 @@ pub(super) struct AdminHarness {
|
||||
proxy_probe: Arc<dyn gateway_admin::ports::proxy::ProxyProbe>,
|
||||
account_runtime: Arc<dyn AccountRuntimeStore>,
|
||||
account_groups: Arc<dyn AccountGroupStore>,
|
||||
snapshot: Arc<dyn SnapshotControl>,
|
||||
auth: Arc<dyn AuthStore>,
|
||||
client_keys: Arc<dyn ClientKeyStore>,
|
||||
observability: Arc<dyn ObservabilityStore>,
|
||||
@@ -125,6 +126,7 @@ impl AdminHarness {
|
||||
proxy_probe: Arc::new(proxies::TestProxies::default()),
|
||||
account_runtime: unavailable.clone(),
|
||||
account_groups: Arc::new(UnavailableAccountGroupStore),
|
||||
snapshot: Arc::new(NoopSnapshot),
|
||||
auth: Arc::new(BootstrapAuthStore::default()),
|
||||
client_keys: unavailable.clone(),
|
||||
observability: unavailable.clone(),
|
||||
@@ -168,6 +170,11 @@ impl AdminHarness {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(mut self, snapshot: Arc<dyn SnapshotControl>) -> Self {
|
||||
self.snapshot = snapshot;
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn auth(mut self, store: Arc<dyn AuthStore>) -> Self {
|
||||
self.auth = store;
|
||||
self
|
||||
@@ -291,7 +298,7 @@ impl AdminHarness {
|
||||
self.providers,
|
||||
)
|
||||
.unwrap(),
|
||||
snapshot: Arc::new(NoopSnapshot),
|
||||
snapshot: self.snapshot,
|
||||
account_probe: self.probe,
|
||||
proxy_probe: self.proxy_probe,
|
||||
client_distribution: Arc::new(NoopClientDistribution),
|
||||
@@ -826,7 +833,11 @@ impl ObservabilityStore for UnavailableStore {
|
||||
Err(unavailable("usage detail"))
|
||||
}
|
||||
|
||||
async fn usage_summary(&self, _: TimeRange, _: UsageFilter) -> AdminStoreResult<UsageOverview> {
|
||||
async fn usage_summary(
|
||||
&self,
|
||||
_: TimeRange,
|
||||
_: UsageFilter,
|
||||
) -> AdminStoreResult<UsageSummaryObservation> {
|
||||
Err(unavailable("usage summary"))
|
||||
}
|
||||
|
||||
@@ -1033,7 +1044,7 @@ pub(super) fn dashboard_profile_provider() -> Arc<dyn ProviderAdmin> {
|
||||
},
|
||||
user_agent: "gateway-admin-test".to_owned(),
|
||||
attributes: vec![DashboardWireAttribute {
|
||||
label: "Core".to_owned(),
|
||||
key: "core".to_owned(),
|
||||
value: "test".to_owned(),
|
||||
}],
|
||||
verified_at: Some(chrono::Utc::now()),
|
||||
|
||||
@@ -23,7 +23,7 @@ use gateway_admin::{
|
||||
HealthStatus, LatencyPercentiles, ObservabilityPageSize, OpsErrorPage, OpsErrorQuery,
|
||||
PercentileMilliseconds, RequestMetricPoint, RequestMetrics, TimeRange, TrendKind,
|
||||
UsageBilling, UsageDetail, UsageFilter, UsageInsightsObservation, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageProjectedCostPoint, UsageQuery,
|
||||
UsageOverview, UsagePage, UsageProjectedCostPoint, UsageQuery, UsageSummaryObservation,
|
||||
},
|
||||
settings::{
|
||||
AdminApiKey, AdminApiKeyMutation, ReplaceRuntimeSettings, RotationStrategy,
|
||||
@@ -1166,7 +1166,7 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
) -> AdminStoreResult<UsageSummaryObservation> {
|
||||
self.usage_summary_filters
|
||||
.lock()
|
||||
.expect("summary filters")
|
||||
@@ -1178,9 +1178,12 @@ impl ObservabilityStore for FixtureObservabilityStore {
|
||||
if self.summary_fail.load(Ordering::SeqCst) {
|
||||
return Err(super::unavailable("projection summary failed"));
|
||||
}
|
||||
Ok(UsageOverview {
|
||||
let overview = self.overview.lock().expect("overview").clone();
|
||||
Ok(UsageSummaryObservation {
|
||||
range,
|
||||
..self.overview.lock().expect("overview").clone()
|
||||
analytics: overview.analytics,
|
||||
requests: (&overview.requests).into(),
|
||||
attempts: overview.attempts,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1504,8 +1507,8 @@ async fn usage_summary_cache_should_coalesce_identical_queries_and_invalidate_de
|
||||
.usage_summary(range, filter)
|
||||
.await
|
||||
.expect("generation refresh");
|
||||
assert_eq!(refreshed.overview.analytics.generation, 1);
|
||||
assert_eq!(refreshed.overview.requests.request_count, 7);
|
||||
assert_eq!(refreshed.observation.analytics.generation, 1);
|
||||
assert_eq!(refreshed.observation.requests.request_count, 7);
|
||||
assert_eq!(store.usage_summary_filters.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
@@ -1525,13 +1528,17 @@ async fn usage_summary_cache_should_coalesce_when_generation_advances_during_the
|
||||
},
|
||||
);
|
||||
assert_eq!(
|
||||
first.expect("first summary").overview.analytics.generation,
|
||||
first
|
||||
.expect("first summary")
|
||||
.observation
|
||||
.analytics
|
||||
.generation,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
second
|
||||
.expect("coalesced summary")
|
||||
.overview
|
||||
.observation
|
||||
.analytics
|
||||
.generation,
|
||||
1
|
||||
|
||||
@@ -38,6 +38,7 @@ tracing = "0.1.44"
|
||||
url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v7"] }
|
||||
zstd = "=0.13.3"
|
||||
zip = { version = "9.0.4", default-features = false, features = ["deflate-flate2"] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1.89"
|
||||
|
||||
@@ -84,7 +84,7 @@ pub(super) fn account_view(
|
||||
let usage_period = quota.usage_window().map(|(_, period)| period);
|
||||
let mut usage = account_usage_view(usage, usage_period, now, time);
|
||||
if account.authentication_kind == "api_key" {
|
||||
usage.window_label_display = "通用额度".to_owned();
|
||||
usage.window_kind = AccountUsageWindowKind::General;
|
||||
}
|
||||
let (quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now, time);
|
||||
AccountView {
|
||||
@@ -120,6 +120,7 @@ pub(super) fn account_view(
|
||||
},
|
||||
weight: account.weight.get(),
|
||||
model_access: account.model_access,
|
||||
outbound_proxy_id: account.outbound_proxy_id,
|
||||
outbound_proxy_endpoint: account
|
||||
.outbound_proxy
|
||||
.as_ref()
|
||||
@@ -240,6 +241,10 @@ pub(super) fn account_quota_view(
|
||||
let recovery_probe_required = cooldown.is_some_and(|value| value.kind.requires_probe());
|
||||
(
|
||||
AccountQuotaView {
|
||||
refreshed_at: quota.observed_at.map(|value| time.rfc3339(&value)),
|
||||
refreshed_at_relative: quota
|
||||
.observed_at
|
||||
.map(|value| time.relative_view(value, now)),
|
||||
refreshed_at_display,
|
||||
limit_reached: quota.limit_reached,
|
||||
rate_limited_until,
|
||||
@@ -343,12 +348,11 @@ pub(super) fn account_usage_view(
|
||||
"known"
|
||||
};
|
||||
AccountUsageView {
|
||||
window_label_display: match period {
|
||||
Some(AccountUsagePeriod::Weekly) => "周额度窗口",
|
||||
Some(AccountUsagePeriod::Monthly) => "月额度窗口",
|
||||
None => "周/月额度窗口",
|
||||
}
|
||||
.to_owned(),
|
||||
window_kind: match period {
|
||||
Some(AccountUsagePeriod::Weekly) => AccountUsageWindowKind::Weekly,
|
||||
Some(AccountUsagePeriod::Monthly) => AccountUsageWindowKind::Monthly,
|
||||
None => AccountUsageWindowKind::Quota,
|
||||
},
|
||||
request_count: Some(usage.request_count),
|
||||
request_count_display: format_number(usage.request_count),
|
||||
input_tokens: usage.input_tokens,
|
||||
@@ -469,7 +473,7 @@ pub(super) fn display_optional_tokens(value: Option<u64>) -> String {
|
||||
|
||||
pub(super) fn empty_account_usage() -> AccountUsageView {
|
||||
AccountUsageView {
|
||||
window_label_display: "周/月额度窗口".to_owned(),
|
||||
window_kind: AccountUsageWindowKind::Quota,
|
||||
request_count: None,
|
||||
request_count_display: "—".to_owned(),
|
||||
input_tokens: None,
|
||||
|
||||
@@ -267,6 +267,7 @@ pub struct AccountSummaryView {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountView {
|
||||
pub capabilities: AccountCapabilitiesView,
|
||||
pub outbound_proxy_id: Option<String>,
|
||||
pub outbound_proxy_endpoint: Option<String>,
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
@@ -384,6 +385,8 @@ pub use crate::admin::account_groups::AccountGroupRefView;
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountQuotaView {
|
||||
pub refreshed_at: Option<String>,
|
||||
pub refreshed_at_relative: Option<crate::time::RelativeTimeView>,
|
||||
pub refreshed_at_display: String,
|
||||
pub limit_reached: bool,
|
||||
/// 冷却到期或可开始恢复探测的时间;非限流中为 `null`
|
||||
@@ -424,11 +427,21 @@ pub struct AccountQuotaWindowView {
|
||||
pub reset_at_display: String,
|
||||
}
|
||||
|
||||
/// 账号用量窗口的语义标识,展示名称由管理端本地化
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AccountUsageWindowKind {
|
||||
General,
|
||||
Weekly,
|
||||
Monthly,
|
||||
Quota,
|
||||
}
|
||||
|
||||
/// 账号观测用量
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AccountUsageView {
|
||||
pub window_label_display: String,
|
||||
pub window_kind: AccountUsageWindowKind,
|
||||
pub request_count: Option<u64>,
|
||||
pub request_count_display: String,
|
||||
pub input_tokens: Option<u64>,
|
||||
@@ -1033,8 +1046,7 @@ impl AccountExportData {
|
||||
time: crate::time::TimePresenter,
|
||||
) -> Self {
|
||||
let filename = format!(
|
||||
"cpr-accounts-selected-{}-{}.json",
|
||||
bundle.documents.len(),
|
||||
"cpr-accounts-{}.json",
|
||||
time.label(bundle.exported_at, "%Y-%m-%d")
|
||||
);
|
||||
let documents = bundle
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
//! 只打包已经脱敏的诊断文档,摘要与正文使用同一份证据
|
||||
|
||||
use std::io::{self, Cursor, Write};
|
||||
|
||||
use serde_json::Value;
|
||||
use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
pub(super) fn build(document: &Value) -> io::Result<Vec<u8>> {
|
||||
let mut archive = ZipWriter::new(Cursor::new(Vec::new()));
|
||||
let options = SimpleFileOptions::default()
|
||||
.compression_method(CompressionMethod::Deflated)
|
||||
.compression_level(Some(1))
|
||||
.unix_permissions(0o600);
|
||||
archive.start_file("diagnostics.json", options)?;
|
||||
serde_json::to_writer_pretty(&mut archive, document)?;
|
||||
archive.start_file("summary.txt", options)?;
|
||||
write!(
|
||||
archive,
|
||||
"CPR 诊断摘要\n导出时间:{}\n目标:{}\n导出截断:{}\n环境:{}\n",
|
||||
document["collectedAt"],
|
||||
document["target"],
|
||||
document["exportTruncated"],
|
||||
document["environment"]
|
||||
)?;
|
||||
for request in document["requests"].as_array().into_iter().flatten() {
|
||||
let evidence = &request["evidence"];
|
||||
let trace = &evidence["trace"];
|
||||
let detail = &evidence["detail"]["request"];
|
||||
writeln!(
|
||||
archive,
|
||||
"\n结果:{};状态码:{};尝试数:{}\n模型:请求={},发送={},上游报告={}\n传输:{} → {}\n时延:{}",
|
||||
detail["logicalOutcome"],
|
||||
detail["statusCode"],
|
||||
detail["attemptCount"],
|
||||
detail["requestedModel"],
|
||||
detail["upstreamModel"],
|
||||
detail["upstreamResponseModel"],
|
||||
detail["clientTransport"],
|
||||
detail["upstreamTransport"],
|
||||
detail["latencyDetails"]
|
||||
)?;
|
||||
writeln!(
|
||||
archive,
|
||||
"\n请求:{}\n总事件:{};淘汰事件:{};关键证据淘汰:{}\n采集阶段:{}",
|
||||
request["requestId"],
|
||||
trace["totalEvents"],
|
||||
trace["droppedEvents"],
|
||||
trace["droppedEvidenceEvents"],
|
||||
trace["stageCounts"]
|
||||
)?;
|
||||
for event in trace["events"].as_array().into_iter().flatten() {
|
||||
if matches!(
|
||||
event["stage"].as_str(),
|
||||
Some(
|
||||
"request.finished"
|
||||
| "attempt.failed"
|
||||
| "retry.decided"
|
||||
| "provider.response.usage"
|
||||
| "request.usage"
|
||||
| "capture.gap"
|
||||
| "provider.response.parse_gap"
|
||||
)
|
||||
) {
|
||||
writeln!(
|
||||
archive,
|
||||
"attempt={} t={}ms {}:{}",
|
||||
event["attemptIndex"], event["elapsedMs"], event["stage"], event["data"]
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !document["error"].is_null() {
|
||||
writeln!(archive, "\n目标错误:{}", document["error"])?;
|
||||
}
|
||||
writeln!(archive, "\n证据缺口:")?;
|
||||
for gap in document["availability"]["gaps"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
writeln!(archive, "{}:{}", gap["section"], gap["reason"])?;
|
||||
}
|
||||
archive.start_file("README.txt", options)?;
|
||||
archive.write_all("diagnostics.json 为完整结构化证据,summary.txt 为同一证据的摘要\n先检查 availability.gaps、exportTruncated 与 trace.stageCounts\n缺失、字段无效、上游明确返回 0、采集截断和证据淘汰含义不同\nprovider.response.usage 为上游终态观测及共享解析结果,request.usage 为网关最终归集结果\n输入指纹来自已解析对象;outbound 为准备发送的对象,成功发送需结合发送事实与响应判断\n指纹只能比较内容与顺序,不能证明上游缓存节点或缓存是否仍存在\nenvironment 是导出时环境;事发配置见 incidentConfiguration 与 request.routed\n历史记录不会补生成当时未采集的信息,未知业务内容不属于自动脱敏保证范围\n".as_bytes())?;
|
||||
Ok(archive.finish()?.into_inner())
|
||||
}
|
||||
@@ -13,6 +13,7 @@ use super::{
|
||||
};
|
||||
use crate::{auth::SessionState, time::TimePresenter};
|
||||
|
||||
mod archive;
|
||||
mod privacy;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -46,26 +47,43 @@ where
|
||||
.await
|
||||
.map_err(map_service_error)?;
|
||||
let time = TimePresenter::new(state.admin_services().timezone());
|
||||
let id = match &bundle.target {
|
||||
DiagnosticTarget::Request(id) | DiagnosticTarget::Operation(id) => id,
|
||||
};
|
||||
let suffix = id
|
||||
.chars()
|
||||
.take(128)
|
||||
.map(|ch| {
|
||||
if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_') {
|
||||
ch
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
let disposition = format!("attachment; filename=cpr-diagnostics-{suffix}.zip");
|
||||
let document = privacy::redact(document(bundle, time));
|
||||
let body = serde_json::to_vec_pretty(&document).map_err(|_| {
|
||||
map_service_error(gateway_admin::model::AdminError::internal(
|
||||
"诊断文件生成失败",
|
||||
))
|
||||
})?;
|
||||
let body = tokio::task::spawn_blocking(move || archive::build(&document))
|
||||
.await
|
||||
.map_err(archive_error)?
|
||||
.map_err(archive_error)?;
|
||||
Ok((
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/json; charset=utf-8"),
|
||||
(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=diagnostics.json",
|
||||
),
|
||||
(header::CACHE_CONTROL, "private, no-store"),
|
||||
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
|
||||
(header::CONTENT_TYPE, "application/zip".to_owned()),
|
||||
(header::CONTENT_DISPOSITION, disposition),
|
||||
(header::CACHE_CONTROL, "private, no-store".to_owned()),
|
||||
(header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_owned()),
|
||||
],
|
||||
body,
|
||||
))
|
||||
}
|
||||
|
||||
fn archive_error(source: impl Into<gateway_core::error::ErrorSource>) -> AdminError {
|
||||
map_service_error(
|
||||
gateway_admin::model::AdminError::internal("诊断文件生成失败").with_source(source),
|
||||
)
|
||||
}
|
||||
|
||||
fn document(bundle: DiagnosticBundle, time: TimePresenter) -> Value {
|
||||
let target = match &bundle.target {
|
||||
DiagnosticTarget::Request(id) => json!({"requestId": id}),
|
||||
@@ -101,6 +119,7 @@ fn document(bundle: DiagnosticBundle, time: TimePresenter) -> Value {
|
||||
}).collect::<Vec<_>>();
|
||||
json!({
|
||||
"schemaVersion": 5,
|
||||
"exportTruncated": false,
|
||||
"source": "gateway.diagnostics",
|
||||
"exportPolicy": "support_credentials_redacted",
|
||||
"target": target,
|
||||
|
||||
@@ -49,7 +49,7 @@ pub(crate) fn outcome_name(outcome: &domain::RequestOutcome) -> &str {
|
||||
outcome.as_str()
|
||||
}
|
||||
|
||||
pub(crate) fn request_metrics_view(metrics: &domain::RequestMetrics) -> RequestMetricsView {
|
||||
fn request_summary_metrics_view(metrics: &domain::RequestSummaryMetrics) -> RequestMetricsView {
|
||||
RequestMetricsView {
|
||||
request_count: metrics.request_count,
|
||||
success_count: metrics.success_count,
|
||||
@@ -686,20 +686,10 @@ pub(crate) fn trend_point_view(
|
||||
tokens_value: point.metrics.total_tokens,
|
||||
errors: format_compact_number(point.service_failure_count),
|
||||
errors_value: point.service_failure_count,
|
||||
latency: display_duration(point.average_latency_ms),
|
||||
latency_value: point.average_latency_ms,
|
||||
max_latency: display_duration(point.metrics.max_latency_ms),
|
||||
max_latency_value: point.metrics.max_latency_ms,
|
||||
min_latency: display_duration(point.metrics.min_latency_ms),
|
||||
min_latency_value: point.metrics.min_latency_ms,
|
||||
success_rate: success_rate_value
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value:.1}%")),
|
||||
success_rate_value,
|
||||
first_token_p50_ms: point
|
||||
.metrics
|
||||
.first_token_latency_percentiles
|
||||
.p50_ms
|
||||
.map(|value| value.as_f64()),
|
||||
first_token_p95_ms: point
|
||||
.metrics
|
||||
.first_token_latency_percentiles
|
||||
@@ -711,20 +701,6 @@ pub(crate) fn trend_point_view(
|
||||
.p95_ms
|
||||
.map(|value| value.as_f64()),
|
||||
output_throughput_p50: point.metrics.output_throughput_p50,
|
||||
admission_decision_p95_ms: point
|
||||
.metrics
|
||||
.admission_decision_percentiles
|
||||
.p95_ms
|
||||
.map(|value| value.as_f64()),
|
||||
account_selection_wait_p95_ms: point
|
||||
.metrics
|
||||
.account_selection_wait_percentiles
|
||||
.p95_ms
|
||||
.map(|value| value.as_f64()),
|
||||
capacity_utilization: point
|
||||
.metrics
|
||||
.capacity_utilization_p95_basis_points
|
||||
.map(basis_points_rate),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -735,34 +711,34 @@ pub(crate) fn trend_summary_view(
|
||||
match kind {
|
||||
TrendKind::Usage => vec![
|
||||
TrendSummaryView {
|
||||
label: "输入".to_owned(),
|
||||
metric: TrendMetric::InputTokens,
|
||||
value: format_compact_number(summary.input_tokens),
|
||||
ratio: None,
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "输出".to_owned(),
|
||||
metric: TrendMetric::OutputTokens,
|
||||
value: format_compact_number(summary.output_tokens),
|
||||
ratio: None,
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "缓存".to_owned(),
|
||||
metric: TrendMetric::CachedInputTokens,
|
||||
value: format_compact_number(summary.cached_input_tokens),
|
||||
ratio: None,
|
||||
},
|
||||
],
|
||||
TrendKind::Latency => vec![
|
||||
TrendSummaryView {
|
||||
label: "首字 P95".to_owned(),
|
||||
metric: TrendMetric::FirstTokenP95Ms,
|
||||
value: display_duration_f64(summary.peak_first_token_p95_ms),
|
||||
ratio: None,
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "总耗时 P95".to_owned(),
|
||||
metric: TrendMetric::LatencyP95Ms,
|
||||
value: display_duration_f64(summary.peak_latency_p95_ms),
|
||||
ratio: None,
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "速率 P50".to_owned(),
|
||||
metric: TrendMetric::OutputThroughputP50,
|
||||
value: summary
|
||||
.minimum_output_throughput_p50
|
||||
.map_or_else(|| "—".to_owned(), |value| format!("{value} tok/s")),
|
||||
@@ -771,19 +747,19 @@ pub(crate) fn trend_summary_view(
|
||||
],
|
||||
TrendKind::Errors => vec![
|
||||
TrendSummaryView {
|
||||
label: "错误数".to_owned(),
|
||||
metric: TrendMetric::Errors,
|
||||
value: format_compact_number(summary.service_failure_count),
|
||||
ratio: None,
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "成功率".to_owned(),
|
||||
metric: TrendMetric::SuccessRate,
|
||||
value: "—".to_owned(),
|
||||
ratio: summary
|
||||
.success_rate
|
||||
.map(|value| format!("{:.1}%", value * 100.0)),
|
||||
},
|
||||
TrendSummaryView {
|
||||
label: "总请求".to_owned(),
|
||||
metric: TrendMetric::Requests,
|
||||
value: format_compact_number(summary.request_count),
|
||||
ratio: None,
|
||||
},
|
||||
@@ -875,7 +851,7 @@ pub(crate) fn wire_profile_view(
|
||||
.attributes
|
||||
.into_iter()
|
||||
.map(|attribute| DashboardWireAttributeView {
|
||||
label: attribute.label,
|
||||
key: attribute.key,
|
||||
value: attribute.value,
|
||||
})
|
||||
.collect(),
|
||||
@@ -928,8 +904,10 @@ pub(crate) fn dashboard_view(
|
||||
.filter(|window| window.used_percent.is_none())
|
||||
.and_then(|window| window.local_usage.as_ref())
|
||||
.map(|usage| usage.request_count);
|
||||
let (metric_label, metric_value) = match local_requests {
|
||||
let (metric, metric_value_raw, metric_label, metric_value) = match local_requests {
|
||||
Some(count) => (
|
||||
DashboardAccountMetric::Requests,
|
||||
Some(count),
|
||||
"次数".to_owned(),
|
||||
if count > 0 {
|
||||
format_compact_number(count)
|
||||
@@ -938,6 +916,8 @@ pub(crate) fn dashboard_view(
|
||||
},
|
||||
),
|
||||
None => (
|
||||
DashboardAccountMetric::Tokens,
|
||||
credential.total_tokens,
|
||||
"今日 Token".to_owned(),
|
||||
credential
|
||||
.total_tokens
|
||||
@@ -971,8 +951,14 @@ pub(crate) fn dashboard_view(
|
||||
usage_window: credential
|
||||
.quota_window
|
||||
.map(|window| crate::admin::accounts::quota_window_view(window, time)),
|
||||
metric,
|
||||
metric_value_raw,
|
||||
metric_label,
|
||||
metric_value,
|
||||
last_used_at: credential.last_used_at,
|
||||
last_used_relative: credential
|
||||
.last_used_at
|
||||
.map(|value| time.relative_view(value, range.end)),
|
||||
last_used: time.relative_optional(credential.last_used_at, range.end),
|
||||
});
|
||||
}
|
||||
@@ -1050,7 +1036,7 @@ pub(crate) fn usage_summary_view(
|
||||
summary: domain::UsageSummary,
|
||||
time: crate::time::TimePresenter,
|
||||
) -> UsageSummaryView {
|
||||
let overview = summary.overview;
|
||||
let overview = summary.observation;
|
||||
UsageSummaryView {
|
||||
analytics: analytics_view(overview.analytics, time),
|
||||
total_requests: format_compact_number(overview.requests.request_count),
|
||||
@@ -1060,7 +1046,8 @@ pub(crate) fn usage_summary_view(
|
||||
cache_write_input_tokens: format_compact_number(overview.requests.cache_write_input_tokens),
|
||||
total_tokens: format_compact_number(overview.requests.total_tokens),
|
||||
average_latency_ms: display_duration(summary.average_latency_ms),
|
||||
logical_requests: request_metrics_view(&overview.requests),
|
||||
average_latency_ms_value: summary.average_latency_ms,
|
||||
logical_requests: request_summary_metrics_view(&overview.requests),
|
||||
attempts: attempt_metrics_view(&overview.attempts),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,27 +25,28 @@ impl DashboardQuery {
|
||||
}
|
||||
}
|
||||
|
||||
/// 逻辑请求列表、汇总和洞察查询参数
|
||||
/// 请求明细的字段筛选、时间范围与分页参数
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageQuery {
|
||||
pub client_api_key_name: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub current_page: Option<u32>,
|
||||
pub page_size: Option<u16>,
|
||||
pub kind: Option<String>,
|
||||
pub outcome: Option<String>,
|
||||
pub client_api_key_id: Option<String>,
|
||||
pub provider: Option<String>,
|
||||
pub request_id: Option<String>,
|
||||
pub account_id: Option<String>,
|
||||
pub route: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub status_code: Option<i64>,
|
||||
pub transport: Option<String>,
|
||||
pub attempt_index: Option<i64>,
|
||||
pub response_id: Option<String>,
|
||||
pub upstream_request_id: Option<String>,
|
||||
pub search: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
pub start_time: Option<String>,
|
||||
pub end_time: Option<String>,
|
||||
}
|
||||
|
||||
/// 用量汇总与洞察只按时间和 Provider 查询
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct UsageAnalyticsQuery {
|
||||
pub provider: Option<String>,
|
||||
pub period: Option<String>,
|
||||
pub as_of: Option<i64>,
|
||||
pub start_time: Option<String>,
|
||||
@@ -258,16 +259,6 @@ pub(crate) fn domain_diagnostic_dimension(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_outcome(
|
||||
value: Option<String>,
|
||||
) -> Result<Option<domain::RequestOutcome>, WireValidationError> {
|
||||
non_empty(value)
|
||||
.map(|value| {
|
||||
domain::RequestOutcome::new(value).map_err(|_| WireValidationError::new("outcome"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn usage_range(
|
||||
start: Option<&str>,
|
||||
end: Option<&str>,
|
||||
@@ -288,39 +279,14 @@ pub(crate) fn dashboard_today_range(
|
||||
crate::time::query_range(start, end, period, as_of, timezone, "today")
|
||||
}
|
||||
|
||||
pub(crate) fn usage_filter(query: &UsageQuery) -> Result<domain::UsageFilter, WireValidationError> {
|
||||
let outcome = query.outcome.clone().or_else(|| {
|
||||
non_empty(query.kind.clone()).filter(|value| {
|
||||
matches!(
|
||||
value.as_str(),
|
||||
"running" | "succeeded" | "failed" | "cancelled" | "incomplete"
|
||||
)
|
||||
})
|
||||
});
|
||||
Ok(domain::UsageFilter {
|
||||
client_api_key_ref: non_empty(query.client_api_key_id.clone()),
|
||||
request_id: non_empty(query.request_id.clone()),
|
||||
provider_account_ref: non_empty(query.account_id.clone()),
|
||||
operation: non_empty(query.route.clone()),
|
||||
pub(crate) fn usage_filter(query: &UsageQuery) -> domain::UsageFilter {
|
||||
domain::UsageFilter {
|
||||
client_api_key_name: non_empty(query.client_api_key_name.clone()),
|
||||
provider_account_name: non_empty(query.account_name.clone()),
|
||||
provider_kind: non_empty(query.provider.clone()),
|
||||
model: non_empty(query.model.clone()),
|
||||
outcome: request_outcome(outcome)?,
|
||||
status_code: parse_status(query.status_code)?,
|
||||
transport: non_empty(query.transport.clone()),
|
||||
attempt_index: parse_attempt_index(query.attempt_index)?,
|
||||
response_id: non_empty(query.response_id.clone()),
|
||||
upstream_request_id: non_empty(query.upstream_request_id.clone()),
|
||||
search: non_empty(query.search.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn analytics_filter(
|
||||
query: &UsageQuery,
|
||||
) -> Result<domain::UsageFilter, WireValidationError> {
|
||||
if query.search.is_some() {
|
||||
return Err(WireValidationError::new("search"));
|
||||
model_contains: non_empty(query.model.clone()),
|
||||
..Default::default()
|
||||
}
|
||||
usage_filter(query)
|
||||
}
|
||||
|
||||
pub(crate) fn usage_command(
|
||||
@@ -344,7 +310,7 @@ pub(crate) fn usage_command(
|
||||
query.as_of,
|
||||
timezone,
|
||||
)?,
|
||||
filter: usage_filter(query)?,
|
||||
filter: usage_filter(query),
|
||||
current_page,
|
||||
page_size: page_size_value,
|
||||
})
|
||||
|
||||
@@ -142,7 +142,7 @@ where
|
||||
pub(crate) async fn usage_records_summary<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageQuery>,
|
||||
AdminQuery(query): AdminQuery<UsageAnalyticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
@@ -155,7 +155,10 @@ where
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = analytics_filter(&query).map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
..Default::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
@@ -174,7 +177,7 @@ where
|
||||
pub(crate) async fn usage_insights_overview<S>(
|
||||
_auth: AdminAuth,
|
||||
State(state): State<S>,
|
||||
AdminQuery(query): AdminQuery<UsageQuery>,
|
||||
AdminQuery(query): AdminQuery<UsageAnalyticsQuery>,
|
||||
) -> Result<impl IntoResponse, AdminError>
|
||||
where
|
||||
S: SessionState + Send + Sync,
|
||||
@@ -188,7 +191,10 @@ where
|
||||
state.admin_services().timezone(),
|
||||
)
|
||||
.map_err(map_wire_error)?;
|
||||
let filter = analytics_filter(&query).map_err(map_wire_error)?;
|
||||
let filter = domain::UsageFilter {
|
||||
provider_kind: non_empty(query.provider),
|
||||
..Default::default()
|
||||
};
|
||||
let result = state
|
||||
.admin_services()
|
||||
.observability()
|
||||
|
||||
@@ -306,27 +306,33 @@ pub struct TrendPointView {
|
||||
pub tokens_value: u64,
|
||||
pub errors: String,
|
||||
pub errors_value: u64,
|
||||
pub latency: String,
|
||||
pub latency_value: Option<u64>,
|
||||
pub max_latency: String,
|
||||
pub max_latency_value: Option<u64>,
|
||||
pub min_latency: String,
|
||||
pub min_latency_value: Option<u64>,
|
||||
pub success_rate: String,
|
||||
pub success_rate_value: Option<f64>,
|
||||
pub first_token_p50_ms: Option<f64>,
|
||||
pub first_token_p95_ms: Option<f64>,
|
||||
pub latency_p95_ms: Option<f64>,
|
||||
pub output_throughput_p50: Option<u64>,
|
||||
pub admission_decision_p95_ms: Option<f64>,
|
||||
pub account_selection_wait_p95_ms: Option<f64>,
|
||||
pub capacity_utilization: Option<f64>,
|
||||
}
|
||||
|
||||
/// 趋势指标的稳定身份,展示名称由管理端本地化
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TrendMetric {
|
||||
InputTokens,
|
||||
OutputTokens,
|
||||
CachedInputTokens,
|
||||
FirstTokenP95Ms,
|
||||
LatencyP95Ms,
|
||||
OutputThroughputP50,
|
||||
Errors,
|
||||
SuccessRate,
|
||||
Requests,
|
||||
}
|
||||
|
||||
/// Dashboard 趋势摘要
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct TrendSummaryView {
|
||||
pub label: String,
|
||||
pub metric: TrendMetric,
|
||||
pub value: String,
|
||||
pub ratio: Option<String>,
|
||||
}
|
||||
@@ -386,7 +392,15 @@ pub struct DashboardCacheCardView {
|
||||
pub average_first_token_latency_ms: String,
|
||||
}
|
||||
|
||||
/// 旧 Dashboard 账号概览卡片所需的 Provider 安全投影
|
||||
/// Dashboard 账号概览指标的稳定标识
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum DashboardAccountMetric {
|
||||
Requests,
|
||||
Tokens,
|
||||
}
|
||||
|
||||
/// Dashboard 账号概览卡片所需的 Provider 安全投影
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardAccountUsageView {
|
||||
@@ -401,8 +415,12 @@ pub struct DashboardAccountUsageView {
|
||||
pub request_buckets: Vec<crate::time::RequestBucketView>,
|
||||
pub quota_used_percent: Option<f64>,
|
||||
pub usage_window: Option<crate::admin::accounts::AccountQuotaWindowView>,
|
||||
pub metric: DashboardAccountMetric,
|
||||
pub metric_value_raw: Option<u64>,
|
||||
pub metric_label: String,
|
||||
pub metric_value: String,
|
||||
pub last_used_at: Option<DateTime<Utc>>,
|
||||
pub last_used_relative: Option<crate::time::RelativeTimeView>,
|
||||
pub last_used: String,
|
||||
}
|
||||
|
||||
@@ -515,7 +533,7 @@ pub struct DashboardWireProfileView {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DashboardWireAttributeView {
|
||||
pub label: String,
|
||||
pub key: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
@@ -595,6 +613,7 @@ pub struct UsageSummaryView {
|
||||
pub cache_write_input_tokens: String,
|
||||
pub total_tokens: String,
|
||||
pub average_latency_ms: String,
|
||||
pub average_latency_ms_value: Option<u64>,
|
||||
pub logical_requests: RequestMetricsView,
|
||||
pub attempts: AttemptMetricsView,
|
||||
}
|
||||
|
||||
@@ -44,6 +44,8 @@ use crate::{
|
||||
|
||||
const MAXIMUM_BODY_BYTES: usize = 1024 * 1024;
|
||||
const MAXIMUM_MODEL_BODY_BYTES: usize = 8 * 1024 * 1024;
|
||||
// 传输归属由宿主写入,不能用插件可覆盖的安全响应头识别管理错误
|
||||
const PLUGIN_RESPONSE_HEADER: &str = "x-cpr-plugin-response";
|
||||
// HTML/SVG 即使被直接导航打开也不能获得管理端 origin;执行入口由隔离 iframe 的消息桥承载
|
||||
const RESOURCE_CSP: &str = "sandbox allow-scripts; default-src 'none'; script-src 'unsafe-inline' blob:; style-src 'unsafe-inline' blob:; img-src data: blob:; font-src data: blob:; connect-src 'none'; frame-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'self'";
|
||||
|
||||
@@ -124,17 +126,17 @@ async fn model_responses(
|
||||
match decode_request_with_body(&body, &headers, MAXIMUM_MODEL_BODY_BYTES) {
|
||||
Ok(decoded) => decoded,
|
||||
Err(error) => {
|
||||
return Ok(protocol_error_response(
|
||||
return Ok(plugin_response(protocol_error_response(
|
||||
StatusCode::BAD_REQUEST,
|
||||
error.protocol_body(),
|
||||
));
|
||||
)));
|
||||
}
|
||||
};
|
||||
let service = state.openai().clone();
|
||||
let execution = service.execution();
|
||||
let prepared = match execution.prepare_plugin_execution(&client_key_id).await {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return Ok(gateway_error_response(&error)),
|
||||
Err(error) => return Ok(plugin_response(gateway_error_response(&error))),
|
||||
};
|
||||
let authorization: Arc<dyn ResponseAuthorization> =
|
||||
Arc::new(PluginModelsAuthorization { state, target });
|
||||
@@ -158,7 +160,7 @@ async fn model_responses(
|
||||
Some(authorization),
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
Ok(plugin_response(response))
|
||||
}
|
||||
|
||||
struct PluginModelsAuthorization {
|
||||
@@ -431,5 +433,12 @@ fn raw_response(result: PluginManagementResponse) -> Result<Response, AdminError
|
||||
);
|
||||
// 宿主提供默认值,插件显式字段随后覆盖;保留同名多值,只验证 HTTP 语法
|
||||
headers.extend(decode_headers(result.headers).map_err(|_| AdminError::bad_gateway())?);
|
||||
Ok(response)
|
||||
Ok(plugin_response(response))
|
||||
}
|
||||
|
||||
fn plugin_response(mut response: Response) -> Response {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(PLUGIN_RESPONSE_HEADER, HeaderValue::from_static("1"));
|
||||
response
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ pub(crate) async fn hangup(
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let protocol_headers = protocol_header_pairs(&headers);
|
||||
let protocol_headers = filter_protocol_headers(&headers);
|
||||
let outcome = gateway
|
||||
.hangup(LiveHangupRequest {
|
||||
call_id: &call_id,
|
||||
@@ -141,13 +141,8 @@ pub(crate) async fn hangup(
|
||||
let status = StatusCode::from_u16(outcome.status).unwrap_or(StatusCode::OK);
|
||||
let mut response = Response::new(axum::body::Body::from(outcome.body));
|
||||
*response.status_mut() = status;
|
||||
for (name, value) in outcome.headers {
|
||||
if let (Ok(name), Ok(value)) = (
|
||||
header::HeaderName::from_bytes(name.as_bytes()),
|
||||
header::HeaderValue::from_bytes(value.as_bytes()),
|
||||
) {
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
for (name, value) in live_forwarded_response_headers(&outcome.headers) {
|
||||
response.headers_mut().append(name, value);
|
||||
}
|
||||
response
|
||||
}
|
||||
@@ -262,20 +257,7 @@ async fn read_call_body(
|
||||
}
|
||||
}
|
||||
|
||||
/// realtime 协议头的 `(name, value)` 形态;名称小写化以稳定上游顺序。
|
||||
pub(crate) fn protocol_header_pairs(headers: &HeaderMap) -> Vec<(String, String)> {
|
||||
super::filter_protocol_headers(headers)
|
||||
.into_iter()
|
||||
.map(|header| {
|
||||
(
|
||||
header.name().to_ascii_lowercase(),
|
||||
String::from_utf8_lossy(header.value()).into_owned(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 汇集引导响应;只放行白名单响应头,正文按字节透传。
|
||||
/// 汇集引导响应;普通响应头与正文按字节透传
|
||||
async fn collect_live_call_response(started: StartedExecution, path: String) -> Response {
|
||||
let request_id = started.request_id;
|
||||
let mut execution = PendingExecution::new(started.session);
|
||||
|
||||
@@ -34,14 +34,6 @@ pub(crate) const MAX_LIVE_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
pub(crate) const LIVE_CALLS_ENDPOINT: &str = "realtime-calls";
|
||||
/// realtime calls 的官方查询参数;上游按 `architecture=avas` 返回 WebRTC answer。
|
||||
pub(crate) const LIVE_CALLS_QUERY: &str = "intent=quicksilver&architecture=avas";
|
||||
/// 客户端协议头允许清单;语音会话语义字段,转发给上游引导请求。
|
||||
pub(crate) const LIVE_PROTOCOL_HEADERS: [&str; 5] = [
|
||||
"openai-alpha",
|
||||
"x-session-id",
|
||||
"session-id",
|
||||
"thread-id",
|
||||
"openai-safety-identifier",
|
||||
];
|
||||
|
||||
pub(crate) fn router() -> Router<crate::ApiState> {
|
||||
Router::new()
|
||||
@@ -136,22 +128,12 @@ pub(crate) fn realtime_unsupported_response(capability: &str) -> Response {
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// 请求侧协议头过滤;语音语义字段按允许清单转发,其余一律不透传。
|
||||
/// 共用 OpenAI 入站头边界;Content-Type 由正文适配结果单独提供
|
||||
pub(crate) fn filter_protocol_headers(headers: &HeaderMap) -> Vec<ProviderHttpHeader> {
|
||||
LIVE_PROTOCOL_HEADERS
|
||||
.iter()
|
||||
.flat_map(|name| {
|
||||
headers
|
||||
.get_all(*name)
|
||||
.into_iter()
|
||||
.map(|value| (*name, value))
|
||||
})
|
||||
.filter_map(|(name, value)| {
|
||||
let value = value.to_str().ok()?;
|
||||
Some(ProviderHttpHeader::new(
|
||||
name,
|
||||
Bytes::copy_from_slice(value.as_bytes()),
|
||||
))
|
||||
super::responses::passthrough_request_headers(headers)
|
||||
.filter(|(name, _)| *name != header::CONTENT_TYPE)
|
||||
.map(|(name, value)| {
|
||||
ProviderHttpHeader::new(name.as_str(), Bytes::copy_from_slice(value.as_bytes()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -281,20 +263,20 @@ pub(crate) fn encode_call_request(
|
||||
.map_err(|_| invalid("Codex live call request is invalid"))
|
||||
}
|
||||
|
||||
/// 响应头白名单;`Location` 是客户端获知 call id 的合同。
|
||||
/// 共用响应身份与传输边界;Live 原始正文另保留上游 Content-Type
|
||||
pub(crate) fn live_forwarded_response_headers(
|
||||
headers: &[ProviderResponseHeader],
|
||||
) -> Vec<(HeaderName, HeaderValue)> {
|
||||
const FORWARDABLE: [&str; 5] = [
|
||||
"location",
|
||||
"content-type",
|
||||
"retry-after",
|
||||
"x-request-id",
|
||||
"openai-request-id",
|
||||
];
|
||||
let connection_options = super::responses::response_connection_options(headers);
|
||||
headers
|
||||
.iter()
|
||||
.filter(|header| FORWARDABLE.contains(&header.name()))
|
||||
.filter(|header| {
|
||||
header.name().eq_ignore_ascii_case("content-type")
|
||||
|| super::responses::response_header_is_forwardable(
|
||||
header.name(),
|
||||
&connection_options,
|
||||
)
|
||||
})
|
||||
.filter_map(|header| {
|
||||
Some((
|
||||
HeaderName::from_bytes(header.name().as_bytes()).ok()?,
|
||||
|
||||
@@ -128,15 +128,7 @@ async fn open_sideband(
|
||||
client_api_key_id: client.policy().key_id(),
|
||||
account_scope: client.policy().account_scope(),
|
||||
style,
|
||||
protocol_headers: filter_protocol_headers(&headers)
|
||||
.into_iter()
|
||||
.map(|header| {
|
||||
(
|
||||
header.name().to_ascii_lowercase(),
|
||||
String::from_utf8_lossy(header.value()).into_owned(),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
protocol_headers: filter_protocol_headers(&headers),
|
||||
subprotocols: offered_subprotocols(&headers),
|
||||
});
|
||||
let result = tokio::select! {
|
||||
|
||||
@@ -321,24 +321,36 @@ impl AuthorizedResponseBody {
|
||||
impl MiddlewareBody for AuthorizedResponseBody {
|
||||
fn next_frame(&mut self) -> BoxFuture<'_, Result<Option<MiddlewareFrame>, MiddlewareError>> {
|
||||
Box::pin(async move {
|
||||
loop {
|
||||
let elapsed = self.last_authorized_at.elapsed();
|
||||
if elapsed >= AUTHORIZATION_RECHECK_INTERVAL {
|
||||
if let Err(error) = self.authorization.authorize().await {
|
||||
return Err(self.revoke(error).await);
|
||||
}
|
||||
self.last_authorized_at = Instant::now();
|
||||
continue;
|
||||
if self.last_authorized_at.elapsed() >= AUTHORIZATION_RECHECK_INTERVAL {
|
||||
if let Err(error) = self.authorization.authorize().await {
|
||||
return Err(self.revoke(error).await);
|
||||
}
|
||||
self.last_authorized_at = Instant::now();
|
||||
}
|
||||
let error = {
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let wait = AUTHORIZATION_RECHECK_INTERVAL - elapsed;
|
||||
tokio::select! {
|
||||
frame = inner.next_frame() => return frame,
|
||||
() = tokio::time::sleep(wait) => {}
|
||||
// 复核身份只暂停同一次读取,不能丢弃已经消费上游事件的 future
|
||||
let frame = inner.next_frame();
|
||||
tokio::pin!(frame);
|
||||
loop {
|
||||
let elapsed = self.last_authorized_at.elapsed();
|
||||
if elapsed >= AUTHORIZATION_RECHECK_INTERVAL {
|
||||
if let Err(error) = self.authorization.authorize().await {
|
||||
break error;
|
||||
}
|
||||
self.last_authorized_at = Instant::now();
|
||||
continue;
|
||||
}
|
||||
let wait = AUTHORIZATION_RECHECK_INTERVAL - elapsed;
|
||||
tokio::select! {
|
||||
result = &mut frame => return result,
|
||||
() = tokio::time::sleep(wait) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
Err(self.revoke(error).await)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -14,11 +14,11 @@ pub(crate) use http::{
|
||||
ResponseAuthorization, ResponsesHttpRequest, execute_prepared_responses, responses,
|
||||
};
|
||||
pub use http::{collect_execution_response, stream_execution_response};
|
||||
pub(crate) use request::decode_request_with_body;
|
||||
pub use request::{
|
||||
ContinuationIntent, DecodedResponsesRequest, OpenAiRequestHeaders, ResponsesRequestMetadata,
|
||||
decode_request_with_headers,
|
||||
};
|
||||
pub(crate) use request::{decode_request_with_body, passthrough_request_headers};
|
||||
pub use response::OpenAiResponsesEncoder;
|
||||
pub(crate) use websocket::responses_websocket;
|
||||
pub use websocket::{ResponseCreateFrameError, decode_response_create_with_context};
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use std::{borrow::Cow, fmt, net::IpAddr};
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use axum::http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD};
|
||||
use bytes::Bytes;
|
||||
use gateway_core::operation::{
|
||||
@@ -567,6 +567,20 @@ fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
fn passthrough_headers(headers: &HeaderMap) -> Vec<Value> {
|
||||
passthrough_request_headers(headers)
|
||||
.map(|(name, value)| {
|
||||
Value::Array(vec![
|
||||
Value::String(name.as_str().to_owned()),
|
||||
Value::String(STANDARD.encode(value.as_bytes())),
|
||||
])
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// HTTP 与 Live 入站共用的身份、传输边界;保留业务头原始多值字节
|
||||
pub(crate) fn passthrough_request_headers(
|
||||
headers: &HeaderMap,
|
||||
) -> impl Iterator<Item = (&HeaderName, &HeaderValue)> {
|
||||
let connection_headers = headers
|
||||
.get_all("connection")
|
||||
.iter()
|
||||
@@ -579,14 +593,7 @@ fn passthrough_headers(headers: &HeaderMap) -> Vec<Value> {
|
||||
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| passthrough_header_name(name.as_str(), &connection_headers))
|
||||
.map(|(name, value)| {
|
||||
Value::Array(vec![
|
||||
Value::String(name.as_str().to_owned()),
|
||||
Value::String(STANDARD.encode(value.as_bytes())),
|
||||
])
|
||||
})
|
||||
.collect()
|
||||
.filter(move |(name, _)| passthrough_header_name(name.as_str(), &connection_headers))
|
||||
}
|
||||
|
||||
fn passthrough_header_name(name: &str, connection_headers: &[String]) -> bool {
|
||||
|
||||
@@ -101,21 +101,51 @@ impl TimePresenter {
|
||||
self.timezone.local(*value).to_rfc3339()
|
||||
}
|
||||
|
||||
pub(crate) fn relative(self, value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
pub(crate) fn relative_view(
|
||||
self,
|
||||
value: DateTime<Utc>,
|
||||
now: DateTime<Utc>,
|
||||
) -> RelativeTimeView {
|
||||
let elapsed = now.signed_duration_since(value);
|
||||
if elapsed.num_seconds() < 0 {
|
||||
return self.datetime(&value);
|
||||
return RelativeTimeView::Absolute {
|
||||
value: self.datetime(&value),
|
||||
};
|
||||
}
|
||||
if elapsed.num_seconds() < 60 {
|
||||
return "刚刚".to_owned();
|
||||
return RelativeTimeView::JustNow;
|
||||
}
|
||||
if elapsed.num_minutes() < 60 {
|
||||
return format!("{} 分钟前", elapsed.num_minutes());
|
||||
return RelativeTimeView::Elapsed {
|
||||
value: elapsed.num_minutes(),
|
||||
unit: RelativeTimeUnit::Minute,
|
||||
};
|
||||
}
|
||||
if elapsed.num_hours() < 24 {
|
||||
return format!("{} 小时前", elapsed.num_hours());
|
||||
return RelativeTimeView::Elapsed {
|
||||
value: elapsed.num_hours(),
|
||||
unit: RelativeTimeUnit::Hour,
|
||||
};
|
||||
}
|
||||
RelativeTimeView::Elapsed {
|
||||
value: elapsed.num_days(),
|
||||
unit: RelativeTimeUnit::Day,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn relative(self, value: DateTime<Utc>, now: DateTime<Utc>) -> String {
|
||||
match self.relative_view(value, now) {
|
||||
RelativeTimeView::JustNow => "刚刚".to_owned(),
|
||||
RelativeTimeView::Elapsed { value, unit } => {
|
||||
let unit = match unit {
|
||||
RelativeTimeUnit::Minute => "分钟",
|
||||
RelativeTimeUnit::Hour => "小时",
|
||||
RelativeTimeUnit::Day => "天",
|
||||
};
|
||||
format!("{value} {unit}前")
|
||||
}
|
||||
RelativeTimeView::Absolute { value } => value,
|
||||
}
|
||||
format!("{} 天前", elapsed.num_days())
|
||||
}
|
||||
|
||||
pub(crate) fn relative_optional(
|
||||
@@ -127,6 +157,24 @@ impl TimePresenter {
|
||||
}
|
||||
}
|
||||
|
||||
/// 后端按查询锚点与部署时区决定展示方式,前端只本地化分类和单位
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum RelativeTimeView {
|
||||
JustNow,
|
||||
Elapsed { value: i64, unit: RelativeTimeUnit },
|
||||
Absolute { value: String },
|
||||
}
|
||||
|
||||
/// 相对时间保持分钟、小时、天的固定精度,不引入月或年的日历推算
|
||||
#[derive(Debug, Clone, Copy, serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RelativeTimeUnit {
|
||||
Minute,
|
||||
Hour,
|
||||
Day,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RequestBucketView {
|
||||
|
||||
@@ -9,6 +9,67 @@ use tower::ServiceExt as _;
|
||||
|
||||
use super::super::{AdminTestFixture, AdminTestState};
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_proxy_binding_is_distinct_from_the_redacted_endpoint() {
|
||||
use chrono::Utc;
|
||||
use gateway_core::account::{AccountStatusFacts, CredentialState, OutboundProxy, QuotaState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (proxy_id, proxy_url, enabled) in [
|
||||
(
|
||||
Some("proxy-a"),
|
||||
Some("socks5h://alice:secret@127.0.0.1:50101"),
|
||||
true,
|
||||
),
|
||||
(
|
||||
Some("proxy-b"),
|
||||
Some("socks5h://bob:secret@127.0.0.1:50101"),
|
||||
false,
|
||||
),
|
||||
(None, Some("socks5h://alice:secret@127.0.0.1:50101"), true),
|
||||
(None, None, true),
|
||||
] {
|
||||
let now = Utc::now();
|
||||
let facts = AccountStatusFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
};
|
||||
let mut item = account_page_item(facts, now, "oauth");
|
||||
item.account.outbound_proxy_id = proxy_id.map(str::to_owned);
|
||||
item.account.outbound_proxy = proxy_url.map(|url| OutboundProxy::parse(url).unwrap());
|
||||
item.account.outbound_proxy_enabled = enabled;
|
||||
*fixture.account.lock().unwrap() = Some(item);
|
||||
let response = admin::accounts::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/accounts")
|
||||
.header("x-request-id", "req_proxy_binding")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 32768).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
let account = &value["data"]["items"][0];
|
||||
assert_eq!(account["outboundProxyId"], serde_json::json!(proxy_id));
|
||||
assert_eq!(
|
||||
account["outboundProxyEndpoint"],
|
||||
serde_json::json!(proxy_url.map(|_| "socks5h://127.0.0.1:50101"))
|
||||
);
|
||||
assert!(!String::from_utf8(body.to_vec()).unwrap().contains("secret"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_cooldown_returns_an_instant_and_server_formatted_recovery() {
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
@@ -99,6 +160,90 @@ async fn account_cooldown_returns_an_instant_and_server_formatted_recovery() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_quota_refresh_time_uses_server_clock_and_deployment_timezone() {
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use gateway_admin::model::provider_credentials::ProviderQuota;
|
||||
use gateway_core::account::{AccountStatusFacts, CredentialState, QuotaState};
|
||||
use serde_json::json;
|
||||
|
||||
for zone in ["UTC", "Asia/Shanghai", "Asia/Kathmandu", "America/New_York"] {
|
||||
let zone = zone
|
||||
.parse::<gateway_core::time::DeploymentTimeZone>()
|
||||
.unwrap();
|
||||
let fixture = AdminTestFixture::with_timezone(zone).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture.account.lock().unwrap() = Some(account_page_item(
|
||||
AccountStatusFacts {
|
||||
enabled: true,
|
||||
credential_state: CredentialState::Ready,
|
||||
access_token_expires_at: None,
|
||||
quota: QuotaState::unknown(),
|
||||
cooldown: None,
|
||||
last_error_reason: None,
|
||||
last_error_message: None,
|
||||
},
|
||||
now,
|
||||
"oauth",
|
||||
));
|
||||
let future = now + Duration::days(1);
|
||||
let future_display = zone.local(future).format("%Y-%m-%d %H:%M:%S").to_string();
|
||||
for (observed_at, expected, legacy) in [
|
||||
(None, json!(null), "—"),
|
||||
(Some(now), json!({ "kind": "just_now" }), "刚刚"),
|
||||
(
|
||||
Some(now - Duration::seconds(90)),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "minute" }),
|
||||
"1 分钟前",
|
||||
),
|
||||
(
|
||||
Some(now - Duration::hours(25)),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "day" }),
|
||||
"1 天前",
|
||||
),
|
||||
(
|
||||
Some(future),
|
||||
json!({ "kind": "absolute", "value": future_display }),
|
||||
future_display.as_str(),
|
||||
),
|
||||
] {
|
||||
*fixture.provider_quota.lock().unwrap() = Some(ProviderQuota {
|
||||
plan_type: None,
|
||||
observed_at,
|
||||
refresh_token_expires_at: None,
|
||||
windows: Vec::new(),
|
||||
credits: None,
|
||||
limit_reached: false,
|
||||
provider_data: None,
|
||||
});
|
||||
let response = admin::accounts::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/accounts")
|
||||
.header("x-request-id", "req_quota_relative_time")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::ACCEPT_LANGUAGE, "en")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), 32768).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
let quota = &value["data"]["items"][0]["quota"];
|
||||
assert_eq!(quota["refreshedAtRelative"], expected);
|
||||
assert_eq!(quota["refreshedAtDisplay"], legacy);
|
||||
let refreshed_at = quota["refreshedAt"]
|
||||
.as_str()
|
||||
.map(|value| DateTime::parse_from_rfc3339(value).unwrap().to_utc());
|
||||
assert_eq!(refreshed_at, observed_at);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn account_page_item(
|
||||
facts: gateway_core::account::AccountStatusFacts,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
@@ -112,6 +257,7 @@ fn account_page_item(
|
||||
|
||||
AccountPageItem {
|
||||
account: AccountRecord {
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
id: "acct_cooldown".to_owned(),
|
||||
provider_kind: gateway_core::routing::ProviderKind::new("openai").unwrap(),
|
||||
|
||||
@@ -181,7 +181,13 @@ mod query {
|
||||
|
||||
#[test]
|
||||
fn account_query_should_reject_unknown_fields() {
|
||||
assert!(serde_json::from_value::<ListQuery>(json!({"id": "cred_1"})).is_err());
|
||||
let mut query = json!({"page": 1, "pageSize": 20});
|
||||
serde_json::from_value::<ListQuery>(query.clone())
|
||||
.unwrap()
|
||||
.validate()
|
||||
.expect("valid account query");
|
||||
query["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<ListQuery>(query).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -470,13 +476,13 @@ fn single_update_should_reject_oversized_notes_and_control_characters() {
|
||||
}
|
||||
|
||||
mod response {
|
||||
use gateway_api::admin::accounts::AccountUsageView;
|
||||
use gateway_api::admin::accounts::{AccountUsageView, AccountUsageWindowKind};
|
||||
use gateway_api::admin::presenter::format_decimal_currency;
|
||||
|
||||
#[test]
|
||||
fn account_usage_view_should_keep_unobserved_numbers_null() {
|
||||
let view = AccountUsageView {
|
||||
window_label_display: "周/月额度窗口".to_owned(),
|
||||
window_kind: AccountUsageWindowKind::Quota,
|
||||
request_count: None,
|
||||
request_count_display: "-".to_owned(),
|
||||
input_tokens: None,
|
||||
@@ -510,7 +516,7 @@ mod response {
|
||||
models: Vec::new(),
|
||||
};
|
||||
let value = serde_json::to_value(view).expect("serialize account usage");
|
||||
assert_eq!(value["windowLabelDisplay"], "周/月额度窗口");
|
||||
assert_eq!(value["windowKind"], "quota");
|
||||
assert!(value["inputTokens"].is_null());
|
||||
assert!(value["totalTokens"].is_null());
|
||||
assert!(value["reasoningOutputTokens"].is_null());
|
||||
|
||||
@@ -511,6 +511,7 @@ mod provider {
|
||||
total_slots: None,
|
||||
},
|
||||
account: AccountRecord {
|
||||
outbound_proxy_id: None,
|
||||
outbound_proxy_enabled: true,
|
||||
notes: None,
|
||||
model_access: Default::default(),
|
||||
|
||||
@@ -41,7 +41,7 @@ use gateway_admin::{
|
||||
DashboardObservation, DashboardQuery, DecimalAmount, DiagnosticDimension,
|
||||
DiagnosticsObservation, Granularity, OpsError, OpsErrorPage, OpsErrorQuery,
|
||||
RequestMetricPoint, TimeRange, UsageDetail, UsageFilter, UsageListRecord,
|
||||
UsageOverview, UsagePage, UsageQuery,
|
||||
UsageOverview, UsagePage, UsageQuery, UsageSummaryObservation,
|
||||
},
|
||||
provider_credentials::{
|
||||
AuthorizationCommit, AuthorizationStarted, CompleteAuthorization, CredentialDetails,
|
||||
@@ -112,6 +112,7 @@ pub(super) struct AdminTestFixture {
|
||||
pub dashboard_observation: Arc<Mutex<Option<DashboardObservation>>>,
|
||||
pub dashboard_summary_range: Arc<Mutex<Option<TimeRange>>>,
|
||||
pub provider_error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
pub provider_quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
pub account: Arc<Mutex<Option<AccountPageItem>>>,
|
||||
pub account_usage: Arc<Mutex<Vec<AccountUsage>>>,
|
||||
}
|
||||
@@ -187,9 +188,18 @@ impl AdminTestFixture {
|
||||
plugin_ports.clone(),
|
||||
plugin_ports.clone(),
|
||||
);
|
||||
let provider_quota = Arc::new(Mutex::new(None));
|
||||
let providers: Vec<Arc<dyn ProviderAdmin>> = vec![
|
||||
Arc::new(UnusedProvider::new("openai", Arc::clone(&provider_error))),
|
||||
Arc::new(UnusedProvider::new("xai", Arc::clone(&provider_error))),
|
||||
Arc::new(UnusedProvider::new(
|
||||
"openai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
)),
|
||||
Arc::new(UnusedProvider::new(
|
||||
"xai",
|
||||
Arc::clone(&provider_error),
|
||||
Arc::clone(&provider_quota),
|
||||
)),
|
||||
];
|
||||
let bundle = gateway_admin::initialize(
|
||||
AdminConfig {
|
||||
@@ -236,6 +246,7 @@ impl AdminTestFixture {
|
||||
dashboard_observation,
|
||||
dashboard_summary_range,
|
||||
provider_error,
|
||||
provider_quota,
|
||||
account,
|
||||
account_usage,
|
||||
}
|
||||
@@ -1547,12 +1558,17 @@ impl ObservabilityStore for UnusedStore {
|
||||
&self,
|
||||
range: TimeRange,
|
||||
filter: UsageFilter,
|
||||
) -> AdminStoreResult<UsageOverview> {
|
||||
) -> AdminStoreResult<UsageSummaryObservation> {
|
||||
let mut data = self.observations.lock().expect("observations");
|
||||
data.summaries.push((range, filter));
|
||||
data.summary
|
||||
.clone()
|
||||
.map(|summary| UsageOverview { range, ..summary })
|
||||
.map(|summary| UsageSummaryObservation {
|
||||
range,
|
||||
analytics: summary.analytics,
|
||||
requests: (&summary.requests).into(),
|
||||
attempts: summary.attempts,
|
||||
})
|
||||
.ok_or_else(|| unavailable("usage summary"))
|
||||
}
|
||||
|
||||
@@ -1585,13 +1601,19 @@ impl ObservabilityStore for UnusedStore {
|
||||
struct UnusedProvider {
|
||||
kind: ProviderKind,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
}
|
||||
|
||||
impl UnusedProvider {
|
||||
fn new(kind: &str, error: Arc<Mutex<Option<ProviderAdminError>>>) -> Self {
|
||||
fn new(
|
||||
kind: &str,
|
||||
error: Arc<Mutex<Option<ProviderAdminError>>>,
|
||||
quota: Arc<Mutex<Option<ProviderQuota>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind: ProviderKind::new(kind).expect("provider kind"),
|
||||
error,
|
||||
quota,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1686,6 +1708,9 @@ impl ProviderAdmin for UnusedProvider {
|
||||
&self,
|
||||
_: gateway_admin::model::provider_credentials::ProviderQuotaRequest,
|
||||
) -> Result<ProviderQuota, ProviderAdminError> {
|
||||
if let Some(quota) = self.quota.lock().expect("provider quota").clone() {
|
||||
return Ok(quota);
|
||||
}
|
||||
Err(self
|
||||
.error
|
||||
.lock()
|
||||
|
||||
@@ -34,7 +34,35 @@ async fn export(
|
||||
let body = to_bytes(response.into_body(), 20 * 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
(status, headers, serde_json::from_slice(&body).unwrap())
|
||||
let file = if status == StatusCode::OK {
|
||||
assert_eq!(headers[header::CONTENT_TYPE], "application/zip");
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(&body)).unwrap();
|
||||
assert_eq!(archive.len(), 3);
|
||||
let file: Value =
|
||||
serde_json::from_reader(archive.by_name("diagnostics.json").unwrap()).unwrap();
|
||||
for name in ["summary.txt", "README.txt"] {
|
||||
use std::io::Read as _;
|
||||
let mut text = String::new();
|
||||
archive
|
||||
.by_name(name)
|
||||
.unwrap()
|
||||
.read_to_string(&mut text)
|
||||
.unwrap();
|
||||
assert!(!text.is_empty());
|
||||
for marker in [
|
||||
"fixture-authorization",
|
||||
"fixture-refresh",
|
||||
"fixture-password",
|
||||
"fixture@example.invalid",
|
||||
] {
|
||||
assert!(!text.contains(marker), "archive leaked {marker} in {name}");
|
||||
}
|
||||
}
|
||||
file
|
||||
} else {
|
||||
serde_json::from_slice(&body).unwrap()
|
||||
};
|
||||
(status, headers, file)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -85,11 +113,15 @@ async fn diagnostic_export_keeps_retry_evidence_and_original_errors_but_not_cred
|
||||
.lock()
|
||||
.unwrap()
|
||||
.extend([error, unrelated]);
|
||||
let trace = json!({"droppedEvents": 0, "events": [
|
||||
let trace = json!({"droppedEvents": 5, "droppedEvidenceEvents": 1, "stageCountOverflow": 2,
|
||||
"stageCounts": {"provider.response.parse_gap": 1}, "events": [
|
||||
{"stage": "request.routed", "data": {"accountPolicy": {"maxAccountRotations": 3}}},
|
||||
{"stage": "account.selection", "data": {"candidates": [{"id": "acct_high", "weight": 2}, {"id": "acct_low", "weight": 1}]}},
|
||||
{"stage": "attempt.failed", "data": {"kind": "unavailable", "recovery": {"replaySafe": true}}},
|
||||
{"stage": "retry.decided", "data": {"phase": "execute", "retryable": false, "clientRetrySuppressed": true, "replayBarrierReached": false}},
|
||||
{"stage": "provider.response.parse_gap", "data": {"reason": "invalid_json"}},
|
||||
{"stage": "provider.request.input", "data": {"input": {"omittedItems": 10}}},
|
||||
{"stage": "upstream.event", "data": {"truncated": true, "jsonValid": false}},
|
||||
]});
|
||||
fixture.usage_detail.lock().unwrap().replace(UsageDetail {
|
||||
request: usage_record_with_account(
|
||||
@@ -108,10 +140,32 @@ async fn diagnostic_export_keeps_retry_evidence_and_original_errors_but_not_cred
|
||||
assert_eq!(status, StatusCode::OK, "{file}");
|
||||
assert_eq!(
|
||||
headers[header::CONTENT_DISPOSITION],
|
||||
"attachment; filename=diagnostics.json"
|
||||
"attachment; filename=cpr-diagnostics-req_err.zip"
|
||||
);
|
||||
assert_eq!(headers[header::CACHE_CONTROL], "private, no-store");
|
||||
assert_eq!(file["schemaVersion"], 5);
|
||||
assert_eq!(
|
||||
file["exportTruncated"], false,
|
||||
"capture loss is distinct from export truncation"
|
||||
);
|
||||
for reason in [
|
||||
"events_evicted",
|
||||
"critical_evidence_evicted",
|
||||
"stage_coverage_incomplete",
|
||||
"response_parse_failed",
|
||||
"input_prefix_sample_limited",
|
||||
"event_data_truncated",
|
||||
"json_parse_failed",
|
||||
] {
|
||||
assert!(
|
||||
file["availability"]["gaps"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|gap| gap["reason"] == reason),
|
||||
"missing gap {reason}"
|
||||
);
|
||||
}
|
||||
assert_eq!(file["requests"][0]["evidence"]["trace"], trace);
|
||||
assert_eq!(
|
||||
file["requests"][0]["evidence"]["incidentConfiguration"]["revision"],
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
//! 查询 wire 校验的固定合同测试
|
||||
|
||||
use gateway_api::admin::observability::{
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind, UsageQuery,
|
||||
parse_attempt_index, parse_datetime, parse_status,
|
||||
DashboardQuery, DiagnosticDimension, DiagnosticsQuery, OpsQuery, TrendKind,
|
||||
UsageAnalyticsQuery, UsageQuery, parse_attempt_index, parse_datetime, parse_status,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -33,6 +33,7 @@ fn observability_queries_should_reject_unknown_fields() {
|
||||
let query = json!({"unknownFilter": true});
|
||||
assert!(serde_json::from_value::<DashboardQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageAnalyticsQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<DiagnosticsQuery>(query).is_err());
|
||||
}
|
||||
|
||||
@@ -3,6 +3,105 @@
|
||||
use chrono::Utc;
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_summary_keeps_request_attempt_cost_and_average_latency_wire_fields() {
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Duration;
|
||||
use gateway_admin::model::observability::{
|
||||
AttemptMetrics, CostCoverage, CurrencyCost, RequestMetrics, TimeRange, UsageOverview,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
for (sum, count, display, raw) in [
|
||||
(404, 3, "134 ms", Some(134)),
|
||||
(0, 1, "0 ms", Some(0)),
|
||||
(0, 0, "—", None),
|
||||
] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
fixture.observations.lock().unwrap().summary = Some(UsageOverview {
|
||||
analytics: gateway_admin::model::observability::AnalyticsMetadata {
|
||||
generated_at: now,
|
||||
..Default::default()
|
||||
},
|
||||
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
|
||||
requests: RequestMetrics {
|
||||
request_count: 5,
|
||||
success_count: 4,
|
||||
cancelled_count: 1,
|
||||
caller_error_count: 1,
|
||||
input_tokens: 400,
|
||||
output_tokens: 80,
|
||||
cached_input_tokens: 30,
|
||||
cache_write_input_tokens: 7,
|
||||
reasoning_output_tokens: 9,
|
||||
total_tokens: 480,
|
||||
latency_sum_ms: sum,
|
||||
latency_count: count,
|
||||
..Default::default()
|
||||
},
|
||||
attempts: AttemptMetrics {
|
||||
attempt_count: 6,
|
||||
success_count: 4,
|
||||
failure_count: 1,
|
||||
cancelled_count: 1,
|
||||
rate_limited_count: 1,
|
||||
cost_coverage: CostCoverage {
|
||||
provider_reported_count: 3,
|
||||
unavailable_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
costs: vec![CurrencyCost {
|
||||
currency: "USD".into(),
|
||||
amount: "1.25".parse().unwrap(),
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
providers: Vec::new(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records/summary?provider=openai")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_summary_projection")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{value}");
|
||||
assert_eq!(
|
||||
value["data"],
|
||||
json!({
|
||||
"analytics": value["data"]["analytics"],
|
||||
"totalRequests": "5", "inputTokens": "400", "outputTokens": "80",
|
||||
"cachedInputTokens": "30", "cacheWriteInputTokens": "7", "totalTokens": "480",
|
||||
"averageLatencyMs": display,
|
||||
"averageLatencyMsValue": raw,
|
||||
"logicalRequests": {"requestCount":5,"successCount":4,"failureCount":0,"cancelledCount":1,"incompleteCount":0,"callerErrorCount":1,"inputTokens":400,"outputTokens":80,"cachedInputTokens":30,"cacheWriteInputTokens":7,"reasoningOutputTokens":9,"totalTokens":480},
|
||||
"attempts": {"attemptCount":6,"successCount":4,"failureCount":1,"cancelledCount":1,"incompleteCount":0,"rateLimitedCount":1,"authFailureCount":0,"provider5xxCount":0,"costCoverage":{"known":3,"partial":0,"unknown":1,"notBillable":0},"costs":[{"currency":"USD","estimatedAmount":"1.25"}]}
|
||||
})
|
||||
);
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
assert_eq!(observations.summaries.len(), 1);
|
||||
assert_eq!(
|
||||
observations.summaries[0].1.provider_kind.as_deref(),
|
||||
Some("openai")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_include_quota_exhaustion_in_unavailable_headline_count() {
|
||||
use axum::{
|
||||
@@ -725,7 +824,7 @@ pub(super) fn usage_record_with_account(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
async fn usage_route_should_forward_name_and_model_contains_filters() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode, header},
|
||||
@@ -741,7 +840,7 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/api/admin/usage/records?outcome=provider_future_state")
|
||||
.uri("/api/admin/usage/records?model=%20coding%20&clientApiKeyName=%20Key%20%25%20&accountName=%20Account%20Name%20")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header("x-request-id", "req_usage_other_outcome")
|
||||
.body(Body::empty())
|
||||
@@ -751,6 +850,16 @@ async fn usage_route_should_forward_a_bounded_unknown_outcome_filter() {
|
||||
.expect("usage response");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let observations = fixture.observations.lock().unwrap();
|
||||
let filter = &observations.records.last().expect("usage query").filter;
|
||||
assert_eq!(filter.client_api_key_name.as_deref(), Some("Key %"));
|
||||
assert_eq!(filter.model_contains.as_deref(), Some("coding"));
|
||||
assert_eq!(
|
||||
filter.provider_account_name.as_deref(),
|
||||
Some("Account Name")
|
||||
);
|
||||
assert!(filter.client_api_key_ref.is_none());
|
||||
assert!(filter.provider_account_ref.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1360,3 +1469,301 @@ async fn usage_status_query_accepts_three_modes_and_rejects_unknown_values() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_summary_should_identify_metrics_without_localized_labels() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::Duration;
|
||||
use gateway_admin::model::{
|
||||
accounts::AccountUsage,
|
||||
observability::{DashboardAccountUsage, DashboardObservation, TimeRange},
|
||||
provider_credentials::{ProviderQuotaWindow, QuotaLocalUsageAttribution},
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let now = Utc::now();
|
||||
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
|
||||
yesterday: Default::default(),
|
||||
analytics: Default::default(),
|
||||
runtime_slots: None,
|
||||
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: Default::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: [
|
||||
(None, Some(2_000), Some(now)),
|
||||
(None, Some(0), None),
|
||||
(None, None, None),
|
||||
(Some(9), Some(2_000), Some(now)),
|
||||
(Some(0), Some(2_000), None),
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, (requests, tokens, last_used_at))| {
|
||||
let account_id = format!("acct_metric_{index}");
|
||||
DashboardAccountUsage {
|
||||
account_id: account_id.clone(),
|
||||
provider_kind: "openai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
name: account_id.clone(),
|
||||
email: None,
|
||||
plan_type: None,
|
||||
plan_type_display: None,
|
||||
request_count: 10,
|
||||
success_count: 10,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: tokens,
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at,
|
||||
request_buckets: Vec::new(),
|
||||
quota_used_percent: None,
|
||||
quota_window: requests.map(|request_count| ProviderQuotaWindow {
|
||||
key: "rolling".to_owned(),
|
||||
group: "shortTerm".to_owned(),
|
||||
label: "滚动窗口".to_owned(),
|
||||
limit_id: None,
|
||||
limit_name: None,
|
||||
role: None,
|
||||
local_usage_attribution: QuotaLocalUsageAttribution::AccountWide,
|
||||
window_seconds: Some(86_400),
|
||||
used_percent: None,
|
||||
reset_at: None,
|
||||
limit_reached: false,
|
||||
local_usage: Some(AccountUsage {
|
||||
account_id,
|
||||
request_count,
|
||||
success_count: request_count,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: None,
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at,
|
||||
request_buckets: Vec::new(),
|
||||
models: Vec::new(),
|
||||
}),
|
||||
provider_data: None,
|
||||
}),
|
||||
models: Vec::new(),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
|
||||
for (kind, metrics) in [
|
||||
(
|
||||
"usage",
|
||||
["inputTokens", "outputTokens", "cachedInputTokens"],
|
||||
),
|
||||
(
|
||||
"latency",
|
||||
["firstTokenP95Ms", "latencyP95Ms", "outputThroughputP50"],
|
||||
),
|
||||
("errors", ["errors", "successRate", "requests"]),
|
||||
] {
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!("/api/admin/dashboard/summary?kind={kind}"))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::ACCEPT_LANGUAGE, "en")
|
||||
.header("x-request-id", format!("req_trend_metrics_{kind}"))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{kind}: {value}");
|
||||
let accounts = value["data"]["accountUsage"].as_array().unwrap();
|
||||
assert_eq!(accounts.len(), 5);
|
||||
for (account, (metric, raw, display, last_used_at)) in accounts.iter().zip([
|
||||
("tokens", Some(2_000), "2K", Some(now)),
|
||||
("tokens", Some(0), "0", None),
|
||||
("tokens", None, "—", None),
|
||||
("requests", Some(9), "9", Some(now)),
|
||||
("requests", Some(0), "—", None),
|
||||
]) {
|
||||
assert_eq!(account["metric"], metric);
|
||||
assert_eq!(account["metricValueRaw"], json!(raw));
|
||||
assert_eq!(account["metricValue"], display);
|
||||
assert_eq!(account["lastUsedAt"], json!(last_used_at));
|
||||
}
|
||||
let summary = value["data"]["trend"]["summary"].as_array().unwrap();
|
||||
assert_eq!(summary.len(), metrics.len());
|
||||
for (item, metric) in summary.iter().zip(metrics) {
|
||||
assert_eq!(item["metric"], metric);
|
||||
assert!(item.get("label").is_none());
|
||||
assert!(item["value"].is_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_relative_time_preserves_server_anchor_boundaries_and_deployment_timezone() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use chrono::{DateTime, Duration};
|
||||
use gateway_admin::model::observability::{
|
||||
DashboardAccountUsage, DashboardObservation, TimeRange,
|
||||
};
|
||||
use gateway_api::admin::observability;
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
use crate::admin::{AdminTestFixture, AdminTestState};
|
||||
|
||||
let now = "2026-06-15T00:00:00Z".parse::<DateTime<Utc>>().unwrap();
|
||||
for (zone, future_display) in [
|
||||
("UTC", "2026-06-15 00:00:01"),
|
||||
("Asia/Shanghai", "2026-06-15 08:00:01"),
|
||||
("Asia/Kathmandu", "2026-06-15 05:45:01"),
|
||||
("America/New_York", "2026-06-14 20:00:01"),
|
||||
] {
|
||||
for (elapsed_ms, expected, legacy) in [
|
||||
(None, json!(null), "—"),
|
||||
(
|
||||
Some(-1_000),
|
||||
json!({ "kind": "absolute", "value": future_display }),
|
||||
future_display,
|
||||
),
|
||||
(Some(-1), json!({ "kind": "just_now" }), "刚刚"),
|
||||
(Some(0), json!({ "kind": "just_now" }), "刚刚"),
|
||||
(Some(59_999), json!({ "kind": "just_now" }), "刚刚"),
|
||||
(
|
||||
Some(60_000),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "minute" }),
|
||||
"1 分钟前",
|
||||
),
|
||||
(
|
||||
Some(119_999),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "minute" }),
|
||||
"1 分钟前",
|
||||
),
|
||||
(
|
||||
Some(3_599_999),
|
||||
json!({ "kind": "elapsed", "value": 59, "unit": "minute" }),
|
||||
"59 分钟前",
|
||||
),
|
||||
(
|
||||
Some(3_600_000),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "hour" }),
|
||||
"1 小时前",
|
||||
),
|
||||
(
|
||||
Some(86_399_999),
|
||||
json!({ "kind": "elapsed", "value": 23, "unit": "hour" }),
|
||||
"23 小时前",
|
||||
),
|
||||
(
|
||||
Some(86_400_000),
|
||||
json!({ "kind": "elapsed", "value": 1, "unit": "day" }),
|
||||
"1 天前",
|
||||
),
|
||||
(
|
||||
Some(34_560_000_000),
|
||||
json!({ "kind": "elapsed", "value": 400, "unit": "day" }),
|
||||
"400 天前",
|
||||
),
|
||||
] {
|
||||
// 每个边界使用独立缓存,避免相同锚点复用上一轮观测数据
|
||||
let fixture = AdminTestFixture::with_timezone(zone.parse().unwrap()).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let last_used_at = elapsed_ms.map(|value| now - Duration::milliseconds(value));
|
||||
*fixture.dashboard_observation.lock().unwrap() = Some(DashboardObservation {
|
||||
yesterday: Default::default(),
|
||||
analytics: Default::default(),
|
||||
runtime_slots: None,
|
||||
range: TimeRange::new(now - Duration::hours(1), now).unwrap(),
|
||||
totals: Default::default(),
|
||||
provider_accounts: Default::default(),
|
||||
trend: Vec::new(),
|
||||
account_usage: vec![DashboardAccountUsage {
|
||||
account_id: "acct_relative_time".to_owned(),
|
||||
provider_kind: "openai".to_owned(),
|
||||
authentication_kind: "oauth".to_owned(),
|
||||
name: "Relative time fixture".to_owned(),
|
||||
email: None,
|
||||
plan_type: None,
|
||||
plan_type_display: None,
|
||||
request_count: 1,
|
||||
success_count: 1,
|
||||
input_tokens: None,
|
||||
output_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
cache_write_input_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
image_input_tokens: None,
|
||||
image_output_tokens: None,
|
||||
image_request_count: 0,
|
||||
image_request_failed_count: 0,
|
||||
total_tokens: None,
|
||||
cost_coverage: Default::default(),
|
||||
costs: Vec::new(),
|
||||
last_used_at,
|
||||
request_buckets: Vec::new(),
|
||||
quota_used_percent: None,
|
||||
quota_window: None,
|
||||
models: Vec::new(),
|
||||
}],
|
||||
recent_requests: Vec::new(),
|
||||
});
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri(format!(
|
||||
"/api/admin/dashboard/summary?asOf={}",
|
||||
now.timestamp_millis()
|
||||
))
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::ACCEPT_LANGUAGE, "en")
|
||||
.header("x-request-id", "req_relative_time")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(status, StatusCode::OK, "{zone}: {value}");
|
||||
let account = &value["data"]["accountUsage"][0];
|
||||
assert_eq!(
|
||||
account["lastUsedRelative"], expected,
|
||||
"{zone}: {elapsed_ms:?}"
|
||||
);
|
||||
assert_eq!(account["lastUsed"], legacy, "{zone}: {elapsed_ms:?}");
|
||||
assert_eq!(account["lastUsedAt"], json!(last_used_at));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ use gateway_core::routing::extensions::ExtensionSetLease;
|
||||
use gateway_core::routing::extensions::ExtensionSetReference;
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
use tower::ServiceExt as _;
|
||||
|
||||
@@ -41,6 +41,7 @@ use super::super::AdminTestFixture;
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum ModelReply {
|
||||
Complete,
|
||||
SessionError,
|
||||
PendingResponse,
|
||||
PendingBody,
|
||||
}
|
||||
@@ -50,6 +51,7 @@ struct ModelExecution {
|
||||
cancellation: Mutex<Option<CancellationToken>>,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
closed: Arc<AtomicBool>,
|
||||
body_control: Arc<ModelBodyControl>,
|
||||
}
|
||||
|
||||
impl ModelExecution {
|
||||
@@ -59,6 +61,7 @@ impl ModelExecution {
|
||||
cancellation: Mutex::new(None),
|
||||
entered: Arc::default(),
|
||||
closed: Arc::default(),
|
||||
body_control: Arc::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -92,6 +95,7 @@ impl ExecutionService for ModelExecution {
|
||||
reply: self.reply,
|
||||
entered: self.entered.clone(),
|
||||
closed: self.closed.clone(),
|
||||
body_control: self.body_control.clone(),
|
||||
}),
|
||||
ExtensionSetReference::new(
|
||||
ExtensionSetId::new("model-bridge-test".into()).unwrap(),
|
||||
@@ -122,6 +126,7 @@ struct ModelMiddleware {
|
||||
reply: ModelReply,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
closed: Arc<AtomicBool>,
|
||||
body_control: Arc<ModelBodyControl>,
|
||||
}
|
||||
|
||||
impl MiddlewarePlan for ModelMiddleware {
|
||||
@@ -152,12 +157,19 @@ impl MiddlewarePlan for ModelMiddleware {
|
||||
let reply = self.reply;
|
||||
let closed = self.closed.clone();
|
||||
let entered = self.entered.clone();
|
||||
let body_control = self.body_control.clone();
|
||||
Box::pin(async move {
|
||||
entered.notify_one();
|
||||
if matches!(reply, ModelReply::PendingResponse) {
|
||||
return futures::future::pending().await;
|
||||
}
|
||||
let (bytes, framing, terminal) = if matches!(reply, ModelReply::PendingBody) {
|
||||
let (bytes, framing, terminal) = if matches!(reply, ModelReply::SessionError) {
|
||||
(
|
||||
"{\"code\":40101,\"message\":\"plugin error\",\"data\":null}",
|
||||
MiddlewareFraming::JsonDocument,
|
||||
true,
|
||||
)
|
||||
} else if matches!(reply, ModelReply::PendingBody) {
|
||||
(
|
||||
"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_bridge\",\"object\":\"response\",\"status\":\"in_progress\",\"output\":[]}}\n\n",
|
||||
MiddlewareFraming::SseEvent,
|
||||
@@ -172,32 +184,67 @@ impl MiddlewarePlan for ModelMiddleware {
|
||||
};
|
||||
Ok(MiddlewareResponse::new(
|
||||
"openai".into(),
|
||||
200,
|
||||
if matches!(reply, ModelReply::SessionError) {
|
||||
401
|
||||
} else {
|
||||
200
|
||||
},
|
||||
vec![
|
||||
MiddlewareHeader::new("set-cookie", "fixture=value".into()),
|
||||
MiddlewareHeader::new("www-authenticate", "Bearer".into()),
|
||||
MiddlewareHeader::new("x-plugin-response", "complete".into()),
|
||||
MiddlewareHeader::new("x-cpr-plugin-response", "untrusted".into()),
|
||||
],
|
||||
Box::new(ModelBody {
|
||||
frame: Some(MiddlewareFrame::new(bytes.into(), framing, terminal)),
|
||||
pending: !terminal,
|
||||
closed,
|
||||
control: body_control,
|
||||
}),
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct ModelBodyControl {
|
||||
delayed_frame: AtomicBool,
|
||||
pending_reads: AtomicUsize,
|
||||
waiting: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
struct ModelBody {
|
||||
frame: Option<MiddlewareFrame>,
|
||||
pending: bool,
|
||||
closed: Arc<AtomicBool>,
|
||||
control: Arc<ModelBodyControl>,
|
||||
}
|
||||
|
||||
impl MiddlewareBody for ModelBody {
|
||||
fn next_frame(&mut self) -> BoxFuture<'_, Result<Option<MiddlewareFrame>, MiddlewareError>> {
|
||||
let frame = self.frame.take();
|
||||
let delayed_frame = if frame.is_none() && self.pending {
|
||||
self.control.pending_reads.fetch_add(1, Ordering::SeqCst);
|
||||
self.control.waiting.notify_one();
|
||||
// 模拟读取在 future 创建时已经消费事件,取消后不能重新取得原帧
|
||||
self.control.delayed_frame.swap(false, Ordering::SeqCst).then(|| {
|
||||
MiddlewareFrame::new(
|
||||
"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_bridge\",\"object\":\"response\",\"status\":\"completed\",\"output\":[]}}\n\n".into(),
|
||||
MiddlewareFraming::SseEvent,
|
||||
true,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Box::pin(async move {
|
||||
if let Some(frame) = self.frame.take() {
|
||||
if let Some(frame) = frame {
|
||||
return Ok(Some(frame));
|
||||
}
|
||||
if let Some(frame) = delayed_frame {
|
||||
self.control.release.notified().await;
|
||||
self.pending = false;
|
||||
return Ok(Some(frame));
|
||||
}
|
||||
if self.pending {
|
||||
@@ -245,6 +292,7 @@ async fn management_calls_and_callbacks_preserve_headers_and_override_defaults()
|
||||
.header("set-cookie", "first=1")
|
||||
.header("set-cookie", "second=2")
|
||||
.header("content-security-policy", "default-src 'self'")
|
||||
.header("x-cpr-plugin-response", "untrusted")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -273,6 +321,15 @@ async fn management_calls_and_callbacks_preserve_headers_and_override_defaults()
|
||||
response.headers()["content-security-policy"],
|
||||
"default-src 'self'"
|
||||
);
|
||||
assert_eq!(response.headers()["x-cpr-plugin-response"], "1");
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-cpr-plugin-response")
|
||||
.iter()
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,6 +345,7 @@ async fn model_bridge_preserves_complete_request_and_response_headers() {
|
||||
assert_eq!(response.headers()["set-cookie"], "fixture=value");
|
||||
assert_eq!(response.headers()["www-authenticate"], "Bearer");
|
||||
assert_eq!(response.headers()["x-plugin-response"], "complete");
|
||||
assert_eq!(response.headers()["x-cpr-plugin-response"], "1");
|
||||
assert_eq!(response.headers()["cache-control"], "no-store");
|
||||
let body = axum::body::to_bytes(response.into_body(), 4096)
|
||||
.await
|
||||
@@ -298,6 +356,40 @@ async fn model_bridge_preserves_complete_request_and_response_headers() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_bridge_distinguishes_host_authentication_from_plugin_session_errors() {
|
||||
let fixture = published_fixture().await;
|
||||
let router = crate::openai::api_router_with_admin_and_execution(
|
||||
fixture.services,
|
||||
ModelExecution::new(ModelReply::SessionError),
|
||||
);
|
||||
let mut unauthenticated = model_request(false);
|
||||
unauthenticated.headers_mut().remove("cookie");
|
||||
unauthenticated.headers_mut().remove("authorization");
|
||||
unauthenticated.headers_mut().remove("x-api-key");
|
||||
let response = router.clone().oneshot(unauthenticated).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert!(!response.headers().contains_key("x-cpr-plugin-response"));
|
||||
let body = axum::body::to_bytes(response.into_body(), 4096)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).unwrap()["code"],
|
||||
40101
|
||||
);
|
||||
|
||||
let response = router.oneshot(model_request(false)).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(response.headers()["x-cpr-plugin-response"], "1");
|
||||
let body = axum::body::to_bytes(response.into_body(), 4096)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).unwrap()["code"],
|
||||
40101
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn model_bridge_revokes_while_waiting_for_the_first_response() {
|
||||
let fixture = published_fixture().await;
|
||||
@@ -358,6 +450,52 @@ async fn model_bridge_revokes_and_closes_a_pending_stream() {
|
||||
assert!(execution.closed.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn model_bridge_authorization_rechecks_preserve_the_pending_frame_read() {
|
||||
let fixture = published_fixture().await;
|
||||
let execution = ModelExecution::new(ModelReply::PendingBody);
|
||||
execution
|
||||
.body_control
|
||||
.delayed_frame
|
||||
.store(true, Ordering::SeqCst);
|
||||
let response =
|
||||
crate::openai::api_router_with_admin_and_execution(fixture.services, execution.clone())
|
||||
.oneshot(model_request(true))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = tokio::spawn(axum::body::to_bytes(response.into_body(), 4096));
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
execution.body_control.waiting.notified(),
|
||||
)
|
||||
.await
|
||||
.expect("第二帧读取已开始");
|
||||
for _ in 0..3 {
|
||||
tokio::time::advance(std::time::Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
execution.body_control.pending_reads.load(Ordering::SeqCst),
|
||||
1
|
||||
);
|
||||
assert!(!body.is_finished());
|
||||
}
|
||||
execution.body_control.release.notify_one();
|
||||
let body = tokio::time::timeout(std::time::Duration::from_secs(5), body)
|
||||
.await
|
||||
.expect("原先等待的终态帧应完成交付")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body = std::str::from_utf8(&body).unwrap();
|
||||
assert_eq!(body.matches("event: response.completed").count(), 1);
|
||||
assert!(!body.contains("response.failed"));
|
||||
assert_eq!(
|
||||
execution.body_control.pending_reads.load(Ordering::SeqCst),
|
||||
1
|
||||
);
|
||||
assert!(execution.closed.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
pub(super) fn resource_fixture(
|
||||
path: &str,
|
||||
) -> gateway_admin::model::plugins::management::PluginManagementResponse {
|
||||
|
||||
@@ -472,16 +472,15 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
fn settings_service_contract_flattens_values_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::settings::{ReplaceRuntimeSettings, RuntimeSettings};
|
||||
|
||||
let current = super::test_runtime_settings();
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<RuntimeSettings>(&serde_json::to_vec(¤t).unwrap()).unwrap(),
|
||||
current
|
||||
);
|
||||
let mut record = serde_json::to_value(¤t).unwrap();
|
||||
assert!(record.get("values").is_none());
|
||||
assert_eq!(
|
||||
record["refresh_margin_seconds"],
|
||||
current.values.refresh_margin_seconds
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<RuntimeSettings>(record.clone()).unwrap(),
|
||||
current
|
||||
@@ -490,13 +489,11 @@ fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
assert!(serde_json::from_value::<RuntimeSettings>(record).is_err());
|
||||
|
||||
let command = ReplaceRuntimeSettings::from(current);
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<ReplaceRuntimeSettings>(&serde_json::to_vec(&command).unwrap())
|
||||
.unwrap(),
|
||||
command
|
||||
);
|
||||
let mut wire = serde_json::to_value(&command).unwrap();
|
||||
assert!(wire.get("values").is_none());
|
||||
assert_eq!(
|
||||
wire["refresh_margin_seconds"],
|
||||
command.values.refresh_margin_seconds
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ReplaceRuntimeSettings>(wire.clone()).unwrap(),
|
||||
command
|
||||
@@ -1430,10 +1427,10 @@ async fn guardian_reservation_round_trips_and_rejects_invalid_values() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_affinity_and_rotation_budget_round_trip_and_reject_invalid_values() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
for (mode, rotations, ttl) in [("relaxed", 0, 1), ("preferred", 3, 24), ("strict", 31, 720)] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body["openaiAccountAffinity"] = json!(mode);
|
||||
body["maxAccountRotations"] = json!(rotations);
|
||||
body["openaiSessionAffinityTtlHours"] = json!(ttl);
|
||||
@@ -1441,7 +1438,7 @@ async fn account_affinity_and_rotation_budget_round_trip_and_reject_invalid_valu
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
Some(body.clone()),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1450,7 +1447,9 @@ async fn account_affinity_and_rotation_budget_round_trip_and_reject_invalid_valu
|
||||
assert_eq!(response["data"]["openaiAccountAffinity"], mode);
|
||||
assert_eq!(response["data"]["maxAccountRotations"], rotations);
|
||||
assert_eq!(response["data"]["openaiSessionAffinityTtlHours"], ttl);
|
||||
body["configRevision"] = response["data"]["configRevision"].clone();
|
||||
}
|
||||
let saved = fixture.settings.settings.lock().unwrap().clone();
|
||||
// 类型错误沿用 AdminJson 的 422,合法类型越界由字段校验返回 400
|
||||
for (field, value, expected_status) in [
|
||||
(
|
||||
@@ -1485,20 +1484,19 @@ async fn account_affinity_and_rotation_budget_round_trip_and_reject_invalid_valu
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
),
|
||||
] {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let mut body = update_body();
|
||||
body[field] = value;
|
||||
let mut invalid = body.clone();
|
||||
invalid[field] = value;
|
||||
let response = app(fixture.state())
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
Some(invalid),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), expected_status, "{field}");
|
||||
}
|
||||
assert_eq!(*fixture.settings.settings.lock().unwrap(), saved);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,29 +1,7 @@
|
||||
//! 验证管理响应封装、分页结构与公开错误码保持稳定
|
||||
|
||||
mod common {
|
||||
use gateway_api::admin::PageMeta;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn page_meta_should_keep_the_existing_camel_case_wire_shape() {
|
||||
let value = serde_json::to_value(PageMeta::new(2, 10, 21, 3))
|
||||
.expect("page metadata should serialize");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"page": 2,
|
||||
"pageSize": 10,
|
||||
"total": 21,
|
||||
"totalPages": 3
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod error {
|
||||
use gateway_api::admin::{AdminErrorBody, AdminErrorCode};
|
||||
use serde_json::json;
|
||||
use gateway_api::admin::AdminErrorCode;
|
||||
|
||||
#[test]
|
||||
fn error_codes_should_keep_all_published_numeric_values() {
|
||||
@@ -56,24 +34,6 @@ mod error {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_admin_api_key_should_keep_code_and_null_data() {
|
||||
let value = serde_json::to_value(AdminErrorBody::new(
|
||||
AdminErrorCode::INVALID_API_KEY,
|
||||
"管理 API Key 无效",
|
||||
))
|
||||
.expect("admin error body should serialize");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"code": 40103,
|
||||
"message": "管理 API Key 无效",
|
||||
"data": null
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mod response {
|
||||
@@ -85,21 +45,6 @@ mod response {
|
||||
use gateway_api::admin::{AdminEnvelope, AdminError, AdminPageData, AdminResponse, PageMeta};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn success_envelope_should_keep_the_existing_wire_shape() {
|
||||
let value = serde_json::to_value(AdminEnvelope::ok(json!({ "id": "resource_1" })))
|
||||
.expect("admin success envelope should serialize");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"code": 200,
|
||||
"message": "OK",
|
||||
"data": { "id": "resource_1" }
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_envelope_should_keep_items_and_page_nested_under_data() {
|
||||
let data = AdminPageData::new(
|
||||
@@ -145,8 +90,15 @@ mod response {
|
||||
let body = response_json(response).await;
|
||||
|
||||
assert_eq!(
|
||||
(status, body["code"].as_u64()),
|
||||
(StatusCode::CREATED, Some(200))
|
||||
(status, body),
|
||||
(
|
||||
StatusCode::CREATED,
|
||||
json!({
|
||||
"code": 200,
|
||||
"message": "OK",
|
||||
"data": { "id": "resource_created" }
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -228,11 +228,14 @@ async fn overview_scopes_every_query_and_projects_only_key_visible_fields() {
|
||||
assert!(filter.provider_account_ref.is_none());
|
||||
assert!(filter.provider_kind.is_none());
|
||||
}
|
||||
assert_eq!(observations.summaries[0].1.model.as_deref(), Some("coding"));
|
||||
assert_eq!(
|
||||
observations.summaries[0].1.model_contains.as_deref(),
|
||||
Some("coding")
|
||||
);
|
||||
let health = observations
|
||||
.trends
|
||||
.iter()
|
||||
.find(|(_, filter)| filter.model.is_none())
|
||||
.find(|(_, filter)| filter.model_contains.is_none())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
health.0.start,
|
||||
@@ -346,7 +349,7 @@ async fn records_keep_pagination_and_hide_admin_and_upstream_data() {
|
||||
]) {
|
||||
assert_eq!(query.status, status);
|
||||
assert_eq!(query.filter.client_api_key_ref.as_deref(), Some("key-42"));
|
||||
assert_eq!(query.filter.model.as_deref(), Some("coding"));
|
||||
assert_eq!(query.filter.model_contains.as_deref(), Some("coding"));
|
||||
assert!(query.filter.provider_account_ref.is_none());
|
||||
assert!(query.filter.provider_kind.is_none());
|
||||
}
|
||||
|
||||
@@ -2,18 +2,22 @@
|
||||
|
||||
mod websocket;
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
body::{Body, Bytes, to_bytes},
|
||||
http::{Request, StatusCode, header::AUTHORIZATION},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::engine::execution::{
|
||||
AuthenticatedClient, ClientAuthenticationError, ExecutionRequestMetadata, ExecutionService,
|
||||
StartExecution, StartProviderExecution, StartedExecution,
|
||||
ExecutionSession, StartExecution, StartProviderExecution, StartedExecution,
|
||||
};
|
||||
use gateway_core::engine::{CoordinatedEvent, EngineError, ModelRequestId};
|
||||
use gateway_core::error::{GatewayError, GatewayErrorKind};
|
||||
use gateway_core::event::{ProtocolWireEvent, ProviderEvent, ProviderResponseHeader};
|
||||
use gateway_core::operation::Operation;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
@@ -29,6 +33,7 @@ const LIVE_KEY: &str = "sk_live_test";
|
||||
struct LiveCapture {
|
||||
captured: Arc<Mutex<Option<StartProviderExecution>>>,
|
||||
gateway: Option<Arc<dyn gateway_core::live::LiveGateway>>,
|
||||
response_headers: Option<Vec<ProviderResponseHeader>>,
|
||||
}
|
||||
|
||||
impl LiveCapture {
|
||||
@@ -84,7 +89,18 @@ impl ExecutionService for LiveCapture {
|
||||
request: StartProviderExecution,
|
||||
) -> BoxFuture<'_, Result<StartedExecution, GatewayError>> {
|
||||
*self.captured.lock().expect("capture") = Some(request);
|
||||
Box::pin(async {
|
||||
Box::pin(async move {
|
||||
if let Some(headers) = &self.response_headers {
|
||||
return Ok(StartedExecution {
|
||||
request_id: ModelRequestId::new("req_live_headers").unwrap(),
|
||||
created_at: SystemTime::now(),
|
||||
stream: false,
|
||||
session: Box::new(LiveSession {
|
||||
headers: headers.clone(),
|
||||
finalized: AtomicBool::new(false),
|
||||
}),
|
||||
});
|
||||
}
|
||||
Err(GatewayError::new(
|
||||
GatewayErrorKind::Internal,
|
||||
"live test captured the provider endpoint request",
|
||||
@@ -93,6 +109,214 @@ impl ExecutionService for LiveCapture {
|
||||
}
|
||||
}
|
||||
|
||||
struct LiveSession {
|
||||
headers: Vec<ProviderResponseHeader>,
|
||||
finalized: AtomicBool,
|
||||
}
|
||||
|
||||
impl ExecutionSession for LiveSession {
|
||||
fn next_event(&mut self) -> BoxFuture<'_, Result<Option<CoordinatedEvent>, EngineError>> {
|
||||
Box::pin(async { unreachable!("Live bootstrap is buffered") })
|
||||
}
|
||||
|
||||
fn collect_uncommitted(&mut self) -> BoxFuture<'_, Result<Vec<ProviderEvent>, EngineError>> {
|
||||
Box::pin(async {
|
||||
Ok(vec![ProviderEvent::wire(
|
||||
ProtocolWireEvent::raw_http_body("openai", Bytes::from_static(b"v=0\r\n")).unwrap(),
|
||||
)])
|
||||
})
|
||||
}
|
||||
|
||||
fn response_headers(&self) -> &[ProviderResponseHeader] {
|
||||
&self.headers
|
||||
}
|
||||
|
||||
fn response_status_code(&self) -> Option<u16> {
|
||||
Some(201)
|
||||
}
|
||||
|
||||
fn commit_downstream(&mut self, _: Option<u16>) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async move {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn is_finalized(&self) -> bool {
|
||||
self.finalized.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn record_client_status(&mut self, _: u16) -> BoxFuture<'_, Result<(), EngineError>> {
|
||||
Box::pin(async { Ok(()) })
|
||||
}
|
||||
|
||||
fn cancel(&self) {
|
||||
self.finalized.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn detach_finalize(self: Box<Self>) -> BoxFuture<'static, ()> {
|
||||
Box::pin(async move { self.cancel() })
|
||||
}
|
||||
}
|
||||
|
||||
fn live_response_headers() -> Vec<ProviderResponseHeader> {
|
||||
[
|
||||
("content-type", b"application/sdp".as_slice()),
|
||||
("location", b"https://api.openai.com/v1/live/call_headers"),
|
||||
("x-test-response", b"first"),
|
||||
("x-test-response", b"\x80second"),
|
||||
("set-cookie", b"private=1"),
|
||||
("openai-organization", b"private-org"),
|
||||
("connection", b"x-test-hop"),
|
||||
("x-test-hop", b"private"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, value)| ProviderResponseHeader::new(name, Bytes::from_static(value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
struct HangupGateway;
|
||||
|
||||
impl gateway_core::live::LiveGateway for HangupGateway {
|
||||
fn open_sideband<'a>(
|
||||
&'a self,
|
||||
_: gateway_core::live::LiveSidebandRequest<'a>,
|
||||
) -> BoxFuture<'a, Result<gateway_core::live::LiveRelay, gateway_core::live::LiveGatewayError>>
|
||||
{
|
||||
Box::pin(async { unreachable!("hangup only") })
|
||||
}
|
||||
|
||||
fn hangup<'a>(
|
||||
&'a self,
|
||||
request: gateway_core::live::LiveHangupRequest<'a>,
|
||||
) -> BoxFuture<
|
||||
'a,
|
||||
Result<gateway_core::live::LiveCallOutcome, gateway_core::live::LiveGatewayError>,
|
||||
> {
|
||||
Box::pin(async move {
|
||||
assert_eq!(request.call_id, "call_headers");
|
||||
assert_eq!(
|
||||
request
|
||||
.protocol_headers
|
||||
.iter()
|
||||
.filter(|header| header.name() == "x-test-business")
|
||||
.map(|header| header.value())
|
||||
.collect::<Vec<_>>(),
|
||||
[b"first".as_slice(), b"\x80second".as_slice()]
|
||||
);
|
||||
assert!(
|
||||
request
|
||||
.protocol_headers
|
||||
.iter()
|
||||
.any(|header| header.name() == "voice-session-id"
|
||||
&& header.value().as_ref() == b"voice-test")
|
||||
);
|
||||
assert!(request.protocol_headers.iter().all(|header| {
|
||||
!["authorization", "connection", "x-test-hop"].contains(&header.name())
|
||||
}));
|
||||
Ok(gateway_core::live::LiveCallOutcome {
|
||||
status: 200,
|
||||
headers: live_response_headers(),
|
||||
body: Bytes::new(),
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_hangup_should_preserve_business_header_bytes_in_both_directions() {
|
||||
let capture = Arc::new(LiveCapture {
|
||||
gateway: Some(Arc::new(HangupGateway)),
|
||||
..LiveCapture::default()
|
||||
});
|
||||
let response = api_router(capture)
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/realtime/calls/call_headers/hangup")
|
||||
.header(AUTHORIZATION, format!("Bearer {LIVE_KEY}"))
|
||||
.header("voice-session-id", "voice-test")
|
||||
.header("x-test-business", "first")
|
||||
.header(
|
||||
"x-test-business",
|
||||
axum::http::HeaderValue::from_bytes(b"\x80second").unwrap(),
|
||||
)
|
||||
.header("connection", "x-test-hop")
|
||||
.header("x-test-hop", "private")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-test-response")
|
||||
.iter()
|
||||
.map(|value| value.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
[b"first".as_slice(), b"\x80second".as_slice()]
|
||||
);
|
||||
for name in [
|
||||
"set-cookie",
|
||||
"openai-organization",
|
||||
"connection",
|
||||
"x-test-hop",
|
||||
] {
|
||||
assert!(!response.headers().contains_key(name));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_response_should_preserve_business_headers_and_rewrite_realtime_location() {
|
||||
let capture = Arc::new(LiveCapture {
|
||||
response_headers: Some(live_response_headers()),
|
||||
..LiveCapture::default()
|
||||
});
|
||||
for path in ["/v1/live", "/v1/realtime/calls"] {
|
||||
let response = api_router(capture.clone())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post(path)
|
||||
.header(AUTHORIZATION, format!("Bearer {LIVE_KEY}"))
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(r#"{"sdp":"v=0"}"#))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
assert_eq!(response.headers()["content-type"], "application/sdp");
|
||||
let location = if path == "/v1/live" {
|
||||
"https://api.openai.com/v1/live/call_headers"
|
||||
} else {
|
||||
"/v1/realtime/calls/call_headers"
|
||||
};
|
||||
assert_eq!(response.headers()["location"], location);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get_all("x-test-response")
|
||||
.iter()
|
||||
.map(|value| value.as_bytes())
|
||||
.collect::<Vec<_>>(),
|
||||
[b"first".as_slice(), b"\x80second".as_slice()]
|
||||
);
|
||||
for name in [
|
||||
"set-cookie",
|
||||
"openai-organization",
|
||||
"connection",
|
||||
"x-test-hop",
|
||||
] {
|
||||
assert!(!response.headers().contains_key(name));
|
||||
}
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), usize::MAX).await.unwrap(),
|
||||
b"v=0\r\n".as_slice()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn capture_live_call(
|
||||
body: Body,
|
||||
content_type: &str,
|
||||
@@ -324,7 +548,7 @@ async fn live_post_only_accepts_declared_methods() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_should_strip_downstream_identity_headers() {
|
||||
async fn live_should_preserve_business_headers_without_downstream_identity_or_transport() {
|
||||
let capture = LiveCapture::new();
|
||||
api_router(Arc::clone(&capture) as Arc<dyn ExecutionService>)
|
||||
.await
|
||||
@@ -336,6 +560,14 @@ async fn live_should_strip_downstream_identity_headers() {
|
||||
.header("openai-organization", "downstream-org")
|
||||
.header("openai-project", "downstream-project")
|
||||
.header("x-oai-attestation", "downstream-attestation")
|
||||
.header("voice-session-id", "voice-test")
|
||||
.header("x-test-business", "first")
|
||||
.header(
|
||||
"x-test-business",
|
||||
axum::http::HeaderValue::from_bytes(b"\x80second").unwrap(),
|
||||
)
|
||||
.header("connection", "x-hop-test")
|
||||
.header("x-hop-test", "hop-only")
|
||||
.body(Body::from(json!({"sdp":"v=0"}).to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -354,6 +586,9 @@ async fn live_should_strip_downstream_identity_headers() {
|
||||
"openai-organization",
|
||||
"openai-project",
|
||||
"x-oai-attestation",
|
||||
"authorization",
|
||||
"connection",
|
||||
"x-hop-test",
|
||||
]
|
||||
.contains(h)
|
||||
})
|
||||
@@ -362,4 +597,18 @@ async fn live_should_strip_downstream_identity_headers() {
|
||||
leaked.is_empty(),
|
||||
"downstream identity headers reach upstream operation: {leaked:?}"
|
||||
);
|
||||
assert!(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.any(|header| header.name() == "voice-session-id"
|
||||
&& header.value().as_ref() == b"voice-test")
|
||||
);
|
||||
let values: Vec<_> = request
|
||||
.headers()
|
||||
.iter()
|
||||
.filter(|header| header.name() == "x-test-business")
|
||||
.map(|header| header.value())
|
||||
.collect();
|
||||
assert_eq!(values, [b"first".as_slice(), b"\x80second".as_slice()]);
|
||||
}
|
||||
|
||||
@@ -92,6 +92,31 @@ impl LiveGateway for Gateway {
|
||||
request: LiveSidebandRequest<'a>,
|
||||
) -> BoxFuture<'a, Result<LiveRelay, LiveGatewayError>> {
|
||||
Box::pin(async move {
|
||||
assert!(
|
||||
request
|
||||
.protocol_headers
|
||||
.iter()
|
||||
.any(|header| header.name() == "voice-session-id"
|
||||
&& header.value().as_ref() == b"voice-test")
|
||||
);
|
||||
assert_eq!(
|
||||
request
|
||||
.protocol_headers
|
||||
.iter()
|
||||
.filter(|header| header.name() == "x-test-business")
|
||||
.map(|header| header.value())
|
||||
.collect::<Vec<_>>(),
|
||||
[b"first".as_slice(), b"second".as_slice()]
|
||||
);
|
||||
assert!(request.protocol_headers.iter().all(|header| {
|
||||
![
|
||||
"authorization",
|
||||
"originator",
|
||||
"connection",
|
||||
"sec-websocket-key",
|
||||
]
|
||||
.contains(&header.name())
|
||||
}));
|
||||
assert!(
|
||||
request.account_scope.allows(
|
||||
&gateway_core::account::ProviderAccountId::new("acct_api_test").unwrap()
|
||||
@@ -154,6 +179,14 @@ async fn blocked_relay(
|
||||
"authorization",
|
||||
format!("Bearer {LIVE_KEY}").parse().unwrap(),
|
||||
);
|
||||
for (name, value) in [
|
||||
("voice-session-id", "voice-test"),
|
||||
("x-test-business", "first"),
|
||||
("x-test-business", "second"),
|
||||
("originator", "foreign-client"),
|
||||
] {
|
||||
request.headers_mut().append(name, value.parse().unwrap());
|
||||
}
|
||||
let (mut socket, _) = connect_async(request).await.unwrap();
|
||||
if trace.block_close {
|
||||
socket.send(Message::Close(None)).await.unwrap();
|
||||
|
||||
@@ -3214,12 +3214,12 @@ async fn compressed_http_request_above_default_limit_should_reach_execution_afte
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_middleware_preserves_heartbeats_before_and_after_rewriting() {
|
||||
let admin = crate::admin::AdminTestFixture::new().await.services;
|
||||
for opaque in [
|
||||
b": ping\n\n".as_slice(),
|
||||
b"id: next\nretry: 1000\n\n".as_slice(),
|
||||
b"event: vendor.future\ndata: future non-JSON payload\n\n".as_slice(),
|
||||
] {
|
||||
let admin = crate::admin::AdminTestFixture::new().await.services;
|
||||
let trace = Arc::new(Trace::default());
|
||||
let heartbeat = Bytes::copy_from_slice(opaque);
|
||||
let future = Bytes::from_static(
|
||||
|
||||
+11
-2
@@ -442,8 +442,17 @@ async fn input_plugin_runs_while_a_real_socket_write_is_blocked() {
|
||||
let (trace, mut socket, server) =
|
||||
start_active_response_with_middleware(Some(frozen), "/custom/websocket").await;
|
||||
trace.release_terminal.notify_one();
|
||||
// 客户端不读取 64 MiB 输出,真实 TCP 写入将受接收窗口背压
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
// 仅窥视帧头以确认写入开始,不消费 64 MiB 输出,保留真实 TCP 接收窗口背压
|
||||
let MaybeTlsStream::Plain(transport) = socket.get_ref() else {
|
||||
panic!("test server must use plain TCP")
|
||||
};
|
||||
let mut frame_header = [0; 1];
|
||||
let received = tokio::time::timeout(Duration::from_secs(2), transport.peek(&mut frame_header))
|
||||
.await
|
||||
.expect("terminal output must start")
|
||||
.expect("peek terminal frame");
|
||||
assert_eq!(received, 1);
|
||||
assert_eq!(frame_header[0] & 0x0f, 2);
|
||||
assert!(!trace.finalized.load(Ordering::SeqCst));
|
||||
socket
|
||||
.send(ClientMessage::Text("signal".into()))
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
//! 诊断内容的统一筛选边界
|
||||
//! 未知键名与值只保留有界结构和摘要
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -12,6 +15,10 @@ pub fn body_fingerprint(bytes: &[u8]) -> Value {
|
||||
|
||||
fn fingerprint_hex(bytes: &[u8]) -> String {
|
||||
let digest = Sha256::digest(bytes);
|
||||
digest_hex(&digest)
|
||||
}
|
||||
|
||||
fn digest_hex(digest: &[u8]) -> String {
|
||||
digest
|
||||
.iter()
|
||||
.flat_map(|byte| {
|
||||
@@ -24,6 +31,36 @@ fn fingerprint_hex(bytes: &[u8]) -> String {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 从已解析对象流式计算 JSON 指纹,不复制或保留序列化正文
|
||||
#[must_use]
|
||||
pub fn json_fingerprint<T: Serialize + ?Sized>(value: &T) -> Value {
|
||||
struct FingerprintWriter {
|
||||
digest: Sha256,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl Write for FingerprintWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.digest.update(bytes);
|
||||
self.bytes = self.bytes.saturating_add(bytes.len());
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
let mut writer = FingerprintWriter {
|
||||
digest: Sha256::new(),
|
||||
bytes: 0,
|
||||
};
|
||||
if serde_json::to_writer(&mut writer, value).is_err() {
|
||||
return json!({"omitted": true, "reason": "fingerprint_serialization_failed"});
|
||||
}
|
||||
json!({"bytes": writer.bytes, "sha256": digest_hex(&writer.digest.finalize())})
|
||||
}
|
||||
|
||||
/// 仅保留有限协议骨架;任意 JSON(包括 headers 和 metadata)不能声明自己可信
|
||||
#[must_use]
|
||||
pub fn diagnostic_json(value: &Value) -> Value {
|
||||
|
||||
@@ -6,7 +6,7 @@ mod selection;
|
||||
mod stream;
|
||||
mod trace;
|
||||
|
||||
pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json};
|
||||
pub use capture::{body_fingerprint, diagnostic_headers, diagnostic_json, json_fingerprint};
|
||||
pub use failure::{OperationalDiagnostics, OperationalFailure};
|
||||
pub use stream::{StreamCapture, StreamFormat};
|
||||
pub use trace::TraceContext;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 首部与最近事件同时保留,流式 delta 合并,所有淘汰均有计数
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
collections::{BTreeMap, VecDeque},
|
||||
sync::{Arc, Mutex},
|
||||
time::{Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
@@ -19,6 +19,9 @@ const MAX_EVENTS: usize = 128;
|
||||
const HEAD_EVENTS: usize = 8;
|
||||
const MAX_DATA_BYTES: usize = 4096;
|
||||
const MAX_BUFFER_BYTES: usize = 64 * 1024;
|
||||
const MAX_EVIDENCE_EVENTS: usize = 64;
|
||||
const MAX_EVIDENCE_DATA_BYTES: usize = 16 * 1024;
|
||||
const MAX_EVIDENCE_BUFFER_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// 显式传给异步任务的关联上下文;默认值禁用捕获,避免全局请求映射
|
||||
#[derive(Clone, Default)]
|
||||
@@ -46,7 +49,12 @@ struct TraceState {
|
||||
wire_frames: u64,
|
||||
wire_bytes: u64,
|
||||
dropped_events: u64,
|
||||
dropped_evidence_events: u64,
|
||||
bytes: usize,
|
||||
evidence_bytes: usize,
|
||||
evidence_events: usize,
|
||||
stage_counts: BTreeMap<&'static str, u64>,
|
||||
stage_count_overflow: u64,
|
||||
events: VecDeque<TraceEvent>,
|
||||
}
|
||||
|
||||
@@ -64,6 +72,8 @@ struct TraceEvent {
|
||||
data: Value,
|
||||
#[serde(skip)]
|
||||
bytes: usize,
|
||||
#[serde(skip)]
|
||||
evidence: bool,
|
||||
}
|
||||
|
||||
impl TraceContext {
|
||||
@@ -89,7 +99,12 @@ impl TraceContext {
|
||||
wire_frames: 0,
|
||||
wire_bytes: 0,
|
||||
dropped_events: 0,
|
||||
dropped_evidence_events: 0,
|
||||
bytes: 0,
|
||||
evidence_bytes: 0,
|
||||
evidence_events: 0,
|
||||
stage_counts: BTreeMap::new(),
|
||||
stage_count_overflow: 0,
|
||||
events: VecDeque::new(),
|
||||
}))),
|
||||
..Self::default()
|
||||
@@ -125,6 +140,11 @@ impl TraceContext {
|
||||
self.push(stage, data, false);
|
||||
}
|
||||
|
||||
/// 仅接收 owner 提取的有界安全事实,预留预算避免被常规流事件挤掉
|
||||
pub fn record_evidence(&self, stage: &'static str, data: Value) {
|
||||
self.push_with_retention(stage, data, false, true, MAX_EVIDENCE_DATA_BYTES);
|
||||
}
|
||||
|
||||
/// 诊断正文与导出分类来自同一失败,导出路径不接收任意 JSON 或上游文案
|
||||
pub fn record_provider_failure(&self, error: &ProviderError) {
|
||||
if !self.is_enabled() {
|
||||
@@ -270,21 +290,44 @@ impl TraceContext {
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
Some(json!({
|
||||
"schemaVersion": 2, "requestId": state.request_id,
|
||||
"schemaVersion": 3, "requestId": state.request_id,
|
||||
"startedAtUnixMs": state.started_at_ms, "totalEvents": state.sequence,
|
||||
"droppedEvents": state.dropped_events, "maxEvents": MAX_EVENTS,
|
||||
"captureMode": "sanitized", "events": state.events,
|
||||
"wireDumpEnabled": tracing::enabled!(target: "request_dump", tracing::Level::INFO),
|
||||
"wireFrames": state.wire_frames, "wireBytes": state.wire_bytes,
|
||||
"maxBufferBytes": MAX_BUFFER_BYTES, "headEvents": HEAD_EVENTS,
|
||||
"maxBufferBytes": MAX_BUFFER_BYTES + MAX_EVIDENCE_BUFFER_BYTES, "headEvents": HEAD_EVENTS,
|
||||
"maxEvidenceEvents": MAX_EVIDENCE_EVENTS,
|
||||
"maxEvidenceDataBytes": MAX_EVIDENCE_DATA_BYTES,
|
||||
"maxEvidenceBufferBytes": MAX_EVIDENCE_BUFFER_BYTES,
|
||||
"droppedEvidenceEvents": state.dropped_evidence_events,
|
||||
"stageCounts": state.stage_counts,
|
||||
"stageCountOverflow": state.stage_count_overflow,
|
||||
"snapshotBoundary": "execution_finalization",
|
||||
}))
|
||||
}
|
||||
|
||||
fn push(&self, stage: &'static str, mut data: Value, coalesce: bool) {
|
||||
fn push(&self, stage: &'static str, data: Value, coalesce: bool) {
|
||||
self.push_with_retention(
|
||||
stage,
|
||||
data,
|
||||
coalesce,
|
||||
important_stage(stage),
|
||||
MAX_DATA_BYTES,
|
||||
);
|
||||
}
|
||||
|
||||
fn push_with_retention(
|
||||
&self,
|
||||
stage: &'static str,
|
||||
mut data: Value,
|
||||
coalesce: bool,
|
||||
evidence: bool,
|
||||
data_limit: usize,
|
||||
) {
|
||||
let Some(state) = &self.state else { return };
|
||||
let mut encoded = serde_json::to_vec(&data).unwrap_or_default();
|
||||
if encoded.len() > MAX_DATA_BYTES {
|
||||
if encoded.len() > data_limit {
|
||||
let mut summary = json!({"truncated": true, "summary": body_fingerprint(&encoded),
|
||||
"eventType": data.get("eventType"), "body": data.get("body"),
|
||||
"jsonValid": data.get("jsonValid"),
|
||||
@@ -341,11 +384,17 @@ impl TraceContext {
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.sequence += 1;
|
||||
if state.stage_counts.contains_key(stage) || state.stage_counts.len() < 64 {
|
||||
*state.stage_counts.entry(stage).or_default() += 1;
|
||||
} else {
|
||||
state.stage_count_overflow += 1;
|
||||
}
|
||||
let sequence = state.sequence;
|
||||
let elapsed_ms = u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
if coalesce
|
||||
&& let Some(last) = state.events.iter_mut().rev().find(|event| {
|
||||
event.stage == stage
|
||||
!event.evidence
|
||||
&& event.stage == stage
|
||||
&& event.attempt_index == self.attempt_index
|
||||
&& event.exchange_id == self.exchange_id
|
||||
})
|
||||
@@ -365,6 +414,10 @@ impl TraceContext {
|
||||
sequence, elapsed_ms, stage, data = %data, "request trace");
|
||||
let bytes = encoded.len() + 256;
|
||||
state.bytes += bytes;
|
||||
if evidence {
|
||||
state.evidence_bytes += bytes;
|
||||
state.evidence_events += 1;
|
||||
}
|
||||
state.events.push_back(TraceEvent {
|
||||
sequence,
|
||||
last_sequence: sequence,
|
||||
@@ -376,39 +429,76 @@ impl TraceContext {
|
||||
count: 1,
|
||||
data,
|
||||
bytes,
|
||||
evidence,
|
||||
});
|
||||
while state.events.len() > MAX_EVENTS || state.bytes > MAX_BUFFER_BYTES {
|
||||
while state.events.len() > MAX_EVENTS
|
||||
|| state.bytes.saturating_sub(state.evidence_bytes) > MAX_BUFFER_BYTES
|
||||
|| state.evidence_bytes > MAX_EVIDENCE_BUFFER_BYTES
|
||||
|| state.evidence_events > MAX_EVIDENCE_EVENTS
|
||||
{
|
||||
if state.evidence_bytes > MAX_EVIDENCE_BUFFER_BYTES
|
||||
|| state.evidence_events > MAX_EVIDENCE_EVENTS
|
||||
{
|
||||
if let Some(index) = state
|
||||
.events
|
||||
.iter()
|
||||
.position(|event| event.evidence && event.stage != "request.routed")
|
||||
.or_else(|| state.events.iter().position(|event| event.evidence))
|
||||
{
|
||||
remove_event(&mut state, index);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// 相比流中间的常规事件,优先保留失败与重试决策
|
||||
// 首尾事件与重要阶段共用同一内存上限
|
||||
// 关键事实使用独立预算,只有自身超限才会淘汰
|
||||
let index = (HEAD_EVENTS..state.events.len().saturating_sub(8))
|
||||
.find(|index| {
|
||||
!matches!(
|
||||
state.events[*index].stage,
|
||||
"attempt.started"
|
||||
| "account.selection"
|
||||
| "account.selected"
|
||||
| "attempt.failed"
|
||||
| "retry.decided"
|
||||
| "retry.policy"
|
||||
| "upstream.close"
|
||||
| "upstream.read.failed"
|
||||
| "capture.gap"
|
||||
| "request.finished"
|
||||
)
|
||||
})
|
||||
.unwrap_or(if state.events.len() > HEAD_EVENTS {
|
||||
HEAD_EVENTS
|
||||
} else {
|
||||
0
|
||||
.find(|index| !state.events[*index].evidence)
|
||||
.or_else(|| {
|
||||
(HEAD_EVENTS..state.events.len())
|
||||
.chain(0..HEAD_EVENTS.min(state.events.len()))
|
||||
.find(|index| !state.events[*index].evidence)
|
||||
});
|
||||
if let Some(event) = state.events.remove(index) {
|
||||
state.bytes = state.bytes.saturating_sub(event.bytes);
|
||||
state.dropped_events += event.count;
|
||||
if let Some(index) = index {
|
||||
remove_event(&mut state, index);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_event(state: &mut TraceState, index: usize) {
|
||||
if let Some(event) = state.events.remove(index) {
|
||||
state.bytes = state.bytes.saturating_sub(event.bytes);
|
||||
state.dropped_events += event.count;
|
||||
if event.evidence {
|
||||
state.evidence_bytes = state.evidence_bytes.saturating_sub(event.bytes);
|
||||
state.evidence_events = state.evidence_events.saturating_sub(1);
|
||||
state.dropped_evidence_events += event.count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn important_stage(stage: &str) -> bool {
|
||||
matches!(
|
||||
stage,
|
||||
"request.routed"
|
||||
| "attempt.started"
|
||||
| "account.selection"
|
||||
| "account.selected"
|
||||
| "attempt.failed"
|
||||
| "retry.decided"
|
||||
| "retry.policy"
|
||||
| "request.finished"
|
||||
| "upstream.exchange.started"
|
||||
| "upstream.response.headers"
|
||||
| "upstream.close"
|
||||
| "upstream.read.failed"
|
||||
| "upstream.payload.sent"
|
||||
| "capture.gap"
|
||||
)
|
||||
}
|
||||
|
||||
impl Drop for TraceState {
|
||||
fn drop(&mut self) {
|
||||
tracing::info!(target: "request_trace", request_id = %self.request_id,
|
||||
|
||||
@@ -18,6 +18,7 @@ struct State {
|
||||
pub struct ConnectionBudget {
|
||||
state: Arc<Mutex<State>>,
|
||||
now: fn() -> Instant,
|
||||
reserve_next_opening: bool,
|
||||
}
|
||||
|
||||
impl Default for ConnectionBudget {
|
||||
@@ -37,9 +38,26 @@ impl ConnectionBudget {
|
||||
Self {
|
||||
state: Arc::default(),
|
||||
now,
|
||||
reserve_next_opening: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅本次恢复使用一半剩余时间并留下一次 opening;已启动的后台连接不受影响
|
||||
#[must_use]
|
||||
pub fn reserve_next_opening(mut self) -> Self {
|
||||
self.reserve_next_opening = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn remaining_openings(&self) -> u32 {
|
||||
let state = self
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
MAX_OPENINGS.saturating_sub(state.openings)
|
||||
}
|
||||
|
||||
/// 在可能建立连接前预约一次 opening;返回本次连接允许使用的剩余时间
|
||||
/// HTTP 连接池隐藏了是否冷建连,因此一次 HTTP opening 也计入次数上限
|
||||
pub fn begin(&self) -> Result<Option<Duration>, ConnectionBudgetExhausted> {
|
||||
@@ -53,11 +71,16 @@ impl ConnectionBudget {
|
||||
let started = *state.started.get_or_insert_with(self.now);
|
||||
let remaining =
|
||||
STARTUP_TIMEOUT.saturating_sub((self.now)().saturating_duration_since(started));
|
||||
if state.openings >= MAX_OPENINGS || remaining.is_zero() {
|
||||
let opening_limit = MAX_OPENINGS - u32::from(self.reserve_next_opening);
|
||||
if state.openings >= opening_limit || remaining.is_zero() {
|
||||
return Err(ConnectionBudgetExhausted);
|
||||
}
|
||||
state.openings += 1;
|
||||
Ok(Some(remaining))
|
||||
Ok(Some(if self.reserve_next_opening {
|
||||
remaining / 2
|
||||
} else {
|
||||
remaining
|
||||
}))
|
||||
}
|
||||
|
||||
/// 仅连接成功不足以清零;确认业务发送后退出新增的 NotSent 恢复策略
|
||||
|
||||
@@ -318,6 +318,7 @@ where
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget::default(),
|
||||
connection_budget: super::connection::ConnectionBudget::default(),
|
||||
connection_retries: 0,
|
||||
connection_rotation: None,
|
||||
observation: ResponseObservation::new(timing_started_at),
|
||||
request_observation,
|
||||
budget_prior_attempts_usd: Decimal::ZERO,
|
||||
@@ -384,6 +385,7 @@ where
|
||||
last_retryable_failure_events: Vec::new(),
|
||||
last_observed_provider: None,
|
||||
pending_terminal_failure: None,
|
||||
pending_stream_failure: None,
|
||||
client_retry: None,
|
||||
};
|
||||
|
||||
@@ -424,6 +426,7 @@ struct CurrentAttempt {
|
||||
metadata: ProviderCallMetadata,
|
||||
trigger: AttemptTrigger,
|
||||
transport: AttemptTransport,
|
||||
connection_rotation_reserved: bool,
|
||||
index: NonZeroU32,
|
||||
started_at: SystemTime,
|
||||
send_observed: bool,
|
||||
@@ -458,6 +461,7 @@ struct PendingAttemptRetry {
|
||||
transport: AttemptTransport,
|
||||
delay: Duration,
|
||||
transport_recovery: bool,
|
||||
reserve_connection_rotation: bool,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话
|
||||
@@ -471,6 +475,8 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget,
|
||||
connection_budget: super::connection::ConnectionBudget,
|
||||
connection_retries: u32,
|
||||
/// 原账号恢复仍建连失败后尝试重选;选号不可用时消费该状态,恢复原账号剩余预算
|
||||
connection_rotation: Option<PendingAttemptRetry>,
|
||||
observation: ResponseObservation,
|
||||
request_observation: Option<RequestObservationDispatch>,
|
||||
budget_prior_attempts_usd: Decimal,
|
||||
@@ -550,12 +556,38 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
last_observed_provider: Option<crate::identity::ProviderKind>,
|
||||
/// 原子失败批次已交给协议层、但尚待下游提交后收敛的原 Provider 错误
|
||||
pending_terminal_failure: Option<PendingTerminalFailure>,
|
||||
/// 已消费的流错误及其异步决策由会话持有,取消读取只暂停处理
|
||||
pending_stream_failure: Option<PendingStreamFailure>,
|
||||
client_retry: Option<ClientRetryGuard>,
|
||||
}
|
||||
|
||||
struct PendingTerminalFailure {
|
||||
error: ProviderError,
|
||||
error: Arc<ProviderError>,
|
||||
send_state: UpstreamSendState,
|
||||
events: Vec<ProviderEvent>,
|
||||
}
|
||||
|
||||
struct PendingStreamFailure {
|
||||
completion: StreamFailureDecision,
|
||||
error: ProviderError,
|
||||
events: Vec<ProviderEvent>,
|
||||
send_state: UpstreamSendState,
|
||||
retry: StreamRetry,
|
||||
trace: serde_json::Value,
|
||||
}
|
||||
|
||||
enum StreamFailureDecision {
|
||||
Pending(BoxFuture<'static, Result<bool, EngineError>>),
|
||||
Retry(bool),
|
||||
Interrupted(Arc<EngineError>),
|
||||
}
|
||||
|
||||
enum StreamRetry {
|
||||
SameAccount,
|
||||
Pending(PendingAttemptRetry),
|
||||
ConnectionRotation(PendingAttemptRetry),
|
||||
ExcludeAccount,
|
||||
Continuation,
|
||||
}
|
||||
|
||||
enum RequestFinalization {
|
||||
@@ -597,14 +629,14 @@ where
|
||||
if self.delivery_pending {
|
||||
return Err(EngineError::DownstreamCommitRequired);
|
||||
}
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
self.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(pending.error));
|
||||
}
|
||||
if self.is_finalized() {
|
||||
self.pending_terminal_failure.take();
|
||||
self.pending_stream_failure.take();
|
||||
return Ok(None);
|
||||
}
|
||||
if self.pending_terminal_failure.is_some() {
|
||||
return self.next_terminal_failure().await;
|
||||
}
|
||||
loop {
|
||||
match self.pull().await? {
|
||||
PullOutcome::Events(events) => {
|
||||
@@ -624,17 +656,12 @@ where
|
||||
error,
|
||||
send_state,
|
||||
} => {
|
||||
self.record_client_retry_failure(&error).await;
|
||||
let requirement = if self.delivery_committed {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
self.delivery_pending = true;
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
};
|
||||
self.replay_barrier_reached = true;
|
||||
self.pending_terminal_failure =
|
||||
Some(PendingTerminalFailure { error, send_state });
|
||||
return CoordinatedEvent::try_batch(events, requirement).map(Some);
|
||||
self.pending_terminal_failure = Some(PendingTerminalFailure {
|
||||
error: Arc::new(error),
|
||||
send_state,
|
||||
events,
|
||||
});
|
||||
return self.next_terminal_failure().await;
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if self.delivery_committed {
|
||||
@@ -668,9 +695,14 @@ where
|
||||
PullOutcome::TerminalFailure {
|
||||
error, send_state, ..
|
||||
} => {
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
return Err(provider_engine_error(error));
|
||||
self.pending_terminal_failure = Some(PendingTerminalFailure {
|
||||
error: Arc::new(error),
|
||||
send_state,
|
||||
events: Vec::new(),
|
||||
});
|
||||
return Err(provider_engine_error(
|
||||
self.finalize_pending_terminal_failure().await?,
|
||||
));
|
||||
}
|
||||
PullOutcome::End => {
|
||||
if events.is_empty() {
|
||||
@@ -868,21 +900,25 @@ where
|
||||
/// 观测写入失败只记录告警,不改变取消结果
|
||||
pub async fn cancel_and_finalize(&mut self) -> Result<(), EngineError> {
|
||||
self.cancellation.cancel();
|
||||
// 已观测事实归属 session,显式取消不再保留失败决策中的存储或插件调用
|
||||
self.pending_stream_failure.take();
|
||||
self.resume_finalization().await;
|
||||
if self.is_finalized() {
|
||||
return Ok(());
|
||||
}
|
||||
self.delivery_pending = false;
|
||||
if let Some(pending) = self.pending_terminal_failure.take() {
|
||||
return self
|
||||
.finish_provider_error_with_send_state(&pending.error, pending.send_state)
|
||||
.await;
|
||||
if self.pending_terminal_failure.is_some() {
|
||||
self.finalize_pending_terminal_failure().await?;
|
||||
return Ok(());
|
||||
}
|
||||
self.finish_interruption(&EngineError::Cancelled).await
|
||||
}
|
||||
|
||||
async fn pull(&mut self) -> Result<PullOutcome, EngineError> {
|
||||
loop {
|
||||
if self.pending_stream_failure.is_some() {
|
||||
return self.resume_stream_failure().await;
|
||||
}
|
||||
if self.current.is_none() {
|
||||
match self.prepare_attempt().await {
|
||||
Ok(Some(outcome)) => return Ok(outcome),
|
||||
@@ -914,7 +950,7 @@ where
|
||||
}
|
||||
PollBoundary::Deadline => {
|
||||
// 会话 deadline 是网关自身的请求预算,不是上游超时;
|
||||
// 真正的上游超时会作为流错误进入 `handle_stream_error` 记账
|
||||
// 真正的上游超时会作为流错误进入 `begin_stream_failure` 记账
|
||||
// 这里不写 provider 失败,避免把本地预算到期归因为上游故障
|
||||
self.finish_interruption(&EngineError::Deadline).await?;
|
||||
return Err(EngineError::Deadline);
|
||||
@@ -976,14 +1012,8 @@ where
|
||||
}
|
||||
}
|
||||
PollBoundary::Item(Some(Err(error))) => {
|
||||
match self.handle_stream_error(error).await? {
|
||||
StreamErrorOutcome::AttemptDiscarded => {
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
terminal @ StreamErrorOutcome::TerminalFailure { .. } => {
|
||||
return Ok(terminal.into_pull_outcome());
|
||||
}
|
||||
}
|
||||
self.begin_stream_failure(error)?;
|
||||
return self.resume_stream_failure().await;
|
||||
}
|
||||
PollBoundary::Item(None) => {
|
||||
self.record_current_provider_success();
|
||||
@@ -999,6 +1029,9 @@ where
|
||||
let is_transport_recovery = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.transport_recovery);
|
||||
let connection_rotation_reserved = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.reserve_connection_rotation);
|
||||
if !is_transport_recovery && self.routing_attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
@@ -1060,7 +1093,9 @@ where
|
||||
self.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
AttemptTransport::Default,
|
||||
self.connection_rotation
|
||||
.as_ref()
|
||||
.map_or(AttemptTransport::Default, |recovery| recovery.transport),
|
||||
),
|
||||
}
|
||||
};
|
||||
@@ -1122,8 +1157,13 @@ where
|
||||
.with_pricing(self.plan.pricing())
|
||||
.with_request_location(self.plan.request_location().cloned())
|
||||
.with_privacy(self.plan.privacy())
|
||||
.with_request_body_limit(self.plan.request_body_limit())
|
||||
.with_concurrency_wait_budget(self.concurrency_wait_budget.clone())
|
||||
.with_connection_budget(self.connection_budget.clone())
|
||||
.with_connection_budget(if connection_rotation_reserved {
|
||||
self.connection_budget.clone().reserve_next_opening()
|
||||
} else {
|
||||
self.connection_budget.clone()
|
||||
})
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_request_policy(self.request_policy.clone())
|
||||
.with_execution_effects(self.execution_effects.as_ref().map(Arc::clone))
|
||||
@@ -1186,9 +1226,21 @@ where
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::QuotaExhausted
|
||||
| ProviderErrorKind::ConcurrencyQueueFull
|
||||
| ProviderErrorKind::ConcurrencyQueueTimeout
|
||||
);
|
||||
let continuation_retry = !error.retry_is_prohibited()
|
||||
&& self.prepare_unavailable_native_continuation_replay(&error);
|
||||
let candidate_retry = !error.retry_is_prohibited() && !continuation_retry
|
||||
let candidate_retry = !error.retry_is_prohibited() && !connection_fallback && !continuation_retry
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -1224,12 +1276,15 @@ where
|
||||
.get()
|
||||
.saturating_sub(self.routing_attempts),
|
||||
remaining_deadline: Duration::ZERO,
|
||||
retry_allowed: continuation_retry || candidate_retry,
|
||||
retry_allowed: connection_fallback
|
||||
|| continuation_retry
|
||||
|| candidate_retry,
|
||||
})
|
||||
.await?;
|
||||
attempt_trace.record("retry.decided", json!({
|
||||
"phase": "prepare", "retryable": retryable,
|
||||
"continuationRetry": continuation_retry, "candidateRetry": candidate_retry,
|
||||
"connectionFallback": connection_fallback,
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sendState": error.send_state().as_str(),
|
||||
"requestSendState": self.current_send_state().as_str(),
|
||||
@@ -1239,6 +1294,17 @@ where
|
||||
"remainingRoutingAttempts": self.plan.max_attempts().get().saturating_sub(self.routing_attempts),
|
||||
}));
|
||||
if retryable {
|
||||
if connection_fallback {
|
||||
let recovery = self
|
||||
.connection_rotation
|
||||
.take()
|
||||
.expect("pending connection rotation");
|
||||
self.excluded_accounts.remove(&recovery.account);
|
||||
self.pending_retry = Some(recovery);
|
||||
} else if candidate_retry {
|
||||
// 原账号只属于原 Provider,跨候选后不能拿旧钉选作恢复兜底
|
||||
self.connection_rotation = None;
|
||||
}
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
@@ -1306,6 +1372,7 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
self.connection_rotation = None;
|
||||
attempt_trace.record("account.selected", json!({
|
||||
"provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(),
|
||||
"transport": metadata.transport().as_str(),
|
||||
@@ -1429,6 +1496,7 @@ where
|
||||
metadata,
|
||||
trigger,
|
||||
transport: attempt_transport,
|
||||
connection_rotation_reserved,
|
||||
index: next_attempt,
|
||||
started_at: SystemTime::now(),
|
||||
send_observed: false,
|
||||
@@ -1529,16 +1597,24 @@ where
|
||||
}
|
||||
|
||||
/// 失败可重试时要求丢弃本 attempt;预算耗尽时可返回最后一批原始失败事件
|
||||
async fn handle_stream_error(
|
||||
&mut self,
|
||||
mut error: ProviderError,
|
||||
) -> Result<StreamErrorOutcome, EngineError> {
|
||||
fn begin_stream_failure(&mut self, mut error: ProviderError) -> Result<(), EngineError> {
|
||||
// 原始 wire 只活在 request-local 决策状态;clone、attempt 记录与持久化终态
|
||||
// 均只接触已剥离的稳定错误字段
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.record_provider_failure(&error);
|
||||
let mut atomic_client_events = error.take_atomic_client_events();
|
||||
let atomic_client_events = error.take_atomic_client_events();
|
||||
// 失败随附的事实也属于当前 attempt,取消观测写入时仍须保留 usage 与费用
|
||||
// 回退仍可能交付该批事件,因此这里只借用观测,确认终态交付后再移除
|
||||
for event in &atomic_client_events {
|
||||
if let Some(observation) = event.response_observation() {
|
||||
self.observe_response(observation.clone());
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
self.observation.observe_identity(fact);
|
||||
self.observation.observe_event(fact);
|
||||
}
|
||||
}
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
if self.request_observation.is_some() {
|
||||
let websocket_attempt = websocket_observation_attempt(¤t);
|
||||
@@ -1565,16 +1641,6 @@ where
|
||||
}
|
||||
let execution_effect_observed = self.execution_effect_observed();
|
||||
let send_state = self.raise_send_watermark(attempt_send_state);
|
||||
if self.request_persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.mark_send_state(&self.request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let provider_proved_replay_safe = provider_proved_replay_safe(&error);
|
||||
let continuation_retry = !execution_effect_observed
|
||||
&& self.prepare_continuation_retry(
|
||||
@@ -1642,6 +1708,18 @@ where
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let can_rotate_connection = connection_retry_requested
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
|
||||
let connection_rotation = transport_recovery
|
||||
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
|
||||
let reserve_connection_rotation = can_rotate_connection
|
||||
&& !current.connection_rotation_reserved
|
||||
&& self.connection_budget.remaining_openings() >= 2;
|
||||
let transport_recovery = transport_recovery.filter(|_| connection_rotation.is_none());
|
||||
let ordinary_retry = !self.connection_budget.exhausted()
|
||||
&& !connection_retry_requested
|
||||
&& !execution_effect_observed
|
||||
@@ -1714,42 +1792,34 @@ where
|
||||
|| same_account_retry
|
||||
|| transient_retry.is_some()
|
||||
|| transport_recovery.is_some()
|
||||
|| connection_rotation.is_some()
|
||||
|| rotation_retry);
|
||||
|
||||
let retryable = match self
|
||||
.apply_retry_policy(super::policy::RetryFacts {
|
||||
attempt_index: current.index,
|
||||
provider: current.metadata.provider().clone(),
|
||||
model: current
|
||||
.metadata
|
||||
.upstream_model()
|
||||
.map(|model| model.as_str().to_owned()),
|
||||
error_kind: error.kind(),
|
||||
upstream_status: error.upstream_status(),
|
||||
send_state: attempt_send_state,
|
||||
remaining_routing_attempts: self
|
||||
.plan
|
||||
.max_attempts()
|
||||
.get()
|
||||
.saturating_sub(self.routing_attempts),
|
||||
remaining_deadline: Duration::ZERO,
|
||||
retry_allowed: retryable,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(retryable) => retryable,
|
||||
Err(error) => {
|
||||
self.current = Some(current);
|
||||
self.finish_interruption(&error).await?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
self.trace.attempt(current.index.get()).record("retry.decided", json!({
|
||||
let policy = self.apply_retry_policy(super::policy::RetryFacts {
|
||||
attempt_index: current.index,
|
||||
provider: current.metadata.provider().clone(),
|
||||
model: current
|
||||
.metadata
|
||||
.upstream_model()
|
||||
.map(|model| model.as_str().to_owned()),
|
||||
error_kind: error.kind(),
|
||||
upstream_status: error.upstream_status(),
|
||||
send_state: attempt_send_state,
|
||||
remaining_routing_attempts: self
|
||||
.plan
|
||||
.max_attempts()
|
||||
.get()
|
||||
.saturating_sub(self.routing_attempts),
|
||||
remaining_deadline: Duration::ZERO,
|
||||
retry_allowed: retryable,
|
||||
});
|
||||
let trace = json!({
|
||||
"phase": "execute",
|
||||
"retryable": retryable, "continuationRetry": continuation_retry,
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
@@ -1772,133 +1842,269 @@ where
|
||||
"credentialRecoveryAttempted": self.credential_recovery_attempted_accounts.contains(current.metadata.provider_account_id()),
|
||||
"hasRetryPolicy": self.request_policy.is_some(),
|
||||
"sendState": format!("{attempt_send_state:?}"),
|
||||
}));
|
||||
if retryable {
|
||||
for event in &atomic_client_events {
|
||||
for fact in event.canonical_facts() {
|
||||
self.observation.observe_event(fact);
|
||||
}
|
||||
}
|
||||
// 原始 wire/HTTP response 由 request-local 所有权保留到下一次 attempt
|
||||
// 成功,或最终空选路时返回客户端;持久化只取得稳定事实快照
|
||||
let persistence_error = self.request_persisted.then(|| error.stable_snapshot());
|
||||
if same_account_retry {
|
||||
let account = current.metadata.provider_account_id().clone();
|
||||
self.credential_recovery_attempted_accounts
|
||||
.insert(account.clone());
|
||||
// 只钉住紧随其后的 replay attempt;replay 再遇可重试错误时,
|
||||
// ordinary/continuation 重试门不受影响,仍可换号
|
||||
self.recovery_account = Some(account);
|
||||
} else if let Some(delay) = transient_retry {
|
||||
self.pending_retry = Some(PendingAttemptRetry {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport: current.transport,
|
||||
delay,
|
||||
transport_recovery: false,
|
||||
});
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
self.pending_retry = Some(PendingAttemptRetry {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
});
|
||||
} else if !continuation_retry {
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
self.last_retryable_failure_events = atomic_client_events;
|
||||
self.last_retryable_failure = Some(error);
|
||||
if let Some(error) = persistence_error {
|
||||
best_effort_store_write(
|
||||
"record_intermediate_failure",
|
||||
&self.request_id,
|
||||
self.engine
|
||||
.store()
|
||||
.record_intermediate_failure(IntermediateFailure {
|
||||
account_snapshot: current.metadata.account_snapshot().cloned(),
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: Some(current.metadata.provider_account_id().clone()),
|
||||
upstream_model_id: current.metadata.upstream_model().cloned(),
|
||||
upstream_status_code: error.upstream_status().or_else(|| {
|
||||
current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code)
|
||||
}),
|
||||
upstream_request_id: error
|
||||
.upstream_request_id()
|
||||
.map(|id| id.as_str())
|
||||
.or_else(|| current.upstream_request_id())
|
||||
.map(str::to_owned),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.reset_uncommitted_observations();
|
||||
return Ok(StreamErrorOutcome::AttemptDiscarded);
|
||||
}
|
||||
|
||||
self.current = Some(current);
|
||||
if !atomic_client_events.is_empty() {
|
||||
self.observe_atomic_terminal_events(&mut atomic_client_events);
|
||||
return Ok(StreamErrorOutcome::TerminalFailure {
|
||||
events: atomic_client_events,
|
||||
});
|
||||
let retry = if same_account_retry {
|
||||
StreamRetry::SameAccount
|
||||
} else if let Some(delay) = transient_retry {
|
||||
StreamRetry::Pending(PendingAttemptRetry {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport: current.transport,
|
||||
delay,
|
||||
transport_recovery: false,
|
||||
reserve_connection_rotation: false,
|
||||
})
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
StreamRetry::Pending(PendingAttemptRetry {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation,
|
||||
})
|
||||
} else if let Some((transport, delay)) = connection_rotation {
|
||||
StreamRetry::ConnectionRotation(PendingAttemptRetry {
|
||||
account: current.metadata.provider_account_id().clone(),
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation: false,
|
||||
})
|
||||
} else if continuation_retry {
|
||||
StreamRetry::Continuation
|
||||
} else {
|
||||
StreamRetry::ExcludeAccount
|
||||
};
|
||||
let intermediate = self.request_persisted.then(|| {
|
||||
let error = error.stable_snapshot();
|
||||
IntermediateFailure {
|
||||
account_snapshot: current.metadata.account_snapshot().cloned(),
|
||||
request_id: self.request_id.clone(),
|
||||
attempt_index: current.index,
|
||||
trigger: current.trigger,
|
||||
provider_kind: current.metadata.provider().clone(),
|
||||
account_id: Some(current.metadata.provider_account_id().clone()),
|
||||
upstream_model_id: current.metadata.upstream_model().cloned(),
|
||||
upstream_status_code: error.upstream_status().or_else(|| {
|
||||
current
|
||||
.response_observation
|
||||
.as_ref()
|
||||
.and_then(ProviderResponseObservation::status_code)
|
||||
}),
|
||||
upstream_request_id: error
|
||||
.upstream_request_id()
|
||||
.map(|id| id.as_str())
|
||||
.or_else(|| current.upstream_request_id())
|
||||
.map(str::to_owned),
|
||||
latency: current.started_at.elapsed().unwrap_or_default(),
|
||||
error,
|
||||
send_state,
|
||||
});
|
||||
}
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
Err(provider_engine_error(error))
|
||||
}
|
||||
});
|
||||
let store = Arc::clone(self.engine.store());
|
||||
let request_id = self.request_id.clone();
|
||||
let persisted = self.request_persisted;
|
||||
// 错误、账号 lease 与决策 future 都跨 next_event 的取消保留
|
||||
// 在持久化及策略回调完成前,不能再次拉流或建立新的 attempt
|
||||
self.current = Some(current);
|
||||
self.pending_stream_failure = Some(PendingStreamFailure {
|
||||
completion: StreamFailureDecision::Pending(Box::pin(async move {
|
||||
if persisted {
|
||||
best_effort_store_write(
|
||||
"mark_send_state",
|
||||
&request_id,
|
||||
store.mark_send_state(&request_id, send_state),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let retryable = policy.await?;
|
||||
if retryable && let Some(intermediate) = intermediate {
|
||||
best_effort_store_write(
|
||||
"record_intermediate_failure",
|
||||
&request_id,
|
||||
store.record_intermediate_failure(intermediate),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(retryable)
|
||||
})),
|
||||
error,
|
||||
events: atomic_client_events,
|
||||
send_state,
|
||||
retry,
|
||||
trace,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_retry_policy(
|
||||
&mut self,
|
||||
mut facts: super::policy::RetryFacts,
|
||||
) -> Result<bool, EngineError> {
|
||||
let Some(policy) = self.request_policy.clone() else {
|
||||
return Ok(facts.retry_allowed);
|
||||
async fn resume_stream_failure(&mut self) -> Result<PullOutcome, EngineError> {
|
||||
let pending = self
|
||||
.pending_stream_failure
|
||||
.as_mut()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
if let StreamFailureDecision::Pending(completion) = &mut pending.completion {
|
||||
pending.completion = match completion.as_mut().await {
|
||||
Ok(retryable) => StreamFailureDecision::Retry(retryable),
|
||||
Err(error) => StreamFailureDecision::Interrupted(Arc::new(error)),
|
||||
};
|
||||
}
|
||||
if let StreamFailureDecision::Interrupted(error) = &pending.completion {
|
||||
let error = Arc::clone(error);
|
||||
self.finish_interruption(&error).await?;
|
||||
self.pending_stream_failure.take();
|
||||
return Err(Arc::try_unwrap(error).expect("interruption is owned by the session"));
|
||||
}
|
||||
let mut pending = self
|
||||
.pending_stream_failure
|
||||
.take()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
let StreamFailureDecision::Retry(retryable) = pending.completion else {
|
||||
unreachable!("completed stream failure has a retry decision");
|
||||
};
|
||||
facts.remaining_deadline = self.deadline.remaining().unwrap_or(Duration::MAX);
|
||||
facts.retry_allowed &= !facts.remaining_deadline.is_zero()
|
||||
&& !self.cancellation.is_cancelled()
|
||||
&& !self.execution_effect_observed();
|
||||
let allowed = facts.retry_allowed;
|
||||
let attempt_index = facts.attempt_index;
|
||||
let decision = {
|
||||
let cancellation = self.cancellation.clone();
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let decision = policy.retry_decision(facts).fuse();
|
||||
let mut deadline_timer = &mut self.deadline_timer;
|
||||
pin_mut!(cancelled, decision);
|
||||
select_biased! {
|
||||
() = cancelled => Err(EngineError::Cancelled),
|
||||
() = deadline_timer => Err(EngineError::Deadline),
|
||||
decision = decision => Ok(decision),
|
||||
pending.trace["retryable"] = json!(retryable);
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.record("retry.decided", pending.trace);
|
||||
if retryable {
|
||||
let current = self.current.take().ok_or(EngineError::NoActiveAttempt)?;
|
||||
match pending.retry {
|
||||
StreamRetry::SameAccount => {
|
||||
let account = current.metadata.provider_account_id().clone();
|
||||
self.credential_recovery_attempted_accounts
|
||||
.insert(account.clone());
|
||||
self.recovery_account = Some(account);
|
||||
}
|
||||
StreamRetry::Pending(retry) => self.pending_retry = Some(retry),
|
||||
StreamRetry::ConnectionRotation(retry) => {
|
||||
self.excluded_accounts.insert(retry.account.clone());
|
||||
self.connection_rotation = Some(retry);
|
||||
}
|
||||
StreamRetry::ExcludeAccount => {
|
||||
self.excluded_accounts
|
||||
.insert(current.metadata.provider_account_id().clone());
|
||||
}
|
||||
StreamRetry::Continuation => {}
|
||||
}
|
||||
}?;
|
||||
self.trace.attempt(attempt_index.get()).record(
|
||||
"retry.policy",
|
||||
json!({
|
||||
"hostAllowed": allowed,
|
||||
"policyStopped": decision == super::policy::RetryDecision::Stop,
|
||||
"executionEffectObserved": self.execution_effect_observed(),
|
||||
"cancelled": self.cancellation.is_cancelled(),
|
||||
"deadlineElapsed": self.deadline.is_elapsed(),
|
||||
}),
|
||||
);
|
||||
// 策略只收窄宿主已允许的恢复,返回后仍复核期限和外部副作用
|
||||
Ok(allowed
|
||||
&& decision != super::policy::RetryDecision::Stop
|
||||
&& !self.execution_effect_observed()
|
||||
&& !self.cancellation.is_cancelled()
|
||||
&& !self.deadline.is_elapsed())
|
||||
self.last_retryable_failure_events = pending.events;
|
||||
self.last_retryable_failure = Some(pending.error);
|
||||
self.reset_uncommitted_observations();
|
||||
return Ok(PullOutcome::AttemptDiscarded);
|
||||
}
|
||||
if !pending.events.is_empty() {
|
||||
for event in &mut pending.events {
|
||||
let _ = event.take_observation();
|
||||
}
|
||||
return Ok(PullOutcome::TerminalFailure {
|
||||
events: pending.events,
|
||||
error: pending.error,
|
||||
send_state: pending.send_state,
|
||||
});
|
||||
}
|
||||
self.pending_terminal_failure = Some(PendingTerminalFailure {
|
||||
error: Arc::new(pending.error),
|
||||
send_state: pending.send_state,
|
||||
events: Vec::new(),
|
||||
});
|
||||
Err(provider_engine_error(
|
||||
self.finalize_pending_terminal_failure().await?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn next_terminal_failure(&mut self) -> Result<Option<CoordinatedEvent>, EngineError> {
|
||||
let pending = self
|
||||
.pending_terminal_failure
|
||||
.as_ref()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
if pending.events.is_empty() {
|
||||
return Err(provider_engine_error(
|
||||
self.finalize_pending_terminal_failure().await?,
|
||||
));
|
||||
}
|
||||
let error = Arc::clone(&pending.error);
|
||||
self.record_client_retry_failure(&error).await;
|
||||
let pending = self
|
||||
.pending_terminal_failure
|
||||
.as_mut()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
let events = std::mem::take(&mut pending.events);
|
||||
let requirement = if self.delivery_committed {
|
||||
CommitRequirement::AlreadyCommitted
|
||||
} else {
|
||||
self.delivery_pending = true;
|
||||
CommitRequirement::CommitBeforeDelivery
|
||||
};
|
||||
self.replay_barrier_reached = true;
|
||||
CoordinatedEvent::try_batch(events, requirement).map(Some)
|
||||
}
|
||||
|
||||
async fn finalize_pending_terminal_failure(&mut self) -> Result<ProviderError, EngineError> {
|
||||
let pending = self
|
||||
.pending_terminal_failure
|
||||
.as_ref()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
let error = Arc::clone(&pending.error);
|
||||
let send_state = pending.send_state;
|
||||
self.finish_provider_error_with_send_state(&error, send_state)
|
||||
.await?;
|
||||
drop(error);
|
||||
let pending = self
|
||||
.pending_terminal_failure
|
||||
.take()
|
||||
.ok_or(EngineError::NoActiveAttempt)?;
|
||||
Ok(Arc::try_unwrap(pending.error).expect("terminal failure is owned by the session"))
|
||||
}
|
||||
|
||||
fn apply_retry_policy(
|
||||
&self,
|
||||
mut facts: super::policy::RetryFacts,
|
||||
) -> BoxFuture<'static, Result<bool, EngineError>> {
|
||||
let policy = self.request_policy.clone();
|
||||
let deadline = self.deadline;
|
||||
let cancellation = self.cancellation.clone();
|
||||
let effects = self.execution_effects.clone();
|
||||
let baseline = self.execution_effects_baseline;
|
||||
let trace = self.trace.clone();
|
||||
Box::pin(async move {
|
||||
let Some(policy) = policy else {
|
||||
return Ok(facts.retry_allowed);
|
||||
};
|
||||
let effect_observed = || {
|
||||
effects
|
||||
.as_ref()
|
||||
.is_some_and(|effects| effects.epoch() != baseline)
|
||||
};
|
||||
facts.remaining_deadline = deadline.remaining().unwrap_or(Duration::MAX);
|
||||
facts.retry_allowed &= !facts.remaining_deadline.is_zero()
|
||||
&& !cancellation.is_cancelled()
|
||||
&& !effect_observed();
|
||||
let allowed = facts.retry_allowed;
|
||||
let attempt_index = facts.attempt_index;
|
||||
let decision = {
|
||||
let cancelled = cancellation.cancelled().fuse();
|
||||
let decision = policy.retry_decision(facts).fuse();
|
||||
let mut deadline_timer = deadline.wait().fuse();
|
||||
pin_mut!(cancelled, decision);
|
||||
select_biased! {
|
||||
() = cancelled => Err(EngineError::Cancelled),
|
||||
() = deadline_timer => Err(EngineError::Deadline),
|
||||
decision = decision => Ok(decision),
|
||||
}
|
||||
}?;
|
||||
trace.attempt(attempt_index.get()).record(
|
||||
"retry.policy",
|
||||
json!({
|
||||
"hostAllowed": allowed,
|
||||
"policyStopped": decision == super::policy::RetryDecision::Stop,
|
||||
"executionEffectObserved": effect_observed(),
|
||||
"cancelled": cancellation.is_cancelled(),
|
||||
"deadlineElapsed": deadline.is_elapsed(),
|
||||
}),
|
||||
);
|
||||
// 策略只收窄宿主已允许的恢复,返回后仍复核期限和外部副作用
|
||||
Ok(allowed
|
||||
&& decision != super::policy::RetryDecision::Stop
|
||||
&& !effect_observed()
|
||||
&& !cancellation.is_cancelled()
|
||||
&& !deadline.is_elapsed())
|
||||
})
|
||||
}
|
||||
|
||||
fn observe_atomic_terminal_events(&mut self, events: &mut [ProviderEvent]) {
|
||||
@@ -2073,6 +2279,7 @@ where
|
||||
self.current_transport_observation();
|
||||
let service_tier = self.current_service_tier();
|
||||
let provider_metadata_json = self.current_provider_metadata_json();
|
||||
self.record_final_usage();
|
||||
self.trace.record(
|
||||
"request.finished",
|
||||
json!({"outcome": "succeeded", "attempts": self.attempts}),
|
||||
@@ -2240,6 +2447,7 @@ where
|
||||
if self.is_finalized() {
|
||||
return Ok(());
|
||||
}
|
||||
self.record_final_usage();
|
||||
self.trace.record(
|
||||
"request.finished",
|
||||
json!({
|
||||
@@ -2448,6 +2656,29 @@ where
|
||||
self.send_state_watermark
|
||||
}
|
||||
|
||||
fn record_final_usage(&self) {
|
||||
if !self.trace.is_enabled() {
|
||||
return;
|
||||
}
|
||||
let usage = &self.observation.usage;
|
||||
let mut facts = json!({
|
||||
"inputTokens": usage.input_tokens,
|
||||
"outputTokens": usage.output_tokens,
|
||||
"cachedInputTokens": usage.cached_input_tokens,
|
||||
"cacheWriteInputTokens": usage.cache_write_input_tokens,
|
||||
"reasoningOutputTokens": usage.reasoning_output_tokens,
|
||||
"imageInputTokens": usage.image_input_tokens,
|
||||
"imageOutputTokens": usage.image_output_tokens,
|
||||
"totalTokens": usage.total_tokens,
|
||||
});
|
||||
facts["boundary"] = json!("finalization_prepared");
|
||||
facts["costSource"] = json!(self.observation.cost.source().as_str());
|
||||
facts["downstreamCommitted"] = json!(self.delivery_committed);
|
||||
self.trace
|
||||
.attempt(self.attempts)
|
||||
.record_evidence("request.usage", facts);
|
||||
}
|
||||
|
||||
fn record_current_provider_success(&mut self) {
|
||||
let provider_kind = self
|
||||
.current
|
||||
@@ -2503,32 +2734,6 @@ enum PullOutcome {
|
||||
End,
|
||||
}
|
||||
|
||||
enum StreamErrorOutcome {
|
||||
AttemptDiscarded,
|
||||
TerminalFailure {
|
||||
events: Vec<ProviderEvent>,
|
||||
error: ProviderError,
|
||||
send_state: UpstreamSendState,
|
||||
},
|
||||
}
|
||||
|
||||
impl StreamErrorOutcome {
|
||||
fn into_pull_outcome(self) -> PullOutcome {
|
||||
match self {
|
||||
Self::AttemptDiscarded => PullOutcome::AttemptDiscarded,
|
||||
Self::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
} => PullOutcome::TerminalFailure {
|
||||
events,
|
||||
error,
|
||||
send_state,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PollBoundary {
|
||||
Item(Option<Result<ProviderEvent, ProviderError>>),
|
||||
Cancelled,
|
||||
|
||||
@@ -65,6 +65,22 @@ pub struct ExecutionRequestMetadata {
|
||||
pub previous_response_id: Option<PreviousResponseId>,
|
||||
}
|
||||
|
||||
impl ExecutionRequestMetadata {
|
||||
pub(super) fn inherit_request_origin(&mut self, scope: &ExtensionCallScope) {
|
||||
let Some(request) = scope.request() else {
|
||||
return;
|
||||
};
|
||||
// 执行入口可以是内部调用,客户端来源仍属于原始入口快照
|
||||
self.client_ip = request.client_ip;
|
||||
self.user_agent = request
|
||||
.headers
|
||||
.iter()
|
||||
.find(|header| header.name().eq_ignore_ascii_case("user-agent"))
|
||||
.and_then(|header| std::str::from_utf8(header.value()).ok())
|
||||
.map(str::to_owned);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AuthenticatedClient {
|
||||
pub(super) settings: Option<RequestSettings>,
|
||||
|
||||
@@ -197,6 +197,15 @@ pub struct BoundModelExecutionContext {
|
||||
}
|
||||
|
||||
impl BoundModelExecutionContext {
|
||||
/// 与实际执行共用绑定时的快照,不能回退到后续发布的配置
|
||||
#[must_use]
|
||||
pub fn request_body_limit(&self) -> usize {
|
||||
self.authority
|
||||
.client
|
||||
.snapshot
|
||||
.responses_max_decompressed_body_bytes()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn request_settings(&self) -> Option<&RequestSettings> {
|
||||
self.authority.client.request_settings()
|
||||
|
||||
@@ -448,6 +448,12 @@ impl DefaultExecutionService {
|
||||
account_group_ids.clone(),
|
||||
)
|
||||
.with_extension_scope(authorization.extension_scope.clone())
|
||||
.with_request_body_limit(
|
||||
request
|
||||
.client
|
||||
.snapshot
|
||||
.responses_max_decompressed_body_bytes(),
|
||||
)
|
||||
.with_execution_effects(Arc::clone(&authorization.graph.as_ref()?.effects)),
|
||||
)
|
||||
});
|
||||
@@ -995,8 +1001,7 @@ impl DefaultExecutionService {
|
||||
let execution_effects_baseline = parent.graph.effects.epoch();
|
||||
metadata.endpoint = "host.model".to_owned();
|
||||
metadata.transport = ClientTransport::InternalPlugin;
|
||||
metadata.client_ip = None;
|
||||
metadata.user_agent = None;
|
||||
metadata.inherit_request_origin(&extension_scope);
|
||||
let authorization = AuthorizedExecution {
|
||||
account_scope,
|
||||
response_control: None,
|
||||
@@ -1149,8 +1154,7 @@ impl DefaultExecutionService {
|
||||
let execution_effects_baseline = graph.effects.epoch();
|
||||
metadata.endpoint = "host.model".to_owned();
|
||||
metadata.transport = ClientTransport::InternalPlugin;
|
||||
metadata.client_ip = None;
|
||||
metadata.user_agent = None;
|
||||
metadata.inherit_request_origin(&authority.extension_scope);
|
||||
self.client_api_key_usage
|
||||
.record_used(authority.client.policy.key_id());
|
||||
self.start_authorized(
|
||||
|
||||
@@ -559,9 +559,22 @@ pub struct MiddlewareContext {
|
||||
deadline: crate::lifecycle::Deadline,
|
||||
extension_scope: ExtensionCallScope,
|
||||
execution_effects: Option<Arc<ExecutionEffects>>,
|
||||
request_body_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl MiddlewareContext {
|
||||
/// attempt 只继承执行预算,不开放 request 设置改写
|
||||
#[must_use]
|
||||
pub const fn with_request_body_limit(mut self, bytes: Option<usize>) -> Self {
|
||||
self.request_body_limit = bytes;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> Option<usize> {
|
||||
self.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn plan(&self) -> Option<FrozenMiddlewarePlan> {
|
||||
self.plan.clone()
|
||||
@@ -590,6 +603,7 @@ impl MiddlewareContext {
|
||||
} = authority;
|
||||
Self {
|
||||
plan: None,
|
||||
request_body_limit: None,
|
||||
request_id,
|
||||
mount,
|
||||
attempt_index,
|
||||
|
||||
@@ -286,6 +286,7 @@ impl ContinuationAttempt {
|
||||
/// Provider 每次执行可见的 request-local context
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestAttemptContext {
|
||||
request_body_limit: Option<usize>,
|
||||
lease_metadata: Option<Arc<crate::provider_ports::ProviderLeaseMetadata>>,
|
||||
response_control: Option<response_control::ResponseControl>,
|
||||
pricing: Arc<crate::metering::PricingOverrides>,
|
||||
@@ -311,6 +312,12 @@ pub struct RequestAttemptContext {
|
||||
}
|
||||
|
||||
impl RequestAttemptContext {
|
||||
#[must_use]
|
||||
pub const fn with_request_body_limit(mut self, bytes: usize) -> Self {
|
||||
self.request_body_limit = Some(bytes);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_lease_metadata(
|
||||
mut self,
|
||||
@@ -387,6 +394,7 @@ impl RequestAttemptContext {
|
||||
pub fn new(request_id: ModelRequestId, client_api_key_ref: ClientApiKeyId) -> Self {
|
||||
Self {
|
||||
response_control: None,
|
||||
request_body_limit: None,
|
||||
lease_metadata: None,
|
||||
request_id,
|
||||
client_api_key_ref,
|
||||
@@ -542,6 +550,11 @@ pub struct AttemptContext {
|
||||
}
|
||||
|
||||
impl AttemptContext {
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> Option<usize> {
|
||||
self.request.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn lease_metadata(&self) -> Option<Arc<crate::provider_ports::ProviderLeaseMetadata>> {
|
||||
self.request.lease_metadata.clone()
|
||||
@@ -747,7 +760,8 @@ impl AttemptContext {
|
||||
extension_scope: self.request.extension_scope.clone(),
|
||||
execution_effects: self.request.execution_effects.as_ref().map(Arc::clone),
|
||||
},
|
||||
);
|
||||
)
|
||||
.with_request_body_limit(self.request.request_body_limit);
|
||||
provider::execute_attempt_middleware(
|
||||
self.request.middleware.as_ref(),
|
||||
context,
|
||||
|
||||
@@ -34,9 +34,15 @@ pub struct ModelRouteInput {
|
||||
requested_model: PublicModelId,
|
||||
available_providers: BTreeSet<ProviderKind>,
|
||||
execution_effects: Arc<ExecutionEffects>,
|
||||
request_body_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl ModelRouteInput {
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> Option<usize> {
|
||||
self.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> &super::ModelRequestId {
|
||||
&self.request_id
|
||||
@@ -188,9 +194,15 @@ pub struct AccountScheduleInput {
|
||||
model: Option<String>,
|
||||
candidates: Vec<AccountScheduleCandidate>,
|
||||
execution_effects: Arc<ExecutionEffects>,
|
||||
request_body_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl AccountScheduleInput {
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> Option<usize> {
|
||||
self.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn request_id(&self) -> &super::ModelRequestId {
|
||||
&self.request_id
|
||||
@@ -282,9 +294,15 @@ pub struct RetryInput {
|
||||
pub extension_scope: ExtensionCallScope,
|
||||
pub facts: RetryFacts,
|
||||
execution_effects: Arc<ExecutionEffects>,
|
||||
request_body_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl RetryInput {
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> Option<usize> {
|
||||
self.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn execution_effects(&self) -> Arc<ExecutionEffects> {
|
||||
Arc::clone(&self.execution_effects)
|
||||
@@ -339,6 +357,7 @@ pub struct RequestPolicyContext {
|
||||
account_group_ids: Arc<[AccountGroupId]>,
|
||||
extension_scope: ExtensionCallScope,
|
||||
execution_effects: Arc<ExecutionEffects>,
|
||||
request_body_limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RequestPolicyContext {
|
||||
@@ -353,6 +372,13 @@ impl fmt::Debug for RequestPolicyContext {
|
||||
}
|
||||
|
||||
impl RequestPolicyContext {
|
||||
/// 路由、调度和重试沿用同一次模型执行的冻结预算
|
||||
#[must_use]
|
||||
pub const fn with_request_body_limit(mut self, bytes: usize) -> Self {
|
||||
self.request_body_limit = Some(bytes);
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn retry_decision(&self, facts: RetryFacts) -> RetryDecision {
|
||||
self.plan
|
||||
.retry_decision(RetryInput {
|
||||
@@ -362,6 +388,7 @@ impl RequestPolicyContext {
|
||||
extension_scope: self.extension_scope.clone(),
|
||||
facts,
|
||||
execution_effects: Arc::clone(&self.execution_effects),
|
||||
request_body_limit: self.request_body_limit,
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
@@ -391,6 +418,7 @@ impl RequestPolicyContext {
|
||||
account_group_ids: account_group_ids.into(),
|
||||
extension_scope: ExtensionCallScope::default(),
|
||||
execution_effects: Arc::new(ExecutionEffects::default()),
|
||||
request_body_limit: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,6 +456,7 @@ impl RequestPolicyContext {
|
||||
requested_model,
|
||||
available_providers,
|
||||
execution_effects: Arc::clone(&self.execution_effects),
|
||||
request_body_limit: self.request_body_limit,
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -474,6 +503,7 @@ impl RequestPolicyContext {
|
||||
model: model.map(str::to_owned),
|
||||
candidates: projected,
|
||||
execution_effects: Arc::clone(&self.execution_effects),
|
||||
request_body_limit: self.request_body_limit,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| AccountPolicyError::Fault)?;
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::fmt;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt, future::BoxFuture};
|
||||
use futures::{Stream, StreamExt, TryStreamExt, channel::oneshot, future::BoxFuture};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::account::{
|
||||
@@ -485,6 +485,9 @@ fn provider_middleware_next(
|
||||
operation: Operation,
|
||||
transport: ClientTransport,
|
||||
terminal: ProviderMiddlewareTerminal,
|
||||
prepared: oneshot::Sender<ProviderCallMetadata>,
|
||||
activate: oneshot::Receiver<()>,
|
||||
observation: Arc<Mutex<MiddlewareSendObservation>>,
|
||||
) -> MiddlewareNext {
|
||||
crate::middleware::compose(Vec::new(), move |request| {
|
||||
Box::pin(async move {
|
||||
@@ -501,11 +504,17 @@ fn provider_middleware_next(
|
||||
.await
|
||||
.map_err(MiddlewareError::Provider)?;
|
||||
let metadata = stream.metadata().clone();
|
||||
prepared
|
||||
.send(metadata.clone())
|
||||
.map_err(|_| MiddlewareError::InvalidState)?;
|
||||
// Core 先校验 metadata 并登记 attempt,首次轮询外层冷流后才放行 next
|
||||
activate.await.map_err(|_| MiddlewareError::InvalidState)?;
|
||||
let protocol = stream.middleware_protocol(&fallback_protocol);
|
||||
let body = ProviderMiddlewareBody {
|
||||
stream,
|
||||
pending: VecDeque::new(),
|
||||
transport,
|
||||
observation,
|
||||
};
|
||||
Ok(
|
||||
MiddlewareResponse::new(protocol, 200, Vec::new(), Box::new(body))
|
||||
@@ -550,6 +559,40 @@ struct ProviderMiddlewareBody {
|
||||
stream: ProviderStream,
|
||||
pending: VecDeque<(ProviderEvent, bool)>,
|
||||
transport: ClientTransport,
|
||||
observation: Arc<Mutex<MiddlewareSendObservation>>,
|
||||
}
|
||||
|
||||
struct MiddlewareSendObservation {
|
||||
observed: UpstreamSendState,
|
||||
reading: bool,
|
||||
}
|
||||
|
||||
impl MiddlewareSendObservation {
|
||||
fn state(&self) -> UpstreamSendState {
|
||||
if self.reading && self.observed == UpstreamSendState::NotSent {
|
||||
UpstreamSendState::Ambiguous
|
||||
} else {
|
||||
self.observed
|
||||
}
|
||||
}
|
||||
|
||||
fn complete(&mut self, observed: UpstreamSendState) {
|
||||
self.reading = false;
|
||||
self.observed = match (self.observed, observed) {
|
||||
(UpstreamSendState::Sent, _) | (_, UpstreamSendState::Sent) => UpstreamSendState::Sent,
|
||||
(UpstreamSendState::Ambiguous, _) | (_, UpstreamSendState::Ambiguous) => {
|
||||
UpstreamSendState::Ambiguous
|
||||
}
|
||||
_ => UpstreamSendState::NotSent,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn middleware_send_state(observation: &Mutex<MiddlewareSendObservation>) -> UpstreamSendState {
|
||||
observation
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.state()
|
||||
}
|
||||
|
||||
impl MiddlewareBody for ProviderMiddlewareBody {
|
||||
@@ -561,7 +604,21 @@ impl MiddlewareBody for ProviderMiddlewareBody {
|
||||
.map(Some)
|
||||
.map_err(MiddlewareError::Provider);
|
||||
}
|
||||
let Some(event) = self.stream.next().await else {
|
||||
self.observation
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.reading = true;
|
||||
let event = self.stream.next().await;
|
||||
let observed = match &event {
|
||||
Some(Ok(_)) => UpstreamSendState::Sent,
|
||||
Some(Err(error)) => error.send_state(),
|
||||
None => UpstreamSendState::Ambiguous,
|
||||
};
|
||||
self.observation
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.complete(observed);
|
||||
let Some(event) = event else {
|
||||
return Ok(None);
|
||||
};
|
||||
let event = event.map_err(MiddlewareError::Provider)?;
|
||||
@@ -689,30 +746,71 @@ pub async fn execute_attempt_middleware(
|
||||
.middleware_body()
|
||||
.map_err(|_| middleware_protocol_error(UpstreamSendState::NotSent))?,
|
||||
);
|
||||
let response = plan
|
||||
.handle(
|
||||
context,
|
||||
request,
|
||||
provider_middleware_next(operation, transport, terminal),
|
||||
)
|
||||
.await
|
||||
.map_err(middleware_prepare_error)?;
|
||||
let (protocol, status_code, headers, body, envelope) = response.into_parts();
|
||||
if status_code != 200 || !headers.is_empty() {
|
||||
return Err(middleware_protocol_error(UpstreamSendState::NotSent));
|
||||
}
|
||||
crate::validation::validate_text(&protocol, 64, true, None)
|
||||
.map_err(|_| middleware_protocol_error(UpstreamSendState::NotSent))?;
|
||||
let metadata = envelope
|
||||
.ok_or_else(|| middleware_protocol_error(UpstreamSendState::NotSent))?
|
||||
.into_provider_metadata();
|
||||
let events = futures::stream::try_unfold((body, protocol), |(mut body, protocol)| async move {
|
||||
let Some(frame) = body.next_frame().await.map_err(middleware_body_error)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let event = middleware_frame_to_provider_event(frame, &protocol)?;
|
||||
Ok(Some((event, (body, protocol))))
|
||||
});
|
||||
let (prepared, metadata) = oneshot::channel();
|
||||
let (activate, activation) = oneshot::channel();
|
||||
let observation = Arc::new(Mutex::new(MiddlewareSendObservation {
|
||||
observed: UpstreamSendState::NotSent,
|
||||
reading: false,
|
||||
}));
|
||||
let response = plan.handle(
|
||||
context,
|
||||
request,
|
||||
provider_middleware_next(
|
||||
operation,
|
||||
transport,
|
||||
terminal,
|
||||
prepared,
|
||||
activation,
|
||||
observation.clone(),
|
||||
),
|
||||
);
|
||||
let (metadata, response) = match futures::future::select(metadata, response).await {
|
||||
futures::future::Either::Left((Ok(metadata), response)) => (metadata, response),
|
||||
futures::future::Either::Left((Err(_), response)) => {
|
||||
return Err(unprepared_middleware_response(response.await));
|
||||
}
|
||||
futures::future::Either::Right((response, _)) => {
|
||||
return Err(unprepared_middleware_response(response));
|
||||
}
|
||||
};
|
||||
let expected_metadata = metadata.clone();
|
||||
// 不创建后台任务,未激活的 handler、stream 与账号 lease 随返回的冷流一起回收
|
||||
let events = futures::stream::once(async move {
|
||||
activate
|
||||
.send(())
|
||||
.map_err(|_| middleware_protocol_error(middleware_send_state(&observation)))?;
|
||||
let response = response
|
||||
.await
|
||||
.map_err(|error| middleware_error(error, middleware_send_state(&observation)))?;
|
||||
let (protocol, status_code, headers, body, envelope) = response.into_parts();
|
||||
if status_code != 200
|
||||
|| !headers.is_empty()
|
||||
|| envelope
|
||||
.map(|envelope| envelope.into_provider_metadata())
|
||||
.as_ref()
|
||||
!= Some(&expected_metadata)
|
||||
{
|
||||
return Err(middleware_protocol_error(middleware_send_state(
|
||||
&observation,
|
||||
)));
|
||||
}
|
||||
crate::validation::validate_text(&protocol, 64, true, None)
|
||||
.map_err(|_| middleware_protocol_error(middleware_send_state(&observation)))?;
|
||||
Ok(futures::stream::try_unfold(
|
||||
(body, protocol, observation),
|
||||
|(mut body, protocol, observation)| async move {
|
||||
let Some(frame) = body.next_frame().await.map_err(|error| {
|
||||
middleware_error(error, middleware_send_state(&observation))
|
||||
})?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let event = middleware_frame_to_provider_event(frame, &protocol)?;
|
||||
Ok(Some((event, (body, protocol, observation))))
|
||||
},
|
||||
))
|
||||
})
|
||||
.try_flatten();
|
||||
Ok(ProviderStream::new(metadata, events, ()))
|
||||
}
|
||||
|
||||
@@ -786,37 +884,26 @@ fn middleware_frame_to_provider_event(
|
||||
Ok(event)
|
||||
}
|
||||
|
||||
fn middleware_prepare_error(error: MiddlewareError) -> ProviderError {
|
||||
match error {
|
||||
MiddlewareError::Provider(error) => error,
|
||||
MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::RequestPolicyDenied,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
}
|
||||
MiddlewareError::Fault
|
||||
| MiddlewareError::Remote { .. }
|
||||
| MiddlewareError::InvalidState
|
||||
| MiddlewareError::Gateway(_)
|
||||
| MiddlewareError::Engine(_) => middleware_protocol_error(UpstreamSendState::NotSent),
|
||||
fn unprepared_middleware_response(
|
||||
response: Result<MiddlewareResponse, MiddlewareError>,
|
||||
) -> ProviderError {
|
||||
match response {
|
||||
Err(error) => middleware_error(error, UpstreamSendState::NotSent),
|
||||
Ok(_) => middleware_protocol_error(UpstreamSendState::NotSent),
|
||||
}
|
||||
}
|
||||
|
||||
fn middleware_body_error(error: MiddlewareError) -> ProviderError {
|
||||
fn middleware_error(error: MiddlewareError, send_state: UpstreamSendState) -> ProviderError {
|
||||
match error {
|
||||
MiddlewareError::Provider(error) => error,
|
||||
MiddlewareError::Rejected | MiddlewareError::Remote { rejected: true, .. } => {
|
||||
ProviderError::new(
|
||||
ProviderErrorKind::RequestPolicyDenied,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)
|
||||
ProviderError::new(ProviderErrorKind::RequestPolicyDenied, send_state)
|
||||
}
|
||||
MiddlewareError::Fault
|
||||
| MiddlewareError::Remote { .. }
|
||||
| MiddlewareError::InvalidState
|
||||
| MiddlewareError::Gateway(_)
|
||||
| MiddlewareError::Engine(_) => middleware_protocol_error(UpstreamSendState::Ambiguous),
|
||||
| MiddlewareError::Engine(_) => middleware_protocol_error(send_state),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -547,7 +547,7 @@ pub struct ProviderResponseTimings {
|
||||
/// 这样 Provider 专有协议不会渗入路由
|
||||
/// 或管理领域
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderResponseMetadata(String);
|
||||
pub struct ProviderResponseMetadata(Arc<str>);
|
||||
|
||||
impl ProviderResponseMetadata {
|
||||
const MAX_BYTES: usize = 32 * 1024;
|
||||
@@ -561,7 +561,7 @@ impl ProviderResponseMetadata {
|
||||
serde_json::from_str::<Value>(&json)
|
||||
.ok()
|
||||
.filter(Value::is_object)
|
||||
.map(|_| Self(json))
|
||||
.map(|_| Self(json.into()))
|
||||
}
|
||||
|
||||
/// 返回不透明的 JSON 文本,供持久化 adapter 原样验证和写入
|
||||
@@ -626,7 +626,8 @@ pub struct ProviderResponseObservation {
|
||||
service_tier: Option<String>,
|
||||
upstream_response_model: Option<String>,
|
||||
timings: ProviderResponseTimings,
|
||||
client_headers: Vec<ProviderResponseHeader>,
|
||||
// 流式事件分别持有快照,共享不可变头集合,避免逐事件复制头名和集合
|
||||
client_headers: Arc<[ProviderResponseHeader]>,
|
||||
provider_metadata: Option<ProviderResponseMetadata>,
|
||||
}
|
||||
|
||||
@@ -642,7 +643,7 @@ impl ProviderResponseObservation {
|
||||
service_tier: None,
|
||||
upstream_response_model: None,
|
||||
timings: ProviderResponseTimings::default(),
|
||||
client_headers: Vec::new(),
|
||||
client_headers: Arc::default(),
|
||||
provider_metadata: None,
|
||||
}
|
||||
}
|
||||
@@ -719,7 +720,7 @@ impl ProviderResponseObservation {
|
||||
|
||||
#[must_use]
|
||||
pub fn with_client_headers(mut self, client_headers: Vec<ProviderResponseHeader>) -> Self {
|
||||
self.client_headers = client_headers;
|
||||
self.client_headers = client_headers.into();
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::account::scope::FrozenAccountScope;
|
||||
use crate::event::ProviderResponseHeader;
|
||||
use crate::operation::ProviderHttpHeader;
|
||||
use crate::policy::ClientApiKeyId;
|
||||
|
||||
/// sideband 中继在客户端与上游 WebSocket 之间搬运的帧。
|
||||
@@ -41,7 +43,7 @@ pub enum LiveSidebandStyle {
|
||||
RealtimeQuery,
|
||||
}
|
||||
|
||||
/// sideband 拨号请求;协议头已由协议 adapter 按允许清单过滤。
|
||||
/// sideband 拨号请求;协议 adapter 已剥离入站身份与传输头
|
||||
#[derive(Debug)]
|
||||
pub struct LiveSidebandRequest<'a> {
|
||||
pub call_id: &'a str,
|
||||
@@ -49,7 +51,7 @@ pub struct LiveSidebandRequest<'a> {
|
||||
/// 本次认证冻结的账号范围与模型政策,重连也必须重新校验
|
||||
pub account_scope: &'a FrozenAccountScope,
|
||||
pub style: LiveSidebandStyle,
|
||||
pub protocol_headers: Vec<(String, String)>,
|
||||
pub protocol_headers: Vec<ProviderHttpHeader>,
|
||||
/// 客户端 offer 的 `Sec-WebSocket-Protocol`,原样透传给上游。
|
||||
pub subprotocols: Vec<String>,
|
||||
}
|
||||
@@ -61,14 +63,14 @@ pub struct LiveHangupRequest<'a> {
|
||||
pub client_api_key_id: &'a ClientApiKeyId,
|
||||
pub content_type: Option<String>,
|
||||
pub body: Bytes,
|
||||
pub protocol_headers: Vec<(String, String)>,
|
||||
pub protocol_headers: Vec<ProviderHttpHeader>,
|
||||
}
|
||||
|
||||
/// 一次性账号级上游调用的透传结果。
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LiveCallOutcome {
|
||||
pub status: u16,
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub headers: Vec<ProviderResponseHeader>,
|
||||
pub body: Bytes,
|
||||
}
|
||||
|
||||
|
||||
@@ -830,12 +830,8 @@ impl ProviderHttpRequest {
|
||||
&& headers
|
||||
.iter()
|
||||
.try_fold(0_usize, |total, header| {
|
||||
let valid_name = !header.name.is_empty()
|
||||
&& header.name.len() <= 128
|
||||
&& header
|
||||
.name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-');
|
||||
let valid_name = header.name.len() <= 128
|
||||
&& http::HeaderName::from_bytes(header.name.as_bytes()).is_ok();
|
||||
let valid_value = header.value.len() <= 8 * 1024
|
||||
&& !header
|
||||
.value
|
||||
|
||||
@@ -88,6 +88,15 @@ pub struct ExtensionPreparationError;
|
||||
|
||||
/// 准备阶段读取与请求快照相同的持久 revision;不得自行发布当前代次
|
||||
pub trait ExtensionPreparationPort: Send + Sync {
|
||||
/// 初始发布尚无可读快照,注册阶段沿用本次编译的正文预算
|
||||
fn prepare_with_request_body_limit(
|
||||
&self,
|
||||
revision: ConfigRevision,
|
||||
_request_body_limit: usize,
|
||||
) -> BoxFuture<'_, Result<ExtensionSetReference, ExtensionPreparationError>> {
|
||||
self.prepare(revision)
|
||||
}
|
||||
|
||||
fn prepare(
|
||||
&self,
|
||||
revision: ConfigRevision,
|
||||
|
||||
@@ -561,6 +561,7 @@ impl ProviderCandidate {
|
||||
/// 一次请求冻结的 Provider 尝试顺序
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RoutingPlan {
|
||||
request_body_limit: usize,
|
||||
pricing: Arc<crate::metering::PricingOverrides>,
|
||||
request_location: Option<crate::account::RequestLocation>,
|
||||
privacy: Option<Arc<dyn crate::settings::privacy::CompiledPrivacyPolicy>>,
|
||||
@@ -573,6 +574,11 @@ pub struct RoutingPlan {
|
||||
}
|
||||
|
||||
impl RoutingPlan {
|
||||
#[must_use]
|
||||
pub const fn request_body_limit(&self) -> usize {
|
||||
self.request_body_limit
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn pricing(&self) -> Arc<crate::metering::PricingOverrides> {
|
||||
Arc::clone(&self.pricing)
|
||||
|
||||
@@ -331,7 +331,13 @@ impl RuntimeSnapshotCompiler {
|
||||
});
|
||||
if must_prepare {
|
||||
let prepared = preparer
|
||||
.prepare(revision)
|
||||
.prepare_with_request_body_limit(
|
||||
revision,
|
||||
usize::try_from(
|
||||
facts.settings.responses_max_decompressed_body_bytes,
|
||||
)
|
||||
.unwrap_or(usize::MAX),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| RuntimeSnapshotCompileError::ExtensionsUnavailable)?;
|
||||
prepared_extensions = Some((revision, prepared));
|
||||
@@ -1201,6 +1207,7 @@ impl RuntimeSnapshot {
|
||||
}
|
||||
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
@@ -1282,6 +1289,7 @@ impl RuntimeSnapshot {
|
||||
account_scope: Arc::clone(&account_scope),
|
||||
};
|
||||
Ok(RoutingPlan {
|
||||
request_body_limit: self.responses_max_decompressed_body_bytes(),
|
||||
config_revision: self.revision,
|
||||
pricing: Arc::clone(&self.settings.values.pricing),
|
||||
request_location: self.settings.request_location.clone(),
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
mod publications;
|
||||
mod publisher;
|
||||
|
||||
pub use publications::{RuntimeSnapshotHandle, RuntimeSnapshotUnavailable};
|
||||
pub use publications::{
|
||||
RuntimeSnapshotHandle, RuntimeSnapshotUnavailable, WeakRuntimeSnapshotHandle,
|
||||
};
|
||||
pub use publisher::{
|
||||
RuntimeSnapshotPublisher, SnapshotControl, SnapshotRevisionStream, SnapshotSubscriptionError,
|
||||
SnapshotSubscriptionPort, runtime_revision_needs_refresh,
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::{
|
||||
use futures::future::BoxFuture;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex as SyncMutex, RwLock},
|
||||
sync::{Arc, Mutex as SyncMutex, RwLock, Weak},
|
||||
};
|
||||
|
||||
/// 请求级冻结失败;此状态必须 fail closed
|
||||
@@ -23,7 +23,27 @@ pub struct RuntimeSnapshotHandle {
|
||||
publications: Arc<SyncMutex<Vec<futures::channel::mpsc::Sender<()>>>>,
|
||||
}
|
||||
|
||||
/// 不延长发布集合生命周期的只读句柄,供集合持有的扩展读取后续配置
|
||||
#[derive(Clone, Default)]
|
||||
pub struct WeakRuntimeSnapshotHandle {
|
||||
current: Weak<RwLock<Option<Arc<RuntimeSnapshot>>>>,
|
||||
}
|
||||
|
||||
impl WeakRuntimeSnapshotHandle {
|
||||
pub fn acquire(&self) -> Result<Arc<RuntimeSnapshot>, RuntimeSnapshotUnavailable> {
|
||||
let current = self.current.upgrade().ok_or(RuntimeSnapshotUnavailable)?;
|
||||
acquire_current(¤t)
|
||||
}
|
||||
}
|
||||
|
||||
impl RuntimeSnapshotHandle {
|
||||
#[must_use]
|
||||
pub fn downgrade(&self) -> WeakRuntimeSnapshotHandle {
|
||||
WeakRuntimeSnapshotHandle {
|
||||
current: Arc::downgrade(&self.current),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn new(initial: RuntimeSnapshot) -> Self {
|
||||
Self {
|
||||
@@ -86,10 +106,7 @@ impl RuntimeSnapshotHandle {
|
||||
|
||||
/// 冻结当前 Arc;后续发布不改变已经开始的请求
|
||||
pub fn acquire(&self) -> Result<Arc<RuntimeSnapshot>, RuntimeSnapshotUnavailable> {
|
||||
read_unpoisoned(&self.current)
|
||||
.clone()
|
||||
.filter(|snapshot| snapshot.extensions().is_none_or(|set| set.can_serve()))
|
||||
.ok_or(RuntimeSnapshotUnavailable)
|
||||
acquire_current(&self.current)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +126,15 @@ impl HealthProbe for RuntimeSnapshotHandle {
|
||||
}
|
||||
}
|
||||
|
||||
fn acquire_current(
|
||||
current: &RwLock<Option<Arc<RuntimeSnapshot>>>,
|
||||
) -> Result<Arc<RuntimeSnapshot>, RuntimeSnapshotUnavailable> {
|
||||
read_unpoisoned(current)
|
||||
.clone()
|
||||
.filter(|snapshot| snapshot.extensions().is_none_or(|set| set.can_serve()))
|
||||
.ok_or(RuntimeSnapshotUnavailable)
|
||||
}
|
||||
|
||||
fn read_unpoisoned<T>(lock: &RwLock<T>) -> std::sync::RwLockReadGuard<'_, T> {
|
||||
lock.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
|
||||
@@ -372,8 +372,12 @@ struct RuntimeSnapshotSubscriptionTask {
|
||||
}
|
||||
|
||||
impl DaemonTask for RuntimeSnapshotSubscriptionTask {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
fn run(
|
||||
&self,
|
||||
context: crate::task::WorkerDaemonContext,
|
||||
) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
let cancellation = context.cancellation();
|
||||
let mut retry_delay = INITIAL_BACKOFF;
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -383,10 +387,11 @@ impl DaemonTask for RuntimeSnapshotSubscriptionTask {
|
||||
let mut subscription = match subscription {
|
||||
Ok(subscription) => {
|
||||
retry_delay = INITIAL_BACKOFF;
|
||||
context.succeeded();
|
||||
subscription
|
||||
}
|
||||
Err(_) => {
|
||||
wait_or_cancel(&cancellation, retry_delay).await;
|
||||
wait_or_cancel(cancellation, retry_delay).await;
|
||||
retry_delay = (retry_delay * 2).min(MAXIMUM_BACKOFF);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -6,8 +6,11 @@
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU64;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use event_listener::Event;
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
use crate::lifecycle::CancellationToken;
|
||||
@@ -341,11 +344,60 @@ pub trait ScheduledTask: Send + Sync {
|
||||
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>>;
|
||||
}
|
||||
|
||||
/// 一次 daemon 运行的取消与成功进度,重启时必须创建新上下文
|
||||
#[derive(Clone)]
|
||||
pub struct WorkerDaemonContext {
|
||||
cancellation: CancellationToken,
|
||||
progress: Arc<DaemonProgress>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DaemonProgress {
|
||||
completed: AtomicU64,
|
||||
changed: Event,
|
||||
}
|
||||
|
||||
impl WorkerDaemonContext {
|
||||
#[must_use]
|
||||
pub fn new(cancellation: CancellationToken) -> Self {
|
||||
Self {
|
||||
cancellation,
|
||||
progress: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cancellation(&self) -> &CancellationToken {
|
||||
&self.cancellation
|
||||
}
|
||||
|
||||
/// 仅在实际工作或依赖检查成功后通知 Host;进入 run 不代表恢复
|
||||
pub fn succeeded(&self) {
|
||||
if !self.cancellation.is_cancelled() {
|
||||
self.progress.completed.fetch_add(1, Ordering::Release);
|
||||
self.progress.changed.notify(usize::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn completed_cycles(&self) -> u64 {
|
||||
self.progress.completed.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// 先注册再检查计数,避免报告与等待交错时丢失成功通知
|
||||
pub async fn progress_since(&self, completed: u64) {
|
||||
let changed = self.progress.changed.listen();
|
||||
if self.completed_cycles() == completed {
|
||||
changed.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 长驻任务
|
||||
///
|
||||
/// 实现负责连接级重连退避;Host 负责任务退出或 panic 后的重启退避
|
||||
pub trait DaemonTask: Send + Sync {
|
||||
fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>>;
|
||||
fn run(&self, context: WorkerDaemonContext) -> BoxFuture<'_, Result<(), WorkerTaskError>>;
|
||||
}
|
||||
|
||||
/// Host 可执行的两种任务形态,无法组合出“守护任务 + 周期调度”等非法状态
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user