mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-09 16:06:27 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af35cde374 | ||
|
|
5c115fb38f | ||
|
|
ed558e5396 | ||
|
|
929f781a63 | ||
|
|
e0326ecba4 | ||
|
|
6057088300 | ||
|
|
e3a49a6217 | ||
|
|
b32f87fa8a | ||
|
|
91e8105e5d | ||
|
|
c4d9576639 | ||
|
|
015cd60be1 | ||
|
|
bad2a245ab | ||
|
|
b49b4528e7 | ||
|
|
d678f350c1 | ||
|
|
17b05d4671 | ||
|
|
7148303ad6 | ||
|
|
f6538d433d | ||
|
|
406256bcc0 | ||
|
|
d912fa0c59 | ||
|
|
a166580aed | ||
|
|
d1aec7311c | ||
|
|
16d8a3f873 | ||
|
|
ac6033f6fa | ||
|
|
cef9f0b21e | ||
|
|
591f95cc18 | ||
|
|
99ba5b1120 |
@@ -15,6 +15,7 @@ description: Codex Proxy RS 前后端开发、排障、代码审查和文档修
|
||||
| 修改服务端接口、路由、调度、账号与用量规则、持久化、后台任务或协议行为;目标是 `backend/` 中的实现、配置、迁移,或 Rust 源码 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)、所属模块、[后端自审](../../../docs/architecture.md#后端自审);涉及 Rust 时使用 `$rust-best-practices` 和下方后端注释要求,验证时读[命令](../../../docs/architecture.md#验证命令) |
|
||||
| 修改页面接口调用,或开发、排查、审查面向管理端或模型客户端的 HTTP/WS API 合同 | [API](../../../docs/api.md)对应章节;优先复用已有接口,核对调用方 → 服务端解析 → 实际查询或执行,不能只凭前端类型认定参数生效;按实际改动叠加前后端规范 |
|
||||
| 模块或调用链变更 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)及所属模块章节,不通读整份架构 |
|
||||
| 新增、调整、清理或审查测试 | 下方[测试维护](#测试维护),按当前行为、职责归属与已有覆盖判断保留、合并或删除 |
|
||||
| 修改部署行为、`deploy/` 配置或数据库迁移 | 按范围读取[部署](../../../deploy/README.md)、[迁移](../../../backend/migrations/README.md)对应章节 |
|
||||
| 开发、排查或审查宿主 SDK、Runtime、扩展接口或独立插件;目标在 `backend/crates/gateway-plugin/` | [插件职责边界](references/plugin-boundaries.md)及所用 [SDK](../../../backend/crates/gateway-plugin/sdk/README.md) 类型与能力合同;宿主守住自身合同、资源与稳定性,不猜测或补偿插件业务错误;仅独立插件创作、排查另用 [cpr-plugin-dev](../cpr-plugin-dev/SKILL.md) |
|
||||
| 修改或审查文档 | [文档检查](references/documentation.md)及目标章节,不默认读取代码开发、页面验收或发布流程 |
|
||||
@@ -35,6 +36,16 @@ description: Codex Proxy RS 前后端开发、排障、代码审查和文档修
|
||||
4. 按变更范围执行[验证](../../../CONTRIBUTING.md#验证)及其停止条件,沿用本轮有效证据与用户已确定的验收分工,区分通过、失败、跳过与未执行;提交时补读[提交约定](../../../CONTRIBUTING.md#项目约定)
|
||||
5. 交付实际结果与验证缺口;前端按[前端流程](references/frontend.md)核对最终差异和运行证据,后端按上表检查生命周期,不把构建通过当作行为验收
|
||||
|
||||
## 测试维护
|
||||
|
||||
- 测试随当前功能维护,名称、夹具和断言描述当前输入、行为与结果;功能变更时优先更新所属模块的现有用例,不为每次改动叠加一组记录变更经过的测试
|
||||
- 清理只证明旧字段、旧路由或旧实现已经消失的用例与附带断言;未知字段校验使用合法请求基线加无业务含义的未知字段,避免因缺少必填项等其他错误而误通过
|
||||
- 同一层对同一行为的重复覆盖合并为清晰的场景矩阵,复用相邻测试夹具;跨层测试确有不同职责或失败路径时保留,不为统一而扩大测试辅助抽象
|
||||
- 断言可观察的行为与业务不变量,避免复述结构体赋值、罗列表列或字段清单、匹配 SQL 文本与源码布局;项目明确要求的架构边界检查继续维护
|
||||
- 当前协议兼容、数据读取兼容、安全边界、并发和资源生命周期仍需覆盖,不按 `old`、`legacy` 等名称机械删除,也不能用改名代替无效用例清理
|
||||
- 不在仓库中新增或保留迁移 SQL 专项测试,包括迁移文本、schema 演变与指定版本升级用例;需要验证迁移执行、数据保留或回滚时,在临时目录编写一次性验证并使用隔离数据库,测试通过即可,临时脚本不纳入维护测试集
|
||||
- 持久化功能测试使用当前完整 schema,验证现行读写、事务与数据约束;插件运行时状态迁移等业务能力按当前功能测试,不归为迁移 SQL 专项测试
|
||||
|
||||
## 后端注释
|
||||
|
||||
- 每个 Rust 文件顶层使用简短的中文 `//!` 注释说明文件职责,覆盖生产源码、测试与构建脚本;测试文件说明测试范围或辅助用途,生成文件同步维护生成器
|
||||
|
||||
@@ -18,7 +18,7 @@ runs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
@@ -38,7 +38,7 @@ runs:
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: inputs.upload-artifact == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist
|
||||
|
||||
@@ -242,7 +242,7 @@ runs:
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
|
||||
@@ -52,7 +52,7 @@ updates:
|
||||
- patch
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /
|
||||
directory: /frontend
|
||||
labels: []
|
||||
schedule:
|
||||
interval: weekly
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -156,6 +156,20 @@ jobs:
|
||||
echo "CPR_TEST_REDIS_URL=redis://:${redis_password}@127.0.0.1:6379"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Expose matching PostgreSQL backup and restore tools
|
||||
shell: bash
|
||||
env:
|
||||
CPR_TEST_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
client_dir="$RUNNER_TEMP/pg-client"
|
||||
mkdir -p "$client_dir"
|
||||
for tool in pg_dump psql; do
|
||||
printf '#!/bin/sh\nexec docker exec -i --env PGPASSWORD "%s" %s "$@"\n' "$CPR_TEST_POSTGRES_CONTAINER" "$tool" > "$client_dir/$tool"
|
||||
chmod +x "$client_dir/$tool"
|
||||
done
|
||||
echo "$client_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Rust tests
|
||||
working-directory: backend
|
||||
run: |
|
||||
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
- name: Setup Rust
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload sealed plugin assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -219,19 +219,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/frontend
|
||||
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: codex-proxy-rs-*
|
||||
path: dist/binaries
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
subject-path: dist/release/*
|
||||
|
||||
- name: Upload release assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
@@ -283,19 +283,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/docker/web/dist
|
||||
|
||||
- name: Download Docker binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -362,7 +362,7 @@ jobs:
|
||||
printf '%s\n' "$IMAGE_DIGEST" > "dist/digests/linux-${IMAGE_ARCH}.txt"
|
||||
|
||||
- name: Upload verified image digest
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: image-digest-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/digests/*.txt
|
||||
@@ -390,13 +390,13 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download release assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
|
||||
- name: Download verified image digests
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: image-digest-*
|
||||
path: dist/digests
|
||||
|
||||
@@ -33,14 +33,23 @@ jobs:
|
||||
...params,
|
||||
per_page: 100,
|
||||
});
|
||||
// 按 PR 作者识别官方机器人,维护者重开或编辑 PR 时仍使用同一判断
|
||||
const isDependabot = pr.user.login === 'dependabot[bot]'
|
||||
&& pr.user.type === 'Bot'
|
||||
&& pr.head.repo?.full_name === `${owner}/${repo}`;
|
||||
const isDependencyUpdate = file => isDependabot
|
||||
&& file.status === 'modified'
|
||||
&& (/^\.github\/workflows\/[^/]+\.ya?ml$/.test(file.filename)
|
||||
|| /^\.github\/actions\/.+\/(?:action\.ya?ml|Dockerfile)$/.test(file.filename));
|
||||
const isProtectedRule = path => typeof path === 'string'
|
||||
&& (path === 'AGENTS.md' || path.endsWith('/AGENTS.md')
|
||||
|| path === 'CONTRIBUTING.md'
|
||||
|| path === '.agents/skills' || path.startsWith('.agents/skills/')
|
||||
|| path === '.github' || path.startsWith('.github/'));
|
||||
|
||||
// 同时检查重命名前后的路径,避免通过移走规则文件绕过检查。
|
||||
if (!files.some(file => isProtectedRule(file.filename) || isProtectedRule(file.previous_filename))) {
|
||||
// 依赖更新只豁免已有文件的修改,重命名前后的受保护路径仍参与检查
|
||||
if (!files.some(file => !isDependencyUpdate(file)
|
||||
&& (isProtectedRule(file.filename) || isProtectedRule(file.previous_filename)))) {
|
||||
core.info('No protected contribution rule changes detected.');
|
||||
return;
|
||||
}
|
||||
@@ -56,7 +65,7 @@ jobs:
|
||||
const marker = '<!-- agents-md-guard -->';
|
||||
const body = [
|
||||
marker,
|
||||
'佬友,我看到这个 PR 涉及受保护的协作规则文件,按仓库约定需要先关闭',
|
||||
'我看到这个 PR 涉及受保护的协作规则文件,按仓库约定需要先关闭',
|
||||
'',
|
||||
'保护范围包括根目录或子目录的 `AGENTS.md`、根目录的 `CONTRIBUTING.md`,以及 `.agents/skills/` 和 `.github/` 中的所有文件,涵盖新增、修改、删除和重命名',
|
||||
'',
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
|
||||
+5
-2
@@ -23,7 +23,8 @@
|
||||
|
||||
AI 草拟或发布 Issue、PR 讨论及审查意见的回复时:
|
||||
|
||||
- 用第一人称“我”,自然称呼对方为“佬友”,不逐句重复,也不用“我们”“本助手”代称自己
|
||||
- 用第一人称“我”,不用“我们”“本助手”代称自己
|
||||
- 普通回复直接回应问题,不加问候语或称呼;向对方提问时称呼“佬”,同一条回复不重复称呼
|
||||
- 客气、委婉地回应具体问题,补充信息或提出不同意见时说明缘由,避免命令式、责备式表达
|
||||
- 使用自然短句,简单回复用短段,并列信息较多时再分点;少用句号,避免客服套话、机械模板和空泛保证
|
||||
- 只把实际完成的排查、复现、修改和验证写成“我已……”;未采纳的意见说明依据,未确认、待处理或未验证的内容如实交代,说清结论、限制和下一步
|
||||
@@ -84,7 +85,9 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名:
|
||||
- `.agents/skills/` 中的所有文件,包括技能入口、引用文档、配置和脚本
|
||||
- `.github/` 中的所有文件,包括工作流、Actions、Issue/PR 模板和依赖更新配置
|
||||
|
||||
调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
官方 `dependabot[bot]` 从本仓库分支发起的依赖更新可修改 `.github/workflows/` 中已有的 YAML 工作流,以及 `.github/actions/` 下已有的 `action.yml`、`action.yaml` 和 `Dockerfile`,继续执行正常审查与 CI;其他受保护文件以及新增、删除、重命名不在例外范围内
|
||||
|
||||
除此之外,调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
|
||||
源码变更遵守现有架构边界与迁移冻结规则,由对应 CI 检查执行;不额外设立整块业务目录的 PR 禁改清单。构建、测试和安全扫描复用现有工作流
|
||||
|
||||
|
||||
Generated
+22
-19
@@ -198,7 +198,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -856,7 +856,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -910,7 +910,7 @@ dependencies = [
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1657,7 +1657,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1720,7 +1720,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1903,6 +1903,7 @@ dependencies = [
|
||||
name = "gateway-store"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
@@ -1915,6 +1916,7 @@ dependencies = [
|
||||
"gateway-admin",
|
||||
"gateway-core",
|
||||
"hex",
|
||||
"percent-encoding",
|
||||
"redis",
|
||||
"secrecy",
|
||||
"serde",
|
||||
@@ -3266,7 +3268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3626,7 +3628,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3999,7 +4001,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4014,10 +4016,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
@@ -4098,9 +4101,9 @@ checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.11.0"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
|
||||
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
@@ -4480,9 +4483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -4580,7 +4583,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4734,9 +4737,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.19"
|
||||
version = "0.7.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
|
||||
checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -5051,9 +5054,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.27.0"
|
||||
version = "1.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ opt-level = 2
|
||||
[profile.dev.package.blake2]
|
||||
opt-level = 2
|
||||
|
||||
[profile.dev.package."sha2@0.11.0"]
|
||||
[profile.dev.package."sha2@0.11.1"]
|
||||
opt-level = 2
|
||||
|
||||
[profile.dev.package."miniz_oxide@0.9.1"]
|
||||
|
||||
@@ -106,8 +106,11 @@ fn config_loader_should_apply_only_explicit_topology_overrides() {
|
||||
"postgres://codex_proxy@127.0.0.1:5432/codex_proxy",
|
||||
)
|
||||
.env("CPR_REDIS_URL", "redis://127.0.0.1:6379/")
|
||||
.env("CPR_DATABASE_PASSWORD", POSTGRES_PASSWORD)
|
||||
.env("CPR_REDIS_PASSWORD", REDIS_PASSWORD)
|
||||
.env(
|
||||
"CPR_DATABASE_PASSWORD",
|
||||
"RdsTest9!abcdefGhRdsTest9!abcdefGh",
|
||||
)
|
||||
.env("CPR_REDIS_PASSWORD", "ManagedRedis9!abcdefGh")
|
||||
.status()
|
||||
.expect("run isolated environment override test");
|
||||
assert!(status.success());
|
||||
@@ -152,14 +155,6 @@ fn config_loader_should_ignore_unknown_fields_in_configuration_sections() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_ignore_removed_tls_and_fingerprint_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["tls"] = serde_json::json!({});
|
||||
document["openai"]["fingerprint"] = serde_json::json!({"browser": "removed"});
|
||||
parse_config(&document.to_string()).expect("removed provider settings are ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_known_fields_alongside_unknown_fields() {
|
||||
for port in [serde_json::json!(0), serde_json::json!("not-a-port")] {
|
||||
@@ -189,7 +184,7 @@ fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
] {
|
||||
let mut document = valid_config_document();
|
||||
if case == "unused" {
|
||||
document["openai"]["wire_profile"]["location"] = serde_json::json!(UNUSED_SECRET);
|
||||
document["openai"]["unused_setting"] = serde_json::json!(UNUSED_SECRET);
|
||||
} else if case == "missing" {
|
||||
document["host"]["listen"]
|
||||
.as_object_mut()
|
||||
@@ -229,7 +224,7 @@ fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
);
|
||||
match case {
|
||||
"unused" => assert!(
|
||||
stderr.contains("openai.wire_profile") && stderr.contains("已忽略"),
|
||||
stderr.contains("openai.unused_setting") && stderr.contains("已忽略"),
|
||||
"{stderr}"
|
||||
),
|
||||
"missing" => assert!(stderr.contains("host.listen.port"), "{stderr}"),
|
||||
@@ -253,19 +248,7 @@ fn config_loader_should_allow_missing_provider_configuration() {
|
||||
let mut document = valid_config_document();
|
||||
document.as_object_mut().unwrap().remove("openai");
|
||||
document.as_object_mut().unwrap().remove("xai");
|
||||
parse_config(&document.to_string()).expect("provider defaults require no YAML identity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_ignore_removed_provider_identity_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["wire_profile"] = serde_json::json!({"codex_version":"legacy-invalid"});
|
||||
document["xai"] = serde_json::json!({"wire_profile": {"client_identifier": "legacy-client"}});
|
||||
let (config, _directory) =
|
||||
parse_config(&document.to_string()).expect("removed identities ignored");
|
||||
let debug = format!("{config:?}");
|
||||
assert!(!debug.contains("legacy-invalid"));
|
||||
assert!(!debug.contains("legacy-client"));
|
||||
parse_config(&document.to_string()).expect("providers use defaults when omitted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -282,13 +265,53 @@ fn config_loader_should_reject_embedded_database_password() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_non_hex_postgres_password() {
|
||||
assert_rejected(valid_config().replace(POSTGRES_PASSWORD, &"g".repeat(48)));
|
||||
fn config_loader_should_accept_managed_store_passwords() {
|
||||
for password in [
|
||||
"RdsTest9!abcdefGh".to_owned(),
|
||||
"RdsTest9!abcdefGhRdsTest9!abcdefGh".to_owned(),
|
||||
POSTGRES_PASSWORD.to_owned(),
|
||||
"x".repeat(128),
|
||||
"Managed@:/?#%&+Password9!".to_owned(),
|
||||
"数据库密码".repeat(4),
|
||||
] {
|
||||
for field in ["database", "redis"] {
|
||||
let mut document = valid_config_document();
|
||||
document["store"][field]["password"] = serde_json::json!(password);
|
||||
let (config, _directory) =
|
||||
parse_config(&document.to_string()).expect("managed store password");
|
||||
assert!(!format!("{config:?}").contains(&password));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_wrong_length_redis_password() {
|
||||
assert_rejected(valid_config().replace(REDIS_PASSWORD, "1234"));
|
||||
fn config_loader_should_reject_invalid_store_passwords() {
|
||||
for password in [
|
||||
String::new(),
|
||||
"x".repeat(15),
|
||||
"x".repeat(129),
|
||||
" ".repeat(32),
|
||||
"\u{3000}".repeat(6),
|
||||
"数据库密码".repeat(9),
|
||||
"RdsTest9!abcdefGh\u{0000}".to_owned(),
|
||||
"RdsTest9!abcdefGh\n".to_owned(),
|
||||
"RdsTest9!abcdefGh\t".to_owned(),
|
||||
"RdsTest9!abcdefGh\u{007f}".to_owned(),
|
||||
] {
|
||||
for field in ["database", "redis"] {
|
||||
let mut document = valid_config_document();
|
||||
document["store"][field]["password"] = serde_json::json!(password);
|
||||
assert_rejected(document.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_embedded_redis_password() {
|
||||
assert_rejected(valid_config().replace(
|
||||
"redis://127.0.0.1:6379/",
|
||||
"redis://default:embedded@127.0.0.1:6379/",
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -706,6 +706,10 @@ impl BackupTask {
|
||||
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
|
||||
match error.code() {
|
||||
code::CANCELLED => (code::CANCELLED, "备份已取消"),
|
||||
code::ANALYTICS_NOT_READY => (
|
||||
code::ANALYTICS_NOT_READY,
|
||||
"用量统计正在重建,请在统计就绪后重试备份",
|
||||
),
|
||||
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
|
||||
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ pub const STATE_CONFLICT: &str = "backup.state_conflict";
|
||||
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
|
||||
/// `pg_dump` 非零退出或失败
|
||||
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
|
||||
/// 用量汇总尚未完整发布
|
||||
pub const ANALYTICS_NOT_READY: &str = "backup.analytics_not_ready";
|
||||
/// 暂存磁盘空间不足
|
||||
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
|
||||
/// S3 认证失败
|
||||
|
||||
@@ -499,7 +499,7 @@ impl BackupError {
|
||||
/// 构造稳定错误码常量
|
||||
pub mod code;
|
||||
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.sql.gz`
|
||||
///
|
||||
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
|
||||
/// 输出统一使用 `/` 分隔且不含空段
|
||||
@@ -510,7 +510,7 @@ pub fn build_object_key(
|
||||
) -> Result<String, BackupError> {
|
||||
let prefix = normalize_prefix(prefix)?;
|
||||
Ok(format!(
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.sql.gz",
|
||||
at.format("%Y"),
|
||||
at.format("%m"),
|
||||
at.format("%d"),
|
||||
@@ -519,10 +519,10 @@ pub fn build_object_key(
|
||||
))
|
||||
}
|
||||
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.sql.gz`,与对象 key 的文件名一致
|
||||
#[must_use]
|
||||
pub fn build_download_file_name(backup_id: &str) -> String {
|
||||
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
|
||||
format!("codex-proxy-rs_{}.sql.gz", short_backup_id(backup_id))
|
||||
}
|
||||
|
||||
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex
|
||||
|
||||
@@ -722,10 +722,6 @@ pub struct UsageListRecord {
|
||||
pub upstream_response_ms: Option<u64>,
|
||||
pub upstream_api_overhead_ms: Option<f64>,
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub admission_decision_ms: Option<u64>,
|
||||
pub account_selection_wait_ms: Option<u64>,
|
||||
@@ -803,10 +799,6 @@ pub struct UsageRecord {
|
||||
pub upstream_response_ms: Option<u64>,
|
||||
pub upstream_api_overhead_ms: Option<f64>,
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
pub admission_decision_ms: Option<u64>,
|
||||
pub account_selection_wait_ms: Option<u64>,
|
||||
|
||||
@@ -5,20 +5,16 @@ use std::str::FromStr as _;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_core::account::OpaqueProviderData;
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::AdminApiKey,
|
||||
use gateway_admin::model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
ProviderDocument, ProviderQuota, ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore},
|
||||
settings::AdminApiKey,
|
||||
};
|
||||
|
||||
mod backup;
|
||||
@@ -99,20 +95,6 @@ fn login_command_debug_should_redact_password() {
|
||||
assert!(!format!("{command:?}").contains("secret-password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_store_capability_traits_should_be_object_safe() {
|
||||
fn assert_object_safe<T: ?Sized + Send + Sync>() {}
|
||||
|
||||
assert_object_safe::<dyn AccountStore>();
|
||||
assert_object_safe::<dyn AuthStore>();
|
||||
assert_object_safe::<dyn ClientKeyStore>();
|
||||
assert_object_safe::<dyn ObservabilityStore>();
|
||||
assert_object_safe::<dyn SettingsStore>();
|
||||
|
||||
fn assert_send_object_safe<T: ?Sized + Send>() {}
|
||||
assert_send_object_safe::<dyn CredentialCommitGuard>();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_document_debug_should_not_expose_opaque_material() {
|
||||
let document = ProviderDocument::new(OpaqueProviderData::new(Default::default()));
|
||||
|
||||
@@ -58,7 +58,7 @@ fn object_key_shape() {
|
||||
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
|
||||
assert_eq!(
|
||||
key,
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.sql.gz"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1390,10 +1390,6 @@ fn total_record(
|
||||
upstream_response_ms: None,
|
||||
upstream_api_overhead_ms: None,
|
||||
upstream_engine_ms: None,
|
||||
upstream_engine_iapi_ttft_ms: None,
|
||||
upstream_engine_service_ttft_ms: None,
|
||||
upstream_engine_iapi_tbt_ms: None,
|
||||
upstream_engine_service_tbt_ms: None,
|
||||
latency_ms: Some(100),
|
||||
admission_decision_ms: None,
|
||||
account_selection_wait_ms: None,
|
||||
|
||||
@@ -21,7 +21,7 @@ fn counters_preserve_integer_precision_and_reject_json_numbers() {
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_absent() {
|
||||
async fn usage_mutations_require_a_trusted_actor() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let denied = registry
|
||||
@@ -38,21 +38,6 @@ async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_a
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let ops = registry
|
||||
.operations()
|
||||
.filter(|op| op.starts_with("usage.") || *op == "pricing.evaluate")
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
ops,
|
||||
std::collections::BTreeSet::from([
|
||||
"usage.records.get",
|
||||
"usage.records.list",
|
||||
"usage.records.write",
|
||||
"usage.aggregates.list",
|
||||
"usage.aggregates.write",
|
||||
"pricing.evaluate"
|
||||
])
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn sdk_wire_types_match_host_contracts() {
|
||||
|
||||
@@ -304,10 +304,6 @@ pub(crate) fn usage_list_record_view(
|
||||
upstream_response_ms: record.upstream_response_ms,
|
||||
upstream_api_overhead_ms: record.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: record.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: record.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: record.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: record.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: record.upstream_engine_service_tbt_ms,
|
||||
},
|
||||
first_token_latency_ms: record.first_token_ms,
|
||||
latency_ms: record.latency_ms,
|
||||
@@ -411,10 +407,6 @@ pub(crate) fn usage_record_view(
|
||||
upstream_response_ms: record.upstream_response_ms,
|
||||
upstream_api_overhead_ms: record.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: record.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: record.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: record.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: record.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: record.upstream_engine_service_tbt_ms,
|
||||
},
|
||||
attempt_index: None,
|
||||
attempt_count: u64::from(record.attempt_count),
|
||||
|
||||
@@ -230,14 +230,6 @@ pub struct UsageLatencyDetailsView {
|
||||
pub upstream_api_overhead_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// 单次上游尝试展示
|
||||
|
||||
@@ -213,14 +213,6 @@ struct OutputTimingView {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
upstream_engine_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
upstream_engine_service_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
upstream_engine_service_tbt_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
first_event_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
first_reasoning_ms: Option<u64>,
|
||||
@@ -260,10 +252,6 @@ fn usage_record(value: UsageListRecord, time: crate::time::TimePresenter) -> Rec
|
||||
upstream_response_ms: value.upstream_response_ms,
|
||||
upstream_api_overhead_ms: value.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: value.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: value.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: value.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: value.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: value.upstream_engine_service_tbt_ms,
|
||||
first_event_ms: value.first_event_ms,
|
||||
first_reasoning_ms: value.first_reasoning_ms,
|
||||
first_text_ms: value.first_text_ms,
|
||||
|
||||
@@ -58,7 +58,8 @@ fn decode_response_create_inner(
|
||||
else {
|
||||
return Err(ResponseCreateFrameError::ExpectedObject);
|
||||
};
|
||||
match body.remove("type") {
|
||||
// 保持剩余字段顺序,避免末尾元数据交换到模型等路由字段之前
|
||||
match body.shift_remove("type") {
|
||||
Some(Value::String(message_type)) if message_type == "response.create" => {}
|
||||
_ => return Err(ResponseCreateFrameError::UnsupportedType),
|
||||
}
|
||||
|
||||
@@ -46,7 +46,6 @@ async fn list_route_should_keep_camel_case_group_and_page_wire() {
|
||||
value["data"]["items"][0]["usage"]["retainedTotalUsd"],
|
||||
"5.5"
|
||||
);
|
||||
assert!(value["data"]["items"][0].get("member_count").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -149,7 +149,7 @@ fn account_page_item(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_usage_returns_distinct_cache_read_and_write_tokens_without_aliases() {
|
||||
async fn account_usage_preserves_cache_read_and_write_counts_and_missing_values() {
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::accounts::AccountUsage;
|
||||
use gateway_core::account::{AccountStatusFacts, CredentialState, QuotaState};
|
||||
@@ -215,37 +215,9 @@ async fn account_usage_returns_distinct_cache_read_and_write_tokens_without_alia
|
||||
usage["cacheWriteInputTokensDisplay"],
|
||||
if write.is_some() { "7" } else { "—" }
|
||||
);
|
||||
for alias in [
|
||||
"createdTokens",
|
||||
"createdTokensDisplay",
|
||||
"readTokens",
|
||||
"readTokensDisplay",
|
||||
] {
|
||||
assert!(usage.get(alias).is_none(), "obsolete token alias: {alias}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_credential_rotation_route_is_not_exposed() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = admin::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/accounts/rotate")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_update_requires_admin_and_validates_before_calling_the_service() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
|
||||
@@ -248,9 +248,8 @@ mod profile_statistics {
|
||||
value["activityInsights"]["invocations"][0]["usageCount"],
|
||||
29
|
||||
);
|
||||
assert!(value.get("cycle").is_none());
|
||||
assert!(value.get("models").is_none());
|
||||
assert!(value.get("estimatedCost").is_none());
|
||||
assert_eq!(value["activityInsights"]["fastModePercent"], 0.0);
|
||||
assert!(value["activityInsights"]["invocations"][0]["skillId"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -375,7 +374,7 @@ mod batch_update {
|
||||
assert!(request.concurrency_limit.is_none());
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"], "enabled": true, "legacy": true
|
||||
"accountIds": ["acct_test"], "enabled": true, "unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
@@ -583,12 +582,12 @@ mod actions {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_actions_should_require_frozen_account_ids_and_reject_unknown_revision() {
|
||||
fn account_actions_should_validate_account_ids_and_reject_unknown_fields() {
|
||||
let id: AccountIdQuery =
|
||||
serde_json::from_value(json!({ "accountId": "acct_1" })).expect("decode ID query");
|
||||
assert!(id.validate().is_ok());
|
||||
let action: AccountActionRequest =
|
||||
serde_json::from_value(json!({ "accountId": "legacy-id" })).expect("decode action");
|
||||
serde_json::from_value(json!({ "accountId": "invalid-id" })).expect("decode action");
|
||||
assert_eq!(action.validate().unwrap_err().field(), "accountId");
|
||||
let refresh: AccountRefreshRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1"
|
||||
@@ -598,7 +597,7 @@ mod actions {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountRefreshRequest>(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
"unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
@@ -772,15 +771,14 @@ mod actions {
|
||||
assert_eq!(invalid.validate().unwrap_err().field(), "data");
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"document": {}
|
||||
"provider": "openai"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"unknownField": true,
|
||||
"data": {}
|
||||
}))
|
||||
.is_err()
|
||||
|
||||
@@ -11,7 +11,7 @@ use gateway_api::admin::accounts::{
|
||||
use gateway_core::operation::RawJsonPayload;
|
||||
|
||||
#[test]
|
||||
fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
fn quota_forecast_projection_formats_capacity_and_preserves_null_zero() {
|
||||
let now = "2026-09-12T00:00:00Z".parse().unwrap();
|
||||
let forecast = AccountQuotaForecast {
|
||||
period: AccountUsagePeriod::Weekly,
|
||||
@@ -54,16 +54,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
assert_eq!(week["estimatedTokensDisplay"], "1M");
|
||||
assert!(week["estimatedUsd"].is_null());
|
||||
assert_eq!(week["estimatedUsdDisplay"], "—");
|
||||
for forecast in value["forecasts"].as_array().unwrap() {
|
||||
for field in [
|
||||
"remainingTokens",
|
||||
"remainingTokensDisplay",
|
||||
"remainingUsd",
|
||||
"remainingUsdDisplay",
|
||||
] {
|
||||
assert!(forecast.get(field).is_none(), "unexpected field: {field}");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
week["source"],
|
||||
serde_json::json!({
|
||||
@@ -77,9 +67,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
"usdDisplay": "$0.1234"
|
||||
})
|
||||
);
|
||||
assert!(week.get("method").is_none());
|
||||
assert!(week.get("methodDisplay").is_none());
|
||||
assert!(value.get("generatedAtDisplay").is_none());
|
||||
assert_eq!(value["forecasts"][1]["estimatedTokens"], 0);
|
||||
assert_eq!(value["forecasts"][1]["estimatedTokensDisplay"], "0");
|
||||
assert_eq!(value["forecasts"][1]["estimatedUsd"], 0.0);
|
||||
@@ -87,7 +74,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
assert_eq!(value["forecasts"][1]["period"], "monthly");
|
||||
assert_eq!(value["forecasts"][1]["targetDays"], 30.0);
|
||||
assert_eq!(value["forecasts"][1]["extrapolated"], true);
|
||||
assert!(value.get("account").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -375,18 +375,17 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<UpdateClientKeyRequest>(json!({
|
||||
"id": "key_1",
|
||||
"expectedConfigRevision": 1,
|
||||
"name": "key",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0,
|
||||
"tokensPerMinute": 0
|
||||
"unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"unknownField": true,
|
||||
"name": "terminal key",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 2,
|
||||
@@ -458,7 +457,6 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
));
|
||||
let list =
|
||||
serde_json::to_value(ClientKeyListData::new(vec![view], None, 1)).expect("serialize list");
|
||||
assert!(list.get("configRevision").is_none());
|
||||
assert_eq!(list["total"], 1);
|
||||
assert_eq!(list["items"][0]["id"], "key_visible");
|
||||
assert_eq!(
|
||||
@@ -468,12 +466,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
assert_eq!(list["items"][0]["routingScope"], "all");
|
||||
assert_eq!(list["items"][0]["maxConcurrency"], 2);
|
||||
assert_eq!(list["items"][0]["requestsPerMinute"], 60);
|
||||
assert!(list["items"][0].get("tokensPerMinute").is_none());
|
||||
assert!(list["items"][0].get("policyJson").is_none());
|
||||
assert_eq!(list["items"][0]["dailyLimitUsd"], "0");
|
||||
assert_eq!(list["items"][0]["weeklyLimitUsd"], "0");
|
||||
assert_eq!(list["items"][0]["dailyUsedUsd"], "0");
|
||||
assert!(list["items"][0].get("unresolvedRequests").is_none());
|
||||
assert_eq!(
|
||||
DateTime::parse_from_rfc3339(
|
||||
list["items"][0]["lastUsedAt"]
|
||||
|
||||
@@ -7,7 +7,7 @@ use gateway_api::admin::observability::{
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
fn dashboard_query_should_parse_error_trends() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
@@ -19,7 +19,7 @@ fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_accept_element_plus_pagination() {
|
||||
fn usage_query_should_accept_page_number_and_size() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"currentPage": 129,
|
||||
"pageSize": 10
|
||||
@@ -29,32 +29,20 @@ fn usage_query_should_accept_element_plus_pagination() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_removed_total_toggle() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"includeTotal": true})).is_err());
|
||||
fn observability_queries_should_reject_unknown_fields() {
|
||||
let query = json!({"unknownFilter": true});
|
||||
assert!(serde_json::from_value::<DashboardQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<DiagnosticsQuery>(query).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_removed_cursor_contract() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"cursor": "opaque"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_nonstandard_page_name() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"page": 1})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
fn ops_query_should_reject_page_size_above_maximum() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_removed_filters() {
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"route": "/v1/responses"})).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"failureClass": "rate_limited"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 响应 DTO 与固定 wire 形状的合同测试
|
||||
//! 验证观测响应的指标、展示投影与查询时间边界
|
||||
|
||||
use chrono::Utc;
|
||||
use serde_json::json;
|
||||
@@ -288,10 +288,6 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
record.upstream_response_ms = Some(1_000);
|
||||
record.upstream_api_overhead_ms = Some(120.25);
|
||||
record.upstream_engine_ms = Some(6400.0);
|
||||
record.upstream_engine_iapi_ttft_ms = Some(650.5);
|
||||
record.upstream_engine_service_ttft_ms = Some(720.25);
|
||||
record.upstream_engine_iapi_tbt_ms = Some(18.45);
|
||||
record.upstream_engine_service_tbt_ms = Some(20.12);
|
||||
fixture
|
||||
.usage_detail
|
||||
.lock()
|
||||
@@ -362,29 +358,6 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
value["data"]["latencyDetails"]["upstreamEngineMs"],
|
||||
json!(6400.0)
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["latencyDetails"]["upstreamEngineIapiTtftMs"],
|
||||
json!(650.5)
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["latencyDetails"]["upstreamEngineServiceTtftMs"],
|
||||
json!(720.25)
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["latencyDetails"]["upstreamEngineIapiTbtMs"],
|
||||
json!(18.45)
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["latencyDetails"]["upstreamEngineServiceTbtMs"],
|
||||
json!(20.12)
|
||||
);
|
||||
assert!(value["data"].get("firstTokenMs").is_none());
|
||||
assert!(value["data"].get("firstTokenLatencyMsDisplay").is_none());
|
||||
assert!(
|
||||
value["data"]["latencyDetails"]
|
||||
.get("firstTokenMs")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"accountId": value["data"]["accountId"],
|
||||
@@ -490,8 +463,6 @@ async fn zero_attempt_failure_detail_omits_trace_and_keeps_missing_upstream_fact
|
||||
assert_eq!(data["clientStatusCode"], 503);
|
||||
assert_eq!(data["message"], "no upstream account is available");
|
||||
assert_eq!(data["model"], "coding");
|
||||
assert!(data.get("upstreamStatusCode").is_none());
|
||||
assert!(data.get("httpVersion").is_none());
|
||||
for field in [
|
||||
"provider",
|
||||
"accountId",
|
||||
@@ -642,7 +613,6 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert!(value["data"]["items"][0].get("impactScore").is_none());
|
||||
assert_eq!(value["data"]["items"][0]["requestShare"], 0.5);
|
||||
assert_eq!(value["data"]["items"][0]["retryCount"], 3);
|
||||
assert_eq!(value["data"]["items"][0]["retryRate"], 0.5);
|
||||
@@ -734,10 +704,6 @@ pub(super) fn usage_record_with_account(
|
||||
upstream_response_ms: None,
|
||||
upstream_api_overhead_ms: None,
|
||||
upstream_engine_ms: None,
|
||||
upstream_engine_iapi_ttft_ms: None,
|
||||
upstream_engine_service_ttft_ms: None,
|
||||
upstream_engine_iapi_tbt_ms: None,
|
||||
upstream_engine_service_tbt_ms: None,
|
||||
latency_ms: None,
|
||||
admission_decision_ms: None,
|
||||
account_selection_wait_ms: None,
|
||||
@@ -877,10 +843,6 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
upstream_response_ms: Some(2_000),
|
||||
upstream_api_overhead_ms: None,
|
||||
upstream_engine_ms: None,
|
||||
upstream_engine_iapi_ttft_ms: None,
|
||||
upstream_engine_service_ttft_ms: None,
|
||||
upstream_engine_iapi_tbt_ms: None,
|
||||
upstream_engine_service_tbt_ms: None,
|
||||
latency_ms: Some(31),
|
||||
admission_decision_ms: Some(1),
|
||||
account_selection_wait_ms: Some(2),
|
||||
|
||||
@@ -35,7 +35,7 @@ use std::{collections::BTreeMap, sync::Arc};
|
||||
mod management;
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
fn artifact_metadata_wire_preserves_identity_icons_and_contributions() {
|
||||
use gateway_admin::model::plugins::{
|
||||
PluginArtifactIcon, PluginArtifactIconVariants, PluginArtifactMetadata, PluginContribution,
|
||||
};
|
||||
@@ -70,7 +70,7 @@ fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
secret_fields: vec![],
|
||||
state_namespaces: vec![],
|
||||
};
|
||||
let value = serde_json::to_value(metadata).expect("v2 artifact metadata JSON");
|
||||
let value = serde_json::to_value(metadata).expect("artifact metadata JSON");
|
||||
assert_eq!(value["displayName"], "Example");
|
||||
assert_eq!(value["publisher"], "test");
|
||||
assert!(value["author"].is_null());
|
||||
@@ -80,33 +80,29 @@ fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
value["contributes"]["middleware"]["id"],
|
||||
"test.example.middleware"
|
||||
);
|
||||
assert!(value.get("capabilities").is_none());
|
||||
assert!(value.get("permissionDescriptions").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn instance_binding_wire_uses_contribution_without_capability_alias() {
|
||||
fn instance_binding_requires_a_contribution_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::plugins::instances::PluginCapabilityBinding;
|
||||
|
||||
let binding = serde_json::from_value::<PluginCapabilityBinding>(serde_json::json!({
|
||||
let input = serde_json::json!({
|
||||
"contribution": "test.example.usage",
|
||||
"stage": "observation",
|
||||
"order": 0,
|
||||
"failurePolicy": "observe"
|
||||
}))
|
||||
.expect("v2 contribution binding");
|
||||
});
|
||||
let binding = serde_json::from_value::<PluginCapabilityBinding>(input.clone())
|
||||
.expect("contribution binding");
|
||||
assert_eq!(binding.contribution, "test.example.usage");
|
||||
|
||||
assert!(
|
||||
serde_json::from_value::<PluginCapabilityBinding>(serde_json::json!({
|
||||
"capability": "usage",
|
||||
"stage": "observation",
|
||||
"order": 0,
|
||||
"failurePolicy": "observe"
|
||||
}))
|
||||
.is_err(),
|
||||
"legacy capability binding must be rejected"
|
||||
);
|
||||
let mut missing = input.clone();
|
||||
missing.as_object_mut().unwrap().remove("contribution");
|
||||
assert!(serde_json::from_value::<PluginCapabilityBinding>(missing).is_err());
|
||||
|
||||
let mut unknown = input;
|
||||
unknown["unknownField"] = serde_json::json!(true);
|
||||
assert!(serde_json::from_value::<PluginCapabilityBinding>(unknown).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -381,7 +377,7 @@ async fn plugin_upload_endpoints_require_admin_and_bound_raw_bodies() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_acceptance_returns_the_flat_install_contract() {
|
||||
async fn artifact_acceptance_returns_installed_artifact_and_configuration_status() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
@@ -418,22 +414,11 @@ async fn artifact_acceptance_returns_the_flat_install_contract() {
|
||||
assert_eq!(result["defaultInstanceId"], serde_json::Value::Null);
|
||||
assert_eq!(result["configurationRequired"], false);
|
||||
assert_eq!(result["artifact"]["metadata"]["sha256"], "a".repeat(64));
|
||||
assert!(
|
||||
result["artifact"]["metadata"]
|
||||
.get("permissionDescriptions")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
result["artifact"]["metadata"]
|
||||
.get("requestedPermissions")
|
||||
.is_none()
|
||||
);
|
||||
assert!(result["artifact"]["acceptedAt"].is_string());
|
||||
assert!(result.get("mutation").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_instance_policy_fields_and_extra_accept_fields_are_rejected() {
|
||||
async fn plugin_mutations_reject_unknown_fields() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
@@ -442,24 +427,25 @@ async fn removed_instance_policy_fields_and_extra_accept_fields_are_rejected() {
|
||||
|
||||
let fixture = super::AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (path, body) in [
|
||||
(
|
||||
"/api/admin/plugins/instances",
|
||||
serde_json::json!({
|
||||
"name":"Example",
|
||||
"artifactSha256":"a".repeat(64),
|
||||
"enabled":false,
|
||||
"trustedProcess":true,
|
||||
"configuration":{},
|
||||
"grants":[],
|
||||
"bindings":[]
|
||||
}),
|
||||
),
|
||||
let instance = serde_json::json!({
|
||||
"name":"Example",
|
||||
"artifactSha256":"a".repeat(64),
|
||||
"enabled":false,
|
||||
"configuration":{},
|
||||
"bindings":[]
|
||||
});
|
||||
serde_json::from_value::<gateway_admin::model::plugins::instances::ConfigurePluginInstance>(
|
||||
instance.clone(),
|
||||
)
|
||||
.expect("valid instance request");
|
||||
for (path, mut body) in [
|
||||
("/api/admin/plugins/instances", instance),
|
||||
(
|
||||
"/api/admin/plugins/artifacts/accept",
|
||||
serde_json::json!({"sha256":"a".repeat(64),"trusted":true}),
|
||||
serde_json::json!({"sha256":"a".repeat(64)}),
|
||||
),
|
||||
] {
|
||||
body["unknownField"] = serde_json::json!(true);
|
||||
let response = gateway_api::admin::router::<super::AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
|
||||
@@ -181,7 +181,7 @@ fn privacy_policy_is_required_and_rejects_unknown_contract_fields() {
|
||||
.remove("codexPrivacyPolicy");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(missing).is_err());
|
||||
let mut unknown = update_body();
|
||||
unknown["codexPrivacyPolicy"]["protectedFields"] = json!([]);
|
||||
unknown["codexPrivacyPolicy"]["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(unknown).is_err());
|
||||
}
|
||||
|
||||
@@ -471,110 +471,6 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::account::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(update_body()).expect("decode settings");
|
||||
request.validate().expect("fixture settings must validate");
|
||||
|
||||
let request_fields: BTreeSet<String> = update_body()
|
||||
.as_object()
|
||||
.expect("request body object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
.iter()
|
||||
.map(|(public, upstream)| {
|
||||
Ok((
|
||||
PublicModelId::new(public.clone())?,
|
||||
UpstreamModelId::new(upstream.clone())?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
|
||||
.expect("valid model mappings"),
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
updated_at: chrono::Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: request.values.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.values.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.values.max_concurrent_per_account)
|
||||
.expect("u32"),
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: request.values.smart_scheduling,
|
||||
min_codex_desktop_version: request.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: request.values.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(request.values.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.values.ops_event_retention_days)
|
||||
.expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.values.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
};
|
||||
|
||||
let response_fields: BTreeSet<String> = serde_json::to_value(RuntimeSettingsView::from((
|
||||
settings,
|
||||
gateway_api::TimePresenter::new(Default::default()),
|
||||
)))
|
||||
.expect("serialize view")
|
||||
.as_object()
|
||||
.expect("view object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut expected_fields = request_fields;
|
||||
expected_fields.insert("providerRequestProfiles".to_owned());
|
||||
expected_fields.insert("openaiClientProfile".to_owned());
|
||||
expected_fields.insert("xaiClientProfile".to_owned());
|
||||
expected_fields.insert("updatedAt".to_owned());
|
||||
expected_fields.insert("updatedAtDisplay".to_owned());
|
||||
expected_fields.insert("smartSchedulingDefaults".to_owned());
|
||||
|
||||
assert_eq!(response_fields, expected_fields);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_accepts_json_bytes_with_flattened_fields() {
|
||||
serde_json::from_slice::<UpdateRuntimeSettingsRequest>(
|
||||
&serde_json::to_vec(&update_body()).unwrap(),
|
||||
)
|
||||
.expect("settings JSON bytes must decode like the HTTP request body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::settings::{ReplaceRuntimeSettings, RuntimeSettings};
|
||||
@@ -610,17 +506,9 @@ fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
fn settings_request_should_reject_unknown_fields() {
|
||||
let mut body = update_body();
|
||||
body["expectedConfigRevision"] = json!(7);
|
||||
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_removed_bucket_retention() {
|
||||
let mut body = update_body();
|
||||
body["bucketRetentionDays"] = json!(365);
|
||||
body["unknownSetting"] = json!(true);
|
||||
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
@@ -960,36 +848,23 @@ fn decompression_setting_should_reject_invalid_values() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn settings_reject_removed_global_fast_policy() {
|
||||
async fn settings_update_rejects_unknown_fields_without_writing() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let app = app(fixture.state());
|
||||
let initial = fixture.settings.settings.lock().unwrap().clone();
|
||||
for value in [json!(true), json!(false), Value::Null] {
|
||||
let mut body = update_body();
|
||||
body["disableFast"] = value;
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(*fixture.settings.settings.lock().unwrap(), initial);
|
||||
}
|
||||
let mut body = update_body();
|
||||
body["unknownSetting"] = json!(true);
|
||||
let response = app
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(
|
||||
response_json(response).await["data"]
|
||||
.get("disableFast")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(*fixture.settings.settings.lock().unwrap(), initial);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -86,7 +86,7 @@ async fn update_event_stream_should_preserve_event_id_and_download_progress_perc
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_return_accepted_and_report_restart_only_through_status() {
|
||||
async fn update_should_return_accepted_and_expose_restart_status() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = app(fixture.state());
|
||||
@@ -110,7 +110,6 @@ async fn update_should_return_accepted_and_report_restart_only_through_status()
|
||||
.expect("JSON");
|
||||
assert_eq!(body["data"]["operationId"], "update-1");
|
||||
assert_eq!(body["data"]["targetVersion"], "0.2.0");
|
||||
assert!(body["data"].get("needRestart").is_none());
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
|
||||
@@ -297,15 +297,15 @@ async fn unified_login_returns_server_identity_and_rotates_the_previous_session(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn login_rejects_legacy_type_without_replacing_the_current_session() {
|
||||
async fn login_rejects_missing_mode_and_unknown_fields_without_replacing_the_session() {
|
||||
let (app, _) = auth_app().await;
|
||||
let cookie =
|
||||
session_cookie(&login(&app, json!({"mode": "key", "apiKey": RAW_KEY}), None).await);
|
||||
for body in [
|
||||
json!({"type": "key", "apiKey": RAW_KEY}),
|
||||
json!({"type": "admin", "password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "type": "key", "apiKey": RAW_KEY}),
|
||||
json!({"mode": "admin", "type": "admin", "password": "strong-admin-password"}),
|
||||
json!({"apiKey": RAW_KEY}),
|
||||
json!({"password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "unknownField": true, "apiKey": RAW_KEY}),
|
||||
json!({"mode": "admin", "unknownField": true, "password": "strong-admin-password"}),
|
||||
] {
|
||||
let response = login(&app, body, Some(&cookie)).await;
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
@@ -458,26 +458,26 @@ async fn key_store_outage_is_unavailable_not_expired_or_authenticated() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_auth_routes_do_not_fall_back_to_spa_or_accept_old_cookies() {
|
||||
async fn unknown_auth_routes_return_json_not_found() {
|
||||
let (app, _) = auth_app().await;
|
||||
let response = app
|
||||
.oneshot(empty_request(Method::GET, "/api/auth/unknown"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response_json(response).await["code"], 40401);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_authentication_requires_the_configured_cookie_name() {
|
||||
let (app, _) = auth_app().await;
|
||||
for path in ["/api/admin/auth/login", "/api/auth/unknown"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(empty_request(Method::GET, path))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response_json(response).await["code"], 40401);
|
||||
}
|
||||
let cookie =
|
||||
session_cookie(&login(&app, json!({"mode": "key", "apiKey": RAW_KEY}), None).await);
|
||||
for old_name in ["cpr_admin_session", "cpr_client_session"] {
|
||||
let old_cookie = cookie.replace("cpr_session", old_name);
|
||||
assert_eq!(
|
||||
response_json(get(&app, "/api/auth/status", &old_cookie).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
}
|
||||
let unrelated_cookie = cookie.replace("cpr_session", "unrelated_session");
|
||||
assert_eq!(
|
||||
response_json(get(&app, "/api/auth/status", &unrelated_cookie).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -140,11 +140,7 @@ fn usage_record() -> UsageListRecord {
|
||||
provider_processing_ms: None,
|
||||
upstream_response_ms: Some(1_000),
|
||||
upstream_api_overhead_ms: None,
|
||||
upstream_engine_ms: None,
|
||||
upstream_engine_iapi_ttft_ms: None,
|
||||
upstream_engine_service_ttft_ms: None,
|
||||
upstream_engine_iapi_tbt_ms: Some(2.450638),
|
||||
upstream_engine_service_tbt_ms: None,
|
||||
upstream_engine_ms: Some(2.450638),
|
||||
latency_ms: Some(1250),
|
||||
admission_decision_ms: None,
|
||||
account_selection_wait_ms: None,
|
||||
|
||||
@@ -320,7 +320,7 @@ async fn records_keep_pagination_and_hide_admin_and_upstream_data() {
|
||||
&record["latencyDetails"],
|
||||
&[
|
||||
"upstreamResponseMs",
|
||||
"upstreamEngineIapiTbtMs",
|
||||
"upstreamEngineMs",
|
||||
"firstEventMs",
|
||||
"firstReasoningMs",
|
||||
"firstTextMs",
|
||||
@@ -328,10 +328,7 @@ async fn records_keep_pagination_and_hide_admin_and_upstream_data() {
|
||||
);
|
||||
assert_eq!(record["latencyDetails"]["firstEventMs"], 100);
|
||||
assert_eq!(record["latencyDetails"]["upstreamResponseMs"], 1000);
|
||||
assert_eq!(
|
||||
record["latencyDetails"]["upstreamEngineIapiTbtMs"],
|
||||
2.450638
|
||||
);
|
||||
assert_eq!(record["latencyDetails"]["upstreamEngineMs"], 2.450638);
|
||||
assert!(record["statusCode"].is_null());
|
||||
} else {
|
||||
assert_eq!(record["statusCode"], 502);
|
||||
|
||||
@@ -799,7 +799,7 @@ async fn websocket_failure_before_first_event_should_return_observed_metadata_th
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
async fn large_websocket_response_create_frames_should_reach_execution() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
@@ -829,15 +829,14 @@ async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit(
|
||||
let payload = json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "x".repeat(16 * 1024 * 1024 + 1),
|
||||
"input": "x".repeat(17 * 1024 * 1024),
|
||||
})
|
||||
.to_string();
|
||||
assert!(payload.len() > 16 * 1024 * 1024);
|
||||
|
||||
socket
|
||||
.send(ClientMessage::Text(payload.into()))
|
||||
.await
|
||||
.expect("send response.create above the former frame limit");
|
||||
.expect("send large response.create frame");
|
||||
|
||||
let mut saw_upstream_event = false;
|
||||
for _ in 0..3 {
|
||||
|
||||
@@ -8,6 +8,26 @@ use serde_json::json;
|
||||
|
||||
use super::decode_response_create;
|
||||
|
||||
#[test]
|
||||
fn response_create_should_preserve_field_order_when_removing_frame_type() {
|
||||
let expected = r#"{"model":"smart-code","stream":true,"input":"hello","future_field":{"z":1,"a":2},"client_metadata":{"thread_id":"thread","turn_id":"turn"}}"#;
|
||||
for payload in [
|
||||
r#"{"type":"response.create","model":"smart-code","stream":true,"input":"hello","future_field":{"z":1,"a":2},"client_metadata":{"thread_id":"thread","turn_id":"turn"}}"#,
|
||||
r#"{"model":"smart-code","stream":true,"type":"response.create","input":"hello","future_field":{"z":1,"a":2},"client_metadata":{"thread_id":"thread","turn_id":"turn"}}"#,
|
||||
r#"{"model":"smart-code","stream":true,"input":"hello","future_field":{"z":1,"a":2},"client_metadata":{"thread_id":"thread","turn_id":"turn"},"type":"response.create"}"#,
|
||||
] {
|
||||
let decoded = decode_response_create(payload).expect("decode response.create");
|
||||
let body = decoded
|
||||
.operation()
|
||||
.middleware_body()
|
||||
.expect("encode transparent body");
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&body).expect("UTF-8 JSON body"),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_retry_identity_should_follow_each_frame_instead_of_handshake_metadata() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{Method, Request, StatusCode},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
const LARGE_BODY_BYTES: usize = 17 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_origin_controls_http_admin_sessions_without_configuration() {
|
||||
@@ -117,64 +117,16 @@ async fn browser_origin_controls_http_admin_sessions_without_configuration() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_responses_review_route_should_not_reach_the_responses_handler() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed review request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_models_catalog_extension_should_use_standard_model_lookup() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/catalog")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model catalog request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model catalog request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_model_info_route_should_return_not_found() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/model-a/info")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model info request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model info request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
async fn large_responses_bodies_should_reach_authentication() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; REMOVED_BODY_LIMIT_BYTES + 1]))
|
||||
.expect("build request above the removed limit"),
|
||||
.body(Body::from(vec![b'a'; LARGE_BODY_BYTES]))
|
||||
.expect("build large request"),
|
||||
)
|
||||
.await
|
||||
.expect("route request above the removed limit");
|
||||
.expect("route large request");
|
||||
|
||||
// 请求进入处理器后应因缺少凭据而被拒绝
|
||||
// 若提前施加正文大小限制,则会在认证前返回 413
|
||||
|
||||
@@ -18,6 +18,7 @@ struct State {
|
||||
pub struct ConnectionBudget {
|
||||
state: Arc<Mutex<State>>,
|
||||
now: fn() -> Instant,
|
||||
reserve_next_opening: bool,
|
||||
}
|
||||
|
||||
impl Default for ConnectionBudget {
|
||||
@@ -37,9 +38,26 @@ impl ConnectionBudget {
|
||||
Self {
|
||||
state: Arc::default(),
|
||||
now,
|
||||
reserve_next_opening: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅本次恢复使用一半剩余时间并留下一次 opening;已启动的后台连接不受影响
|
||||
#[must_use]
|
||||
pub fn reserve_next_opening(mut self) -> Self {
|
||||
self.reserve_next_opening = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn remaining_openings(&self) -> u32 {
|
||||
let state = self
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
MAX_OPENINGS.saturating_sub(state.openings)
|
||||
}
|
||||
|
||||
/// 在可能建立连接前预约一次 opening;返回本次连接允许使用的剩余时间
|
||||
/// HTTP 连接池隐藏了是否冷建连,因此一次 HTTP opening 也计入次数上限
|
||||
pub fn begin(&self) -> Result<Option<Duration>, ConnectionBudgetExhausted> {
|
||||
@@ -53,11 +71,16 @@ impl ConnectionBudget {
|
||||
let started = *state.started.get_or_insert_with(self.now);
|
||||
let remaining =
|
||||
STARTUP_TIMEOUT.saturating_sub((self.now)().saturating_duration_since(started));
|
||||
if state.openings >= MAX_OPENINGS || remaining.is_zero() {
|
||||
let opening_limit = MAX_OPENINGS - u32::from(self.reserve_next_opening);
|
||||
if state.openings >= opening_limit || remaining.is_zero() {
|
||||
return Err(ConnectionBudgetExhausted);
|
||||
}
|
||||
state.openings += 1;
|
||||
Ok(Some(remaining))
|
||||
Ok(Some(if self.reserve_next_opening {
|
||||
remaining / 2
|
||||
} else {
|
||||
remaining
|
||||
}))
|
||||
}
|
||||
|
||||
/// 仅连接成功不足以清零;确认业务发送后退出新增的 NotSent 恢复策略
|
||||
|
||||
@@ -318,6 +318,7 @@ where
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget::default(),
|
||||
connection_budget: super::connection::ConnectionBudget::default(),
|
||||
connection_retries: 0,
|
||||
connection_rotation: None,
|
||||
observation: ResponseObservation::new(timing_started_at),
|
||||
request_observation,
|
||||
budget_prior_attempts_usd: Decimal::ZERO,
|
||||
@@ -424,6 +425,7 @@ struct CurrentAttempt {
|
||||
metadata: ProviderCallMetadata,
|
||||
trigger: AttemptTrigger,
|
||||
transport: AttemptTransport,
|
||||
connection_rotation_reserved: bool,
|
||||
index: NonZeroU32,
|
||||
started_at: SystemTime,
|
||||
send_observed: bool,
|
||||
@@ -458,6 +460,7 @@ struct PendingAttemptRetry {
|
||||
transport: AttemptTransport,
|
||||
delay: Duration,
|
||||
transport_recovery: bool,
|
||||
reserve_connection_rotation: bool,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话
|
||||
@@ -471,6 +474,8 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget,
|
||||
connection_budget: super::connection::ConnectionBudget,
|
||||
connection_retries: u32,
|
||||
/// 原账号恢复仍建连失败后尝试重选;选号不可用时消费该状态,恢复原账号剩余预算
|
||||
connection_rotation: Option<PendingAttemptRetry>,
|
||||
observation: ResponseObservation,
|
||||
request_observation: Option<RequestObservationDispatch>,
|
||||
budget_prior_attempts_usd: Decimal,
|
||||
@@ -999,6 +1004,9 @@ where
|
||||
let is_transport_recovery = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.transport_recovery);
|
||||
let connection_rotation_reserved = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.reserve_connection_rotation);
|
||||
if !is_transport_recovery && self.routing_attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
@@ -1060,7 +1068,9 @@ where
|
||||
self.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
AttemptTransport::Default,
|
||||
self.connection_rotation
|
||||
.as_ref()
|
||||
.map_or(AttemptTransport::Default, |recovery| recovery.transport),
|
||||
),
|
||||
}
|
||||
};
|
||||
@@ -1123,7 +1133,11 @@ where
|
||||
.with_request_location(self.plan.request_location().cloned())
|
||||
.with_privacy(self.plan.privacy())
|
||||
.with_concurrency_wait_budget(self.concurrency_wait_budget.clone())
|
||||
.with_connection_budget(self.connection_budget.clone())
|
||||
.with_connection_budget(if connection_rotation_reserved {
|
||||
self.connection_budget.clone().reserve_next_opening()
|
||||
} else {
|
||||
self.connection_budget.clone()
|
||||
})
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_request_policy(self.request_policy.clone())
|
||||
.with_execution_effects(self.execution_effects.as_ref().map(Arc::clone))
|
||||
@@ -1186,9 +1200,21 @@ where
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::QuotaExhausted
|
||||
| ProviderErrorKind::ConcurrencyQueueFull
|
||||
| ProviderErrorKind::ConcurrencyQueueTimeout
|
||||
);
|
||||
let continuation_retry = !error.retry_is_prohibited()
|
||||
&& self.prepare_unavailable_native_continuation_replay(&error);
|
||||
let candidate_retry = !error.retry_is_prohibited() && !continuation_retry
|
||||
let candidate_retry = !error.retry_is_prohibited() && !connection_fallback && !continuation_retry
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -1224,12 +1250,15 @@ where
|
||||
.get()
|
||||
.saturating_sub(self.routing_attempts),
|
||||
remaining_deadline: Duration::ZERO,
|
||||
retry_allowed: continuation_retry || candidate_retry,
|
||||
retry_allowed: connection_fallback
|
||||
|| continuation_retry
|
||||
|| candidate_retry,
|
||||
})
|
||||
.await?;
|
||||
attempt_trace.record("retry.decided", json!({
|
||||
"phase": "prepare", "retryable": retryable,
|
||||
"continuationRetry": continuation_retry, "candidateRetry": candidate_retry,
|
||||
"connectionFallback": connection_fallback,
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sendState": error.send_state().as_str(),
|
||||
"requestSendState": self.current_send_state().as_str(),
|
||||
@@ -1239,6 +1268,17 @@ where
|
||||
"remainingRoutingAttempts": self.plan.max_attempts().get().saturating_sub(self.routing_attempts),
|
||||
}));
|
||||
if retryable {
|
||||
if connection_fallback {
|
||||
let recovery = self
|
||||
.connection_rotation
|
||||
.take()
|
||||
.expect("pending connection rotation");
|
||||
self.excluded_accounts.remove(&recovery.account);
|
||||
self.pending_retry = Some(recovery);
|
||||
} else if candidate_retry {
|
||||
// 原账号只属于原 Provider,跨候选后不能拿旧钉选作恢复兜底
|
||||
self.connection_rotation = None;
|
||||
}
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
@@ -1306,6 +1346,7 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
self.connection_rotation = None;
|
||||
attempt_trace.record("account.selected", json!({
|
||||
"provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(),
|
||||
"transport": metadata.transport().as_str(),
|
||||
@@ -1429,6 +1470,7 @@ where
|
||||
metadata,
|
||||
trigger,
|
||||
transport: attempt_transport,
|
||||
connection_rotation_reserved,
|
||||
index: next_attempt,
|
||||
started_at: SystemTime::now(),
|
||||
send_observed: false,
|
||||
@@ -1642,6 +1684,18 @@ where
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let can_rotate_connection = connection_retry_requested
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
|
||||
let connection_rotation = transport_recovery
|
||||
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
|
||||
let reserve_connection_rotation = can_rotate_connection
|
||||
&& !current.connection_rotation_reserved
|
||||
&& self.connection_budget.remaining_openings() >= 2;
|
||||
let transport_recovery = transport_recovery.filter(|_| connection_rotation.is_none());
|
||||
let ordinary_retry = !self.connection_budget.exhausted()
|
||||
&& !connection_retry_requested
|
||||
&& !execution_effect_observed
|
||||
@@ -1714,6 +1768,7 @@ where
|
||||
|| same_account_retry
|
||||
|| transient_retry.is_some()
|
||||
|| transport_recovery.is_some()
|
||||
|| connection_rotation.is_some()
|
||||
|| rotation_retry);
|
||||
|
||||
let retryable = match self
|
||||
@@ -1750,6 +1805,8 @@ where
|
||||
"retryable": retryable, "continuationRetry": continuation_retry,
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
@@ -1795,6 +1852,7 @@ where
|
||||
transport: current.transport,
|
||||
delay,
|
||||
transport_recovery: false,
|
||||
reserve_connection_rotation: false,
|
||||
});
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
self.pending_retry = Some(PendingAttemptRetry {
|
||||
@@ -1802,6 +1860,17 @@ where
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation,
|
||||
});
|
||||
} else if let Some((transport, delay)) = connection_rotation {
|
||||
let account = current.metadata.provider_account_id().clone();
|
||||
self.excluded_accounts.insert(account.clone());
|
||||
self.connection_rotation = Some(PendingAttemptRetry {
|
||||
account,
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation: false,
|
||||
});
|
||||
} else if !continuation_retry {
|
||||
self.excluded_accounts
|
||||
|
||||
@@ -955,10 +955,6 @@ pub struct ModelRequestTimings {
|
||||
pub upstream_response_ms: Option<u64>,
|
||||
pub upstream_api_overhead_ms: Option<f64>,
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
pub latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
|
||||
@@ -807,10 +807,6 @@ impl ResponseObservation {
|
||||
self.timings.upstream_response_ms = None;
|
||||
self.timings.upstream_api_overhead_ms = None;
|
||||
self.timings.upstream_engine_ms = None;
|
||||
self.timings.upstream_engine_iapi_ttft_ms = None;
|
||||
self.timings.upstream_engine_service_ttft_ms = None;
|
||||
self.timings.upstream_engine_iapi_tbt_ms = None;
|
||||
self.timings.upstream_engine_service_tbt_ms = None;
|
||||
}
|
||||
|
||||
pub(super) fn observe_response(&mut self, observation: &ProviderResponseObservation) {
|
||||
@@ -848,17 +844,5 @@ impl ResponseObservation {
|
||||
if let Some(value) = observed.upstream_engine_ms {
|
||||
self.timings.upstream_engine_ms = Some(value);
|
||||
}
|
||||
if let Some(value) = observed.upstream_engine_iapi_ttft_ms {
|
||||
self.timings.upstream_engine_iapi_ttft_ms = Some(value);
|
||||
}
|
||||
if let Some(value) = observed.upstream_engine_service_ttft_ms {
|
||||
self.timings.upstream_engine_service_ttft_ms = Some(value);
|
||||
}
|
||||
if let Some(value) = observed.upstream_engine_iapi_tbt_ms {
|
||||
self.timings.upstream_engine_iapi_tbt_ms = Some(value);
|
||||
}
|
||||
if let Some(value) = observed.upstream_engine_service_tbt_ms {
|
||||
self.timings.upstream_engine_service_tbt_ms = Some(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -536,12 +536,8 @@ pub struct ProviderResponseTimings {
|
||||
pub upstream_response_ms: Option<u64>,
|
||||
/// 上游排除引擎与客户端工具时间后的 API 耗时
|
||||
pub upstream_api_overhead_ms: Option<f64>,
|
||||
/// 上游响应级引擎耗时;以下专项指标保留毫秒小数及各自内部计量层级
|
||||
/// 上游响应级引擎耗时,保留毫秒小数
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
}
|
||||
|
||||
/// Provider 已筛选的响应观测 JSON
|
||||
|
||||
@@ -27,6 +27,40 @@ fn elapsed_recovery_window_is_shared_without_waiting_for_real_time() {
|
||||
assert_eq!(fallback.startup_remaining(), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_account_retry_preserves_time_and_an_opening_for_rotation() {
|
||||
use std::{
|
||||
cell::Cell,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
thread_local! {
|
||||
static NOW: Cell<Instant> = Cell::new(Instant::now());
|
||||
}
|
||||
fn now() -> Instant {
|
||||
NOW.get()
|
||||
}
|
||||
let request = ConnectionBudget::with_clock(now);
|
||||
// 后台 WS 先占一次,HTTP 首次建连再占一次,模拟代理持续不响应
|
||||
let background = request.clone();
|
||||
assert_eq!(background.begin().unwrap(), Some(Duration::from_secs(30)));
|
||||
NOW.set(now() + Duration::from_secs(1));
|
||||
assert_eq!(request.begin().unwrap(), Some(Duration::from_secs(29)));
|
||||
NOW.set(now() + Duration::from_secs(15));
|
||||
assert_eq!(request.remaining_openings(), 2);
|
||||
let retry = request.clone().reserve_next_opening();
|
||||
assert_eq!(retry.begin().unwrap(), Some(Duration::from_secs(7)));
|
||||
assert_eq!(request.remaining_openings(), 1);
|
||||
assert!(retry.begin().is_err());
|
||||
assert!(!request.exhausted());
|
||||
NOW.set(now() + Duration::from_secs(7));
|
||||
// 重选使用原预算,既不继承 A 的局部时间限制,也不能重置总次数
|
||||
assert_eq!(request.begin().unwrap(), Some(Duration::from_secs(7)));
|
||||
assert!(request.exhausted());
|
||||
assert!(background.begin().is_err());
|
||||
request.complete();
|
||||
assert_eq!(retry.begin().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_transports_share_one_opening_budget_but_other_requests_do_not() {
|
||||
let request = ConnectionBudget::default();
|
||||
|
||||
@@ -505,10 +505,6 @@ struct FinalState {
|
||||
upstream_response_ms: Option<u64>,
|
||||
upstream_api_overhead_ms: Option<f64>,
|
||||
upstream_engine_ms: Option<f64>,
|
||||
upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
upstream_engine_service_ttft_ms: Option<f64>,
|
||||
upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
upstream_engine_service_tbt_ms: Option<f64>,
|
||||
provider_metadata_json: Option<String>,
|
||||
cost_source: CostSource,
|
||||
cost_ticks: Option<u128>,
|
||||
@@ -684,12 +680,6 @@ impl ExecutionStore for FakeStore {
|
||||
upstream_response_ms: finalization.timings.upstream_response_ms,
|
||||
upstream_api_overhead_ms: finalization.timings.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: finalization.timings.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: finalization.timings.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: finalization
|
||||
.timings
|
||||
.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: finalization.timings.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: finalization.timings.upstream_engine_service_tbt_ms,
|
||||
provider_metadata_json: finalization.provider_metadata_json,
|
||||
cost_source: finalization.cost.source(),
|
||||
cost_ticks: finalization
|
||||
@@ -2067,10 +2057,6 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
upstream_response_ms: Some(7_000),
|
||||
upstream_api_overhead_ms: Some(120.25),
|
||||
upstream_engine_ms: Some(6400.0),
|
||||
upstream_engine_iapi_ttft_ms: Some(650.5),
|
||||
upstream_engine_service_ttft_ms: Some(720.25),
|
||||
upstream_engine_iapi_tbt_ms: Some(18.45),
|
||||
upstream_engine_service_tbt_ms: Some(20.12),
|
||||
..ProviderResponseTimings::default()
|
||||
});
|
||||
let second_observation = ProviderResponseObservation::new(
|
||||
@@ -2164,10 +2150,6 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
assert_eq!(finalization.upstream_response_ms, None);
|
||||
assert_eq!(finalization.upstream_api_overhead_ms, None);
|
||||
assert_eq!(finalization.upstream_engine_ms, None);
|
||||
assert_eq!(finalization.upstream_engine_iapi_ttft_ms, None);
|
||||
assert_eq!(finalization.upstream_engine_service_ttft_ms, None);
|
||||
assert_eq!(finalization.upstream_engine_iapi_tbt_ms, None);
|
||||
assert_eq!(finalization.upstream_engine_service_tbt_ms, None);
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 2);
|
||||
assert_eq!(
|
||||
@@ -5849,11 +5831,11 @@ fn provider_default_profile_is_applied_once_and_frozen_across_account_retries()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
fn connection_recovery_rotates_only_after_retrying_the_first_account() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_only",
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
@@ -5861,11 +5843,10 @@ fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_only",
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
@@ -5881,37 +5862,250 @@ fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 4);
|
||||
let account = ProviderAccountId::new("acct_only").unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
let first = ProviderAccountId::new("acct_first").unwrap();
|
||||
assert_eq!(contexts[1].required_account(), Some(&first));
|
||||
assert!(!contexts[1].excluded_accounts().contains(&first));
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert!(contexts[2].excluded_accounts().contains(&first));
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert_eq!(context.transport(), AttemptTransport::Fallback);
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(store.state.lock().unwrap().intermediate_failures, 2);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 4);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
for unavailable in [
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
ProviderErrorKind::AccountCapacityUnavailable,
|
||||
ProviderErrorKind::QuotaExhausted,
|
||||
ProviderErrorKind::ConcurrencyQueueFull,
|
||||
ProviderErrorKind::ConcurrencyQueueTimeout,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Error(ProviderError::new(unavailable, UpstreamSendState::NotSent)),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 5);
|
||||
let account = ProviderAccountId::new("acct_only").unwrap();
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert!(contexts[2].excluded_accounts().contains(&account));
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Fallback);
|
||||
for index in [1, 3, 4] {
|
||||
let context = &contexts[index];
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert_eq!(context.transport(), AttemptTransport::Fallback);
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_rotation_does_not_bypass_selection_replay_prohibition() {
|
||||
for send_state in [
|
||||
UpstreamSendState::NotSent,
|
||||
UpstreamSendState::Sent,
|
||||
UpstreamSendState::Ambiguous,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let selection_error = ProviderError::new(ProviderErrorKind::NoEligibleAccount, send_state);
|
||||
let selection_error = if send_state == UpstreamSendState::NotSent {
|
||||
selection_error.with_retry_prohibited()
|
||||
} else {
|
||||
selection_error
|
||||
};
|
||||
let (coordinator, _, provider) =
|
||||
coordinator(vec![failure(), failure(), Script::Error(selection_error)]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(block_on(session.collect_uncommitted()).is_err());
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 3);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_does_not_delay_existing_account_rotation() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
let first = ProviderAccountId::new("acct_first").unwrap();
|
||||
// 临时建连失败先在原账号恢复,后续明确可重放的限流拒绝仍按已有规则换号
|
||||
assert_eq!(contexts[1].required_account(), Some(&first));
|
||||
assert_eq!(contexts[1].transport(), AttemptTransport::Fallback);
|
||||
assert!(!contexts[1].excluded_accounts().contains(&first));
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Default);
|
||||
assert!(contexts[2].excluded_accounts().contains(&first));
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 2);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_stays_on_account_when_rotation_is_not_allowed() {
|
||||
for required in [true, false] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = if required {
|
||||
plan(&operation)
|
||||
} else {
|
||||
plan_with_policy(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::new(2).unwrap())
|
||||
.with_max_account_rotations(0),
|
||||
)
|
||||
};
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_required",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_required",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let account = ProviderAccountId::new("acct_required").unwrap();
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
required.then_some(account.clone()),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
assert_eq!(contexts[0].required_account(), required.then_some(&account));
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_exhaustion_preserves_transport_failure() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(
|
||||
(0..4)
|
||||
.map(|_| Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
})
|
||||
.collect(),
|
||||
let mut scripts: Vec<_> = (0..4)
|
||||
.map(|_| Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
})
|
||||
.collect();
|
||||
scripts.insert(
|
||||
2,
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
);
|
||||
let (coordinator, store, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
@@ -5925,7 +6119,7 @@ fn connection_recovery_exhaustion_preserves_transport_failure() {
|
||||
assert!(
|
||||
matches!(error, EngineError::Provider(ref error) if error.kind() == ProviderErrorKind::Transport)
|
||||
);
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 4);
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 5);
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
4
|
||||
|
||||
@@ -10,9 +10,7 @@ use gateway_core::upstream::UpstreamTransport;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn provider_event_keeps_large_wire_and_observation_payloads_behind_one_indirection() {
|
||||
assert!(std::mem::size_of::<ProviderEvent>() <= 64);
|
||||
|
||||
fn provider_event_should_distinguish_client_delivery_from_observation() {
|
||||
let wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json("openai", None, json!({"type": "response.created"}))
|
||||
.expect("wire event"),
|
||||
|
||||
@@ -378,19 +378,38 @@ impl PendingResponse {
|
||||
|
||||
impl HttpBody {
|
||||
pub async fn read(&mut self, maximum: usize) -> Result<Option<Bytes>, HttpError> {
|
||||
self.read_until(maximum, self.deadline).await
|
||||
}
|
||||
|
||||
/// 资源 owner 已接管总寿命时,每次读取使用独立预算,取消仍由 owner 回收正文
|
||||
pub async fn read_with_timeout(
|
||||
&mut self,
|
||||
maximum: usize,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<Bytes>, HttpError> {
|
||||
let deadline = Instant::now()
|
||||
.checked_add(timeout)
|
||||
.ok_or_else(|| HttpError::sent("read timeout"))?;
|
||||
self.read_until(maximum, deadline).await
|
||||
}
|
||||
|
||||
async fn read_until(
|
||||
&mut self,
|
||||
maximum: usize,
|
||||
deadline: Instant,
|
||||
) -> Result<Option<Bytes>, HttpError> {
|
||||
if maximum == 0 || maximum > 64 * 1024 {
|
||||
return Err(HttpError::sent("read size"));
|
||||
}
|
||||
loop {
|
||||
// 已缓冲的数据和就绪帧也受同一期限约束,慢读者不能无限续用 HTTP 资源
|
||||
if Instant::now() >= self.deadline {
|
||||
// 已缓冲的数据和就绪帧也受本次期限约束,不能绕过 owner 的读取预算
|
||||
if Instant::now() >= deadline {
|
||||
return Err(HttpError::sent("response deadline").with_kind(HttpErrorKind::Timeout));
|
||||
}
|
||||
if !self.pending.is_empty() {
|
||||
return Ok(Some(self.pending.split_to(maximum.min(self.pending.len()))));
|
||||
}
|
||||
self.pending = match tokio::time::timeout_at(self.deadline, self.response.frame()).await
|
||||
{
|
||||
self.pending = match tokio::time::timeout_at(deadline, self.response.frame()).await {
|
||||
Ok(Some(Ok(frame))) => match frame.into_data() {
|
||||
Ok(chunk) => chunk,
|
||||
Err(_) => continue,
|
||||
|
||||
@@ -33,7 +33,8 @@ pub fn worker(store: Arc<dyn RetentionStore>) -> Result<WorkerContribution, Work
|
||||
// 保持稳定的租约身份,滚动升级期间仍与已有进程互斥
|
||||
let id = WorkerId::try_new(WorkerKind::Retention, "postgres")?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(60 * 60),
|
||||
// 有界单轮结束后一分钟继续检查,避免日级写入量超过清理容量
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(15 * 60),
|
||||
|
||||
@@ -162,20 +162,15 @@ fn logging_rejects_zero_retention_windows() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_defaults_to_time_retention_and_ignores_removed_count_retention() {
|
||||
let mut value = serde_json::json!({
|
||||
fn logging_defaults_to_time_retention() {
|
||||
let value = serde_json::json!({
|
||||
"level": "info", "stdout": true,
|
||||
"file": { "enabled": true, "directory": "logs", "max_file_size_mb": 20 }
|
||||
});
|
||||
let config: LoggingConfig = serde_json::from_value(value.clone()).unwrap();
|
||||
let config: LoggingConfig = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(config.request_dump_retention_days, 1.0);
|
||||
assert_eq!(config.file.retention_days, 7);
|
||||
assert!(!config.oauth_recovery);
|
||||
value["file"]["max_files"] = serde_json::json!(20);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<LoggingConfig>(value).unwrap(),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -221,6 +221,63 @@ async fn buffered_body_cannot_extend_the_request_deadline() {
|
||||
assert_eq!(error.kind(), HttpErrorKind::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_body_reads_outlive_opening_but_keep_a_per_read_timeout() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (send_next, next) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut head = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0 && head.len() < 16 * 1024);
|
||||
head.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\na")
|
||||
.await
|
||||
.unwrap();
|
||||
next.await.unwrap();
|
||||
socket.write_all(b"b").await.unwrap();
|
||||
assert_eq!(socket.read(&mut buffer).await.unwrap(), 0);
|
||||
});
|
||||
let mut response = HttpClient::new()
|
||||
.unwrap()
|
||||
.open(request(url), None, &local_network(), Duration::from_secs(1))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.body.read(1).await.unwrap().unwrap(), "a");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
send_next.send(()).unwrap();
|
||||
assert_eq!(
|
||||
response
|
||||
.body
|
||||
.read_with_timeout(1, Duration::from_secs(1))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
"b"
|
||||
);
|
||||
let error = response
|
||||
.body
|
||||
.read_with_timeout(1, Duration::from_millis(100))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), HttpErrorKind::Timeout);
|
||||
assert_eq!(error.send_state, UpstreamSendState::Sent);
|
||||
drop(response);
|
||||
tokio::time::timeout(Duration::from_secs(1), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_dns_releases_the_request_slot_without_starting_http() {
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -543,10 +543,6 @@ fn wire_timings(observation: &RequestObservation) -> RequestTimings {
|
||||
upstream_response_ms: timings.upstream_response_ms,
|
||||
upstream_api_overhead_ms: timings.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: timings.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: timings.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: timings.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: timings.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: timings.upstream_engine_service_tbt_ms,
|
||||
latency_ms: timings.latency_ms,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +177,8 @@ impl Execution {
|
||||
));
|
||||
}
|
||||
let mut context = self.adapter.session.context(Stage::Upstream, timeout);
|
||||
// 初始回复后由模型执行持有流的期限与取消,RPC 预算只约束建立阶段
|
||||
context.resource_stream = true;
|
||||
context.request =
|
||||
crate::adapter::request_context(invocation.context.extension_scope().request());
|
||||
context.request_id = Some(invocation.context.request_id().as_str().to_owned());
|
||||
|
||||
@@ -132,10 +132,6 @@ fn observation(value: &ProviderResponseObservation) -> facts::ResponseObservatio
|
||||
upstream_response_ms: timings.upstream_response_ms,
|
||||
upstream_api_overhead_ms: timings.upstream_api_overhead_ms,
|
||||
upstream_engine_ms: timings.upstream_engine_ms,
|
||||
upstream_engine_iapi_ttft_ms: timings.upstream_engine_iapi_ttft_ms,
|
||||
upstream_engine_service_ttft_ms: timings.upstream_engine_service_ttft_ms,
|
||||
upstream_engine_iapi_tbt_ms: timings.upstream_engine_iapi_tbt_ms,
|
||||
upstream_engine_service_tbt_ms: timings.upstream_engine_service_tbt_ms,
|
||||
latency_ms: None,
|
||||
},
|
||||
client_headers: value
|
||||
|
||||
@@ -142,11 +142,12 @@ impl HttpCallbacks<'_> {
|
||||
let mut body = stream.body.try_lock().map_err(|_| {
|
||||
PluginFault::new(ErrorCode::Capacity, "HTTP stream already has a reader")
|
||||
})?;
|
||||
let timeout = call.timeout()?;
|
||||
let mut closed = stream.closed.subscribe();
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = closed.wait_for(|closed| *closed) => Err(denied()),
|
||||
result = body.as_mut().ok_or_else(denied)?.read(request.maximum_bytes as usize) => result.map_err(http_error),
|
||||
result = body.as_mut().ok_or_else(denied)?.read_with_timeout(request.maximum_bytes as usize, timeout) => result.map_err(http_error),
|
||||
};
|
||||
if !matches!(&result, Ok(Some(_))) {
|
||||
body.take();
|
||||
|
||||
@@ -48,6 +48,14 @@ async fn setup_with_restart_circuit(
|
||||
async fn setup_with_contributions_and_restart_circuit(
|
||||
contributes: Contributions,
|
||||
restart_circuit: PluginRestartCircuitConfig,
|
||||
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
||||
setup_with_limits(contributes, restart_circuit, RpcLimits::default()).await
|
||||
}
|
||||
|
||||
async fn setup_with_limits(
|
||||
contributes: Contributions,
|
||||
restart_circuit: PluginRestartCircuitConfig,
|
||||
rpc_limits: RpcLimits,
|
||||
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let artifact = PackageInspector::new(PackageLimits::default(), "1.0.0".parse().unwrap())
|
||||
@@ -83,7 +91,7 @@ async fn setup_with_contributions_and_restart_circuit(
|
||||
cache_directory: cache.path().to_owned(),
|
||||
host_version: "1.0.0".parse().unwrap(),
|
||||
package_limits: PackageLimits::default(),
|
||||
rpc_limits: RpcLimits::default(),
|
||||
rpc_limits,
|
||||
restart_circuit,
|
||||
},
|
||||
Arc::new(gateway_host::outbound::HttpClient::new().unwrap()),
|
||||
|
||||
@@ -817,7 +817,7 @@ async fn completed_observation_contains_terminal_cost_timings_and_failure() {
|
||||
.with_timings(ModelRequestTimings {
|
||||
first_text_ms: Some(12),
|
||||
upstream_response_ms: Some(7_000),
|
||||
upstream_engine_iapi_tbt_ms: Some(2.450638),
|
||||
upstream_engine_ms: Some(2.450638),
|
||||
latency_ms: Some(34),
|
||||
..ModelRequestTimings::default()
|
||||
}),
|
||||
@@ -831,10 +831,7 @@ async fn completed_observation_contains_terminal_cost_timings_and_failure() {
|
||||
assert_eq!(observed["usage"]["cost"]["total"]["currency"], "USD");
|
||||
assert_eq!(observed["usage"]["timings"]["first_text_ms"], 12);
|
||||
assert_eq!(observed["usage"]["timings"]["upstream_response_ms"], 7_000);
|
||||
assert_eq!(
|
||||
observed["usage"]["timings"]["upstream_engine_iapi_tbt_ms"],
|
||||
2.450638
|
||||
);
|
||||
assert_eq!(observed["usage"]["timings"]["upstream_engine_ms"], 2.450638);
|
||||
assert_eq!(observed["usage"]["timings"]["latency_ms"], 34);
|
||||
assert_eq!(observed["usage"]["failure"]["outcome"], "failed");
|
||||
assert_eq!(observed["usage"]["failure"]["attempt_count"], 2);
|
||||
|
||||
@@ -103,10 +103,24 @@ async fn middleware_preserves_three_state_fast_and_sources() {
|
||||
Some(90_000)
|
||||
);
|
||||
let sources = settings.inspect();
|
||||
assert!(sources["execution"].get("disable_fast").is_none());
|
||||
if mode != FastMode::Default && mode == expected {
|
||||
assert_eq!(sources["execution"]["fast_mode"]["instance_id"], "seed");
|
||||
let fast_source = &sources["execution"]["fast_mode"];
|
||||
if mode == FastMode::Default && expected == FastMode::Default {
|
||||
assert!(fast_source.is_null());
|
||||
} else {
|
||||
assert_eq!(
|
||||
fast_source["instance_id"],
|
||||
if mode == expected {
|
||||
"seed"
|
||||
} else {
|
||||
"fast-settings"
|
||||
}
|
||||
);
|
||||
assert_eq!(fast_source["value"], expected.as_str());
|
||||
}
|
||||
assert_eq!(
|
||||
sources["execution"]["timeout_ms"]["instance_id"],
|
||||
"fast-settings"
|
||||
);
|
||||
next.run(request).await
|
||||
})
|
||||
}),
|
||||
|
||||
@@ -186,7 +186,7 @@ async fn facts_are_readable_without_replacing_or_duplicating_the_original_execut
|
||||
.with_timings(ProviderResponseTimings {
|
||||
headers_ms: Some(19),
|
||||
upstream_response_ms: Some(7_000),
|
||||
upstream_engine_iapi_tbt_ms: Some(2.450638),
|
||||
upstream_engine_ms: Some(2.450638),
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
@@ -273,7 +273,7 @@ async fn facts_are_readable_without_replacing_or_duplicating_the_original_execut
|
||||
);
|
||||
assert_eq!(facts["observation"]["timings"]["headers_ms"], 19);
|
||||
assert_eq!(
|
||||
facts["observation"]["timings"]["upstream_engine_iapi_tbt_ms"],
|
||||
facts["observation"]["timings"]["upstream_engine_ms"],
|
||||
2.450638
|
||||
);
|
||||
assert_eq!(
|
||||
|
||||
@@ -101,6 +101,17 @@ async fn setup(
|
||||
tempfile::TempDir,
|
||||
Arc<super::Store>,
|
||||
gateway_plugin_runtime::PluginRuntime,
|
||||
) {
|
||||
setup_with_limits(config, gateway_plugin_runtime::RpcLimits::default()).await
|
||||
}
|
||||
|
||||
async fn setup_with_limits(
|
||||
config: Value,
|
||||
limits: gateway_plugin_runtime::RpcLimits,
|
||||
) -> (
|
||||
tempfile::TempDir,
|
||||
Arc<super::Store>,
|
||||
gateway_plugin_runtime::PluginRuntime,
|
||||
) {
|
||||
let contributes = Contributions::from([crate::support::contribution(
|
||||
Capability::UpstreamAdapter,
|
||||
@@ -109,7 +120,7 @@ async fn setup(
|
||||
vec!["openai".into()],
|
||||
)]);
|
||||
let (cache, store, runtime) =
|
||||
super::setup_with_contributions_and_restart_circuit(contributes, Default::default()).await;
|
||||
super::setup_with_limits(contributes, Default::default(), limits).await;
|
||||
{
|
||||
let mut snapshot = store.snapshot.lock().unwrap();
|
||||
let instance = &mut snapshot.instances[0];
|
||||
@@ -309,6 +320,132 @@ async fn adapter_http_and_sse_are_cold_authenticated_and_settled_once() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_sse_outlives_rpc_budget_and_cancellation_closes_its_resources() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
for cancel in [false, true] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (started, receiving) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
let head_end = loop {
|
||||
if let Some(end) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
|
||||
break end + 4;
|
||||
}
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0 && request.len() < 16 * 1024);
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
};
|
||||
let length: usize = std::str::from_utf8(&request[..head_end])
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.unwrap()
|
||||
.1
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
while request.len() < head_end + length {
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0);
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n: heartbeat\n\n")
|
||||
.await
|
||||
.unwrap();
|
||||
started.send(()).unwrap();
|
||||
for _ in 0..6 {
|
||||
tokio::select! {
|
||||
result = socket.read(&mut buffer) => {
|
||||
assert_eq!(result.unwrap(), 0);
|
||||
return false;
|
||||
}
|
||||
() = tokio::time::sleep(Duration::from_millis(200)) => {}
|
||||
}
|
||||
if socket.write_all(b": heartbeat\n\n").await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
socket.shutdown().await.unwrap();
|
||||
true
|
||||
});
|
||||
let mut config = configuration(&base, true);
|
||||
// 此夹具汇总 HTTP 正文后产出规范事件,不把多个心跳伪装成一个 SSE 帧
|
||||
config["upstream_events"][1]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("wire");
|
||||
let (cache, _, runtime) = setup_with_limits(
|
||||
config,
|
||||
gateway_plugin_runtime::RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(500),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let generation =
|
||||
ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let account = Account::new();
|
||||
let context = context(&runtime, &generation, "key-one");
|
||||
let cancellation = context.cancellation().clone();
|
||||
let mut stream = execute_with_context(account.clone(), context, None);
|
||||
let consume = async {
|
||||
let mut completed = 0;
|
||||
while let Some(event) = stream.next().await {
|
||||
completed += event?
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.filter(|fact| matches!(fact, GatewayEvent::Completed(_)))
|
||||
.count();
|
||||
}
|
||||
Ok::<_, ProviderError>(completed)
|
||||
};
|
||||
let interrupt = async {
|
||||
receiving.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
if cancel {
|
||||
cancellation.cancel();
|
||||
}
|
||||
};
|
||||
let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::join!(consume, interrupt)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if cancel {
|
||||
assert_eq!(result.unwrap_err().kind(), ProviderErrorKind::Cancelled);
|
||||
} else {
|
||||
assert_eq!(result.unwrap(), 1);
|
||||
}
|
||||
drop(stream);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(2), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
!cancel
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while Arc::strong_count(&account) != 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(generation);
|
||||
runtime.shutdown().await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_accepts_service_tier_resolved_by_the_terminal_response() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -278,7 +278,7 @@ fn icon_content_must_fully_decode_match_its_type_and_fit_dimensions() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obsolete_manifest_fields_are_rejected_without_aliases() {
|
||||
fn manifest_rejects_unknown_fields_before_package_validation() {
|
||||
let manifest = serde_json::json!({
|
||||
"manifestVersion":2,
|
||||
"name": "example",
|
||||
@@ -290,7 +290,8 @@ fn obsolete_manifest_fields_are_rejected_without_aliases() {
|
||||
"engines": {"codex-proxy-rs": ">=1.0.0, <2.0.0"},
|
||||
"main": "bin/worker",
|
||||
"runtime": "trustedProcess",
|
||||
"capabilities": [{"capability":"request_normalizer"}]
|
||||
"contributes": {},
|
||||
"unknownField": true
|
||||
});
|
||||
let archive = crate::support::archive(BTreeMap::from([(
|
||||
"plugin.json".to_owned(),
|
||||
|
||||
@@ -82,6 +82,10 @@ let (message, ()) = tokio::try_join!(
|
||||
包含标准事实与可选原始 wire;事件需符合开始、内容、工具、用量和唯一结束的顺序。
|
||||
原始 JSON / SSE 字节与标准事实分开传递,错误使用 `UpstreamFailure`;宿主网络观测决定发送状态,插件不能将其降为未发送
|
||||
|
||||
初始 RPC 回复受插件处理预算约束,默认上限为 120 秒;持续接收与解析应放在返回的流中。
|
||||
初始流式回复后,流的寿命遵循模型请求期限与取消,受管 HTTP 正文每次读取单独计时。
|
||||
持续有数据的流不会因初始 RPC 预算耗尽而结束,单次读取超时、客户端取消和流结束仍会回收资源
|
||||
|
||||
用量提交标准 `Usage`,费用由内置 Provider 按已冻结的上游模型与价格计算,响应回显模型不切换计价。
|
||||
`service_tier` 记录上游回显的最新档位,允许上游在完成时确定实际值;OpenAI 计价沿用请求策略,回显档位仅作观测。
|
||||
未知用量或价格保留未知。宿主确认 RPC 正常结束后才发布完成事实,插件不能独立结算或组织换号重试
|
||||
|
||||
@@ -110,14 +110,6 @@ pub struct RequestTimings {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_iapi_ttft_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_service_ttft_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_iapi_tbt_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_engine_service_tbt_ms: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub latency_ms: Option<u64>,
|
||||
}
|
||||
|
||||
|
||||
@@ -204,13 +204,13 @@ fn cost_timings_and_failure_keep_unknown_facts_explicit_and_safe() {
|
||||
let timings = RequestTimings {
|
||||
first_text_ms: Some(12),
|
||||
upstream_response_ms: Some(7_000),
|
||||
upstream_engine_iapi_tbt_ms: Some(2.450638),
|
||||
upstream_engine_ms: Some(2.450638),
|
||||
latency_ms: Some(34),
|
||||
..RequestTimings::default()
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(&timings).unwrap(),
|
||||
json!({"first_text_ms":12,"upstream_response_ms":7000,"upstream_engine_iapi_tbt_ms":2.450638,"latency_ms":34})
|
||||
json!({"first_text_ms":12,"upstream_response_ms":7000,"upstream_engine_ms":2.450638,"latency_ms":34})
|
||||
);
|
||||
|
||||
let failure = RequestFailure {
|
||||
|
||||
@@ -132,52 +132,40 @@ fn installation_checks_protocol_engine_and_target() {
|
||||
|
||||
#[test]
|
||||
fn capability_contract_versions_are_checked_independently_of_rpc_version() {
|
||||
let mut manifest = packaged_manifest();
|
||||
assert!(manifest.validate().is_ok());
|
||||
for version in [0, 1, 2, 3, 4, 6, u32::MAX] {
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::Middleware)
|
||||
.unwrap()
|
||||
.version = version;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
for (capability, stage, supported, needs_formats) in [
|
||||
(Capability::Middleware, Stage::Request, 5, true),
|
||||
(Capability::Management, Stage::Management, 2, false),
|
||||
(Capability::UpstreamAdapter, Stage::Upstream, 3, true),
|
||||
(Capability::Scheduler, Stage::Scheduling, 3, false),
|
||||
] {
|
||||
for mut manifest in [source_manifest(), packaged_manifest()] {
|
||||
let formats = if needs_formats {
|
||||
vec!["openai".into()]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
capability,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.contract".into(),
|
||||
version: supported,
|
||||
stages: vec![stage],
|
||||
input_formats: formats.clone(),
|
||||
output_formats: formats,
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok(), "{capability:?} v{supported}");
|
||||
for version in (0..supported).chain([supported + 1, u32::MAX]) {
|
||||
manifest.contributes.get_mut(&capability).unwrap().version = version;
|
||||
assert_eq!(
|
||||
manifest.validate(),
|
||||
Err(ManifestError::Invalid),
|
||||
"{capability:?} v{version}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::Management,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.management".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Management],
|
||||
input_formats: vec![],
|
||||
output_formats: vec![],
|
||||
},
|
||||
);
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_upstream_contract_requires_an_author_upgrade() {
|
||||
let mut manifest = source_manifest();
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::UpstreamAdapter,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.upstream".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Upstream],
|
||||
input_formats: vec!["openai".into()],
|
||||
output_formats: vec!["openai".into()],
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok());
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::UpstreamAdapter)
|
||||
.unwrap()
|
||||
.version = 1;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -371,18 +359,19 @@ fn state_namespaces_are_base_infrastructure_with_bounded_versions() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_manifest_fields_use_only_the_v3_camel_case_shape() {
|
||||
let value = serde_json::to_value(source_manifest()).unwrap();
|
||||
assert!(value.get("manifestVersion").is_some());
|
||||
assert!(value.get("displayName").is_some());
|
||||
assert!(value.get("configurationSchema").is_some());
|
||||
assert!(value.get("schema_version").is_none());
|
||||
fn manifest_round_trips_public_fields_and_rejects_unknown_fields() {
|
||||
let manifest = source_manifest();
|
||||
let mut value = serde_json::to_value(&manifest).unwrap();
|
||||
assert_eq!(value["manifestVersion"], MANIFEST_VERSION);
|
||||
assert_eq!(value["displayName"], manifest.display_name);
|
||||
assert_eq!(value["configurationSchema"], manifest.configuration_schema);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<Manifest>(value.clone()).unwrap(),
|
||||
manifest
|
||||
);
|
||||
|
||||
let mut old = value;
|
||||
old.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("schema_version".into(), json!(3));
|
||||
assert!(serde_json::from_value::<Manifest>(old).is_err());
|
||||
value["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<Manifest>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -439,26 +428,3 @@ fn contribution_declaration_uses_camel_case_description_fields() {
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scheduler_requires_the_current_contract() {
|
||||
let mut manifest = packaged_manifest();
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::Scheduler,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.scheduler".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Scheduling],
|
||||
input_formats: vec![],
|
||||
output_formats: vec![],
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok());
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::Scheduler)
|
||||
.unwrap()
|
||||
.version = 1;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
async-stream = "0.3.6"
|
||||
async-trait = "0.1.89"
|
||||
async-compression = { version = "0.4.50", default-features = false, features = ["tokio", "gzip"] }
|
||||
aws-credential-types = "1.2"
|
||||
aws-sdk-s3 = { version = "1.140", default-features = false, features = ["rt-tokio", "default-https-client"] }
|
||||
aws-smithy-runtime-api = "1.14"
|
||||
@@ -37,6 +38,7 @@ url = "2.5.8"
|
||||
uuid = { version = "1.23.3", features = ["v4", "v7"] }
|
||||
|
||||
[dev-dependencies]
|
||||
percent-encoding = "2.3"
|
||||
tempfile = "3.23"
|
||||
tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "net", "process", "time", "test-util"] }
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
//! 有界流式归档写入:压缩后限额与摘要覆盖实际上传的字节
|
||||
|
||||
use async_compression::tokio::write::GzipEncoder;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt as _, BufWriter};
|
||||
|
||||
struct ArchiveSink {
|
||||
file: BufWriter<tokio::fs::File>,
|
||||
maximum: u64,
|
||||
size: u64,
|
||||
hash: Sha256,
|
||||
}
|
||||
|
||||
impl AsyncWrite for ArchiveSink {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
if bytes.len() as u64 > self.maximum.saturating_sub(self.size) {
|
||||
return Poll::Ready(Err(io::Error::from(io::ErrorKind::FileTooLarge)));
|
||||
}
|
||||
match Pin::new(&mut self.file).poll_write(cx, bytes) {
|
||||
Poll::Ready(Ok(count)) => {
|
||||
self.size += count as u64;
|
||||
self.hash.update(&bytes[..count]);
|
||||
Poll::Ready(Ok(count))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.file).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.file).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ArchiveWriter(GzipEncoder<ArchiveSink>);
|
||||
|
||||
impl ArchiveWriter {
|
||||
pub fn new(file: std::fs::File, maximum: u64) -> Self {
|
||||
Self(GzipEncoder::new(ArchiveSink {
|
||||
file: BufWriter::new(tokio::fs::File::from_std(file)),
|
||||
maximum,
|
||||
size: 0,
|
||||
hash: Sha256::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn finish(mut self) -> io::Result<(u64, String)> {
|
||||
self.shutdown().await?;
|
||||
let sink = self.0.into_inner();
|
||||
sink.file.get_ref().sync_all().await?;
|
||||
Ok((sink.size, hex::encode(sink.hash.finalize())))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for ArchiveWriter {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(&mut self.0).poll_write(cx, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
//! S3 对象存储与 `pg_dump` 基础设施适配器
|
||||
|
||||
mod archive;
|
||||
pub mod pg_dump;
|
||||
pub mod s3;
|
||||
mod snapshot;
|
||||
pub mod staging;
|
||||
|
||||
@@ -16,11 +16,12 @@ use tokio::process::Command;
|
||||
use gateway_admin::model::backup::{BackupError, code};
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpArtifact, DumpRequest, StagedArtifact};
|
||||
|
||||
use super::staging::StagingArea;
|
||||
use super::{archive::ArchiveWriter, snapshot::BusinessSnapshot, staging::StagingArea};
|
||||
|
||||
/// `pg_dump` 导出适配器
|
||||
pub struct PgDumpAdapter {
|
||||
staging: Arc<StagingArea>,
|
||||
pool: sqlx::PgPool,
|
||||
/// 不含密码的 PostgreSQL URL,用于 `--dbname`
|
||||
database_url: String,
|
||||
/// 数据库密码,仅通过 `PGPASSWORD` 注入
|
||||
@@ -34,9 +35,11 @@ impl PgDumpAdapter {
|
||||
staging: Arc<StagingArea>,
|
||||
database_url: impl Into<String>,
|
||||
database_password: impl Into<String>,
|
||||
pool: sqlx::PgPool,
|
||||
) -> Self {
|
||||
Self {
|
||||
staging,
|
||||
pool,
|
||||
database_url: database_url.into(),
|
||||
database_password: database_password.into(),
|
||||
}
|
||||
@@ -64,8 +67,14 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
.open(&partial)
|
||||
.map_err(|_| pg_dump_failed("无法创建暂存文件"))?;
|
||||
let _partial_cleanup = PartialArchive(partial.clone());
|
||||
let mut child = Command::new("pg_dump")
|
||||
.arg("--format=custom")
|
||||
let snapshot = tokio::select! {
|
||||
biased;
|
||||
_ = request.cancellation.cancelled() => return Err(cancelled()),
|
||||
result = BusinessSnapshot::begin(&self.pool, &request.backup_id) => result?,
|
||||
};
|
||||
let mut command = Command::new("pg_dump");
|
||||
command
|
||||
.arg("--format=plain")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--dbname")
|
||||
@@ -74,17 +83,22 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
.env("PGCONNECT_TIMEOUT", "10")
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.kill_on_drop(true)
|
||||
.kill_on_drop(true);
|
||||
command.arg("--lock-wait-timeout=5s");
|
||||
snapshot.configure(&mut command);
|
||||
let mut child = command
|
||||
.spawn()
|
||||
.map_err(|_| pg_dump_failed("无法启动 pg_dump 进程"))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut size: u64 = 0;
|
||||
let export = async {
|
||||
let stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| pg_dump_failed("无法读取 pg_dump 输出"))?;
|
||||
let mut writer = tokio::io::BufWriter::new(tokio::fs::File::from_std(file));
|
||||
let mut writer = ArchiveWriter::new(file, self.staging.max_archive_bytes());
|
||||
writer
|
||||
.write_all(snapshot.restore_preamble().as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
let mut reader = tokio::io::BufReader::new(stdout);
|
||||
let mut buffer = vec![0_u8; 64 * 1024];
|
||||
loop {
|
||||
@@ -95,20 +109,8 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
size = size.saturating_add(n as u64);
|
||||
if size > self.staging.max_archive_bytes() {
|
||||
return Err(staging_space_exhausted());
|
||||
}
|
||||
hasher.update(&buffer[..n]);
|
||||
writer
|
||||
.write_all(&buffer[..n])
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("写入暂存文件失败"))?;
|
||||
writer.write_all(&buffer[..n]).await.map_err(write_failed)?;
|
||||
}
|
||||
writer
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("刷新暂存文件失败"))?;
|
||||
let status = child
|
||||
.wait()
|
||||
.await
|
||||
@@ -116,22 +118,26 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
if !status.success() {
|
||||
return Err(pg_dump_failed("pg_dump 非零退出"));
|
||||
}
|
||||
Ok(())
|
||||
snapshot.append(&mut writer).await?;
|
||||
writer.finish().await.map_err(write_failed)
|
||||
};
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = request.cancellation.cancelled() => Err(cancelled()),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(15 * 60)) => Err(pg_dump_failed("备份导出超过 15 分钟期限")),
|
||||
result = export => result,
|
||||
};
|
||||
if let Err(error) = result {
|
||||
// 正常错误和取消必须等到进程退出;任务被强制丢弃时由 kill_on_drop 接管
|
||||
if let Err(source) = child.kill().await {
|
||||
tracing::warn!(backup_id = %request.backup_id, error = %source, "终止 pg_dump 失败");
|
||||
let (size, sha256) = match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
// 正常错误和取消必须等到进程退出;任务被强制丢弃时由 kill_on_drop 接管
|
||||
if let Err(source) = child.kill().await {
|
||||
tracing::warn!(backup_id = %request.backup_id, error = %source, "终止 pg_dump 失败");
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
std::fs::rename(&partial, &final_path).map_err(|_| pg_dump_failed("暂存归档改名失败"))?;
|
||||
let sha256 = hex::encode(hasher.finalize());
|
||||
Ok(DumpArtifact {
|
||||
path: final_path,
|
||||
size_bytes: size,
|
||||
@@ -199,7 +205,7 @@ async fn hash_file(path: &std::path::Path) -> Result<String, BackupError> {
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
fn pg_dump_failed(message: &'static str) -> BackupError {
|
||||
pub(super) fn pg_dump_failed(message: &'static str) -> BackupError {
|
||||
BackupError::new(code::PG_DUMP_FAILED, message.to_owned())
|
||||
}
|
||||
|
||||
@@ -213,3 +219,14 @@ fn staging_space_exhausted() -> BackupError {
|
||||
fn cancelled() -> BackupError {
|
||||
BackupError::new(code::CANCELLED, "备份导出已取消".to_owned())
|
||||
}
|
||||
|
||||
pub(super) fn write_failed(error: std::io::Error) -> BackupError {
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::FileTooLarge | std::io::ErrorKind::StorageFull
|
||||
) {
|
||||
staging_space_exhausted()
|
||||
} else {
|
||||
pg_dump_failed("写入暂存归档失败")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! 与业务数据共用快照导出已发布的小时总量,不扫描请求明细或逐请求投影
|
||||
|
||||
use futures::TryStreamExt as _;
|
||||
use gateway_admin::model::backup::{BackupError, code};
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt as _};
|
||||
|
||||
use super::pg_dump::{pg_dump_failed, write_failed};
|
||||
|
||||
/// 只排除表数据,保留完整 schema 及迁移历史;费用窗口独立于逐请求去重记录保留
|
||||
const EXCLUDED_DATA: &[&str] = &[
|
||||
"model_requests",
|
||||
"model_request_leases",
|
||||
"ops_events",
|
||||
"admin_audit_events",
|
||||
"client_api_key_charge_events",
|
||||
"request_analytics_facts",
|
||||
"request_analytics_15m",
|
||||
"request_analytics_1h",
|
||||
"request_analytics_1d_counts",
|
||||
"request_analytics_pending_requests",
|
||||
"request_analytics_pending_buckets",
|
||||
"request_analytics_build_state",
|
||||
];
|
||||
|
||||
pub(super) struct BusinessSnapshot {
|
||||
transaction: Transaction<'static, Postgres>,
|
||||
snapshot_id: String,
|
||||
schema: String,
|
||||
}
|
||||
|
||||
impl BusinessSnapshot {
|
||||
pub async fn begin(pool: &PgPool, backup_id: &str) -> Result<Self, BackupError> {
|
||||
let mut transaction = pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法开始备份快照"))?;
|
||||
sqlx::query("set transaction isolation level repeatable read, read only")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法隔离备份快照"))?;
|
||||
sqlx::query("set local statement_timeout = '30s'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法设置备份查询期限"))?;
|
||||
if !crate::postgres::backup_projection_ready(&mut transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法读取统计发布状态"))?
|
||||
{
|
||||
return Err(BackupError::new(
|
||||
code::ANALYTICS_NOT_READY,
|
||||
"用量统计正在重建,请在统计就绪后重试备份".to_owned(),
|
||||
));
|
||||
}
|
||||
sqlx::query("select set_config('cpr.backup_id',$1,true)")
|
||||
.bind(backup_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法标记备份汇总"))?;
|
||||
let (snapshot_id, schema): (String, String) =
|
||||
sqlx::query_as("select pg_export_snapshot(), current_schema()")
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法导出一致性快照"))?;
|
||||
Ok(Self {
|
||||
transaction,
|
||||
snapshot_id,
|
||||
schema: format!("\"{}\"", schema.replace('"', "\"\"")),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn configure(&self, command: &mut tokio::process::Command) {
|
||||
command
|
||||
.arg("--snapshot")
|
||||
.arg(&self.snapshot_id)
|
||||
.arg("--schema")
|
||||
.arg(&self.schema);
|
||||
for table in EXCLUDED_DATA {
|
||||
command
|
||||
.arg("--exclude-table-data")
|
||||
.arg(format!("{}.\"{table}\"", self.schema));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn restore_preamble(&self) -> String {
|
||||
// 新库自带 public;只替换空 schema,含已有对象时由 PostgreSQL 拒绝删除
|
||||
format!("DROP SCHEMA IF EXISTS {};\n", self.schema)
|
||||
}
|
||||
|
||||
pub async fn append(
|
||||
mut self,
|
||||
writer: &mut (impl AsyncWrite + Unpin),
|
||||
) -> Result<(), BackupError> {
|
||||
let header = format!(
|
||||
"\n-- CPR 业务备份:已发布的全局小时用量与维护贡献\nCOPY {}.request_analytics_1h (bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) FROM stdin;\n",
|
||||
self.schema
|
||||
);
|
||||
writer
|
||||
.write_all(header.as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
// 原生统计只合并 global,不能把按账号、Key 等重复投影的同一请求再次相加
|
||||
// 贡献身份包含本次备份 ID,恢复后再备份时与上一次恢复的基线互不覆盖
|
||||
let mut stream = self.transaction.copy_out_raw(r#"COPY (
|
||||
WITH native AS MATERIALIZED (
|
||||
SELECT bucket_start, counts, costs FROM request_analytics_1h WHERE collection='global'
|
||||
), counters AS (
|
||||
SELECT bucket_start, c.key, sum(c.value::numeric) AS value
|
||||
FROM native CROSS JOIN LATERAL jsonb_each_text(counts) c
|
||||
WHERE c.key IN ('request_count','success_count','failure_count','cancelled_count','incomplete_count',
|
||||
'completed_request_count','input_tokens','output_tokens','cached_input_tokens','cache_write_input_tokens',
|
||||
'reasoning_output_tokens','image_input_tokens','image_output_tokens','total_tokens')
|
||||
GROUP BY bucket_start,c.key
|
||||
), amounts AS (
|
||||
SELECT bucket_start, c.key, sum(c.value::numeric)::text AS value
|
||||
FROM native CROSS JOIN LATERAL jsonb_each_text(costs) c
|
||||
WHERE c.key NOT IN ('standard_usd','no_cache_usd') GROUP BY bucket_start,c.key
|
||||
), totals AS (
|
||||
SELECT bucket_start, jsonb_object_agg(key,value) AS counts FROM counters GROUP BY bucket_start
|
||||
), costs AS (
|
||||
SELECT bucket_start, jsonb_object_agg(key,value) AS costs FROM amounts GROUP BY bucket_start
|
||||
)
|
||||
SELECT bucket_start, 'usage_contribution', ARRAY['','','','','','','backup:'||current_setting('cpr.backup_id')],
|
||||
'',0,0,counts,coalesce(costs.costs,'{}'::jsonb),'{}'::jsonb
|
||||
FROM totals LEFT JOIN costs USING(bucket_start)
|
||||
UNION ALL
|
||||
SELECT bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions
|
||||
FROM request_analytics_1h WHERE collection='usage_contribution'
|
||||
) TO STDOUT"#).await.map_err(|_| pg_dump_failed("无法导出用量汇总"))?;
|
||||
while let Some(bytes) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("读取用量汇总失败"))?
|
||||
{
|
||||
writer.write_all(&bytes).await.map_err(write_failed)?;
|
||||
}
|
||||
drop(stream);
|
||||
let footer = format!(
|
||||
"\\.\nINSERT INTO {}.request_analytics_build_state (singleton) VALUES (true);\n",
|
||||
self.schema
|
||||
);
|
||||
writer
|
||||
.write_all(footer.as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
self.transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("关闭备份快照失败"))
|
||||
}
|
||||
}
|
||||
@@ -52,7 +52,7 @@ impl StagingArea {
|
||||
/// 完成归档的暂存路径
|
||||
#[must_use]
|
||||
pub fn final_path(&self, backup_id: &str) -> PathBuf {
|
||||
self.base_dir.join(format!("{backup_id}.dump"))
|
||||
self.base_dir.join(format!("{backup_id}.sql.gz"))
|
||||
}
|
||||
|
||||
/// 单任务暂存归档上限
|
||||
|
||||
@@ -341,11 +341,12 @@ pub(crate) fn backup_ports(
|
||||
config.backup_staging_dir().to_path_buf(),
|
||||
backup::staging::DEFAULT_MAX_ARCHIVE_BYTES,
|
||||
)?);
|
||||
let repository = Arc::new(postgres::PgBackupRepository::new(pool));
|
||||
let repository = Arc::new(postgres::PgBackupRepository::new(pool.clone()));
|
||||
let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new(
|
||||
staging,
|
||||
config.database.url.clone(),
|
||||
config.database.password.clone(),
|
||||
pool,
|
||||
));
|
||||
let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new());
|
||||
Ok(BackupStorePorts::new(repository, dump, object_store))
|
||||
|
||||
@@ -184,12 +184,17 @@ pub(crate) struct StoreConnectionConfig {
|
||||
impl StoreConnectionConfig {
|
||||
fn validate(&self, field: &'static str) -> StoreResult<()> {
|
||||
require_nonempty("store config", field, &self.url)?;
|
||||
if self.password.len() != 48 || !self.password.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
// 托管存储的具体密码策略由服务端约束,客户端共用长度与控制字符边界
|
||||
if !(16..=128).contains(&self.password.len())
|
||||
|| self.password.chars().any(char::is_control)
|
||||
|| self.password.trim().is_empty()
|
||||
{
|
||||
return Err(StoreError::InvalidData {
|
||||
source: None,
|
||||
entity: "store config",
|
||||
message: format!("{field}.password must be exactly 48 hexadecimal characters"),
|
||||
message: format!(
|
||||
"{field}.password must be 16 to 128 bytes, not blank, without control characters"
|
||||
),
|
||||
});
|
||||
}
|
||||
self.connection_url(field).map(|_| ())
|
||||
@@ -208,7 +213,8 @@ impl StoreConnectionConfig {
|
||||
message: format!("{field}.url must not contain a password"),
|
||||
});
|
||||
}
|
||||
url.set_password(Some(&self.password))
|
||||
// set_password 保留已有百分号序列,先转义字面量 %,避免驱动解码后改变原始密码
|
||||
url.set_password(Some(&self.password.replace('%', "%25")))
|
||||
.map_err(|()| StoreError::InvalidData {
|
||||
source: None,
|
||||
entity: "store config",
|
||||
|
||||
@@ -46,4 +46,4 @@ pub use bundle::*;
|
||||
pub use config::*;
|
||||
pub use value::*;
|
||||
pub(crate) use workers::*;
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe, run_request_analytics_cycle};
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe, RequestAnalyticsCycle};
|
||||
|
||||
@@ -107,14 +107,6 @@ struct UpstreamTimings {
|
||||
api_overhead_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
engine_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
engine_iapi_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
engine_service_ttft_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
engine_iapi_tbt_ms: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
engine_service_tbt_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
@@ -229,16 +221,9 @@ impl RequestObservation {
|
||||
value.retry_after_ms,
|
||||
])?;
|
||||
// serde_json 会把非有限浮点数写成 null,先拒绝以免把非法指标伪装成缺失
|
||||
for value in [
|
||||
timings.upstream_api_overhead_ms,
|
||||
timings.upstream_engine_ms,
|
||||
timings.upstream_engine_iapi_ttft_ms,
|
||||
timings.upstream_engine_service_ttft_ms,
|
||||
timings.upstream_engine_iapi_tbt_ms,
|
||||
timings.upstream_engine_service_tbt_ms,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
for value in [timings.upstream_api_overhead_ms, timings.upstream_engine_ms]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if !value.is_finite() || value < 0.0 || value >= i64::MAX as f64 {
|
||||
return Err(invalid(
|
||||
@@ -263,10 +248,6 @@ impl RequestObservation {
|
||||
response_ms: timings.upstream_response_ms,
|
||||
api_overhead_ms: timings.upstream_api_overhead_ms,
|
||||
engine_ms: timings.upstream_engine_ms,
|
||||
engine_iapi_ttft_ms: timings.upstream_engine_iapi_ttft_ms,
|
||||
engine_service_ttft_ms: timings.upstream_engine_service_ttft_ms,
|
||||
engine_iapi_tbt_ms: timings.upstream_engine_iapi_tbt_ms,
|
||||
engine_service_tbt_ms: timings.upstream_engine_service_tbt_ms,
|
||||
}),
|
||||
}),
|
||||
transport: Some(TransportObservation {
|
||||
|
||||
@@ -192,6 +192,8 @@ async fn apply_changes(
|
||||
}
|
||||
let mut buckets =
|
||||
BTreeMap::<DateTime<Utc>, BTreeMap<(String, Vec<String>, String), MetricState>>::new();
|
||||
let mut empty_groups = Vec::new();
|
||||
let mut rebuild = Vec::new();
|
||||
for (key, change) in changes {
|
||||
if change.previous.encode()? == change.current.encode()?
|
||||
&& change.previous.last_used_at() == change.current.last_used_at()
|
||||
@@ -202,21 +204,17 @@ async fn apply_changes(
|
||||
let needs_rebuild = subtract(&mut state, &change.previous, &change.current)?;
|
||||
state.merge(&change.current)?;
|
||||
if state.count("request_count") == 0 {
|
||||
let statement = format!(
|
||||
"delete from {table} where bucket_start=$1 and collection=$2 and dimensions=$3 and outcome=$4"
|
||||
);
|
||||
sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(key.0)
|
||||
.bind(&key.1)
|
||||
.bind(&key.2)
|
||||
.bind(&key.3)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("remove empty analytics group", error))?;
|
||||
empty_groups.push(serde_json::json!({
|
||||
"bucket_start": key.0,
|
||||
"collection": key.1,
|
||||
"dimensions": key.2,
|
||||
"outcome": key.3,
|
||||
}));
|
||||
} else {
|
||||
// 预测状态只有可加计数和美元总额,零金额无需修复币种存在性
|
||||
if needs_rebuild && key.1 != forecast::COLLECTION {
|
||||
state = rebuild_group(tx, hourly, progress, &key).await?;
|
||||
rebuild.push(key);
|
||||
continue;
|
||||
}
|
||||
buckets
|
||||
.entry(key.0)
|
||||
@@ -224,6 +222,24 @@ async fn apply_changes(
|
||||
.insert((key.1, key.2, key.3), state);
|
||||
}
|
||||
}
|
||||
for (key, state) in rebuild_groups(tx, hourly, progress, rebuild).await? {
|
||||
buckets
|
||||
.entry(key.0)
|
||||
.or_default()
|
||||
.insert((key.1, key.2, key.3), state);
|
||||
}
|
||||
// 保留期缩短会同时清空大量分组,集合删除避免 SQL 往返随分组数放大
|
||||
for groups in empty_groups.chunks(1_000) {
|
||||
let statement = format!(
|
||||
"delete from {table} a using jsonb_to_recordset($1::jsonb) k(bucket_start timestamptz,collection text,dimensions text[],outcome text)
|
||||
where a.bucket_start=k.bucket_start and a.collection=k.collection and a.dimensions=k.dimensions and a.outcome=k.outcome"
|
||||
);
|
||||
sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(sqlx::types::Json(groups))
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("remove empty analytics groups", error))?;
|
||||
}
|
||||
write_bucket_groups(tx, table, buckets).await
|
||||
}
|
||||
|
||||
@@ -311,61 +327,97 @@ pub(super) fn subtract(
|
||||
Ok(rebuild)
|
||||
}
|
||||
|
||||
async fn rebuild_group(
|
||||
async fn rebuild_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
hourly: bool,
|
||||
progress: &BTreeMap<DateTime<Utc>, Progress>,
|
||||
key: &GroupKey,
|
||||
) -> StoreResult<MetricState> {
|
||||
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
let mut cursor = None;
|
||||
if hourly {
|
||||
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
|
||||
&& let Some(time) = owner.started_at
|
||||
{
|
||||
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
end = if owner.contains_quarter(&candidate) {
|
||||
time + TimeDelta::minutes(15)
|
||||
} else {
|
||||
time
|
||||
};
|
||||
keys: Vec<GroupKey>,
|
||||
) -> StoreResult<BTreeMap<GroupKey, MetricState>> {
|
||||
// 同一统计桶的分组共享时间范围,按桶查询才能让事实表范围扫描命中 started_at 索引
|
||||
let mut by_bucket = BTreeMap::<(&str, DateTime<Utc>), Vec<&GroupKey>>::new();
|
||||
for key in &keys {
|
||||
by_bucket.entry((&key.1, key.0)).or_default().push(key);
|
||||
}
|
||||
let mut rebuilt = BTreeMap::new();
|
||||
for ((collection, bucket_start), keys) in by_bucket {
|
||||
let columns = worker::COLLECTION_COLUMNS
|
||||
.iter()
|
||||
.find_map(|(name, columns)| (*name == collection).then_some(*columns))
|
||||
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
|
||||
let bucket_end = bucket_start + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
// 同类分组共用维度匹配和聚合 SQL,各组仍独立遵守已发布的构建前缀
|
||||
for keys in keys.chunks(1_000) {
|
||||
let mut input = Vec::with_capacity(keys.len());
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
let mut cursor = None;
|
||||
if hourly {
|
||||
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
|
||||
&& let Some(time) = owner.started_at
|
||||
{
|
||||
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
end = if owner.contains_quarter(&candidate) {
|
||||
time + TimeDelta::minutes(15)
|
||||
} else {
|
||||
time
|
||||
};
|
||||
}
|
||||
} else if let Some(stage) = progress.get(&key.0)
|
||||
&& stage.phase == 1
|
||||
{
|
||||
cursor = stage.started_at.zip(stage.id.as_deref());
|
||||
}
|
||||
input.push(serde_json::json!({
|
||||
"bucket_start": key.0,
|
||||
"end_at": end,
|
||||
"cursor_time": cursor.map(|(time, _)| time),
|
||||
"cursor_id": cursor.map(|(_, id)| id),
|
||||
"dimensions": key.2,
|
||||
"outcome": key.3,
|
||||
"name": index.to_string(),
|
||||
}));
|
||||
rebuilt.insert((*key).clone(), MetricState::default());
|
||||
}
|
||||
// 等值桶键避免范围连接反复比较其他时间桶,范围和游标仍限定已发布前缀
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"with matching as materialized (select mr.*,k.bucket_start as output_bucket,k.name,coalesce((",
|
||||
);
|
||||
query.push(completed_usage_fact_predicate("mr"))
|
||||
.push("),false) as is_completed_usage from jsonb_to_recordset(")
|
||||
.push_bind(serde_json::Value::Array(input))
|
||||
.push("::jsonb) k(bucket_start timestamptz,end_at timestamptz,cursor_time timestamptz,cursor_id text,dimensions text[],outcome text,name text)
|
||||
join request_analytics_facts mr on date_bin(make_interval(secs => ")
|
||||
.push_bind(if hourly { 3600_f64 } else { 900_f64 })
|
||||
.push("),mr.started_at,timestamptz '2000-01-01 UTC')=k.bucket_start
|
||||
and mr.started_at >= k.bucket_start and mr.started_at < k.end_at
|
||||
and mr.recovered_at is null and mr.outcome=k.outcome
|
||||
and (k.cursor_time is null or (mr.started_at,mr.id collate \"C\") <= (k.cursor_time,k.cursor_id collate \"C\"))
|
||||
and mr.started_at >= ")
|
||||
.push_bind(bucket_start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(bucket_end);
|
||||
for (index, column) in columns.iter().enumerate() {
|
||||
query.push(format!(
|
||||
" and coalesce(mr.{column}::text,'')=k.dimensions[{}]",
|
||||
index + 1
|
||||
));
|
||||
}
|
||||
query.push(")");
|
||||
aggregate::push_fact_aggregation(&mut query, false, true);
|
||||
let rows = query.build().fetch_all(&mut **tx).await.map_err(|error| {
|
||||
postgres_unavailable("rebuild changed analytics extrema", error)
|
||||
})?;
|
||||
for row in rows {
|
||||
let name: String = get(&row, "name")?;
|
||||
let index = name
|
||||
.parse::<usize>()
|
||||
.map_err(|_| invalid("invalid analytics rebuild group"))?;
|
||||
let key = keys
|
||||
.get(index)
|
||||
.ok_or_else(|| invalid("unknown analytics rebuild group"))?;
|
||||
rebuilt.insert((*key).clone(), MetricState::decode(&row)?);
|
||||
}
|
||||
}
|
||||
} else if let Some(stage) = progress.get(&key.0)
|
||||
&& stage.phase == 1
|
||||
{
|
||||
cursor = stage.started_at.zip(stage.id.as_deref());
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new("with matching as materialized (select mr.*,");
|
||||
query.push_bind(key.0).push("::timestamptz as output_bucket,''::text as name,coalesce((")
|
||||
.push(completed_usage_fact_predicate("mr")).push("),false) as is_completed_usage from request_analytics_facts mr where mr.started_at >= ")
|
||||
.push_bind(key.0).push(" and mr.started_at < ").push_bind(end)
|
||||
.push(" and mr.recovered_at is null and mr.outcome=").push_bind(&key.3);
|
||||
if let Some((time, id)) = cursor {
|
||||
query
|
||||
.push(" and (mr.started_at,mr.id collate \"C\") <= (")
|
||||
.push_bind(time)
|
||||
.push(",")
|
||||
.push_bind(id)
|
||||
.push(")");
|
||||
}
|
||||
let columns = worker::COLLECTION_COLUMNS
|
||||
.iter()
|
||||
.find_map(|(collection, columns)| (*collection == key.1).then_some(*columns))
|
||||
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
|
||||
for (column, value) in columns.iter().zip(&key.2) {
|
||||
query
|
||||
.push(format!(" and coalesce(mr.{column}::text,'')="))
|
||||
.push_bind(value);
|
||||
}
|
||||
query.push(")");
|
||||
aggregate::push_fact_aggregation(&mut query, false, true);
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("rebuild changed analytics extrema", error))?;
|
||||
row.as_ref()
|
||||
.map(MetricState::decode)
|
||||
.transpose()
|
||||
.map(Option::unwrap_or_default)
|
||||
Ok(rebuilt)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,15 @@ pub(crate) const AGGREGATE_COLLECTION: &str = "usage_contribution";
|
||||
const SCHEMA_VERSION: i32 = 1;
|
||||
const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION;
|
||||
|
||||
/// 业务备份只消费已发布小时汇总,重建中的前缀不能作为完整历史导出
|
||||
pub(crate) async fn backup_projection_ready(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
) -> StoreResult<bool> {
|
||||
sqlx::query_scalar("select exists(select 1 from request_analytics_build_state where singleton and backfill_complete and reset_phase=0 and schema_version=$1 and histogram_version=$2) and not exists(select 1 from request_analytics_pending_buckets)")
|
||||
.bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).fetch_one(&mut **tx).await
|
||||
.map_err(|error| postgres_unavailable("check backup analytics readiness", error))
|
||||
}
|
||||
|
||||
pub(super) async fn begin_snapshot(
|
||||
connection: &mut ObservabilityConnection,
|
||||
) -> StoreResult<sqlx::Transaction<'_, Postgres>> {
|
||||
|
||||
@@ -83,7 +83,17 @@ impl PgRequestAnalyticsRepository {
|
||||
return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() });
|
||||
}
|
||||
let mut result = AnalyticsMaintenance::default();
|
||||
let rows = sqlx::query("select request_id, change_version from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
|
||||
// 历史删除可能一次排入大量旧任务,给仍存在的请求预留半批名额,避免实时统计饥饿
|
||||
// 其余名额仍按最早入队消费,两类任务都能前进;同一快照内选版本,沿用精确版本 ACK
|
||||
let rows = sqlx::query("with live as materialized (
|
||||
select pending.request_id, pending.change_version from request_analytics_pending_requests pending
|
||||
join model_requests source on source.id=pending.request_id
|
||||
order by pending.oldest_pending_at, pending.request_id limit ($1::bigint / 2)
|
||||
), oldest as (
|
||||
select pending.request_id, pending.change_version from request_analytics_pending_requests pending
|
||||
where not exists (select 1 from live where live.request_id=pending.request_id)
|
||||
order by pending.oldest_pending_at, pending.request_id limit ($1::bigint - (select count(*) from live))
|
||||
) select request_id, change_version from live union all select request_id, change_version from oldest")
|
||||
.bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
|
||||
@@ -350,10 +350,6 @@ pub(crate) fn usage_list_record_from_row(
|
||||
upstream_response_ms: optional_unsigned(row, "upstream_response_ms")?,
|
||||
upstream_api_overhead_ms: get(row, "upstream_api_overhead_ms")?,
|
||||
upstream_engine_ms: get(row, "upstream_engine_ms")?,
|
||||
upstream_engine_iapi_ttft_ms: get(row, "upstream_engine_iapi_ttft_ms")?,
|
||||
upstream_engine_service_ttft_ms: get(row, "upstream_engine_service_ttft_ms")?,
|
||||
upstream_engine_iapi_tbt_ms: get(row, "upstream_engine_iapi_tbt_ms")?,
|
||||
upstream_engine_service_tbt_ms: get(row, "upstream_engine_service_tbt_ms")?,
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
admission_decision_ms: optional_unsigned(row, "admission_decision_ms")?,
|
||||
account_selection_wait_ms: optional_unsigned(row, "account_selection_wait_ms")?,
|
||||
@@ -446,10 +442,6 @@ pub(crate) fn usage_record_from_row(row: &sqlx::postgres::PgRow) -> StoreResult<
|
||||
upstream_response_ms: optional_unsigned(row, "upstream_response_ms")?,
|
||||
upstream_api_overhead_ms: get(row, "upstream_api_overhead_ms")?,
|
||||
upstream_engine_ms: get(row, "upstream_engine_ms")?,
|
||||
upstream_engine_iapi_ttft_ms: get(row, "upstream_engine_iapi_ttft_ms")?,
|
||||
upstream_engine_service_ttft_ms: get(row, "upstream_engine_service_ttft_ms")?,
|
||||
upstream_engine_iapi_tbt_ms: get(row, "upstream_engine_iapi_tbt_ms")?,
|
||||
upstream_engine_service_tbt_ms: get(row, "upstream_engine_service_tbt_ms")?,
|
||||
latency_ms: optional_unsigned(row, "latency_ms")?,
|
||||
admission_decision_ms: optional_unsigned(row, "admission_decision_ms")?,
|
||||
account_selection_wait_ms: optional_unsigned(row, "account_selection_wait_ms")?,
|
||||
|
||||
@@ -34,6 +34,7 @@ pub use admin_adapter::*;
|
||||
pub(crate) use analytics::accounts::{
|
||||
account_group_usage, account_usage_sort_values, account_usage_windows,
|
||||
};
|
||||
pub(crate) use analytics::backup_projection_ready;
|
||||
pub(crate) use analytics::{AGGREGATE_COLLECTION, AnalyticsSnapshot};
|
||||
pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
pub(crate) use mapping::*;
|
||||
|
||||
@@ -128,7 +128,7 @@ pub(crate) const USAGE_LIST_RECORD_SELECT: &str =
|
||||
mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms,
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
mr.provider_processing_ms, mr.upstream_response_ms,
|
||||
mr.upstream_api_overhead_ms, mr.upstream_engine_ms, mr.upstream_engine_iapi_ttft_ms, mr.upstream_engine_service_ttft_ms, mr.upstream_engine_iapi_tbt_ms, mr.upstream_engine_service_tbt_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.upstream_api_overhead_ms, mr.upstream_engine_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.account_selection_wait_ms, mr.capacity_used_slots, mr.capacity_total_slots,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.subagent_kind, mr.compact,
|
||||
@@ -157,7 +157,7 @@ pub(crate) const USAGE_RECORD_DETAIL_SELECT: &str =
|
||||
mr.cost_currency, mr.transport_decision_wait_ms, mr.connect_ms, mr.headers_ms,
|
||||
mr.first_event_ms, mr.first_reasoning_ms, mr.first_text_ms, mr.first_token_ms,
|
||||
mr.provider_processing_ms, mr.upstream_response_ms,
|
||||
mr.upstream_api_overhead_ms, mr.upstream_engine_ms, mr.upstream_engine_iapi_ttft_ms, mr.upstream_engine_service_ttft_ms, mr.upstream_engine_iapi_tbt_ms, mr.upstream_engine_service_tbt_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.upstream_api_overhead_ms, mr.upstream_engine_ms, mr.latency_ms, mr.admission_decision_ms,
|
||||
mr.account_selection_wait_ms, mr.capacity_used_slots, mr.capacity_total_slots,
|
||||
host(mr.client_ip) as client_ip,
|
||||
mr.user_agent, mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
|
||||
@@ -61,7 +61,9 @@ impl RetentionStore for PgRetentionRepository {
|
||||
select ctid from model_requests
|
||||
where outcome <> 'running'
|
||||
and completed_at < $1 - ($2 * interval '1 day')
|
||||
order by completed_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
RetentionTarget::OpsEvents => {
|
||||
@@ -70,7 +72,9 @@ impl RetentionStore for PgRetentionRepository {
|
||||
select ctid from ops_events
|
||||
where model_request_id is null
|
||||
and created_at < $1 - ($2 * interval '1 day')
|
||||
order by created_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
RetentionTarget::AdminAuditEvents => {
|
||||
@@ -78,18 +82,39 @@ impl RetentionStore for PgRetentionRepository {
|
||||
where ctid in (
|
||||
select ctid from admin_audit_events
|
||||
where created_at < $1 - ($2 * interval '1 day')
|
||||
order by created_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
};
|
||||
sqlx::query(sql)
|
||||
// 跳过正在写入的父行;级联与投影触发器仍可能等锁,单批需有服务端截止时间
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|source| unavailable("begin retention batch", source))?;
|
||||
sqlx::query("set local statement_timeout = '5s'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| unavailable("set retention statement timeout", source))?;
|
||||
sqlx::query("set local lock_timeout = '500ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| unavailable("set retention lock timeout", source))?;
|
||||
let deleted = sqlx::query(sql)
|
||||
.bind(now)
|
||||
.bind(i64::from(policy.days(target)))
|
||||
.bind(i64::from(limit.get()))
|
||||
.execute(&self.pool)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map(|result| result.rows_affected())
|
||||
.map_err(|source| unavailable("delete expired records", source))
|
||||
.map_err(|source| unavailable("delete expired records", source))?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|source| unavailable("commit retention batch", source))?;
|
||||
Ok(deleted)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use gateway_core::task::DaemonTask;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
const COMMAND_DRAIN_TIMEOUT: Duration = Duration::from_secs(4);
|
||||
|
||||
@@ -325,12 +326,16 @@ pub(crate) fn request_analytics_worker(
|
||||
Ok(WorkerContribution::Registration(scheduled_worker(
|
||||
id,
|
||||
Duration::from_secs(5),
|
||||
Box::new(RequestAnalyticsTask { repository }),
|
||||
Box::new(RequestAnalyticsTask {
|
||||
repository,
|
||||
cycle: RequestAnalyticsCycle::default(),
|
||||
}),
|
||||
)?))
|
||||
}
|
||||
|
||||
struct RequestAnalyticsTask {
|
||||
repository: postgres::PgRequestAnalyticsRepository,
|
||||
cycle: RequestAnalyticsCycle,
|
||||
}
|
||||
|
||||
impl ScheduledTask for RequestAnalyticsTask {
|
||||
@@ -341,7 +346,7 @@ impl ScheduledTask for RequestAnalyticsTask {
|
||||
Box::pin(async move {
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => Ok(()),
|
||||
result = run_request_analytics_cycle(&self.repository) => result.map(|_| ()).map_err(|source| {
|
||||
result = self.cycle.run(&self.repository) => result.map(|_| ()).map_err(|source| {
|
||||
WorkerTaskError::safe("request analytics maintenance failed").with_source(source)
|
||||
}),
|
||||
}
|
||||
@@ -349,43 +354,107 @@ impl ScheduledTask for RequestAnalyticsTask {
|
||||
}
|
||||
}
|
||||
|
||||
/// 生产与隔离压测共用的单周期推进预算;各批独立提交并归还观测槽位
|
||||
pub async fn run_request_analytics_cycle(
|
||||
repository: &postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<postgres::AnalyticsMaintenance> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut total = postgres::AnalyticsMaintenance::default();
|
||||
loop {
|
||||
let result = match tokio::time::timeout_at(deadline, repository.maintain(1_000)).await {
|
||||
Ok(result) => result?,
|
||||
Err(source) => {
|
||||
if total.projected_requests == 0
|
||||
&& total.backfilled_requests == 0
|
||||
&& total.rebuilt_buckets == 0
|
||||
&& total.advanced_buckets == 0
|
||||
&& !total.reset_progress
|
||||
{
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "request analytics cycle timed out without progress".to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
});
|
||||
}
|
||||
return Ok(total);
|
||||
}
|
||||
};
|
||||
let idle = result.projected_requests == 0
|
||||
&& result.backfilled_requests == 0
|
||||
&& result.rebuilt_buckets == 0
|
||||
&& result.advanced_buckets == 0
|
||||
&& !result.reset_progress;
|
||||
total.projected_requests += result.projected_requests;
|
||||
total.backfilled_requests += result.backfilled_requests;
|
||||
total.rebuilt_buckets += result.rebuilt_buckets;
|
||||
total.advanced_buckets += result.advanced_buckets;
|
||||
total.reset_progress |= result.reset_progress;
|
||||
if idle || tokio::time::Instant::now() >= deadline {
|
||||
return Ok(total);
|
||||
/// 统计消费周期预算,跨轮保留批量上限以便超时后的下一轮提交进展
|
||||
pub struct RequestAnalyticsCycle {
|
||||
batch_size: AtomicU32,
|
||||
}
|
||||
|
||||
impl Default for RequestAnalyticsCycle {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
batch_size: AtomicU32::new(1_000),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestAnalyticsCycle {
|
||||
/// 生产与隔离压测共用的推进路径;各批独立提交并归还观测槽位
|
||||
pub async fn run(
|
||||
&self,
|
||||
repository: &postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<postgres::AnalyticsMaintenance> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut total = postgres::AnalyticsMaintenance::default();
|
||||
loop {
|
||||
let batch_size = self.batch_size.load(Ordering::Relaxed);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result =
|
||||
match tokio::time::timeout_at(deadline, repository.maintain(batch_size)).await {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(error)) => {
|
||||
if analytics_maintenance_timed_out(&error) {
|
||||
self.reduce_batch_size(batch_size);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Err(source) => {
|
||||
if total.projected_requests == 0
|
||||
&& total.backfilled_requests == 0
|
||||
&& total.rebuilt_buckets == 0
|
||||
&& total.advanced_buckets == 0
|
||||
&& !total.reset_progress
|
||||
{
|
||||
// 只在整轮没有返回进展时退让,避免尾批剩余时间短导致持续缩批
|
||||
self.reduce_batch_size(batch_size);
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "request analytics cycle timed out without progress"
|
||||
.to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
});
|
||||
}
|
||||
return Ok(total);
|
||||
}
|
||||
};
|
||||
let idle = result.projected_requests == 0
|
||||
&& result.backfilled_requests == 0
|
||||
&& result.rebuilt_buckets == 0
|
||||
&& result.advanced_buckets == 0
|
||||
&& !result.reset_progress;
|
||||
total.projected_requests += result.projected_requests;
|
||||
total.backfilled_requests += result.backfilled_requests;
|
||||
total.rebuilt_buckets += result.rebuilt_buckets;
|
||||
total.advanced_buckets += result.advanced_buckets;
|
||||
total.reset_progress |= result.reset_progress;
|
||||
// 满批快速完成才恢复吞吐,空结果也可能只是其他实例持有维护锁
|
||||
if (result.projected_requests >= u64::from(batch_size)
|
||||
|| result.backfilled_requests >= u64::from(batch_size))
|
||||
&& started.elapsed() < Duration::from_secs(2)
|
||||
{
|
||||
self.batch_size
|
||||
.store((batch_size * 2).min(1_000), Ordering::Relaxed);
|
||||
}
|
||||
if idle {
|
||||
return Ok(total);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_batch_size(&self, batch_size: u32) {
|
||||
let next_batch_size = (batch_size / 2).max(1);
|
||||
self.batch_size.store(next_batch_size, Ordering::Relaxed);
|
||||
tracing::warn!(
|
||||
batch_size,
|
||||
next_batch_size,
|
||||
"request analytics batch timed out"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn analytics_maintenance_timed_out(error: &StoreError) -> bool {
|
||||
let StoreError::Unavailable {
|
||||
source: Some(source),
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
source.is::<tokio::time::error::Elapsed>()
|
||||
|| source
|
||||
.downcast_ref::<sqlx::Error>()
|
||||
.and_then(sqlx::Error::as_database_error)
|
||||
.is_some_and(|error| error.code().as_deref() == Some("57014"))
|
||||
}
|
||||
|
||||
@@ -5,11 +5,13 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::postgres::{TestDatabase, drain_analytics};
|
||||
use gateway_admin::model::backup::code;
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpRequest};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_store::backup::{pg_dump::PgDumpAdapter, staging::StagingArea};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
#[tokio::test]
|
||||
async fn dump_process_lifecycle() {
|
||||
@@ -41,14 +43,31 @@ async fn dump_process_lifecycle() {
|
||||
assert!(status.success());
|
||||
return;
|
||||
};
|
||||
let Some(database) = TestDatabase::create("dump_lifecycle").await else {
|
||||
return;
|
||||
};
|
||||
drain_analytics(&database.pool).await;
|
||||
let directory = Path::new(&directory);
|
||||
let staging = Arc::new(StagingArea::open(directory.join("staging"), 64).unwrap());
|
||||
let dump = |mode: &str| PgDumpAdapter::new(staging.clone(), mode, "test-password");
|
||||
let staging = Arc::new(StagingArea::open(directory.join("staging"), 1024 * 1024).unwrap());
|
||||
let dump = |mode: &str| {
|
||||
PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
mode,
|
||||
"test-password",
|
||||
database.pool.clone(),
|
||||
)
|
||||
};
|
||||
|
||||
let artifact = dump("success").dump(request("success")).await.unwrap();
|
||||
assert_eq!(std::fs::read(&artifact.path).unwrap(), b"archive");
|
||||
assert_eq!(artifact.size_bytes, 7);
|
||||
assert_eq!(artifact.sha256, hex::encode(Sha256::digest(b"archive")));
|
||||
let compressed = std::fs::read(&artifact.path).unwrap();
|
||||
let mut decoded = String::new();
|
||||
async_compression::tokio::bufread::GzipDecoder::new(compressed.as_slice())
|
||||
.read_to_string(&mut decoded)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(decoded.contains("\narchive\n"));
|
||||
assert_eq!(artifact.size_bytes, compressed.len() as u64);
|
||||
assert_eq!(artifact.sha256, hex::encode(Sha256::digest(&compressed)));
|
||||
assert!(!staging.partial_path("success").exists());
|
||||
assert_eq!(
|
||||
dump("success")
|
||||
@@ -65,10 +84,15 @@ async fn dump_process_lifecycle() {
|
||||
assert!(!staging.partial_path("fail").exists());
|
||||
|
||||
let small = Arc::new(StagingArea::open(directory.join("small"), 1).unwrap());
|
||||
let error = PgDumpAdapter::new(small.clone(), "success", "test-password")
|
||||
.dump(request("oversized"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
let error = PgDumpAdapter::new(
|
||||
small.clone(),
|
||||
"success",
|
||||
"test-password",
|
||||
database.pool.clone(),
|
||||
)
|
||||
.dump(request("oversized"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), code::STAGING_SPACE_EXHAUSTED);
|
||||
assert!(!small.partial_path("oversized").exists());
|
||||
|
||||
@@ -119,13 +143,21 @@ async fn dump_process_lifecycle() {
|
||||
artifact.path.exists(),
|
||||
"completed archives remain available for recovery"
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_reports_failure_and_still_removes_the_other_archive() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let staging = Arc::new(StagingArea::open(directory.path().to_path_buf(), 64).unwrap());
|
||||
let adapter = PgDumpAdapter::new(staging.clone(), "unused", "unused");
|
||||
let adapter = PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
"unused",
|
||||
"unused",
|
||||
sqlx::postgres::PgPoolOptions::new()
|
||||
.connect_lazy("postgres://localhost/unused")
|
||||
.unwrap(),
|
||||
);
|
||||
adapter.cleanup_staging("missing").await.unwrap();
|
||||
std::fs::create_dir(staging.partial_path("blocked")).unwrap();
|
||||
std::fs::write(staging.final_path("blocked"), b"archive").unwrap();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证只读检查 Bundle 不执行迁移且拒绝业务写入
|
||||
//! 验证 Store 认证、凭据编码与只读检查 Bundle 的写入边界
|
||||
|
||||
use gateway_core::{
|
||||
account::{
|
||||
@@ -33,11 +33,6 @@ async fn inspection_bundle_does_not_migrate_and_rejects_business_writes() {
|
||||
database
|
||||
.query_pairs_mut()
|
||||
.append_pair("options", &format!("-csearch_path={schema}"));
|
||||
let config_connection = |mut url: url::Url| {
|
||||
let password = url.password().unwrap().to_owned();
|
||||
url.set_password(None).unwrap();
|
||||
json!({"url":url.as_str(),"password":password})
|
||||
};
|
||||
let mut config: StoreConfig = serde_json::from_value(json!({
|
||||
"database":config_connection(database), "redis":config_connection(url::Url::parse(&redis).unwrap()),
|
||||
})).unwrap();
|
||||
@@ -109,3 +104,136 @@ async fn inspection_bundle_does_not_migrate_and_rejects_business_writes() {
|
||||
.unwrap();
|
||||
admin.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_bundle_authenticates_configured_and_redis_acl_credentials() {
|
||||
let (Some(database), Some(redis_url)) = (
|
||||
crate::support::test_env("CPR_TEST_DATABASE_URL"),
|
||||
crate::support::test_env("CPR_TEST_REDIS_URL"),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let document = json!({
|
||||
"database": config_connection(url::Url::parse(&database).unwrap()),
|
||||
"redis": config_connection(url::Url::parse(&redis_url).unwrap()),
|
||||
});
|
||||
let mut admin = redis::Client::open(redis_url.as_str())
|
||||
.unwrap()
|
||||
.get_connection_manager()
|
||||
.await
|
||||
.unwrap();
|
||||
let username = format!("cpr_password_{}", uuid::Uuid::new_v4().simple());
|
||||
let password = "Managed%41%25%FF@:/?#%&+$'\"\\ secret9!";
|
||||
redis::cmd("ACL")
|
||||
.arg("SETUSER")
|
||||
.arg(&username)
|
||||
.arg("on")
|
||||
.arg(format!(">{password}"))
|
||||
.arg("~*")
|
||||
.arg("&*")
|
||||
.arg("+@all")
|
||||
.query_async::<()>(&mut admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut acl_url = url::Url::parse(&redis_url).unwrap();
|
||||
acl_url.set_username(&username).unwrap();
|
||||
acl_url.set_password(None).unwrap();
|
||||
let mut acl_document = document.clone();
|
||||
acl_document["redis"] = json!({"url": acl_url.as_str(), "password": password});
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let mut results = Vec::new();
|
||||
// 同时走 Store 的配置解析、URL 构造与真实连接路径,断言前先清理临时 ACL 用户
|
||||
for document in [document, acl_document] {
|
||||
let mut config: StoreConfig = serde_json::from_value(document).unwrap();
|
||||
let result = async {
|
||||
config.resolve_and_validate(directory.path())?;
|
||||
let bundle = gateway_store::initialize_read_only(config).await?;
|
||||
let mut states = Vec::new();
|
||||
for probe in bundle.health_probes() {
|
||||
states.push(probe.check().await);
|
||||
}
|
||||
Ok::<_, gateway_store::StoreError>(states)
|
||||
}
|
||||
.await;
|
||||
results.push(result);
|
||||
}
|
||||
redis::cmd("ACL")
|
||||
.arg("DELUSER")
|
||||
.arg(&username)
|
||||
.query_async::<i64>(&mut admin)
|
||||
.await
|
||||
.unwrap();
|
||||
for result in results {
|
||||
let states = result.expect("Store credentials authenticate without double encoding");
|
||||
assert!(!states.is_empty());
|
||||
assert!(
|
||||
states
|
||||
.iter()
|
||||
.all(|state| *state == gateway_core::health::HealthState::Healthy)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_bundle_authenticates_postgres_password_with_percent_sequences() {
|
||||
let (Some(database), Some(redis_url)) = (
|
||||
crate::support::test_env("CPR_TEST_DATABASE_URL"),
|
||||
crate::support::test_env("CPR_TEST_REDIS_URL"),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let admin = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database)
|
||||
.await
|
||||
.unwrap();
|
||||
let username = format!("cpr_password_{}", uuid::Uuid::new_v4().simple());
|
||||
let password = "Managed%41%25%FFPassword9!";
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"CREATE ROLE {username} LOGIN PASSWORD '{password}'"
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut database_url = url::Url::parse(&database).unwrap();
|
||||
database_url.set_username(&username).unwrap();
|
||||
database_url.set_password(None).unwrap();
|
||||
let mut config: StoreConfig = serde_json::from_value(json!({
|
||||
"database": {"url": database_url.as_str(), "password": password},
|
||||
"redis": config_connection(url::Url::parse(&redis_url).unwrap()),
|
||||
}))
|
||||
.unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let result = async {
|
||||
config.resolve_and_validate(directory.path())?;
|
||||
let bundle = gateway_store::initialize_read_only(config).await?;
|
||||
let mut states = Vec::new();
|
||||
for probe in bundle.health_probes() {
|
||||
states.push(probe.check().await);
|
||||
}
|
||||
Ok::<_, gateway_store::StoreError>(states)
|
||||
}
|
||||
.await;
|
||||
// 连接失败时也先清理临时角色,避免回归断言留下测试账号
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!("DROP ROLE {username}")))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
admin.close().await;
|
||||
let states = result.expect("PostgreSQL password percent sequences remain literal");
|
||||
assert!(!states.is_empty());
|
||||
assert!(
|
||||
states
|
||||
.iter()
|
||||
.all(|state| *state == gateway_core::health::HealthState::Healthy)
|
||||
);
|
||||
}
|
||||
|
||||
fn config_connection(mut url: url::Url) -> serde_json::Value {
|
||||
let password = percent_encoding::percent_decode_str(url.password().unwrap())
|
||||
.decode_utf8()
|
||||
.expect("UTF-8 fixture password")
|
||||
.into_owned();
|
||||
url.set_password(None).unwrap();
|
||||
json!({"url": url.as_str(), "password": password})
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ fn store_config_should_derive_backup_staging_from_runtime_data_dir() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits() {
|
||||
fn store_pool_config_defaults_reserve_capacity_for_observability() {
|
||||
let pool: StorePoolConfig =
|
||||
serde_json::from_value(serde_json::json!({})).expect("default pool configuration");
|
||||
|
||||
@@ -37,13 +37,6 @@ fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits()
|
||||
.observability_max_connections(),
|
||||
40
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<StorePoolConfig>(serde_json::json!({
|
||||
"statement_timeout_seconds": 60,
|
||||
}))
|
||||
.unwrap(),
|
||||
pool,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -27,36 +27,6 @@ use super::TestDatabase;
|
||||
const MIXED_GROUP: &str = "grp_00000000000000000000000000000001";
|
||||
const EMPTY_GROUP: &str = "grp_00000000000000000000000000000002";
|
||||
|
||||
#[tokio::test]
|
||||
async fn fast_mode_migration_preserves_existing_group_choices() {
|
||||
let Some(database) = TestDatabase::create_through("group_fast_mode_upgrade", 21).await else {
|
||||
return;
|
||||
};
|
||||
for (id, disabled) in [(MIXED_GROUP, true), (EMPTY_GROUP, false)] {
|
||||
sqlx::query("insert into account_groups (id, name, color, disable_fast, created_at, updated_at) values ($1, $1, '#2563EBFF', $2, now(), now())")
|
||||
.bind(id)
|
||||
.bind(disabled)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let groups = sqlx::query_as::<_, (String, String)>(
|
||||
"select id, fast_mode from account_groups order by id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
groups,
|
||||
vec![
|
||||
(MIXED_GROUP.to_owned(), "disabled".to_owned()),
|
||||
(EMPTY_GROUP.to_owned(), "default".to_owned())
|
||||
]
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn groups_aggregate_cross_provider_members_and_key_bindings_without_multiplication() {
|
||||
let Some(database) = TestDatabase::create("account_group_aggregate").await else {
|
||||
@@ -840,20 +810,19 @@ async fn image_generation_group_updates_preserve_omitted_values_and_publish_snap
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_generation_migration_allows_existing_and_default_groups() {
|
||||
let Some(database) = TestDatabase::create_through("group_image_upgrade", 31).await else {
|
||||
async fn image_generation_defaults_to_allowed_for_new_groups() {
|
||||
let Some(database) = TestDatabase::create("group_image_default").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, $1, '#2563EBFF', now(), now())")
|
||||
.bind(MIXED_GROUP).execute(&database.pool).await.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, $1, '#2563EBFF', now(), now())")
|
||||
.bind(EMPTY_GROUP).execute(&database.pool).await.unwrap();
|
||||
let allowed =
|
||||
sqlx::query_scalar::<_, bool>("select allow_image_generation from account_groups")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(allowed, vec![true, true]);
|
||||
let allowed = sqlx::query_scalar::<_, bool>(
|
||||
"select allow_image_generation from account_groups where id = $1",
|
||||
)
|
||||
.bind(EMPTY_GROUP)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(allowed);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
//! 小时重建合并、分段快照失效与阻塞桶调度的真实数据库验收
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{ObservabilityRange, PgRequestAnalyticsRepository};
|
||||
use gateway_store::{
|
||||
RequestAnalyticsCycle,
|
||||
postgres::{ObservabilityRange, PgRequestAnalyticsRepository},
|
||||
};
|
||||
|
||||
use super::{
|
||||
TestDatabase, analytics::seed, drain_analytics, observability_query_budget,
|
||||
@@ -352,3 +355,64 @@ async fn incremental_summary_write_failure_rolls_back_facts_and_preserves_queue(
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn extrema_rebuild_reads_only_its_bucket_when_history_is_large() {
|
||||
let Some(database) = TestDatabase::create("analytics_rebuild_bucket_range").await else {
|
||||
return;
|
||||
};
|
||||
// 强制刷新只作用于当前连接,Worker 与计数读取必须复用同一条连接
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(database.pool.connect_options().as_ref().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
for number in 1..=3_i64 {
|
||||
let id = format!("rebuild_range_{number}");
|
||||
seed(&pool, &id, hour() + TimeDelta::minutes(number), &[]).await;
|
||||
sqlx::query("update model_requests set provider_account_ref='rebuild_range_account',request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',$2::bigint)) where id=$1")
|
||||
.bind(id).bind(number).execute(&pool).await.unwrap();
|
||||
}
|
||||
drain_analytics(&pool).await;
|
||||
// 只扩充其他时间桶的事实,规划器拿到真实行数后不能再为一个分组扫描全部历史
|
||||
sqlx::query("insert into request_analytics_facts select ('history_'||n)::text as id,(f.started_at-make_interval(hours=>n/1000+2))::timestamptz,f.completed_at-make_interval(hours=>n/1000+2),f.client_api_key_ref,f.provider_account_ref,f.routing_group_refs,f.operation,f.provider_kind,f.requested_model_id,f.upstream_model_id,f.outcome,f.client_status_code,f.upstream_status_code,f.client_transport,f.upstream_transport,f.attempt_count,null,null,null,f.recovered_at,f.downstream_committed_at,f.request_kind,f.image_generation_requested,f.image_generation_succeeded,f.error_kind,f.input_tokens,f.output_tokens,f.cached_input_tokens,f.cache_write_input_tokens,f.reasoning_output_tokens,f.image_input_tokens,f.image_output_tokens,f.total_tokens,f.cost_source,f.cost_currency,f.cost_amount,f.first_token_ms,f.latency_ms,f.admission_decision_ms,f.account_selection_wait_ms,f.upstream_response_ms,f.capacity_used_slots,f.capacity_total_slots,f.standard_cost_amount,f.no_cache_cost_amount,f.source_version,f.schema_version,f.ops_failure_count,f.ops_rate_limited_count,f.ops_auth_failure_count,f.ops_provider_5xx_count from request_analytics_facts f cross join generate_series(1,300000) n where f.id='rebuild_range_2'")
|
||||
.execute(&pool).await.unwrap();
|
||||
sqlx::query("analyze request_analytics_facts")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("delete from model_requests where id='rebuild_range_1'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = PgRequestAnalyticsRepository::new(pool.clone(), observability_query_budget());
|
||||
let scanned_before = sequentially_read_facts(&pool).await;
|
||||
RequestAnalyticsCycle::default()
|
||||
.run(&worker)
|
||||
.await
|
||||
.expect("extrema rebuild must finish within the cycle budget");
|
||||
// 耗时随机器和数据量变化,用顺序扫描读取的行数判断重建是否越出所在统计桶
|
||||
let scanned = sequentially_read_facts(&pool).await - scanned_before;
|
||||
assert!(
|
||||
scanned < 300_000,
|
||||
"extrema rebuild scanned {scanned} fact rows"
|
||||
);
|
||||
let status = worker.status().await.unwrap();
|
||||
assert_eq!((status.pending_requests, status.pending_buckets), (0, 0));
|
||||
let row: (i64, f64, f64) = sqlx::query_as("select (counts->>'request_count')::bigint,(distributions->'latency_ms'->>'minimum')::double precision,(distributions->'latency_ms'->>'maximum')::double precision from request_analytics_15m where collection='global' and bucket_start=$1")
|
||||
.bind(hour()).fetch_one(&pool).await.unwrap();
|
||||
assert_eq!(row, (2, 2.0, 3.0));
|
||||
pool.close().await;
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn sequentially_read_facts(pool: &sqlx::PgPool) -> i64 {
|
||||
sqlx::query("select pg_stat_force_next_flush()")
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar("select coalesce(seq_tup_read,0) from pg_stat_user_tables where relid='request_analytics_facts'::regclass")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
//! 验证统计升级保留历史事实与回填进度,并重建已有时间桶
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{TEST_MIGRATOR, TestDatabase};
|
||||
|
||||
#[tokio::test]
|
||||
async fn analytics_upgrade_preserves_facts_and_backfill_progress_and_requeues_buckets() {
|
||||
let Some(database) = TestDatabase::create_through("analytics_upgrade", 29).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::raw_sql(
|
||||
"insert into request_analytics_facts(id,started_at,source_version) values
|
||||
('first','2026-10-01 10:01:00+00',1),
|
||||
('same_bucket','2026-10-01 10:02:00+00',2),
|
||||
('next_bucket','2026-10-01 10:16:00+00',3);
|
||||
insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at)
|
||||
values ('2026-10-01 10:00:00+00','2026-10-01 09:00:00+00');
|
||||
update request_analytics_build_state set
|
||||
backfill_complete=true, backfill_cursor_id='next_bucket',
|
||||
backfill_cursor_started_at='2026-10-01 10:16:00+00',
|
||||
coverage_start='2026-10-01 10:00:00+00',
|
||||
coverage_end='2026-10-01 10:30:00+00', generation=7;",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let facts_before: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(f) from request_analytics_facts f order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let progress_before: Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from request_analytics_build_state s")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let pending_before: (i64, DateTime<Utc>) = sqlx::query_as(
|
||||
"select change_version,oldest_pending_at from request_analytics_pending_buckets",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
|
||||
let facts_after: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(f) from request_analytics_facts f order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let progress_after: Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from request_analytics_build_state s")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(facts_after, facts_before);
|
||||
assert_eq!(progress_after, progress_before);
|
||||
let pending: Vec<(DateTime<Utc>, i64, DateTime<Utc>)> = sqlx::query_as(
|
||||
"select bucket_start,change_version,oldest_pending_at
|
||||
from request_analytics_pending_buckets order by bucket_start",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let expected_buckets = ["2026-10-01T10:00:00Z", "2026-10-01T10:15:00Z"]
|
||||
.map(|value| value.parse::<DateTime<Utc>>().unwrap());
|
||||
assert_eq!(
|
||||
pending.iter().map(|row| row.0).collect::<Vec<_>>(),
|
||||
expected_buckets
|
||||
);
|
||||
assert!(pending[0].1 > pending_before.0);
|
||||
assert_eq!(pending[0].2, pending_before.1);
|
||||
|
||||
// 重启不会再次入队,也不会覆盖最早待处理时间
|
||||
TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let pending_after_restart: Vec<(DateTime<Utc>, i64, DateTime<Utc>)> = sqlx::query_as(
|
||||
"select bucket_start,change_version,oldest_pending_at
|
||||
from request_analytics_pending_buckets order by bucket_start",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pending_after_restart, pending);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! 用实际 PostgreSQL 工具验收业务备份、恢复与恢复后的统计续算
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort as _, DumpRequest};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_store::backup::{pg_dump::PgDumpAdapter, staging::StagingArea};
|
||||
use gateway_store::postgres::ObservabilityRange;
|
||||
use std::{process::Stdio, sync::Arc};
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
use super::{TestDatabase, analytics::seed, drain_analytics, observability_repository};
|
||||
|
||||
#[tokio::test]
|
||||
async fn business_backup_restores_core_state_and_usage_without_request_history() {
|
||||
let Some(database) = TestDatabase::create("business_backup").await else {
|
||||
return;
|
||||
};
|
||||
let hour = "2026-10-15T12:00:00Z"
|
||||
.parse::<chrono::DateTime<Utc>>()
|
||||
.unwrap();
|
||||
for id in ["native_a", "native_b"] {
|
||||
seed(&database.pool, id, hour + TimeDelta::minutes(5), &[]).await;
|
||||
}
|
||||
sqlx::raw_sql("insert into client_api_keys(id,name,key,created_at,updated_at) values('restore_key','restore key','sk_backup_test',now(),now());
|
||||
insert into client_api_key_budget_windows values('restore_key',now(),now()+interval '1 day',now(),now()+interval '7 days',2.5,9.5);
|
||||
insert into client_api_key_charge_events(request_id,client_api_key_id,amount_usd,completed_at) values('native_a','restore_key',2.5,now());
|
||||
insert into plugin_update_sources(plugin_id,source_json) values('backup.test','{}');
|
||||
insert into plugin_artifacts(sha256,plugin_id,version,metadata_json,source_json,archive,accepted_at) values(repeat('a',64),'backup.test','1.0.0','{}','{}',decode('010203','hex'),now());
|
||||
update runtime_settings set usage_retention_days=3;
|
||||
update model_requests set cost_source='provider_reported',cost_amount=0.5,cost_currency='USD';
|
||||
insert into request_analytics_1h(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions)
|
||||
values('2026-10-15T12:00:00Z','usage_contribution',array['plugin','model','','','','','maintained'],'',0,0,'{\"request_count\":7,\"total_tokens\":900}','{\"USD\":\"1.25\"}','{}')")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let range = ObservabilityRange::new(hour, hour + TimeDelta::hours(1)).unwrap();
|
||||
let before = observability_repository(&database.pool)
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(before.requests.request_count, 9);
|
||||
assert_eq!(before.requests.total_tokens, 1140);
|
||||
assert_eq!(before.attempts.costs[0].currency, "USD");
|
||||
assert_eq!(before.attempts.costs[0].amount.as_str(), "2.25");
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let staging =
|
||||
Arc::new(StagingArea::open(directory.path().to_owned(), 8 * 1024 * 1024).unwrap());
|
||||
let mut url = url::Url::parse(&std::env::var("CPR_TEST_DATABASE_URL").unwrap()).unwrap();
|
||||
let password = percent_encoding::percent_decode_str(url.password().unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.unwrap()
|
||||
.into_owned();
|
||||
url.set_password(None).unwrap();
|
||||
let adapter = PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
url.to_string(),
|
||||
password.clone(),
|
||||
database.pool.clone(),
|
||||
);
|
||||
sqlx::query("update request_analytics_build_state set backfill_complete=false")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = adapter
|
||||
.dump(DumpRequest {
|
||||
backup_id: "rebuilding".to_owned(),
|
||||
cancellation: CancellationToken::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
gateway_admin::model::backup::code::ANALYTICS_NOT_READY
|
||||
);
|
||||
assert!(!staging.partial_path("rebuilding").exists());
|
||||
sqlx::query("update request_analytics_build_state set backfill_complete=true")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (backup_id, expected_requests, expected_tokens) in
|
||||
[("first", 9, 1140), ("second", 10, 1260)]
|
||||
{
|
||||
let artifact = adapter
|
||||
.dump(DumpRequest {
|
||||
backup_id: backup_id.to_owned(),
|
||||
cancellation: CancellationToken::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let file = tokio::io::BufReader::new(tokio::fs::File::open(&artifact.path).await.unwrap());
|
||||
let mut sql = Vec::new();
|
||||
async_compression::tokio::bufread::GzipDecoder::new(file)
|
||||
.read_to_end(&mut sql)
|
||||
.await
|
||||
.unwrap();
|
||||
let schema: String = sqlx::query_scalar("select current_schema()")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 仅清空本测试随机 schema,模拟新库预先存在 public 的状态
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"drop schema \"{schema}\" cascade; create schema \"{schema}\""
|
||||
)))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut process = tokio::process::Command::new("psql")
|
||||
.args([
|
||||
"-X",
|
||||
"--single-transaction",
|
||||
"--set=ON_ERROR_STOP=1",
|
||||
"--dbname",
|
||||
url.as_str(),
|
||||
])
|
||||
.env("PGPASSWORD", &password)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("PostgreSQL 18 psql must be in PATH");
|
||||
process.stdin.take().unwrap().write_all(&sql).await.unwrap();
|
||||
let output = process.wait_with_output().await.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let retained: (i64,i64,i64) = sqlx::query_as("select (select count(*) from model_requests),(select count(*) from request_analytics_facts),(select count(*) from client_api_key_charge_events)")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(retained, (0, 0, 0));
|
||||
let budget: (String,String) = sqlx::query_as("select daily_used_usd::text,weekly_used_usd::text from client_api_key_budget_windows where client_api_key_id='restore_key'")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(
|
||||
budget,
|
||||
("2.5000000000".to_owned(), "9.5000000000".to_owned())
|
||||
);
|
||||
let archive: Vec<u8> = sqlx::query_scalar(
|
||||
"select archive from plugin_artifacts where plugin_id='backup.test'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(archive, [1, 2, 3]);
|
||||
let retention_days: i64 =
|
||||
sqlx::query_scalar("select usage_retention_days from runtime_settings")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(retention_days, 3);
|
||||
drain_analytics(&database.pool).await;
|
||||
let restored = observability_repository(&database.pool)
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.requests.request_count, expected_requests);
|
||||
assert_eq!(restored.requests.total_tokens, expected_tokens);
|
||||
assert_eq!(restored.attempts.costs, before.attempts.costs);
|
||||
if backup_id == "first" {
|
||||
seed(
|
||||
&database.pool,
|
||||
"after_restore",
|
||||
hour + TimeDelta::minutes(10),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
drain_analytics(&database.pool).await;
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//! PostgreSQL 连接错误、迁移应用与事务归属测试
|
||||
//! PostgreSQL 连接初始化、会话设置与事务取消测试
|
||||
|
||||
use std::error::Error as _;
|
||||
|
||||
@@ -109,32 +109,32 @@ async fn invalid_postgres_options_preserve_the_sqlx_source() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly() {
|
||||
async fn connection_initialization_preserves_settings_when_reopened() {
|
||||
let Some(database_url) = crate::support::test_env("CPR_TEST_DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let database = format!("cpr_store_migrator_{}", Uuid::new_v4().simple());
|
||||
let database = format!("cpr_store_connection_{}", Uuid::new_v4().simple());
|
||||
let admin = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect migration test PostgreSQL");
|
||||
.expect("connect test PostgreSQL");
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"create database \"{database}\""
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("create migration test database");
|
||||
.expect("create connection test database");
|
||||
|
||||
let isolated_url = PgConnectOptions::from_str(&database_url)
|
||||
.expect("parse migration test PostgreSQL URL")
|
||||
.expect("parse test PostgreSQL URL")
|
||||
.database(&database)
|
||||
.to_url_lossy()
|
||||
.to_string();
|
||||
let pool_config = gateway_store::StorePoolConfig::default();
|
||||
let first = connect_and_migrate(&isolated_url, pool_config)
|
||||
.await
|
||||
.expect("apply migrations through production migrator");
|
||||
.expect("initialize production connection");
|
||||
let session_settings = sqlx::query_as::<_, (String, i64, i64, i64)>(
|
||||
"select current_setting('application_name'),
|
||||
extract(epoch from current_setting('statement_timeout')::interval)::bigint,
|
||||
@@ -154,70 +154,20 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
timezone, "UTC",
|
||||
"runtime sessions use an explicit technical time basis"
|
||||
);
|
||||
let first_tables = sqlx::query_scalar::<_, String>(
|
||||
"select table_name
|
||||
from information_schema.tables
|
||||
where table_schema = 'public'
|
||||
order by table_name",
|
||||
)
|
||||
.fetch_all(&first)
|
||||
.await
|
||||
.expect("load migrated tables");
|
||||
sqlx::query("update runtime_settings set refresh_margin_seconds = 1800 where id = 1")
|
||||
.execute(&first)
|
||||
.await
|
||||
.expect("save custom runtime setting");
|
||||
first.close().await;
|
||||
|
||||
let second = connect_and_migrate(&isolated_url, pool_config)
|
||||
.await
|
||||
.expect("reopen database through production migrator");
|
||||
let migration_count =
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from _sqlx_migrations where success")
|
||||
.expect("reopen production connection");
|
||||
let refresh_margin: i64 =
|
||||
sqlx::query_scalar("select refresh_margin_seconds from runtime_settings where id = 1")
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("count successful migrations");
|
||||
let response_id_types = sqlx::query_scalar::<_, String>(
|
||||
"select data_type
|
||||
from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'model_requests'
|
||||
and column_name in ('client_response_id', 'upstream_response_id')
|
||||
order by column_name",
|
||||
)
|
||||
.fetch_all(&second)
|
||||
.await
|
||||
.expect("load opaque response ID column types");
|
||||
let raw_response_id_index_exists = sqlx::query_scalar::<_, bool>(
|
||||
"select exists (
|
||||
select 1
|
||||
from pg_indexes
|
||||
where schemaname = 'public'
|
||||
and indexname = 'model_requests_client_response_uq'
|
||||
)",
|
||||
)
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("check removed raw response ID index");
|
||||
let legacy_key_provider_column_exists = sqlx::query_scalar::<_, bool>(
|
||||
"select exists (
|
||||
select 1 from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'client_api_keys'
|
||||
and column_name = 'provider_kind'
|
||||
)",
|
||||
)
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("check removed client key provider column");
|
||||
let routing_history_columns = sqlx::query_scalar::<_, String>(
|
||||
"select column_name from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'model_requests'
|
||||
and column_name in (
|
||||
'routing_scope', 'routing_group_refs', 'routing_group_names_snapshot'
|
||||
)
|
||||
order by column_name",
|
||||
)
|
||||
.fetch_all(&second)
|
||||
.await
|
||||
.expect("load routing history columns");
|
||||
.expect("load saved runtime setting");
|
||||
second.close().await;
|
||||
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
@@ -225,72 +175,9 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("drop migration test database");
|
||||
.expect("drop connection test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!(
|
||||
first_tables,
|
||||
[
|
||||
"_sqlx_migrations",
|
||||
"account_group_accounts",
|
||||
"account_groups",
|
||||
"admin_audit_events",
|
||||
"admin_users",
|
||||
"backup_records",
|
||||
"backup_settings",
|
||||
"client_api_key_budget_windows",
|
||||
"client_api_key_charge_events",
|
||||
"client_api_key_groups",
|
||||
"client_api_keys",
|
||||
"model_request_leases",
|
||||
"model_request_observations",
|
||||
"model_requests",
|
||||
"ops_events",
|
||||
"outbound_proxies",
|
||||
"plugin_account_group_resources",
|
||||
"plugin_artifact_credentials",
|
||||
"plugin_artifact_platforms",
|
||||
"plugin_artifacts",
|
||||
"plugin_client_api_key_resources",
|
||||
"plugin_instance_secrets",
|
||||
"plugin_instances",
|
||||
"plugin_source_credentials",
|
||||
"plugin_state_generations",
|
||||
"plugin_state_records",
|
||||
"plugin_update_sources",
|
||||
"plugin_version_configurations",
|
||||
"provider_accounts",
|
||||
"provider_authorization_receipts",
|
||||
"request_analytics_15m",
|
||||
"request_analytics_1d_counts",
|
||||
"request_analytics_1h",
|
||||
"request_analytics_build_state",
|
||||
"request_analytics_facts",
|
||||
"request_analytics_pending_buckets",
|
||||
"request_analytics_pending_requests",
|
||||
"runtime_settings",
|
||||
]
|
||||
);
|
||||
assert_eq!(session_settings, ("codex-proxy-rs".to_owned(), 30, 5, 30));
|
||||
assert_eq!(
|
||||
migration_count,
|
||||
i64::try_from(TEST_MIGRATOR.iter().count())
|
||||
.expect("migration count fits PostgreSQL bigint")
|
||||
);
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
assert!(!legacy_key_provider_column_exists);
|
||||
assert!(routing_history_columns.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = TEST_MIGRATOR
|
||||
.iter()
|
||||
.flat_map(|migration| migration.sql.as_str().lines())
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
|
||||
assert_eq!(transaction_statements, 0);
|
||||
assert_eq!(refresh_margin, 1800, "reopening preserves custom settings");
|
||||
}
|
||||
|
||||
@@ -2168,10 +2168,6 @@ async fn shared_core_usage_and_timings_preserve_fields_and_reject_invalid_phases
|
||||
upstream_response_ms: Some(1_000),
|
||||
upstream_api_overhead_ms: Some(120.25),
|
||||
upstream_engine_ms: Some(6400.0),
|
||||
upstream_engine_iapi_ttft_ms: Some(650.5),
|
||||
upstream_engine_service_ttft_ms: Some(720.25),
|
||||
upstream_engine_iapi_tbt_ms: Some(18.45),
|
||||
upstream_engine_service_tbt_ms: Some(20.12),
|
||||
latency_ms: Some(9),
|
||||
};
|
||||
ExecutionStore::finalize_model_request(&store, finalization)
|
||||
@@ -2202,22 +2198,6 @@ async fn shared_core_usage_and_timings_preserve_fields_and_reject_invalid_phases
|
||||
}
|
||||
assert_eq!(persisted["upstream_api_overhead_ms"].as_f64(), Some(120.25));
|
||||
assert_eq!(persisted["upstream_engine_ms"].as_f64(), Some(6400.0));
|
||||
assert_eq!(
|
||||
persisted["upstream_engine_iapi_ttft_ms"].as_f64(),
|
||||
Some(650.5)
|
||||
);
|
||||
assert_eq!(
|
||||
persisted["upstream_engine_service_ttft_ms"].as_f64(),
|
||||
Some(720.25)
|
||||
);
|
||||
assert_eq!(
|
||||
persisted["upstream_engine_iapi_tbt_ms"].as_f64(),
|
||||
Some(18.45)
|
||||
);
|
||||
assert_eq!(
|
||||
persisted["upstream_engine_service_tbt_ms"].as_f64(),
|
||||
Some(20.12)
|
||||
);
|
||||
let invalid_id = "req_invalid_phase";
|
||||
seed_running_request(&database.pool, invalid_id)
|
||||
.await
|
||||
@@ -2233,7 +2213,7 @@ async fn shared_core_usage_and_timings_preserve_fields_and_reject_invalid_phases
|
||||
assert_eq!(stored_row(&database.pool, invalid_id).await, before);
|
||||
for invalid_ms in [-1.0, f64::NAN, f64::INFINITY, i64::MAX as f64] {
|
||||
let mut invalid = successful_core_finalization(invalid_id);
|
||||
invalid.timings.upstream_engine_iapi_tbt_ms = Some(invalid_ms);
|
||||
invalid.timings.upstream_engine_ms = Some(invalid_ms);
|
||||
let error = ExecutionStore::finalize_model_request(&store, invalid)
|
||||
.await
|
||||
.expect_err("invalid official timing must not silently become missing");
|
||||
|
||||
@@ -18,8 +18,8 @@ mod admission_recovery;
|
||||
mod analytics;
|
||||
mod analytics_buckets;
|
||||
mod analytics_reset;
|
||||
mod analytics_upgrade;
|
||||
mod backup;
|
||||
mod backup_export;
|
||||
mod client_budgets;
|
||||
mod client_keys;
|
||||
mod connection;
|
||||
@@ -69,10 +69,6 @@ pub(super) fn admin_account_store(pool: &PgPool) -> PgAdminAccountStore {
|
||||
|
||||
impl TestDatabase {
|
||||
pub(super) async fn create(label: &str) -> Option<Self> {
|
||||
Self::create_through(label, i64::MAX).await
|
||||
}
|
||||
|
||||
pub(super) async fn create_through(label: &str, migration_version: i64) -> Option<Self> {
|
||||
let database_url = crate::support::test_env("CPR_TEST_DATABASE_URL")?;
|
||||
let schema = format!("cpr_store_{label}_{}", Uuid::new_v4().simple());
|
||||
// 临时 schema 验证事务可见性与回滚,不模拟 PostgreSQL 掉电恢复
|
||||
@@ -95,7 +91,7 @@ impl TestDatabase {
|
||||
.await
|
||||
.expect("connect isolated test schema");
|
||||
TEST_MIGRATOR
|
||||
.run_to(migration_version, &pool)
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("apply test migrations");
|
||||
Some(Self {
|
||||
|
||||
@@ -110,7 +110,7 @@ async fn output_throughput_uses_official_response_duration_without_local_fallbac
|
||||
'firstTokenMs', $2::bigint,
|
||||
'latencyMs', $3::bigint),
|
||||
'upstream', coalesce(request_observation_json #> '{timings,upstream}', '{}'::jsonb) || jsonb_build_object(
|
||||
'responseMs', $4::bigint, 'engineIapiTbtMs', 2.450638))))
|
||||
'responseMs', $4::bigint, 'engineMs', 2.450638))))
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.bind(output)
|
||||
@@ -150,8 +150,6 @@ async fn output_throughput_uses_official_response_duration_without_local_fallbac
|
||||
detail.request.upstream_response_ms,
|
||||
record.upstream_response_ms
|
||||
);
|
||||
assert_eq!(record.upstream_engine_iapi_tbt_ms, Some(2.450638));
|
||||
assert_eq!(detail.request.upstream_engine_iapi_tbt_ms, Some(2.450638));
|
||||
|
||||
super::drain_analytics(&database.pool).await;
|
||||
let summary = repository
|
||||
|
||||
@@ -1,58 +1,10 @@
|
||||
//! 观测视图升级兼容性与普通聚合的覆盖索引规划回归测试
|
||||
//! 验证观测视图普通聚合能够使用覆盖索引
|
||||
|
||||
use chrono::Utc;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{TestDatabase, seed_observability_facts};
|
||||
|
||||
#[tokio::test]
|
||||
async fn projection_upgrade_preserves_observation_values() {
|
||||
let Some(db) = TestDatabase::create_through("observation_projection_upgrade", 26).await else {
|
||||
return;
|
||||
};
|
||||
// 此测试固定覆盖 26 → 27 的冻结视图合同,不依赖最新版本的写入夹具
|
||||
sqlx::query("insert into model_requests(id,client_api_key_ref,operation,client_transport,outcome,client_status_code,started_at,deadline_at,completed_at,request_observation_json) values('req_observe_success','historical_key','responses','http_sse','succeeded',200,now()-interval '1 hour',now(),now(),'{\"request\":{\"configRevision\":1,\"protocol\":\"openai\",\"endpoint\":\"/v1/responses\",\"compact\":false},\"routing\":{\"scope\":\"all\",\"groupRefs\":[],\"groupNamesSnapshot\":[]},\"timings\":{\"local\":{\"latencyMs\":800}}}'::jsonb)")
|
||||
.execute(&db.pool).await.unwrap();
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set request_observation_json = jsonb_set(request_observation_json,
|
||||
'{timings,upstream}', '{\"responseMs\":700,\"engineIapiTbtMs\":2.450638}')
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let before: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(mr) from model_request_observations mr order by id")
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
super::super::TEST_MIGRATOR
|
||||
.run_to(27, &db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let after: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(mr) from model_request_observations mr order by id")
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"projection upgrade must preserve every field"
|
||||
);
|
||||
let updatable: String = sqlx::query_scalar(
|
||||
"select is_updatable from information_schema.views
|
||||
where table_schema = current_schema() and table_name = 'model_request_observations'",
|
||||
)
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(updatable, "NO", "observation projection remains read-only");
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plain_observation_aggregates_can_use_a_covering_index() {
|
||||
let Some(db) = TestDatabase::create("observation_covering_index").await else {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证插件制品身份、信任接受、迁移与安装幂等性
|
||||
//! 验证插件制品身份、信任接受与安装幂等性
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -83,12 +83,7 @@ async fn unknown_metadata_fields_do_not_change_artifact_identity() {
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(stored.get("permissionDescriptions").is_none());
|
||||
assert!(stored.get("requestedPermissions").is_none());
|
||||
let mut extended = stored;
|
||||
extended["permissionDescriptions"] =
|
||||
serde_json::json!([{"permission": "models", "label": "旧文案"}]);
|
||||
extended["requestedPermissions"] = serde_json::json!(["models"]);
|
||||
extended["futureMetadata"] = serde_json::json!({"enabled": true});
|
||||
extended["contributes"]["middleware"]["futureField"] = serde_json::json!(true);
|
||||
sqlx::query("update plugin_artifacts set metadata_json=$2 where sha256=$1")
|
||||
@@ -136,94 +131,6 @@ async fn unknown_metadata_fields_do_not_change_artifact_identity() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trust_migration_removes_only_obsolete_metadata_fields() {
|
||||
let Some(database) = TestDatabase::create_through("plugin_trust_upgrade", 19).await else {
|
||||
return;
|
||||
};
|
||||
initialize_revision(&database).await;
|
||||
let store = PgPluginStore::new(database.pool.clone());
|
||||
for (digest, extra) in [
|
||||
('a', serde_json::json!({"requestedPermissions": ["models"]})),
|
||||
(
|
||||
'b',
|
||||
serde_json::json!({"permissionDescriptions": ["旧文案"]}),
|
||||
),
|
||||
(
|
||||
'c',
|
||||
serde_json::json!({"requestedPermissions": [], "permissionDescriptions": []}),
|
||||
),
|
||||
('d', serde_json::json!({})),
|
||||
] {
|
||||
// 每个制品使用独立平台,保留同一版本的平台不可变约束
|
||||
let fixture = artifact(digest, &[&format!("test-{digest}")]);
|
||||
let installed = store
|
||||
.install_artifact(fixture.clone(), PluginSource::Upload, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
if digest == 'a' {
|
||||
store
|
||||
.accept_artifact(&fixture.metadata.sha256, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
sqlx::query("update plugin_artifacts set metadata_json=metadata_json || $2 || '{\"futureMetadata\":true}'::jsonb where sha256=$1")
|
||||
.bind(&fixture.metadata.sha256).bind(extra).execute(&database.pool).await.unwrap();
|
||||
sqlx::query("insert into plugin_instances(id,artifact_sha256,name,enabled,configuration_json,bindings_json,revision) values ($1,$2,'Example',false,'{}','[]',$3)")
|
||||
.bind(uuid::Uuid::now_v7()).bind(&fixture.metadata.sha256)
|
||||
.bind(i64::try_from(installed.config_revision.get()).unwrap())
|
||||
.execute(&database.pool).await.unwrap();
|
||||
}
|
||||
let mut expected: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(a) from plugin_artifacts a order by sha256")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for row in &mut expected {
|
||||
let metadata = row["metadata_json"].as_object_mut().unwrap();
|
||||
metadata.remove("requestedPermissions");
|
||||
metadata.remove("permissionDescriptions");
|
||||
}
|
||||
let instances: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(i) from plugin_instances i order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let revision: i64 =
|
||||
sqlx::query_scalar("select config_revision from runtime_settings where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for _ in 0..2 {
|
||||
super::super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let actual: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(a) from plugin_artifacts a order by sha256")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
actual, expected,
|
||||
"只清理两个旧字段,保留包体、接受事实与其他元数据"
|
||||
);
|
||||
let current_instances: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(i) from plugin_instances i order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(current_instances, instances);
|
||||
let snapshot = store.load_instances().await.unwrap();
|
||||
assert_eq!(snapshot.instances.len(), 4);
|
||||
assert_eq!(
|
||||
snapshot.config_revision.get(),
|
||||
u64::try_from(revision).unwrap()
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_acceptance_is_exact_and_idempotent() {
|
||||
let Some(database) = TestDatabase::create("plugin_artifact_acceptance").await else {
|
||||
@@ -262,51 +169,6 @@ async fn artifact_acceptance_is_exact_and_idempotent() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plugin_migration_preserves_main_settings_and_accepts_new_installations() {
|
||||
let Some(database) = TestDatabase::create_through("plugin_upgrade_from_main", 17).await else {
|
||||
return;
|
||||
};
|
||||
initialize_revision(&database).await;
|
||||
// 0022 会把仍为旧默认 3600 的 margin 收敛到 300;预设自定义值,
|
||||
// 让本测试继续验证「所有已自定义设置都原样保留」。
|
||||
sqlx::query("update runtime_settings set max_concurrent_per_account=0, refresh_margin_seconds=1800 where id=1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let before: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from runtime_settings s where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let after: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from runtime_settings s where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (key, value) in before.as_object().unwrap() {
|
||||
if key == "request_interval_ms" {
|
||||
assert!(after.get(key).is_none(), "removed setting {key}");
|
||||
continue;
|
||||
}
|
||||
assert_eq!(after.get(key), Some(value), "existing setting {key}");
|
||||
}
|
||||
let store = PgPluginStore::new(database.pool.clone());
|
||||
store
|
||||
.install_artifact(
|
||||
artifact('a', &["linux-x86_64"]),
|
||||
PluginSource::Upload,
|
||||
&context(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn source_identity_requires_publisher_and_name() {
|
||||
let Some(database) = TestDatabase::create("plugin_source_identity").await else {
|
||||
|
||||
@@ -1193,63 +1193,6 @@ async fn legacy_urls_join_one_catalog_entry_and_invalid_batch_rolls_back() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_backfills_shared_proxies_without_changing_credentials() {
|
||||
let Some(database) = TestDatabase::create_through("proxy_backfill", 4).await else {
|
||||
return;
|
||||
};
|
||||
// 从真实旧 schema 正向升级,不拆卸最新结构,避免遗漏后续增加的引用约束
|
||||
for id in ["acct_one", "acct_two", "acct_direct"] {
|
||||
let proxy_url = (id != "acct_direct").then_some("http://user:secret@127.0.0.1:8080/");
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id,provider_kind,name,authentication_kind,provider_credentials_json,
|
||||
has_refresh_token,credential_observed_at,created_at,updated_at,outbound_proxy_url
|
||||
) values ($1,'openai',$1,'oauth','{}',false,now(),now(),now(),$2)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(proxy_url)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
assert!(
|
||||
sqlx::query_scalar::<_, bool>("select bool_and(enabled) from outbound_proxies")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let accounts = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let inherited = gateway_core::account::ProviderAccountStore::list_accounts(&accounts)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(inherited.len(), 3);
|
||||
assert!(
|
||||
inherited
|
||||
.iter()
|
||||
.all(|account| account.request_location().is_none())
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from outbound_proxies")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(sqlx::query_scalar::<_, i64>("select count(*) from provider_accounts a join outbound_proxies p on a.outbound_proxy_id = p.id where a.outbound_proxy_url = p.proxy_url and a.credential_revision = 1").fetch_one(&database.pool).await.unwrap(), 2);
|
||||
assert!(
|
||||
sqlx::query_scalar::<_, Option<String>>(
|
||||
"select outbound_proxy_id from provider_accounts where id = 'acct_direct'"
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_proxy_location_projects_to_accounts_and_guards_refresh_results() {
|
||||
use gateway_core::account::{ProviderAccountStore, RequestLocation};
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
//! 验证短保留期升级、分批清理边界与统计异步收敛期间的查询可用性
|
||||
|
||||
use std::num::NonZeroU32;
|
||||
use std::{num::NonZeroU32, time::Duration};
|
||||
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use gateway_admin::{
|
||||
model::retention::{RetentionPolicy, RetentionTarget},
|
||||
ports::retention::RetentionStore as _,
|
||||
ports::{retention::RetentionStore as _, store::ObservabilityStore as _},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
ObservabilityRange, PgRequestAnalyticsRepository, PgRetentionRepository,
|
||||
use gateway_store::{
|
||||
RequestAnalyticsCycle,
|
||||
postgres::{
|
||||
ObservabilityQueryBudget, ObservabilityRange, PgRequestAnalyticsRepository,
|
||||
PgRetentionRepository,
|
||||
},
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -18,10 +22,7 @@ use super::{
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_projected() {
|
||||
use gateway_admin::{
|
||||
model::observability::{AnalyticsState, DashboardQuery, TimeRange},
|
||||
ports::store::ObservabilityStore as _,
|
||||
};
|
||||
use gateway_admin::model::observability::{AnalyticsState, DashboardQuery, TimeRange};
|
||||
|
||||
let Some(database) = TestDatabase::create("short_retention_readable").await else {
|
||||
return;
|
||||
@@ -183,6 +184,107 @@ async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_expired_rows_do_not_block_cleanup_and_are_revisited() {
|
||||
let Some(database) = TestDatabase::create("retention_locked").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for id in ["locked", "available"] {
|
||||
seed(&database.pool, id, now - ChronoDuration::days(40), &[]).await;
|
||||
}
|
||||
let mut writing = database.pool.begin().await.unwrap();
|
||||
sqlx::query("select id from model_requests where id='locked' for update")
|
||||
.execute(&mut *writing)
|
||||
.await
|
||||
.unwrap();
|
||||
let repository = PgRetentionRepository::new(database.pool.clone());
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
let deleted = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
repository.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cleanup must skip the occupied row")
|
||||
.unwrap();
|
||||
assert_eq!(deleted, 1);
|
||||
writing.commit().await.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap()
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocked_cascade_rolls_back_the_batch_and_respects_lock_deadline() {
|
||||
let Some(database) = TestDatabase::create("retention_cascade_deadline").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed(
|
||||
&database.pool,
|
||||
"parent",
|
||||
now - ChronoDuration::days(40),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("insert into ops_events(id,model_request_id,attempt_index,level,component,operation,failure_kind,message,created_at) values('child','parent',1,'warning','routing','responses','rate_limited','synthetic',$1)")
|
||||
.bind(now).execute(&database.pool).await.unwrap();
|
||||
let mut writing = database.pool.begin().await.unwrap();
|
||||
sqlx::query("select id from ops_events where id='child' for update")
|
||||
.execute(&mut *writing)
|
||||
.await
|
||||
.unwrap();
|
||||
let repository = PgRetentionRepository::new(database.pool.clone());
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(3),
|
||||
repository.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cascade lock must have a server deadline");
|
||||
assert!(result.is_err());
|
||||
writing.rollback().await.unwrap();
|
||||
let remaining: i64 =
|
||||
sqlx::query_scalar("select count(*) from model_requests where id='parent'")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1);
|
||||
assert_eq!(
|
||||
repository
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap()
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_batch_deletes_only_expired_rows_up_to_limit() {
|
||||
let Some(database) = TestDatabase::create("retention_cycle_budget").await else {
|
||||
@@ -280,3 +382,271 @@ async fn empty_request_cleanup_does_not_conflict_with_analytics_publication() {
|
||||
publication.commit().await.unwrap();
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scattered_retention_deletions_commit_atomically_and_allow_new_requests_to_advance() {
|
||||
let Some(database) = TestDatabase::create("retention_scattered_groups").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..600 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30) + ChronoDuration::minutes(number * 20),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let worker = PgRequestAnalyticsRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(2, Duration::from_secs(5), Duration::from_secs(120))
|
||||
.unwrap(),
|
||||
);
|
||||
let before = worker.status().await.unwrap();
|
||||
let retention = PgRetentionRepository::new(database.pool.clone());
|
||||
assert_eq!(
|
||||
retention
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
RetentionPolicy::try_new(10, 30, 90).unwrap(),
|
||||
NonZeroU32::new(1000).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
600
|
||||
);
|
||||
seed(
|
||||
&database.pool,
|
||||
"new_request",
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
|
||||
// 小时层失败时,已经执行的多个刻钟删除批次与事实删除也必须一起回滚
|
||||
sqlx::raw_sql("create function reject_hour_deletion() returns trigger language plpgsql as $$begin raise exception 'test hour deletion failure'; end$$;
|
||||
create trigger reject_hour_deletion before delete on request_analytics_1h for each statement execute function reject_hour_deletion()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let cycle = RequestAnalyticsCycle::default();
|
||||
assert!(cycle.run(&worker).await.is_err());
|
||||
let failed = worker.status().await.unwrap();
|
||||
assert_eq!(failed.generation, before.generation);
|
||||
assert_eq!(failed.pending_requests, 601);
|
||||
let statistics = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap();
|
||||
assert_eq!(
|
||||
statistics
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap()
|
||||
.requests
|
||||
.request_count,
|
||||
600
|
||||
);
|
||||
sqlx::raw_sql("drop trigger reject_hour_deletion on request_analytics_1h; drop function reject_hour_deletion();
|
||||
create function slow_summary_statement() returns trigger language plpgsql as $$begin perform pg_sleep(0.002); return null; end$$;
|
||||
create trigger slow_quarter_delete before delete on request_analytics_15m for each statement execute function slow_summary_statement();
|
||||
create trigger slow_hour_delete before delete on request_analytics_1h for each statement execute function slow_summary_statement()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
|
||||
// 按语句注入延迟,覆盖大量分组的 SQL 往返成本及多个集合删除批次
|
||||
cycle
|
||||
.run(&worker)
|
||||
.await
|
||||
.expect("retention backlog must advance within the cycle budget");
|
||||
let after = worker.status().await.unwrap();
|
||||
assert_eq!(after.pending_requests, 0);
|
||||
assert_eq!(after.pending_buckets, 0);
|
||||
assert!(after.generation > before.generation);
|
||||
let actual = statistics
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
assert_eq!(actual.requests.total_tokens, 120);
|
||||
let timeline = admin_observability_store(&database.pool)
|
||||
.dashboard_summary(
|
||||
gateway_admin::model::observability::DashboardQuery::new(
|
||||
gateway_admin::model::observability::TimeRange {
|
||||
start: now - ChronoDuration::days(1),
|
||||
end: now,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap(),
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
timeline
|
||||
.trend
|
||||
.iter()
|
||||
.map(|point| point.metrics.request_count)
|
||||
.sum::<u64>(),
|
||||
1
|
||||
);
|
||||
let remaining: (i64, i64, i64, i64) = sqlx::query_as(
|
||||
"select
|
||||
(select count(*) from request_analytics_facts where id like 'expired_%'),
|
||||
(select count(*) from request_analytics_15m where bucket_start < $1),
|
||||
(select count(*) from request_analytics_1h where bucket_start < $1),
|
||||
(select count(*) from request_analytics_1d_counts where bucket_start < $1)",
|
||||
)
|
||||
.bind(now - ChronoDuration::days(10))
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, (0, 0, 0, 0));
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_requests_and_retention_deletions_both_advance_within_each_batch() {
|
||||
let Some(database) = TestDatabase::create("retention_fair_projection").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..20 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
PgRetentionRepository::new(database.pool.clone())
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
RetentionPolicy::try_new(10, 30, 90).unwrap(),
|
||||
NonZeroU32::new(1000).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
for wave in 1..=3 {
|
||||
for number in 0..2 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("new_{wave}_{number}"),
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
assert_eq!(worker.maintain(4).await.unwrap().projected_requests, 4);
|
||||
let facts: (i64, i64) = sqlx::query_as(
|
||||
"select count(*) filter (where id like 'expired_%'), count(*) filter (where id like 'new_%') from request_analytics_facts",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 历史队列尚未排空时,每批都须撤销旧贡献并发布本轮的新请求
|
||||
assert_eq!(facts, (20 - wave * 2, wave * 2));
|
||||
let current = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(1), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(current.requests.request_count, wave as u64 * 2);
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let final_summary = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(final_summary.requests.request_count, 6);
|
||||
assert_eq!(final_summary.requests.total_tokens, 720);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timed_out_retention_batches_shrink_across_cycles_and_lock_contention() {
|
||||
let Some(database) = TestDatabase::create("retention_batch_timeout").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..600 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::query("delete from model_requests")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
seed(
|
||||
&database.pool,
|
||||
"new_request",
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
// 数据库延迟与实际批量相关,小批量无需等待,以可观察进展验证跨轮退让
|
||||
sqlx::raw_sql("create function slow_large_fact_deletion() returns trigger language plpgsql as $$begin
|
||||
if (select count(*) from removed_facts) > 250 then perform pg_sleep(2); end if;
|
||||
return null; end$$;
|
||||
create trigger slow_large_fact_deletion after delete on request_analytics_facts referencing old table as removed_facts for each statement execute function slow_large_fact_deletion()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker = PgRequestAnalyticsRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(2, Duration::from_secs(1), Duration::from_secs(1))
|
||||
.unwrap(),
|
||||
);
|
||||
let cycle = RequestAnalyticsCycle::default();
|
||||
for _ in 0..2 {
|
||||
assert!(cycle.run(&worker).await.is_err());
|
||||
// 空轮可能来自别的实例持锁,不能因此恢复会反复超时的大批量
|
||||
let mut lock = database.pool.begin().await.unwrap();
|
||||
sqlx::query(
|
||||
"select pg_advisory_xact_lock(hashtextextended(current_schema()::text,382032))",
|
||||
)
|
||||
.execute(&mut *lock)
|
||||
.await
|
||||
.unwrap();
|
||||
let idle = cycle.run(&worker).await.unwrap();
|
||||
assert_eq!(idle.projected_requests, 0);
|
||||
lock.rollback().await.unwrap();
|
||||
}
|
||||
let pending = worker.status().await.unwrap().pending_requests;
|
||||
assert_eq!(pending, 601);
|
||||
// 此轮可以在后续批次超时前已有提交,下一轮重放仍须保持计数正确
|
||||
let _ = cycle.run(&worker).await;
|
||||
assert!(worker.status().await.unwrap().pending_requests < pending);
|
||||
for _ in 0..5 {
|
||||
// 与调度器错误重试一样留出退避时间,让已取消语句完成回滚并释放维护锁
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let _ = cycle.run(&worker).await;
|
||||
if worker.status().await.unwrap().pending_requests == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(worker.status().await.unwrap().pending_requests, 0);
|
||||
let actual = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
assert_eq!(actual.requests.total_tokens, 120);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user