mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-13 08:06:41 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af35cde374 | ||
|
|
5c115fb38f | ||
|
|
ed558e5396 | ||
|
|
929f781a63 | ||
|
|
e0326ecba4 | ||
|
|
6057088300 | ||
|
|
e3a49a6217 | ||
|
|
b32f87fa8a | ||
|
|
91e8105e5d | ||
|
|
c4d9576639 | ||
|
|
015cd60be1 | ||
|
|
bad2a245ab | ||
|
|
b49b4528e7 | ||
|
|
d678f350c1 | ||
|
|
17b05d4671 | ||
|
|
7148303ad6 | ||
|
|
f6538d433d | ||
|
|
406256bcc0 | ||
|
|
d912fa0c59 | ||
|
|
a166580aed | ||
|
|
d1aec7311c |
@@ -15,6 +15,7 @@ description: Codex Proxy RS 前后端开发、排障、代码审查和文档修
|
||||
| 修改服务端接口、路由、调度、账号与用量规则、持久化、后台任务或协议行为;目标是 `backend/` 中的实现、配置、迁移,或 Rust 源码 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)、所属模块、[后端自审](../../../docs/architecture.md#后端自审);涉及 Rust 时使用 `$rust-best-practices` 和下方后端注释要求,验证时读[命令](../../../docs/architecture.md#验证命令) |
|
||||
| 修改页面接口调用,或开发、排查、审查面向管理端或模型客户端的 HTTP/WS API 合同 | [API](../../../docs/api.md)对应章节;优先复用已有接口,核对调用方 → 服务端解析 → 实际查询或执行,不能只凭前端类型认定参数生效;按实际改动叠加前后端规范 |
|
||||
| 模块或调用链变更 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)及所属模块章节,不通读整份架构 |
|
||||
| 新增、调整、清理或审查测试 | 下方[测试维护](#测试维护),按当前行为、职责归属与已有覆盖判断保留、合并或删除 |
|
||||
| 修改部署行为、`deploy/` 配置或数据库迁移 | 按范围读取[部署](../../../deploy/README.md)、[迁移](../../../backend/migrations/README.md)对应章节 |
|
||||
| 开发、排查或审查宿主 SDK、Runtime、扩展接口或独立插件;目标在 `backend/crates/gateway-plugin/` | [插件职责边界](references/plugin-boundaries.md)及所用 [SDK](../../../backend/crates/gateway-plugin/sdk/README.md) 类型与能力合同;宿主守住自身合同、资源与稳定性,不猜测或补偿插件业务错误;仅独立插件创作、排查另用 [cpr-plugin-dev](../cpr-plugin-dev/SKILL.md) |
|
||||
| 修改或审查文档 | [文档检查](references/documentation.md)及目标章节,不默认读取代码开发、页面验收或发布流程 |
|
||||
@@ -35,6 +36,16 @@ description: Codex Proxy RS 前后端开发、排障、代码审查和文档修
|
||||
4. 按变更范围执行[验证](../../../CONTRIBUTING.md#验证)及其停止条件,沿用本轮有效证据与用户已确定的验收分工,区分通过、失败、跳过与未执行;提交时补读[提交约定](../../../CONTRIBUTING.md#项目约定)
|
||||
5. 交付实际结果与验证缺口;前端按[前端流程](references/frontend.md)核对最终差异和运行证据,后端按上表检查生命周期,不把构建通过当作行为验收
|
||||
|
||||
## 测试维护
|
||||
|
||||
- 测试随当前功能维护,名称、夹具和断言描述当前输入、行为与结果;功能变更时优先更新所属模块的现有用例,不为每次改动叠加一组记录变更经过的测试
|
||||
- 清理只证明旧字段、旧路由或旧实现已经消失的用例与附带断言;未知字段校验使用合法请求基线加无业务含义的未知字段,避免因缺少必填项等其他错误而误通过
|
||||
- 同一层对同一行为的重复覆盖合并为清晰的场景矩阵,复用相邻测试夹具;跨层测试确有不同职责或失败路径时保留,不为统一而扩大测试辅助抽象
|
||||
- 断言可观察的行为与业务不变量,避免复述结构体赋值、罗列表列或字段清单、匹配 SQL 文本与源码布局;项目明确要求的架构边界检查继续维护
|
||||
- 当前协议兼容、数据读取兼容、安全边界、并发和资源生命周期仍需覆盖,不按 `old`、`legacy` 等名称机械删除,也不能用改名代替无效用例清理
|
||||
- 不在仓库中新增或保留迁移 SQL 专项测试,包括迁移文本、schema 演变与指定版本升级用例;需要验证迁移执行、数据保留或回滚时,在临时目录编写一次性验证并使用隔离数据库,测试通过即可,临时脚本不纳入维护测试集
|
||||
- 持久化功能测试使用当前完整 schema,验证现行读写、事务与数据约束;插件运行时状态迁移等业务能力按当前功能测试,不归为迁移 SQL 专项测试
|
||||
|
||||
## 后端注释
|
||||
|
||||
- 每个 Rust 文件顶层使用简短的中文 `//!` 注释说明文件职责,覆盖生产源码、测试与构建脚本;测试文件说明测试范围或辅助用途,生成文件同步维护生成器
|
||||
|
||||
@@ -18,7 +18,7 @@ runs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
@@ -38,7 +38,7 @@ runs:
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: inputs.upload-artifact == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist
|
||||
|
||||
@@ -242,7 +242,7 @@ runs:
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
|
||||
@@ -52,7 +52,7 @@ updates:
|
||||
- patch
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /
|
||||
directory: /frontend
|
||||
labels: []
|
||||
schedule:
|
||||
interval: weekly
|
||||
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
run: cargo clippy --all-targets --all-features --locked -- -D warnings
|
||||
|
||||
- name: Setup nextest
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-nextest@0.9.143
|
||||
fallback: none
|
||||
@@ -156,6 +156,20 @@ jobs:
|
||||
echo "CPR_TEST_REDIS_URL=redis://:${redis_password}@127.0.0.1:6379"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Expose matching PostgreSQL backup and restore tools
|
||||
shell: bash
|
||||
env:
|
||||
CPR_TEST_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
client_dir="$RUNNER_TEMP/pg-client"
|
||||
mkdir -p "$client_dir"
|
||||
for tool in pg_dump psql; do
|
||||
printf '#!/bin/sh\nexec docker exec -i --env PGPASSWORD "%s" %s "$@"\n' "$CPR_TEST_POSTGRES_CONTAINER" "$tool" > "$client_dir/$tool"
|
||||
chmod +x "$client_dir/$tool"
|
||||
done
|
||||
echo "$client_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Rust tests
|
||||
working-directory: backend
|
||||
run: |
|
||||
|
||||
@@ -119,7 +119,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
- name: Setup Rust
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload sealed plugin assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
CPR_BUILD_TYPE: ${{ needs.release-info.outputs.build_type }}
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/binaries/${{ matrix.os }}-${{ matrix.arch }}/
|
||||
@@ -219,19 +219,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/frontend
|
||||
|
||||
- name: Download binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: codex-proxy-rs-*
|
||||
path: dist/binaries
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -250,7 +250,7 @@ jobs:
|
||||
subject-path: dist/release/*
|
||||
|
||||
- name: Upload release assets
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
@@ -283,19 +283,19 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: dist/docker/web/dist
|
||||
|
||||
- name: Download Docker binary artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/docker-artifacts/codex-proxy-rs-${{ matrix.os }}-${{ matrix.arch }}
|
||||
|
||||
- name: Download sealed plugin assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: plugin-release-assets
|
||||
path: dist/plugin-release-assets
|
||||
@@ -362,7 +362,7 @@ jobs:
|
||||
printf '%s\n' "$IMAGE_DIGEST" > "dist/digests/linux-${IMAGE_ARCH}.txt"
|
||||
|
||||
- name: Upload verified image digest
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: image-digest-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/digests/*.txt
|
||||
@@ -390,13 +390,13 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
- name: Download release assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
|
||||
- name: Download verified image digests
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
pattern: image-digest-*
|
||||
path: dist/digests
|
||||
|
||||
@@ -33,14 +33,23 @@ jobs:
|
||||
...params,
|
||||
per_page: 100,
|
||||
});
|
||||
// 按 PR 作者识别官方机器人,维护者重开或编辑 PR 时仍使用同一判断
|
||||
const isDependabot = pr.user.login === 'dependabot[bot]'
|
||||
&& pr.user.type === 'Bot'
|
||||
&& pr.head.repo?.full_name === `${owner}/${repo}`;
|
||||
const isDependencyUpdate = file => isDependabot
|
||||
&& file.status === 'modified'
|
||||
&& (/^\.github\/workflows\/[^/]+\.ya?ml$/.test(file.filename)
|
||||
|| /^\.github\/actions\/.+\/(?:action\.ya?ml|Dockerfile)$/.test(file.filename));
|
||||
const isProtectedRule = path => typeof path === 'string'
|
||||
&& (path === 'AGENTS.md' || path.endsWith('/AGENTS.md')
|
||||
|| path === 'CONTRIBUTING.md'
|
||||
|| path === '.agents/skills' || path.startsWith('.agents/skills/')
|
||||
|| path === '.github' || path.startsWith('.github/'));
|
||||
|
||||
// 同时检查重命名前后的路径,避免通过移走规则文件绕过检查。
|
||||
if (!files.some(file => isProtectedRule(file.filename) || isProtectedRule(file.previous_filename))) {
|
||||
// 依赖更新只豁免已有文件的修改,重命名前后的受保护路径仍参与检查
|
||||
if (!files.some(file => !isDependencyUpdate(file)
|
||||
&& (isProtectedRule(file.filename) || isProtectedRule(file.previous_filename)))) {
|
||||
core.info('No protected contribution rule changes detected.');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Install cargo-audit
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
|
||||
with:
|
||||
tool: cargo-audit@0.22.2
|
||||
fallback: none
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
run: ruby .github/scripts/toolchains.rb >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: ${{ steps.toolchains.outputs.node }}
|
||||
cache: pnpm
|
||||
|
||||
+3
-1
@@ -85,7 +85,9 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名:
|
||||
- `.agents/skills/` 中的所有文件,包括技能入口、引用文档、配置和脚本
|
||||
- `.github/` 中的所有文件,包括工作流、Actions、Issue/PR 模板和依赖更新配置
|
||||
|
||||
调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
官方 `dependabot[bot]` 从本仓库分支发起的依赖更新可修改 `.github/workflows/` 中已有的 YAML 工作流,以及 `.github/actions/` 下已有的 `action.yml`、`action.yaml` 和 `Dockerfile`,继续执行正常审查与 CI;其他受保护文件以及新增、删除、重命名不在例外范围内
|
||||
|
||||
除此之外,调整这些文件时先提交 Issue 说明原因和建议,由维护者处理;[规则保护工作流](.github/workflows/rules.yml) 会提示并关闭包含此类变更的 PR。撤销相关变更后可以重新打开 PR,继续正常检查
|
||||
|
||||
源码变更遵守现有架构边界与迁移冻结规则,由对应 CI 检查执行;不额外设立整块业务目录的 PR 禁改清单。构建、测试和安全扫描复用现有工作流
|
||||
|
||||
|
||||
Generated
+21
-19
@@ -198,7 +198,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -856,7 +856,7 @@ dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -910,7 +910,7 @@ dependencies = [
|
||||
"provider-xai",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1657,7 +1657,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1720,7 +1720,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
@@ -1903,6 +1903,7 @@ dependencies = [
|
||||
name = "gateway-store"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
@@ -3267,7 +3268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3627,7 +3628,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4000,7 +4001,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4015,10 +4016,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
@@ -4099,9 +4101,9 @@ checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.11.0"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
|
||||
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.0",
|
||||
@@ -4481,9 +4483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -4581,7 +4583,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4735,9 +4737,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.19"
|
||||
version = "0.7.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
|
||||
checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -5052,9 +5054,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.27.0"
|
||||
version = "1.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a"
|
||||
checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ opt-level = 2
|
||||
[profile.dev.package.blake2]
|
||||
opt-level = 2
|
||||
|
||||
[profile.dev.package."sha2@0.11.0"]
|
||||
[profile.dev.package."sha2@0.11.1"]
|
||||
opt-level = 2
|
||||
|
||||
[profile.dev.package."miniz_oxide@0.9.1"]
|
||||
|
||||
@@ -155,14 +155,6 @@ fn config_loader_should_ignore_unknown_fields_in_configuration_sections() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_ignore_removed_tls_and_fingerprint_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["tls"] = serde_json::json!({});
|
||||
document["openai"]["fingerprint"] = serde_json::json!({"browser": "removed"});
|
||||
parse_config(&document.to_string()).expect("removed provider settings are ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_reject_invalid_known_fields_alongside_unknown_fields() {
|
||||
for port in [serde_json::json!(0), serde_json::json!("not-a-port")] {
|
||||
@@ -192,7 +184,7 @@ fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
] {
|
||||
let mut document = valid_config_document();
|
||||
if case == "unused" {
|
||||
document["openai"]["wire_profile"]["location"] = serde_json::json!(UNUSED_SECRET);
|
||||
document["openai"]["unused_setting"] = serde_json::json!(UNUSED_SECRET);
|
||||
} else if case == "missing" {
|
||||
document["host"]["listen"]
|
||||
.as_object_mut()
|
||||
@@ -232,7 +224,7 @@ fn config_loader_should_report_startup_configuration_diagnostics() {
|
||||
);
|
||||
match case {
|
||||
"unused" => assert!(
|
||||
stderr.contains("openai.wire_profile") && stderr.contains("已忽略"),
|
||||
stderr.contains("openai.unused_setting") && stderr.contains("已忽略"),
|
||||
"{stderr}"
|
||||
),
|
||||
"missing" => assert!(stderr.contains("host.listen.port"), "{stderr}"),
|
||||
@@ -256,19 +248,7 @@ fn config_loader_should_allow_missing_provider_configuration() {
|
||||
let mut document = valid_config_document();
|
||||
document.as_object_mut().unwrap().remove("openai");
|
||||
document.as_object_mut().unwrap().remove("xai");
|
||||
parse_config(&document.to_string()).expect("provider defaults require no YAML identity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_loader_should_ignore_removed_provider_identity_sections() {
|
||||
let mut document = valid_config_document();
|
||||
document["openai"]["wire_profile"] = serde_json::json!({"codex_version":"legacy-invalid"});
|
||||
document["xai"] = serde_json::json!({"wire_profile": {"client_identifier": "legacy-client"}});
|
||||
let (config, _directory) =
|
||||
parse_config(&document.to_string()).expect("removed identities ignored");
|
||||
let debug = format!("{config:?}");
|
||||
assert!(!debug.contains("legacy-invalid"));
|
||||
assert!(!debug.contains("legacy-client"));
|
||||
parse_config(&document.to_string()).expect("providers use defaults when omitted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -706,6 +706,10 @@ impl BackupTask {
|
||||
fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) {
|
||||
match error.code() {
|
||||
code::CANCELLED => (code::CANCELLED, "备份已取消"),
|
||||
code::ANALYTICS_NOT_READY => (
|
||||
code::ANALYTICS_NOT_READY,
|
||||
"用量统计正在重建,请在统计就绪后重试备份",
|
||||
),
|
||||
code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"),
|
||||
_ => (code::PG_DUMP_FAILED, "数据库导出失败"),
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ pub const STATE_CONFLICT: &str = "backup.state_conflict";
|
||||
pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked";
|
||||
/// `pg_dump` 非零退出或失败
|
||||
pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed";
|
||||
/// 用量汇总尚未完整发布
|
||||
pub const ANALYTICS_NOT_READY: &str = "backup.analytics_not_ready";
|
||||
/// 暂存磁盘空间不足
|
||||
pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted";
|
||||
/// S3 认证失败
|
||||
|
||||
@@ -499,7 +499,7 @@ impl BackupError {
|
||||
/// 构造稳定错误码常量
|
||||
pub mod code;
|
||||
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`
|
||||
/// 构建对象 key:`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.sql.gz`
|
||||
///
|
||||
/// 规范化规则:prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符;
|
||||
/// 输出统一使用 `/` 分隔且不含空段
|
||||
@@ -510,7 +510,7 @@ pub fn build_object_key(
|
||||
) -> Result<String, BackupError> {
|
||||
let prefix = normalize_prefix(prefix)?;
|
||||
Ok(format!(
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump",
|
||||
"{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.sql.gz",
|
||||
at.format("%Y"),
|
||||
at.format("%m"),
|
||||
at.format("%d"),
|
||||
@@ -519,10 +519,10 @@ pub fn build_object_key(
|
||||
))
|
||||
}
|
||||
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.dump`,与对象 key 的文件名一致
|
||||
/// 下载文件名:`codex-proxy-rs_{短 id}.sql.gz`,与对象 key 的文件名一致
|
||||
#[must_use]
|
||||
pub fn build_download_file_name(backup_id: &str) -> String {
|
||||
format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id))
|
||||
format!("codex-proxy-rs_{}.sql.gz", short_backup_id(backup_id))
|
||||
}
|
||||
|
||||
/// 截取备份 id 的可读短标识:去掉 `backup_` 前缀,最多保留 8 位 hex
|
||||
|
||||
@@ -5,20 +5,16 @@ use std::str::FromStr as _;
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_core::account::OpaqueProviderData;
|
||||
|
||||
use gateway_admin::{
|
||||
model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
CredentialCommitGuard, ProviderDocument, ProviderQuota, ProviderQuotaWindow,
|
||||
QuotaLocalUsageAttribution,
|
||||
},
|
||||
settings::AdminApiKey,
|
||||
use gateway_admin::model::{
|
||||
PageSize, Revision,
|
||||
accounts::AccountUsage,
|
||||
auth::LoginCommand,
|
||||
client_keys::ClientKeyPageSize,
|
||||
observability::{DecimalAmount, RequestOutcome, TimeRange},
|
||||
provider_credentials::{
|
||||
ProviderDocument, ProviderQuota, ProviderQuotaWindow, QuotaLocalUsageAttribution,
|
||||
},
|
||||
ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore},
|
||||
settings::AdminApiKey,
|
||||
};
|
||||
|
||||
mod backup;
|
||||
@@ -99,20 +95,6 @@ fn login_command_debug_should_redact_password() {
|
||||
assert!(!format!("{command:?}").contains("secret-password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_store_capability_traits_should_be_object_safe() {
|
||||
fn assert_object_safe<T: ?Sized + Send + Sync>() {}
|
||||
|
||||
assert_object_safe::<dyn AccountStore>();
|
||||
assert_object_safe::<dyn AuthStore>();
|
||||
assert_object_safe::<dyn ClientKeyStore>();
|
||||
assert_object_safe::<dyn ObservabilityStore>();
|
||||
assert_object_safe::<dyn SettingsStore>();
|
||||
|
||||
fn assert_send_object_safe<T: ?Sized + Send>() {}
|
||||
assert_send_object_safe::<dyn CredentialCommitGuard>();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_document_debug_should_not_expose_opaque_material() {
|
||||
let document = ProviderDocument::new(OpaqueProviderData::new(Default::default()));
|
||||
|
||||
@@ -58,7 +58,7 @@ fn object_key_shape() {
|
||||
let key = build_object_key("production/backups", "backup_abc", at).unwrap();
|
||||
assert_eq!(
|
||||
key,
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump"
|
||||
"production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.sql.gz"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ fn counters_preserve_integer_precision_and_reject_json_numbers() {
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_absent() {
|
||||
async fn usage_mutations_require_a_trusted_actor() {
|
||||
let admin = super::AdminHarness::new().build().await;
|
||||
let registry = admin.public_services();
|
||||
let denied = registry
|
||||
@@ -38,21 +38,6 @@ async fn usage_mutations_require_trusted_actor_and_old_workflow_operations_are_a
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let ops = registry
|
||||
.operations()
|
||||
.filter(|op| op.starts_with("usage.") || *op == "pricing.evaluate")
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
ops,
|
||||
std::collections::BTreeSet::from([
|
||||
"usage.records.get",
|
||||
"usage.records.list",
|
||||
"usage.records.write",
|
||||
"usage.aggregates.list",
|
||||
"usage.aggregates.write",
|
||||
"pricing.evaluate"
|
||||
])
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn sdk_wire_types_match_host_contracts() {
|
||||
|
||||
@@ -46,7 +46,6 @@ async fn list_route_should_keep_camel_case_group_and_page_wire() {
|
||||
value["data"]["items"][0]["usage"]["retainedTotalUsd"],
|
||||
"5.5"
|
||||
);
|
||||
assert!(value["data"]["items"][0].get("member_count").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -149,7 +149,7 @@ fn account_page_item(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_usage_returns_distinct_cache_read_and_write_tokens_without_aliases() {
|
||||
async fn account_usage_preserves_cache_read_and_write_counts_and_missing_values() {
|
||||
use chrono::Utc;
|
||||
use gateway_admin::model::accounts::AccountUsage;
|
||||
use gateway_core::account::{AccountStatusFacts, CredentialState, QuotaState};
|
||||
@@ -215,37 +215,9 @@ async fn account_usage_returns_distinct_cache_read_and_write_tokens_without_alia
|
||||
usage["cacheWriteInputTokensDisplay"],
|
||||
if write.is_some() { "7" } else { "—" }
|
||||
);
|
||||
for alias in [
|
||||
"createdTokens",
|
||||
"createdTokensDisplay",
|
||||
"readTokens",
|
||||
"readTokensDisplay",
|
||||
] {
|
||||
assert!(usage.get(alias).is_none(), "obsolete token alias: {alias}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_credential_rotation_route_is_not_exposed() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let response = admin::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/accounts/rotate")
|
||||
.header(header::COOKIE, "cpr_session=valid-session")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_update_requires_admin_and_validates_before_calling_the_service() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
|
||||
@@ -248,9 +248,8 @@ mod profile_statistics {
|
||||
value["activityInsights"]["invocations"][0]["usageCount"],
|
||||
29
|
||||
);
|
||||
assert!(value.get("cycle").is_none());
|
||||
assert!(value.get("models").is_none());
|
||||
assert!(value.get("estimatedCost").is_none());
|
||||
assert_eq!(value["activityInsights"]["fastModePercent"], 0.0);
|
||||
assert!(value["activityInsights"]["invocations"][0]["skillId"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -375,7 +374,7 @@ mod batch_update {
|
||||
assert!(request.concurrency_limit.is_none());
|
||||
assert!(
|
||||
serde_json::from_value::<BatchUpdateAccountsRequest>(json!({
|
||||
"accountIds": ["acct_test"], "enabled": true, "legacy": true
|
||||
"accountIds": ["acct_test"], "enabled": true, "unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
@@ -583,12 +582,12 @@ mod actions {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn account_actions_should_require_frozen_account_ids_and_reject_unknown_revision() {
|
||||
fn account_actions_should_validate_account_ids_and_reject_unknown_fields() {
|
||||
let id: AccountIdQuery =
|
||||
serde_json::from_value(json!({ "accountId": "acct_1" })).expect("decode ID query");
|
||||
assert!(id.validate().is_ok());
|
||||
let action: AccountActionRequest =
|
||||
serde_json::from_value(json!({ "accountId": "legacy-id" })).expect("decode action");
|
||||
serde_json::from_value(json!({ "accountId": "invalid-id" })).expect("decode action");
|
||||
assert_eq!(action.validate().unwrap_err().field(), "accountId");
|
||||
let refresh: AccountRefreshRequest = serde_json::from_value(json!({
|
||||
"accountId": "acct_1"
|
||||
@@ -598,7 +597,7 @@ mod actions {
|
||||
assert!(
|
||||
serde_json::from_value::<AccountRefreshRequest>(json!({
|
||||
"accountId": "acct_1",
|
||||
"expectedConfigRevision": 0
|
||||
"unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
@@ -772,15 +771,14 @@ mod actions {
|
||||
assert_eq!(invalid.validate().unwrap_err().field(), "data");
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"document": {}
|
||||
"provider": "openai"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<AccountImportRequest>(json!({
|
||||
"provider": "openai",
|
||||
"expectedConfigRevision": 7,
|
||||
"unknownField": true,
|
||||
"data": {}
|
||||
}))
|
||||
.is_err()
|
||||
|
||||
@@ -11,7 +11,7 @@ use gateway_api::admin::accounts::{
|
||||
use gateway_core::operation::RawJsonPayload;
|
||||
|
||||
#[test]
|
||||
fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
fn quota_forecast_projection_formats_capacity_and_preserves_null_zero() {
|
||||
let now = "2026-09-12T00:00:00Z".parse().unwrap();
|
||||
let forecast = AccountQuotaForecast {
|
||||
period: AccountUsagePeriod::Weekly,
|
||||
@@ -54,16 +54,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
assert_eq!(week["estimatedTokensDisplay"], "1M");
|
||||
assert!(week["estimatedUsd"].is_null());
|
||||
assert_eq!(week["estimatedUsdDisplay"], "—");
|
||||
for forecast in value["forecasts"].as_array().unwrap() {
|
||||
for field in [
|
||||
"remainingTokens",
|
||||
"remainingTokensDisplay",
|
||||
"remainingUsd",
|
||||
"remainingUsdDisplay",
|
||||
] {
|
||||
assert!(forecast.get(field).is_none(), "unexpected field: {field}");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
week["source"],
|
||||
serde_json::json!({
|
||||
@@ -77,9 +67,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
"usdDisplay": "$0.1234"
|
||||
})
|
||||
);
|
||||
assert!(week.get("method").is_none());
|
||||
assert!(week.get("methodDisplay").is_none());
|
||||
assert!(value.get("generatedAtDisplay").is_none());
|
||||
assert_eq!(value["forecasts"][1]["estimatedTokens"], 0);
|
||||
assert_eq!(value["forecasts"][1]["estimatedTokensDisplay"], "0");
|
||||
assert_eq!(value["forecasts"][1]["estimatedUsd"], 0.0);
|
||||
@@ -87,7 +74,6 @@ fn quota_forecast_projection_only_exposes_capacity_and_preserves_null_zero() {
|
||||
assert_eq!(value["forecasts"][1]["period"], "monthly");
|
||||
assert_eq!(value["forecasts"][1]["targetDays"], 30.0);
|
||||
assert_eq!(value["forecasts"][1]["extrapolated"], true);
|
||||
assert!(value.get("account").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -375,18 +375,17 @@ fn client_key_mutations_should_validate_text_limits_and_unknown_fields() {
|
||||
assert!(
|
||||
serde_json::from_value::<UpdateClientKeyRequest>(json!({
|
||||
"id": "key_1",
|
||||
"expectedConfigRevision": 1,
|
||||
"name": "key",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 0,
|
||||
"requestsPerMinute": 0,
|
||||
"tokensPerMinute": 0
|
||||
"unknownField": true
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<CreateClientKeyRequest>(json!({
|
||||
"expectedConfigRevision": 7,
|
||||
"unknownField": true,
|
||||
"name": "terminal key",
|
||||
"groupIds": [],
|
||||
"maxConcurrency": 2,
|
||||
@@ -458,7 +457,6 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
));
|
||||
let list =
|
||||
serde_json::to_value(ClientKeyListData::new(vec![view], None, 1)).expect("serialize list");
|
||||
assert!(list.get("configRevision").is_none());
|
||||
assert_eq!(list["total"], 1);
|
||||
assert_eq!(list["items"][0]["id"], "key_visible");
|
||||
assert_eq!(
|
||||
@@ -468,12 +466,9 @@ fn client_key_responses_should_keep_shape_and_redact_creation_debug() {
|
||||
assert_eq!(list["items"][0]["routingScope"], "all");
|
||||
assert_eq!(list["items"][0]["maxConcurrency"], 2);
|
||||
assert_eq!(list["items"][0]["requestsPerMinute"], 60);
|
||||
assert!(list["items"][0].get("tokensPerMinute").is_none());
|
||||
assert!(list["items"][0].get("policyJson").is_none());
|
||||
assert_eq!(list["items"][0]["dailyLimitUsd"], "0");
|
||||
assert_eq!(list["items"][0]["weeklyLimitUsd"], "0");
|
||||
assert_eq!(list["items"][0]["dailyUsedUsd"], "0");
|
||||
assert!(list["items"][0].get("unresolvedRequests").is_none());
|
||||
assert_eq!(
|
||||
DateTime::parse_from_rfc3339(
|
||||
list["items"][0]["lastUsedAt"]
|
||||
|
||||
@@ -7,7 +7,7 @@ use gateway_api::admin::observability::{
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn dashboard_query_should_parse_terminal_trend_kinds() {
|
||||
fn dashboard_query_should_parse_error_trends() {
|
||||
let query: DashboardQuery = serde_json::from_value(json!({"kind": "errors"})).unwrap();
|
||||
assert_eq!(query.trend_kind().unwrap(), TrendKind::Errors);
|
||||
}
|
||||
@@ -19,7 +19,7 @@ fn dashboard_query_should_reject_unknown_trend_kind() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_accept_element_plus_pagination() {
|
||||
fn usage_query_should_accept_page_number_and_size() {
|
||||
let query: UsageQuery = serde_json::from_value(json!({
|
||||
"currentPage": 129,
|
||||
"pageSize": 10
|
||||
@@ -29,32 +29,20 @@ fn usage_query_should_accept_element_plus_pagination() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_removed_total_toggle() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"includeTotal": true})).is_err());
|
||||
fn observability_queries_should_reject_unknown_fields() {
|
||||
let query = json!({"unknownFilter": true});
|
||||
assert!(serde_json::from_value::<DashboardQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<UsageQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(query.clone()).is_err());
|
||||
assert!(serde_json::from_value::<DiagnosticsQuery>(query).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_removed_cursor_contract() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"cursor": "opaque"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_query_should_reject_nonstandard_page_name() {
|
||||
assert!(serde_json::from_value::<UsageQuery>(json!({"page": 1})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_page_size_above_terminal_limit() {
|
||||
fn ops_query_should_reject_page_size_above_maximum() {
|
||||
let query: OpsQuery = serde_json::from_value(json!({"pageSize": 101})).unwrap();
|
||||
assert_eq!(query.validate_pagination().unwrap_err().field(), "pageSize");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_query_should_reject_removed_filters() {
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"route": "/v1/responses"})).is_err());
|
||||
assert!(serde_json::from_value::<OpsQuery>(json!({"failureClass": "rate_limited"})).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diagnostics_query_should_keep_wire_dimension_name() {
|
||||
let query: DiagnosticsQuery =
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 响应 DTO 与固定 wire 形状的合同测试
|
||||
//! 验证观测响应的指标、展示投影与查询时间边界
|
||||
|
||||
use chrono::Utc;
|
||||
use serde_json::json;
|
||||
@@ -358,13 +358,6 @@ async fn usage_detail_should_keep_attempt_snapshot_contract() {
|
||||
value["data"]["latencyDetails"]["upstreamEngineMs"],
|
||||
json!(6400.0)
|
||||
);
|
||||
assert!(value["data"].get("firstTokenMs").is_none());
|
||||
assert!(value["data"].get("firstTokenLatencyMsDisplay").is_none());
|
||||
assert!(
|
||||
value["data"]["latencyDetails"]
|
||||
.get("firstTokenMs")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::json!({
|
||||
"accountId": value["data"]["accountId"],
|
||||
@@ -470,8 +463,6 @@ async fn zero_attempt_failure_detail_omits_trace_and_keeps_missing_upstream_fact
|
||||
assert_eq!(data["clientStatusCode"], 503);
|
||||
assert_eq!(data["message"], "no upstream account is available");
|
||||
assert_eq!(data["model"], "coding");
|
||||
assert!(data.get("upstreamStatusCode").is_none());
|
||||
assert!(data.get("httpVersion").is_none());
|
||||
for field in [
|
||||
"provider",
|
||||
"accountId",
|
||||
@@ -622,7 +613,6 @@ async fn diagnostics_should_keep_stable_key_and_display_name_contract() {
|
||||
.await
|
||||
.expect("diagnostics body");
|
||||
let value: serde_json::Value = serde_json::from_slice(&body).expect("diagnostics JSON");
|
||||
assert!(value["data"]["items"][0].get("impactScore").is_none());
|
||||
assert_eq!(value["data"]["items"][0]["requestShare"], 0.5);
|
||||
assert_eq!(value["data"]["items"][0]["retryCount"], 3);
|
||||
assert_eq!(value["data"]["items"][0]["retryRate"], 0.5);
|
||||
|
||||
@@ -35,7 +35,7 @@ use std::{collections::BTreeMap, sync::Arc};
|
||||
mod management;
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
fn artifact_metadata_wire_preserves_identity_icons_and_contributions() {
|
||||
use gateway_admin::model::plugins::{
|
||||
PluginArtifactIcon, PluginArtifactIconVariants, PluginArtifactMetadata, PluginContribution,
|
||||
};
|
||||
@@ -70,7 +70,7 @@ fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
secret_fields: vec![],
|
||||
state_namespaces: vec![],
|
||||
};
|
||||
let value = serde_json::to_value(metadata).expect("v2 artifact metadata JSON");
|
||||
let value = serde_json::to_value(metadata).expect("artifact metadata JSON");
|
||||
assert_eq!(value["displayName"], "Example");
|
||||
assert_eq!(value["publisher"], "test");
|
||||
assert!(value["author"].is_null());
|
||||
@@ -80,33 +80,29 @@ fn artifact_metadata_wire_exposes_v2_identity_and_contributions() {
|
||||
value["contributes"]["middleware"]["id"],
|
||||
"test.example.middleware"
|
||||
);
|
||||
assert!(value.get("capabilities").is_none());
|
||||
assert!(value.get("permissionDescriptions").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn instance_binding_wire_uses_contribution_without_capability_alias() {
|
||||
fn instance_binding_requires_a_contribution_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::plugins::instances::PluginCapabilityBinding;
|
||||
|
||||
let binding = serde_json::from_value::<PluginCapabilityBinding>(serde_json::json!({
|
||||
let input = serde_json::json!({
|
||||
"contribution": "test.example.usage",
|
||||
"stage": "observation",
|
||||
"order": 0,
|
||||
"failurePolicy": "observe"
|
||||
}))
|
||||
.expect("v2 contribution binding");
|
||||
});
|
||||
let binding = serde_json::from_value::<PluginCapabilityBinding>(input.clone())
|
||||
.expect("contribution binding");
|
||||
assert_eq!(binding.contribution, "test.example.usage");
|
||||
|
||||
assert!(
|
||||
serde_json::from_value::<PluginCapabilityBinding>(serde_json::json!({
|
||||
"capability": "usage",
|
||||
"stage": "observation",
|
||||
"order": 0,
|
||||
"failurePolicy": "observe"
|
||||
}))
|
||||
.is_err(),
|
||||
"legacy capability binding must be rejected"
|
||||
);
|
||||
let mut missing = input.clone();
|
||||
missing.as_object_mut().unwrap().remove("contribution");
|
||||
assert!(serde_json::from_value::<PluginCapabilityBinding>(missing).is_err());
|
||||
|
||||
let mut unknown = input;
|
||||
unknown["unknownField"] = serde_json::json!(true);
|
||||
assert!(serde_json::from_value::<PluginCapabilityBinding>(unknown).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -381,7 +377,7 @@ async fn plugin_upload_endpoints_require_admin_and_bound_raw_bodies() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_acceptance_returns_the_flat_install_contract() {
|
||||
async fn artifact_acceptance_returns_installed_artifact_and_configuration_status() {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request, StatusCode},
|
||||
@@ -418,22 +414,11 @@ async fn artifact_acceptance_returns_the_flat_install_contract() {
|
||||
assert_eq!(result["defaultInstanceId"], serde_json::Value::Null);
|
||||
assert_eq!(result["configurationRequired"], false);
|
||||
assert_eq!(result["artifact"]["metadata"]["sha256"], "a".repeat(64));
|
||||
assert!(
|
||||
result["artifact"]["metadata"]
|
||||
.get("permissionDescriptions")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
result["artifact"]["metadata"]
|
||||
.get("requestedPermissions")
|
||||
.is_none()
|
||||
);
|
||||
assert!(result["artifact"]["acceptedAt"].is_string());
|
||||
assert!(result.get("mutation").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_instance_policy_fields_and_extra_accept_fields_are_rejected() {
|
||||
async fn plugin_mutations_reject_unknown_fields() {
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
@@ -442,24 +427,25 @@ async fn removed_instance_policy_fields_and_extra_accept_fields_are_rejected() {
|
||||
|
||||
let fixture = super::AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
for (path, body) in [
|
||||
(
|
||||
"/api/admin/plugins/instances",
|
||||
serde_json::json!({
|
||||
"name":"Example",
|
||||
"artifactSha256":"a".repeat(64),
|
||||
"enabled":false,
|
||||
"trustedProcess":true,
|
||||
"configuration":{},
|
||||
"grants":[],
|
||||
"bindings":[]
|
||||
}),
|
||||
),
|
||||
let instance = serde_json::json!({
|
||||
"name":"Example",
|
||||
"artifactSha256":"a".repeat(64),
|
||||
"enabled":false,
|
||||
"configuration":{},
|
||||
"bindings":[]
|
||||
});
|
||||
serde_json::from_value::<gateway_admin::model::plugins::instances::ConfigurePluginInstance>(
|
||||
instance.clone(),
|
||||
)
|
||||
.expect("valid instance request");
|
||||
for (path, mut body) in [
|
||||
("/api/admin/plugins/instances", instance),
|
||||
(
|
||||
"/api/admin/plugins/artifacts/accept",
|
||||
serde_json::json!({"sha256":"a".repeat(64),"trusted":true}),
|
||||
serde_json::json!({"sha256":"a".repeat(64)}),
|
||||
),
|
||||
] {
|
||||
body["unknownField"] = serde_json::json!(true);
|
||||
let response = gateway_api::admin::router::<super::AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
|
||||
@@ -181,7 +181,7 @@ fn privacy_policy_is_required_and_rejects_unknown_contract_fields() {
|
||||
.remove("codexPrivacyPolicy");
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(missing).is_err());
|
||||
let mut unknown = update_body();
|
||||
unknown["codexPrivacyPolicy"]["protectedFields"] = json!([]);
|
||||
unknown["codexPrivacyPolicy"]["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(unknown).is_err());
|
||||
}
|
||||
|
||||
@@ -471,110 +471,6 @@ fn settings_response_should_cover_the_full_runtime_settings_contract() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_and_response_fields_should_stay_in_lockstep() {
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use gateway_admin::model::Revision;
|
||||
use gateway_admin::model::settings::RuntimeSettings;
|
||||
use gateway_api::admin::settings::RuntimeSettingsView;
|
||||
use gateway_core::account::RotationStrategy;
|
||||
use gateway_core::routing::{PublicModelId, UpstreamModelId};
|
||||
|
||||
let request: UpdateRuntimeSettingsRequest =
|
||||
serde_json::from_value(update_body()).expect("decode settings");
|
||||
request.validate().expect("fixture settings must validate");
|
||||
|
||||
let request_fields: BTreeSet<String> = update_body()
|
||||
.as_object()
|
||||
.expect("request body object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = RuntimeSettings {
|
||||
request_profiles: Default::default(),
|
||||
config_revision: Revision::new(7).expect("revision"),
|
||||
model_mappings: request
|
||||
.model_mappings
|
||||
.iter()
|
||||
.map(|(public, upstream)| {
|
||||
Ok((
|
||||
PublicModelId::new(public.clone())?,
|
||||
UpstreamModelId::new(upstream.clone())?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, gateway_core::error::IdentifierError>>()
|
||||
.expect("valid model mappings"),
|
||||
rotation_strategy: RotationStrategy::parse(&request.rotation_strategy)
|
||||
.expect("fixture rotation strategy"),
|
||||
updated_at: chrono::Utc::now(),
|
||||
values: gateway_admin::model::settings::RuntimeSettingsValues {
|
||||
codex_privacy_policy: Default::default(),
|
||||
request_location_enabled: false,
|
||||
request_location: Default::default(),
|
||||
refresh_margin_seconds: request.values.refresh_margin_seconds,
|
||||
refresh_concurrency: u32::try_from(request.values.refresh_concurrency).expect("u32"),
|
||||
max_concurrent_per_account: u32::try_from(request.values.max_concurrent_per_account)
|
||||
.expect("u32"),
|
||||
max_waiting_per_key: 0,
|
||||
max_waiting_per_account: 0,
|
||||
concurrency_wait_timeout_seconds: 30,
|
||||
openai_guardian_reserved_concurrency: 0,
|
||||
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
|
||||
max_account_rotations: 3,
|
||||
openai_session_affinity_ttl_hours: 24,
|
||||
responses_max_decompressed_body_bytes: 64 * 1024 * 1024,
|
||||
smart_scheduling: request.values.smart_scheduling,
|
||||
min_codex_desktop_version: request.values.min_codex_desktop_version,
|
||||
min_codex_cli_version: request.values.min_codex_cli_version,
|
||||
usage_retention_days: u32::try_from(request.values.usage_retention_days).expect("u32"),
|
||||
ops_event_retention_days: u32::try_from(request.values.ops_event_retention_days)
|
||||
.expect("u32"),
|
||||
audit_retention_days: u32::try_from(request.values.audit_retention_days).expect("u32"),
|
||||
account_auto_freeze_enabled: true,
|
||||
account_auto_freeze_threshold: 12,
|
||||
account_auto_freeze_window_seconds: 600,
|
||||
account_auto_freeze_duration_seconds: 7_200,
|
||||
account_auto_freeze_probe_enabled: true,
|
||||
account_auto_freeze_probe_model: None,
|
||||
account_auto_freeze_adaptive_concurrency: true,
|
||||
account_warmup_enabled: false,
|
||||
account_warmup_schedule_time: "08:00".to_owned(),
|
||||
account_warmup_model: None,
|
||||
reset_credit_lead_minutes: 10,
|
||||
reset_credit_max_weekly_remaining_percent: None,
|
||||
},
|
||||
};
|
||||
|
||||
let response_fields: BTreeSet<String> = serde_json::to_value(RuntimeSettingsView::from((
|
||||
settings,
|
||||
gateway_api::TimePresenter::new(Default::default()),
|
||||
)))
|
||||
.expect("serialize view")
|
||||
.as_object()
|
||||
.expect("view object")
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut expected_fields = request_fields;
|
||||
expected_fields.insert("providerRequestProfiles".to_owned());
|
||||
expected_fields.insert("openaiClientProfile".to_owned());
|
||||
expected_fields.insert("xaiClientProfile".to_owned());
|
||||
expected_fields.insert("updatedAt".to_owned());
|
||||
expected_fields.insert("updatedAtDisplay".to_owned());
|
||||
expected_fields.insert("smartSchedulingDefaults".to_owned());
|
||||
|
||||
assert_eq!(response_fields, expected_fields);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_accepts_json_bytes_with_flattened_fields() {
|
||||
serde_json::from_slice::<UpdateRuntimeSettingsRequest>(
|
||||
&serde_json::to_vec(&update_body()).unwrap(),
|
||||
)
|
||||
.expect("settings JSON bytes must decode like the HTTP request body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
use gateway_admin::model::settings::{ReplaceRuntimeSettings, RuntimeSettings};
|
||||
@@ -610,17 +506,9 @@ fn settings_service_contract_stays_flat_and_rejects_unknown_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_unknown_revision_field() {
|
||||
fn settings_request_should_reject_unknown_fields() {
|
||||
let mut body = update_body();
|
||||
body["expectedConfigRevision"] = json!(7);
|
||||
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_request_should_reject_removed_bucket_retention() {
|
||||
let mut body = update_body();
|
||||
body["bucketRetentionDays"] = json!(365);
|
||||
body["unknownSetting"] = json!(true);
|
||||
|
||||
assert!(serde_json::from_value::<UpdateRuntimeSettingsRequest>(body).is_err());
|
||||
}
|
||||
@@ -960,36 +848,23 @@ fn decompression_setting_should_reject_invalid_values() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn settings_reject_removed_global_fast_policy() {
|
||||
async fn settings_update_rejects_unknown_fields_without_writing() {
|
||||
let fixture = AdminTestFixture::new().await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let app = app(fixture.state());
|
||||
let initial = fixture.settings.settings.lock().unwrap().clone();
|
||||
for value in [json!(true), json!(false), Value::Null] {
|
||||
let mut body = update_body();
|
||||
body["disableFast"] = value;
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(*fixture.settings.settings.lock().unwrap(), initial);
|
||||
}
|
||||
let mut body = update_body();
|
||||
body["unknownSetting"] = json!(true);
|
||||
let response = app
|
||||
.oneshot(request(Method::GET, "/api/admin/settings", None))
|
||||
.oneshot(request(
|
||||
Method::POST,
|
||||
"/api/admin/settings/update",
|
||||
Some(body),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(
|
||||
response_json(response).await["data"]
|
||||
.get("disableFast")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
assert_eq!(*fixture.settings.settings.lock().unwrap(), initial);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -86,7 +86,7 @@ async fn update_event_stream_should_preserve_event_id_and_download_progress_perc
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_should_return_accepted_and_report_restart_only_through_status() {
|
||||
async fn update_should_return_accepted_and_expose_restart_status() {
|
||||
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
|
||||
fixture.auth.insert_session("valid-session");
|
||||
let router = app(fixture.state());
|
||||
@@ -110,7 +110,6 @@ async fn update_should_return_accepted_and_report_restart_only_through_status()
|
||||
.expect("JSON");
|
||||
assert_eq!(body["data"]["operationId"], "update-1");
|
||||
assert_eq!(body["data"]["targetVersion"], "0.2.0");
|
||||
assert!(body["data"].get("needRestart").is_none());
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
|
||||
@@ -297,15 +297,15 @@ async fn unified_login_returns_server_identity_and_rotates_the_previous_session(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn login_rejects_legacy_type_without_replacing_the_current_session() {
|
||||
async fn login_rejects_missing_mode_and_unknown_fields_without_replacing_the_session() {
|
||||
let (app, _) = auth_app().await;
|
||||
let cookie =
|
||||
session_cookie(&login(&app, json!({"mode": "key", "apiKey": RAW_KEY}), None).await);
|
||||
for body in [
|
||||
json!({"type": "key", "apiKey": RAW_KEY}),
|
||||
json!({"type": "admin", "password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "type": "key", "apiKey": RAW_KEY}),
|
||||
json!({"mode": "admin", "type": "admin", "password": "strong-admin-password"}),
|
||||
json!({"apiKey": RAW_KEY}),
|
||||
json!({"password": "strong-admin-password"}),
|
||||
json!({"mode": "key", "unknownField": true, "apiKey": RAW_KEY}),
|
||||
json!({"mode": "admin", "unknownField": true, "password": "strong-admin-password"}),
|
||||
] {
|
||||
let response = login(&app, body, Some(&cookie)).await;
|
||||
assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY);
|
||||
@@ -458,26 +458,26 @@ async fn key_store_outage_is_unavailable_not_expired_or_authenticated() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_auth_routes_do_not_fall_back_to_spa_or_accept_old_cookies() {
|
||||
async fn unknown_auth_routes_return_json_not_found() {
|
||||
let (app, _) = auth_app().await;
|
||||
let response = app
|
||||
.oneshot(empty_request(Method::GET, "/api/auth/unknown"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response_json(response).await["code"], 40401);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_authentication_requires_the_configured_cookie_name() {
|
||||
let (app, _) = auth_app().await;
|
||||
for path in ["/api/admin/auth/login", "/api/auth/unknown"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(empty_request(Method::GET, path))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
assert_eq!(response_json(response).await["code"], 40401);
|
||||
}
|
||||
let cookie =
|
||||
session_cookie(&login(&app, json!({"mode": "key", "apiKey": RAW_KEY}), None).await);
|
||||
for old_name in ["cpr_admin_session", "cpr_client_session"] {
|
||||
let old_cookie = cookie.replace("cpr_session", old_name);
|
||||
assert_eq!(
|
||||
response_json(get(&app, "/api/auth/status", &old_cookie).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
}
|
||||
let unrelated_cookie = cookie.replace("cpr_session", "unrelated_session");
|
||||
assert_eq!(
|
||||
response_json(get(&app, "/api/auth/status", &unrelated_cookie).await).await["data"]["authenticated"],
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -799,7 +799,7 @@ async fn websocket_failure_before_first_event_should_return_observed_metadata_th
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit() {
|
||||
async fn large_websocket_response_create_frames_should_reach_execution() {
|
||||
let trace = Arc::new(AtomicFailureTrace::default());
|
||||
let execution = Arc::new(AtomicFailureExecution {
|
||||
client: authenticated_client("sk_ws_atomic"),
|
||||
@@ -829,15 +829,14 @@ async fn websocket_response_create_should_not_have_a_private_16_mib_frame_limit(
|
||||
let payload = json!({
|
||||
"type": "response.create",
|
||||
"model": "model-a",
|
||||
"input": "x".repeat(16 * 1024 * 1024 + 1),
|
||||
"input": "x".repeat(17 * 1024 * 1024),
|
||||
})
|
||||
.to_string();
|
||||
assert!(payload.len() > 16 * 1024 * 1024);
|
||||
|
||||
socket
|
||||
.send(ClientMessage::Text(payload.into()))
|
||||
.await
|
||||
.expect("send response.create above the former frame limit");
|
||||
.expect("send large response.create frame");
|
||||
|
||||
let mut saw_upstream_event = false;
|
||||
for _ in 0..3 {
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Method, Request, StatusCode, header::AUTHORIZATION},
|
||||
http::{Method, Request, StatusCode},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::api_router_with_origins;
|
||||
use super::models::ModelsExecution;
|
||||
|
||||
const REMOVED_BODY_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
const LARGE_BODY_BYTES: usize = 17 * 1024 * 1024;
|
||||
|
||||
#[tokio::test]
|
||||
async fn browser_origin_controls_http_admin_sessions_without_configuration() {
|
||||
@@ -117,64 +117,16 @@ async fn browser_origin_controls_http_admin_sessions_without_configuration() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_responses_review_route_should_not_reach_the_responses_handler() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::post("/v1/responses/review")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed review request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed review request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_models_catalog_extension_should_use_standard_model_lookup() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/catalog")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model catalog request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model catalog request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removed_model_info_route_should_return_not_found() {
|
||||
let response = api_router_with_origins(ModelsExecution::new(), Vec::new())
|
||||
.await
|
||||
.oneshot(
|
||||
Request::get("/v1/models/model-a/info")
|
||||
.header(AUTHORIZATION, "Bearer sk_models_test")
|
||||
.body(Body::empty())
|
||||
.expect("build removed model info request"),
|
||||
)
|
||||
.await
|
||||
.expect("route removed model info request");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn responses_body_should_accept_payload_above_the_removed_private_limit() {
|
||||
async fn large_responses_bodies_should_reach_authentication() {
|
||||
let router = api_router_with_origins(ModelsExecution::new(), Vec::new()).await;
|
||||
let response = router
|
||||
.oneshot(
|
||||
Request::post("/v1/responses")
|
||||
.body(Body::from(vec![b'a'; REMOVED_BODY_LIMIT_BYTES + 1]))
|
||||
.expect("build request above the removed limit"),
|
||||
.body(Body::from(vec![b'a'; LARGE_BODY_BYTES]))
|
||||
.expect("build large request"),
|
||||
)
|
||||
.await
|
||||
.expect("route request above the removed limit");
|
||||
.expect("route large request");
|
||||
|
||||
// 请求进入处理器后应因缺少凭据而被拒绝
|
||||
// 若提前施加正文大小限制,则会在认证前返回 413
|
||||
|
||||
@@ -18,6 +18,7 @@ struct State {
|
||||
pub struct ConnectionBudget {
|
||||
state: Arc<Mutex<State>>,
|
||||
now: fn() -> Instant,
|
||||
reserve_next_opening: bool,
|
||||
}
|
||||
|
||||
impl Default for ConnectionBudget {
|
||||
@@ -37,9 +38,26 @@ impl ConnectionBudget {
|
||||
Self {
|
||||
state: Arc::default(),
|
||||
now,
|
||||
reserve_next_opening: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// 仅本次恢复使用一半剩余时间并留下一次 opening;已启动的后台连接不受影响
|
||||
#[must_use]
|
||||
pub fn reserve_next_opening(mut self) -> Self {
|
||||
self.reserve_next_opening = true;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn remaining_openings(&self) -> u32 {
|
||||
let state = self
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
MAX_OPENINGS.saturating_sub(state.openings)
|
||||
}
|
||||
|
||||
/// 在可能建立连接前预约一次 opening;返回本次连接允许使用的剩余时间
|
||||
/// HTTP 连接池隐藏了是否冷建连,因此一次 HTTP opening 也计入次数上限
|
||||
pub fn begin(&self) -> Result<Option<Duration>, ConnectionBudgetExhausted> {
|
||||
@@ -53,11 +71,16 @@ impl ConnectionBudget {
|
||||
let started = *state.started.get_or_insert_with(self.now);
|
||||
let remaining =
|
||||
STARTUP_TIMEOUT.saturating_sub((self.now)().saturating_duration_since(started));
|
||||
if state.openings >= MAX_OPENINGS || remaining.is_zero() {
|
||||
let opening_limit = MAX_OPENINGS - u32::from(self.reserve_next_opening);
|
||||
if state.openings >= opening_limit || remaining.is_zero() {
|
||||
return Err(ConnectionBudgetExhausted);
|
||||
}
|
||||
state.openings += 1;
|
||||
Ok(Some(remaining))
|
||||
Ok(Some(if self.reserve_next_opening {
|
||||
remaining / 2
|
||||
} else {
|
||||
remaining
|
||||
}))
|
||||
}
|
||||
|
||||
/// 仅连接成功不足以清零;确认业务发送后退出新增的 NotSent 恢复策略
|
||||
|
||||
@@ -318,6 +318,7 @@ where
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget::default(),
|
||||
connection_budget: super::connection::ConnectionBudget::default(),
|
||||
connection_retries: 0,
|
||||
connection_rotation: None,
|
||||
observation: ResponseObservation::new(timing_started_at),
|
||||
request_observation,
|
||||
budget_prior_attempts_usd: Decimal::ZERO,
|
||||
@@ -424,6 +425,7 @@ struct CurrentAttempt {
|
||||
metadata: ProviderCallMetadata,
|
||||
trigger: AttemptTrigger,
|
||||
transport: AttemptTransport,
|
||||
connection_rotation_reserved: bool,
|
||||
index: NonZeroU32,
|
||||
started_at: SystemTime,
|
||||
send_observed: bool,
|
||||
@@ -458,6 +460,7 @@ struct PendingAttemptRetry {
|
||||
transport: AttemptTransport,
|
||||
delay: Duration,
|
||||
transport_recovery: bool,
|
||||
reserve_connection_rotation: bool,
|
||||
}
|
||||
|
||||
/// API 可逐事件消费的 Core 执行会话
|
||||
@@ -471,6 +474,8 @@ pub struct ResponseExecutionSession<S: ?Sized> {
|
||||
concurrency_wait_budget: ConcurrencyWaitBudget,
|
||||
connection_budget: super::connection::ConnectionBudget,
|
||||
connection_retries: u32,
|
||||
/// 原账号恢复仍建连失败后尝试重选;选号不可用时消费该状态,恢复原账号剩余预算
|
||||
connection_rotation: Option<PendingAttemptRetry>,
|
||||
observation: ResponseObservation,
|
||||
request_observation: Option<RequestObservationDispatch>,
|
||||
budget_prior_attempts_usd: Decimal,
|
||||
@@ -999,6 +1004,9 @@ where
|
||||
let is_transport_recovery = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.transport_recovery);
|
||||
let connection_rotation_reserved = pending_retry
|
||||
.as_ref()
|
||||
.is_some_and(|retry| retry.reserve_connection_rotation);
|
||||
if !is_transport_recovery && self.routing_attempts >= self.plan.max_attempts().get() {
|
||||
return Err(EngineError::EmptyRoutingPlan);
|
||||
}
|
||||
@@ -1060,7 +1068,9 @@ where
|
||||
self.recovery_account
|
||||
.take()
|
||||
.or_else(|| self.account_selection.required_account().cloned()),
|
||||
AttemptTransport::Default,
|
||||
self.connection_rotation
|
||||
.as_ref()
|
||||
.map_or(AttemptTransport::Default, |recovery| recovery.transport),
|
||||
),
|
||||
}
|
||||
};
|
||||
@@ -1123,7 +1133,11 @@ where
|
||||
.with_request_location(self.plan.request_location().cloned())
|
||||
.with_privacy(self.plan.privacy())
|
||||
.with_concurrency_wait_budget(self.concurrency_wait_budget.clone())
|
||||
.with_connection_budget(self.connection_budget.clone())
|
||||
.with_connection_budget(if connection_rotation_reserved {
|
||||
self.connection_budget.clone().reserve_next_opening()
|
||||
} else {
|
||||
self.connection_budget.clone()
|
||||
})
|
||||
.with_timing_started_at(self.observation.timing_started_at)
|
||||
.with_request_policy(self.request_policy.clone())
|
||||
.with_execution_effects(self.execution_effects.as_ref().map(Arc::clone))
|
||||
@@ -1186,9 +1200,21 @@ where
|
||||
Ok(stream) => stream,
|
||||
Err(error) => {
|
||||
attempt_trace.record_provider_failure(&error);
|
||||
let connection_fallback = self.connection_rotation.is_some()
|
||||
&& !self.connection_budget.exhausted()
|
||||
&& !error.retry_is_prohibited()
|
||||
&& error.send_state() == UpstreamSendState::NotSent
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::NoEligibleAccount
|
||||
| ProviderErrorKind::AccountCapacityUnavailable
|
||||
| ProviderErrorKind::QuotaExhausted
|
||||
| ProviderErrorKind::ConcurrencyQueueFull
|
||||
| ProviderErrorKind::ConcurrencyQueueTimeout
|
||||
);
|
||||
let continuation_retry = !error.retry_is_prohibited()
|
||||
&& self.prepare_unavailable_native_continuation_replay(&error);
|
||||
let candidate_retry = !error.retry_is_prohibited() && !continuation_retry
|
||||
let candidate_retry = !error.retry_is_prohibited() && !connection_fallback && !continuation_retry
|
||||
&& matches!(
|
||||
error.kind(),
|
||||
ProviderErrorKind::AccountCapacityUnavailable
|
||||
@@ -1224,12 +1250,15 @@ where
|
||||
.get()
|
||||
.saturating_sub(self.routing_attempts),
|
||||
remaining_deadline: Duration::ZERO,
|
||||
retry_allowed: continuation_retry || candidate_retry,
|
||||
retry_allowed: connection_fallback
|
||||
|| continuation_retry
|
||||
|| candidate_retry,
|
||||
})
|
||||
.await?;
|
||||
attempt_trace.record("retry.decided", json!({
|
||||
"phase": "prepare", "retryable": retryable,
|
||||
"continuationRetry": continuation_retry, "candidateRetry": candidate_retry,
|
||||
"connectionFallback": connection_fallback,
|
||||
"retryProhibited": error.retry_is_prohibited(),
|
||||
"sendState": error.send_state().as_str(),
|
||||
"requestSendState": self.current_send_state().as_str(),
|
||||
@@ -1239,6 +1268,17 @@ where
|
||||
"remainingRoutingAttempts": self.plan.max_attempts().get().saturating_sub(self.routing_attempts),
|
||||
}));
|
||||
if retryable {
|
||||
if connection_fallback {
|
||||
let recovery = self
|
||||
.connection_rotation
|
||||
.take()
|
||||
.expect("pending connection rotation");
|
||||
self.excluded_accounts.remove(&recovery.account);
|
||||
self.pending_retry = Some(recovery);
|
||||
} else if candidate_retry {
|
||||
// 原账号只属于原 Provider,跨候选后不能拿旧钉选作恢复兜底
|
||||
self.connection_rotation = None;
|
||||
}
|
||||
return Ok(Some(PullOutcome::AttemptDiscarded));
|
||||
}
|
||||
if matches!(
|
||||
@@ -1306,6 +1346,7 @@ where
|
||||
}
|
||||
|
||||
let metadata = stream.metadata().clone();
|
||||
self.connection_rotation = None;
|
||||
attempt_trace.record("account.selected", json!({
|
||||
"provider": metadata.provider().as_str(), "accountId": metadata.provider_account_id().as_str(),
|
||||
"transport": metadata.transport().as_str(),
|
||||
@@ -1429,6 +1470,7 @@ where
|
||||
metadata,
|
||||
trigger,
|
||||
transport: attempt_transport,
|
||||
connection_rotation_reserved,
|
||||
index: next_attempt,
|
||||
started_at: SystemTime::now(),
|
||||
send_observed: false,
|
||||
@@ -1642,6 +1684,18 @@ where
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let can_rotate_connection = connection_retry_requested
|
||||
&& self.account_selection.required_account().is_none()
|
||||
&& self.routing_attempts < self.plan.max_attempts().get()
|
||||
&& self.account_rotations
|
||||
< self.plan.account_selection_policy().max_account_rotations();
|
||||
// A 先恢复一次,再利用预留预算重选;重选继续走已有降级传输,避免 WS 抢占最后一次 opening
|
||||
let connection_rotation = transport_recovery
|
||||
.filter(|_| can_rotate_connection && current.connection_rotation_reserved);
|
||||
let reserve_connection_rotation = can_rotate_connection
|
||||
&& !current.connection_rotation_reserved
|
||||
&& self.connection_budget.remaining_openings() >= 2;
|
||||
let transport_recovery = transport_recovery.filter(|_| connection_rotation.is_none());
|
||||
let ordinary_retry = !self.connection_budget.exhausted()
|
||||
&& !connection_retry_requested
|
||||
&& !execution_effect_observed
|
||||
@@ -1714,6 +1768,7 @@ where
|
||||
|| same_account_retry
|
||||
|| transient_retry.is_some()
|
||||
|| transport_recovery.is_some()
|
||||
|| connection_rotation.is_some()
|
||||
|| rotation_retry);
|
||||
|
||||
let retryable = match self
|
||||
@@ -1750,6 +1805,8 @@ where
|
||||
"retryable": retryable, "continuationRetry": continuation_retry,
|
||||
"sameAccountRetry": same_account_retry, "accountRotationRetry": account_rotation_retry,
|
||||
"ordinaryRetry": ordinary_retry, "transportRecovery": transport_recovery.is_some(),
|
||||
"connectionRotation": connection_rotation.is_some(),
|
||||
"reserveConnectionRotation": reserve_connection_rotation,
|
||||
"transientRetry": transient_retry.is_some(),
|
||||
"clientRetrySuppressed": suppress_transient_retry,
|
||||
"rotationRetry": rotation_retry, "accountRotations": self.account_rotations,
|
||||
@@ -1795,6 +1852,7 @@ where
|
||||
transport: current.transport,
|
||||
delay,
|
||||
transport_recovery: false,
|
||||
reserve_connection_rotation: false,
|
||||
});
|
||||
} else if let Some((transport, delay)) = transport_recovery {
|
||||
self.pending_retry = Some(PendingAttemptRetry {
|
||||
@@ -1802,6 +1860,17 @@ where
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation,
|
||||
});
|
||||
} else if let Some((transport, delay)) = connection_rotation {
|
||||
let account = current.metadata.provider_account_id().clone();
|
||||
self.excluded_accounts.insert(account.clone());
|
||||
self.connection_rotation = Some(PendingAttemptRetry {
|
||||
account,
|
||||
transport,
|
||||
delay,
|
||||
transport_recovery: true,
|
||||
reserve_connection_rotation: false,
|
||||
});
|
||||
} else if !continuation_retry {
|
||||
self.excluded_accounts
|
||||
|
||||
@@ -27,6 +27,40 @@ fn elapsed_recovery_window_is_shared_without_waiting_for_real_time() {
|
||||
assert_eq!(fallback.startup_remaining(), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_account_retry_preserves_time_and_an_opening_for_rotation() {
|
||||
use std::{
|
||||
cell::Cell,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
thread_local! {
|
||||
static NOW: Cell<Instant> = Cell::new(Instant::now());
|
||||
}
|
||||
fn now() -> Instant {
|
||||
NOW.get()
|
||||
}
|
||||
let request = ConnectionBudget::with_clock(now);
|
||||
// 后台 WS 先占一次,HTTP 首次建连再占一次,模拟代理持续不响应
|
||||
let background = request.clone();
|
||||
assert_eq!(background.begin().unwrap(), Some(Duration::from_secs(30)));
|
||||
NOW.set(now() + Duration::from_secs(1));
|
||||
assert_eq!(request.begin().unwrap(), Some(Duration::from_secs(29)));
|
||||
NOW.set(now() + Duration::from_secs(15));
|
||||
assert_eq!(request.remaining_openings(), 2);
|
||||
let retry = request.clone().reserve_next_opening();
|
||||
assert_eq!(retry.begin().unwrap(), Some(Duration::from_secs(7)));
|
||||
assert_eq!(request.remaining_openings(), 1);
|
||||
assert!(retry.begin().is_err());
|
||||
assert!(!request.exhausted());
|
||||
NOW.set(now() + Duration::from_secs(7));
|
||||
// 重选使用原预算,既不继承 A 的局部时间限制,也不能重置总次数
|
||||
assert_eq!(request.begin().unwrap(), Some(Duration::from_secs(7)));
|
||||
assert!(request.exhausted());
|
||||
assert!(background.begin().is_err());
|
||||
request.complete();
|
||||
assert_eq!(retry.begin().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_transports_share_one_opening_budget_but_other_requests_do_not() {
|
||||
let request = ConnectionBudget::default();
|
||||
|
||||
@@ -5831,11 +5831,11 @@ fn provider_default_profile_is_applied_once_and_frozen_across_account_retries()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
fn connection_recovery_rotates_only_after_retrying_the_first_account() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_only",
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
@@ -5843,11 +5843,10 @@ fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_only",
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
@@ -5863,37 +5862,250 @@ fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 4);
|
||||
let account = ProviderAccountId::new("acct_only").unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
let first = ProviderAccountId::new("acct_first").unwrap();
|
||||
assert_eq!(contexts[1].required_account(), Some(&first));
|
||||
assert!(!contexts[1].excluded_accounts().contains(&first));
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert!(contexts[2].excluded_accounts().contains(&first));
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert_eq!(context.transport(), AttemptTransport::Fallback);
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(store.state.lock().unwrap().intermediate_failures, 2);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 4);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_keeps_one_account_and_http_transport_until_success() {
|
||||
for unavailable in [
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
ProviderErrorKind::AccountCapacityUnavailable,
|
||||
ProviderErrorKind::QuotaExhausted,
|
||||
ProviderErrorKind::ConcurrencyQueueFull,
|
||||
ProviderErrorKind::ConcurrencyQueueTimeout,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Error(ProviderError::new(unavailable, UpstreamSendState::NotSent)),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 5);
|
||||
let account = ProviderAccountId::new("acct_only").unwrap();
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert!(contexts[2].excluded_accounts().contains(&account));
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Fallback);
|
||||
for index in [1, 3, 4] {
|
||||
let context = &contexts[index];
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert_eq!(context.transport(), AttemptTransport::Fallback);
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 3);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_rotation_does_not_bypass_selection_replay_prohibition() {
|
||||
for send_state in [
|
||||
UpstreamSendState::NotSent,
|
||||
UpstreamSendState::Sent,
|
||||
UpstreamSendState::Ambiguous,
|
||||
] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let selection_error = ProviderError::new(ProviderErrorKind::NoEligibleAccount, send_state);
|
||||
let selection_error = if send_state == UpstreamSendState::NotSent {
|
||||
selection_error.with_retry_prohibited()
|
||||
} else {
|
||||
selection_error
|
||||
};
|
||||
let (coordinator, _, provider) =
|
||||
coordinator(vec![failure(), failure(), Script::Error(selection_error)]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(block_on(session.collect_uncommitted()).is_err());
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 3);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_does_not_delay_existing_account_rotation() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(vec![
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_first",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::RateLimited,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_status(429)
|
||||
.with_replay_safe())],
|
||||
},
|
||||
Script::Stream {
|
||||
account_id: "acct_second",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
block_on(session.commit_downstream(Some(200))).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
let first = ProviderAccountId::new("acct_first").unwrap();
|
||||
// 临时建连失败先在原账号恢复,后续明确可重放的限流拒绝仍按已有规则换号
|
||||
assert_eq!(contexts[1].required_account(), Some(&first));
|
||||
assert_eq!(contexts[1].transport(), AttemptTransport::Fallback);
|
||||
assert!(!contexts[1].excluded_accounts().contains(&first));
|
||||
assert!(contexts[2].required_account().is_none());
|
||||
assert_eq!(contexts[2].transport(), AttemptTransport::Default);
|
||||
assert!(contexts[2].excluded_accounts().contains(&first));
|
||||
let state = store.state.lock().unwrap();
|
||||
assert_eq!(state.intermediate_failures, 2);
|
||||
assert_eq!(state.finalizations[0].outcome, ExecutionOutcome::Succeeded);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 2);
|
||||
drop(session);
|
||||
assert_eq!(provider.released_leases.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_stays_on_account_when_rotation_is_not_allowed() {
|
||||
for required in [true, false] {
|
||||
let operation = generate_operation();
|
||||
let route_plan = if required {
|
||||
plan(&operation)
|
||||
} else {
|
||||
plan_with_policy(
|
||||
&operation,
|
||||
AccountSelectionPolicy::new(RotationStrategy::Smart, NonZeroU32::new(2).unwrap())
|
||||
.with_max_account_rotations(0),
|
||||
)
|
||||
};
|
||||
let failure = || Script::Stream {
|
||||
account_id: "acct_required",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
};
|
||||
let (coordinator, _, provider) = coordinator(vec![
|
||||
failure(),
|
||||
failure(),
|
||||
Script::Stream {
|
||||
account_id: "acct_required",
|
||||
items: complete_stream(None),
|
||||
},
|
||||
]);
|
||||
let account = ProviderAccountId::new("acct_required").unwrap();
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
route_plan,
|
||||
required.then_some(account.clone()),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
))
|
||||
.unwrap();
|
||||
block_on(session.collect_uncommitted()).unwrap();
|
||||
let contexts = provider.contexts.lock().unwrap();
|
||||
assert_eq!(contexts.len(), 3);
|
||||
assert_eq!(contexts[0].required_account(), required.then_some(&account));
|
||||
for context in contexts.iter().skip(1) {
|
||||
assert_eq!(context.required_account(), Some(&account));
|
||||
assert!(!context.excluded_accounts().contains(&account));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_recovery_exhaustion_preserves_transport_failure() {
|
||||
let operation = generate_operation();
|
||||
let route_plan = plan(&operation);
|
||||
let (coordinator, store, provider) = coordinator(
|
||||
(0..4)
|
||||
.map(|_| Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
})
|
||||
.collect(),
|
||||
let mut scripts: Vec<_> = (0..4)
|
||||
.map(|_| Script::Stream {
|
||||
account_id: "acct_only",
|
||||
items: vec![Err(ProviderError::new(
|
||||
ProviderErrorKind::Transport,
|
||||
UpstreamSendState::NotSent,
|
||||
)
|
||||
.with_connection_retry(AttemptTransport::Fallback))],
|
||||
})
|
||||
.collect();
|
||||
scripts.insert(
|
||||
2,
|
||||
Script::Error(ProviderError::new(
|
||||
ProviderErrorKind::NoEligibleAccount,
|
||||
UpstreamSendState::NotSent,
|
||||
)),
|
||||
);
|
||||
let (coordinator, store, provider) = coordinator(scripts);
|
||||
let mut session = block_on(coordinator.start(
|
||||
model_request(&operation, SystemTime::now() + Duration::from_secs(30)),
|
||||
operation,
|
||||
@@ -5907,7 +6119,7 @@ fn connection_recovery_exhaustion_preserves_transport_failure() {
|
||||
assert!(
|
||||
matches!(error, EngineError::Provider(ref error) if error.kind() == ProviderErrorKind::Transport)
|
||||
);
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 4);
|
||||
assert_eq!(provider.contexts.lock().unwrap().len(), 5);
|
||||
assert_eq!(
|
||||
store.state.lock().unwrap().finalizations[0].attempt_count,
|
||||
4
|
||||
|
||||
@@ -10,9 +10,7 @@ use gateway_core::upstream::UpstreamTransport;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn provider_event_keeps_large_wire_and_observation_payloads_behind_one_indirection() {
|
||||
assert!(std::mem::size_of::<ProviderEvent>() <= 64);
|
||||
|
||||
fn provider_event_should_distinguish_client_delivery_from_observation() {
|
||||
let wire = ProviderEvent::wire(
|
||||
ProtocolWireEvent::json("openai", None, json!({"type": "response.created"}))
|
||||
.expect("wire event"),
|
||||
|
||||
@@ -378,19 +378,38 @@ impl PendingResponse {
|
||||
|
||||
impl HttpBody {
|
||||
pub async fn read(&mut self, maximum: usize) -> Result<Option<Bytes>, HttpError> {
|
||||
self.read_until(maximum, self.deadline).await
|
||||
}
|
||||
|
||||
/// 资源 owner 已接管总寿命时,每次读取使用独立预算,取消仍由 owner 回收正文
|
||||
pub async fn read_with_timeout(
|
||||
&mut self,
|
||||
maximum: usize,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<Bytes>, HttpError> {
|
||||
let deadline = Instant::now()
|
||||
.checked_add(timeout)
|
||||
.ok_or_else(|| HttpError::sent("read timeout"))?;
|
||||
self.read_until(maximum, deadline).await
|
||||
}
|
||||
|
||||
async fn read_until(
|
||||
&mut self,
|
||||
maximum: usize,
|
||||
deadline: Instant,
|
||||
) -> Result<Option<Bytes>, HttpError> {
|
||||
if maximum == 0 || maximum > 64 * 1024 {
|
||||
return Err(HttpError::sent("read size"));
|
||||
}
|
||||
loop {
|
||||
// 已缓冲的数据和就绪帧也受同一期限约束,慢读者不能无限续用 HTTP 资源
|
||||
if Instant::now() >= self.deadline {
|
||||
// 已缓冲的数据和就绪帧也受本次期限约束,不能绕过 owner 的读取预算
|
||||
if Instant::now() >= deadline {
|
||||
return Err(HttpError::sent("response deadline").with_kind(HttpErrorKind::Timeout));
|
||||
}
|
||||
if !self.pending.is_empty() {
|
||||
return Ok(Some(self.pending.split_to(maximum.min(self.pending.len()))));
|
||||
}
|
||||
self.pending = match tokio::time::timeout_at(self.deadline, self.response.frame()).await
|
||||
{
|
||||
self.pending = match tokio::time::timeout_at(deadline, self.response.frame()).await {
|
||||
Ok(Some(Ok(frame))) => match frame.into_data() {
|
||||
Ok(chunk) => chunk,
|
||||
Err(_) => continue,
|
||||
|
||||
@@ -33,7 +33,8 @@ pub fn worker(store: Arc<dyn RetentionStore>) -> Result<WorkerContribution, Work
|
||||
// 保持稳定的租约身份,滚动升级期间仍与已有进程互斥
|
||||
let id = WorkerId::try_new(WorkerKind::Retention, "postgres")?;
|
||||
let schedule = WorkerSchedule::try_new(
|
||||
Duration::from_secs(60 * 60),
|
||||
// 有界单轮结束后一分钟继续检查,避免日级写入量超过清理容量
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(60),
|
||||
Duration::from_secs(15 * 60),
|
||||
|
||||
@@ -162,20 +162,15 @@ fn logging_rejects_zero_retention_windows() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_defaults_to_time_retention_and_ignores_removed_count_retention() {
|
||||
let mut value = serde_json::json!({
|
||||
fn logging_defaults_to_time_retention() {
|
||||
let value = serde_json::json!({
|
||||
"level": "info", "stdout": true,
|
||||
"file": { "enabled": true, "directory": "logs", "max_file_size_mb": 20 }
|
||||
});
|
||||
let config: LoggingConfig = serde_json::from_value(value.clone()).unwrap();
|
||||
let config: LoggingConfig = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(config.request_dump_retention_days, 1.0);
|
||||
assert_eq!(config.file.retention_days, 7);
|
||||
assert!(!config.oauth_recovery);
|
||||
value["file"]["max_files"] = serde_json::json!(20);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<LoggingConfig>(value).unwrap(),
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -221,6 +221,63 @@ async fn buffered_body_cannot_extend_the_request_deadline() {
|
||||
assert_eq!(error.kind(), HttpErrorKind::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_body_reads_outlive_opening_but_keep_a_per_read_timeout() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (send_next, next) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut head = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0 && head.len() < 16 * 1024);
|
||||
head.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\na")
|
||||
.await
|
||||
.unwrap();
|
||||
next.await.unwrap();
|
||||
socket.write_all(b"b").await.unwrap();
|
||||
assert_eq!(socket.read(&mut buffer).await.unwrap(), 0);
|
||||
});
|
||||
let mut response = HttpClient::new()
|
||||
.unwrap()
|
||||
.open(request(url), None, &local_network(), Duration::from_secs(1))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.body.read(1).await.unwrap().unwrap(), "a");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
send_next.send(()).unwrap();
|
||||
assert_eq!(
|
||||
response
|
||||
.body
|
||||
.read_with_timeout(1, Duration::from_secs(1))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
"b"
|
||||
);
|
||||
let error = response
|
||||
.body
|
||||
.read_with_timeout(1, Duration::from_millis(100))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), HttpErrorKind::Timeout);
|
||||
assert_eq!(error.send_state, UpstreamSendState::Sent);
|
||||
drop(response);
|
||||
tokio::time::timeout(Duration::from_secs(1), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_dns_releases_the_request_slot_without_starting_http() {
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -177,6 +177,8 @@ impl Execution {
|
||||
));
|
||||
}
|
||||
let mut context = self.adapter.session.context(Stage::Upstream, timeout);
|
||||
// 初始回复后由模型执行持有流的期限与取消,RPC 预算只约束建立阶段
|
||||
context.resource_stream = true;
|
||||
context.request =
|
||||
crate::adapter::request_context(invocation.context.extension_scope().request());
|
||||
context.request_id = Some(invocation.context.request_id().as_str().to_owned());
|
||||
|
||||
@@ -142,11 +142,12 @@ impl HttpCallbacks<'_> {
|
||||
let mut body = stream.body.try_lock().map_err(|_| {
|
||||
PluginFault::new(ErrorCode::Capacity, "HTTP stream already has a reader")
|
||||
})?;
|
||||
let timeout = call.timeout()?;
|
||||
let mut closed = stream.closed.subscribe();
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = closed.wait_for(|closed| *closed) => Err(denied()),
|
||||
result = body.as_mut().ok_or_else(denied)?.read(request.maximum_bytes as usize) => result.map_err(http_error),
|
||||
result = body.as_mut().ok_or_else(denied)?.read_with_timeout(request.maximum_bytes as usize, timeout) => result.map_err(http_error),
|
||||
};
|
||||
if !matches!(&result, Ok(Some(_))) {
|
||||
body.take();
|
||||
|
||||
@@ -48,6 +48,14 @@ async fn setup_with_restart_circuit(
|
||||
async fn setup_with_contributions_and_restart_circuit(
|
||||
contributes: Contributions,
|
||||
restart_circuit: PluginRestartCircuitConfig,
|
||||
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
||||
setup_with_limits(contributes, restart_circuit, RpcLimits::default()).await
|
||||
}
|
||||
|
||||
async fn setup_with_limits(
|
||||
contributes: Contributions,
|
||||
restart_circuit: PluginRestartCircuitConfig,
|
||||
rpc_limits: RpcLimits,
|
||||
) -> (tempfile::TempDir, Arc<Store>, PluginRuntime) {
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let artifact = PackageInspector::new(PackageLimits::default(), "1.0.0".parse().unwrap())
|
||||
@@ -83,7 +91,7 @@ async fn setup_with_contributions_and_restart_circuit(
|
||||
cache_directory: cache.path().to_owned(),
|
||||
host_version: "1.0.0".parse().unwrap(),
|
||||
package_limits: PackageLimits::default(),
|
||||
rpc_limits: RpcLimits::default(),
|
||||
rpc_limits,
|
||||
restart_circuit,
|
||||
},
|
||||
Arc::new(gateway_host::outbound::HttpClient::new().unwrap()),
|
||||
|
||||
@@ -103,10 +103,24 @@ async fn middleware_preserves_three_state_fast_and_sources() {
|
||||
Some(90_000)
|
||||
);
|
||||
let sources = settings.inspect();
|
||||
assert!(sources["execution"].get("disable_fast").is_none());
|
||||
if mode != FastMode::Default && mode == expected {
|
||||
assert_eq!(sources["execution"]["fast_mode"]["instance_id"], "seed");
|
||||
let fast_source = &sources["execution"]["fast_mode"];
|
||||
if mode == FastMode::Default && expected == FastMode::Default {
|
||||
assert!(fast_source.is_null());
|
||||
} else {
|
||||
assert_eq!(
|
||||
fast_source["instance_id"],
|
||||
if mode == expected {
|
||||
"seed"
|
||||
} else {
|
||||
"fast-settings"
|
||||
}
|
||||
);
|
||||
assert_eq!(fast_source["value"], expected.as_str());
|
||||
}
|
||||
assert_eq!(
|
||||
sources["execution"]["timeout_ms"]["instance_id"],
|
||||
"fast-settings"
|
||||
);
|
||||
next.run(request).await
|
||||
})
|
||||
}),
|
||||
|
||||
@@ -101,6 +101,17 @@ async fn setup(
|
||||
tempfile::TempDir,
|
||||
Arc<super::Store>,
|
||||
gateway_plugin_runtime::PluginRuntime,
|
||||
) {
|
||||
setup_with_limits(config, gateway_plugin_runtime::RpcLimits::default()).await
|
||||
}
|
||||
|
||||
async fn setup_with_limits(
|
||||
config: Value,
|
||||
limits: gateway_plugin_runtime::RpcLimits,
|
||||
) -> (
|
||||
tempfile::TempDir,
|
||||
Arc<super::Store>,
|
||||
gateway_plugin_runtime::PluginRuntime,
|
||||
) {
|
||||
let contributes = Contributions::from([crate::support::contribution(
|
||||
Capability::UpstreamAdapter,
|
||||
@@ -109,7 +120,7 @@ async fn setup(
|
||||
vec!["openai".into()],
|
||||
)]);
|
||||
let (cache, store, runtime) =
|
||||
super::setup_with_contributions_and_restart_circuit(contributes, Default::default()).await;
|
||||
super::setup_with_limits(contributes, Default::default(), limits).await;
|
||||
{
|
||||
let mut snapshot = store.snapshot.lock().unwrap();
|
||||
let instance = &mut snapshot.instances[0];
|
||||
@@ -309,6 +320,132 @@ async fn adapter_http_and_sse_are_cold_authenticated_and_settled_once() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_sse_outlives_rpc_budget_and_cancellation_closes_its_resources() {
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
for cancel in [false, true] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (started, receiving) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0; 1024];
|
||||
let head_end = loop {
|
||||
if let Some(end) = request.windows(4).position(|bytes| bytes == b"\r\n\r\n") {
|
||||
break end + 4;
|
||||
}
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0 && request.len() < 16 * 1024);
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
};
|
||||
let length: usize = std::str::from_utf8(&request[..head_end])
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.unwrap()
|
||||
.1
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap();
|
||||
while request.len() < head_end + length {
|
||||
let count = socket.read(&mut buffer).await.unwrap();
|
||||
assert!(count > 0);
|
||||
request.extend_from_slice(&buffer[..count]);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n: heartbeat\n\n")
|
||||
.await
|
||||
.unwrap();
|
||||
started.send(()).unwrap();
|
||||
for _ in 0..6 {
|
||||
tokio::select! {
|
||||
result = socket.read(&mut buffer) => {
|
||||
assert_eq!(result.unwrap(), 0);
|
||||
return false;
|
||||
}
|
||||
() = tokio::time::sleep(Duration::from_millis(200)) => {}
|
||||
}
|
||||
if socket.write_all(b": heartbeat\n\n").await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
socket.shutdown().await.unwrap();
|
||||
true
|
||||
});
|
||||
let mut config = configuration(&base, true);
|
||||
// 此夹具汇总 HTTP 正文后产出规范事件,不把多个心跳伪装成一个 SSE 帧
|
||||
config["upstream_events"][1]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("wire");
|
||||
let (cache, _, runtime) = setup_with_limits(
|
||||
config,
|
||||
gateway_plugin_runtime::RpcLimits {
|
||||
maximum_call_timeout: Duration::from_millis(500),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let generation =
|
||||
ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let account = Account::new();
|
||||
let context = context(&runtime, &generation, "key-one");
|
||||
let cancellation = context.cancellation().clone();
|
||||
let mut stream = execute_with_context(account.clone(), context, None);
|
||||
let consume = async {
|
||||
let mut completed = 0;
|
||||
while let Some(event) = stream.next().await {
|
||||
completed += event?
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.filter(|fact| matches!(fact, GatewayEvent::Completed(_)))
|
||||
.count();
|
||||
}
|
||||
Ok::<_, ProviderError>(completed)
|
||||
};
|
||||
let interrupt = async {
|
||||
receiving.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(800)).await;
|
||||
if cancel {
|
||||
cancellation.cancel();
|
||||
}
|
||||
};
|
||||
let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::join!(consume, interrupt)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if cancel {
|
||||
assert_eq!(result.unwrap_err().kind(), ProviderErrorKind::Cancelled);
|
||||
} else {
|
||||
assert_eq!(result.unwrap(), 1);
|
||||
}
|
||||
drop(stream);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(2), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
!cancel
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while Arc::strong_count(&account) != 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
drop(generation);
|
||||
runtime.shutdown().await;
|
||||
super::wait_until_empty(cache.path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_accepts_service_tier_resolved_by_the_terminal_response() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -278,7 +278,7 @@ fn icon_content_must_fully_decode_match_its_type_and_fit_dimensions() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obsolete_manifest_fields_are_rejected_without_aliases() {
|
||||
fn manifest_rejects_unknown_fields_before_package_validation() {
|
||||
let manifest = serde_json::json!({
|
||||
"manifestVersion":2,
|
||||
"name": "example",
|
||||
@@ -290,7 +290,8 @@ fn obsolete_manifest_fields_are_rejected_without_aliases() {
|
||||
"engines": {"codex-proxy-rs": ">=1.0.0, <2.0.0"},
|
||||
"main": "bin/worker",
|
||||
"runtime": "trustedProcess",
|
||||
"capabilities": [{"capability":"request_normalizer"}]
|
||||
"contributes": {},
|
||||
"unknownField": true
|
||||
});
|
||||
let archive = crate::support::archive(BTreeMap::from([(
|
||||
"plugin.json".to_owned(),
|
||||
|
||||
@@ -82,6 +82,10 @@ let (message, ()) = tokio::try_join!(
|
||||
包含标准事实与可选原始 wire;事件需符合开始、内容、工具、用量和唯一结束的顺序。
|
||||
原始 JSON / SSE 字节与标准事实分开传递,错误使用 `UpstreamFailure`;宿主网络观测决定发送状态,插件不能将其降为未发送
|
||||
|
||||
初始 RPC 回复受插件处理预算约束,默认上限为 120 秒;持续接收与解析应放在返回的流中。
|
||||
初始流式回复后,流的寿命遵循模型请求期限与取消,受管 HTTP 正文每次读取单独计时。
|
||||
持续有数据的流不会因初始 RPC 预算耗尽而结束,单次读取超时、客户端取消和流结束仍会回收资源
|
||||
|
||||
用量提交标准 `Usage`,费用由内置 Provider 按已冻结的上游模型与价格计算,响应回显模型不切换计价。
|
||||
`service_tier` 记录上游回显的最新档位,允许上游在完成时确定实际值;OpenAI 计价沿用请求策略,回显档位仅作观测。
|
||||
未知用量或价格保留未知。宿主确认 RPC 正常结束后才发布完成事实,插件不能独立结算或组织换号重试
|
||||
|
||||
@@ -132,52 +132,40 @@ fn installation_checks_protocol_engine_and_target() {
|
||||
|
||||
#[test]
|
||||
fn capability_contract_versions_are_checked_independently_of_rpc_version() {
|
||||
let mut manifest = packaged_manifest();
|
||||
assert!(manifest.validate().is_ok());
|
||||
for version in [0, 1, 2, 3, 4, 6, u32::MAX] {
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::Middleware)
|
||||
.unwrap()
|
||||
.version = version;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
for (capability, stage, supported, needs_formats) in [
|
||||
(Capability::Middleware, Stage::Request, 5, true),
|
||||
(Capability::Management, Stage::Management, 2, false),
|
||||
(Capability::UpstreamAdapter, Stage::Upstream, 3, true),
|
||||
(Capability::Scheduler, Stage::Scheduling, 3, false),
|
||||
] {
|
||||
for mut manifest in [source_manifest(), packaged_manifest()] {
|
||||
let formats = if needs_formats {
|
||||
vec!["openai".into()]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
capability,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.contract".into(),
|
||||
version: supported,
|
||||
stages: vec![stage],
|
||||
input_formats: formats.clone(),
|
||||
output_formats: formats,
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok(), "{capability:?} v{supported}");
|
||||
for version in (0..supported).chain([supported + 1, u32::MAX]) {
|
||||
manifest.contributes.get_mut(&capability).unwrap().version = version;
|
||||
assert_eq!(
|
||||
manifest.validate(),
|
||||
Err(ManifestError::Invalid),
|
||||
"{capability:?} v{version}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::Management,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.management".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Management],
|
||||
input_formats: vec![],
|
||||
output_formats: vec![],
|
||||
},
|
||||
);
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_upstream_contract_requires_an_author_upgrade() {
|
||||
let mut manifest = source_manifest();
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::UpstreamAdapter,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.upstream".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Upstream],
|
||||
input_formats: vec!["openai".into()],
|
||||
output_formats: vec!["openai".into()],
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok());
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::UpstreamAdapter)
|
||||
.unwrap()
|
||||
.version = 1;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -371,18 +359,19 @@ fn state_namespaces_are_base_infrastructure_with_bounded_versions() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_manifest_fields_use_only_the_v3_camel_case_shape() {
|
||||
let value = serde_json::to_value(source_manifest()).unwrap();
|
||||
assert!(value.get("manifestVersion").is_some());
|
||||
assert!(value.get("displayName").is_some());
|
||||
assert!(value.get("configurationSchema").is_some());
|
||||
assert!(value.get("schema_version").is_none());
|
||||
fn manifest_round_trips_public_fields_and_rejects_unknown_fields() {
|
||||
let manifest = source_manifest();
|
||||
let mut value = serde_json::to_value(&manifest).unwrap();
|
||||
assert_eq!(value["manifestVersion"], MANIFEST_VERSION);
|
||||
assert_eq!(value["displayName"], manifest.display_name);
|
||||
assert_eq!(value["configurationSchema"], manifest.configuration_schema);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<Manifest>(value.clone()).unwrap(),
|
||||
manifest
|
||||
);
|
||||
|
||||
let mut old = value;
|
||||
old.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("schema_version".into(), json!(3));
|
||||
assert!(serde_json::from_value::<Manifest>(old).is_err());
|
||||
value["unknownField"] = json!(true);
|
||||
assert!(serde_json::from_value::<Manifest>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -439,26 +428,3 @@ fn contribution_declaration_uses_camel_case_description_fields() {
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scheduler_requires_the_current_contract() {
|
||||
let mut manifest = packaged_manifest();
|
||||
manifest.contributes.clear();
|
||||
manifest.contributes.insert(
|
||||
Capability::Scheduler,
|
||||
ContributionDeclaration {
|
||||
id: "9acme.request-tags.scheduler".into(),
|
||||
version: 3,
|
||||
stages: vec![Stage::Scheduling],
|
||||
input_formats: vec![],
|
||||
output_formats: vec![],
|
||||
},
|
||||
);
|
||||
assert!(manifest.validate().is_ok());
|
||||
manifest
|
||||
.contributes
|
||||
.get_mut(&Capability::Scheduler)
|
||||
.unwrap()
|
||||
.version = 1;
|
||||
assert_eq!(manifest.validate(), Err(ManifestError::Invalid));
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ path = "tests/main.rs"
|
||||
[dependencies]
|
||||
async-stream = "0.3.6"
|
||||
async-trait = "0.1.89"
|
||||
async-compression = { version = "0.4.50", default-features = false, features = ["tokio", "gzip"] }
|
||||
aws-credential-types = "1.2"
|
||||
aws-sdk-s3 = { version = "1.140", default-features = false, features = ["rt-tokio", "default-https-client"] }
|
||||
aws-smithy-runtime-api = "1.14"
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
//! 有界流式归档写入:压缩后限额与摘要覆盖实际上传的字节
|
||||
|
||||
use async_compression::tokio::write::GzipEncoder;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt as _, BufWriter};
|
||||
|
||||
struct ArchiveSink {
|
||||
file: BufWriter<tokio::fs::File>,
|
||||
maximum: u64,
|
||||
size: u64,
|
||||
hash: Sha256,
|
||||
}
|
||||
|
||||
impl AsyncWrite for ArchiveSink {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
if bytes.len() as u64 > self.maximum.saturating_sub(self.size) {
|
||||
return Poll::Ready(Err(io::Error::from(io::ErrorKind::FileTooLarge)));
|
||||
}
|
||||
match Pin::new(&mut self.file).poll_write(cx, bytes) {
|
||||
Poll::Ready(Ok(count)) => {
|
||||
self.size += count as u64;
|
||||
self.hash.update(&bytes[..count]);
|
||||
Poll::Ready(Ok(count))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.file).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.file).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ArchiveWriter(GzipEncoder<ArchiveSink>);
|
||||
|
||||
impl ArchiveWriter {
|
||||
pub fn new(file: std::fs::File, maximum: u64) -> Self {
|
||||
Self(GzipEncoder::new(ArchiveSink {
|
||||
file: BufWriter::new(tokio::fs::File::from_std(file)),
|
||||
maximum,
|
||||
size: 0,
|
||||
hash: Sha256::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn finish(mut self) -> io::Result<(u64, String)> {
|
||||
self.shutdown().await?;
|
||||
let sink = self.0.into_inner();
|
||||
sink.file.get_ref().sync_all().await?;
|
||||
Ok((sink.size, hex::encode(sink.hash.finalize())))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for ArchiveWriter {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
Pin::new(&mut self.0).poll_write(cx, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.0).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
//! S3 对象存储与 `pg_dump` 基础设施适配器
|
||||
|
||||
mod archive;
|
||||
pub mod pg_dump;
|
||||
pub mod s3;
|
||||
mod snapshot;
|
||||
pub mod staging;
|
||||
|
||||
@@ -16,11 +16,12 @@ use tokio::process::Command;
|
||||
use gateway_admin::model::backup::{BackupError, code};
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpArtifact, DumpRequest, StagedArtifact};
|
||||
|
||||
use super::staging::StagingArea;
|
||||
use super::{archive::ArchiveWriter, snapshot::BusinessSnapshot, staging::StagingArea};
|
||||
|
||||
/// `pg_dump` 导出适配器
|
||||
pub struct PgDumpAdapter {
|
||||
staging: Arc<StagingArea>,
|
||||
pool: sqlx::PgPool,
|
||||
/// 不含密码的 PostgreSQL URL,用于 `--dbname`
|
||||
database_url: String,
|
||||
/// 数据库密码,仅通过 `PGPASSWORD` 注入
|
||||
@@ -34,9 +35,11 @@ impl PgDumpAdapter {
|
||||
staging: Arc<StagingArea>,
|
||||
database_url: impl Into<String>,
|
||||
database_password: impl Into<String>,
|
||||
pool: sqlx::PgPool,
|
||||
) -> Self {
|
||||
Self {
|
||||
staging,
|
||||
pool,
|
||||
database_url: database_url.into(),
|
||||
database_password: database_password.into(),
|
||||
}
|
||||
@@ -64,8 +67,14 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
.open(&partial)
|
||||
.map_err(|_| pg_dump_failed("无法创建暂存文件"))?;
|
||||
let _partial_cleanup = PartialArchive(partial.clone());
|
||||
let mut child = Command::new("pg_dump")
|
||||
.arg("--format=custom")
|
||||
let snapshot = tokio::select! {
|
||||
biased;
|
||||
_ = request.cancellation.cancelled() => return Err(cancelled()),
|
||||
result = BusinessSnapshot::begin(&self.pool, &request.backup_id) => result?,
|
||||
};
|
||||
let mut command = Command::new("pg_dump");
|
||||
command
|
||||
.arg("--format=plain")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--dbname")
|
||||
@@ -74,17 +83,22 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
.env("PGCONNECT_TIMEOUT", "10")
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.kill_on_drop(true)
|
||||
.kill_on_drop(true);
|
||||
command.arg("--lock-wait-timeout=5s");
|
||||
snapshot.configure(&mut command);
|
||||
let mut child = command
|
||||
.spawn()
|
||||
.map_err(|_| pg_dump_failed("无法启动 pg_dump 进程"))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut size: u64 = 0;
|
||||
let export = async {
|
||||
let stdout = child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| pg_dump_failed("无法读取 pg_dump 输出"))?;
|
||||
let mut writer = tokio::io::BufWriter::new(tokio::fs::File::from_std(file));
|
||||
let mut writer = ArchiveWriter::new(file, self.staging.max_archive_bytes());
|
||||
writer
|
||||
.write_all(snapshot.restore_preamble().as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
let mut reader = tokio::io::BufReader::new(stdout);
|
||||
let mut buffer = vec![0_u8; 64 * 1024];
|
||||
loop {
|
||||
@@ -95,20 +109,8 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
size = size.saturating_add(n as u64);
|
||||
if size > self.staging.max_archive_bytes() {
|
||||
return Err(staging_space_exhausted());
|
||||
}
|
||||
hasher.update(&buffer[..n]);
|
||||
writer
|
||||
.write_all(&buffer[..n])
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("写入暂存文件失败"))?;
|
||||
writer.write_all(&buffer[..n]).await.map_err(write_failed)?;
|
||||
}
|
||||
writer
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("刷新暂存文件失败"))?;
|
||||
let status = child
|
||||
.wait()
|
||||
.await
|
||||
@@ -116,22 +118,26 @@ impl DatabaseDumpPort for PgDumpAdapter {
|
||||
if !status.success() {
|
||||
return Err(pg_dump_failed("pg_dump 非零退出"));
|
||||
}
|
||||
Ok(())
|
||||
snapshot.append(&mut writer).await?;
|
||||
writer.finish().await.map_err(write_failed)
|
||||
};
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = request.cancellation.cancelled() => Err(cancelled()),
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(15 * 60)) => Err(pg_dump_failed("备份导出超过 15 分钟期限")),
|
||||
result = export => result,
|
||||
};
|
||||
if let Err(error) = result {
|
||||
// 正常错误和取消必须等到进程退出;任务被强制丢弃时由 kill_on_drop 接管
|
||||
if let Err(source) = child.kill().await {
|
||||
tracing::warn!(backup_id = %request.backup_id, error = %source, "终止 pg_dump 失败");
|
||||
let (size, sha256) = match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
// 正常错误和取消必须等到进程退出;任务被强制丢弃时由 kill_on_drop 接管
|
||||
if let Err(source) = child.kill().await {
|
||||
tracing::warn!(backup_id = %request.backup_id, error = %source, "终止 pg_dump 失败");
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
std::fs::rename(&partial, &final_path).map_err(|_| pg_dump_failed("暂存归档改名失败"))?;
|
||||
let sha256 = hex::encode(hasher.finalize());
|
||||
Ok(DumpArtifact {
|
||||
path: final_path,
|
||||
size_bytes: size,
|
||||
@@ -199,7 +205,7 @@ async fn hash_file(path: &std::path::Path) -> Result<String, BackupError> {
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
fn pg_dump_failed(message: &'static str) -> BackupError {
|
||||
pub(super) fn pg_dump_failed(message: &'static str) -> BackupError {
|
||||
BackupError::new(code::PG_DUMP_FAILED, message.to_owned())
|
||||
}
|
||||
|
||||
@@ -213,3 +219,14 @@ fn staging_space_exhausted() -> BackupError {
|
||||
fn cancelled() -> BackupError {
|
||||
BackupError::new(code::CANCELLED, "备份导出已取消".to_owned())
|
||||
}
|
||||
|
||||
pub(super) fn write_failed(error: std::io::Error) -> BackupError {
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::FileTooLarge | std::io::ErrorKind::StorageFull
|
||||
) {
|
||||
staging_space_exhausted()
|
||||
} else {
|
||||
pg_dump_failed("写入暂存归档失败")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! 与业务数据共用快照导出已发布的小时总量,不扫描请求明细或逐请求投影
|
||||
|
||||
use futures::TryStreamExt as _;
|
||||
use gateway_admin::model::backup::{BackupError, code};
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt as _};
|
||||
|
||||
use super::pg_dump::{pg_dump_failed, write_failed};
|
||||
|
||||
/// 只排除表数据,保留完整 schema 及迁移历史;费用窗口独立于逐请求去重记录保留
|
||||
const EXCLUDED_DATA: &[&str] = &[
|
||||
"model_requests",
|
||||
"model_request_leases",
|
||||
"ops_events",
|
||||
"admin_audit_events",
|
||||
"client_api_key_charge_events",
|
||||
"request_analytics_facts",
|
||||
"request_analytics_15m",
|
||||
"request_analytics_1h",
|
||||
"request_analytics_1d_counts",
|
||||
"request_analytics_pending_requests",
|
||||
"request_analytics_pending_buckets",
|
||||
"request_analytics_build_state",
|
||||
];
|
||||
|
||||
pub(super) struct BusinessSnapshot {
|
||||
transaction: Transaction<'static, Postgres>,
|
||||
snapshot_id: String,
|
||||
schema: String,
|
||||
}
|
||||
|
||||
impl BusinessSnapshot {
|
||||
pub async fn begin(pool: &PgPool, backup_id: &str) -> Result<Self, BackupError> {
|
||||
let mut transaction = pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法开始备份快照"))?;
|
||||
sqlx::query("set transaction isolation level repeatable read, read only")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法隔离备份快照"))?;
|
||||
sqlx::query("set local statement_timeout = '30s'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法设置备份查询期限"))?;
|
||||
if !crate::postgres::backup_projection_ready(&mut transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法读取统计发布状态"))?
|
||||
{
|
||||
return Err(BackupError::new(
|
||||
code::ANALYTICS_NOT_READY,
|
||||
"用量统计正在重建,请在统计就绪后重试备份".to_owned(),
|
||||
));
|
||||
}
|
||||
sqlx::query("select set_config('cpr.backup_id',$1,true)")
|
||||
.bind(backup_id)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法标记备份汇总"))?;
|
||||
let (snapshot_id, schema): (String, String) =
|
||||
sqlx::query_as("select pg_export_snapshot(), current_schema()")
|
||||
.fetch_one(&mut *transaction)
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("无法导出一致性快照"))?;
|
||||
Ok(Self {
|
||||
transaction,
|
||||
snapshot_id,
|
||||
schema: format!("\"{}\"", schema.replace('"', "\"\"")),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn configure(&self, command: &mut tokio::process::Command) {
|
||||
command
|
||||
.arg("--snapshot")
|
||||
.arg(&self.snapshot_id)
|
||||
.arg("--schema")
|
||||
.arg(&self.schema);
|
||||
for table in EXCLUDED_DATA {
|
||||
command
|
||||
.arg("--exclude-table-data")
|
||||
.arg(format!("{}.\"{table}\"", self.schema));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn restore_preamble(&self) -> String {
|
||||
// 新库自带 public;只替换空 schema,含已有对象时由 PostgreSQL 拒绝删除
|
||||
format!("DROP SCHEMA IF EXISTS {};\n", self.schema)
|
||||
}
|
||||
|
||||
pub async fn append(
|
||||
mut self,
|
||||
writer: &mut (impl AsyncWrite + Unpin),
|
||||
) -> Result<(), BackupError> {
|
||||
let header = format!(
|
||||
"\n-- CPR 业务备份:已发布的全局小时用量与维护贡献\nCOPY {}.request_analytics_1h (bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions) FROM stdin;\n",
|
||||
self.schema
|
||||
);
|
||||
writer
|
||||
.write_all(header.as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
// 原生统计只合并 global,不能把按账号、Key 等重复投影的同一请求再次相加
|
||||
// 贡献身份包含本次备份 ID,恢复后再备份时与上一次恢复的基线互不覆盖
|
||||
let mut stream = self.transaction.copy_out_raw(r#"COPY (
|
||||
WITH native AS MATERIALIZED (
|
||||
SELECT bucket_start, counts, costs FROM request_analytics_1h WHERE collection='global'
|
||||
), counters AS (
|
||||
SELECT bucket_start, c.key, sum(c.value::numeric) AS value
|
||||
FROM native CROSS JOIN LATERAL jsonb_each_text(counts) c
|
||||
WHERE c.key IN ('request_count','success_count','failure_count','cancelled_count','incomplete_count',
|
||||
'completed_request_count','input_tokens','output_tokens','cached_input_tokens','cache_write_input_tokens',
|
||||
'reasoning_output_tokens','image_input_tokens','image_output_tokens','total_tokens')
|
||||
GROUP BY bucket_start,c.key
|
||||
), amounts AS (
|
||||
SELECT bucket_start, c.key, sum(c.value::numeric)::text AS value
|
||||
FROM native CROSS JOIN LATERAL jsonb_each_text(costs) c
|
||||
WHERE c.key NOT IN ('standard_usd','no_cache_usd') GROUP BY bucket_start,c.key
|
||||
), totals AS (
|
||||
SELECT bucket_start, jsonb_object_agg(key,value) AS counts FROM counters GROUP BY bucket_start
|
||||
), costs AS (
|
||||
SELECT bucket_start, jsonb_object_agg(key,value) AS costs FROM amounts GROUP BY bucket_start
|
||||
)
|
||||
SELECT bucket_start, 'usage_contribution', ARRAY['','','','','','','backup:'||current_setting('cpr.backup_id')],
|
||||
'',0,0,counts,coalesce(costs.costs,'{}'::jsonb),'{}'::jsonb
|
||||
FROM totals LEFT JOIN costs USING(bucket_start)
|
||||
UNION ALL
|
||||
SELECT bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions
|
||||
FROM request_analytics_1h WHERE collection='usage_contribution'
|
||||
) TO STDOUT"#).await.map_err(|_| pg_dump_failed("无法导出用量汇总"))?;
|
||||
while let Some(bytes) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("读取用量汇总失败"))?
|
||||
{
|
||||
writer.write_all(&bytes).await.map_err(write_failed)?;
|
||||
}
|
||||
drop(stream);
|
||||
let footer = format!(
|
||||
"\\.\nINSERT INTO {}.request_analytics_build_state (singleton) VALUES (true);\n",
|
||||
self.schema
|
||||
);
|
||||
writer
|
||||
.write_all(footer.as_bytes())
|
||||
.await
|
||||
.map_err(write_failed)?;
|
||||
self.transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|_| pg_dump_failed("关闭备份快照失败"))
|
||||
}
|
||||
}
|
||||
@@ -52,7 +52,7 @@ impl StagingArea {
|
||||
/// 完成归档的暂存路径
|
||||
#[must_use]
|
||||
pub fn final_path(&self, backup_id: &str) -> PathBuf {
|
||||
self.base_dir.join(format!("{backup_id}.dump"))
|
||||
self.base_dir.join(format!("{backup_id}.sql.gz"))
|
||||
}
|
||||
|
||||
/// 单任务暂存归档上限
|
||||
|
||||
@@ -341,11 +341,12 @@ pub(crate) fn backup_ports(
|
||||
config.backup_staging_dir().to_path_buf(),
|
||||
backup::staging::DEFAULT_MAX_ARCHIVE_BYTES,
|
||||
)?);
|
||||
let repository = Arc::new(postgres::PgBackupRepository::new(pool));
|
||||
let repository = Arc::new(postgres::PgBackupRepository::new(pool.clone()));
|
||||
let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new(
|
||||
staging,
|
||||
config.database.url.clone(),
|
||||
config.database.password.clone(),
|
||||
pool,
|
||||
));
|
||||
let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new());
|
||||
Ok(BackupStorePorts::new(repository, dump, object_store))
|
||||
|
||||
@@ -46,4 +46,4 @@ pub use bundle::*;
|
||||
pub use config::*;
|
||||
pub use value::*;
|
||||
pub(crate) use workers::*;
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe, run_request_analytics_cycle};
|
||||
pub use workers::{CommandStoreDrainError, PostgresHealthProbe, RequestAnalyticsCycle};
|
||||
|
||||
@@ -192,6 +192,8 @@ async fn apply_changes(
|
||||
}
|
||||
let mut buckets =
|
||||
BTreeMap::<DateTime<Utc>, BTreeMap<(String, Vec<String>, String), MetricState>>::new();
|
||||
let mut empty_groups = Vec::new();
|
||||
let mut rebuild = Vec::new();
|
||||
for (key, change) in changes {
|
||||
if change.previous.encode()? == change.current.encode()?
|
||||
&& change.previous.last_used_at() == change.current.last_used_at()
|
||||
@@ -202,21 +204,17 @@ async fn apply_changes(
|
||||
let needs_rebuild = subtract(&mut state, &change.previous, &change.current)?;
|
||||
state.merge(&change.current)?;
|
||||
if state.count("request_count") == 0 {
|
||||
let statement = format!(
|
||||
"delete from {table} where bucket_start=$1 and collection=$2 and dimensions=$3 and outcome=$4"
|
||||
);
|
||||
sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(key.0)
|
||||
.bind(&key.1)
|
||||
.bind(&key.2)
|
||||
.bind(&key.3)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("remove empty analytics group", error))?;
|
||||
empty_groups.push(serde_json::json!({
|
||||
"bucket_start": key.0,
|
||||
"collection": key.1,
|
||||
"dimensions": key.2,
|
||||
"outcome": key.3,
|
||||
}));
|
||||
} else {
|
||||
// 预测状态只有可加计数和美元总额,零金额无需修复币种存在性
|
||||
if needs_rebuild && key.1 != forecast::COLLECTION {
|
||||
state = rebuild_group(tx, hourly, progress, &key).await?;
|
||||
rebuild.push(key);
|
||||
continue;
|
||||
}
|
||||
buckets
|
||||
.entry(key.0)
|
||||
@@ -224,6 +222,24 @@ async fn apply_changes(
|
||||
.insert((key.1, key.2, key.3), state);
|
||||
}
|
||||
}
|
||||
for (key, state) in rebuild_groups(tx, hourly, progress, rebuild).await? {
|
||||
buckets
|
||||
.entry(key.0)
|
||||
.or_default()
|
||||
.insert((key.1, key.2, key.3), state);
|
||||
}
|
||||
// 保留期缩短会同时清空大量分组,集合删除避免 SQL 往返随分组数放大
|
||||
for groups in empty_groups.chunks(1_000) {
|
||||
let statement = format!(
|
||||
"delete from {table} a using jsonb_to_recordset($1::jsonb) k(bucket_start timestamptz,collection text,dimensions text[],outcome text)
|
||||
where a.bucket_start=k.bucket_start and a.collection=k.collection and a.dimensions=k.dimensions and a.outcome=k.outcome"
|
||||
);
|
||||
sqlx::query(sqlx::AssertSqlSafe(statement))
|
||||
.bind(sqlx::types::Json(groups))
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("remove empty analytics groups", error))?;
|
||||
}
|
||||
write_bucket_groups(tx, table, buckets).await
|
||||
}
|
||||
|
||||
@@ -311,61 +327,97 @@ pub(super) fn subtract(
|
||||
Ok(rebuild)
|
||||
}
|
||||
|
||||
async fn rebuild_group(
|
||||
async fn rebuild_groups(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
hourly: bool,
|
||||
progress: &BTreeMap<DateTime<Utc>, Progress>,
|
||||
key: &GroupKey,
|
||||
) -> StoreResult<MetricState> {
|
||||
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
let mut cursor = None;
|
||||
if hourly {
|
||||
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
|
||||
&& let Some(time) = owner.started_at
|
||||
{
|
||||
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
end = if owner.contains_quarter(&candidate) {
|
||||
time + TimeDelta::minutes(15)
|
||||
} else {
|
||||
time
|
||||
};
|
||||
keys: Vec<GroupKey>,
|
||||
) -> StoreResult<BTreeMap<GroupKey, MetricState>> {
|
||||
// 同一统计桶的分组共享时间范围,按桶查询才能让事实表范围扫描命中 started_at 索引
|
||||
let mut by_bucket = BTreeMap::<(&str, DateTime<Utc>), Vec<&GroupKey>>::new();
|
||||
for key in &keys {
|
||||
by_bucket.entry((&key.1, key.0)).or_default().push(key);
|
||||
}
|
||||
let mut rebuilt = BTreeMap::new();
|
||||
for ((collection, bucket_start), keys) in by_bucket {
|
||||
let columns = worker::COLLECTION_COLUMNS
|
||||
.iter()
|
||||
.find_map(|(name, columns)| (*name == collection).then_some(*columns))
|
||||
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
|
||||
let bucket_end = bucket_start + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
// 同类分组共用维度匹配和聚合 SQL,各组仍独立遵守已发布的构建前缀
|
||||
for keys in keys.chunks(1_000) {
|
||||
let mut input = Vec::with_capacity(keys.len());
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
let mut end = key.0 + TimeDelta::seconds(if hourly { 3600 } else { 900 });
|
||||
let mut cursor = None;
|
||||
if hourly {
|
||||
if let Some((_, owner)) = progress.range(key.0..end).find(|(_, p)| p.phase == 4)
|
||||
&& let Some(time) = owner.started_at
|
||||
{
|
||||
let candidate = (time, key.1.clone(), key.2.clone(), key.3.clone());
|
||||
end = if owner.contains_quarter(&candidate) {
|
||||
time + TimeDelta::minutes(15)
|
||||
} else {
|
||||
time
|
||||
};
|
||||
}
|
||||
} else if let Some(stage) = progress.get(&key.0)
|
||||
&& stage.phase == 1
|
||||
{
|
||||
cursor = stage.started_at.zip(stage.id.as_deref());
|
||||
}
|
||||
input.push(serde_json::json!({
|
||||
"bucket_start": key.0,
|
||||
"end_at": end,
|
||||
"cursor_time": cursor.map(|(time, _)| time),
|
||||
"cursor_id": cursor.map(|(_, id)| id),
|
||||
"dimensions": key.2,
|
||||
"outcome": key.3,
|
||||
"name": index.to_string(),
|
||||
}));
|
||||
rebuilt.insert((*key).clone(), MetricState::default());
|
||||
}
|
||||
// 等值桶键避免范围连接反复比较其他时间桶,范围和游标仍限定已发布前缀
|
||||
let mut query = QueryBuilder::<Postgres>::new(
|
||||
"with matching as materialized (select mr.*,k.bucket_start as output_bucket,k.name,coalesce((",
|
||||
);
|
||||
query.push(completed_usage_fact_predicate("mr"))
|
||||
.push("),false) as is_completed_usage from jsonb_to_recordset(")
|
||||
.push_bind(serde_json::Value::Array(input))
|
||||
.push("::jsonb) k(bucket_start timestamptz,end_at timestamptz,cursor_time timestamptz,cursor_id text,dimensions text[],outcome text,name text)
|
||||
join request_analytics_facts mr on date_bin(make_interval(secs => ")
|
||||
.push_bind(if hourly { 3600_f64 } else { 900_f64 })
|
||||
.push("),mr.started_at,timestamptz '2000-01-01 UTC')=k.bucket_start
|
||||
and mr.started_at >= k.bucket_start and mr.started_at < k.end_at
|
||||
and mr.recovered_at is null and mr.outcome=k.outcome
|
||||
and (k.cursor_time is null or (mr.started_at,mr.id collate \"C\") <= (k.cursor_time,k.cursor_id collate \"C\"))
|
||||
and mr.started_at >= ")
|
||||
.push_bind(bucket_start)
|
||||
.push(" and mr.started_at < ")
|
||||
.push_bind(bucket_end);
|
||||
for (index, column) in columns.iter().enumerate() {
|
||||
query.push(format!(
|
||||
" and coalesce(mr.{column}::text,'')=k.dimensions[{}]",
|
||||
index + 1
|
||||
));
|
||||
}
|
||||
query.push(")");
|
||||
aggregate::push_fact_aggregation(&mut query, false, true);
|
||||
let rows = query.build().fetch_all(&mut **tx).await.map_err(|error| {
|
||||
postgres_unavailable("rebuild changed analytics extrema", error)
|
||||
})?;
|
||||
for row in rows {
|
||||
let name: String = get(&row, "name")?;
|
||||
let index = name
|
||||
.parse::<usize>()
|
||||
.map_err(|_| invalid("invalid analytics rebuild group"))?;
|
||||
let key = keys
|
||||
.get(index)
|
||||
.ok_or_else(|| invalid("unknown analytics rebuild group"))?;
|
||||
rebuilt.insert((*key).clone(), MetricState::decode(&row)?);
|
||||
}
|
||||
}
|
||||
} else if let Some(stage) = progress.get(&key.0)
|
||||
&& stage.phase == 1
|
||||
{
|
||||
cursor = stage.started_at.zip(stage.id.as_deref());
|
||||
}
|
||||
let mut query = QueryBuilder::<Postgres>::new("with matching as materialized (select mr.*,");
|
||||
query.push_bind(key.0).push("::timestamptz as output_bucket,''::text as name,coalesce((")
|
||||
.push(completed_usage_fact_predicate("mr")).push("),false) as is_completed_usage from request_analytics_facts mr where mr.started_at >= ")
|
||||
.push_bind(key.0).push(" and mr.started_at < ").push_bind(end)
|
||||
.push(" and mr.recovered_at is null and mr.outcome=").push_bind(&key.3);
|
||||
if let Some((time, id)) = cursor {
|
||||
query
|
||||
.push(" and (mr.started_at,mr.id collate \"C\") <= (")
|
||||
.push_bind(time)
|
||||
.push(",")
|
||||
.push_bind(id)
|
||||
.push(")");
|
||||
}
|
||||
let columns = worker::COLLECTION_COLUMNS
|
||||
.iter()
|
||||
.find_map(|(collection, columns)| (*collection == key.1).then_some(*columns))
|
||||
.ok_or_else(|| invalid("unknown incremental analytics collection"))?;
|
||||
for (column, value) in columns.iter().zip(&key.2) {
|
||||
query
|
||||
.push(format!(" and coalesce(mr.{column}::text,'')="))
|
||||
.push_bind(value);
|
||||
}
|
||||
query.push(")");
|
||||
aggregate::push_fact_aggregation(&mut query, false, true);
|
||||
let row = query
|
||||
.build()
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_err(|error| postgres_unavailable("rebuild changed analytics extrema", error))?;
|
||||
row.as_ref()
|
||||
.map(MetricState::decode)
|
||||
.transpose()
|
||||
.map(Option::unwrap_or_default)
|
||||
Ok(rebuilt)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,15 @@ pub(crate) const AGGREGATE_COLLECTION: &str = "usage_contribution";
|
||||
const SCHEMA_VERSION: i32 = 1;
|
||||
const HISTOGRAM_VERSION: i32 = histogram::HISTOGRAM_VERSION;
|
||||
|
||||
/// 业务备份只消费已发布小时汇总,重建中的前缀不能作为完整历史导出
|
||||
pub(crate) async fn backup_projection_ready(
|
||||
tx: &mut sqlx::Transaction<'_, Postgres>,
|
||||
) -> StoreResult<bool> {
|
||||
sqlx::query_scalar("select exists(select 1 from request_analytics_build_state where singleton and backfill_complete and reset_phase=0 and schema_version=$1 and histogram_version=$2) and not exists(select 1 from request_analytics_pending_buckets)")
|
||||
.bind(SCHEMA_VERSION).bind(HISTOGRAM_VERSION).fetch_one(&mut **tx).await
|
||||
.map_err(|error| postgres_unavailable("check backup analytics readiness", error))
|
||||
}
|
||||
|
||||
pub(super) async fn begin_snapshot(
|
||||
connection: &mut ObservabilityConnection,
|
||||
) -> StoreResult<sqlx::Transaction<'_, Postgres>> {
|
||||
|
||||
@@ -83,7 +83,17 @@ impl PgRequestAnalyticsRepository {
|
||||
return Ok(AnalyticsMaintenance { reset_progress: true, ..Default::default() });
|
||||
}
|
||||
let mut result = AnalyticsMaintenance::default();
|
||||
let rows = sqlx::query("select request_id, change_version from request_analytics_pending_requests order by oldest_pending_at, request_id limit $1")
|
||||
// 历史删除可能一次排入大量旧任务,给仍存在的请求预留半批名额,避免实时统计饥饿
|
||||
// 其余名额仍按最早入队消费,两类任务都能前进;同一快照内选版本,沿用精确版本 ACK
|
||||
let rows = sqlx::query("with live as materialized (
|
||||
select pending.request_id, pending.change_version from request_analytics_pending_requests pending
|
||||
join model_requests source on source.id=pending.request_id
|
||||
order by pending.oldest_pending_at, pending.request_id limit ($1::bigint / 2)
|
||||
), oldest as (
|
||||
select pending.request_id, pending.change_version from request_analytics_pending_requests pending
|
||||
where not exists (select 1 from live where live.request_id=pending.request_id)
|
||||
order by pending.oldest_pending_at, pending.request_id limit ($1::bigint - (select count(*) from live))
|
||||
) select request_id, change_version from live union all select request_id, change_version from oldest")
|
||||
.bind(i64::from(batch_size)).fetch_all(&mut *transaction).await.map_err(|error| postgres_unavailable("load analytics request queue", error))?;
|
||||
let mut jobs = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
|
||||
@@ -34,6 +34,7 @@ pub use admin_adapter::*;
|
||||
pub(crate) use analytics::accounts::{
|
||||
account_group_usage, account_usage_sort_values, account_usage_windows,
|
||||
};
|
||||
pub(crate) use analytics::backup_projection_ready;
|
||||
pub(crate) use analytics::{AGGREGATE_COLLECTION, AnalyticsSnapshot};
|
||||
pub use analytics::{AnalyticsMaintenance, AnalyticsStatus, PgRequestAnalyticsRepository};
|
||||
pub(crate) use mapping::*;
|
||||
|
||||
@@ -61,7 +61,9 @@ impl RetentionStore for PgRetentionRepository {
|
||||
select ctid from model_requests
|
||||
where outcome <> 'running'
|
||||
and completed_at < $1 - ($2 * interval '1 day')
|
||||
order by completed_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
RetentionTarget::OpsEvents => {
|
||||
@@ -70,7 +72,9 @@ impl RetentionStore for PgRetentionRepository {
|
||||
select ctid from ops_events
|
||||
where model_request_id is null
|
||||
and created_at < $1 - ($2 * interval '1 day')
|
||||
order by created_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
RetentionTarget::AdminAuditEvents => {
|
||||
@@ -78,18 +82,39 @@ impl RetentionStore for PgRetentionRepository {
|
||||
where ctid in (
|
||||
select ctid from admin_audit_events
|
||||
where created_at < $1 - ($2 * interval '1 day')
|
||||
order by created_at
|
||||
limit $3
|
||||
for update skip locked
|
||||
)"
|
||||
}
|
||||
};
|
||||
sqlx::query(sql)
|
||||
// 跳过正在写入的父行;级联与投影触发器仍可能等锁,单批需有服务端截止时间
|
||||
let mut transaction = self
|
||||
.pool
|
||||
.begin()
|
||||
.await
|
||||
.map_err(|source| unavailable("begin retention batch", source))?;
|
||||
sqlx::query("set local statement_timeout = '5s'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| unavailable("set retention statement timeout", source))?;
|
||||
sqlx::query("set local lock_timeout = '500ms'")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|source| unavailable("set retention lock timeout", source))?;
|
||||
let deleted = sqlx::query(sql)
|
||||
.bind(now)
|
||||
.bind(i64::from(policy.days(target)))
|
||||
.bind(i64::from(limit.get()))
|
||||
.execute(&self.pool)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map(|result| result.rows_affected())
|
||||
.map_err(|source| unavailable("delete expired records", source))
|
||||
.map_err(|source| unavailable("delete expired records", source))?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|source| unavailable("commit retention batch", source))?;
|
||||
Ok(deleted)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use gateway_core::task::DaemonTask;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
|
||||
const COMMAND_DRAIN_TIMEOUT: Duration = Duration::from_secs(4);
|
||||
|
||||
@@ -325,12 +326,16 @@ pub(crate) fn request_analytics_worker(
|
||||
Ok(WorkerContribution::Registration(scheduled_worker(
|
||||
id,
|
||||
Duration::from_secs(5),
|
||||
Box::new(RequestAnalyticsTask { repository }),
|
||||
Box::new(RequestAnalyticsTask {
|
||||
repository,
|
||||
cycle: RequestAnalyticsCycle::default(),
|
||||
}),
|
||||
)?))
|
||||
}
|
||||
|
||||
struct RequestAnalyticsTask {
|
||||
repository: postgres::PgRequestAnalyticsRepository,
|
||||
cycle: RequestAnalyticsCycle,
|
||||
}
|
||||
|
||||
impl ScheduledTask for RequestAnalyticsTask {
|
||||
@@ -341,7 +346,7 @@ impl ScheduledTask for RequestAnalyticsTask {
|
||||
Box::pin(async move {
|
||||
tokio::select! {
|
||||
() = context.cancellation().cancelled() => Ok(()),
|
||||
result = run_request_analytics_cycle(&self.repository) => result.map(|_| ()).map_err(|source| {
|
||||
result = self.cycle.run(&self.repository) => result.map(|_| ()).map_err(|source| {
|
||||
WorkerTaskError::safe("request analytics maintenance failed").with_source(source)
|
||||
}),
|
||||
}
|
||||
@@ -349,43 +354,107 @@ impl ScheduledTask for RequestAnalyticsTask {
|
||||
}
|
||||
}
|
||||
|
||||
/// 生产与隔离压测共用的单周期推进预算;各批独立提交并归还观测槽位
|
||||
pub async fn run_request_analytics_cycle(
|
||||
repository: &postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<postgres::AnalyticsMaintenance> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut total = postgres::AnalyticsMaintenance::default();
|
||||
loop {
|
||||
let result = match tokio::time::timeout_at(deadline, repository.maintain(1_000)).await {
|
||||
Ok(result) => result?,
|
||||
Err(source) => {
|
||||
if total.projected_requests == 0
|
||||
&& total.backfilled_requests == 0
|
||||
&& total.rebuilt_buckets == 0
|
||||
&& total.advanced_buckets == 0
|
||||
&& !total.reset_progress
|
||||
{
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "request analytics cycle timed out without progress".to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
});
|
||||
}
|
||||
return Ok(total);
|
||||
}
|
||||
};
|
||||
let idle = result.projected_requests == 0
|
||||
&& result.backfilled_requests == 0
|
||||
&& result.rebuilt_buckets == 0
|
||||
&& result.advanced_buckets == 0
|
||||
&& !result.reset_progress;
|
||||
total.projected_requests += result.projected_requests;
|
||||
total.backfilled_requests += result.backfilled_requests;
|
||||
total.rebuilt_buckets += result.rebuilt_buckets;
|
||||
total.advanced_buckets += result.advanced_buckets;
|
||||
total.reset_progress |= result.reset_progress;
|
||||
if idle || tokio::time::Instant::now() >= deadline {
|
||||
return Ok(total);
|
||||
/// 统计消费周期预算,跨轮保留批量上限以便超时后的下一轮提交进展
|
||||
pub struct RequestAnalyticsCycle {
|
||||
batch_size: AtomicU32,
|
||||
}
|
||||
|
||||
impl Default for RequestAnalyticsCycle {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
batch_size: AtomicU32::new(1_000),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestAnalyticsCycle {
|
||||
/// 生产与隔离压测共用的推进路径;各批独立提交并归还观测槽位
|
||||
pub async fn run(
|
||||
&self,
|
||||
repository: &postgres::PgRequestAnalyticsRepository,
|
||||
) -> StoreResult<postgres::AnalyticsMaintenance> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut total = postgres::AnalyticsMaintenance::default();
|
||||
loop {
|
||||
let batch_size = self.batch_size.load(Ordering::Relaxed);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result =
|
||||
match tokio::time::timeout_at(deadline, repository.maintain(batch_size)).await {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(error)) => {
|
||||
if analytics_maintenance_timed_out(&error) {
|
||||
self.reduce_batch_size(batch_size);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Err(source) => {
|
||||
if total.projected_requests == 0
|
||||
&& total.backfilled_requests == 0
|
||||
&& total.rebuilt_buckets == 0
|
||||
&& total.advanced_buckets == 0
|
||||
&& !total.reset_progress
|
||||
{
|
||||
// 只在整轮没有返回进展时退让,避免尾批剩余时间短导致持续缩批
|
||||
self.reduce_batch_size(batch_size);
|
||||
return Err(StoreError::Unavailable {
|
||||
backend: StoreBackend::PostgreSql,
|
||||
message: "request analytics cycle timed out without progress"
|
||||
.to_owned(),
|
||||
source: Some(gateway_core::error::ErrorSource::new(source)),
|
||||
});
|
||||
}
|
||||
return Ok(total);
|
||||
}
|
||||
};
|
||||
let idle = result.projected_requests == 0
|
||||
&& result.backfilled_requests == 0
|
||||
&& result.rebuilt_buckets == 0
|
||||
&& result.advanced_buckets == 0
|
||||
&& !result.reset_progress;
|
||||
total.projected_requests += result.projected_requests;
|
||||
total.backfilled_requests += result.backfilled_requests;
|
||||
total.rebuilt_buckets += result.rebuilt_buckets;
|
||||
total.advanced_buckets += result.advanced_buckets;
|
||||
total.reset_progress |= result.reset_progress;
|
||||
// 满批快速完成才恢复吞吐,空结果也可能只是其他实例持有维护锁
|
||||
if (result.projected_requests >= u64::from(batch_size)
|
||||
|| result.backfilled_requests >= u64::from(batch_size))
|
||||
&& started.elapsed() < Duration::from_secs(2)
|
||||
{
|
||||
self.batch_size
|
||||
.store((batch_size * 2).min(1_000), Ordering::Relaxed);
|
||||
}
|
||||
if idle {
|
||||
return Ok(total);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(total);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reduce_batch_size(&self, batch_size: u32) {
|
||||
let next_batch_size = (batch_size / 2).max(1);
|
||||
self.batch_size.store(next_batch_size, Ordering::Relaxed);
|
||||
tracing::warn!(
|
||||
batch_size,
|
||||
next_batch_size,
|
||||
"request analytics batch timed out"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn analytics_maintenance_timed_out(error: &StoreError) -> bool {
|
||||
let StoreError::Unavailable {
|
||||
source: Some(source),
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
source.is::<tokio::time::error::Elapsed>()
|
||||
|| source
|
||||
.downcast_ref::<sqlx::Error>()
|
||||
.and_then(sqlx::Error::as_database_error)
|
||||
.is_some_and(|error| error.code().as_deref() == Some("57014"))
|
||||
}
|
||||
|
||||
@@ -5,11 +5,13 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::postgres::{TestDatabase, drain_analytics};
|
||||
use gateway_admin::model::backup::code;
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort, DumpRequest};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_store::backup::{pg_dump::PgDumpAdapter, staging::StagingArea};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
#[tokio::test]
|
||||
async fn dump_process_lifecycle() {
|
||||
@@ -41,14 +43,31 @@ async fn dump_process_lifecycle() {
|
||||
assert!(status.success());
|
||||
return;
|
||||
};
|
||||
let Some(database) = TestDatabase::create("dump_lifecycle").await else {
|
||||
return;
|
||||
};
|
||||
drain_analytics(&database.pool).await;
|
||||
let directory = Path::new(&directory);
|
||||
let staging = Arc::new(StagingArea::open(directory.join("staging"), 64).unwrap());
|
||||
let dump = |mode: &str| PgDumpAdapter::new(staging.clone(), mode, "test-password");
|
||||
let staging = Arc::new(StagingArea::open(directory.join("staging"), 1024 * 1024).unwrap());
|
||||
let dump = |mode: &str| {
|
||||
PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
mode,
|
||||
"test-password",
|
||||
database.pool.clone(),
|
||||
)
|
||||
};
|
||||
|
||||
let artifact = dump("success").dump(request("success")).await.unwrap();
|
||||
assert_eq!(std::fs::read(&artifact.path).unwrap(), b"archive");
|
||||
assert_eq!(artifact.size_bytes, 7);
|
||||
assert_eq!(artifact.sha256, hex::encode(Sha256::digest(b"archive")));
|
||||
let compressed = std::fs::read(&artifact.path).unwrap();
|
||||
let mut decoded = String::new();
|
||||
async_compression::tokio::bufread::GzipDecoder::new(compressed.as_slice())
|
||||
.read_to_string(&mut decoded)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(decoded.contains("\narchive\n"));
|
||||
assert_eq!(artifact.size_bytes, compressed.len() as u64);
|
||||
assert_eq!(artifact.sha256, hex::encode(Sha256::digest(&compressed)));
|
||||
assert!(!staging.partial_path("success").exists());
|
||||
assert_eq!(
|
||||
dump("success")
|
||||
@@ -65,10 +84,15 @@ async fn dump_process_lifecycle() {
|
||||
assert!(!staging.partial_path("fail").exists());
|
||||
|
||||
let small = Arc::new(StagingArea::open(directory.join("small"), 1).unwrap());
|
||||
let error = PgDumpAdapter::new(small.clone(), "success", "test-password")
|
||||
.dump(request("oversized"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
let error = PgDumpAdapter::new(
|
||||
small.clone(),
|
||||
"success",
|
||||
"test-password",
|
||||
database.pool.clone(),
|
||||
)
|
||||
.dump(request("oversized"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), code::STAGING_SPACE_EXHAUSTED);
|
||||
assert!(!small.partial_path("oversized").exists());
|
||||
|
||||
@@ -119,13 +143,21 @@ async fn dump_process_lifecycle() {
|
||||
artifact.path.exists(),
|
||||
"completed archives remain available for recovery"
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_reports_failure_and_still_removes_the_other_archive() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let staging = Arc::new(StagingArea::open(directory.path().to_path_buf(), 64).unwrap());
|
||||
let adapter = PgDumpAdapter::new(staging.clone(), "unused", "unused");
|
||||
let adapter = PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
"unused",
|
||||
"unused",
|
||||
sqlx::postgres::PgPoolOptions::new()
|
||||
.connect_lazy("postgres://localhost/unused")
|
||||
.unwrap(),
|
||||
);
|
||||
adapter.cleanup_staging("missing").await.unwrap();
|
||||
std::fs::create_dir(staging.partial_path("blocked")).unwrap();
|
||||
std::fs::write(staging.final_path("blocked"), b"archive").unwrap();
|
||||
|
||||
@@ -20,7 +20,7 @@ fn store_config_should_derive_backup_staging_from_runtime_data_dir() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits() {
|
||||
fn store_pool_config_defaults_reserve_capacity_for_observability() {
|
||||
let pool: StorePoolConfig =
|
||||
serde_json::from_value(serde_json::json!({})).expect("default pool configuration");
|
||||
|
||||
@@ -37,13 +37,6 @@ fn store_pool_config_ignores_unknown_options_and_keeps_connection_pool_limits()
|
||||
.observability_max_connections(),
|
||||
40
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<StorePoolConfig>(serde_json::json!({
|
||||
"statement_timeout_seconds": 60,
|
||||
}))
|
||||
.unwrap(),
|
||||
pool,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -27,36 +27,6 @@ use super::TestDatabase;
|
||||
const MIXED_GROUP: &str = "grp_00000000000000000000000000000001";
|
||||
const EMPTY_GROUP: &str = "grp_00000000000000000000000000000002";
|
||||
|
||||
#[tokio::test]
|
||||
async fn fast_mode_migration_preserves_existing_group_choices() {
|
||||
let Some(database) = TestDatabase::create_through("group_fast_mode_upgrade", 21).await else {
|
||||
return;
|
||||
};
|
||||
for (id, disabled) in [(MIXED_GROUP, true), (EMPTY_GROUP, false)] {
|
||||
sqlx::query("insert into account_groups (id, name, color, disable_fast, created_at, updated_at) values ($1, $1, '#2563EBFF', $2, now(), now())")
|
||||
.bind(id)
|
||||
.bind(disabled)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let groups = sqlx::query_as::<_, (String, String)>(
|
||||
"select id, fast_mode from account_groups order by id",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
groups,
|
||||
vec![
|
||||
(MIXED_GROUP.to_owned(), "disabled".to_owned()),
|
||||
(EMPTY_GROUP.to_owned(), "default".to_owned())
|
||||
]
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn groups_aggregate_cross_provider_members_and_key_bindings_without_multiplication() {
|
||||
let Some(database) = TestDatabase::create("account_group_aggregate").await else {
|
||||
@@ -840,20 +810,19 @@ async fn image_generation_group_updates_preserve_omitted_values_and_publish_snap
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_generation_migration_allows_existing_and_default_groups() {
|
||||
let Some(database) = TestDatabase::create_through("group_image_upgrade", 31).await else {
|
||||
async fn image_generation_defaults_to_allowed_for_new_groups() {
|
||||
let Some(database) = TestDatabase::create("group_image_default").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, $1, '#2563EBFF', now(), now())")
|
||||
.bind(MIXED_GROUP).execute(&database.pool).await.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
sqlx::query("insert into account_groups (id, name, color, created_at, updated_at) values ($1, $1, '#2563EBFF', now(), now())")
|
||||
.bind(EMPTY_GROUP).execute(&database.pool).await.unwrap();
|
||||
let allowed =
|
||||
sqlx::query_scalar::<_, bool>("select allow_image_generation from account_groups")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(allowed, vec![true, true]);
|
||||
let allowed = sqlx::query_scalar::<_, bool>(
|
||||
"select allow_image_generation from account_groups where id = $1",
|
||||
)
|
||||
.bind(EMPTY_GROUP)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(allowed);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
//! 小时重建合并、分段快照失效与阻塞桶调度的真实数据库验收
|
||||
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use gateway_store::postgres::{ObservabilityRange, PgRequestAnalyticsRepository};
|
||||
use gateway_store::{
|
||||
RequestAnalyticsCycle,
|
||||
postgres::{ObservabilityRange, PgRequestAnalyticsRepository},
|
||||
};
|
||||
|
||||
use super::{
|
||||
TestDatabase, analytics::seed, drain_analytics, observability_query_budget,
|
||||
@@ -352,3 +355,64 @@ async fn incremental_summary_write_failure_rolls_back_facts_and_preserves_queue(
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn extrema_rebuild_reads_only_its_bucket_when_history_is_large() {
|
||||
let Some(database) = TestDatabase::create("analytics_rebuild_bucket_range").await else {
|
||||
return;
|
||||
};
|
||||
// 强制刷新只作用于当前连接,Worker 与计数读取必须复用同一条连接
|
||||
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect_with(database.pool.connect_options().as_ref().clone())
|
||||
.await
|
||||
.unwrap();
|
||||
for number in 1..=3_i64 {
|
||||
let id = format!("rebuild_range_{number}");
|
||||
seed(&pool, &id, hour() + TimeDelta::minutes(number), &[]).await;
|
||||
sqlx::query("update model_requests set provider_account_ref='rebuild_range_account',request_observation_json=jsonb_set(request_observation_json,'{timings,local}',jsonb_build_object('latencyMs',$2::bigint,'firstTokenMs',$2::bigint)) where id=$1")
|
||||
.bind(id).bind(number).execute(&pool).await.unwrap();
|
||||
}
|
||||
drain_analytics(&pool).await;
|
||||
// 只扩充其他时间桶的事实,规划器拿到真实行数后不能再为一个分组扫描全部历史
|
||||
sqlx::query("insert into request_analytics_facts select ('history_'||n)::text as id,(f.started_at-make_interval(hours=>n/1000+2))::timestamptz,f.completed_at-make_interval(hours=>n/1000+2),f.client_api_key_ref,f.provider_account_ref,f.routing_group_refs,f.operation,f.provider_kind,f.requested_model_id,f.upstream_model_id,f.outcome,f.client_status_code,f.upstream_status_code,f.client_transport,f.upstream_transport,f.attempt_count,null,null,null,f.recovered_at,f.downstream_committed_at,f.request_kind,f.image_generation_requested,f.image_generation_succeeded,f.error_kind,f.input_tokens,f.output_tokens,f.cached_input_tokens,f.cache_write_input_tokens,f.reasoning_output_tokens,f.image_input_tokens,f.image_output_tokens,f.total_tokens,f.cost_source,f.cost_currency,f.cost_amount,f.first_token_ms,f.latency_ms,f.admission_decision_ms,f.account_selection_wait_ms,f.upstream_response_ms,f.capacity_used_slots,f.capacity_total_slots,f.standard_cost_amount,f.no_cache_cost_amount,f.source_version,f.schema_version,f.ops_failure_count,f.ops_rate_limited_count,f.ops_auth_failure_count,f.ops_provider_5xx_count from request_analytics_facts f cross join generate_series(1,300000) n where f.id='rebuild_range_2'")
|
||||
.execute(&pool).await.unwrap();
|
||||
sqlx::query("analyze request_analytics_facts")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("delete from model_requests where id='rebuild_range_1'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker = PgRequestAnalyticsRepository::new(pool.clone(), observability_query_budget());
|
||||
let scanned_before = sequentially_read_facts(&pool).await;
|
||||
RequestAnalyticsCycle::default()
|
||||
.run(&worker)
|
||||
.await
|
||||
.expect("extrema rebuild must finish within the cycle budget");
|
||||
// 耗时随机器和数据量变化,用顺序扫描读取的行数判断重建是否越出所在统计桶
|
||||
let scanned = sequentially_read_facts(&pool).await - scanned_before;
|
||||
assert!(
|
||||
scanned < 300_000,
|
||||
"extrema rebuild scanned {scanned} fact rows"
|
||||
);
|
||||
let status = worker.status().await.unwrap();
|
||||
assert_eq!((status.pending_requests, status.pending_buckets), (0, 0));
|
||||
let row: (i64, f64, f64) = sqlx::query_as("select (counts->>'request_count')::bigint,(distributions->'latency_ms'->>'minimum')::double precision,(distributions->'latency_ms'->>'maximum')::double precision from request_analytics_15m where collection='global' and bucket_start=$1")
|
||||
.bind(hour()).fetch_one(&pool).await.unwrap();
|
||||
assert_eq!(row, (2, 2.0, 3.0));
|
||||
pool.close().await;
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
async fn sequentially_read_facts(pool: &sqlx::PgPool) -> i64 {
|
||||
sqlx::query("select pg_stat_force_next_flush()")
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query_scalar("select coalesce(seq_tup_read,0) from pg_stat_user_tables where relid='request_analytics_facts'::regclass")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
//! 验证统计升级保留历史事实与回填进度,并重建已有时间桶
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{TEST_MIGRATOR, TestDatabase};
|
||||
|
||||
#[tokio::test]
|
||||
async fn analytics_upgrade_preserves_facts_and_backfill_progress_and_requeues_buckets() {
|
||||
let Some(database) = TestDatabase::create_through("analytics_upgrade", 29).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::raw_sql(
|
||||
"insert into request_analytics_facts(id,started_at,source_version) values
|
||||
('first','2026-10-01 10:01:00+00',1),
|
||||
('same_bucket','2026-10-01 10:02:00+00',2),
|
||||
('next_bucket','2026-10-01 10:16:00+00',3);
|
||||
insert into request_analytics_pending_buckets(bucket_start,oldest_pending_at)
|
||||
values ('2026-10-01 10:00:00+00','2026-10-01 09:00:00+00');
|
||||
update request_analytics_build_state set
|
||||
backfill_complete=true, backfill_cursor_id='next_bucket',
|
||||
backfill_cursor_started_at='2026-10-01 10:16:00+00',
|
||||
coverage_start='2026-10-01 10:00:00+00',
|
||||
coverage_end='2026-10-01 10:30:00+00', generation=7;",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let facts_before: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(f) from request_analytics_facts f order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let progress_before: Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from request_analytics_build_state s")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let pending_before: (i64, DateTime<Utc>) = sqlx::query_as(
|
||||
"select change_version,oldest_pending_at from request_analytics_pending_buckets",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
|
||||
let facts_after: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(f) from request_analytics_facts f order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let progress_after: Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from request_analytics_build_state s")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(facts_after, facts_before);
|
||||
assert_eq!(progress_after, progress_before);
|
||||
let pending: Vec<(DateTime<Utc>, i64, DateTime<Utc>)> = sqlx::query_as(
|
||||
"select bucket_start,change_version,oldest_pending_at
|
||||
from request_analytics_pending_buckets order by bucket_start",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let expected_buckets = ["2026-10-01T10:00:00Z", "2026-10-01T10:15:00Z"]
|
||||
.map(|value| value.parse::<DateTime<Utc>>().unwrap());
|
||||
assert_eq!(
|
||||
pending.iter().map(|row| row.0).collect::<Vec<_>>(),
|
||||
expected_buckets
|
||||
);
|
||||
assert!(pending[0].1 > pending_before.0);
|
||||
assert_eq!(pending[0].2, pending_before.1);
|
||||
|
||||
// 重启不会再次入队,也不会覆盖最早待处理时间
|
||||
TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let pending_after_restart: Vec<(DateTime<Utc>, i64, DateTime<Utc>)> = sqlx::query_as(
|
||||
"select bucket_start,change_version,oldest_pending_at
|
||||
from request_analytics_pending_buckets order by bucket_start",
|
||||
)
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pending_after_restart, pending);
|
||||
database.close().await;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! 用实际 PostgreSQL 工具验收业务备份、恢复与恢复后的统计续算
|
||||
|
||||
use chrono::{TimeDelta, Utc};
|
||||
use gateway_admin::ports::backup::{DatabaseDumpPort as _, DumpRequest};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_store::backup::{pg_dump::PgDumpAdapter, staging::StagingArea};
|
||||
use gateway_store::postgres::ObservabilityRange;
|
||||
use std::{process::Stdio, sync::Arc};
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
use super::{TestDatabase, analytics::seed, drain_analytics, observability_repository};
|
||||
|
||||
#[tokio::test]
|
||||
async fn business_backup_restores_core_state_and_usage_without_request_history() {
|
||||
let Some(database) = TestDatabase::create("business_backup").await else {
|
||||
return;
|
||||
};
|
||||
let hour = "2026-10-15T12:00:00Z"
|
||||
.parse::<chrono::DateTime<Utc>>()
|
||||
.unwrap();
|
||||
for id in ["native_a", "native_b"] {
|
||||
seed(&database.pool, id, hour + TimeDelta::minutes(5), &[]).await;
|
||||
}
|
||||
sqlx::raw_sql("insert into client_api_keys(id,name,key,created_at,updated_at) values('restore_key','restore key','sk_backup_test',now(),now());
|
||||
insert into client_api_key_budget_windows values('restore_key',now(),now()+interval '1 day',now(),now()+interval '7 days',2.5,9.5);
|
||||
insert into client_api_key_charge_events(request_id,client_api_key_id,amount_usd,completed_at) values('native_a','restore_key',2.5,now());
|
||||
insert into plugin_update_sources(plugin_id,source_json) values('backup.test','{}');
|
||||
insert into plugin_artifacts(sha256,plugin_id,version,metadata_json,source_json,archive,accepted_at) values(repeat('a',64),'backup.test','1.0.0','{}','{}',decode('010203','hex'),now());
|
||||
update runtime_settings set usage_retention_days=3;
|
||||
update model_requests set cost_source='provider_reported',cost_amount=0.5,cost_currency='USD';
|
||||
insert into request_analytics_1h(bucket_start,collection,dimensions,outcome,schema_version,histogram_version,counts,costs,distributions)
|
||||
values('2026-10-15T12:00:00Z','usage_contribution',array['plugin','model','','','','','maintained'],'',0,0,'{\"request_count\":7,\"total_tokens\":900}','{\"USD\":\"1.25\"}','{}')")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
drain_analytics(&database.pool).await;
|
||||
let range = ObservabilityRange::new(hour, hour + TimeDelta::hours(1)).unwrap();
|
||||
let before = observability_repository(&database.pool)
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(before.requests.request_count, 9);
|
||||
assert_eq!(before.requests.total_tokens, 1140);
|
||||
assert_eq!(before.attempts.costs[0].currency, "USD");
|
||||
assert_eq!(before.attempts.costs[0].amount.as_str(), "2.25");
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let staging =
|
||||
Arc::new(StagingArea::open(directory.path().to_owned(), 8 * 1024 * 1024).unwrap());
|
||||
let mut url = url::Url::parse(&std::env::var("CPR_TEST_DATABASE_URL").unwrap()).unwrap();
|
||||
let password = percent_encoding::percent_decode_str(url.password().unwrap_or_default())
|
||||
.decode_utf8()
|
||||
.unwrap()
|
||||
.into_owned();
|
||||
url.set_password(None).unwrap();
|
||||
let adapter = PgDumpAdapter::new(
|
||||
staging.clone(),
|
||||
url.to_string(),
|
||||
password.clone(),
|
||||
database.pool.clone(),
|
||||
);
|
||||
sqlx::query("update request_analytics_build_state set backfill_complete=false")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = adapter
|
||||
.dump(DumpRequest {
|
||||
backup_id: "rebuilding".to_owned(),
|
||||
cancellation: CancellationToken::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
gateway_admin::model::backup::code::ANALYTICS_NOT_READY
|
||||
);
|
||||
assert!(!staging.partial_path("rebuilding").exists());
|
||||
sqlx::query("update request_analytics_build_state set backfill_complete=true")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (backup_id, expected_requests, expected_tokens) in
|
||||
[("first", 9, 1140), ("second", 10, 1260)]
|
||||
{
|
||||
let artifact = adapter
|
||||
.dump(DumpRequest {
|
||||
backup_id: backup_id.to_owned(),
|
||||
cancellation: CancellationToken::new(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let file = tokio::io::BufReader::new(tokio::fs::File::open(&artifact.path).await.unwrap());
|
||||
let mut sql = Vec::new();
|
||||
async_compression::tokio::bufread::GzipDecoder::new(file)
|
||||
.read_to_end(&mut sql)
|
||||
.await
|
||||
.unwrap();
|
||||
let schema: String = sqlx::query_scalar("select current_schema()")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 仅清空本测试随机 schema,模拟新库预先存在 public 的状态
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"drop schema \"{schema}\" cascade; create schema \"{schema}\""
|
||||
)))
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut process = tokio::process::Command::new("psql")
|
||||
.args([
|
||||
"-X",
|
||||
"--single-transaction",
|
||||
"--set=ON_ERROR_STOP=1",
|
||||
"--dbname",
|
||||
url.as_str(),
|
||||
])
|
||||
.env("PGPASSWORD", &password)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("PostgreSQL 18 psql must be in PATH");
|
||||
process.stdin.take().unwrap().write_all(&sql).await.unwrap();
|
||||
let output = process.wait_with_output().await.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let retained: (i64,i64,i64) = sqlx::query_as("select (select count(*) from model_requests),(select count(*) from request_analytics_facts),(select count(*) from client_api_key_charge_events)")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(retained, (0, 0, 0));
|
||||
let budget: (String,String) = sqlx::query_as("select daily_used_usd::text,weekly_used_usd::text from client_api_key_budget_windows where client_api_key_id='restore_key'")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(
|
||||
budget,
|
||||
("2.5000000000".to_owned(), "9.5000000000".to_owned())
|
||||
);
|
||||
let archive: Vec<u8> = sqlx::query_scalar(
|
||||
"select archive from plugin_artifacts where plugin_id='backup.test'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(archive, [1, 2, 3]);
|
||||
let retention_days: i64 =
|
||||
sqlx::query_scalar("select usage_retention_days from runtime_settings")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(retention_days, 3);
|
||||
drain_analytics(&database.pool).await;
|
||||
let restored = observability_repository(&database.pool)
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.requests.request_count, expected_requests);
|
||||
assert_eq!(restored.requests.total_tokens, expected_tokens);
|
||||
assert_eq!(restored.attempts.costs, before.attempts.costs);
|
||||
if backup_id == "first" {
|
||||
seed(
|
||||
&database.pool,
|
||||
"after_restore",
|
||||
hour + TimeDelta::minutes(10),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
drain_analytics(&database.pool).await;
|
||||
}
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//! PostgreSQL 连接错误、迁移应用与事务归属测试
|
||||
//! PostgreSQL 连接初始化、会话设置与事务取消测试
|
||||
|
||||
use std::error::Error as _;
|
||||
|
||||
@@ -109,32 +109,32 @@ async fn invalid_postgres_options_preserve_the_sqlx_source() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly() {
|
||||
async fn connection_initialization_preserves_settings_when_reopened() {
|
||||
let Some(database_url) = crate::support::test_env("CPR_TEST_DATABASE_URL") else {
|
||||
return;
|
||||
};
|
||||
let database = format!("cpr_store_migrator_{}", Uuid::new_v4().simple());
|
||||
let database = format!("cpr_store_connection_{}", Uuid::new_v4().simple());
|
||||
let admin = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect migration test PostgreSQL");
|
||||
.expect("connect test PostgreSQL");
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
"create database \"{database}\""
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("create migration test database");
|
||||
.expect("create connection test database");
|
||||
|
||||
let isolated_url = PgConnectOptions::from_str(&database_url)
|
||||
.expect("parse migration test PostgreSQL URL")
|
||||
.expect("parse test PostgreSQL URL")
|
||||
.database(&database)
|
||||
.to_url_lossy()
|
||||
.to_string();
|
||||
let pool_config = gateway_store::StorePoolConfig::default();
|
||||
let first = connect_and_migrate(&isolated_url, pool_config)
|
||||
.await
|
||||
.expect("apply migrations through production migrator");
|
||||
.expect("initialize production connection");
|
||||
let session_settings = sqlx::query_as::<_, (String, i64, i64, i64)>(
|
||||
"select current_setting('application_name'),
|
||||
extract(epoch from current_setting('statement_timeout')::interval)::bigint,
|
||||
@@ -154,70 +154,20 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
timezone, "UTC",
|
||||
"runtime sessions use an explicit technical time basis"
|
||||
);
|
||||
let first_tables = sqlx::query_scalar::<_, String>(
|
||||
"select table_name
|
||||
from information_schema.tables
|
||||
where table_schema = 'public'
|
||||
order by table_name",
|
||||
)
|
||||
.fetch_all(&first)
|
||||
.await
|
||||
.expect("load migrated tables");
|
||||
sqlx::query("update runtime_settings set refresh_margin_seconds = 1800 where id = 1")
|
||||
.execute(&first)
|
||||
.await
|
||||
.expect("save custom runtime setting");
|
||||
first.close().await;
|
||||
|
||||
let second = connect_and_migrate(&isolated_url, pool_config)
|
||||
.await
|
||||
.expect("reopen database through production migrator");
|
||||
let migration_count =
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from _sqlx_migrations where success")
|
||||
.expect("reopen production connection");
|
||||
let refresh_margin: i64 =
|
||||
sqlx::query_scalar("select refresh_margin_seconds from runtime_settings where id = 1")
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("count successful migrations");
|
||||
let response_id_types = sqlx::query_scalar::<_, String>(
|
||||
"select data_type
|
||||
from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'model_requests'
|
||||
and column_name in ('client_response_id', 'upstream_response_id')
|
||||
order by column_name",
|
||||
)
|
||||
.fetch_all(&second)
|
||||
.await
|
||||
.expect("load opaque response ID column types");
|
||||
let raw_response_id_index_exists = sqlx::query_scalar::<_, bool>(
|
||||
"select exists (
|
||||
select 1
|
||||
from pg_indexes
|
||||
where schemaname = 'public'
|
||||
and indexname = 'model_requests_client_response_uq'
|
||||
)",
|
||||
)
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("check removed raw response ID index");
|
||||
let legacy_key_provider_column_exists = sqlx::query_scalar::<_, bool>(
|
||||
"select exists (
|
||||
select 1 from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'client_api_keys'
|
||||
and column_name = 'provider_kind'
|
||||
)",
|
||||
)
|
||||
.fetch_one(&second)
|
||||
.await
|
||||
.expect("check removed client key provider column");
|
||||
let routing_history_columns = sqlx::query_scalar::<_, String>(
|
||||
"select column_name from information_schema.columns
|
||||
where table_schema = 'public'
|
||||
and table_name = 'model_requests'
|
||||
and column_name in (
|
||||
'routing_scope', 'routing_group_refs', 'routing_group_names_snapshot'
|
||||
)
|
||||
order by column_name",
|
||||
)
|
||||
.fetch_all(&second)
|
||||
.await
|
||||
.expect("load routing history columns");
|
||||
.expect("load saved runtime setting");
|
||||
second.close().await;
|
||||
|
||||
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
|
||||
@@ -225,72 +175,9 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly
|
||||
)))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.expect("drop migration test database");
|
||||
.expect("drop connection test database");
|
||||
admin.close().await;
|
||||
|
||||
assert_eq!(
|
||||
first_tables,
|
||||
[
|
||||
"_sqlx_migrations",
|
||||
"account_group_accounts",
|
||||
"account_groups",
|
||||
"admin_audit_events",
|
||||
"admin_users",
|
||||
"backup_records",
|
||||
"backup_settings",
|
||||
"client_api_key_budget_windows",
|
||||
"client_api_key_charge_events",
|
||||
"client_api_key_groups",
|
||||
"client_api_keys",
|
||||
"model_request_leases",
|
||||
"model_request_observations",
|
||||
"model_requests",
|
||||
"ops_events",
|
||||
"outbound_proxies",
|
||||
"plugin_account_group_resources",
|
||||
"plugin_artifact_credentials",
|
||||
"plugin_artifact_platforms",
|
||||
"plugin_artifacts",
|
||||
"plugin_client_api_key_resources",
|
||||
"plugin_instance_secrets",
|
||||
"plugin_instances",
|
||||
"plugin_source_credentials",
|
||||
"plugin_state_generations",
|
||||
"plugin_state_records",
|
||||
"plugin_update_sources",
|
||||
"plugin_version_configurations",
|
||||
"provider_accounts",
|
||||
"provider_authorization_receipts",
|
||||
"request_analytics_15m",
|
||||
"request_analytics_1d_counts",
|
||||
"request_analytics_1h",
|
||||
"request_analytics_build_state",
|
||||
"request_analytics_facts",
|
||||
"request_analytics_pending_buckets",
|
||||
"request_analytics_pending_requests",
|
||||
"runtime_settings",
|
||||
]
|
||||
);
|
||||
assert_eq!(session_settings, ("codex-proxy-rs".to_owned(), 30, 5, 30));
|
||||
assert_eq!(
|
||||
migration_count,
|
||||
i64::try_from(TEST_MIGRATOR.iter().count())
|
||||
.expect("migration count fits PostgreSQL bigint")
|
||||
);
|
||||
assert_eq!(response_id_types, ["bytea", "bytea"]);
|
||||
assert!(!raw_response_id_index_exists);
|
||||
assert!(!legacy_key_provider_column_exists);
|
||||
assert!(routing_history_columns.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrations_should_leave_transaction_ownership_to_sqlx() {
|
||||
let transaction_statements = TEST_MIGRATOR
|
||||
.iter()
|
||||
.flat_map(|migration| migration.sql.as_str().lines())
|
||||
.map(str::trim)
|
||||
.filter(|line| matches!(*line, "begin;" | "commit;"))
|
||||
.count();
|
||||
|
||||
assert_eq!(transaction_statements, 0);
|
||||
assert_eq!(refresh_margin, 1800, "reopening preserves custom settings");
|
||||
}
|
||||
|
||||
@@ -18,8 +18,8 @@ mod admission_recovery;
|
||||
mod analytics;
|
||||
mod analytics_buckets;
|
||||
mod analytics_reset;
|
||||
mod analytics_upgrade;
|
||||
mod backup;
|
||||
mod backup_export;
|
||||
mod client_budgets;
|
||||
mod client_keys;
|
||||
mod connection;
|
||||
@@ -69,10 +69,6 @@ pub(super) fn admin_account_store(pool: &PgPool) -> PgAdminAccountStore {
|
||||
|
||||
impl TestDatabase {
|
||||
pub(super) async fn create(label: &str) -> Option<Self> {
|
||||
Self::create_through(label, i64::MAX).await
|
||||
}
|
||||
|
||||
pub(super) async fn create_through(label: &str, migration_version: i64) -> Option<Self> {
|
||||
let database_url = crate::support::test_env("CPR_TEST_DATABASE_URL")?;
|
||||
let schema = format!("cpr_store_{label}_{}", Uuid::new_v4().simple());
|
||||
// 临时 schema 验证事务可见性与回滚,不模拟 PostgreSQL 掉电恢复
|
||||
@@ -95,7 +91,7 @@ impl TestDatabase {
|
||||
.await
|
||||
.expect("connect isolated test schema");
|
||||
TEST_MIGRATOR
|
||||
.run_to(migration_version, &pool)
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("apply test migrations");
|
||||
Some(Self {
|
||||
|
||||
@@ -1,58 +1,10 @@
|
||||
//! 观测视图升级兼容性与普通聚合的覆盖索引规划回归测试
|
||||
//! 验证观测视图普通聚合能够使用覆盖索引
|
||||
|
||||
use chrono::Utc;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{TestDatabase, seed_observability_facts};
|
||||
|
||||
#[tokio::test]
|
||||
async fn projection_upgrade_preserves_observation_values() {
|
||||
let Some(db) = TestDatabase::create_through("observation_projection_upgrade", 26).await else {
|
||||
return;
|
||||
};
|
||||
// 此测试固定覆盖 26 → 27 的冻结视图合同,不依赖最新版本的写入夹具
|
||||
sqlx::query("insert into model_requests(id,client_api_key_ref,operation,client_transport,outcome,client_status_code,started_at,deadline_at,completed_at,request_observation_json) values('req_observe_success','historical_key','responses','http_sse','succeeded',200,now()-interval '1 hour',now(),now(),'{\"request\":{\"configRevision\":1,\"protocol\":\"openai\",\"endpoint\":\"/v1/responses\",\"compact\":false},\"routing\":{\"scope\":\"all\",\"groupRefs\":[],\"groupNamesSnapshot\":[]},\"timings\":{\"local\":{\"latencyMs\":800}}}'::jsonb)")
|
||||
.execute(&db.pool).await.unwrap();
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set request_observation_json = jsonb_set(request_observation_json,
|
||||
'{timings,upstream}', '{\"responseMs\":700,\"engineIapiTbtMs\":2.450638}')
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let before: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(mr) from model_request_observations mr order by id")
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
super::super::TEST_MIGRATOR
|
||||
.run_to(27, &db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let after: Vec<Value> =
|
||||
sqlx::query_scalar("select to_jsonb(mr) from model_request_observations mr order by id")
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"projection upgrade must preserve every field"
|
||||
);
|
||||
let updatable: String = sqlx::query_scalar(
|
||||
"select is_updatable from information_schema.views
|
||||
where table_schema = current_schema() and table_name = 'model_request_observations'",
|
||||
)
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(updatable, "NO", "observation projection remains read-only");
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plain_observation_aggregates_can_use_a_covering_index() {
|
||||
let Some(db) = TestDatabase::create("observation_covering_index").await else {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证插件制品身份、信任接受、迁移与安装幂等性
|
||||
//! 验证插件制品身份、信任接受与安装幂等性
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -83,12 +83,7 @@ async fn unknown_metadata_fields_do_not_change_artifact_identity() {
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(stored.get("permissionDescriptions").is_none());
|
||||
assert!(stored.get("requestedPermissions").is_none());
|
||||
let mut extended = stored;
|
||||
extended["permissionDescriptions"] =
|
||||
serde_json::json!([{"permission": "models", "label": "旧文案"}]);
|
||||
extended["requestedPermissions"] = serde_json::json!(["models"]);
|
||||
extended["futureMetadata"] = serde_json::json!({"enabled": true});
|
||||
extended["contributes"]["middleware"]["futureField"] = serde_json::json!(true);
|
||||
sqlx::query("update plugin_artifacts set metadata_json=$2 where sha256=$1")
|
||||
@@ -136,94 +131,6 @@ async fn unknown_metadata_fields_do_not_change_artifact_identity() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn trust_migration_removes_only_obsolete_metadata_fields() {
|
||||
let Some(database) = TestDatabase::create_through("plugin_trust_upgrade", 19).await else {
|
||||
return;
|
||||
};
|
||||
initialize_revision(&database).await;
|
||||
let store = PgPluginStore::new(database.pool.clone());
|
||||
for (digest, extra) in [
|
||||
('a', serde_json::json!({"requestedPermissions": ["models"]})),
|
||||
(
|
||||
'b',
|
||||
serde_json::json!({"permissionDescriptions": ["旧文案"]}),
|
||||
),
|
||||
(
|
||||
'c',
|
||||
serde_json::json!({"requestedPermissions": [], "permissionDescriptions": []}),
|
||||
),
|
||||
('d', serde_json::json!({})),
|
||||
] {
|
||||
// 每个制品使用独立平台,保留同一版本的平台不可变约束
|
||||
let fixture = artifact(digest, &[&format!("test-{digest}")]);
|
||||
let installed = store
|
||||
.install_artifact(fixture.clone(), PluginSource::Upload, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
if digest == 'a' {
|
||||
store
|
||||
.accept_artifact(&fixture.metadata.sha256, &context())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
sqlx::query("update plugin_artifacts set metadata_json=metadata_json || $2 || '{\"futureMetadata\":true}'::jsonb where sha256=$1")
|
||||
.bind(&fixture.metadata.sha256).bind(extra).execute(&database.pool).await.unwrap();
|
||||
sqlx::query("insert into plugin_instances(id,artifact_sha256,name,enabled,configuration_json,bindings_json,revision) values ($1,$2,'Example',false,'{}','[]',$3)")
|
||||
.bind(uuid::Uuid::now_v7()).bind(&fixture.metadata.sha256)
|
||||
.bind(i64::try_from(installed.config_revision.get()).unwrap())
|
||||
.execute(&database.pool).await.unwrap();
|
||||
}
|
||||
let mut expected: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(a) from plugin_artifacts a order by sha256")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for row in &mut expected {
|
||||
let metadata = row["metadata_json"].as_object_mut().unwrap();
|
||||
metadata.remove("requestedPermissions");
|
||||
metadata.remove("permissionDescriptions");
|
||||
}
|
||||
let instances: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(i) from plugin_instances i order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let revision: i64 =
|
||||
sqlx::query_scalar("select config_revision from runtime_settings where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for _ in 0..2 {
|
||||
super::super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let actual: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(a) from plugin_artifacts a order by sha256")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
actual, expected,
|
||||
"只清理两个旧字段,保留包体、接受事实与其他元数据"
|
||||
);
|
||||
let current_instances: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(i) from plugin_instances i order by id")
|
||||
.fetch_all(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(current_instances, instances);
|
||||
let snapshot = store.load_instances().await.unwrap();
|
||||
assert_eq!(snapshot.instances.len(), 4);
|
||||
assert_eq!(
|
||||
snapshot.config_revision.get(),
|
||||
u64::try_from(revision).unwrap()
|
||||
);
|
||||
}
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn artifact_acceptance_is_exact_and_idempotent() {
|
||||
let Some(database) = TestDatabase::create("plugin_artifact_acceptance").await else {
|
||||
@@ -262,51 +169,6 @@ async fn artifact_acceptance_is_exact_and_idempotent() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plugin_migration_preserves_main_settings_and_accepts_new_installations() {
|
||||
let Some(database) = TestDatabase::create_through("plugin_upgrade_from_main", 17).await else {
|
||||
return;
|
||||
};
|
||||
initialize_revision(&database).await;
|
||||
// 0022 会把仍为旧默认 3600 的 margin 收敛到 300;预设自定义值,
|
||||
// 让本测试继续验证「所有已自定义设置都原样保留」。
|
||||
sqlx::query("update runtime_settings set max_concurrent_per_account=0, refresh_margin_seconds=1800 where id=1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let before: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from runtime_settings s where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let after: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(s) from runtime_settings s where id=1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for (key, value) in before.as_object().unwrap() {
|
||||
if key == "request_interval_ms" {
|
||||
assert!(after.get(key).is_none(), "removed setting {key}");
|
||||
continue;
|
||||
}
|
||||
assert_eq!(after.get(key), Some(value), "existing setting {key}");
|
||||
}
|
||||
let store = PgPluginStore::new(database.pool.clone());
|
||||
store
|
||||
.install_artifact(
|
||||
artifact('a', &["linux-x86_64"]),
|
||||
PluginSource::Upload,
|
||||
&context(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn source_identity_requires_publisher_and_name() {
|
||||
let Some(database) = TestDatabase::create("plugin_source_identity").await else {
|
||||
|
||||
@@ -1193,63 +1193,6 @@ async fn legacy_urls_join_one_catalog_entry_and_invalid_batch_rolls_back() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_backfills_shared_proxies_without_changing_credentials() {
|
||||
let Some(database) = TestDatabase::create_through("proxy_backfill", 4).await else {
|
||||
return;
|
||||
};
|
||||
// 从真实旧 schema 正向升级,不拆卸最新结构,避免遗漏后续增加的引用约束
|
||||
for id in ["acct_one", "acct_two", "acct_direct"] {
|
||||
let proxy_url = (id != "acct_direct").then_some("http://user:secret@127.0.0.1:8080/");
|
||||
sqlx::query(
|
||||
"insert into provider_accounts (
|
||||
id,provider_kind,name,authentication_kind,provider_credentials_json,
|
||||
has_refresh_token,credential_observed_at,created_at,updated_at,outbound_proxy_url
|
||||
) values ($1,'openai',$1,'oauth','{}',false,now(),now(),now(),$2)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(proxy_url)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
assert!(
|
||||
sqlx::query_scalar::<_, bool>("select bool_and(enabled) from outbound_proxies")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let accounts = PgProviderAccountRepository::new(database.pool.clone());
|
||||
let inherited = gateway_core::account::ProviderAccountStore::list_accounts(&accounts)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(inherited.len(), 3);
|
||||
assert!(
|
||||
inherited
|
||||
.iter()
|
||||
.all(|account| account.request_location().is_none())
|
||||
);
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i64>("select count(*) from outbound_proxies")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
assert_eq!(sqlx::query_scalar::<_, i64>("select count(*) from provider_accounts a join outbound_proxies p on a.outbound_proxy_id = p.id where a.outbound_proxy_url = p.proxy_url and a.credential_revision = 1").fetch_one(&database.pool).await.unwrap(), 2);
|
||||
assert!(
|
||||
sqlx::query_scalar::<_, Option<String>>(
|
||||
"select outbound_proxy_id from provider_accounts where id = 'acct_direct'"
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_proxy_location_projects_to_accounts_and_guards_refresh_results() {
|
||||
use gateway_core::account::{ProviderAccountStore, RequestLocation};
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
//! 验证短保留期升级、分批清理边界与统计异步收敛期间的查询可用性
|
||||
|
||||
use std::num::NonZeroU32;
|
||||
use std::{num::NonZeroU32, time::Duration};
|
||||
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use gateway_admin::{
|
||||
model::retention::{RetentionPolicy, RetentionTarget},
|
||||
ports::retention::RetentionStore as _,
|
||||
ports::{retention::RetentionStore as _, store::ObservabilityStore as _},
|
||||
};
|
||||
use gateway_store::postgres::{
|
||||
ObservabilityRange, PgRequestAnalyticsRepository, PgRetentionRepository,
|
||||
use gateway_store::{
|
||||
RequestAnalyticsCycle,
|
||||
postgres::{
|
||||
ObservabilityQueryBudget, ObservabilityRange, PgRequestAnalyticsRepository,
|
||||
PgRetentionRepository,
|
||||
},
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -18,10 +22,7 @@ use super::{
|
||||
|
||||
#[tokio::test]
|
||||
async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_projected() {
|
||||
use gateway_admin::{
|
||||
model::observability::{AnalyticsState, DashboardQuery, TimeRange},
|
||||
ports::store::ObservabilityStore as _,
|
||||
};
|
||||
use gateway_admin::model::observability::{AnalyticsState, DashboardQuery, TimeRange};
|
||||
|
||||
let Some(database) = TestDatabase::create("short_retention_readable").await else {
|
||||
return;
|
||||
@@ -183,6 +184,107 @@ async fn short_retention_keeps_statistics_readable_until_each_deletion_batch_is_
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_expired_rows_do_not_block_cleanup_and_are_revisited() {
|
||||
let Some(database) = TestDatabase::create("retention_locked").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for id in ["locked", "available"] {
|
||||
seed(&database.pool, id, now - ChronoDuration::days(40), &[]).await;
|
||||
}
|
||||
let mut writing = database.pool.begin().await.unwrap();
|
||||
sqlx::query("select id from model_requests where id='locked' for update")
|
||||
.execute(&mut *writing)
|
||||
.await
|
||||
.unwrap();
|
||||
let repository = PgRetentionRepository::new(database.pool.clone());
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
let deleted = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
repository.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cleanup must skip the occupied row")
|
||||
.unwrap();
|
||||
assert_eq!(deleted, 1);
|
||||
writing.commit().await.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap()
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blocked_cascade_rolls_back_the_batch_and_respects_lock_deadline() {
|
||||
let Some(database) = TestDatabase::create("retention_cascade_deadline").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed(
|
||||
&database.pool,
|
||||
"parent",
|
||||
now - ChronoDuration::days(40),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
sqlx::query("insert into ops_events(id,model_request_id,attempt_index,level,component,operation,failure_kind,message,created_at) values('child','parent',1,'warning','routing','responses','rate_limited','synthetic',$1)")
|
||||
.bind(now).execute(&database.pool).await.unwrap();
|
||||
let mut writing = database.pool.begin().await.unwrap();
|
||||
sqlx::query("select id from ops_events where id='child' for update")
|
||||
.execute(&mut *writing)
|
||||
.await
|
||||
.unwrap();
|
||||
let repository = PgRetentionRepository::new(database.pool.clone());
|
||||
let policy = RetentionPolicy::try_new(31, 7, 90).unwrap();
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(3),
|
||||
repository.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cascade lock must have a server deadline");
|
||||
assert!(result.is_err());
|
||||
writing.rollback().await.unwrap();
|
||||
let remaining: i64 =
|
||||
sqlx::query_scalar("select count(*) from model_requests where id='parent'")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1);
|
||||
assert_eq!(
|
||||
repository
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
policy,
|
||||
NonZeroU32::new(10).unwrap()
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retention_batch_deletes_only_expired_rows_up_to_limit() {
|
||||
let Some(database) = TestDatabase::create("retention_cycle_budget").await else {
|
||||
@@ -280,3 +382,271 @@ async fn empty_request_cleanup_does_not_conflict_with_analytics_publication() {
|
||||
publication.commit().await.unwrap();
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scattered_retention_deletions_commit_atomically_and_allow_new_requests_to_advance() {
|
||||
let Some(database) = TestDatabase::create("retention_scattered_groups").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..600 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30) + ChronoDuration::minutes(number * 20),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let worker = PgRequestAnalyticsRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(2, Duration::from_secs(5), Duration::from_secs(120))
|
||||
.unwrap(),
|
||||
);
|
||||
let before = worker.status().await.unwrap();
|
||||
let retention = PgRetentionRepository::new(database.pool.clone());
|
||||
assert_eq!(
|
||||
retention
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
RetentionPolicy::try_new(10, 30, 90).unwrap(),
|
||||
NonZeroU32::new(1000).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
600
|
||||
);
|
||||
seed(
|
||||
&database.pool,
|
||||
"new_request",
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
|
||||
// 小时层失败时,已经执行的多个刻钟删除批次与事实删除也必须一起回滚
|
||||
sqlx::raw_sql("create function reject_hour_deletion() returns trigger language plpgsql as $$begin raise exception 'test hour deletion failure'; end$$;
|
||||
create trigger reject_hour_deletion before delete on request_analytics_1h for each statement execute function reject_hour_deletion()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let cycle = RequestAnalyticsCycle::default();
|
||||
assert!(cycle.run(&worker).await.is_err());
|
||||
let failed = worker.status().await.unwrap();
|
||||
assert_eq!(failed.generation, before.generation);
|
||||
assert_eq!(failed.pending_requests, 601);
|
||||
let statistics = observability_repository(&database.pool);
|
||||
let range = ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap();
|
||||
assert_eq!(
|
||||
statistics
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap()
|
||||
.requests
|
||||
.request_count,
|
||||
600
|
||||
);
|
||||
sqlx::raw_sql("drop trigger reject_hour_deletion on request_analytics_1h; drop function reject_hour_deletion();
|
||||
create function slow_summary_statement() returns trigger language plpgsql as $$begin perform pg_sleep(0.002); return null; end$$;
|
||||
create trigger slow_quarter_delete before delete on request_analytics_15m for each statement execute function slow_summary_statement();
|
||||
create trigger slow_hour_delete before delete on request_analytics_1h for each statement execute function slow_summary_statement()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
|
||||
// 按语句注入延迟,覆盖大量分组的 SQL 往返成本及多个集合删除批次
|
||||
cycle
|
||||
.run(&worker)
|
||||
.await
|
||||
.expect("retention backlog must advance within the cycle budget");
|
||||
let after = worker.status().await.unwrap();
|
||||
assert_eq!(after.pending_requests, 0);
|
||||
assert_eq!(after.pending_buckets, 0);
|
||||
assert!(after.generation > before.generation);
|
||||
let actual = statistics
|
||||
.usage_summary(range, Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
assert_eq!(actual.requests.total_tokens, 120);
|
||||
let timeline = admin_observability_store(&database.pool)
|
||||
.dashboard_summary(
|
||||
gateway_admin::model::observability::DashboardQuery::new(
|
||||
gateway_admin::model::observability::TimeRange {
|
||||
start: now - ChronoDuration::days(1),
|
||||
end: now,
|
||||
},
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap(),
|
||||
now,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
timeline
|
||||
.trend
|
||||
.iter()
|
||||
.map(|point| point.metrics.request_count)
|
||||
.sum::<u64>(),
|
||||
1
|
||||
);
|
||||
let remaining: (i64, i64, i64, i64) = sqlx::query_as(
|
||||
"select
|
||||
(select count(*) from request_analytics_facts where id like 'expired_%'),
|
||||
(select count(*) from request_analytics_15m where bucket_start < $1),
|
||||
(select count(*) from request_analytics_1h where bucket_start < $1),
|
||||
(select count(*) from request_analytics_1d_counts where bucket_start < $1)",
|
||||
)
|
||||
.bind(now - ChronoDuration::days(10))
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, (0, 0, 0, 0));
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_requests_and_retention_deletions_both_advance_within_each_batch() {
|
||||
let Some(database) = TestDatabase::create("retention_fair_projection").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..20 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
PgRetentionRepository::new(database.pool.clone())
|
||||
.purge_batch(
|
||||
RetentionTarget::ModelRequests,
|
||||
now,
|
||||
RetentionPolicy::try_new(10, 30, 90).unwrap(),
|
||||
NonZeroU32::new(1000).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let worker =
|
||||
PgRequestAnalyticsRepository::new(database.pool.clone(), observability_query_budget());
|
||||
for wave in 1..=3 {
|
||||
for number in 0..2 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("new_{wave}_{number}"),
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
assert_eq!(worker.maintain(4).await.unwrap().projected_requests, 4);
|
||||
let facts: (i64, i64) = sqlx::query_as(
|
||||
"select count(*) filter (where id like 'expired_%'), count(*) filter (where id like 'new_%') from request_analytics_facts",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
// 历史队列尚未排空时,每批都须撤销旧贡献并发布本轮的新请求
|
||||
assert_eq!(facts, (20 - wave * 2, wave * 2));
|
||||
let current = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(1), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(current.requests.request_count, wave as u64 * 2);
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
let final_summary = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(final_summary.requests.request_count, 6);
|
||||
assert_eq!(final_summary.requests.total_tokens, 720);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timed_out_retention_batches_shrink_across_cycles_and_lock_contention() {
|
||||
let Some(database) = TestDatabase::create("retention_batch_timeout").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
for number in 0..600 {
|
||||
seed(
|
||||
&database.pool,
|
||||
&format!("expired_{number:04}"),
|
||||
now - ChronoDuration::days(30),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drain_analytics(&database.pool).await;
|
||||
sqlx::query("delete from model_requests")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
seed(
|
||||
&database.pool,
|
||||
"new_request",
|
||||
now - ChronoDuration::minutes(2),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
// 数据库延迟与实际批量相关,小批量无需等待,以可观察进展验证跨轮退让
|
||||
sqlx::raw_sql("create function slow_large_fact_deletion() returns trigger language plpgsql as $$begin
|
||||
if (select count(*) from removed_facts) > 250 then perform pg_sleep(2); end if;
|
||||
return null; end$$;
|
||||
create trigger slow_large_fact_deletion after delete on request_analytics_facts referencing old table as removed_facts for each statement execute function slow_large_fact_deletion()")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
let worker = PgRequestAnalyticsRepository::new(
|
||||
database.pool.clone(),
|
||||
ObservabilityQueryBudget::try_new(2, Duration::from_secs(1), Duration::from_secs(1))
|
||||
.unwrap(),
|
||||
);
|
||||
let cycle = RequestAnalyticsCycle::default();
|
||||
for _ in 0..2 {
|
||||
assert!(cycle.run(&worker).await.is_err());
|
||||
// 空轮可能来自别的实例持锁,不能因此恢复会反复超时的大批量
|
||||
let mut lock = database.pool.begin().await.unwrap();
|
||||
sqlx::query(
|
||||
"select pg_advisory_xact_lock(hashtextextended(current_schema()::text,382032))",
|
||||
)
|
||||
.execute(&mut *lock)
|
||||
.await
|
||||
.unwrap();
|
||||
let idle = cycle.run(&worker).await.unwrap();
|
||||
assert_eq!(idle.projected_requests, 0);
|
||||
lock.rollback().await.unwrap();
|
||||
}
|
||||
let pending = worker.status().await.unwrap().pending_requests;
|
||||
assert_eq!(pending, 601);
|
||||
// 此轮可以在后续批次超时前已有提交,下一轮重放仍须保持计数正确
|
||||
let _ = cycle.run(&worker).await;
|
||||
assert!(worker.status().await.unwrap().pending_requests < pending);
|
||||
for _ in 0..5 {
|
||||
// 与调度器错误重试一样留出退避时间,让已取消语句完成回滚并释放维护锁
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let _ = cycle.run(&worker).await;
|
||||
if worker.status().await.unwrap().pending_requests == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(worker.status().await.unwrap().pending_requests, 0);
|
||||
let actual = observability_repository(&database.pool)
|
||||
.usage_summary(
|
||||
ObservabilityRange::new(now - ChronoDuration::days(31), now).unwrap(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(actual.requests.request_count, 1);
|
||||
assert_eq!(actual.requests.total_tokens, 120);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! 验证运行设置的数据库约束、升级保留与快照发布
|
||||
//! 验证运行设置的读写、数据库约束与快照发布
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -61,56 +61,12 @@ fn runtime_settings_keep_account_rotation_global() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_request_interval_preserves_settings_and_snapshot() {
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create_through("drop_interval", 29).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("update runtime_settings set request_interval_ms = 60000, max_concurrent_per_account = 7, refresh_concurrency = 12, refresh_margin_seconds = 1234, config_revision = 70 where id = 1")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let column_exists: bool = sqlx::query_scalar("select exists(select 1 from information_schema.columns where table_schema = current_schema() and table_name = 'runtime_settings' and column_name = 'request_interval_ms')")
|
||||
.fetch_one(&database.pool).await.unwrap();
|
||||
assert!(!column_exists);
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let settings = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(settings.values.max_concurrent_per_account, 7);
|
||||
assert_eq!(settings.values.refresh_concurrency, 12);
|
||||
assert_eq!(settings.values.refresh_margin_seconds, 1234);
|
||||
let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone())
|
||||
.load_runtime_snapshot()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
serde_json::to_value(snapshot.settings)
|
||||
.unwrap()
|
||||
.get("request_interval_ms")
|
||||
.is_none()
|
||||
);
|
||||
repository
|
||||
.update_runtime_settings(settings_with_margin(600))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.refresh_margin_seconds,
|
||||
600
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn privacy_policy_upgrade_defaults_off_and_round_trips_in_snapshot() {
|
||||
async fn privacy_policy_defaults_off_and_round_trips_in_snapshot() {
|
||||
use gateway_core::settings::privacy::*;
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create_through("privacy_policy", 25).await else {
|
||||
let Some(database) = TestDatabase::create("privacy_policy").await else {
|
||||
return;
|
||||
};
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(
|
||||
@@ -151,167 +107,18 @@ async fn privacy_policy_upgrade_defaults_off_and_round_trips_in_snapshot() {
|
||||
serde_json::to_value(snapshot.settings).unwrap()["codex_privacy_policy"],
|
||||
serde_json::to_value(expected).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
before.values.codex_privacy_policy,
|
||||
CodexPrivacyPolicy::default()
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn privacy_steps_migration_preserves_rules_and_other_settings() {
|
||||
let Some(database) = TestDatabase::create_through("privacy_steps", 30).await else {
|
||||
return;
|
||||
};
|
||||
let legacy = serde_json::json!({
|
||||
"enabled":true, "onError":"skip_rule", "rules":[
|
||||
{"id":"first","name":"旧规则一","enabled":true,"scope":"request_header","selector":"x-private",
|
||||
"action":"remove_field","pattern":null,"replacement":"","value":null,
|
||||
"replaceAll":true,"caseInsensitive":false,"multiLine":false},
|
||||
{"id":"second","name":"旧规则二","enabled":false,"scope":"request_body","selector":"$.owner",
|
||||
"action":"regex_replace","pattern":"alice","replacement":"user","value":null,
|
||||
"replaceAll":false,"caseInsensitive":true,"multiLine":true}
|
||||
]
|
||||
});
|
||||
let mut expected = serde_json::json!({
|
||||
"enabled":true, "onError":"skip_rule", "rules":[
|
||||
{"id":"first","name":"旧规则一","enabled":true,"steps":[
|
||||
{"scope":"request_header","selector":"x-private","action":"remove_field",
|
||||
"pattern":null,"replacement":"","value":null,
|
||||
"replaceAll":true,"caseInsensitive":false,"multiLine":false}]},
|
||||
{"id":"second","name":"旧规则二","enabled":false,"steps":[
|
||||
{"scope":"request_body","selector":"$.owner","action":"regex_replace",
|
||||
"pattern":"alice","replacement":"user","value":null,
|
||||
"replaceAll":false,"caseInsensitive":true,"multiLine":true}]}
|
||||
]
|
||||
});
|
||||
// 同时包含聚合规则,迁移不能嵌套包装或拆散现有步骤
|
||||
let canonical = serde_json::json!({
|
||||
"id":"aggregate","name":"通用聚合规则","enabled":true,"steps":[
|
||||
{"scope":"request_body","selector":"$.metadata","action":"set_value",
|
||||
"pattern":null,"replacement":"","value":{"owner":"user","items":[1,2]},
|
||||
"replaceAll":true,"caseInsensitive":false,"multiLine":false},
|
||||
{"scope":"request_header","selector":"x-private","action":"remove_field",
|
||||
"pattern":null,"replacement":"","value":null,
|
||||
"replaceAll":true,"caseInsensitive":false,"multiLine":false}
|
||||
]
|
||||
});
|
||||
let mut mixed = legacy;
|
||||
mixed["rules"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.insert(1, canonical.clone());
|
||||
expected["rules"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.insert(1, canonical);
|
||||
sqlx::query("update runtime_settings set codex_privacy_policy_json = $1, config_revision = 70, refresh_margin_seconds = 1234 where id = 1")
|
||||
.bind(&mixed).execute(&database.pool).await.unwrap();
|
||||
let mut expected_row: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(runtime_settings) from runtime_settings where id = 1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
expected_row["codex_privacy_policy_json"] = expected.clone();
|
||||
expected_row["reset_credit_lead_minutes"] = serde_json::json!(10);
|
||||
expected_row["reset_credit_max_weekly_remaining_percent"] = serde_json::Value::Null;
|
||||
// 完整升级还会清理自动回切字段,其余运行设置必须原样保留
|
||||
expected_row["smart_scheduling_json"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("preferHigherWeight");
|
||||
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let stored: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(runtime_settings) from runtime_settings where id = 1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored, expected_row);
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let loaded = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&loaded.values.codex_privacy_policy).unwrap(),
|
||||
expected
|
||||
);
|
||||
// 服务重启后再次检查迁移,不产生写入或额外嵌套
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let stored: serde_json::Value =
|
||||
sqlx::query_scalar("select to_jsonb(runtime_settings) from runtime_settings where id = 1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored, expected_row);
|
||||
let snapshot = gateway_store::postgres::RuntimeSnapshotRepository::load_runtime_snapshot(
|
||||
&gateway_store::postgres::PgRuntimeSnapshotRepository::new(database.pool.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(snapshot.settings).unwrap()["codex_privacy_policy"],
|
||||
expected
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn privacy_steps_migration_rejects_non_object_without_losing_data() {
|
||||
let Some(database) = TestDatabase::create_through("privacy_invalid", 30).await else {
|
||||
return;
|
||||
};
|
||||
let invalid = serde_json::json!({
|
||||
"enabled":false,"onError":"reject_request","rules":[{},42]
|
||||
});
|
||||
sqlx::query("update runtime_settings set codex_privacy_policy_json = $1 where id = 1")
|
||||
.bind(&invalid)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(super::TEST_MIGRATOR.run(&database.pool).await.is_err());
|
||||
let stored: serde_json::Value =
|
||||
sqlx::query_scalar("select codex_privacy_policy_json from runtime_settings where id = 1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored, invalid);
|
||||
let applied: bool =
|
||||
sqlx::query_scalar("select exists(select 1 from _sqlx_migrations where version = 31)")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!applied);
|
||||
sqlx::query("update runtime_settings set codex_privacy_policy_json = jsonb_set(codex_privacy_policy_json, '{rules}', '[]') where id = 1")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let loaded = PgRuntimeSettingsRepository::new(database.pool.clone())
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!loaded.values.codex_privacy_policy.enabled);
|
||||
assert!(loaded.values.codex_privacy_policy.rules.is_empty());
|
||||
assert_eq!(
|
||||
loaded.values.codex_privacy_policy.on_error,
|
||||
gateway_core::settings::privacy::PrivacyFailureMode::RejectRequest
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn smart_settings_upgrade_preserves_selection_and_publishes_custom_config() {
|
||||
async fn smart_settings_round_trip_publish_and_restore_database_defaults() {
|
||||
use gateway_core::account::SmartSchedulingConfig;
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create_through("smart_config", 18).await else {
|
||||
let Some(database) = TestDatabase::create("smart_config").await else {
|
||||
return;
|
||||
};
|
||||
// 升级前不能用包含新列的 Repository,直接写入旧版本已有字段
|
||||
sqlx::query("update runtime_settings set rotation_strategy = 'sticky', refresh_margin_seconds = 3600 where id = 1")
|
||||
.execute(&database.pool).await.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(before.rotation_strategy, "sticky");
|
||||
// 0022 将仍为旧默认 3600 的行迁移到 300;管理员自定义值才会原样保留。
|
||||
assert_eq!(before.values.refresh_margin_seconds, 300);
|
||||
assert_eq!(
|
||||
before.values.smart_scheduling,
|
||||
SmartSchedulingConfig::default()
|
||||
@@ -333,8 +140,17 @@ async fn smart_settings_upgrade_preserves_selection_and_publishes_custom_config(
|
||||
serde_json::to_value(expected).unwrap()
|
||||
);
|
||||
assert!(snapshot.config_revision > before.config_revision);
|
||||
sqlx::query("update runtime_settings set smart_scheduling_json = default where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
before.values.smart_scheduling,
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.smart_scheduling,
|
||||
SmartSchedulingConfig::default()
|
||||
);
|
||||
database.close().await;
|
||||
@@ -434,52 +250,33 @@ async fn unlimited_default_account_concurrency_round_trips_without_relaxing_othe
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migrations_should_narrow_the_default_refresh_margin_to_the_codex_baseline() {
|
||||
async fn refresh_margin_defaults_apply_to_initial_settings_and_database_resets() {
|
||||
let Some(database) = TestDatabase::create("refresh_margin_default").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
// 新装实例:0001 种子行携带旧默认 3600,0022 数据迁移统一收敛到 300。
|
||||
let settings = repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("load settings");
|
||||
assert_eq!(settings.values.refresh_margin_seconds, 300);
|
||||
// 列默认值同步收窄,重建行不会回退到旧值;目录查询限定本测试 schema,
|
||||
// 同库并行测试里停留在旧迁移版本的 schema 仍保留 3600 旧默认,不能被读到。
|
||||
let column_default: String = sqlx::query_scalar(
|
||||
"select column_default from information_schema.columns \
|
||||
where table_schema = current_schema() \
|
||||
and table_name = 'runtime_settings' \
|
||||
and column_name = 'refresh_margin_seconds'",
|
||||
)
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.expect("read column default");
|
||||
assert_eq!(column_default, "300");
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_margin_migration_should_preserve_admin_customized_values() {
|
||||
// 升级语义:0021 之前部署的实例经 0022 迁移后,仅旧默认 3600 收敛到 300,
|
||||
// 管理员自定义值原样保留。
|
||||
let Some(database) = TestDatabase::create_through("refresh_margin_custom", 21).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("update runtime_settings set refresh_margin_seconds = 1800 where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("persist customized margin");
|
||||
super::TEST_MIGRATOR
|
||||
.run(&database.pool)
|
||||
.unwrap();
|
||||
sqlx::query("update runtime_settings set refresh_margin_seconds = default where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.expect("apply remaining migrations");
|
||||
let settings = PgRuntimeSettingsRepository::new(database.pool.clone())
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.expect("load settings");
|
||||
assert_eq!(settings.values.refresh_margin_seconds, 1_800);
|
||||
.expect("reset refresh margin to the database default");
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.refresh_margin_seconds,
|
||||
300
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
@@ -714,7 +511,9 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
),
|
||||
(0, 0, 30, 0)
|
||||
);
|
||||
let mut update = settings_with_margin(3600);
|
||||
let mut update = settings_with_margin(1234);
|
||||
update.values.max_concurrent_per_account = 7;
|
||||
update.values.refresh_concurrency = 12;
|
||||
update.values.max_waiting_per_key = 5;
|
||||
update.values.max_waiting_per_account = 7;
|
||||
update.values.concurrency_wait_timeout_seconds = 12;
|
||||
@@ -723,12 +522,15 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
let settings = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
settings.values.max_concurrent_per_account,
|
||||
settings.values.refresh_concurrency,
|
||||
settings.values.refresh_margin_seconds,
|
||||
settings.values.max_waiting_per_key,
|
||||
settings.values.max_waiting_per_account,
|
||||
settings.values.concurrency_wait_timeout_seconds,
|
||||
settings.values.openai_guardian_reserved_concurrency,
|
||||
),
|
||||
(5, 7, 12, 1)
|
||||
(7, 12, 1234, 5, 7, 12, 1)
|
||||
);
|
||||
let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone())
|
||||
.load_runtime_snapshot()
|
||||
@@ -737,6 +539,9 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
let snapshot_values = serde_json::to_value(&snapshot.settings).unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
snapshot_values["max_concurrent_per_account"]
|
||||
.as_u64()
|
||||
.unwrap(),
|
||||
snapshot_values["max_waiting_per_key"].as_u64().unwrap(),
|
||||
snapshot_values["max_waiting_per_account"].as_u64().unwrap(),
|
||||
snapshot_values["concurrency_wait_timeout_seconds"]
|
||||
@@ -746,7 +551,7 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
|
||||
.as_u64()
|
||||
.unwrap(),
|
||||
),
|
||||
(5, 7, 12, 1)
|
||||
(7, 5, 7, 12, 1)
|
||||
);
|
||||
assert!(snapshot.config_revision > before.config_revision);
|
||||
database.close().await;
|
||||
@@ -1235,48 +1040,6 @@ async fn warmup_cursor_survives_restart_and_settings_updates_without_rewinding()
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_scheduling_upgrade_preserves_settings_without_a_warmup_table() {
|
||||
let Some(database) = TestDatabase::create_through("runtime_scheduling_upgrade", 20).await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
sqlx::query(
|
||||
"update runtime_settings set rotation_strategy = 'sticky',
|
||||
account_warmup_enabled = true, account_warmup_schedule_time = '08:00,13:00',
|
||||
account_warmup_model = 'test-model' where id = 1",
|
||||
)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let warmup_table: Option<String> =
|
||||
sqlx::query_scalar("select to_regclass('account_warmup_slots')::text")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(warmup_table.is_none());
|
||||
let cursor: Option<DateTime<Utc>> =
|
||||
sqlx::query_scalar("select account_warmup_cursor from runtime_settings where id = 1")
|
||||
.fetch_one(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(cursor.is_none());
|
||||
let settings = PgRuntimeSettingsRepository::new(database.pool.clone())
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(settings.rotation_strategy, "sticky");
|
||||
assert!(settings.values.account_warmup_enabled);
|
||||
assert_eq!(settings.values.account_warmup_schedule_time, "08:00,13:00");
|
||||
assert_eq!(
|
||||
settings.values.account_warmup_model.as_deref(),
|
||||
Some("test-model")
|
||||
);
|
||||
assert_eq!(settings.values.openai_guardian_reserved_concurrency, 0);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warmup_cursor_resolves_dst_and_deduplicates_across_timezones() {
|
||||
use gateway_core::{provider_ports::ProviderRuntimePolicyPort, time::DeploymentTimeZone};
|
||||
@@ -1320,23 +1083,17 @@ async fn warmup_cursor_resolves_dst_and_deduplicates_across_timezones() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn account_affinity_upgrade_defaults_and_updates_reach_the_snapshot() {
|
||||
async fn account_affinity_defaults_and_updates_reach_the_snapshot() {
|
||||
use gateway_core::account::AccountAffinity;
|
||||
use gateway_store::postgres::{PgRuntimeSnapshotRepository, RuntimeSnapshotRepository};
|
||||
let Some(database) = TestDatabase::create_through("account_affinity_upgrade", 22).await else {
|
||||
let Some(database) = TestDatabase::create("account_affinity").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("update runtime_settings set rotation_strategy = 'sticky', config_revision = config_revision + 1 where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let before = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(before.rotation_strategy, "sticky");
|
||||
assert_eq!(
|
||||
before.values.openai_account_affinity,
|
||||
AccountAffinity::Relaxed
|
||||
AccountAffinity::Strict
|
||||
);
|
||||
assert_eq!(before.values.max_account_rotations, 3);
|
||||
assert_eq!(before.values.openai_session_affinity_ttl_hours, 24);
|
||||
@@ -1402,105 +1159,12 @@ async fn account_affinity_upgrade_defaults_and_updates_reach_the_snapshot() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preferred_affinity_migration_defaults_to_strict_and_preserves_saved_modes() {
|
||||
use gateway_core::account::AccountAffinity;
|
||||
let Some(fresh) = TestDatabase::create("affinity_default").await else {
|
||||
async fn reset_credit_weekly_threshold_defaults_unset_and_round_trips() {
|
||||
let Some(database) = TestDatabase::create("reset_weekly_threshold").await else {
|
||||
return;
|
||||
};
|
||||
let repository = PgRuntimeSettingsRepository::new(fresh.pool.clone());
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.openai_account_affinity,
|
||||
AccountAffinity::Strict
|
||||
);
|
||||
fresh.close().await;
|
||||
|
||||
for mode in [AccountAffinity::Relaxed, AccountAffinity::Strict] {
|
||||
let database = TestDatabase::create_through("affinity_preserve", 23)
|
||||
.await
|
||||
.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
// 升级前只写当时已有列,现行 Repository 依赖完整迁移后的 schema
|
||||
sqlx::query("update runtime_settings set openai_account_affinity = $1, config_revision = config_revision + 1, updated_at = now() where id = 1")
|
||||
.bind(match mode {
|
||||
AccountAffinity::Relaxed => "relaxed",
|
||||
_ => "strict",
|
||||
})
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.openai_account_affinity,
|
||||
mode
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn removing_switchback_preserves_custom_weights_and_replaces_database_default() {
|
||||
let Some(database) = TestDatabase::create_through("remove_switchback", 31).await else {
|
||||
return;
|
||||
};
|
||||
let expected =
|
||||
gateway_core::account::SmartSchedulingConfig::new([0.0, 2.0, 1.0, 0.5, 1.2, 2.3]).unwrap();
|
||||
let mut legacy = serde_json::to_value(expected).unwrap();
|
||||
legacy["preferHigherWeight"] = serde_json::json!(true);
|
||||
sqlx::query("update runtime_settings set smart_scheduling_json = $1 where id = 1")
|
||||
.bind(legacy)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.smart_scheduling,
|
||||
expected
|
||||
);
|
||||
sqlx::query("update runtime_settings set smart_scheduling_json = default where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
repository
|
||||
.load_runtime_settings()
|
||||
.await
|
||||
.unwrap()
|
||||
.values
|
||||
.smart_scheduling,
|
||||
gateway_core::account::SmartSchedulingConfig::default()
|
||||
);
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reset_credit_weekly_threshold_upgrade_defaults_unset_and_round_trips() {
|
||||
let Some(database) = TestDatabase::create_through("reset_weekly_threshold", 34).await else {
|
||||
return;
|
||||
};
|
||||
sqlx::query("update runtime_settings set reset_credit_lead_minutes = 20 where id = 1")
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run(&database.pool).await.unwrap();
|
||||
let repository = PgRuntimeSettingsRepository::new(database.pool.clone());
|
||||
let settings = repository.load_runtime_settings().await.unwrap();
|
||||
assert_eq!(settings.values.reset_credit_lead_minutes, 20);
|
||||
assert_eq!(
|
||||
settings.values.reset_credit_max_weekly_remaining_percent,
|
||||
None
|
||||
|
||||
@@ -2,59 +2,6 @@
|
||||
|
||||
use super::TestDatabase;
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_details_migration_preserves_existing_request_and_event_text() {
|
||||
let Some(db) = TestDatabase::create_through("error_details_upgrade", 22).await else {
|
||||
return;
|
||||
};
|
||||
seed_legacy_request(&db.pool).await;
|
||||
let raw = "{ \"error\": {\"code\":\"Vendor.Unknown\",\"message\":\"原始错误\"} }";
|
||||
sqlx::query("update model_requests set raw_upstream_error = $1 where id = 'req_integrity'")
|
||||
.bind(raw)
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"insert into ops_events (id, level, component, operation, failure_kind, message, created_at, raw_upstream_error)
|
||||
values ('error_details_upgrade', 'error', 'provider', 'generate', 'unavailable', 'safe message', now(), $1)",
|
||||
).bind(raw).execute(&db.pool).await.unwrap();
|
||||
|
||||
super::TEST_MIGRATOR.run(&db.pool).await.unwrap();
|
||||
let details: Vec<String> = sqlx::query_scalar(
|
||||
"select error_details from model_requests where id = 'req_integrity'
|
||||
union all select error_details from ops_events where id = 'error_details_upgrade'",
|
||||
)
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(details, vec![raw.to_owned(), raw.to_owned()]);
|
||||
let old_columns: i64 = sqlx::query_scalar(
|
||||
"select count(*) from information_schema.columns
|
||||
where table_schema = current_schema() and table_name in ('model_requests', 'ops_events')
|
||||
and column_name = 'raw_upstream_error'",
|
||||
)
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(old_columns, 0);
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
async fn seed_legacy_request(pool: &sqlx::PgPool) {
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
id, client_api_key_ref, config_revision, protocol, operation, endpoint,
|
||||
client_transport, started_at, deadline_at, outcome, completed_at, routing_scope
|
||||
) values (
|
||||
'req_integrity', 'deleted_key', 1, 'openai', 'responses', '/v1/responses',
|
||||
'http_sse', now(), now() + interval '1 hour', 'failed', now(), 'all'
|
||||
)",
|
||||
)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed request without optional facts");
|
||||
}
|
||||
|
||||
async fn seed_request(pool: &sqlx::PgPool) {
|
||||
sqlx::query(
|
||||
"insert into model_requests (
|
||||
@@ -191,11 +138,6 @@ async fn historical_references_are_validated_without_live_foreign_keys() {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(references, "already_deleted_account");
|
||||
let live_columns: i64 = sqlx::query_scalar("select count(*) from information_schema.columns where table_schema = current_schema()
|
||||
and ((table_name = 'model_requests' and column_name in ('client_api_key_id','provider_account_id'))
|
||||
or (table_name = 'ops_events' and column_name = 'provider_account_id'))")
|
||||
.fetch_one(&db.pool).await.unwrap();
|
||||
assert_eq!(live_columns, 0);
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
@@ -252,89 +194,6 @@ async fn backup_completion_cannot_precede_its_start() {
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_observation_migration_preserves_every_historical_field() {
|
||||
let Some(db) = TestDatabase::create_through("request_observation_upgrade", 24).await else {
|
||||
return;
|
||||
};
|
||||
seed_legacy_request(&db.pool).await;
|
||||
sqlx::raw_sql(
|
||||
"insert into model_requests
|
||||
select (jsonb_populate_record(null::model_requests,
|
||||
to_jsonb(mr) || jsonb_build_object('id', 'req_empty'))).*
|
||||
from model_requests mr;
|
||||
update model_requests set config_revision = 7,
|
||||
provider_account_name_snapshot = 'historical account',
|
||||
provider_account_email_snapshot = 'history@example.invalid',
|
||||
provider_account_authentication_kind_snapshot = 'oauth',
|
||||
routing_scope = 'groups', routing_group_refs = array['deleted_group'],
|
||||
routing_group_names_snapshot = '[\"历史组\"]',
|
||||
provider_error_code = 'rate_limit', error_message = 'safe error', retry_after_ms = 0,
|
||||
client_ip = '192.0.2.10/24', user_agent = 'fixture', reasoning_effort = 'high',
|
||||
reasoning_preset = 'balanced', subagent_kind = 'review', compact = true,
|
||||
transport_decision_wait_ms = 0, connect_ms = 2, headers_ms = 3,
|
||||
first_event_ms = 4, first_reasoning_ms = 5, first_text_ms = 6,
|
||||
first_token_ms = 5, latency_ms = 1000, provider_processing_ms = 7,
|
||||
admission_decision_ms = 0, account_selection_wait_ms = 8,
|
||||
capacity_used_slots = 0, capacity_total_slots = 20,
|
||||
http_version = 'HTTP/2', websocket_pool = 'reuse',
|
||||
upstream_connection_id = 'connection', upstream_connection_exit_reason = 'peer_closed',
|
||||
upstream_connection_age_ms = 10, upstream_connection_idle_ms = 0,
|
||||
upstream_response_model = 'response-model', continuation_requested = true,
|
||||
continuation_previous_response_id_hash = repeat('a', 64),
|
||||
continuation_unavailable_reason = 'missing_response',
|
||||
recovery_request_id = 'recovery', recovered_at = completed_at + interval '1 second',
|
||||
recovery_attempt_count = 1, recovery_retry_delay_ms = 0, recovery_total_latency_ms = 1000,
|
||||
input_tokens = 10, output_tokens = 20, total_tokens = 30,
|
||||
cost_source = 'provider_reported', cost_amount = 0.0123456789, cost_currency = 'USD',
|
||||
error_details = '{ \"opaque\": true }', provider_observation_json = '{\"vendor\":true}',
|
||||
diagnostic_trace_json = '{\"attempts\":[]}'
|
||||
where id = 'req_integrity'",
|
||||
).execute(&db.pool).await.expect("populate all moved fields in the released schema");
|
||||
let before: Vec<serde_json::Value> =
|
||||
sqlx::query_scalar("select to_jsonb(mr) from model_requests mr order by id")
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
super::TEST_MIGRATOR.run_to(27, &db.pool).await.unwrap();
|
||||
let after: Vec<serde_json::Value> = sqlx::query_scalar(
|
||||
"select to_jsonb(mr) - 'request_observation_json' - 'upstream_response_ms' - 'upstream_api_overhead_ms' - 'upstream_engine_ms' - 'upstream_engine_iapi_ttft_ms' - 'upstream_engine_service_ttft_ms' - 'upstream_engine_iapi_tbt_ms' - 'upstream_engine_service_tbt_ms'
|
||||
from model_request_observations mr order by id",
|
||||
)
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"all 90 historical facts survive the migration"
|
||||
);
|
||||
let columns: i64 = sqlx::query_scalar(
|
||||
"select count(*) from information_schema.columns
|
||||
where table_schema = current_schema() and table_name = 'model_requests'",
|
||||
)
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(columns, 49);
|
||||
let observation: serde_json::Value = sqlx::query_scalar(
|
||||
"select request_observation_json from model_requests where id = 'req_integrity'",
|
||||
)
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
observation["timings"]["local"]["transportDecisionWaitMs"],
|
||||
0
|
||||
);
|
||||
assert_eq!(observation["scheduling"]["capacityUsedSlots"], 0);
|
||||
assert!(
|
||||
observation["timings"]["upstream"]
|
||||
.get("responseMs")
|
||||
.is_none()
|
||||
);
|
||||
db.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_observation_reject_malformed_documents_and_keep_upstream_clock_independent() {
|
||||
use serde_json::json;
|
||||
|
||||
@@ -8,7 +8,7 @@ use gateway_store::postgres::{
|
||||
use super::TestDatabase;
|
||||
|
||||
#[test]
|
||||
fn snapshot_client_policy_contains_only_common_limits() {
|
||||
fn client_key_snapshot_debug_redacts_plaintext_key() {
|
||||
let policy = ClientApiKeySnapshot {
|
||||
name_snapshot: None,
|
||||
request_profiles: Default::default(),
|
||||
@@ -20,8 +20,6 @@ fn snapshot_client_policy_contains_only_common_limits() {
|
||||
requests_per_minute: 60,
|
||||
},
|
||||
};
|
||||
assert_eq!(policy.limits.max_concurrency, 3);
|
||||
assert!(policy.group_ids.is_empty());
|
||||
assert!(!format!("{policy:?}").contains("sk_snapshot_secret"));
|
||||
}
|
||||
|
||||
@@ -50,6 +48,8 @@ async fn runtime_snapshot_loads_enabled_plaintext_key_without_debug_exposure() {
|
||||
Some("snapshot")
|
||||
);
|
||||
assert!(snapshot.client_api_keys[0].group_ids.is_empty());
|
||||
assert_eq!(snapshot.client_api_keys[0].limits.max_concurrency, 2);
|
||||
assert_eq!(snapshot.client_api_keys[0].limits.requests_per_minute, 60);
|
||||
assert_eq!(
|
||||
snapshot.client_api_keys[0].plaintext_key.expose_for_auth(),
|
||||
plaintext
|
||||
|
||||
@@ -891,8 +891,10 @@ impl CodexProvider {
|
||||
);
|
||||
if codex_oauth {
|
||||
normalize_selected_codex_downstream_body(
|
||||
upstream_request.body_mut(),
|
||||
&mut upstream_request,
|
||||
generate.protocol_payload().context(),
|
||||
&middleware_headers,
|
||||
context.request_id().as_str(),
|
||||
);
|
||||
}
|
||||
if let Some(location) = lease
|
||||
|
||||
@@ -153,8 +153,8 @@ impl OpenAiResponseObservationState {
|
||||
started_at: Instant,
|
||||
) -> bool {
|
||||
let mut changed = false;
|
||||
// 首字采用官方输出项边界,真实推理与正文仍分别计时
|
||||
if signals.output_start {
|
||||
// 首字采用首响应载荷边界,真实推理与正文仍分别计时
|
||||
if signals.response_data {
|
||||
changed |= insert_first_timing(&mut self.timings.first_token_ms, started_at);
|
||||
}
|
||||
if signals.reasoning_output {
|
||||
|
||||
@@ -342,6 +342,14 @@ impl CodexCanonicalDecoder {
|
||||
raw_sse_frame: Option<RawResponseFrame>,
|
||||
output: &mut Vec<ProviderEvent>,
|
||||
) -> Result<(), CodexCanonicalError> {
|
||||
// 首字只要求响应载荷,不依赖 JSON、事件类型或实际输出内容
|
||||
let data = event.data.trim();
|
||||
self.timing_signals.response_data |=
|
||||
if matches!(raw_sse_frame, Some(RawResponseFrame::WebSocket(_))) {
|
||||
!event.data.is_empty()
|
||||
} else {
|
||||
!data.is_empty() && !data.starts_with("[DONE]")
|
||||
};
|
||||
if event.data.trim() == "[DONE]"
|
||||
&& !matches!(raw_sse_frame, Some(RawResponseFrame::WebSocket(_)))
|
||||
{
|
||||
@@ -371,7 +379,7 @@ impl CodexCanonicalDecoder {
|
||||
.or_else(|| value.get("type").and_then(Value::as_str));
|
||||
if event_type == Some("codex.rate_limits") {
|
||||
// HTTP 传输已将此控制帧投影为本地额度事实
|
||||
// 此帧不能成为客户端输出,也不能启动首个输出计时
|
||||
// 此帧只贡献首响应观测,不能成为客户端输出
|
||||
return Ok(());
|
||||
}
|
||||
if event_type == Some("responsesapi.websocket_timing") && self.started && !self.completed {
|
||||
@@ -438,7 +446,6 @@ impl CodexCanonicalDecoder {
|
||||
|
||||
fn merge_timing_signals(&mut self, signals: ResponseEventSignals) {
|
||||
self.timing_signals.protocol_progress |= signals.protocol_progress;
|
||||
self.timing_signals.output_start |= signals.output_start;
|
||||
self.timing_signals.semantic_output |= signals.semantic_output;
|
||||
self.timing_signals.reasoning_output |= signals.reasoning_output;
|
||||
self.timing_signals.text_output |= signals.text_output;
|
||||
|
||||
@@ -4,17 +4,25 @@
|
||||
mod body;
|
||||
mod grok;
|
||||
mod headers;
|
||||
mod search_history;
|
||||
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
pub(super) use body::normalize_codex_request_body;
|
||||
|
||||
pub(crate) fn normalize_selected_codex_downstream_body(
|
||||
body: &mut Map<String, Value>,
|
||||
request: &mut CodexResponsesRequest,
|
||||
context: &Map<String, Value>,
|
||||
middleware_headers: &[MiddlewareHeader],
|
||||
request_id: &str,
|
||||
) {
|
||||
let body = request.body_mut();
|
||||
body::normalize_non_codex_request_body(body);
|
||||
grok::normalize_request_body(body, context);
|
||||
search_history::normalize_search_history(request, middleware_headers, request_id);
|
||||
}
|
||||
|
||||
pub(super) fn is_non_codex_request_header(name: &str) -> bool {
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Codex OAuth 搜索历史缺少工具声明时的临时兼容
|
||||
|
||||
use gateway_core::engine::middleware::MiddlewareHeader;
|
||||
use gateway_protocol::openai::X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::transport::protocol::responses::CodexResponsesRequest;
|
||||
|
||||
// TODO(https://github.com/openai/codex/issues/51962): 官方修复历史搜索项与工具声明不匹配后,
|
||||
// 用含 web_search_call 的普通请求和 Lite 压缩请求验证直连均可完成,再移除此补偿及对应合同测试
|
||||
// Codex 0.163.0 本地压缩可保留搜索历史却使用空工具;真实上游对照中补声明可消除相应 500
|
||||
pub(super) fn normalize_search_history(
|
||||
request: &mut CodexResponsesRequest,
|
||||
middleware_headers: &[MiddlewareHeader],
|
||||
request_id: &str,
|
||||
) {
|
||||
// 按最终请求头选择声明位置,中间件可以覆盖入站 Lite 标记
|
||||
let responses_lite = middleware_headers
|
||||
.iter()
|
||||
.find(|header| {
|
||||
header
|
||||
.name()
|
||||
.eq_ignore_ascii_case(X_OPENAI_INTERNAL_CODEX_RESPONSES_LITE_HEADER)
|
||||
})
|
||||
.map_or(
|
||||
request.responses_lite.as_deref() == Some("true"),
|
||||
|header| header.value().as_ref() == b"true",
|
||||
);
|
||||
if normalize_body(request.body_mut(), responses_lite) {
|
||||
tracing::debug!(
|
||||
request_id,
|
||||
compatibility = "web_search_history",
|
||||
responses_lite,
|
||||
"OpenAI search history declaration supplemented"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_body(body: &mut Map<String, Value>, lite: bool) -> bool {
|
||||
// 原生续接的历史声明可能仅在上游连接中,不能从本轮增量推断工具缺失
|
||||
if body
|
||||
.get("previous_response_id")
|
||||
.is_some_and(|id| !id.is_null())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(input) = body.get("input").and_then(Value::as_array) else {
|
||||
return false;
|
||||
};
|
||||
if !input.iter().any(|item| item["type"] == "web_search_call") {
|
||||
return false;
|
||||
}
|
||||
let tools = match body.get("tools") {
|
||||
None | Some(Value::Null) => &[][..],
|
||||
Some(Value::Array(tools)) => tools.as_slice(),
|
||||
_ => return false,
|
||||
};
|
||||
if declares_search(tools) {
|
||||
return false;
|
||||
}
|
||||
let mut has_tools = !tools.is_empty();
|
||||
for item in input {
|
||||
if matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some("additional_tools" | "tool_search_output")
|
||||
) {
|
||||
let Some(tools) = item.get("tools").and_then(Value::as_array) else {
|
||||
// 不修补非法工具结构,保留上游原有校验
|
||||
return false;
|
||||
};
|
||||
if declares_search(tools) {
|
||||
return false;
|
||||
}
|
||||
has_tools |= !tools.is_empty();
|
||||
}
|
||||
}
|
||||
let disable_tools = !has_tools;
|
||||
if disable_tools
|
||||
&& !body
|
||||
.get("tool_choice")
|
||||
.is_none_or(|choice| matches!(choice.as_str(), Some("auto" | "none")))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let insert_at = input.len()
|
||||
- usize::from(
|
||||
input
|
||||
.last()
|
||||
.is_some_and(|item| item["type"] == "compaction_trigger"),
|
||||
);
|
||||
let tool = json!({"type":"web_search", "external_web_access":false});
|
||||
if lite {
|
||||
let Some(input) = body.get_mut("input").and_then(Value::as_array_mut) else {
|
||||
return false;
|
||||
};
|
||||
// 独立追加声明,不改写已有历史项及其 ID,且保留末尾压缩触发项的位置
|
||||
input.insert(
|
||||
insert_at,
|
||||
json!({"type":"additional_tools", "role":"developer", "tools":[tool]}),
|
||||
);
|
||||
} else if let Some(Value::Array(tools)) = body.get_mut("tools") {
|
||||
tools.push(tool);
|
||||
} else {
|
||||
body.insert("tools".to_owned(), json!([tool]));
|
||||
}
|
||||
// 无工具请求只获得历史解释声明;已有工具请求保留调用方的工具选择,新增搜索仅可读缓存
|
||||
if disable_tools {
|
||||
body.insert("tool_choice".to_owned(), json!("none"));
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn declares_search(tools: &[Value]) -> bool {
|
||||
tools.iter().any(|tool| {
|
||||
tool.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|kind| kind == "web_search" || kind.starts_with("web_search_preview"))
|
||||
})
|
||||
}
|
||||
@@ -222,14 +222,14 @@ pub(crate) fn is_replaceable_response_prelude(event_type: &str, value: &Value) -
|
||||
}
|
||||
}
|
||||
|
||||
/// 单个 Responses 事件对计时系统提供的稳定语义信号
|
||||
/// Responses 响应载荷与实际内容的观测信号
|
||||
///
|
||||
/// `output_start` 只表示官方输出项开始事件;`semantic_output` 仍要求
|
||||
/// `response_data` 表示收到非空响应载荷;`semantic_output` 仍要求
|
||||
/// 实际内容,供重试与交付边界使用,不能与首字观测互换
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct ResponseEventSignals {
|
||||
pub protocol_progress: bool,
|
||||
pub output_start: bool,
|
||||
pub response_data: bool,
|
||||
pub semantic_output: bool,
|
||||
pub reasoning_output: bool,
|
||||
pub text_output: bool,
|
||||
@@ -313,12 +313,11 @@ impl ResponseTimingMetrics {
|
||||
|
||||
/// 从已解析的 Responses 事件提取计时语义
|
||||
///
|
||||
/// 首字采用官方 Codex 的首个 output_item.added 边界
|
||||
/// 首响应载荷由解码器在 JSON 解析前观测
|
||||
/// 语义输出仍要求文本、推理、工具参数、图片结果或工具执行
|
||||
pub fn response_event_signals(event_type: Option<&str>, value: &Value) -> ResponseEventSignals {
|
||||
let mut signals = ResponseEventSignals {
|
||||
protocol_progress: !matches!(event_type, Some("response.failed" | "error")),
|
||||
output_start: event_type == Some("response.output_item.added"),
|
||||
..ResponseEventSignals::default()
|
||||
};
|
||||
match event_type {
|
||||
|
||||
@@ -2,10 +2,7 @@
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
@@ -41,7 +38,7 @@ pub enum WebSocketOriginBreakerDecision {
|
||||
HalfOpenBusy,
|
||||
}
|
||||
|
||||
/// origin 级 WebSocket 快路径熔断器
|
||||
/// origin 级 WebSocket 建连熔断器
|
||||
#[derive(Clone)]
|
||||
pub struct WebSocketOriginBreaker {
|
||||
inner: Arc<Mutex<HashMap<String, CircuitState>>>,
|
||||
@@ -77,81 +74,90 @@ impl WebSocketOriginBreaker {
|
||||
.or_insert_with(CircuitState::closed);
|
||||
|
||||
match state {
|
||||
CircuitState::Closed { failures } => {
|
||||
CircuitState::Closed {
|
||||
generation,
|
||||
failures,
|
||||
} => {
|
||||
retain_window(failures, now, self.config.failure_window);
|
||||
WebSocketOriginBreakerDecision::Allowed(WebSocketOriginBreakerPermit::new(
|
||||
self.clone(),
|
||||
origin_key.to_string(),
|
||||
None,
|
||||
*generation,
|
||||
))
|
||||
}
|
||||
CircuitState::Open { until } if now < *until => WebSocketOriginBreakerDecision::Open,
|
||||
CircuitState::Open { .. } => {
|
||||
let probe_id = Uuid::new_v4();
|
||||
*state = CircuitState::HalfOpen { probe_id };
|
||||
let generation = Uuid::new_v4();
|
||||
*state = CircuitState::HalfOpen { generation };
|
||||
WebSocketOriginBreakerDecision::Allowed(WebSocketOriginBreakerPermit::new(
|
||||
self.clone(),
|
||||
origin_key.to_string(),
|
||||
Some(probe_id),
|
||||
generation,
|
||||
))
|
||||
}
|
||||
CircuitState::HalfOpen { .. } => WebSocketOriginBreakerDecision::HalfOpenBusy,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_success(&self, origin_key: &str, probe_id: Option<Uuid>) {
|
||||
fn record_success(&self, origin_key: &str, generation: Uuid) {
|
||||
let mut circuits = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if permit_still_owns_state(circuits.get(origin_key), probe_id) {
|
||||
circuits.insert(origin_key.to_string(), CircuitState::closed());
|
||||
let Some(state) = circuits.get_mut(origin_key) else {
|
||||
return;
|
||||
};
|
||||
if !permit_still_owns_state(Some(state), generation) {
|
||||
return;
|
||||
}
|
||||
match state {
|
||||
CircuitState::Closed { failures, .. } => {
|
||||
// 保留同轮尚未完成的 opening,让其后续结果继续参与计数
|
||||
failures.clear();
|
||||
}
|
||||
CircuitState::HalfOpen { .. } => *state = CircuitState::closed(),
|
||||
CircuitState::Open { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_failure(&self, origin_key: &str, probe_id: Option<Uuid>) {
|
||||
fn record_failure(&self, origin_key: &str, generation: Uuid) {
|
||||
let now = Instant::now();
|
||||
let mut circuits = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !permit_still_owns_state(circuits.get(origin_key), probe_id) {
|
||||
return;
|
||||
}
|
||||
if probe_id.is_some() {
|
||||
circuits.insert(
|
||||
origin_key.to_string(),
|
||||
CircuitState::Open {
|
||||
until: now + self.config.open_duration,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let state = circuits
|
||||
.entry(origin_key.to_string())
|
||||
.or_insert_with(CircuitState::closed);
|
||||
let CircuitState::Closed { failures } = state else {
|
||||
let Some(state) = circuits.get_mut(origin_key) else {
|
||||
return;
|
||||
};
|
||||
retain_window(failures, now, self.config.failure_window);
|
||||
failures.push_back(now);
|
||||
if failures.len() >= self.config.failure_threshold {
|
||||
*state = CircuitState::Open {
|
||||
until: now + self.config.open_duration,
|
||||
};
|
||||
if !permit_still_owns_state(Some(state), generation) {
|
||||
return;
|
||||
}
|
||||
match state {
|
||||
CircuitState::HalfOpen { .. } => {
|
||||
*state = CircuitState::Open {
|
||||
until: now + self.config.open_duration,
|
||||
};
|
||||
}
|
||||
CircuitState::Closed { failures, .. } => {
|
||||
retain_window(failures, now, self.config.failure_window);
|
||||
failures.push_back(now);
|
||||
if failures.len() >= self.config.failure_threshold {
|
||||
*state = CircuitState::Open {
|
||||
until: now + self.config.open_duration,
|
||||
};
|
||||
}
|
||||
}
|
||||
CircuitState::Open { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel_probe(&self, origin_key: &str, probe_id: Option<Uuid>) {
|
||||
let Some(probe_id) = probe_id else {
|
||||
return;
|
||||
};
|
||||
fn cancel_probe(&self, origin_key: &str, generation: Uuid) {
|
||||
let mut circuits = self
|
||||
.inner
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if permit_still_owns_state(circuits.get(origin_key), Some(probe_id)) {
|
||||
if matches!(circuits.get(origin_key), Some(CircuitState::HalfOpen { generation: active }) if *active == generation)
|
||||
{
|
||||
circuits.insert(
|
||||
origin_key.to_string(),
|
||||
CircuitState::Open {
|
||||
@@ -166,95 +172,71 @@ impl WebSocketOriginBreaker {
|
||||
pub struct WebSocketOriginBreakerPermit {
|
||||
breaker: WebSocketOriginBreaker,
|
||||
origin_key: String,
|
||||
probe_id: Option<Uuid>,
|
||||
degradation_recorded: Arc<AtomicBool>,
|
||||
generation: Uuid,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl WebSocketOriginBreakerPermit {
|
||||
fn new(breaker: WebSocketOriginBreaker, origin_key: String, probe_id: Option<Uuid>) -> Self {
|
||||
fn new(breaker: WebSocketOriginBreaker, origin_key: String, generation: Uuid) -> Self {
|
||||
Self {
|
||||
breaker,
|
||||
origin_key,
|
||||
probe_id,
|
||||
degradation_recorded: Arc::new(AtomicBool::new(false)),
|
||||
generation,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 创建可跨前台预算与后台 opening 生命周期共享的快路径观察器
|
||||
pub(crate) fn fast_path_reporter(&self) -> WebSocketOriginFastPathReporter {
|
||||
WebSocketOriginFastPathReporter {
|
||||
breaker: self.breaker.clone(),
|
||||
origin_key: self.origin_key.clone(),
|
||||
probe_id: self.probe_id,
|
||||
degradation_recorded: Arc::clone(&self.degradation_recorded),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn succeed(mut self) {
|
||||
if !self.degradation_recorded.load(Ordering::Acquire) {
|
||||
self.breaker.record_success(&self.origin_key, self.probe_id);
|
||||
}
|
||||
self.breaker
|
||||
.record_success(&self.origin_key, self.generation);
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
pub fn fast_timeout(mut self) {
|
||||
self.fast_path_reporter().missed();
|
||||
self.armed = false;
|
||||
/// 非池化 opening 已被前台时限终止,按连接失败计数
|
||||
pub fn fast_timeout(self) {
|
||||
self.fail();
|
||||
}
|
||||
|
||||
pub fn fail(mut self) {
|
||||
self.record_failure_once();
|
||||
self.breaker
|
||||
.record_failure(&self.origin_key, self.generation);
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
/// opening 因账号驱逐或服务关闭而取消,不污染 origin 可用性
|
||||
pub(crate) fn cancel(mut self) {
|
||||
self.breaker.cancel_probe(&self.origin_key, self.probe_id);
|
||||
self.breaker.cancel_probe(&self.origin_key, self.generation);
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
fn record_failure_once(&self) {
|
||||
if !self.degradation_recorded.swap(true, Ordering::AcqRel) {
|
||||
self.breaker.record_failure(&self.origin_key, self.probe_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketOriginBreakerPermit {
|
||||
fn drop(&mut self) {
|
||||
if self.armed {
|
||||
self.record_failure_once();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 前台等待结束后仍可报告同一次后台 opening 错过快路径预算
|
||||
pub(crate) struct WebSocketOriginFastPathReporter {
|
||||
breaker: WebSocketOriginBreaker,
|
||||
origin_key: String,
|
||||
probe_id: Option<Uuid>,
|
||||
degradation_recorded: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WebSocketOriginFastPathReporter {
|
||||
pub(crate) fn missed(&self) {
|
||||
if !self.degradation_recorded.swap(true, Ordering::AcqRel) {
|
||||
self.breaker.record_failure(&self.origin_key, self.probe_id);
|
||||
self.breaker
|
||||
.record_failure(&self.origin_key, self.generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum CircuitState {
|
||||
Closed { failures: VecDeque<Instant> },
|
||||
Open { until: Instant },
|
||||
HalfOpen { probe_id: Uuid },
|
||||
Closed {
|
||||
generation: Uuid,
|
||||
failures: VecDeque<Instant>,
|
||||
},
|
||||
Open {
|
||||
until: Instant,
|
||||
},
|
||||
HalfOpen {
|
||||
generation: Uuid,
|
||||
},
|
||||
}
|
||||
|
||||
impl CircuitState {
|
||||
fn closed() -> Self {
|
||||
Self::Closed {
|
||||
// 初始化或半开恢复时创建新代次,隔离之前熔断周期的迟到结果
|
||||
generation: Uuid::new_v4(),
|
||||
failures: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
@@ -269,10 +251,14 @@ fn retain_window(timeouts: &mut VecDeque<Instant>, now: Instant, window: Duratio
|
||||
}
|
||||
}
|
||||
|
||||
fn permit_still_owns_state(state: Option<&CircuitState>, probe_id: Option<Uuid>) -> bool {
|
||||
match (state, probe_id) {
|
||||
(Some(CircuitState::Closed { .. }), None) => true,
|
||||
(Some(CircuitState::HalfOpen { probe_id: active }), Some(expected)) => *active == expected,
|
||||
fn permit_still_owns_state(state: Option<&CircuitState>, generation: Uuid) -> bool {
|
||||
match state {
|
||||
Some(
|
||||
CircuitState::Closed {
|
||||
generation: active, ..
|
||||
}
|
||||
| CircuitState::HalfOpen { generation: active },
|
||||
) => *active == generation,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ use tokio::{
|
||||
use super::{
|
||||
breaker::{
|
||||
WebSocketOriginBreaker, WebSocketOriginBreakerDecision, WebSocketOriginBreakerPermit,
|
||||
WebSocketOriginFastPathReporter,
|
||||
},
|
||||
error::CodexWebSocketExchangeError,
|
||||
exchange::{
|
||||
@@ -306,7 +305,6 @@ async fn prepare_pooled_websocket(
|
||||
struct PooledWebSocketConnectWaiter {
|
||||
started_at: tokio::time::Instant,
|
||||
receiver: oneshot::Receiver<Result<PooledWebSocketHandoff, CodexWebSocketExchangeError>>,
|
||||
fast_path_reporter: Option<WebSocketOriginFastPathReporter>,
|
||||
}
|
||||
|
||||
struct PooledWebSocketHandoff {
|
||||
@@ -324,9 +322,7 @@ impl PooledWebSocketConnectWaiter {
|
||||
match wait_for_fast_path(self.started_at, fast_path_budget, &mut self.receiver).await {
|
||||
WebSocketFastPath::Ready(received) => received,
|
||||
WebSocketFastPath::Missed => {
|
||||
if let Some(reporter) = &self.fast_path_reporter {
|
||||
reporter.missed();
|
||||
}
|
||||
// 熔断许可仍归后台 opening 持有,前台停止等待不等于建连失败
|
||||
return Ok(WebSocketFastPath::Missed);
|
||||
}
|
||||
};
|
||||
@@ -347,9 +343,6 @@ fn start_pooled_websocket_connect(
|
||||
let task_key = connect_lease.key().clone();
|
||||
let started_at = connect_lease.started_at();
|
||||
let cancellation = connect_lease.cancellation_token();
|
||||
let fast_path_reporter = permit
|
||||
.as_ref()
|
||||
.map(WebSocketOriginBreakerPermit::fast_path_reporter);
|
||||
let keepalive = pool.keepalive();
|
||||
let context = PumpLogContext::new(
|
||||
Some(task_key.account_id().to_owned()),
|
||||
@@ -437,7 +430,6 @@ fn start_pooled_websocket_connect(
|
||||
PooledWebSocketConnectWaiter {
|
||||
started_at,
|
||||
receiver,
|
||||
fast_path_reporter,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,14 +4,6 @@ use std::path::Path;
|
||||
|
||||
use provider_openai::config::{DEFAULT_STREAM_MAX_RETRIES, MAX_STREAM_MAX_RETRIES, OpenAiConfig};
|
||||
|
||||
#[test]
|
||||
fn openai_config_ignores_removed_yaml_identity_fields() {
|
||||
let config: OpenAiConfig = serde_json::from_value(serde_json::json!({
|
||||
"wire_profile": {"originator": "legacy-client", "codex_version": "invalid", "residency": "us"}
|
||||
})).unwrap();
|
||||
assert_eq!(config, OpenAiConfig::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_config_keeps_residency_separate_from_client_identity() {
|
||||
use provider_openai::transport::profile::CodexResidency;
|
||||
@@ -98,10 +90,6 @@ fn openai_config_defaults_to_the_provider_owned_operating_values() {
|
||||
u32::try_from(DEFAULT_STREAM_MAX_RETRIES).expect("default retry budget fits u32"),
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
provider_openai::transport::profile::CodexWireProfile::default().user_agent(),
|
||||
"Codex Desktop/0.153.4 (Mac OS 15.7.1; arm64) unknown (Codex Desktop; 26.901.51231)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,6 +12,7 @@ mod precommit;
|
||||
mod privacy;
|
||||
mod recovery;
|
||||
mod response_interrupt;
|
||||
mod search_history;
|
||||
mod search_model_access;
|
||||
mod selection_retry;
|
||||
mod session_binding;
|
||||
@@ -929,6 +930,34 @@ async fn capture_replay_compatibility_request(
|
||||
input: Value,
|
||||
api_key: bool,
|
||||
websocket: bool,
|
||||
) -> Value {
|
||||
let original = json!({
|
||||
"model":"gpt-5.4","store":false,"stream":true,"input":input,
|
||||
"instructions":"Preserve the original instructions.",
|
||||
"tools":[{"type":"function","name":"echo","parameters":{"type":"object","properties":{"status":{"type":"string"},"content":{"type":"array"}}}}],
|
||||
"future_field":{"status":"keep","content":[1]},
|
||||
"client_metadata":{"extension":{"status":"keep","content":[1]}}
|
||||
});
|
||||
let actual = capture_downstream_request(original.clone(), api_key, websocket, false).await;
|
||||
for field in ["model", "store", "instructions", "tools", "future_field"] {
|
||||
assert_eq!(
|
||||
actual[field].to_string(),
|
||||
original[field].to_string(),
|
||||
"unexpected change in {field}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
actual["client_metadata"]["extension"],
|
||||
original["client_metadata"]["extension"]
|
||||
);
|
||||
actual
|
||||
}
|
||||
|
||||
async fn capture_downstream_request(
|
||||
original: Value,
|
||||
api_key: bool,
|
||||
websocket: bool,
|
||||
lite: bool,
|
||||
) -> Value {
|
||||
let http = MockServer::start().await;
|
||||
let (base_url, websocket_server) = if websocket {
|
||||
@@ -980,33 +1009,33 @@ async fn capture_replay_compatibility_request(
|
||||
} else {
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
}
|
||||
let original = json!({
|
||||
"model":"gpt-5.4","store":false,"stream":true,"input":input,
|
||||
"instructions":"Preserve the original instructions.",
|
||||
"tools":[{"type":"function","name":"echo","parameters":{"type":"object","properties":{"status":{"type":"string"},"content":{"type":"array"}}}}],
|
||||
"future_field":{"status":"keep","content":[1]},
|
||||
"client_metadata":{"extension":{"status":"keep","content":[1]}}
|
||||
});
|
||||
let operation = Operation::Generate(GenerateRequest::from_protocol_payload(
|
||||
ProtocolPayload::json_object("openai", original.as_object().unwrap().clone())
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([(
|
||||
"use_websocket".to_owned(),
|
||||
json!(websocket),
|
||||
)])),
|
||||
.with_context(Map::from_iter([
|
||||
("use_websocket".to_owned(), json!(websocket)),
|
||||
(
|
||||
"responses_lite".to_owned(),
|
||||
json!(if lite { "true" } else { "false" }),
|
||||
),
|
||||
])),
|
||||
));
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_replay_compatibility", CancellationToken::new()),
|
||||
context_with_state_owner_and_location(
|
||||
"req_replay_compatibility",
|
||||
"acct_provider_contract",
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
while let Some(event) = stream.next().await {
|
||||
event.expect("successful upstream completion");
|
||||
}
|
||||
let actual = if let Some(server) = websocket_server {
|
||||
if let Some(server) = websocket_server {
|
||||
timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -1019,19 +1048,7 @@ async fn capture_replay_compatibility_request(
|
||||
"compatibility must run before the first send, without retry"
|
||||
);
|
||||
captured_request_body(&requests[0])
|
||||
};
|
||||
for field in ["model", "store", "instructions", "tools", "future_field"] {
|
||||
assert_eq!(
|
||||
actual[field].to_string(),
|
||||
original[field].to_string(),
|
||||
"unexpected change in {field}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
actual["client_metadata"]["extension"],
|
||||
original["client_metadata"]["extension"]
|
||||
);
|
||||
actual
|
||||
}
|
||||
|
||||
const CAPTURE_COMPLETED_SSE: &str = concat!(
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
//! 验证 OAuth 搜索历史补偿的实际 HTTP/WS 出站、Lite 压缩及透传边界
|
||||
|
||||
use super::*;
|
||||
|
||||
fn search_request() -> Value {
|
||||
json!({
|
||||
"model":"gpt-5.4", "store":false, "stream":true,
|
||||
"input":[
|
||||
{"type":"message","role":"user","content":"Find the Rust website"},
|
||||
{"type":"web_search_call","id":"ws_history","status":"completed",
|
||||
"action":{"type":"search","query":"Rust official website"}},
|
||||
{"type":"message","role":"assistant","content":[
|
||||
{"type":"output_text","text":"https://www.rust-lang.org/",
|
||||
"annotations":[{"type":"url_citation","url":"https://www.rust-lang.org/","title":"Rust","start_index":0,"end_index":26}]}]},
|
||||
{"type":"message","role":"user","content":"Repeat the domain"}
|
||||
],
|
||||
"instructions":"Synthetic search history", "future_field":{"keep":true}
|
||||
})
|
||||
}
|
||||
|
||||
fn cached_search() -> Value {
|
||||
json!({"type":"web_search","external_web_access":false})
|
||||
}
|
||||
|
||||
fn echo_tool() -> Value {
|
||||
json!({"type":"function","name":"echo","parameters":{"type":"object"}})
|
||||
}
|
||||
|
||||
fn assert_request_fields(actual: &Value, expected: &Value) {
|
||||
for field in [
|
||||
"input",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"instructions",
|
||||
"future_field",
|
||||
] {
|
||||
assert_eq!(actual.get(field), expected.get(field), "{field}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_without_tools_completes_with_cached_declaration_and_no_execution() {
|
||||
for websocket in [false, true] {
|
||||
for lite in [false, true] {
|
||||
for compact in [false, true] {
|
||||
let mut original = search_request();
|
||||
if compact {
|
||||
original["input"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(json!({"type":"compaction_trigger"}));
|
||||
original["tool_choice"] = json!("auto");
|
||||
original["tools"] = json!([]);
|
||||
}
|
||||
let mut expected = original.clone();
|
||||
expected["tool_choice"] = json!("none");
|
||||
if lite {
|
||||
let input = expected["input"].as_array_mut().unwrap();
|
||||
input.insert(
|
||||
input.len() - usize::from(compact),
|
||||
json!({
|
||||
"type":"additional_tools","role":"developer","tools":[cached_search()]
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
expected["tools"] = json!([cached_search()]);
|
||||
}
|
||||
let actual = capture_downstream_request(original, false, websocket, lite).await;
|
||||
assert_request_fields(&actual, &expected);
|
||||
// 将适配后的正文再次回放,不重复补入声明或改变引用
|
||||
let repeated =
|
||||
capture_downstream_request(expected.clone(), false, websocket, lite).await;
|
||||
assert_request_fields(&repeated, &expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_preserves_existing_tools_and_explicit_choices() {
|
||||
for lite in [false, true] {
|
||||
for choice in [
|
||||
json!("auto"),
|
||||
json!("none"),
|
||||
json!("required"),
|
||||
json!({"type":"function","name":"echo"}),
|
||||
] {
|
||||
let mut original = search_request();
|
||||
original["tool_choice"] = choice;
|
||||
if lite {
|
||||
original["input"].as_array_mut().unwrap().insert(0, json!({
|
||||
"type":"additional_tools", "id":"at_original", "role":"developer", "tools":[echo_tool()]
|
||||
}));
|
||||
} else {
|
||||
original["tools"] = json!([echo_tool()]);
|
||||
}
|
||||
let mut expected = original.clone();
|
||||
if lite {
|
||||
expected["input"].as_array_mut().unwrap().push(json!({
|
||||
"type":"additional_tools","role":"developer","tools":[cached_search()]
|
||||
}));
|
||||
} else {
|
||||
expected["tools"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(cached_search());
|
||||
}
|
||||
let actual = capture_downstream_request(original, false, true, lite).await;
|
||||
assert_request_fields(&actual, &expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_keeps_existing_search_declarations_in_all_supported_locations() {
|
||||
for kind in ["web_search", "web_search_preview"] {
|
||||
for location in ["top", "additional_tools", "tool_search_output"] {
|
||||
let mut original = search_request();
|
||||
let tool = json!({"type":kind,"external_web_access":true,"search_context_size":"high"});
|
||||
if location == "top" {
|
||||
original["tools"] = json!([tool]);
|
||||
} else {
|
||||
original["input"].as_array_mut().unwrap().insert(
|
||||
0,
|
||||
json!({
|
||||
"type":location, "tools":[tool], "id":"existing_declaration"
|
||||
}),
|
||||
);
|
||||
}
|
||||
let actual = capture_downstream_request(original.clone(), false, false, true).await;
|
||||
assert_request_fields(&actual, &original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_compensation_does_not_apply_to_api_key_accounts() {
|
||||
for websocket in [false, true] {
|
||||
for lite in [false, true] {
|
||||
let original = search_request();
|
||||
let actual = capture_downstream_request(original.clone(), true, websocket, lite).await;
|
||||
assert_request_fields(&actual, &original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_keeps_unrecognized_tools_and_unfulfillable_choices_unchanged() {
|
||||
let mut cases = Vec::new();
|
||||
for tools in [json!({"future":true}), json!("invalid")] {
|
||||
let mut body = search_request();
|
||||
body["tools"] = tools;
|
||||
cases.push(body);
|
||||
}
|
||||
for choice in [
|
||||
json!(null),
|
||||
json!("required"),
|
||||
json!("future-choice"),
|
||||
json!({"type":"function","name":"absent"}),
|
||||
] {
|
||||
let mut body = search_request();
|
||||
body["tool_choice"] = choice;
|
||||
cases.push(body);
|
||||
}
|
||||
for kind in ["additional_tools", "tool_search_output"] {
|
||||
let mut body = search_request();
|
||||
body["input"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(json!({"type":kind,"tools":{"future":true}}));
|
||||
cases.push(body);
|
||||
}
|
||||
for body in cases {
|
||||
let actual = capture_downstream_request(body.clone(), false, false, true).await;
|
||||
assert_request_fields(&actual, &body);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requests_without_hosted_search_history_do_not_gain_tools() {
|
||||
for input in [
|
||||
json!([{ "type":"message", "role":"user", "content":"web_search_call" }]),
|
||||
json!([{ "type":"custom_tool_call", "name":"web.run", "call_id":"call_search", "input":"{}" }]),
|
||||
json!([{ "type":"function_call_output", "call_id":"call_search", "output":{"type":"web_search_call"} }]),
|
||||
json!([]),
|
||||
json!(null),
|
||||
json!({"type":"web_search_call"}),
|
||||
] {
|
||||
let mut original = search_request();
|
||||
original["input"] = input;
|
||||
let actual = capture_downstream_request(original.clone(), false, false, true).await;
|
||||
assert_request_fields(&actual, &original);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn search_history_does_not_infer_missing_declarations_from_native_continuation() {
|
||||
let mut original = search_request();
|
||||
original["previous_response_id"] = json!("resp_existing");
|
||||
original["store"] = json!(true);
|
||||
let actual = capture_downstream_request(original.clone(), false, true, true).await;
|
||||
assert_request_fields(&actual, &original);
|
||||
assert_eq!(
|
||||
actual["previous_response_id"],
|
||||
original["previous_response_id"]
|
||||
);
|
||||
}
|
||||
@@ -79,53 +79,8 @@ async fn cyber_policy_rotation_preserves_upstream_response_and_next_request_stay
|
||||
let coordinator =
|
||||
AttemptCoordinator::new(GatewayEngine::new(Arc::new(NoopExecutionStore), providers));
|
||||
let operation = turn_request("policy-session", "policy-turn");
|
||||
let kind = ProviderKind::new("openai").unwrap();
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).unwrap(),
|
||||
SettingsValues::new(3, "smart", Default::default(), None, None)
|
||||
.with_request_location(global_request_location(), true),
|
||||
vec![kind.clone()],
|
||||
vec![ProviderModel::new(
|
||||
kind,
|
||||
UpstreamModelId::new("gpt-5.4").unwrap(),
|
||||
ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation(),
|
||||
)],
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let plan = snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-5.4").unwrap(),
|
||||
&operation,
|
||||
contract_account_scope(),
|
||||
&RoutingContext::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let request = |id| NewModelRequest {
|
||||
admission_decision_ms: None,
|
||||
id: ModelRequestId::new(id).unwrap(),
|
||||
client_api_key_name_snapshot: None,
|
||||
client_api_key_ref: ClientApiKeyId::new("key_openai_contract").unwrap(),
|
||||
config_revision: ConfigRevision::new(1).unwrap(),
|
||||
routing: AccountRoutingSnapshot::all(),
|
||||
protocol: "openai".to_owned(),
|
||||
operation: operation.kind(),
|
||||
endpoint: "responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model: Some(PublicModelId::new("gpt-5.4").unwrap()),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at: SystemTime::now(),
|
||||
deadline_at: (SystemTime::now() + Duration::from_secs(10)).into(),
|
||||
};
|
||||
let plan = execution_plan(&operation);
|
||||
let request = |id| execution_request(id, &operation, Duration::from_secs(10));
|
||||
let mut session = coordinator
|
||||
.start(
|
||||
request("req_policy_first"),
|
||||
@@ -183,3 +138,163 @@ async fn cyber_policy_rotation_preserves_upstream_response_and_next_request_stay
|
||||
);
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stalled_http_proxy_retries_same_account_before_using_reserved_rotation_budget() {
|
||||
stalled_proxy_recovery(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stalled_pooled_websocket_and_http_leave_one_opening_for_rotation() {
|
||||
stalled_proxy_recovery(true).await;
|
||||
}
|
||||
|
||||
async fn stalled_proxy_recovery(websocket: bool) {
|
||||
use provider_openai::credential::ResponsesTransport;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_subagent_a").await;
|
||||
create_account_with_enabled(&store, "acct_subagent_b", false).await;
|
||||
if !websocket {
|
||||
store.set_oauth_transport("acct_subagent_a", ResponsesTransport::Http);
|
||||
}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
store.set_egress(
|
||||
"acct_subagent_a",
|
||||
Some(
|
||||
gateway_core::account::OutboundProxy::parse(&format!(
|
||||
"socks5h://{}",
|
||||
listener.local_addr().unwrap()
|
||||
))
|
||||
.unwrap(),
|
||||
),
|
||||
None,
|
||||
);
|
||||
let openings = Arc::new(AtomicUsize::new(0));
|
||||
let proxy_openings = Arc::clone(&openings);
|
||||
let proxy_store = Arc::clone(&store);
|
||||
let proxy = tokio::spawn(async move {
|
||||
let mut sockets = Vec::new();
|
||||
loop {
|
||||
sockets.push(listener.accept().await.unwrap().0);
|
||||
if proxy_openings.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
// A 已选定后再开放 B,避免测试依赖选号权重或随机顺序
|
||||
proxy_store
|
||||
.set_enabled(&ProviderAccountId::new("acct_subagent_b").unwrap(), true)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/codex/responses"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("content-type", "text/event-stream")
|
||||
.set_body_string(CAPTURE_COMPLETED_SSE),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let provider = provider_with_base_url(&store, server.uri());
|
||||
let providers =
|
||||
ProviderRegistry::new([provider as Arc<dyn gateway_core::engine::provider::Provider>])
|
||||
.unwrap();
|
||||
let coordinator =
|
||||
AttemptCoordinator::new(GatewayEngine::new(Arc::new(NoopExecutionStore), providers));
|
||||
let operation = Operation::Generate(generate_with_session_context(
|
||||
"budget-session",
|
||||
Some("budget-session"),
|
||||
None,
|
||||
));
|
||||
let plan = execution_plan(&operation);
|
||||
let started = std::time::Instant::now();
|
||||
let mut session = coordinator
|
||||
.start(
|
||||
execution_request("req_reserved_rotation", &operation, Duration::from_secs(40)),
|
||||
operation.clone(),
|
||||
plan,
|
||||
None,
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = timeout(Duration::from_secs(30), session.collect_uncommitted()).await;
|
||||
let elapsed = started.elapsed();
|
||||
proxy.abort();
|
||||
let _ = proxy.await;
|
||||
result
|
||||
.expect("rotation must finish within the original startup window")
|
||||
.unwrap();
|
||||
let expected_openings = if websocket { 3 } else { 2 };
|
||||
assert_eq!(openings.load(Ordering::SeqCst), expected_openings);
|
||||
assert!(
|
||||
elapsed >= Duration::from_secs(20),
|
||||
"A must get its own retry: {elapsed:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(28),
|
||||
"B must retain useful time: {elapsed:?}"
|
||||
);
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1, "B must skip a fresh WS opening");
|
||||
assert_eq!(requests[0].method, "POST");
|
||||
eprintln!(
|
||||
"stalled proxy: websocket={websocket}, A openings={expected_openings}, one B payload, elapsed={elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn execution_plan(operation: &Operation) -> gateway_core::routing::RoutingPlan {
|
||||
let kind = ProviderKind::new("openai").unwrap();
|
||||
let snapshot = RuntimeSnapshot::new(
|
||||
ConfigRevision::new(1).unwrap(),
|
||||
SettingsValues::new(3, "smart", Default::default(), None, None)
|
||||
.with_request_location(global_request_location(), true),
|
||||
vec![kind.clone()],
|
||||
vec![ProviderModel::new(
|
||||
kind,
|
||||
UpstreamModelId::new("gpt-5.4").unwrap(),
|
||||
ModelCapabilities::new(BTreeSet::from([operation.kind()]), Some(32_000))
|
||||
.with_upstream_feature_validation(),
|
||||
)],
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
snapshot
|
||||
.plan(
|
||||
&PublicModelId::new("gpt-5.4").unwrap(),
|
||||
operation,
|
||||
contract_account_scope(),
|
||||
&RoutingContext::default(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn execution_request(id: &str, operation: &Operation, timeout: Duration) -> NewModelRequest {
|
||||
NewModelRequest {
|
||||
admission_decision_ms: None,
|
||||
id: ModelRequestId::new(id).unwrap(),
|
||||
client_api_key_name_snapshot: None,
|
||||
client_api_key_ref: ClientApiKeyId::new("key_openai_contract").unwrap(),
|
||||
config_revision: ConfigRevision::new(1).unwrap(),
|
||||
routing: AccountRoutingSnapshot::all(),
|
||||
protocol: "openai".to_owned(),
|
||||
operation: operation.kind(),
|
||||
endpoint: "responses".to_owned(),
|
||||
client_transport: "http_sse".to_owned(),
|
||||
requested_model: Some(PublicModelId::new("gpt-5.4").unwrap()),
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
reasoning_effort: None,
|
||||
reasoning_preset: None,
|
||||
request_kind: None,
|
||||
subagent_kind: None,
|
||||
compact: false,
|
||||
continuation: Default::default(),
|
||||
image_generation_requested: false,
|
||||
started_at: SystemTime::now(),
|
||||
deadline_at: (SystemTime::now() + timeout).into(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,7 +138,6 @@ async fn official_response_duration_is_observed_independently_of_the_request_clo
|
||||
assert_eq!(timings.upstream_response_ms, Some(7_000));
|
||||
assert_eq!(timings.upstream_api_overhead_ms, Some(120.25));
|
||||
assert_eq!(timings.upstream_engine_ms, Some(6400.0));
|
||||
assert_eq!(timings.first_token_ms, None);
|
||||
assert!(timings.first_event_ms.is_some_and(|value| value >= 20_000));
|
||||
}
|
||||
}
|
||||
@@ -195,7 +194,6 @@ async fn response_critical_path_is_observed_without_logical_turn_totals_on_both_
|
||||
assert_eq!(timings.upstream_engine_ms, Some(6710.68507));
|
||||
assert!((timings.upstream_api_overhead_ms.unwrap() - 81.409594).abs() < 0.000001);
|
||||
assert_eq!(timings.upstream_response_ms, Some(7000));
|
||||
assert_eq!(timings.first_token_ms, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,12 +287,39 @@ async fn timing_frames_outside_the_active_response_do_not_enter_request_metrics(
|
||||
let timings = finish_timings(&mut stream, ProviderResponseTimings::default()).await;
|
||||
assert_eq!(timings.upstream_api_overhead_ms, None);
|
||||
assert_eq!(timings.upstream_engine_ms, None);
|
||||
assert_eq!(timings.first_token_ms, None);
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn content_without_output_item_start_keeps_ttft_unknown_on_every_attempt() {
|
||||
async fn sse_keepalive_and_empty_data_do_not_start_ttft() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let (base_url, release, _, server) = paused_chunked_sse_server(
|
||||
": ping\n\nevent: response.created\n\ndata: \t\n\ndata: [DONE]\n\n".to_owned(),
|
||||
CAPTURE_COMPLETED_SSE.to_owned(),
|
||||
)
|
||||
.await;
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", http_generate_operation()),
|
||||
timed_context(Instant::now() - Duration::from_secs(3), 1),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let first = first_upstream_timings(&mut stream).await;
|
||||
assert_eq!(first.first_token_ms, None);
|
||||
release.send(()).unwrap();
|
||||
let final_timings = finish_timings(&mut stream, first).await;
|
||||
assert!(
|
||||
final_timings
|
||||
.first_token_ms
|
||||
.is_some_and(|value| value >= 3_000)
|
||||
);
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_response_uses_request_clock_on_every_attempt() {
|
||||
for attempt in [1, 2] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
@@ -318,7 +343,7 @@ async fn content_without_output_item_start_keeps_ttft_unknown_on_every_attempt()
|
||||
.unwrap();
|
||||
let first = first_upstream_timings(&mut stream).await;
|
||||
assert!(first.first_event_ms.is_some_and(|value| value >= 3_000));
|
||||
assert_eq!(first.first_token_ms, None);
|
||||
assert!(first.first_token_ms.is_some_and(|value| value >= 3_000));
|
||||
assert!(first.first_text_ms.is_some_and(|value| value >= 3_000));
|
||||
release.send(()).unwrap();
|
||||
let final_timings = finish_timings(&mut stream, first).await;
|
||||
@@ -328,7 +353,7 @@ async fn content_without_output_item_start_keeps_ttft_unknown_on_every_attempt()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn structural_frames_start_ttft_before_content_on_http_and_websocket() {
|
||||
async fn created_response_starts_ttft_before_output_on_http_and_websocket() {
|
||||
for websocket in [false, true] {
|
||||
for (output, has_content) in [
|
||||
(
|
||||
@@ -382,13 +407,11 @@ async fn structural_frames_start_ttft_before_content_on_http_and_websocket() {
|
||||
)
|
||||
.await;
|
||||
ws.next().await.unwrap().unwrap();
|
||||
for event in [created, added] {
|
||||
ws.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
ws.send(Message::Text(created.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
released.await.unwrap();
|
||||
for event in [output, completed] {
|
||||
for event in [added, output, completed] {
|
||||
ws.send(Message::Text(event.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -397,8 +420,8 @@ async fn structural_frames_start_ttft_before_content_on_http_and_websocket() {
|
||||
(base_url, release, server)
|
||||
} else {
|
||||
let (base_url, release, _, server) = paused_chunked_sse_server(
|
||||
format!("data: {created}\n\ndata: {added}\n\n"),
|
||||
format!("data: {output}\n\ndata: {completed}\n\n"),
|
||||
format!("data: {created}\n\n"),
|
||||
format!("data: {added}\n\ndata: {output}\n\ndata: {completed}\n\n"),
|
||||
)
|
||||
.await;
|
||||
(base_url, release, server)
|
||||
@@ -423,10 +446,10 @@ async fn structural_frames_start_ttft_before_content_on_http_and_websocket() {
|
||||
return observation.timings();
|
||||
}
|
||||
}
|
||||
panic!("missing structural first-token observation");
|
||||
panic!("missing first-response observation");
|
||||
})
|
||||
.await
|
||||
.expect("structural first token before releasing content");
|
||||
.expect("first response before releasing output");
|
||||
assert!(first.first_token_ms.is_some_and(|value| value >= 3_000));
|
||||
assert_eq!(first.first_text_ms, None);
|
||||
assert_eq!(first.first_reasoning_ms, None);
|
||||
|
||||
@@ -141,11 +141,42 @@ fn official_response_duration_does_not_cross_response_identity_or_incomplete_bou
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn official_timing_frames_do_not_start_first_output_wait() {
|
||||
fn first_response_observation_uses_nonempty_data_before_json_or_event_classification() {
|
||||
for data in [
|
||||
r#"{"type":"response.created","response":{"id":"resp_first"}}"#,
|
||||
r#"{"type":"response.failed","response":{"error":{"message":"failed"}}}"#,
|
||||
r#"{"type":"future.event"}"#,
|
||||
"not-json",
|
||||
] {
|
||||
for websocket in [false, true] {
|
||||
let mut decoder = CodexCanonicalDecoder::new("fallback").with_raw_sse_passthrough();
|
||||
if websocket {
|
||||
let _ = decoder.push_websocket(data);
|
||||
} else {
|
||||
let _ = decoder
|
||||
.push(b": ping\r\nevent: response.created\r\ndata: \t\r\n\r\ndata: [DONE]\n\n");
|
||||
assert!(!decoder.take_timing_signals().response_data);
|
||||
let frame = format!("data: {data}\r\n\r\n");
|
||||
let split = frame.len() / 2;
|
||||
let _ = decoder.push(&frame.as_bytes()[..split]);
|
||||
assert!(!decoder.take_timing_signals().response_data);
|
||||
let _ = decoder.push(&frame.as_bytes()[split..]);
|
||||
}
|
||||
let signals = decoder.take_timing_signals();
|
||||
assert!(signals.response_data, "websocket={websocket}, data={data}");
|
||||
assert!(!signals.semantic_output);
|
||||
assert!(!signals.reasoning_output);
|
||||
assert!(!signals.text_output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn official_timing_frames_start_first_response_without_content() {
|
||||
let mut decoder = CodexCanonicalDecoder::new("gpt-5.4");
|
||||
let events = decoder.push(b"data: {\"type\":\"responsesapi.websocket_timing\",\"timing_metrics\":{\"total_turn_time_s\":20}}\n\n").expect("timing frame");
|
||||
assert!(!events.is_empty());
|
||||
assert!(!decoder.take_timing_signals().output_start);
|
||||
assert!(decoder.take_timing_signals().response_data);
|
||||
assert_eq!(decoder.upstream_response_ms(), None);
|
||||
}
|
||||
|
||||
@@ -169,7 +200,7 @@ fn decoder_should_not_forward_codex_rate_limit_metadata_fixture_as_openai_wire()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_sse_passthrough_should_drop_rate_limit_control_without_timing_side_effects() {
|
||||
fn raw_sse_passthrough_should_observe_rate_limit_response_without_forwarding_it() {
|
||||
let frame = concat!(
|
||||
"event: codex.rate_limits\n",
|
||||
"data: {\"type\":\"codex.rate_limits\",\"rate_limits\":{\"primary\":{\"used_percent\":90}}}\n\n",
|
||||
@@ -183,7 +214,7 @@ fn raw_sse_passthrough_should_drop_rate_limit_control_without_timing_side_effect
|
||||
|
||||
assert!(events.is_empty());
|
||||
assert!(!signals.protocol_progress);
|
||||
assert!(!signals.output_start);
|
||||
assert!(signals.response_data);
|
||||
assert!(!signals.semantic_output);
|
||||
}
|
||||
|
||||
|
||||
@@ -117,7 +117,7 @@ async fn cold_websocket_should_fall_back_without_recording_a_successful_connect(
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn downstream_websocket_new_chain_should_preserve_continuation_after_slow_opening() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
@@ -190,7 +190,7 @@ async fn downstream_websocket_new_chain_should_preserve_continuation_after_slow_
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[tokio::test]
|
||||
async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
@@ -227,67 +227,102 @@ async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn timed_out_websocket_should_finish_in_background_and_serve_the_next_request() {
|
||||
#[tokio::test]
|
||||
async fn slow_pooled_openings_should_remain_healthy_and_serve_subsequent_requests() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (websocket_ready_tx, websocket_ready_rx) = tokio::sync::oneshot::channel();
|
||||
let openings = WebSocketOriginBreakerConfig::default().failure_threshold + 1;
|
||||
let (websocket_ready_tx, mut websocket_ready_rx) = tokio::sync::mpsc::channel(1);
|
||||
let server = tokio::spawn(async move {
|
||||
let (websocket_stream, _) = listener.accept().await.unwrap();
|
||||
let websocket_server = tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let mut websocket = accept_codex_test_websocket(websocket_stream).await;
|
||||
websocket_ready_tx.send(()).unwrap();
|
||||
let _payload = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_background_ready", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
for _ in 0..openings {
|
||||
let (mut websocket_stream, _) = listener.accept().await.unwrap();
|
||||
let mut prefix = [0; 4];
|
||||
loop {
|
||||
let read = websocket_stream.peek(&mut prefix).await.unwrap();
|
||||
assert!(read > 0, "opening socket closed before the request prefix");
|
||||
if read == prefix.len() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
if prefix == *b"POST" {
|
||||
read_http_request(&mut websocket_stream).await;
|
||||
write_completed_sse_response(&mut websocket_stream).await;
|
||||
continue;
|
||||
}
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let ready = websocket_ready_tx.clone();
|
||||
let websocket_server = tokio::spawn(async move {
|
||||
release_rx.await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(websocket_stream).await;
|
||||
ready.send(()).await.unwrap();
|
||||
// 回退的请求只发 HTTP,WS 的首个业务帧必须来自后续请求
|
||||
let payload = websocket.next().await.unwrap().unwrap();
|
||||
let payload: Value = serde_json::from_str(payload.to_text().unwrap()).unwrap();
|
||||
assert_eq!(payload["input"][0]["content"], "subsequent request");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_background_ready", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let (mut http, _) = listener.accept().await.unwrap();
|
||||
let request = read_http_request(&mut http).await;
|
||||
assert!(request.starts_with("POST /codex/responses HTTP/1.1"));
|
||||
write_completed_sse_response(&mut http).await;
|
||||
websocket_server.await.unwrap();
|
||||
let (mut http, _) = listener.accept().await.unwrap();
|
||||
let request = read_http_request(&mut http).await;
|
||||
assert!(request.starts_with("POST /codex/responses HTTP/1.1"));
|
||||
write_completed_sse_response(&mut http).await;
|
||||
release_tx.send(()).unwrap();
|
||||
websocket_server.await.unwrap();
|
||||
}
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let request = new_chain_request("conversation-background-ready");
|
||||
for index in 0..openings {
|
||||
let request = new_chain_request(&format!("conversation-background-{index}"));
|
||||
let first = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_background_http", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("foreground request should use HTTP after its fast-path budget");
|
||||
assert_eq!(first.transport, CodexBackendTransport::HttpSse);
|
||||
assert_eq!(first.transport_metrics.ws_connect_ms, None);
|
||||
assert_eq!(
|
||||
first.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2WebSocketBudgetExhausted)
|
||||
);
|
||||
websocket_ready_rx.recv().await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
let mut subsequent = codex_request(
|
||||
"gpt-5.5",
|
||||
"be brief",
|
||||
vec![json!({"role":"user","content":"subsequent request"})],
|
||||
);
|
||||
subsequent.use_websocket = true;
|
||||
subsequent.local_conversation_id = request.local_conversation_id.clone();
|
||||
let second = backend
|
||||
.create_response(
|
||||
&subsequent,
|
||||
request_context("req_background_reuse", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("next request should reuse the background websocket");
|
||||
|
||||
let first = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_background_http", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("foreground request should use HTTP after its fast-path budget");
|
||||
websocket_ready_rx.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_background_reuse", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("next request should reuse the background websocket");
|
||||
assert_eq!(second.transport, CodexBackendTransport::WebSocket);
|
||||
assert_eq!(
|
||||
second.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::ReusedWebSocket)
|
||||
);
|
||||
assert!(second.body.contains("resp_background_ready"));
|
||||
}
|
||||
server.await.unwrap();
|
||||
|
||||
assert_eq!(first.transport, CodexBackendTransport::HttpSse);
|
||||
assert_eq!(first.transport_metrics.ws_connect_ms, None);
|
||||
assert_eq!(second.transport, CodexBackendTransport::WebSocket);
|
||||
assert_eq!(
|
||||
second.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::ReusedWebSocket)
|
||||
);
|
||||
assert!(second.body.contains("resp_background_ready"));
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -1265,6 +1300,201 @@ async fn origin_breaker_should_open_then_allow_only_one_half_open_probe() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slow_half_open_probe_should_remain_singleflight_and_recover_on_success() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let config = WebSocketOriginBreakerConfig::default();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
reject_websocket_openings(&listener, config.failure_threshold).await;
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let probe = tokio::spawn(async move {
|
||||
release_rx.await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
ready_tx.send(()).unwrap();
|
||||
let _payload = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_probe_reused", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
// 探测未完成前,预算耗尽的原请求和其他新会话都只能发送 HTTP
|
||||
for _ in 0..2 {
|
||||
let (mut http, _) = listener.accept().await.unwrap();
|
||||
assert!(
|
||||
read_http_request(&mut http)
|
||||
.await
|
||||
.starts_with("POST /codex/responses")
|
||||
);
|
||||
write_completed_sse_response(&mut http).await;
|
||||
}
|
||||
probe.await.unwrap();
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _payload = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_after_recovery", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
record_websocket_origin_failures(&backend, config.failure_threshold, "slow-half-open").await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(config.open_duration).await;
|
||||
tokio::time::resume();
|
||||
|
||||
let probe_request = new_chain_request("slow-half-open-probe");
|
||||
let fallback = backend
|
||||
.create_response(
|
||||
&probe_request,
|
||||
request_context("req_slow_probe", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fallback.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2WebSocketBudgetExhausted)
|
||||
);
|
||||
let suppressed = backend
|
||||
.create_response(
|
||||
&new_chain_request("other-cold-conversation"),
|
||||
request_context("req_probe_busy", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
suppressed.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2BreakerOpen)
|
||||
);
|
||||
release_tx.send(()).unwrap();
|
||||
ready_rx.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
let reused = backend
|
||||
.create_response(
|
||||
&probe_request,
|
||||
request_context("req_probe_reuse", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
reused.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::ReusedWebSocket)
|
||||
);
|
||||
let fresh = backend
|
||||
.create_response(
|
||||
&new_chain_request("after-probe-recovery"),
|
||||
request_context("req_after_probe", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(fresh.transport, CodexBackendTransport::WebSocket);
|
||||
assert_eq!(
|
||||
fresh.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::ConnectedWebSocket)
|
||||
);
|
||||
server.await.unwrap();
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn slow_half_open_probe_should_reopen_after_the_hard_connect_timeout() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let config = WebSocketOriginBreakerConfig::default();
|
||||
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
reject_websocket_openings(&listener, config.failure_threshold).await;
|
||||
let (mut opening, _) = listener.accept().await.unwrap();
|
||||
assert!(
|
||||
read_http_request(&mut opening)
|
||||
.await
|
||||
.starts_with("GET /codex/responses")
|
||||
);
|
||||
let stalled = tokio::spawn(async move {
|
||||
let mut byte = [0];
|
||||
assert_eq!(opening.read(&mut byte).await.unwrap(), 0);
|
||||
closed_tx.send(()).unwrap();
|
||||
});
|
||||
for _ in 0..3 {
|
||||
let (mut http, _) = listener.accept().await.unwrap();
|
||||
assert!(
|
||||
read_http_request(&mut http)
|
||||
.await
|
||||
.starts_with("POST /codex/responses")
|
||||
);
|
||||
write_completed_sse_response(&mut http).await;
|
||||
}
|
||||
stalled.await.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
record_websocket_origin_failures(&backend, config.failure_threshold, "hard-timeout").await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(config.open_duration).await;
|
||||
tokio::time::resume();
|
||||
|
||||
let fallback = backend
|
||||
.create_response(
|
||||
&new_chain_request("hard-timeout-probe"),
|
||||
request_context("req_hard_probe", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fallback.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2WebSocketBudgetExhausted)
|
||||
);
|
||||
let busy = backend
|
||||
.create_response(
|
||||
&new_chain_request("while-hard-probe-pending"),
|
||||
request_context("req_hard_busy", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
busy.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2BreakerOpen)
|
||||
);
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
tokio::time::resume();
|
||||
timeout(Duration::from_secs(1), closed_rx)
|
||||
.await
|
||||
.expect("15s hard timeout must close the opening socket")
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
let suppressed = backend
|
||||
.create_response(
|
||||
&new_chain_request("after-hard-probe-failure"),
|
||||
request_context("req_after_hard_failure", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
suppressed.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2BreakerOpen)
|
||||
);
|
||||
server.await.unwrap();
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_half_open_opening_should_allow_another_probe() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -1272,6 +1502,7 @@ async fn cancelled_half_open_opening_should_allow_another_probe() {
|
||||
let breaker_config = WebSocketOriginBreakerConfig::default();
|
||||
|
||||
let (opening_started_tx, opening_started_rx) = tokio::sync::oneshot::channel();
|
||||
let (opening_closed_tx, opening_closed_rx) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
reject_websocket_openings(&listener, breaker_config.failure_threshold).await;
|
||||
|
||||
@@ -1280,12 +1511,21 @@ async fn cancelled_half_open_opening_should_allow_another_probe() {
|
||||
assert!(request.starts_with("GET /codex/responses HTTP/1.1"));
|
||||
opening_started_tx.send(()).unwrap();
|
||||
|
||||
let (mut http, _) = listener.accept().await.unwrap();
|
||||
assert!(
|
||||
read_http_request(&mut http)
|
||||
.await
|
||||
.starts_with("POST /codex/responses")
|
||||
);
|
||||
write_completed_sse_response(&mut http).await;
|
||||
|
||||
let mut byte = [0_u8; 1];
|
||||
let read = timeout(Duration::from_secs(1), opening.read(&mut byte))
|
||||
.await
|
||||
.expect("account eviction should close the half-open probe")
|
||||
.unwrap();
|
||||
assert_eq!(read, 0);
|
||||
opening_closed_tx.send(()).unwrap();
|
||||
|
||||
let (fresh_stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(fresh_stream).await;
|
||||
@@ -1318,23 +1558,27 @@ async fn cancelled_half_open_opening_should_allow_another_probe() {
|
||||
let attempt = tokio::spawn(async move {
|
||||
probe_backend
|
||||
.create_response(
|
||||
&explicit_websocket_warmup_request("conversation-cancelled-half-open"),
|
||||
&new_chain_request("conversation-cancelled-half-open"),
|
||||
request_context("req_cancelled_half_open", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
opening_started_rx.await.unwrap();
|
||||
pool.evict_account("chatgpt-account").await;
|
||||
timeout(Duration::from_secs(1), attempt)
|
||||
let fallback = attempt
|
||||
.await
|
||||
.expect("cancelled request should finish")
|
||||
.unwrap()
|
||||
.expect_err("cancelled half-open opening should fail the request");
|
||||
.expect("slow probe should fall back to HTTP");
|
||||
assert_eq!(
|
||||
fallback.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::Http2WebSocketBudgetExhausted)
|
||||
);
|
||||
pool.evict_account("chatgpt-account").await;
|
||||
opening_closed_rx.await.unwrap();
|
||||
|
||||
let response = backend
|
||||
.create_response(
|
||||
&explicit_websocket_warmup_request("conversation-next-half-open-probe"),
|
||||
&new_chain_request("conversation-next-half-open-probe"),
|
||||
request_context("req_next_half_open_probe", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
@@ -1531,7 +1775,7 @@ async fn half_open_upstream_response_should_close_origin_breaker() {
|
||||
);
|
||||
let response = backend
|
||||
.create_response(
|
||||
&explicit_websocket_warmup_request("conversation-after-half-open-response"),
|
||||
&new_chain_request("conversation-after-half-open-response"),
|
||||
request_context("req_after_half_open_response", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -119,24 +119,23 @@ fn raw_identity_rejects_injection_conflicting_headers_and_unknown_modes() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_configuration_and_old_frozen_profile_keep_their_existing_identity() {
|
||||
fn preset_identity_round_trip_preserves_request_headers_and_official_preview() {
|
||||
let state = CodexWireProfileState::new(CodexWireProfile::default());
|
||||
let legacy = ClientProfileSelection::default();
|
||||
let resolved = legacy.resolve(&state).unwrap();
|
||||
let configuration = ClientProfileSelection::default().document().unwrap();
|
||||
let resolved = RequestProfileSelection::parse(&configuration)
|
||||
.unwrap()
|
||||
.resolve(&state)
|
||||
.unwrap();
|
||||
let restored: CodexWireProfile =
|
||||
serde_json::from_value(serde_json::to_value(&resolved).unwrap()).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
RequestProfileSelection::parse(&legacy.document().unwrap())
|
||||
.unwrap()
|
||||
.resolve(&state)
|
||||
.unwrap(),
|
||||
resolved
|
||||
build_codex_model_headers(&restored, "Bearer fixture", None).unwrap(),
|
||||
build_codex_model_headers(&resolved, "Bearer fixture", None).unwrap()
|
||||
);
|
||||
let old_document = serde_json::to_value(&resolved).unwrap();
|
||||
assert!(old_document.get("exact_user_agent").is_none());
|
||||
let restored: CodexWireProfile = serde_json::from_value(old_document).unwrap();
|
||||
assert_eq!(restored.user_agent(), resolved.user_agent());
|
||||
assert_eq!(
|
||||
state
|
||||
.preview_selection(&legacy.document().unwrap())
|
||||
.preview_selection(&configuration)
|
||||
.unwrap()
|
||||
.expose_to_provider()["versionSource"],
|
||||
"official"
|
||||
|
||||
@@ -88,47 +88,6 @@ fn semantic_output_should_ignore_structural_empty_and_terminal_frames() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_signals_should_count_structural_start_without_semantic_output() {
|
||||
let signals = response_event_signals(
|
||||
Some("response.output_item.added"),
|
||||
&json!({"item": {"type": "message", "content": []}}),
|
||||
);
|
||||
|
||||
assert!(signals.protocol_progress);
|
||||
assert!(signals.output_start);
|
||||
assert!(!signals.semantic_output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_token_should_use_only_the_official_output_item_added_boundary() {
|
||||
for event in [
|
||||
"response.created",
|
||||
"response.in_progress",
|
||||
"keepalive",
|
||||
"codex.rate_limits",
|
||||
"codex.response.metadata",
|
||||
"response.metadata",
|
||||
"responsesapi.websocket_timing",
|
||||
"response.content_part.added",
|
||||
"response.reasoning_summary_part.added",
|
||||
"response.output_text.delta",
|
||||
"response.output_item.done",
|
||||
"response.completed",
|
||||
"response.failed",
|
||||
"error",
|
||||
] {
|
||||
let signals = response_event_signals(Some(event), &json!({"type": event}));
|
||||
assert!(!signals.output_start, "not an output item start: {event}");
|
||||
}
|
||||
let signals = response_event_signals(
|
||||
Some("response.output_item.added"),
|
||||
&json!({"type": "response.output_item.added", "item": {"type": "message", "content": []}}),
|
||||
);
|
||||
assert!(signals.output_start);
|
||||
assert!(!signals.semantic_output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_signals_should_count_text_inside_a_completed_output_item() {
|
||||
let signals = response_event_signals(
|
||||
|
||||
@@ -529,98 +529,30 @@ fn encoder_should_restore_conversation_fallback_after_body_and_context_values()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_accept_metadata_session_and_thread_ids_but_ignore_legacy_context_aliases() {
|
||||
fn encoder_should_extract_websocket_context_from_client_metadata() {
|
||||
let metadata = json!({
|
||||
"session_id": "metadata-session",
|
||||
"thread_id": "metadata-thread",
|
||||
"x-codex-turn-state": "metadata-turn-state",
|
||||
"turn_id": "metadata-turn-id",
|
||||
"future_context": {"keep": true}
|
||||
});
|
||||
let request = request(Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
("turn_state".to_owned(), json!("legacy-turn-state")),
|
||||
(
|
||||
"x-codex-turn-state".to_owned(),
|
||||
json!("legacy-header-turn-state"),
|
||||
),
|
||||
("turn_metadata".to_owned(), json!("legacy-turn-metadata")),
|
||||
("beta_features".to_owned(), json!("legacy-beta")),
|
||||
("include_timing_metrics".to_owned(), json!("legacy-timing")),
|
||||
("codex_window_id".to_owned(), json!("legacy-window")),
|
||||
(
|
||||
"x-codex-window-id".to_owned(),
|
||||
json!("legacy-header-window"),
|
||||
),
|
||||
("parent_thread_id".to_owned(), json!("legacy-parent")),
|
||||
(
|
||||
"x-codex-parent-thread-id".to_owned(),
|
||||
json!("legacy-header-parent"),
|
||||
),
|
||||
("conversationId".to_owned(), json!("legacy-conversation")),
|
||||
("sessionId".to_owned(), json!("legacy-session")),
|
||||
("threadId".to_owned(), json!("legacy-thread")),
|
||||
("client_request_id".to_owned(), json!("legacy-request")),
|
||||
("clientRequestId".to_owned(), json!("legacy-camel-request")),
|
||||
("turnId".to_owned(), json!("legacy-turn")),
|
||||
("x-codex-turn-id".to_owned(), json!("legacy-header-turn")),
|
||||
(
|
||||
"client_metadata".to_owned(),
|
||||
json!({
|
||||
"turnState": "metadata-turn-state",
|
||||
"turnMetadata": "metadata-turn-metadata",
|
||||
"conversation_id": "metadata-conversation",
|
||||
"session_id": "metadata-session",
|
||||
"thread_id": "metadata-thread",
|
||||
"x-codex-window-id": "metadata-window"
|
||||
}),
|
||||
),
|
||||
("client_metadata".to_owned(), metadata.clone()),
|
||||
]));
|
||||
|
||||
let encoded = encode_generate_request(&request, "gpt-test", None).expect("encode");
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
encoded.turn_state,
|
||||
encoded.turn_metadata,
|
||||
encoded.beta_features,
|
||||
encoded.include_timing_metrics,
|
||||
encoded.codex_window_id,
|
||||
encoded.parent_thread_id,
|
||||
encoded.client_conversation_id,
|
||||
encoded.client_session_id,
|
||||
encoded.client_thread_id,
|
||||
encoded.client_request_id,
|
||||
encoded.client_turn_id,
|
||||
),
|
||||
(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some("metadata-session".to_owned()),
|
||||
Some("metadata-thread".to_owned()),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
encoded.client_session_id.as_deref(),
|
||||
Some("metadata-session")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_extract_official_websocket_context_projection() {
|
||||
let request = request(Map::from_iter([
|
||||
("model".to_owned(), json!("client-model")),
|
||||
("input".to_owned(), json!("prompt")),
|
||||
(
|
||||
"client_metadata".to_owned(),
|
||||
json!({
|
||||
"x-codex-turn-state": "metadata-turn-state",
|
||||
"turn_id": "metadata-turn-id"
|
||||
}),
|
||||
),
|
||||
]));
|
||||
|
||||
let encoded = encode_generate_request(&request, "gpt-test", None).expect("encode");
|
||||
|
||||
assert_eq!(encoded.client_thread_id.as_deref(), Some("metadata-thread"));
|
||||
assert_eq!(encoded.turn_state.as_deref(), Some("metadata-turn-state"));
|
||||
assert_eq!(encoded.client_turn_id.as_deref(), Some("metadata-turn-id"));
|
||||
assert_eq!(encoded.body().get("client_metadata"), Some(&metadata));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
//! 验证 WebSocket 熔断器的并发建连结果与跨轮次隔离
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use provider_openai::transport::websocket::{
|
||||
WebSocketOriginBreaker, WebSocketOriginBreakerConfig, WebSocketOriginBreakerDecision,
|
||||
WebSocketOriginBreakerPermit,
|
||||
};
|
||||
|
||||
const ORIGIN: &str = "https://example.test:443";
|
||||
|
||||
fn acquire(breaker: &WebSocketOriginBreaker) -> WebSocketOriginBreakerPermit {
|
||||
let WebSocketOriginBreakerDecision::Allowed(permit) = breaker.try_acquire(ORIGIN) else {
|
||||
panic!("expected an opening permit");
|
||||
};
|
||||
permit
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closed_success_should_keep_counting_concurrent_failures() {
|
||||
let config = WebSocketOriginBreakerConfig::default();
|
||||
let breaker = WebSocketOriginBreaker::with_config(config);
|
||||
let success = acquire(&breaker);
|
||||
let pending_failures = (0..config.failure_threshold)
|
||||
.map(|_| acquire(&breaker))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
success.succeed();
|
||||
for permit in pending_failures {
|
||||
permit.fail();
|
||||
}
|
||||
|
||||
assert!(matches!(
|
||||
breaker.try_acquire(ORIGIN),
|
||||
WebSocketOriginBreakerDecision::Open
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_closed_success_should_clear_previous_failures() {
|
||||
let config = WebSocketOriginBreakerConfig::default();
|
||||
let breaker = WebSocketOriginBreaker::with_config(config);
|
||||
let first_success = acquire(&breaker);
|
||||
let pending_success = acquire(&breaker);
|
||||
|
||||
first_success.succeed();
|
||||
for _ in 0..config.failure_threshold - 1 {
|
||||
acquire(&breaker).fail();
|
||||
}
|
||||
pending_success.succeed();
|
||||
for _ in 0..config.failure_threshold - 1 {
|
||||
acquire(&breaker).fail();
|
||||
}
|
||||
|
||||
acquire(&breaker).succeed();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stale_closed_permits_should_not_change_recovered_failure_history() {
|
||||
for outcome in ["success", "failure", "drop"] {
|
||||
let breaker = WebSocketOriginBreaker::with_config(WebSocketOriginBreakerConfig {
|
||||
failure_threshold: 2,
|
||||
failure_window: Duration::from_secs(30),
|
||||
open_duration: Duration::from_secs(30),
|
||||
});
|
||||
let stale = acquire(&breaker);
|
||||
acquire(&breaker).fail();
|
||||
acquire(&breaker).fail();
|
||||
assert!(matches!(
|
||||
breaker.try_acquire(ORIGIN),
|
||||
WebSocketOriginBreakerDecision::Open
|
||||
));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
acquire(&breaker).succeed();
|
||||
acquire(&breaker).fail();
|
||||
match outcome {
|
||||
"success" => stale.succeed(),
|
||||
"failure" => stale.fail(),
|
||||
_ => drop(stale),
|
||||
}
|
||||
|
||||
// 新代次的一次失败不能被旧成功清除,也不能被旧失败或 Drop 凑满阈值
|
||||
acquire(&breaker).fail();
|
||||
assert!(matches!(
|
||||
breaker.try_acquire(ORIGIN),
|
||||
WebSocketOriginBreakerDecision::Open
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stale_closed_permits_should_not_finish_a_half_open_probe() {
|
||||
for outcome in ["success", "failure", "drop"] {
|
||||
let breaker = WebSocketOriginBreaker::with_config(WebSocketOriginBreakerConfig {
|
||||
failure_threshold: 1,
|
||||
failure_window: Duration::from_secs(30),
|
||||
open_duration: Duration::from_secs(30),
|
||||
});
|
||||
let stale = acquire(&breaker);
|
||||
acquire(&breaker).fail();
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
let probe = acquire(&breaker);
|
||||
match outcome {
|
||||
"success" => stale.succeed(),
|
||||
"failure" => stale.fail(),
|
||||
_ => drop(stale),
|
||||
}
|
||||
assert!(matches!(
|
||||
breaker.try_acquire(ORIGIN),
|
||||
WebSocketOriginBreakerDecision::HalfOpenBusy
|
||||
));
|
||||
probe.succeed();
|
||||
acquire(&breaker).succeed();
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ use provider_openai::transport::{
|
||||
websocket::write_websocket_audit_artifact_for_dir,
|
||||
};
|
||||
|
||||
mod breaker;
|
||||
mod diagnostics;
|
||||
mod fingerprint;
|
||||
mod metadata;
|
||||
@@ -365,8 +366,8 @@ async fn websocket_execute_response_create_request_should_collect_completed_sse(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_exchange_should_accept_upstream_frame_above_removed_private_limit() {
|
||||
const REMOVED_FRAME_LIMIT_BYTES: usize = 16 * 1024 * 1024;
|
||||
async fn websocket_exchange_should_deliver_large_upstream_frames() {
|
||||
const LARGE_DELTA_BYTES: usize = 17 * 1024 * 1024;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
@@ -376,14 +377,13 @@ async fn websocket_exchange_should_accept_upstream_frame_above_removed_private_l
|
||||
let mut websocket =
|
||||
accept_codex_test_websocket_with(stream, |_request, _response| {}).await;
|
||||
let _message = websocket.next().await.unwrap().unwrap();
|
||||
let oversized_event = json!({
|
||||
let large_event = json!({
|
||||
"type": "response.output_text.delta",
|
||||
"delta": "x".repeat(REMOVED_FRAME_LIMIT_BYTES + 1),
|
||||
"delta": "x".repeat(LARGE_DELTA_BYTES),
|
||||
})
|
||||
.to_string();
|
||||
assert!(oversized_event.len() > REMOVED_FRAME_LIMIT_BYTES);
|
||||
websocket
|
||||
.send(Message::Text(oversized_event.into()))
|
||||
.send(Message::Text(large_event.into()))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
@@ -391,7 +391,7 @@ async fn websocket_exchange_should_accept_upstream_frame_above_removed_private_l
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "resp_after_oversized_frame",
|
||||
"id": "resp_after_large_frame",
|
||||
"object": "response",
|
||||
"output": [],
|
||||
"usage": {
|
||||
@@ -421,11 +421,11 @@ async fn websocket_exchange_should_accept_upstream_frame_above_removed_private_l
|
||||
|
||||
let response = execute_response_create_request(&prepared)
|
||||
.await
|
||||
.expect("upstream frame above the former private limit should remain forwardable");
|
||||
.expect("large upstream frame should remain forwardable");
|
||||
server.await.unwrap();
|
||||
|
||||
assert!(response.body.len() > REMOVED_FRAME_LIMIT_BYTES);
|
||||
assert!(response.body.contains("resp_after_oversized_frame"));
|
||||
assert!(response.body.len() > LARGE_DELTA_BYTES);
|
||||
assert!(response.body.contains("resp_after_large_frame"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -228,39 +228,16 @@ async fn quota_refresh_persists_subscription_without_user_profile_and_reads_it_f
|
||||
assert!(document.get("userId").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn billing_percent_does_not_rewrite_quota_access_fact() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("still-exhausted", "still-exhausted")]).await;
|
||||
let id = account_id("still-exhausted");
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
|
||||
crate::support::grok_quota_service(
|
||||
repository,
|
||||
QueueBillingTransport::success(br#"{"config":{"creditUsagePercent":100}}"#),
|
||||
)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh exhausted quota");
|
||||
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").quota().access(),
|
||||
gateway_core::account::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn billing_access_and_scheduling_use_current_quota_fields() {
|
||||
use gateway_core::account::QuotaAccessState::{Allowed, Exhausted};
|
||||
|
||||
for (config, expected_percent, expected_access) in [
|
||||
(
|
||||
serde_json::json!({"creditUsagePercent": 0}),
|
||||
Some(0.0),
|
||||
Allowed,
|
||||
),
|
||||
(
|
||||
serde_json::json!({"creditUsagePercent": 100}),
|
||||
Some(100.0),
|
||||
@@ -306,6 +283,10 @@ async fn billing_access_and_scheduling_use_current_quota_fields() {
|
||||
let account = store.account(&id).expect("refreshed account");
|
||||
|
||||
assert_eq!(snapshot.billing().used_percent(), expected_percent);
|
||||
assert!(
|
||||
snapshot.billing().has_authoritative_quota(),
|
||||
"config: {config}"
|
||||
);
|
||||
assert_eq!(
|
||||
account.quota().access(),
|
||||
expected_access,
|
||||
@@ -526,66 +507,10 @@ async fn quota_refresh_rejects_a_concurrent_credential_revision() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_billing_fields_do_not_supply_quota_evidence_or_period() {
|
||||
for (limit, used) in [
|
||||
(
|
||||
serde_json::json!({"val": 10000}),
|
||||
serde_json::json!({"val": 2500}),
|
||||
),
|
||||
(
|
||||
serde_json::json!("uninterpreted"),
|
||||
serde_json::json!({"val": -1}),
|
||||
),
|
||||
] {
|
||||
let (store, repository) = repository_with_accounts(&[("legacy", "legacy")]).await;
|
||||
let id = account_id("legacy");
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
SystemTime::now(),
|
||||
None,
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
let body = serde_json::to_vec(&serde_json::json!({"config": {
|
||||
"monthlyLimit": limit, "used": used,
|
||||
"billingPeriodStart": "2026-07-01T00:00:00Z",
|
||||
"billingPeriodEnd": "2099-08-01T00:00:00Z"
|
||||
}}))
|
||||
.expect("billing document");
|
||||
let service =
|
||||
crate::support::grok_quota_service(repository, QueueBillingTransport::success(&body));
|
||||
let snapshot = service
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh opaque billing");
|
||||
let account = store.account(&id).expect("account");
|
||||
|
||||
assert_eq!(snapshot.billing().used_percent(), None);
|
||||
assert!(!snapshot.billing().has_authoritative_quota());
|
||||
assert_eq!(
|
||||
snapshot.billing().period_kind(),
|
||||
provider_xai::GrokQuotaPeriodKind::Other
|
||||
);
|
||||
assert_eq!(snapshot.billing().period_start(), None);
|
||||
assert_eq!(snapshot.billing().period_end(), None);
|
||||
assert_eq!(service.scheduling_signals(&account), None);
|
||||
assert_eq!(
|
||||
account.quota().access(),
|
||||
gateway_core::account::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_quota_period_does_not_fill_missing_dates_from_legacy_fields() {
|
||||
async fn quota_period_dates_remain_absent_when_not_reported() {
|
||||
let (_, repository) = repository_with_accounts(&[("current-period", "current-period")]).await;
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"creditUsagePercent":25,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY"},"billingPeriodStart":"2026-07-01T00:00:00Z","billingPeriodEnd":"2099-08-01T00:00:00Z"}}"#,
|
||||
br#"{"config":{"creditUsagePercent":25,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY"}}}"#,
|
||||
);
|
||||
let snapshot = crate::support::grok_quota_service(repository, transport)
|
||||
.refresh_account(&account_id("current-period"))
|
||||
@@ -658,61 +583,59 @@ async fn expired_billing_window_does_not_participate_in_scheduling_rank() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_authoritative_quota_refresh_does_not_clear_existing_quota_exhaustion() {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("free-quota", "subject-free-quota")]).await;
|
||||
let id = account_id("free-quota");
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(QuotaEvidence::ProviderDenied, SystemTime::now(), None),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"onDemandCap":{"val":0},"onDemandUsed":{"val":0},"prepaidBalance":{"val":0}}}"#,
|
||||
);
|
||||
let snapshot = crate::support::grok_quota_service(repository, transport)
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh Free quota");
|
||||
async fn non_authoritative_quota_refresh_preserves_exhaustion_without_scheduling_evidence() {
|
||||
use provider_xai::GrokQuotaPeriodKind::{Other, Weekly};
|
||||
|
||||
assert!(!snapshot.billing().has_authoritative_quota());
|
||||
assert_eq!(
|
||||
store.account(&id).expect("account").quota().access(),
|
||||
gateway_core::account::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
for (config, expected_period) in [
|
||||
(
|
||||
serde_json::json!({"future_quota": {"remaining": 100}}),
|
||||
Other,
|
||||
),
|
||||
(
|
||||
serde_json::json!({
|
||||
"currentPeriod": {"type": "USAGE_PERIOD_TYPE_WEEKLY"},
|
||||
"onDemandCap": {"val": 0},
|
||||
"onDemandUsed": {"val": 0},
|
||||
"prepaidBalance": {"val": 0}
|
||||
}),
|
||||
Weekly,
|
||||
),
|
||||
] {
|
||||
let (store, repository) =
|
||||
repository_with_accounts(&[("free-quota", "subject-free-quota")]).await;
|
||||
let id = account_id("free-quota");
|
||||
store
|
||||
.apply_quota_access(QuotaAccessChange {
|
||||
account_id: id.clone(),
|
||||
expected_revision: CredentialRevision::new(1).expect("revision"),
|
||||
state: QuotaState::exhausted(
|
||||
QuotaEvidence::ProviderDenied,
|
||||
SystemTime::now(),
|
||||
None,
|
||||
),
|
||||
})
|
||||
.await
|
||||
.expect("mark quota exhausted");
|
||||
let body = serde_json::to_vec(&serde_json::json!({"config": config})).expect("billing");
|
||||
let service =
|
||||
crate::support::grok_quota_service(repository, QueueBillingTransport::success(&body));
|
||||
let snapshot = service
|
||||
.refresh_account(&id)
|
||||
.await
|
||||
.expect("refresh Free quota");
|
||||
let account = store.account(&id).expect("account");
|
||||
|
||||
#[tokio::test]
|
||||
async fn reported_zero_percent_is_authoritative_quota() {
|
||||
let (_, repository) =
|
||||
repository_with_accounts(&[("zero-percent", "subject-zero-percent")]).await;
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"creditUsagePercent":0,"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"}}}"#,
|
||||
);
|
||||
let snapshot = crate::support::grok_quota_service(repository, transport)
|
||||
.refresh_account(&account_id("zero-percent"))
|
||||
.await
|
||||
.expect("refresh reported quota");
|
||||
|
||||
assert!(snapshot.billing().has_authoritative_quota());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn positive_prepaid_balance_is_authoritative_quota() {
|
||||
let (_, repository) =
|
||||
repository_with_accounts(&[("prepaid-quota", "subject-prepaid-quota")]).await;
|
||||
let transport = QueueBillingTransport::success(
|
||||
br#"{"config":{"currentPeriod":{"type":"USAGE_PERIOD_TYPE_WEEKLY","start":"2026-07-15T00:00:00Z","end":"2026-07-22T00:00:00Z"},"prepaidBalance":{"val":500}}}"#,
|
||||
);
|
||||
let snapshot = crate::support::grok_quota_service(repository, transport)
|
||||
.refresh_account(&account_id("prepaid-quota"))
|
||||
.await
|
||||
.expect("refresh prepaid quota");
|
||||
|
||||
assert!(snapshot.billing().has_authoritative_quota());
|
||||
assert!(!snapshot.billing().has_authoritative_quota());
|
||||
assert_eq!(snapshot.billing().used_percent(), None);
|
||||
assert_eq!(snapshot.billing().period_kind(), expected_period);
|
||||
assert_eq!(snapshot.billing().period_start(), None);
|
||||
assert_eq!(snapshot.billing().period_end(), None);
|
||||
assert_eq!(service.scheduling_signals(&account), None);
|
||||
assert_eq!(
|
||||
account.quota().access(),
|
||||
gateway_core::account::QuotaAccessState::Exhausted
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -177,109 +177,40 @@ fn encoder_should_strip_all_client_metadata_before_grok_build() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_apply_build_defaults_and_normalize_reasoning_effort() {
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"include": ["web_search_call.action.sources"],
|
||||
"reasoning": {"effort": "XHIGH", "summary": "auto"}
|
||||
}));
|
||||
fn encoder_should_apply_build_defaults_and_preserve_requested_includes() {
|
||||
for (fields, expected_include) in [
|
||||
(json!({}), json!(["reasoning.encrypted_content"])),
|
||||
(
|
||||
json!({"store": null, "include": null}),
|
||||
json!(["reasoning.encrypted_content"]),
|
||||
),
|
||||
(
|
||||
json!({"include": ["web_search_call.action.sources"]}),
|
||||
json!([
|
||||
"web_search_call.action.sources",
|
||||
"reasoning.encrypted_content"
|
||||
]),
|
||||
),
|
||||
] {
|
||||
let mut body = fields;
|
||||
body["input"] = json!("hello");
|
||||
let encoded = GrokResponsesRequest::encode(&raw_request(body), "grok-4.7", &client_key())
|
||||
.expect("build defaults");
|
||||
|
||||
let encoded = GrokResponsesRequest::encode(&request, "grok-4.5", &client_key())
|
||||
.expect("normalized request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
|
||||
assert_eq!(body.pointer("/store"), Some(&json!(false)));
|
||||
assert_eq!(
|
||||
body.pointer("/include"),
|
||||
Some(&json!([
|
||||
"web_search_call.action.sources",
|
||||
"reasoning.encrypted_content"
|
||||
]))
|
||||
);
|
||||
assert_eq!(body.pointer("/reasoning/effort"), Some(&json!("xhigh")));
|
||||
assert_eq!(body.pointer("/reasoning/summary"), Some(&json!("auto")));
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"reasoning": {"effort": "xhigh"}
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "xai/grok-4.6-latest", &client_key())
|
||||
.expect("routed model request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
assert_eq!(body.pointer("/model"), Some(&json!("xai/grok-4.6-latest")));
|
||||
assert_eq!(body.pointer("/reasoning/effort"), Some(&json!("xhigh")));
|
||||
assert_eq!(body.pointer("/reasoning_effort"), None);
|
||||
assert_eq!(body.pointer("/reasoningEffort"), None);
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"reasoning": {"effort": "high", "summary": "auto"}
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "grok-composer-2.5-fast", &client_key())
|
||||
.expect("Composer request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
assert_eq!(
|
||||
body.pointer("/reasoning"),
|
||||
Some(&json!({"effort":"high", "summary":"auto"}))
|
||||
);
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"store": null,
|
||||
"include": null,
|
||||
"reasoning": {"effort": "max"}
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "grok-composer-2.5-fast", &client_key())
|
||||
.expect("empty Composer reasoning");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
assert_eq!(body.pointer("/store"), Some(&json!(false)));
|
||||
assert_eq!(
|
||||
body.pointer("/include"),
|
||||
Some(&json!(["reasoning.encrypted_content"]))
|
||||
);
|
||||
assert_eq!(body.pointer("/reasoning"), Some(&json!({"effort":"max"})));
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"reasoning": {}
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "grok-composer-2.5-fast", &client_key())
|
||||
.expect("Composer reasoning without effort");
|
||||
assert_eq!(
|
||||
Value::Object(encoded.body().clone()).pointer("/reasoning"),
|
||||
Some(&json!({}))
|
||||
);
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"reasoning": {"effort": "max"}
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "tenant/foo/grok-4.6", &client_key())
|
||||
.expect("unknown provider prefix");
|
||||
assert_eq!(
|
||||
Value::Object(encoded.body().clone()).pointer("/reasoning/effort"),
|
||||
Some(&json!("max"))
|
||||
);
|
||||
assert_eq!(encoded.body()["store"], json!(false));
|
||||
assert_eq!(encoded.body()["include"], expected_include);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_strip_codex_only_fields_without_model_guesses() {
|
||||
fn encoder_should_strip_codex_only_fields_and_preserve_business_extensions() {
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"prompt_cache_retention": "24h",
|
||||
"safety_identifier": "user-1",
|
||||
"presence_penalty": 0.1,
|
||||
"presencePenalty": 0.2,
|
||||
"frequency_penalty": 0.3,
|
||||
"frequencyPenalty": 0.4,
|
||||
"stop": ["done"],
|
||||
"provider_options": {"providers": {"xai": {"transport": "websocket"}}},
|
||||
"future_official_field": {"keep": true},
|
||||
"external_web_access": true,
|
||||
"metadata": {"external_web_access": false},
|
||||
"tools": [{
|
||||
@@ -299,6 +230,7 @@ fn encoder_should_strip_codex_only_fields_without_model_guesses() {
|
||||
for pointer in [
|
||||
"/prompt_cache_retention",
|
||||
"/safety_identifier",
|
||||
"/provider_options",
|
||||
"/external_web_access",
|
||||
"/metadata/external_web_access",
|
||||
"/tools/0/external_web_access",
|
||||
@@ -306,18 +238,10 @@ fn encoder_should_strip_codex_only_fields_without_model_guesses() {
|
||||
assert_eq!(body.pointer(pointer), None, "field survived at {pointer}");
|
||||
}
|
||||
|
||||
let request = raw_request(json!({
|
||||
"model": "client-model",
|
||||
"input": "hello",
|
||||
"logprobs": true,
|
||||
"top_logprobs": 5
|
||||
}));
|
||||
let encoded = GrokResponsesRequest::encode(&request, "grok-4.20-reasoning", &client_key())
|
||||
.expect("sanitized 4.20 request");
|
||||
let body = Value::Object(encoded.body().clone());
|
||||
assert_eq!(body.pointer("/model"), Some(&json!("grok-4.20-reasoning")));
|
||||
assert_eq!(body.pointer("/logprobs"), Some(&json!(true)));
|
||||
assert_eq!(body.pointer("/top_logprobs"), Some(&json!(5)));
|
||||
assert_eq!(
|
||||
body.pointer("/future_official_field"),
|
||||
Some(&json!({"keep": true}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -444,31 +368,6 @@ fn request_debug_should_not_expose_prompt_or_unknown_values() {
|
||||
assert!(!debug.contains("must-not-leak"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_should_remove_retired_gateway_provider_options_before_xai_conversion() {
|
||||
let request = raw_request(Value::Object(Map::from_iter([
|
||||
("model".to_owned(), json!("client")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
(
|
||||
"provider_options".to_owned(),
|
||||
json!({
|
||||
"version": "future-version",
|
||||
"providers": {"xai": {"transport": "websocket", "turn_index": "7"}}
|
||||
}),
|
||||
),
|
||||
("future_official_field".to_owned(), json!({"keep": true})),
|
||||
])));
|
||||
|
||||
let encoded =
|
||||
GrokResponsesRequest::encode(&request, "grok-routed", &client_key()).expect("request");
|
||||
|
||||
assert!(encoded.body().get("provider_options").is_none());
|
||||
assert_eq!(
|
||||
encoded.body().get("future_official_field"),
|
||||
Some(&json!({"keep": true}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_session_should_be_tenant_isolated_and_stable_across_turns() {
|
||||
let first = raw_request(json!({
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user