Compare commits

..
24 Commits
Author SHA1 Message Date
ZyyandCodex 81de0f5331 build(release): keep the plugin manifest inside distribution packages
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 03:00:01 +08:00
ZyyandCodex 69c1ee15bb docs(release): clarify same-version retries and simplify release notes
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 02:56:49 +08:00
ZyyandCodex ba87cfde6a chore(ci): remove the release script test suite
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 02:41:37 +08:00
ZyyandCodex d0a588fbc3 test(plugin): signal crashes after host registration
Drive each worker exit from the prepared generation so scheduling delays cannot turn a post-registration crash into a registration failure.

Co-authored-by: Codex <noreply@openai.com>
2026-11-07 02:28:00 +08:00
ZyyandCodex 7cee430947 chore(release): v3.42.0
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 01:59:00 +08:00
ZyyandCodex ac98a764ff fix(update): discover channels from verified release assets
Publish an immutable channel snapshot with each release and advance a fixed release pointer only after public assets pass verification. Check the exact platform package before notifying clients.

Co-authored-by: Codex <noreply@openai.com>
2026-11-07 01:59:00 +08:00
ZyyandCodex 56ef1bffa2 test(openai): align service tier fixture with OAuth normalization
Use client_metadata to verify nested service tier fields survive fast-mode handling while preserving outbound tier and billing assertions.

Co-authored-by: Codex <noreply@openai.com>
2026-11-07 01:46:54 +08:00
ZyyandCodex e275a6dada chore(release): v3.42.0
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 01:21:29 +08:00
555269739c fix(openai): normalize unsupported Codex OAuth request options (#630)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 01:16:25 +08:00
ZyyandCodex d03d8408fc docs: clarify current configuration and update behavior
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 00:55:33 +08:00
ZyyandCodex 228eea90c1 chore(db): consolidate unreleased migrations
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 00:48:32 +08:00
ZyyandCodex 0d25061ef0 docs: bound UI verification and generalize AI attribution
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 00:37:46 +08:00
ZyyandZyy e97b1faf0d feat(openai): derive installation identities from key seeds (#629)
* feat(openai): derive installation identities from key seeds

* docs: explain installation seed migration effects

* docs: fix the key configuration reference

---------

Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-07 00:33:17 +08:00
ZyyandZyy 90ecac978a feat(accounts): make quota credit use opt-in per account (#628)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-07 00:20:43 +08:00
ZyyandZyy 4326eb1e4b fix(update): bound GitHub lookups with a published channel index (#627)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-07 00:20:19 +08:00
ZyyandCodex 2a8d5d8643 ci(release): publish the channel index after verifying release assets
Co-authored-by: Codex <noreply@openai.com>
2026-11-07 00:13:47 +08:00
ZyyandCodex b7fa5872a3 refactor(ui): reuse shared button components
Co-authored-by: Codex <noreply@openai.com>
2026-11-06 23:41:36 +08:00
ZyyandCodex e907f419ab fix(ui): show account group names in popovers
Co-authored-by: Codex <noreply@openai.com>
2026-11-06 23:20:13 +08:00
ZyyandZyy ee7e837d1f feat(scheduling): split queue timeouts and allow account fallback (#625)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-06 23:09:29 +08:00
ZyyandCodex 1240f22580 fix(openai): honor affinity mode for sessionless requests
Use the configured affinity mode for content and cache-key bindings, including strict waiting and binding store failures. Cover queue rejection, cancellation, migration, and preferred spillover, and document the scheduling boundaries.

Refs #623

Co-authored-by: Codex <noreply@openai.com>
2026-11-06 22:05:15 +08:00
ZyyandCodex e390417821 fix(openai): prefer the same account for sessionless requests
Use the client cache key or the opening messages as a tenant-scoped scheduling preference. Keep explicit sessions and continuation ownership separate, and allow spillover when the preferred account is busy.

Refs #622

Co-authored-by: Codex <noreply@openai.com>
2026-11-06 20:01:01 +08:00
ZyyandCodex e3f1299a0c fix(ui): retain custom model choices until save
Keep deselected model options in the editing session and remove the nested model list scrollbar.

Co-authored-by: Codex <noreply@openai.com>
2026-11-06 15:53:42 +08:00
Zyy 4e4cd96478 fix(openai): recheck cross-account capacity while queued 2026-11-06 15:02:16 +08:00
ZyyandZyy 6de1beb1df perf(openai): reduce scheduling reads and stabilize session affinity (#618)
Co-authored-by: Zyy <zyycn@users.noreply.github.com>
2026-11-06 14:44:42 +08:00
184 changed files with 6570 additions and 2297 deletions
+3 -2
View File
@@ -20,7 +20,7 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
| 领域或操作,行为影响与文件路径满足其一即可 | 确定方案前读取与执行 |
| --- | --- |
| 调整页面、弹窗、表单、按钮、文案、样式或交互;目标在 `frontend/`、`modules/ui/`、插件前端,或是 `.vue` 文件 | [前端流程](references/frontend.md),覆盖组件复用、文案、Token、样式、视觉状态和运行证据;局部修改同样执行 |
| 调整页面、弹窗、表单、按钮、文案、样式或交互;目标在 `frontend/`、`modules/ui/`、插件前端,或是 `.vue` 文件 | [前端流程](references/frontend.md),局部修改同样按影响范围检查组件复用、文案、Token、样式、视觉状态和运行证据 |
| 开发、排查或审查插件接入、宿主 SDK、Runtime、扩展接口或独立网关插件;目标在 `backend/crates/gateway-plugin/`、`modules/plugins/` | 完整读取[插件职责边界](references/plugin-boundaries.md)及所用 [SDK](../../../backend/crates/gateway-plugin/sdk/README.md) 类型与能力合同;宿主维护自身合同、资源与稳定性,插件业务错误在插件侧修复,不在宿主、SDK 或 Runtime 中补偿;独立插件开发、排查、审查另用 [cpr-plugin-dev](../cpr-plugin-dev/SKILL.md) |
| 开发、排查或审查服务端接口、路由、调度、账号与用量规则、持久化、后台任务或协议行为;目标是 `backend/` 中的实现、配置、迁移,或 Rust 源码 | [Workspace 边界](../../../docs/architecture.md#3-workspace-边界)的模块与依赖规则、所属模块及[后端自审](../../../docs/architecture.md#后端自审);Rust 使用 `$rust-best-practices` 和下方后端注释要求,宿主 Rust 同时读取 [Rust 模块组织约定](../../../docs/architecture.md#32-rust-模块组织约定) |
| 修改页面接口调用,或开发、排查、审查面向管理端或模型客户端的 HTTP/WS API 合同 | [API](../../../docs/api.md)对应章节;优先复用已有接口,核对调用方 → 服务端解析 → 实际查询或执行,不能只凭前端类型认定参数生效;按实际改动叠加前后端规范 |
@@ -45,8 +45,9 @@ description: Codex Proxy RS 开发、修复、排障、代码审查和文档修
1. 遵守已提供的 [AGENTS.md](../../../AGENTS.md),按其 CodeGraph 约定定位代码;确认工作区、任务范围与已有改动,找到事实归属、同类实现和调用链,补齐上表命中规范及[项目约定](../../../CONTRIBUTING.md#项目约定)对应条目后再修改
2. 以最终差异重新匹配上表,补齐新增领域的规范并检查实际实现,再处理本次引入的职责越界、重复规则、冗余状态与无依据分支;删除或替换功能时一并清理失效的配置、类型、调用、测试与文档,已部署数据和公开合同仍遵守所属迁移及兼容规则;仅审查时报告,不自行修复
3. 按最终行为执行[文档检查](references/documentation.md),核对所属正文及相关概述、示例和引用,在本任务内完成必要同步;不以尚无文档差异跳过检查。内部重构等未改变文档所述事实时无需改文档;常规文档只写当前状态,普通开发不自动修改 `release/notes.md`
4. 按变更范围执行[验证](../../../CONTRIBUTING.md#验证)及其停止条件,后端命令见[验证命令](../../../docs/architecture.md#验证命令);复用仍适用的验证证据与用户已确定的验收分工,区分通过、失败、跳过与未执行;提交前确认已覆盖[提交约定](../../../CONTRIBUTING.md#项目约定)
4. 按变更范围执行[验证](../../../CONTRIBUTING.md#验证)及其停止条件,后端命令见[验证命令](../../../docs/architecture.md#验证命令);复用仍适用的验证证据与用户已确定的验收分工,区分通过、失败、跳过与未执行
5. 交付前按[首要写作要求](#首要写作要求言简意赅)通读本次新增或修改的文字。交代实际结果与验证缺口;前端按[前端流程](references/frontend.md)核对运行证据,后端按上表检查生命周期,不把构建通过当作行为验收
6. 按授权完成提交与交付;提交前按[提交约定](../../../CONTRIBUTING.md#项目约定)准备提交信息与实际共同作者署名,提交后核对实际提交消息,PR 的 AI 披露不能替代提交署名
## 测试维护
@@ -4,7 +4,7 @@
## 领域必读
确定页面方案或审查结论前,读取[前端职责](../../../../docs/architecture.md#34-前端模块职责)、[文案与信息层级](../../../../docs/theme.md#界面文案与信息层级)和[界面验证](../../../../CONTRIBUTING.md#界面验证)的完整章节;只改文案或局部样式也执行。明确本次用户操作、原页面、最接近的同类实现及可复用基础组件,沿用既有设计与请求层,按本轮验收分工留存所需截图
确定页面方案或审查结论前,读取[前端职责](../../../../docs/architecture.md#34-前端模块职责)、[文案与信息层级](../../../../docs/theme.md#界面文案与信息层级)和[界面验证](../../../../CONTRIBUTING.md#界面验证)的完整章节,局部修改同样读取。明确本次用户操作、原页面、最接近的同类实现及可复用基础组件,沿用既有设计与请求层,按本轮验收分工留存所需截图
## 按改动补齐规范
@@ -21,9 +21,15 @@
普通维护和新增业务页面沿用现有设计;设计技能中的配色、字体、布局探索不覆盖项目规范,主题算法只在涉及时读取
## 验收范围与收尾
验证前按[界面验证](../../../../CONTRIBUTING.md#界面验证)明确本次受影响的用户路径、状态和通过条件。局部修改围绕受影响区域及相关操作验收;主题、窗口尺寸和空/加载/错误状态仅在受影响时补充,共用主题或基础组件变更按主题规范扩大范围
通过后按[验证停止条件](../../../../CONTRIBUTING.md#验证)收尾;只有新改动、失败或未解疑点时补验相关部分。补交截图或 PR 材料时复用仍有效的运行证据
## 交付前
对照最终差异和实际页面逐项核对,不以读取过规范代替检查:
对照最终差异和实际页面核对适用项,不以读取过规范代替检查:
- **复用与范围**:是否沿用参考页面和基础组件,已有组件能承载时不手写替代控件;共用缺陷按职责修复,修复后清理本次已失效的页面补丁,不顺带重排无关区域或新增重复入口
- **文案与层级**:标点是否合规,标题、状态和帮助是否重复,详细说明与当前决策必需提示的位置是否合适
+2 -1
View File
@@ -19,7 +19,8 @@ description: Codex Proxy RS 的版本规划、发布说明、正式或预发行
- Codex 兼容性变化按[问题与方案依据](../../../CONTRIBUTING.md#问题与方案依据)核对已有依据与验证范围,缺少依据列为发行风险,不以 CI 通过代替
- `release/publish --check <version>` 只读检查发行准备条件;不带 `--check` 会提交、打 tag、推送并触发工作流,实际发布仍需相应授权
- 同名版本、超时或中断先查本地与远端状态,不删已发布 tag、不移动 tag、不强推掩盖冲突
- 同版本失败先区分[续跑原任务与修复源码后重发](references/operations.md#恢复与验收):前者沿用原提交;后者仅在版本尚未公开时清理失败的版本提交,再生成唯一的版本提交。已发布的 tag 和产物保持不变
- 备版只更新源码的构建版本;通道快照是版本 Release 的 `version.yaml` 附件,验收时核对固定预发行页 `update-index` 指向的公开快照
- 预发行不得覆盖 GitHub Latest 或镜像 `latest`,完成后回读核对
- `release/notes.md` 只承载当前发行的说明;常规文档修改另按开发入口执行[文档检查](../cpr-dev-guide/references/documentation.md)
- 交付版本、提交、Release/工作流链接与实际结果,未完成构建或缺失产物时不称发布成功
@@ -27,7 +27,7 @@
应用内更新仅允许同一轮实验,规则见[版本命名与升级规则](../../../../deploy/README.md#版本命名与升级规则)。
发版前核对上一实验版是否已包含该更新器;仍禁用在线更新的旧构建需要先手动升级。
- GitHub Release 标为 Pre-release,且不设为 Latest;Docker 发布版本标签,不覆盖稳定版 `latest`。
正式版和 alpha、beta、rc 均不接收实验版更新;环境变量不能改变发行线。
发布任务应将该基线的最高公开实验版写入版本 Release 通道快照的 `published[大版本].exp[基线]`,不推进普通通道。
- 复用正式版平台清单、质量检查、完整安装包与多架构镜像,以及部署文件、校验和和现有产物证明。
对照本次清单确认平台,不以“实验版”为由仅发布单架构镜像。
- 部署说明交代与稳定实例并存的目录、配置、端口和数据库要求;需要 Compose 隔离配置时,
@@ -20,7 +20,7 @@
调用入口前重新核对当前脚本的副作用、目标版本和待推送提交,保留无关工作区内容,不自动清理或暂存用户文件。
先编辑 `release/notes.md`,保留为未提交改动(已暂存或未暂存均可);其他已跟踪文件的改动应先单独提交。
脚本会更新 `release/version.yaml`,按[接口弃用规则](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)校验并维护 `plugin-api-deprecations.json`,将这些差异与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
脚本只改 `release/version.yaml` 的顶层 `version`,按[接口弃用规则](../../../../backend/crates/gateway-plugin/sdk/docs/manifest.md#接口弃用)校验并维护 `plugin-api-deprecations.json`,将这些差异与最终工作区中的发布说明一起提交为 `chore(release): vX.Y.Z`。
不要提前单独提交说明或把版本文件改成目标版本;脚本要求说明相对 HEAD 有改动,并自行更新版本号。
本任务需要的提交按[项目约定](../../../../CONTRIBUTING.md#项目约定)保留英文 Conventional Commits 与相应署名。
@@ -38,13 +38,28 @@ release/publish "$release_version"
## 恢复与验收
- 脚本中断或网络超时时,先核对本地提交与 tag、远端分支与 tag、对应工作流 run 及 Release,确定已完成到哪一步。
若本地已创建版本提交或 tag,不直接重跑入口,也不通过删 tag、回退版本或强推来消除报错。
- 提交和 tag 已推送但工作流未触发时,核对目标后使用脚本提示的 dispatch 命令补触发;
工作流已存在则跟踪或恢复对应 run,避免重复发布。发现 tag 指向不同提交时报告冲突,不移动已发布 tag。
- 跟踪与本次 tag 对应的 run ID,核对工作流解析的目标提交。触发工作流成功只表示已开始,不能当作发布完成。
- 构建与打包中间产物保留 7 天,在有效期内可重跑失败任务;产物过期后重跑完整工作流,不移动原 tag。
先核对本地与远端的分支头、版本提交、tag,以及对应的工作流、GitHub Release 和版本镜像。超时或任务失败不等于版本未公开;即使 Release 还是草稿,版本镜像也可能已经推送
### 续跑原任务
源码无需修改时保留版本提交和 tag,核对两者指向后从断点继续:
- 尚未推送:补齐缺失的步骤,按入口原子推送分支和 tag
- 已推送但未触发:使用入口提示的 dispatch 命令
- 已有 run:跟踪对应提交的任务,失败时重跑失败任务。构建产物保留 7 天,过期后从原 tag 重跑完整工作流
### 修复源码后同版本重发
1. 确认旧工作流已停止、该版本没有公开 Release 或版本镜像,且待清理提交不在其他已发布 tag 的历史中。已有公开产物时保留原提交和 tag,代码修复另发新版本;发布状态不明时先查清
2. 从失败版本提交的父提交重新接上后续功能和修复,跳过失败的 `chore(release): vX.Y.Z`。保留最终说明为未提交改动,版本号和插件弃用清单交给发行入口。核对差异,确保修复未丢失、失败提交已不在待发历史中
3. 只清理本次失败的 tag 和草稿。涉及远端改写时,先核对授权,以事先记录的分支头和 tag 对象 SHA 作显式租约,原子更新分支并删除未发行 tag。租约不匹配就停止,不合并旧线
4. 修复验证通过后执行 `release/publish --check <version>`,再按[执行发版](#执行发版)重发。核对上一发行至新 tag 的历史中只有一个本版本的发行提交,且与工作流构建的提交一致
### 验收
- 发布完成后核对 GitHub Release 的版本、正式版/预发行状态和中文变更正文;对照平台清单与工作流确认归档、校验和及镜像产物。
工作流附加的安装说明和自动提交列表按其当前配置处理,不用它们替代手写的中文变更说明。
- 回读固定预发行页 `update-index`(不是 GitHub Latest),按正文 JSON 的 `current` 下载对应 `v<current>` 的 `version.yaml`。核对快照版本及 `published`:正式版参与同一大版本四个普通通道的比较;RC 参与 RC、Beta、Alpha,Beta 参与 Beta、Alpha,Alpha 只参与自身。实验版只推进 `exp["X.Y.Z"]`。晚完成的发行不得回退较高候选或丢失其他发行线
- 只有指针写入失败时,在既有授权内执行 `gh workflow run release.yml --ref main -f tag="v$release_version" -f index_only=true`。恢复任务与发布共用锁,只核对已公开版本的附件和待发布快照摘要,不重新打包、改版本 tag 或写 main。完成后回读指针与附件;待发布记录不能当作成功。索引附件最多 128 KiB
- 返回版本、tag 对应提交、Release/工作流链接及实际结果。失败、仍在构建或缺失产物时明确说明,不能称为发布成功;
Release 和镜像已生成也不表示用户的运行实例已经升级。
@@ -5,7 +5,7 @@
## 基线与版本
1. 确认类型、分支、upstream、目标仓库、工作区与完整待发布范围,保留无关改动;未跟踪文件也需核对
2. 读取 [version.yaml](../../../../release/version.yaml)、相关 tag 和实际 Release,确定上一发布基线与本次目标提交,再核对二者间的最终 diff,不只看最近几条提交
2. 读取 [version.yaml](../../../../release/version.yaml)、相关 tag 和实际 Release,区分源码构建版本和 `update-index` 指向的发行附件中的 `published` 通道表,结合目标通道确定上一发布基线与本次目标提交,再核对二者间的最终 diff,不只看最近几条提交
3. 使用用户确定的目标版本,建议版本时说明兼容性、能力或修复依据;同名 tag 先区分重试与新发行,不递增版本掩盖冲突
4. 命名与晋级按[版本命名与升级规则](../../../../deploy/README.md#版本命名与升级规则),实验版另读[实验流程](experimental.md#分支与版本)
@@ -14,7 +14,9 @@
读取当前 [notes.md](../../../../release/notes.md)与[项目约定](../../../../CONTRIBUTING.md#项目约定)中的发布说明条目
- 只在发布说明任务中修改 `release/notes.md`,替换为本次版本内容,首行为目标 tag,正文使用中文
- 按用户可感知的最终变化归并,省略空章节,不逐条翻译提交,不写排查过程或未经验证的承诺
- 常规版本按「新增功能」「优化改进」「问题修复」「升级须知」归类,省略空章节;单一改动可直接写一句。安装命令和文档入口由工作流附加
- 每条用一句短语讲清一个用户可感知的结果,不加句号,例如「系统更新:确认当前平台有安装包后才提示升级」
- 同一变化只写一次,保留会影响使用或升级的默认值和限制,不展开实现过程、逐条翻译提交或写未经验证的承诺
- 剔除回退、重复及未进入发行范围的内容,升级事项须能追溯到实际迁移、配置或兼容变化
- 只翻译已有准确说明时保持事实与范围,不混入其他变更
- 常规文档仍只描述当前状态;必要修订执行[文档检查](../../cpr-dev-guide/references/documentation.md),不复制发布说明中的版本变化列表
@@ -26,6 +28,8 @@
进入实际发行准备后,读取目标分支的 [release/publish](../../../../release/publish)、[工作流](../../../../.github/workflows/release.yml)与[平台清单](../../../../release/platforms.yaml),按当前入口执行,不另写脚本或绕过检查
在上一发行基线到 HEAD 的提交中检查是否已有完整标题为 `chore(release): v<目标版本>` 的提交,即使同名 tag 已不存在也要查。命中时先核对该提交对应的发行状态,按[失败恢复](operations.md#恢复与验收)处理,不在旧提交上再叠一个同名提交
发布说明准备好、其他代码已提交后,执行 `release/publish --check <version>`,检查版本、说明、工作区、远端分支与 tag、工具链和插件兼容窗口。
预检不修改文件、Git 引用或远端状态;远端存在本地未知提交时先同步再检查。它不替代 CI、发行物验证或实际发布授权
+3 -4
View File
@@ -1,10 +1,10 @@
require 'digest'
require 'fileutils'
require 'json'
require_relative 'release-metadata'
# 归档与镜像消费相同二进制和封口清单,不在打包时重新解释插件分发策略
module ReleasePackage
APP_NAME = 'codex-proxy-rs'
APP_NAME = ReleaseMetadata::APP_NAME
MANIFEST = 'dist/plugin-release-assets/plugin-release-manifest.json'
def self.install_manifest(directory)
@@ -34,7 +34,6 @@ module ReleasePackage
default_image = "${CPR_IMAGE:-#{image}:latest}"
raise 'Expected exactly one default application image in deploy/compose.yaml' unless compose.scan(default_image).length == 1
File.write("#{release_dir}/compose.yaml", compose.sub(default_image, "${CPR_IMAGE:-#{image}:#{ENV.fetch('DOCKER_VERSION')}}"))
FileUtils.cp(MANIFEST, release_dir)
platforms.each do |platform|
raise "Unsupported archive format: #{platform['archive']}" unless platform['archive'] == 'tar.gz'
@@ -47,7 +46,7 @@ module ReleasePackage
FileUtils.cp_r('dist/frontend/.', "#{root}/web/dist")
FileUtils.cp(config, "#{root}/deploy")
install_manifest("#{root}/plugins/official")
archive = "#{release_dir}/#{APP_NAME}_#{ENV.fetch('VERSION')}_#{platform['os']}_#{platform['arch']}.tar.gz"
archive = File.join(release_dir, ReleaseMetadata.archive_name(ENV.fetch('VERSION'), platform))
raise "Failed to create #{archive}" unless system('tar', '-czf', archive, '-C', File.dirname(root), APP_NAME)
end
+170
View File
@@ -0,0 +1,170 @@
require 'base64'
require 'digest'
require 'uri'
require_relative 'release-metadata'
# 通道快照随版本封存,固定 Release 只记录已公开快照和可恢复的待发布摘要
module PublishedChannels
POINTER = 'update-index'
LIMIT = 128 * 1024
def self.advance(document, version)
version = ReleaseMetadata.version(version)
major, = version.split('.', 2)
core, stage = version.split('+', 2).first.split('-', 2)
channels = (document.fetch('published')[major] ||= {})
if stage&.start_with?('exp.')
channels = (channels['exp'] ||= {})
keys = [core]
else
stages = %w[stable rc beta alpha]
keys = stages.drop(stages.index(stage ? stage.split('.').first : 'stable'))
end
incoming = ReleaseMetadata.sort_key(version)
changed = false
keys.each do |key|
current = channels[key]
if current
comparison = incoming <=> ReleaseMetadata.sort_key(current)
raise "Conflicting release identities: #{current}, #{version}" if comparison.zero? && current != version
next unless comparison.positive?
end
channels[key] = version
changed = true
end
changed
end
class Publisher
def api(endpoint, method: 'GET', data: nil, optional: false)
args = ['gh', 'api', endpoint, '--method', method]
args.concat(%w[--input -]) if data
out, error, status = Open3.capture3(*args, stdin_data: data ? JSON.generate(data) : '')
return nil if optional && !status.success? && error.include?('(HTTP 404)')
raise "Cannot access #{endpoint}: #{error.strip}" unless status.success?
JSON.parse(out)
end
def initialize(repository)
@repository = repository
@base = "repos/#{repository}"
release = api("#{@base}/releases/tags/#{POINTER}", optional: true)
if release
raise 'Invalid update-index release' unless release.fetch('tag_name') == POINTER &&
release.fetch('prerelease') && !release.fetch('draft')
@id = release.fetch('id')
@state = JSON.parse(release.fetch('body'))
@state.fetch('current')
else
@state = { 'current' => nil }
end
end
def save(current:, pending: nil)
state = { 'current' => current }
state['pending'] = pending if pending
body = { body: JSON.generate(state), make_latest: 'false' }
if @id
api("#{@base}/releases/#{@id}", method: 'PATCH', data: body)
else
release = api("#{@base}/releases", method: 'POST', data: body.merge(
tag_name: POINTER, name: '更新通道索引', target_commitish: 'main', prerelease: true
))
@id = release.fetch('id')
end
@state = state
end
def release(version)
api("#{@base}/releases/tags/v#{ReleaseMetadata.version(version)}", optional: true)
end
def verify_assets(version, release, index:)
tag = "v#{version}"
raise "Release is not public or tag differs: #{tag}" unless release && release.fetch('tag_name') == tag &&
!release.fetch('draft') && release.fetch('prerelease') == ReleaseMetadata.prerelease?(version)
data = api("#{@base}/contents/release/platforms.yaml?ref=#{URI.encode_www_form_component(tag)}")
platforms = YAML.safe_load(Base64.decode64(data.fetch('content'))).fetch('platforms')
raise "Missing platforms for #{tag}" if platforms.empty?
expected = platforms.map { |p| ReleaseMetadata.archive_name(version, p) }
expected.concat(%w[checksums.txt compose.yaml config.example.yaml])
expected << 'version.yaml' if index
assets = release.fetch('assets').select { |a| a.fetch('state') == 'uploaded' && a.fetch('size').positive? }.to_h { |a| [a.fetch('name'), a] }
missing = expected - assets.keys
raise "Incomplete release #{tag}: #{missing.join(', ')}" unless missing.empty?
assets
end
def download(version)
content, error, status = Open3.capture3('gh', 'release', 'download', "v#{version}", '--repo', @repository,
'--pattern', 'version.yaml', '--output', '-')
raise "Cannot download v#{version} snapshot: #{error.strip}" unless status.success?
content
end
def read_snapshot(version, asset)
raise 'Invalid version.yaml size' unless (1..LIMIT).cover?(asset.fetch('size'))
content = download(version)
digest = Digest::SHA256.hexdigest(content)
raise "Snapshot size or digest differs for v#{version}" unless content.bytesize == asset.fetch('size') && asset.fetch('digest') == "sha256:#{digest}"
doc = YAML.safe_load(content)
raise 'Invalid channel snapshot' unless doc.fetch('version') == version && doc.fetch('published').is_a?(Hash)
[doc, digest]
end
def previous
if (current = @state.fetch('current'))
assets = verify_assets(current, release(current), index: true)
return read_snapshot(current, assets.fetch('version.yaml')).first
end
# 首次接入时从实际公开的稳定版引导;没有历史 Release 的仓库从空表开始
doc = { 'published' => {} }
latest = api("#{@base}/releases/latest", optional: true)
if latest
version = ReleaseMetadata.version(latest.fetch('tag_name'))
verify_assets(version, latest, index: false)
PublishedChannels.advance(doc, version)
end
doc
end
# 调用方持有同一仓库的发布锁;快照必须在取锁后生成,不能在并行构建中预先合并
def prepare(version, directory)
if (pending = @state.dig('pending', 'version')) && (old = release(pending)) && !old.fetch('draft')
finish(pending)
end
existing = release(version)
raise "Release v#{version} already public; use index-only recovery" if existing && !existing.fetch('draft')
doc = previous
doc['version'] = version
PublishedChannels.advance(doc, version)
content = YAML.dump(doc)
raise 'Published index exceeds the client size limit' if content.bytesize > LIMIT
digest = Digest::SHA256.hexdigest(content)
File.write(File.join(directory, 'version.yaml'), content)
File.open(File.join(directory, 'checksums.txt'), 'a') { |file| file.puts "#{digest} version.yaml" }
save(current: @state.fetch('current'), pending: { 'version' => version, 'sha256' => digest })
end
def finish(version)
assets = verify_assets(version, release(version), index: true)
_, digest = read_snapshot(version, assets.fetch('version.yaml'))
return if @state.fetch('current') == version
unless @state['pending'] == { 'version' => version, 'sha256' => digest }
raise "v#{version} is not the reserved snapshot; refusing to overwrite another publication"
end
save(current: version)
end
end
end
if $PROGRAM_NAME == __FILE__
mode, raw = ARGV
version = ReleaseMetadata.version(raw || '')
publisher = PublishedChannels::Publisher.new(ENV.fetch('GITHUB_REPOSITORY'))
case mode
when 'prepare' then publisher.prepare(version, 'dist/release')
when 'finish' then publisher.finish(version)
else abort 'usage: published-channels.rb prepare|finish VERSION'
end
end
-193
View File
@@ -1,193 +0,0 @@
require 'fileutils'
require 'json'
require 'open3'
require 'tmpdir'
require_relative 'release-quality'
# 使用临时 Git 仓库验证远端状态和只读边界,不访问真实 GitHub 或修改当前仓库
ROOT = File.expand_path('../..', __dir__)
def assert(condition, message)
raise message unless condition
end
def git(*arguments)
output, error, status = Open3.capture3('git', *arguments)
raise "git #{arguments.first}: #{error}" unless status.success?
output.strip
end
def commit(message)
git('add', '.')
git('commit', '-qm', message)
end
Dir.mktmpdir('release-checks-') do |directory|
remote = File.join(directory, 'remote.git')
system('git', 'init', '--bare', '-q', remote, exception: true)
checkout = File.join(directory, 'checkout')
FileUtils.mkdir_p(checkout)
Dir.chdir(checkout) do
git('init', '-q', '-b', 'main')
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Release test')
files = %w[
release/publish release/version.yaml release/notes.md release/platforms.yaml
.github/scripts/toolchains.rb .github/scripts/release-metadata.rb
backend/rust-toolchain.toml deploy/Dockerfile
backend/crates/gateway-plugin/runtime/plugin-api-lifecycle.rb
backend/crates/gateway-plugin/runtime/plugin-host-compatibility.json
backend/crates/gateway-plugin/runtime/plugin-api-deprecations.json
]
files.each do |path|
FileUtils.mkdir_p(File.dirname(path))
FileUtils.cp(File.join(ROOT, path), path)
end
commit('initial source')
assert(ReleaseQuality.evidence('owner/repo').nil?, 'root commit should require full checks')
git('remote', 'add', 'origin', remote)
git('push', '-qu', 'origin', 'main')
File.write('release/notes.md', "# v99.0.0\n\nFixture notes\n")
bin = File.join(directory, 'bin')
FileUtils.mkdir_p(bin)
File.write(File.join(bin, 'gh'), <<~RUBY)
#!/usr/bin/env ruby
case ARGV
when ['auth', 'status'] then exit 0
when ['repo', 'view', #{remote.inspect}, '--json', 'nameWithOwner', '--jq', '.nameWithOwner']
puts 'owner/repo'
else
abort "unexpected gh mutation: \#{ARGV.inspect}"
end
RUBY
File.chmod(0o755, File.join(bin, 'gh'))
environment = { 'PATH' => "#{bin}:#{ENV.fetch('PATH')}" }
snapshot = lambda do
[git('show-ref'), git('status', '--porcelain'), File.binread('.git/index'),
files.to_h { |path| [path, File.binread(path)] },
git('ls-remote', 'origin'), File.exist?('.git/FETCH_HEAD')]
end
check = lambda do |success, message = nil|
before = snapshot.call
output, error, status = Open3.capture3(environment, 'bash', 'release/publish', '--check', '99.0.0')
assert(status.success? == success, "preflight outcome: #{output}#{error}")
assert("#{output}#{error}".include?(message), "missing error: #{message}; #{error}") if message
assert(snapshot.call == before, 'preflight modified files, refs, index, or FETCH_HEAD')
end
check.call(true, 'Preflight passed')
File.write('deploy/Dockerfile', File.read('deploy/Dockerfile') + "\n# fixture\n")
check.call(false, 'unstaged changes')
git('checkout', '--', 'deploy/Dockerfile')
git('tag', 'v99.0.0')
check.call(false, 'local tag already exists')
git('push', '-q', 'origin', 'refs/tags/v99.0.0')
git('tag', '-d', 'v99.0.0')
check.call(false, 'remote tag already exists')
git('push', '-q', 'origin', ':refs/tags/v99.0.0')
# 未履行的兼容窗口必须在只读预检中失败,不能等待提交版本后再发现
deprecations = 'backend/crates/gateway-plugin/runtime/plugin-api-deprecations.json'
previous = File.read(deprecations)
File.write(deprecations, JSON.generate([{
capability: 'removed-fixture', version: 1, replacement_version: 2, remaining_releases: 7
}]))
git('add', deprecations)
git('commit', '-qm', 'invalid lifecycle fixture')
check.call(false, 'replacement v2 must be available')
File.write(deprecations, previous)
commit('restore lifecycle and notes')
File.write('release/notes.md', "# v99.0.0\n\nUpdated fixture notes\n")
# 远端前进但没有本地对象时,提示同步而不隐式 fetch
other = File.join(directory, 'other')
git('clone', '-q', '-b', 'main', remote, other)
Dir.chdir(other) do
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Release test')
File.write('remote-only', 'change')
commit('remote advance')
git('push', '-q')
end
check.call(false, 'unseen commits')
git('fetch', '-q', 'origin')
check.call(false, 'diverged')
FileUtils.mv(remote, "#{remote}.hidden")
# 读取远端失败不能当作 tag 不存在,仍须保持本地不变
before = [git('show-ref'), git('status', '--porcelain'), File.binread('.git/index')]
_, error, status = Open3.capture3(environment, 'bash', 'release/publish', '--check', '99.0.0')
assert(!status.success? && error.include?('failed to read remote refs'), 'remote failure was ignored')
assert(before == [git('show-ref'), git('status', '--porcelain'), File.binread('.git/index')], 'remote failure changed local state')
FileUtils.mv("#{remote}.hidden", remote)
File.write('release/version.yaml', "version: \"99.0.0\"\n")
commit('release metadata')
parent = git('rev-parse', 'HEAD^')
run = { 'head_sha' => parent, 'conclusion' => 'success', 'head_repository' => { 'full_name' => 'owner/repo' }, 'id' => 42 }
job = { 'name' => 'quality / backend', 'conclusion' => 'success', 'html_url' => 'https://example.invalid/42' }
runs = [run]
jobs = [job]
original_capture = ReleaseQuality.method(:capture)
ReleaseQuality.define_singleton_method(:capture) do |*arguments|
if arguments.first == 'gh'
endpoint = arguments.find { |argument| argument.start_with?('repos/') }
if endpoint.include?('/workflows/ci.yml/runs?')
assert(endpoint.include?("event=push&branch=main&head_sha=#{parent}"), 'wrong CI evidence scope')
JSON.generate('workflow_runs' => runs)
else
assert(endpoint == 'repos/owner/repo/actions/runs/42/jobs?per_page=100', 'wrong run jobs')
jobs.map { |entry| JSON.generate(entry) }.join("\n")
end
else
original_capture.call(*arguments)
end
end
assert(ReleaseQuality.evidence('owner/repo') == job['html_url'], 'valid CI evidence not reused')
%w[skipped failure cancelled].each do |conclusion|
job['conclusion'] = conclusion
assert(ReleaseQuality.evidence('owner/repo').nil?, "reused #{conclusion} backend")
end
job['conclusion'] = 'success'
jobs = []
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused missing backend job')
jobs = [job]
run['conclusion'] = 'failure'
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused failed CI run')
run['conclusion'] = nil
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused unfinished CI run')
run['conclusion'] = 'success'
run['head_repository']['full_name'] = 'fork/repo'
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused foreign repository CI')
run['head_repository']['full_name'] = 'owner/repo'
runs = []
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused absent CI run')
runs = [run]
output, _, status = Open3.capture3(environment.merge('GITHUB_REPOSITORY' => 'owner/repo'), 'ruby', File.join(ROOT, '.github/scripts/release-quality.rb'))
assert(status.success? && output.strip == 'run_backend=true', 'API failure did not fall back to full checks')
File.write('backend/source.rs', '// changed source')
commit('source changed')
assert(ReleaseQuality.evidence('owner/repo').nil?, 'reused CI across source changes')
merge = git('commit-tree', 'HEAD^{tree}', '-p', 'HEAD', '-p', 'HEAD^', '-m', 'merge fixture')
git('update-ref', 'HEAD', merge)
assert(ReleaseQuality.evidence('owner/repo').nil?, 'merge commit should require full checks')
# 在同一个隔离仓库走实际提交与原子推送路径,dispatch 仍由本地替身接收
git('merge', '--no-edit', 'origin/main')
File.write('release/notes.md', "# v100.0.0\n\nPublish fixture notes\n")
dispatch = File.join(directory, 'dispatch.json')
mock = File.join(bin, 'gh')
File.write(mock, File.read(mock).sub("else\n", <<~RUBY))
when ['workflow', 'run', 'release.yml', '--repo', 'owner/repo', '--ref', 'main', '--raw-field', 'tag=v100.0.0']
File.write(#{dispatch.inspect}, ARGV.join(' '))
else
RUBY
output, error, status = Open3.capture3(environment, 'bash', 'release/publish', '100.0.0')
assert(status.success?, "publish failed: #{output}#{error}")
assert(File.exist?(dispatch), 'workflow was not dispatched')
assert(git('log', '-1', '--format=%s') == 'chore(release): v100.0.0', 'wrong release commit')
assert(git('status', '--porcelain').empty?, 'release left tracked changes')
assert(git('rev-parse', 'HEAD') == git('rev-parse', 'v100.0.0^{commit}'), 'tag differs from release commit')
assert(git('ls-remote', 'origin', 'refs/tags/v100.0.0').split.first == git('rev-parse', 'v100.0.0'), 'tag not pushed')
end
end
puts 'Release preflight and CI evidence checks passed'
+4
View File
@@ -62,6 +62,10 @@ module ReleaseMetadata
entries
end
def self.archive_name(version, platform)
"#{APP_NAME}_#{version}_#{platform.fetch('os')}_#{platform.fetch('arch')}.#{platform.fetch('archive')}"
end
def self.outputs(tag)
value = version(YAML.load_file('release/version.yaml').fetch('version'))
raise "Release tag #{tag} does not match v#{value}" unless tag == "v#{value}"
-3
View File
@@ -259,6 +259,3 @@ jobs:
for script in .github/scripts/*.rb; do ruby -c "$script"; done
ruby .github/scripts/toolchains.rb
ruby -c backend/crates/gateway-plugin/runtime/plugin-api-lifecycle.rb
- name: Verify release preflight and CI evidence
run: ruby .github/scripts/release-checks-test.rb
+40 -21
View File
@@ -7,6 +7,10 @@ on:
description: 'Tag to release, for example v0.1.0'
required: true
type: string
index_only:
description: 'Recover the index for an already public release without rebuilding'
type: boolean
default: false
permissions:
contents: read
@@ -21,6 +25,7 @@ env:
jobs:
release-info:
if: ${{ !inputs.index_only }}
runs-on: ubuntu-26.04
timeout-minutes: 10
permissions:
@@ -239,8 +244,6 @@ jobs:
timeout-minutes: 20
permissions:
contents: read
id-token: write
attestations: write
steps:
- name: Checkout
@@ -274,11 +277,6 @@ jobs:
PLATFORM_MATRIX: ${{ needs.release-info.outputs.platform_matrix }}
run: ruby .github/scripts/package-release.rb archives
- name: Generate release asset provenance
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-path: dist/release/*
- name: Upload release assets
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
@@ -510,24 +508,12 @@ jobs:
shell: bash
env:
INSTALLATION_NOTES: |
## 📥 安装
## Linux 一键安装(Docker Compose)
**Docker 部署:**
```bash
# 从 GitHub 容器镜像仓库拉取
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.release-info.outputs.docker_version }}
curl -fsSLO https://raw.githubusercontent.com/${{ github.repository }}/${{ needs.release-info.outputs.tag }}/deploy/install.sh && CPR_RELEASE_TAG=${{ needs.release-info.outputs.tag }} bash install.sh
```
请从本版本附件下载 `compose.yaml` 和 `config.example.yaml`。Compose 默认固定到本版本镜像;升级前请对比模板与已有配置。
**手动下载:**
请从下方附件中下载适合当前平台的归档包。
附件包含与当前宿主版本及提交绑定的 `plugin-release-manifest.json`,当前没有默认内置插件。
教学示例由独立的 `codex-proxy-plugins` 仓库维护和发布,不随宿主发行物分发。
发布归档提供 GitHub 构建产物证明。Docker 镜像附带软件物料清单(SBOM)、构建来源证明和 cosign 无密钥签名。
## 📚 文档
- [GitHub 仓库](https://github.com/${{ github.repository }})
@@ -540,6 +526,17 @@ jobs:
File.write(output, "#{notes}\n\n#{installation}\n")
RUBY
- name: Prepare versioned channel snapshot under the publication lock
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ needs.release-info.outputs.version }}
run: ruby .github/scripts/published-channels.rb prepare "$VERSION"
- name: Generate release asset provenance
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-path: dist/release/*
- name: Upload GitHub Release assets
id: release
uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
@@ -563,3 +560,25 @@ jobs:
DIGEST: ${{ steps.manifest.outputs.digest }}
RELEASE_ID: ${{ steps.release.outputs.id }}
run: ruby .github/scripts/publish-release.rb
- name: Advance update-index to the verified snapshot
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ needs.release-info.outputs.version }}
run: ruby .github/scripts/published-channels.rb finish "$VERSION"
recover-index:
if: inputs.index_only
runs-on: ubuntu-26.04
concurrency:
group: release-publication
queue: max
permissions:
contents: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Verify public assets and finish pending publication
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ inputs.tag }}
run: ruby .github/scripts/published-channels.rb finish "$TAG"
+2 -1
View File
@@ -126,7 +126,8 @@ PR 不接受以下协作规则文件的新增、修改、删除或重命名:
- AI 进行 Rust 开发时,加载并使用 `$rust-best-practices` 技能。模块、可见性、依赖方向和测试目录按架构文档执行,使用现有 Rustfmt 与 Clippy 配置
- Vue 使用 TypeScript、`<script setup>` 和现有 ESLint 配置;API 请求通过已有模块及 `request` 封装,沿用逐行请求配置格式
- 业务数据的解析、校验和展示优先复用已有逻辑。新增抽象放在拥有该职责的模块,避免重复事实源或无实际需求的通用层
- 新增代码注释使用中文。提交信息使用英文 Conventional Commits,参考近期历史,按完整功能归并;确由 Codex 参与的提交保留 `Co-authored-by: Codex <noreply@openai.com>`
- 新增代码注释使用中文。提交信息使用英文 Conventional Commits,参考近期历史,按完整功能归并
- AI 参与的提交按实际助手添加 `Co-authored-by: 名称 <邮箱>`,名称与邮箱采用该助手已知的署名身份,并保留其他实际共同作者署名;PR 的 AI 使用披露不替代提交署名
- 发布说明写入 `release/notes.md`,标题首行为目标版本的 `# vX.Y.Z`,章节标题和正文使用中文;命令、配置键、协议名等保留原文。只说明本次版本的实际变化,按影响交代升级事项
- PR 聚焦目标;无关格式化、重构和依赖升级单独处理
+1
View File
@@ -1834,6 +1834,7 @@ dependencies = [
"tracing-appender",
"tracing-subscriber",
"wiremock",
"yaml-rust2",
]
[[package]]
@@ -21,6 +21,7 @@ pub use gateway_core::account::{
/// 账号可用的管理操作
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct ProviderAccountCapabilities {
pub quota_credits: bool,
pub quota: bool,
pub quota_refresh: bool,
pub profile: bool,
@@ -110,6 +111,7 @@ pub enum AccountGroupFilter {
/// 账号公共存储投影;Provider 专属字段不进入此结构
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AccountRecord {
pub auto_use_quota_credits: bool,
pub id: String,
pub provider_kind: ProviderKind,
pub groups: Vec<AccountGroupRef>,
@@ -249,6 +251,7 @@ pub struct AccountUpdateResult {
/// 仅修改显式字段的批量账号设置命令
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BatchUpdateAccounts {
pub auto_use_quota_credits: Option<bool>,
pub account_ids: Vec<String>,
pub enabled: Option<bool>,
pub concurrency_limit: Option<Option<AccountConcurrencyLimit>>,
@@ -53,7 +53,9 @@ pub struct RuntimeSettingsValues {
pub max_concurrent_per_account: u32,
pub max_waiting_per_key: u32,
pub max_waiting_per_account: u32,
pub concurrency_wait_timeout_seconds: u32,
pub key_queue_timeout_seconds: u32,
pub account_queue_timeout_seconds: u32,
pub switch_account_on_queue_failure: bool,
pub max_retry_after_seconds: u32,
pub openai_guardian_reserved_concurrency: u32,
pub openai_account_affinity: gateway_core::account::AccountAffinity,
@@ -201,7 +203,8 @@ impl RuntimeSettingsValues {
gateway_core::settings::validate_request_limits(
self.max_waiting_per_key,
self.max_waiting_per_account,
self.concurrency_wait_timeout_seconds,
self.key_queue_timeout_seconds,
self.account_queue_timeout_seconds,
self.max_account_rotations,
self.openai_session_affinity_ttl_hours,
)?;
@@ -3,6 +3,14 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
/// 摘要只读取发布索引,完整详情可按请求刷新
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SystemUpdateCheck {
Summary,
Detail,
Refresh,
}
/// 本次检查或安装选择的更新稳定性范围,具体版本准入由 Host 判定
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
@@ -69,6 +77,7 @@ pub struct SystemUpdateDetail {
pub update_supported: bool,
pub unsupported_reason: Option<String>,
pub warning: Option<String>,
pub retry_at: Option<DateTime<Utc>>,
}
/// 系统操作类型
@@ -3,12 +3,13 @@
use std::{pin::Pin, sync::Arc};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures::Stream;
use crate::model::Revision;
use crate::model::system::{
SystemOperationAccepted, SystemUpdateChannel, SystemUpdateDetail, SystemUpdateEvent,
SystemUpdateStatus, SystemVersion,
SystemOperationAccepted, SystemUpdateChannel, SystemUpdateCheck, SystemUpdateDetail,
SystemUpdateEvent, SystemUpdateStatus, SystemVersion,
};
/// Host 已校验并解包的同一更新候选;字节只含发行清单,不含运行配置或凭据
@@ -43,6 +44,7 @@ pub enum SystemOperationErrorKind {
pub struct SystemOperationError {
kind: SystemOperationErrorKind,
message: String,
retry_at: Option<DateTime<Utc>>,
source: Option<gateway_core::error::ErrorSource>,
sensitive_context_redacted: bool,
}
@@ -53,11 +55,23 @@ impl SystemOperationError {
Self {
kind,
message: message.into(),
retry_at: None,
source: None,
sensitive_context_redacted: false,
}
}
#[must_use]
pub fn with_retry_at(mut self, retry_at: DateTime<Utc>) -> Self {
self.retry_at = Some(retry_at);
self
}
#[must_use]
pub const fn retry_at(&self) -> Option<DateTime<Utc>> {
self.retry_at
}
#[must_use]
pub fn with_source(mut self, source: impl Into<gateway_core::error::ErrorSource>) -> Self {
self.source = Some(source.into());
@@ -135,7 +149,7 @@ pub trait SystemOperations: Send + Sync {
async fn update_detail(
&self,
refresh: bool,
check: SystemUpdateCheck,
channel: Option<SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError>;
@@ -759,6 +759,7 @@ impl AccountsService for DefaultAccountsService {
self.accounts
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
account_ids: vec![account_id.to_string()],
enabled: Some(true),
concurrency_limit: None,
@@ -837,15 +838,28 @@ impl AccountsService for DefaultAccountsService {
(
std::sync::Arc<dyn crate::ports::provider::ProviderAdmin>,
Vec<ProviderAccountId>,
Vec<ProviderAccountId>,
),
>::new();
for account_id in &account_ids {
let (item, provider) = self.provider_for_account(account_id).await?;
providers
if command.auto_use_quota_credits.is_some()
&& !provider
.account_capabilities(account_id, &item.account.authentication_kind)
.quota_credits
{
return Err(AdminError::invalid("该账号不支持使用额度点数"));
}
let (_, selected, exhausted) = providers
.entry(item.account.provider_kind)
.or_insert_with(|| (provider, Vec::new()))
.1
.push(account_id.clone());
.or_insert_with(|| (provider, Vec::new(), Vec::new()));
selected.push(account_id.clone());
if command.auto_use_quota_credits == Some(true)
&& !item.account.auto_use_quota_credits
&& item.account.quota.is_exhausted()
{
exhausted.push(account_id.clone());
}
}
let enabled = command.enabled;
let result = self
@@ -853,7 +867,7 @@ impl AccountsService for DefaultAccountsService {
.batch_update_accounts(command, context)
.await
.map_err(|error| map_store_error(error, "provider accounts"))?;
for (provider, provider_ids) in providers.values() {
for (provider, provider_ids, _) in providers.values() {
if enabled == Some(false) {
for account_id in provider_ids {
super::notify_account_unavailable(
@@ -872,6 +886,18 @@ impl AccountsService for DefaultAccountsService {
.await;
}
publish_committed(self.snapshot.as_ref(), result.config_revision).await?;
for (provider, _, exhausted) in providers.into_values() {
if exhausted.is_empty() {
continue;
}
super::credential_mutation::observe_account_quotas(
self.upstream_info.as_ref(),
&provider,
&self.accounts,
&exhausted,
context.request_id.as_str(),
);
}
Ok(result)
}
@@ -30,7 +30,18 @@ pub(super) async fn publish_credentials_and_observe_quota(
notify_account_facts_changed(peer.map(AsRef::as_ref), provider.as_ref(), account_ids).await;
publish_committed(snapshot, revision).await?;
// 凭据已经提交;额度只是可重建的观察,不能拖住管理请求或回滚提交结果
observe_account_quotas(peer, provider, accounts, account_ids, request_id);
Ok(())
}
pub(super) fn observe_account_quotas(
peer: Option<&std::sync::Arc<dyn crate::ports::peer::PeerUpstreamService>>,
provider: &std::sync::Arc<dyn ProviderAdmin>,
accounts: &std::sync::Arc<dyn AccountStore>,
account_ids: &[ProviderAccountId],
request_id: &str,
) {
// 配置已提交,额度查询失败不回滚配置
let provider = provider.clone();
let peer = peer.cloned();
let accounts = accounts.clone();
@@ -48,14 +59,14 @@ pub(super) async fn publish_credentials_and_observe_quota(
Ok(Some(account)) => account,
Ok(None) => continue,
Err(error) => {
tracing::warn!(request_id, account_id = %account_id, error = %error, "initial quota account lookup failed");
tracing::warn!(request_id, account_id = %account_id, error = %error, "quota observation account lookup failed");
continue;
}
};
if !account.enabled || &account.provider_kind != provider.provider_kind() {
continue;
}
// 首查与手动刷新共用认证能力分派,互联协议由 Admin 解释
// 后台查询与手动刷新共用认证能力分派,互联协议由 Admin 解释
super::peer_link::upstream::read_quota(
Some(peer),
provider.as_ref(),
@@ -72,12 +83,11 @@ pub(super) async fn publish_credentials_and_observe_quota(
provider = %provider.provider_kind().as_str(),
account_id = %account_id.as_str(),
quota_error = ?error.kind(),
"initial quota observation failed"
"account quota observation failed"
);
}
}
});
Ok(())
}
pub(super) async fn required_credential(
@@ -10,8 +10,8 @@ use crate::{
model::{
AdminError, AdminErrorKind,
system::{
SystemOperationAccepted, SystemUpdateChannel, SystemUpdateDetail, SystemUpdateStatus,
SystemVersion,
SystemOperationAccepted, SystemUpdateChannel, SystemUpdateCheck, SystemUpdateDetail,
SystemUpdateStatus, SystemVersion,
},
},
ports::system::{
@@ -25,7 +25,7 @@ pub trait SystemService: Send + Sync {
async fn version(&self) -> Result<SystemVersion, AdminError>;
async fn update_detail(
&self,
refresh: bool,
check: SystemUpdateCheck,
channel: Option<SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, AdminError>;
fn update_events(&self) -> SystemUpdateEventStream;
@@ -71,11 +71,11 @@ impl SystemService for DefaultSystemService {
async fn update_detail(
&self,
refresh: bool,
check: SystemUpdateCheck,
channel: Option<SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, AdminError> {
self.operations
.update_detail(refresh, channel)
.update_detail(check, channel)
.await
.map_err(map_system_error)
}
@@ -1166,7 +1166,9 @@ impl SettingsStore for StaticSettingsStore {
max_concurrent_per_account: 1,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -1906,6 +1908,7 @@ async fn accounts_batch_update_should_commit_once_and_notify_each_provider() {
.batch_update(
&context("batch-update-request"),
BatchUpdateAccounts {
auto_use_quota_credits: None,
model_access: Default::default(),
outbound_proxy: None,
account_ids: vec!["acct_openai".to_owned(), "acct_xai".to_owned()],
@@ -3396,6 +3399,7 @@ fn quota_local_usage(account_id: &str, total_tokens: u64) -> AccountUsage {
pub(super) fn account_record(kind: &str) -> AccountRecord {
let now = Utc::now();
AccountRecord {
auto_use_quota_credits: false,
outbound_proxy_id: None,
outbound_proxy_enabled: true,
notes: None,
@@ -41,7 +41,9 @@ fn runtime_settings(enabled: bool, probe_enabled: bool) -> RuntimeSettings {
max_concurrent_per_account: 5,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -1179,7 +1179,7 @@ impl SystemOperations for UnavailableSystem {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
Err(unavailable_system())
@@ -1420,7 +1420,9 @@ impl SettingsStore for FixtureSettingsStore {
max_concurrent_per_account: self.max_concurrent_per_account,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -456,6 +456,31 @@ impl ProviderAccountStore for TestAccounts {
.cloned()
.collect())
}
async fn list_for_provider_accounts(
&self,
provider: &ProviderKind,
account_ids: &[ProviderAccountId],
) -> Result<Vec<ProviderAccount>, StoreError> {
if account_ids.is_empty() {
return Ok(Vec::new());
}
if Some(provider.as_str()) == self.failed_provider {
return Err(StoreError::new(StoreErrorKind::Unavailable));
}
Ok(self
.accounts
.lock()
.unwrap()
.iter()
.filter(|account| {
account.provider() == provider
&& account.enabled()
&& account_ids.contains(account.id())
})
.cloned()
.collect())
}
async fn list_refresh_candidates(
&self,
_query: ProviderRefreshQuery,
@@ -333,7 +333,9 @@ async fn settings_should_reject_zero_refresh_margin_before_store_call() {
max_concurrent_per_account: 1,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -41,7 +41,7 @@ impl SystemOperations for RecordingSystemOperations {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
Ok(SystemUpdateDetail {
@@ -60,6 +60,7 @@ impl SystemOperations for RecordingSystemOperations {
update_supported: false,
unsupported_reason: Some("source build".to_owned()),
warning: None,
retry_at: None,
})
}
@@ -250,7 +250,7 @@ impl SystemOperations for PreflightingSystem {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
unreachable!()
@@ -116,6 +116,7 @@ pub(super) fn account_view(
let (mut quota, refresh_token_expires_at) = account_quota_view(quota, cooldown, now, time);
quota.protection_rules = protection_rules;
AccountView {
auto_use_quota_credits: account.auto_use_quota_credits,
upstream,
capabilities: capabilities.into(),
id: account.id.clone(),
@@ -61,6 +61,7 @@ pub struct ListQuery {
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BatchUpdateAccountsRequest {
pub auto_use_quota_credits: Option<bool>,
pub outbound_proxy_id: Option<String>,
pub outbound_proxy_url: Option<AccountProxyUpdate>,
pub account_ids: Vec<String>,
@@ -117,7 +118,8 @@ impl BatchUpdateAccountsRequest {
.map(parse_concurrency_limit)
.transpose()?;
self.weight.map(parse_account_weight).transpose()?;
if self.enabled.is_none()
if self.auto_use_quota_credits.is_none()
&& self.enabled.is_none()
&& self.concurrency_limit.is_none()
&& self.weight.is_none()
&& self.group_ids.is_none()
@@ -133,6 +135,7 @@ impl BatchUpdateAccountsRequest {
pub(super) fn into_command(self) -> Result<BatchUpdateAccounts, WireValidationError> {
self.validate()?;
Ok(BatchUpdateAccounts {
auto_use_quota_credits: self.auto_use_quota_credits,
outbound_proxy: proxy_selection(self.outbound_proxy_id, self.outbound_proxy_url)?,
account_ids: self.account_ids,
enabled: self.enabled,
@@ -266,6 +269,7 @@ pub struct AccountSummaryView {
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountView {
pub auto_use_quota_credits: bool,
pub upstream: Option<super::AccountUpstreamInfoView>,
pub capabilities: AccountCapabilitiesView,
pub outbound_proxy_id: Option<String>,
@@ -318,6 +322,7 @@ pub struct AccountCapacityView {
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct AccountCapabilitiesView {
pub quota_credits: bool,
pub quota: bool,
pub quota_refresh: bool,
pub profile: bool,
@@ -336,6 +341,7 @@ impl From<gateway_admin::model::accounts::ProviderAccountCapabilities> for Accou
subscription: value.subscription,
avatar: value.avatar,
reset_credits: value.reset_credits,
quota_credits: value.quota_credits,
consume_reset_credit: value.consume_reset_credit,
}
}
@@ -44,7 +44,9 @@ pub struct RuntimeSettingsFields {
pub max_concurrent_per_account: u64,
pub max_waiting_per_key: u32,
pub max_waiting_per_account: u32,
pub concurrency_wait_timeout_seconds: u32,
pub key_queue_timeout_seconds: u32,
pub account_queue_timeout_seconds: u32,
pub switch_account_on_queue_failure: bool,
pub max_retry_after_seconds: u32,
pub openai_guardian_reserved_concurrency: u32,
pub openai_account_affinity: gateway_core::account::AccountAffinity,
@@ -751,7 +753,9 @@ impl RuntimeSettingsFields {
.map_err(|_| WireValidationError::new("settingsMaxConcurrencyOverflow"))?,
max_waiting_per_key: self.max_waiting_per_key,
max_waiting_per_account: self.max_waiting_per_account,
concurrency_wait_timeout_seconds: self.concurrency_wait_timeout_seconds,
key_queue_timeout_seconds: self.key_queue_timeout_seconds,
account_queue_timeout_seconds: self.account_queue_timeout_seconds,
switch_account_on_queue_failure: self.switch_account_on_queue_failure,
max_retry_after_seconds: self.max_retry_after_seconds,
openai_guardian_reserved_concurrency: self.openai_guardian_reserved_concurrency,
openai_account_affinity: self.openai_account_affinity,
@@ -791,7 +795,9 @@ impl From<gateway_admin::model::settings::RuntimeSettingsValues> for RuntimeSett
max_concurrent_per_account: u64::from(settings.max_concurrent_per_account),
max_waiting_per_key: settings.max_waiting_per_key,
max_waiting_per_account: settings.max_waiting_per_account,
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
key_queue_timeout_seconds: settings.key_queue_timeout_seconds,
account_queue_timeout_seconds: settings.account_queue_timeout_seconds,
switch_account_on_queue_failure: settings.switch_account_on_queue_failure,
max_retry_after_seconds: settings.max_retry_after_seconds,
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
openai_account_affinity: settings.openai_account_affinity,
@@ -824,7 +830,8 @@ fn settings_validation_error(field: &'static str) -> WireValidationError {
"reset_credit_max_weekly_remaining_percent" => "resetCreditMaxWeeklyRemainingPercent",
"account_warmup_schedule_time" => "accountWarmupScheduleTime",
"audit_retention_days" => "auditRetentionDays",
"concurrency_wait_timeout_seconds" => "concurrencyWaitTimeoutSeconds",
"key_queue_timeout_seconds" => "keyQueueTimeoutSeconds",
"account_queue_timeout_seconds" => "accountQueueTimeoutSeconds",
"max_account_rotations" => "maxAccountRotations",
"max_concurrent_per_account" => "maxConcurrentPerAccount",
"max_waiting_per_account" => "maxWaitingPerAccount",
+27 -10
View File
@@ -20,8 +20,9 @@ use axum::{
use futures::{Stream, StreamExt};
use gateway_admin::model::system::{
SystemOperationAccepted, SystemOperationKind, SystemOperationState, SystemOperationStatus,
SystemRestartPlan, SystemUpdateChannel, SystemUpdateDetail, SystemUpdateEvent,
SystemUpdateEventLevel, SystemUpdatePolicy, SystemUpdateStatus, SystemVersion,
SystemRestartPlan, SystemUpdateChannel, SystemUpdateCheck, SystemUpdateDetail,
SystemUpdateEvent, SystemUpdateEventLevel, SystemUpdatePolicy, SystemUpdateStatus,
SystemVersion,
};
use serde::{Deserialize, Serialize};
@@ -34,7 +35,10 @@ use super::{
#[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct UpdateDetailQuery {
refresh: Option<bool>,
#[serde(default)]
refresh: bool,
#[serde(default)]
summary_only: bool,
channel: Option<SystemUpdateChannel>,
}
@@ -44,10 +48,16 @@ impl UpdateDetailQuery {
self.channel
}
/// 是否强制从发布源刷新
/// 摘要只读取发布索引;手动刷新有独立的最短间隔
#[must_use]
pub fn refresh(&self) -> bool {
self.refresh.unwrap_or(false)
pub const fn check(&self) -> SystemUpdateCheck {
if self.summary_only {
SystemUpdateCheck::Summary
} else if self.refresh {
SystemUpdateCheck::Refresh
} else {
SystemUpdateCheck::Detail
}
}
}
@@ -116,10 +126,12 @@ struct SystemUpdateDetailView {
update_supported: bool,
unsupported_reason: Option<String>,
warning: Option<String>,
retry_at: Option<String>,
retry_at_display: Option<String>,
}
impl From<SystemUpdateDetail> for SystemUpdateDetailView {
fn from(detail: SystemUpdateDetail) -> Self {
impl From<(SystemUpdateDetail, crate::time::TimePresenter)> for SystemUpdateDetailView {
fn from((detail, time): (SystemUpdateDetail, crate::time::TimePresenter)) -> Self {
Self {
deployment_mode_label: deployment_mode_label(&detail.deployment_mode).to_owned(),
build_type_label: build_type_label(&detail.build_type),
@@ -135,6 +147,8 @@ impl From<SystemUpdateDetail> for SystemUpdateDetailView {
update_supported: detail.update_supported,
unsupported_reason: detail.unsupported_reason,
warning: detail.warning,
retry_at: detail.retry_at.map(|at| at.to_rfc3339()),
retry_at_display: detail.retry_at.map(|at| time.label(at, "%m-%d %H:%M:%S")),
}
}
}
@@ -284,12 +298,15 @@ where
let detail = state
.admin_services()
.system()
.update_detail(query.refresh(), query.channel())
.update_detail(query.check(), query.channel())
.await
.map_err(map_system_error)?;
Ok(AdminResponse::new(
StatusCode::OK,
AdminEnvelope::ok(SystemUpdateDetailView::from(detail)),
AdminEnvelope::ok(SystemUpdateDetailView::from((
detail,
crate::time::TimePresenter::new(state.admin_services().timezone()),
))),
))
}
@@ -406,6 +406,7 @@ fn account_page_item(
AccountPageItem {
protection_rules: Vec::new(),
account: AccountRecord {
auto_use_quota_credits: false,
outbound_proxy_id: None,
outbound_proxy_enabled: true,
id: "acct_cooldown".to_owned(),
@@ -578,6 +578,7 @@ mod provider {
total_slots: None,
},
account: AccountRecord {
auto_use_quota_credits: false,
outbound_proxy_id: None,
outbound_proxy_enabled: true,
notes: None,
@@ -650,7 +650,9 @@ impl SettingsStore for MemorySettingsStore {
max_concurrent_per_account: command.values.max_concurrent_per_account,
max_waiting_per_key: command.values.max_waiting_per_key,
max_waiting_per_account: command.values.max_waiting_per_account,
concurrency_wait_timeout_seconds: command.values.concurrency_wait_timeout_seconds,
key_queue_timeout_seconds: command.values.key_queue_timeout_seconds,
account_queue_timeout_seconds: command.values.account_queue_timeout_seconds,
switch_account_on_queue_failure: command.values.switch_account_on_queue_failure,
max_retry_after_seconds: command.values.max_retry_after_seconds,
openai_guardian_reserved_concurrency: command
.values
@@ -1831,7 +1833,7 @@ impl SystemOperations for UnusedSystem {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
Err(unavailable_system())
@@ -1895,7 +1897,9 @@ fn test_runtime_settings() -> RuntimeSettings {
max_concurrent_per_account: 3,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -56,7 +56,9 @@ fn update_body() -> Value {
"openaiAccountAffinity": "relaxed",
"maxAccountRotations": 3,
"openaiSessionAffinityTtlHours": 24,
"concurrencyWaitTimeoutSeconds": 30,
"keyQueueTimeoutSeconds": 30,
"accountQueueTimeoutSeconds": 30,
"switchAccountOnQueueFailure": false,
"maxRetryAfterSeconds": 240,
"responsesMaxDecompressedBodyBytes": 67108864,
"rotationStrategy": "round_robin",
@@ -659,8 +661,10 @@ fn concurrency_queue_fields_validate_bounds_and_accept_disabled_queues() {
for (field, invalid) in [
("maxWaitingPerKey", 1001),
("maxWaitingPerAccount", 1001),
("concurrencyWaitTimeoutSeconds", 0),
("concurrencyWaitTimeoutSeconds", 121),
("keyQueueTimeoutSeconds", 0),
("accountQueueTimeoutSeconds", 0),
("keyQueueTimeoutSeconds", 121),
("accountQueueTimeoutSeconds", 121),
] {
let mut body = update_body();
body[field] = json!(invalid);
@@ -670,7 +674,7 @@ fn concurrency_queue_fields_validate_bounds_and_accept_disabled_queues() {
let mut body = update_body();
body["maxWaitingPerKey"] = json!(0);
body["maxWaitingPerAccount"] = json!(1000);
body["concurrencyWaitTimeoutSeconds"] = json!(120);
body["accountQueueTimeoutSeconds"] = json!(120);
serde_json::from_value::<UpdateRuntimeSettingsRequest>(body)
.unwrap()
.validate()
@@ -27,7 +27,22 @@ use super::{AdminTestFixture, AdminTestState, unavailable_system};
#[test]
fn update_detail_query_should_default_to_no_refresh_and_reject_unknown_fields() {
let query: UpdateDetailQuery = serde_json::from_str("{}").expect("empty query object");
assert!(!query.refresh());
use gateway_admin::model::system::SystemUpdateCheck;
assert_eq!(query.check(), SystemUpdateCheck::Detail);
for (json, mode) in [
(r#"{"refresh":true}"#, SystemUpdateCheck::Refresh),
(
r#"{"summaryOnly":true,"refresh":true}"#,
SystemUpdateCheck::Summary,
),
] {
assert_eq!(
serde_json::from_str::<UpdateDetailQuery>(json)
.unwrap()
.check(),
mode
);
}
assert!(serde_json::from_str::<UpdateDetailQuery>(r#"{"refresh":true,"extra":1}"#).is_err());
}
@@ -191,7 +206,7 @@ impl SystemOperations for ProgressSystem {
async fn update_detail(
&self,
_: bool,
check: gateway_admin::model::system::SystemUpdateCheck,
channel: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
use gateway_admin::model::system::{SystemUpdateChannel, SystemUpdatePolicy};
@@ -206,11 +221,16 @@ impl SystemOperations for ProgressSystem {
deployment_mode: "docker".into(),
build_type: "release".into(),
release_url: None,
notes: self.notes.clone(),
notes: if check == gateway_admin::model::system::SystemUpdateCheck::Summary {
None
} else {
self.notes.clone()
},
cached: false,
update_supported: true,
unsupported_reason: None,
warning: None,
retry_at: None,
})
}
@@ -281,7 +301,10 @@ impl SystemOperations for ProgressSystem {
#[tokio::test]
async fn update_detail_should_validate_and_forward_the_temporary_channel() {
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem::default())).await;
let fixture = AdminTestFixture::with_system(Arc::new(ProgressSystem {
notes: Some("notes".into()),
}))
.await;
fixture.auth.insert_session("valid-session");
let router = app(fixture.state());
for (authenticated, query, expected) in [
@@ -289,6 +312,8 @@ async fn update_detail_should_validate_and_forward_the_temporary_channel() {
(true, "channel=nightly", StatusCode::BAD_REQUEST),
(true, "channel=beta&extra=true", StatusCode::BAD_REQUEST),
(true, "channel=beta", StatusCode::OK),
(true, "channel=beta&refresh=true", StatusCode::OK),
(true, "channel=beta&summaryOnly=true", StatusCode::OK),
(true, "", StatusCode::OK),
] {
let mut request = Request::builder()
@@ -307,6 +332,10 @@ async fn update_detail_should_validate_and_forward_the_temporary_channel() {
let body: serde_json::Value =
serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.expect("body"))
.expect("json");
assert_eq!(
body["data"]["notes"].is_null(),
query.contains("summaryOnly")
);
assert_eq!(
body["data"]["policy"]["channel"],
if query.is_empty() { "stable" } else { "beta" }
@@ -208,7 +208,9 @@ async fn request_settings_reach_query_terminal_without_mutating_authenticated_ba
settings: Some(|settings| {
let mut runtime = serde_json::to_value(&settings.runtime).unwrap();
runtime["model_mappings"] = serde_json::json!({"model-a":"plugin-model"});
runtime["concurrency_wait_timeout_seconds"] = serde_json::json!(30);
runtime["key_queue_timeout_seconds"] = serde_json::json!(30);
runtime["account_queue_timeout_seconds"] = serde_json::json!(30);
runtime["switch_account_on_queue_failure"] = serde_json::json!(false);
settings.runtime = serde_json::from_value(runtime).unwrap();
settings.fast_mode = FastMode::Disabled;
}),
@@ -97,6 +97,7 @@ async fn fixture(sharing: ClientPeerSharing, plaintext: &str) -> (AdminTestFixtu
},
projection: resolve_account_status(&facts, now.into()),
account: AccountRecord {
auto_use_quota_credits: false,
id: "acct_private_sentinel".into(),
provider_kind: provider,
groups: vec![group.clone()],
+1 -1
View File
@@ -105,7 +105,7 @@ impl SystemOperations for VersionSystem {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
unreachable!("client route must not request update details")
@@ -364,9 +364,11 @@ impl QuotaAccessState {
}
}
/// 权威额度耗尽证据
/// 账号额度访问依据
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum QuotaEvidence {
/// 套餐已耗尽,但上游确认点数可用
CreditsAvailable,
ProviderDenied,
AccountLimitReached,
UsageLimitReached,
@@ -377,6 +379,7 @@ impl QuotaEvidence {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::CreditsAvailable => "credits_available",
Self::ProviderDenied => "provider_denied",
Self::AccountLimitReached => "account_limit_reached",
Self::UsageLimitReached => "usage_limit_reached",
@@ -387,6 +390,7 @@ impl QuotaEvidence {
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"credits_available" => Some(Self::CreditsAvailable),
"provider_denied" => Some(Self::ProviderDenied),
"account_limit_reached" => Some(Self::AccountLimitReached),
"usage_limit_reached" => Some(Self::UsageLimitReached),
@@ -456,6 +460,17 @@ impl QuotaState {
}
}
/// 点数是唯一放行依据时保留套餐重置时间,开关变化不伪造新观测
#[must_use]
pub const fn credits_only(observed_at: SystemTime, reset_at: Option<SystemTime>) -> Self {
Self {
access: QuotaAccessState::Allowed,
evidence: Some(QuotaEvidence::CreditsAvailable),
observed_at: Some(observed_at),
reset_at,
}
}
/// 从持久化列恢复额度状态;非法组合返回 `None`
#[must_use]
pub const fn from_persisted(
@@ -464,32 +479,25 @@ impl QuotaState {
observed_at: Option<SystemTime>,
reset_at: Option<SystemTime>,
) -> Option<Self> {
match access {
QuotaAccessState::Unknown if evidence.is_none() && reset_at.is_none() => Some(Self {
let valid = match (access, evidence) {
(QuotaAccessState::Unknown, None) => reset_at.is_none(),
(QuotaAccessState::Allowed, None) => observed_at.is_some() && reset_at.is_none(),
(QuotaAccessState::Allowed, Some(QuotaEvidence::CreditsAvailable)) => {
observed_at.is_some()
}
(QuotaAccessState::Exhausted, Some(QuotaEvidence::CreditsAvailable)) => false,
(QuotaAccessState::Exhausted, Some(_)) => observed_at.is_some(),
_ => false,
};
if valid {
Some(Self {
access,
evidence,
observed_at,
reset_at,
}),
QuotaAccessState::Allowed
if evidence.is_none() && observed_at.is_some() && reset_at.is_none() =>
{
Some(Self {
access,
evidence,
observed_at,
reset_at,
})
}
QuotaAccessState::Exhausted if evidence.is_some() && observed_at.is_some() => {
Some(Self {
access,
evidence,
observed_at,
reset_at,
})
}
_ => None,
})
} else {
None
}
}
@@ -813,6 +821,7 @@ pub struct ProviderAccount {
enabled: bool,
concurrency_limit: Option<AccountConcurrencyLimit>,
weight: AccountWeight,
auto_use_quota_credits: bool,
model_access: super::AccountModelAccess,
credential_state: CredentialState,
quota: QuotaState,
@@ -851,6 +860,7 @@ impl ProviderAccount {
enabled: true,
concurrency_limit: None,
weight: AccountWeight::DEFAULT,
auto_use_quota_credits: false,
model_access: super::AccountModelAccess::all(),
credential_state: CredentialState::Unknown,
quota: QuotaState::unknown(),
@@ -864,6 +874,17 @@ impl ProviderAccount {
}
}
#[must_use]
pub const fn with_auto_use_quota_credits(mut self, enabled: bool) -> Self {
self.auto_use_quota_credits = enabled;
self
}
#[must_use]
pub const fn auto_use_quota_credits(&self) -> bool {
self.auto_use_quota_credits
}
#[must_use]
pub fn with_notes(mut self, notes: Option<String>) -> Self {
self.notes = notes;
@@ -6,6 +6,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, RwLock};
use std::time::{Duration, Instant, SystemTime};
use futures::FutureExt;
use crate::concurrency::{CapacityWait, ConcurrencyQueuePolicy, QueueRejection};
use crate::identity::ProviderKind;
@@ -92,6 +94,7 @@ pub struct AccountSelectionPolicy {
queue_policy: ConcurrencyQueuePolicy,
openai_guardian_reserved_concurrency: u32,
openai_account_affinity: AccountAffinity,
switch_account_on_queue_failure: bool,
max_account_rotations: u32,
openai_session_affinity_ttl: Duration,
}
@@ -108,6 +111,7 @@ impl AccountSelectionPolicy {
max_concurrent_per_account: max_concurrent_per_account.into(),
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: AccountAffinity::default(),
switch_account_on_queue_failure: false,
max_account_rotations: 3,
openai_session_affinity_ttl: Duration::from_secs(24 * 3600),
queue_policy: ConcurrencyQueuePolicy {
@@ -128,6 +132,45 @@ impl AccountSelectionPolicy {
self.openai_account_affinity
}
#[must_use]
pub const fn with_switch_account_on_queue_failure(mut self, enabled: bool) -> Self {
self.switch_account_on_queue_failure = enabled;
self
}
/// 原队位由首次选号释放。仅对允许换号的请求调用,重选不排队,也不延长请求总期限
pub async fn reselect_after_queue_rejection<
T,
E: From<QueueRejection>,
F: Future<Output = Result<T, E>>,
>(
self,
rejection: QueueRejection,
remaining: Option<Duration>,
select: impl FnOnce(ConcurrencyQueuePolicy) -> F,
no_account_available: impl FnOnce(&E) -> bool,
) -> Result<T, E> {
if !self.switch_account_on_queue_failure || remaining.is_some_and(|time| time.is_zero()) {
return Err(rejection.into());
}
let queue = ConcurrencyQueuePolicy {
max_waiting: 0,
..self.queue_policy
};
let selection = select(queue);
let result = match remaining {
Some(limit) => futures::select_biased! {
() = futures_timer::Delay::new(limit).fuse() => return Err(rejection.into()),
result = selection.fuse() => result,
},
None => selection.await,
};
match result {
Err(error) if no_account_available(&error) => Err(rejection.into()),
result => result,
}
}
#[must_use]
pub const fn with_max_account_rotations(mut self, rotations: u32) -> Self {
self.max_account_rotations = rotations;
@@ -29,6 +29,13 @@ pub trait ProviderAccountStore: Send + Sync {
provider: &ProviderKind,
) -> Result<Vec<ProviderAccount>, StoreError>;
/// 只读取指定 Provider 冻结范围内的已启用账号,空范围直接返回空池
async fn list_for_provider_accounts(
&self,
provider: &ProviderKind,
account_ids: &[ProviderAccountId],
) -> Result<Vec<ProviderAccount>, StoreError>;
/// 一次有界查询返回账号事实和 revision-fenced 明文 credential
async fn list_refresh_candidates(
&self,
+35 -10
View File
@@ -11,6 +11,7 @@ use futures_timer::Delay;
const MAX_TOTAL_WAITING: usize = 1_024;
const CAPACITY_RECHECK_INTERVAL: Duration = Duration::from_millis(100);
const NON_HEAD_RECHECK_INTERVAL: Duration = Duration::from_millis(500);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct ConcurrencyQueuePolicy {
@@ -18,7 +19,7 @@ pub struct ConcurrencyQueuePolicy {
pub timeout: Duration,
}
/// 从首次入队开始计时,密钥、账号与后续重试共享同一个截止时刻
/// 从首次入队开始计时,同一层的重新排队与重试共享截止时刻
#[derive(Debug, Clone, Default)]
pub struct ConcurrencyWaitBudget {
deadline: Arc<OnceLock<Instant>>,
@@ -223,6 +224,9 @@ impl<K: Eq + Hash> WaitTicket<K> {
}
pub async fn retry(&self) -> Result<(), QueueRejection> {
if !self.is_head() {
return self.turn().await;
}
// 租约释放可能经过后台队列;有界重查同时覆盖释放通知之前及 TTL 到期的空位
Delay::new(
CAPACITY_RECHECK_INTERVAL.min(self.deadline.saturating_duration_since(Instant::now())),
@@ -230,6 +234,28 @@ impl<K: Eq + Hash> WaitTicket<K> {
.await;
self.turn().await
}
async fn retry_or_recheck(&self) -> Result<(), QueueRejection> {
if !self.is_head() {
// 非队首由位置释放唤醒,低频重查覆盖外部调度状态变化
let turn = self.turn().fuse();
let recheck = Delay::new(NON_HEAD_RECHECK_INTERVAL).fuse();
pin_mut!(turn, recheck);
return select_biased! {
result = turn => result,
() = recheck => if Instant::now() < self.deadline { Ok(()) } else { Err(QueueRejection::Timeout) },
};
}
Delay::new(
CAPACITY_RECHECK_INTERVAL.min(self.deadline.saturating_duration_since(Instant::now())),
)
.await;
if Instant::now() < self.deadline {
Ok(())
} else {
Err(QueueRejection::Timeout)
}
}
}
impl<K: Eq + Hash> Drop for WaitTicket<K> {
@@ -300,13 +326,19 @@ impl<'a, K: Clone + Eq + Hash> CapacityWait<'a, K> {
self
}
/// 外部绑定可在队列等待期间变化,周期重读仍由 can_try 保持队首资格
/// 允许非队首低频复核外部调度状态,当前队列准入仍由 can_try 裁决
#[must_use]
pub const fn with_periodic_recheck(mut self, enabled: bool) -> Self {
self.periodic_recheck = enabled;
self
}
/// 当前等待位置,供调用方按请求归属限定重查范围
#[must_use]
pub fn queued_key(&self) -> Option<&K> {
self.ticket.as_ref().map(WaitTicket::key)
}
#[must_use]
pub fn can_try(&self, key: &K) -> bool {
if let Some(ticket) = self.ticket.as_ref().filter(|ticket| ticket.key() == key) {
@@ -356,14 +388,7 @@ impl<'a, K: Clone + Eq + Hash> CapacityWait<'a, K> {
}
if let Some(ticket) = &self.ticket {
if self.periodic_recheck {
Delay::new(
CAPACITY_RECHECK_INTERVAL
.min(ticket.deadline.saturating_duration_since(Instant::now())),
)
.await;
if Instant::now() >= ticket.deadline {
return Err(QueueRejection::Timeout);
}
ticket.retry_or_recheck().await?;
} else {
ticket.retry().await?;
}
@@ -600,12 +600,6 @@ impl<S: ?Sized> ResponseExecutionSession<S>
where
S: ExecutionStore + 'static,
{
/// 首次驱动前接入密钥准入使用的预算,所有账号尝试随后共享它
pub(crate) fn with_concurrency_wait_budget(mut self, budget: ConcurrencyWaitBudget) -> Self {
self.concurrency_wait_budget = budget;
self
}
/// 传输首字节事实可在 next_event 等待期间由协议层提交
pub fn response_commit(&self) -> super::execution::ResponseCommit {
self.response_commit.clone()
@@ -5,7 +5,6 @@ use super::contract::{AuthenticatedClient, ExecutionRequestMetadata};
use super::{MAX_CONCURRENT_NESTED_EXECUTIONS, MAX_NESTED_EXECUTIONS};
use crate::engine::budget::{ClientBudgetCharge, ClientBudgetPort};
use crate::{
concurrency::ConcurrencyWaitBudget,
engine::{
ModelRequestId, ProviderAccountId, coordinator::CoordinationExtensions,
extensions::ExtensionCallScope, nested::ExecutionEffects,
@@ -75,7 +74,6 @@ pub(super) struct AuthorizedExecution {
pub(super) struct ExecutionStartGuard {
pub(super) admission: ExecutionAdmission,
pub(super) active_request: Option<ActiveRequestLease>,
pub(super) concurrency_wait_budget: ConcurrencyWaitBudget,
pub(super) admission_decision_ms: Option<u64>,
}
@@ -724,7 +724,6 @@ impl DefaultExecutionService {
request_id: &ModelRequestId,
authorization: &mut AuthorizedExecution,
) -> Result<ExecutionStartGuard, GatewayError> {
let concurrency_wait_budget = ConcurrencyWaitBudget::default();
let (admission, admission_decision_ms) = if authorization.nested.is_some() {
let permit = authorization.nested_permit.take().ok_or_else(|| {
GatewayError::new(
@@ -741,7 +740,6 @@ impl DefaultExecutionService {
request_id,
authorization.deadline_at,
authorization.cancellation.clone(),
&concurrency_wait_budget,
)
.await?;
let admission = if let Some(permit) = authorization.nested_permit.take() {
@@ -774,7 +772,6 @@ impl DefaultExecutionService {
Ok(ExecutionStartGuard {
admission,
active_request,
concurrency_wait_budget,
admission_decision_ms,
})
}
@@ -802,7 +799,6 @@ impl DefaultExecutionService {
let ExecutionStartGuard {
admission,
active_request,
concurrency_wait_budget,
admission_decision_ms,
} = start_guard;
let observation = plan
@@ -884,7 +880,7 @@ impl DefaultExecutionService {
)
.await
{
Ok(core) => core.with_concurrency_wait_budget(concurrency_wait_budget),
Ok(core) => core,
Err(error) => {
if let Some(budget) = &self.budget {
settle_budget(
@@ -1245,13 +1241,13 @@ impl DefaultExecutionService {
request_id: &ModelRequestId,
deadline_at: Deadline,
cancellation: CancellationToken,
budget: &ConcurrencyWaitBudget,
) -> Result<AdmissionLease, GatewayError> {
let policy = client.snapshot.client_queue_policy();
let limits = client.policy.limits();
let key = client.policy.key_id();
let budget = ConcurrencyWaitBudget::default();
let mut waiting =
CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), budget);
CapacityWait::new(&self.admission_waiting, policy, deadline_at.at(), &budget);
let mut admission = AdmissionLease {
port: Arc::clone(&self.admissions),
diagnostics: self.diagnostics.clone(),
@@ -1566,7 +1562,12 @@ impl ExecutionService for DefaultExecutionService {
for kind in scope.provider_kinds() {
let provider = providers.get(kind).ok_or(ProviderCatalogUnavailable)?;
let Some(models) = provider
.query_client_model_catalog(scope, protocol, client_version)
.query_client_model_catalog(
client.policy.key_id(),
scope,
protocol,
client_version,
)
.await?
else {
for profile in client.snapshot.profiles_for_provider_scope(kind, scope) {
@@ -880,7 +880,7 @@ impl AttemptContext {
self.deadline
}
/// 返回密钥准入、账号选择与重试共用的排队预算
/// 返回账号选择与后续重试共用的排队预算
#[must_use]
pub const fn concurrency_wait_budget(&self) -> &crate::concurrency::ConcurrencyWaitBudget {
&self.request.concurrency_wait_budget
@@ -1219,6 +1219,7 @@ pub trait Provider: Send + Sync {
/// `None` 表示不提供该协议的原生目录;读取失败不能伪装成不支持
async fn query_client_model_catalog(
&self,
_client_api_key_id: &ClientApiKeyId,
_scope: &crate::account::scope::FrozenAccountScope,
_protocol: &str,
_client_version: &str,
@@ -219,6 +219,13 @@ pub trait ProviderCooldownPort: Send + Sync {
account_id: &'a ProviderAccountId,
) -> BoxFuture<'a, Result<Option<ProviderCooldown>, ProviderStoreError>>;
/// 按输入顺序返回每项读取结果,单项失败不丢弃其余账号事实
/// 实现应限制单批读取规模,取消后不启动后续批次
fn read_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> BoxFuture<'a, Vec<Result<Option<ProviderCooldown>, ProviderStoreError>>>;
fn clear<'a>(
&'a self,
account_id: &'a ProviderAccountId,
@@ -236,6 +243,13 @@ pub trait ProviderCooldownPort: Send + Sync {
scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Result<Option<ProviderScopedCooldown>, ProviderStoreError>>;
/// 与账号级批量读取使用相同的顺序、单项失败和取消合同
fn read_scoped_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Vec<Result<Option<ProviderScopedCooldown>, ProviderStoreError>>>;
fn clear_scoped<'a>(
&'a self,
account_id: &'a ProviderAccountId,
@@ -74,13 +74,12 @@ impl CompiledSettings {
super::validate_request_limits(
settings.max_waiting_per_key,
settings.max_waiting_per_account,
settings.concurrency_wait_timeout_seconds,
settings.key_queue_timeout_seconds,
settings.account_queue_timeout_seconds,
settings.max_account_rotations,
settings.openai_session_affinity_ttl_hours,
)
.map_err(|_| InvalidSettings)?;
let queue_timeout =
Duration::from_secs(u64::from(settings.concurrency_wait_timeout_seconds));
let responses_max_decompressed_body_bytes =
super::response_body_limit(settings.responses_max_decompressed_body_bytes)
.map_err(|_| InvalidSettings)?;
@@ -104,11 +103,12 @@ impl CompiledSettings {
.with_smart_scheduling(settings.smart_scheduling)
.with_queue(ConcurrencyQueuePolicy {
max_waiting: settings.max_waiting_per_account,
timeout: queue_timeout,
});
timeout: Duration::from_secs(u64::from(settings.account_queue_timeout_seconds)),
})
.with_switch_account_on_queue_failure(settings.switch_account_on_queue_failure);
let client_queue_policy = ConcurrencyQueuePolicy {
max_waiting: settings.max_waiting_per_key,
timeout: queue_timeout,
timeout: Duration::from_secs(u64::from(settings.key_queue_timeout_seconds)),
};
let request_location = settings
.request_location_enabled
@@ -20,7 +20,9 @@ pub struct SettingsValues {
pub(crate) max_concurrent_per_account: u32,
pub(crate) max_waiting_per_key: u32,
pub(crate) max_waiting_per_account: u32,
pub(crate) concurrency_wait_timeout_seconds: u32,
pub(crate) key_queue_timeout_seconds: u32,
pub(crate) account_queue_timeout_seconds: u32,
pub(crate) switch_account_on_queue_failure: bool,
pub(crate) max_retry_after_seconds: u32,
pub(crate) openai_guardian_reserved_concurrency: u32,
pub(crate) openai_account_affinity: crate::account::AccountAffinity,
@@ -158,11 +160,15 @@ impl SettingsValues {
mut self,
max_waiting_per_key: u32,
max_waiting_per_account: u32,
timeout_seconds: u32,
key_timeout_seconds: u32,
account_timeout_seconds: u32,
switch_on_failure: bool,
) -> Self {
self.max_waiting_per_key = max_waiting_per_key;
self.max_waiting_per_account = max_waiting_per_account;
self.concurrency_wait_timeout_seconds = timeout_seconds;
self.key_queue_timeout_seconds = key_timeout_seconds;
self.account_queue_timeout_seconds = account_timeout_seconds;
self.switch_account_on_queue_failure = switch_on_failure;
self
}
@@ -185,7 +191,9 @@ impl SettingsValues {
max_concurrent_per_account,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: crate::account::AccountAffinity::default(),
@@ -221,7 +229,8 @@ impl SettingsValues {
pub fn validate_request_limits(
max_waiting_per_key: u32,
max_waiting_per_account: u32,
timeout_seconds: u32,
key_timeout_seconds: u32,
account_timeout_seconds: u32,
max_account_rotations: u32,
affinity_ttl_hours: u32,
) -> Result<(), &'static str> {
@@ -229,8 +238,12 @@ pub fn validate_request_limits(
(max_waiting_per_key <= 1_000, "max_waiting_per_key"),
(max_waiting_per_account <= 1_000, "max_waiting_per_account"),
(
(1..=120).contains(&timeout_seconds),
"concurrency_wait_timeout_seconds",
(1..=120).contains(&key_timeout_seconds),
"key_queue_timeout_seconds",
),
(
(1..=120).contains(&account_timeout_seconds),
"account_queue_timeout_seconds",
),
(
max_account_rotations <= crate::account::MAX_ACCOUNT_ROTATIONS,
@@ -48,6 +48,69 @@ fn cancelling_a_middle_waiter_preserves_fifo_and_reclaims_capacity() {
});
}
#[test]
fn non_head_waiters_resume_on_promotion_without_waiting_for_the_recheck_timer() {
for periodic in [false, true] {
let queue = ConcurrencyWaitQueue::default();
let deadline = Instant::now() + Duration::from_secs(2);
let head = queue.enqueue(&["account"], 2, deadline).unwrap();
let budget = ConcurrencyWaitBudget::default();
let mut waiting = CapacityWait::new(
&queue,
ConcurrencyQueuePolicy {
max_waiting: 2,
timeout: Duration::from_secs(2),
},
None,
&budget,
)
.with_periodic_recheck(periodic);
assert!(waiting.wait(&["account"]).now_or_never().is_none());
assert_eq!(waiting.queued_key(), Some(&"account"));
let retry = waiting.wait(&["account"]);
futures::pin_mut!(retry);
assert!(retry.as_mut().now_or_never().is_none());
drop(head);
assert_eq!(retry.as_mut().now_or_never(), Some(Ok(())));
}
}
#[test]
fn non_head_binding_rechecks_preserve_position_and_shared_deadline() {
block_on(async {
let queue = ConcurrencyWaitQueue::default();
let head = queue
.enqueue(&["account"], 2, Instant::now() + Duration::from_secs(2))
.unwrap();
let budget = ConcurrencyWaitBudget::default();
let mut waiting = CapacityWait::new(
&queue,
ConcurrencyQueuePolicy {
max_waiting: 2,
timeout: Duration::from_millis(750),
},
None,
&budget,
)
.with_periodic_recheck(true);
let started = Instant::now();
waiting.wait(&["account"]).await.unwrap();
assert!(started.elapsed() >= Duration::from_millis(500));
assert_eq!(waiting.queued_key(), Some(&"account"));
assert!(!waiting.can_try(&"account"));
assert!(matches!(
queue.enqueue(&["account"], 2, Instant::now() + Duration::from_secs(2)),
Err(QueueRejection::Full)
));
assert_eq!(
waiting.wait(&["account"]).await,
Err(QueueRejection::Timeout)
);
drop((head, waiting));
assert!(!queue.has_waiters(&"account"));
});
}
#[test]
fn wait_deadline_applies_before_and_after_becoming_head() {
block_on(async {
@@ -109,9 +172,8 @@ fn request_deadline_bounds_waiting_and_dropped_future_releases_its_ticket() {
}
#[test]
fn new_layers_and_retries_cannot_restart_a_requests_wait_budget() {
fn account_reselection_and_retries_share_the_account_wait_budget() {
block_on(async {
let keys = ConcurrencyWaitQueue::default();
let accounts = ConcurrencyWaitQueue::default();
let policy = ConcurrencyQueuePolicy {
max_waiting: 1,
@@ -119,11 +181,11 @@ fn new_layers_and_retries_cannot_restart_a_requests_wait_budget() {
};
let request_deadline = SystemTime::now() + Duration::from_secs(5);
let budget = ConcurrencyWaitBudget::default();
let mut key_wait = CapacityWait::new(&keys, policy, request_deadline, &budget);
key_wait.wait(&["key"]).await.unwrap();
drop(key_wait);
let mut first = CapacityWait::new(&accounts, policy, request_deadline, &budget);
first.wait(&["first_account"]).await.unwrap();
drop(first);
// 第一层已取得重试机会,后续准备跨过总时限后不能在账号层重新排队
// 第一次账号等待已启动时限,重试或改选其他账号都不能重新计时
futures_timer::Delay::new(policy.timeout).await;
let next_attempt_budget = budget.clone();
let mut account_wait =
@@ -36,6 +36,21 @@ pub(crate) struct Feedback {
}
impl ProviderCooldownPort for Feedback {
fn read_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> BoxFuture<'a, Vec<Result<Option<ProviderCooldown>, ProviderStoreError>>> {
Box::pin(async move { account_ids.iter().map(|_| Ok(None)).collect() })
}
fn read_scoped_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
_scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Vec<Result<Option<ProviderScopedCooldown>, ProviderStoreError>>> {
Box::pin(async move { account_ids.iter().map(|_| Ok(None)).collect() })
}
fn publish_protection_policy<'a>(
&'a self,
_: &'a AccountErrorProtection,
@@ -3340,6 +3340,7 @@ impl Provider for CatalogProvider {
}
async fn query_client_model_catalog(
&self,
_client_api_key_id: &ClientApiKeyId,
scope: &FrozenAccountScope,
protocol: &str,
version: &str,
@@ -4390,10 +4391,7 @@ impl Provider for QueuedAccountProvider {
) -> Result<ProviderStream, ProviderError> {
let mut waiting = gateway_core::concurrency::CapacityWait::new(
&self.waiting,
gateway_core::concurrency::ConcurrencyQueuePolicy {
max_waiting: 1,
timeout: Duration::from_secs(5),
},
context.account_selection_policy().queue_policy(),
context.deadline().at(),
context.concurrency_wait_budget(),
);
@@ -4406,7 +4404,7 @@ impl Provider for QueuedAccountProvider {
}
#[test]
fn account_wait_inherits_the_budget_spent_during_client_admission() {
fn account_wait_starts_its_own_budget_after_client_admission() {
use futures::FutureExt;
block_on(async {
let admissions = Arc::new(BoundedAdmissions::default());
@@ -4419,7 +4417,10 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() {
},
FastMode::Default,
);
let settings = snapshot.settings().clone().with_concurrency_queues(1, 0, 1);
let settings = snapshot
.settings()
.clone()
.with_concurrency_queues(1, 1, 1, 2, false);
let snapshot = snapshot.with_settings(&settings).unwrap();
let service = DefaultExecutionService::new(
RuntimeSnapshotHandle::new(snapshot),
@@ -4437,12 +4438,12 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() {
.start(request(&service, ClientTransport::HttpSse))
.await
.unwrap();
let started_at = Instant::now();
let mut queued = service.start(request(&service, ClientTransport::HttpSse));
assert!(queued.as_mut().now_or_never().is_none());
futures_timer::Delay::new(Duration::from_millis(200)).await;
drop(running);
let mut started = queued.await.unwrap();
let account_started = Instant::now();
let error = started.session.collect_uncommitted().await.unwrap_err();
assert!(
matches!(
@@ -4453,9 +4454,8 @@ fn account_wait_inherits_the_budget_spent_during_client_admission() {
"{error:?}"
);
assert!(
started_at.elapsed() < Duration::from_secs(3),
"账号层不能重新获得自己的 5 秒预算,实际耗时 {:?}",
started_at.elapsed()
account_started.elapsed() >= Duration::from_secs(2),
"账号排队应获得独立的两秒预算"
);
assert!(admissions.active.lock().unwrap().is_empty());
});
@@ -4532,6 +4532,8 @@ fn queue_service(
max_waiting,
0,
u32::try_from(timeout.as_secs()).unwrap(),
30,
false,
);
let snapshot = snapshot.with_settings(&settings).unwrap();
let service = DefaultExecutionService::new(
@@ -4704,7 +4706,7 @@ fn request_settings_recompute_routing_and_admission_without_changing_sibling_or_
let settings = snapshot
.settings()
.clone()
.with_concurrency_queues(0, 0, 30);
.with_concurrency_queues(0, 0, 30, 30, false);
let snapshot = snapshot.with_settings(&settings).unwrap();
let snapshots = RuntimeSnapshotHandle::new(snapshot);
let admissions = Arc::new(Admissions::default());
+1
View File
@@ -46,6 +46,7 @@ tokio-tungstenite = { version = "0.29.0", default-features = false, features = [
tracing = "0.1.44"
tracing-appender = "0.2.5"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
yaml-rust2 = "0.11.0"
[dev-dependencies]
filetime = "0.2.27"
@@ -0,0 +1,241 @@
//! 单个宿主实例共享发布索引和通道详情,版本探测只读内存
use gateway_admin::model::system::{
SystemUpdateChannel as Channel, SystemUpdateCheck as Check, SystemUpdateDetail,
};
use semver::Version;
use std::{collections::VecDeque, sync::Arc, time::Duration};
use tokio::{
sync::{Mutex, RwLock},
time::{Instant, timeout},
};
use yaml_rust2::Yaml;
use super::diagnostics::Diagnostics;
use super::metadata::{Metadata, published_version};
use super::release::{
GitHubRelease, base_update_detail, detail_from_release, update_target_allowed,
};
use super::{OperationError, SystemUpdateConfig, conflict, upstream};
const NORMAL: Duration = Duration::from_secs(1200);
const REFRESH: Duration = Duration::from_secs(60);
const BUDGET: Duration = Duration::from_secs(20);
struct Entry<T> {
at: Instant,
result: Result<T, OperationError>,
}
struct Slot<T> {
value: RwLock<Option<Entry<T>>>,
fetching: Mutex<()>,
}
impl<T> Default for Slot<T> {
fn default() -> Self {
Self {
value: RwLock::new(None),
fetching: Mutex::new(()),
}
}
}
impl<T: Clone> Slot<T> {
async fn cached(&self, age: Duration) -> Option<Result<T, OperationError>> {
let value = self.value.read().await;
let entry = value.as_ref()?;
let ttl = match &entry.result {
Err(error) if error.retry_at().is_some_and(|at| at <= chrono::Utc::now()) => {
return None;
}
Err(_) => REFRESH,
Ok(_) => age,
};
(entry.at.elapsed() < ttl).then(|| entry.result.clone())
}
async fn get(
&self,
age: Duration,
fetch: impl Future<Output = Result<T, OperationError>>,
diagnostics: &Diagnostics,
) -> Result<(T, bool), OperationError> {
if let Some(result) = self.cached(age).await {
return result.map(|value| (value, true));
}
let guard = self.fetching.lock().await;
if let Some(result) = self.cached(age).await {
return result.map(|value| (value, true));
}
let result = fetch.await;
*self.value.write().await = Some(Entry {
at: Instant::now(),
result: result.clone(),
});
drop(guard);
if let Err(error) = &result {
diagnostics.record("check_release", None, error).await;
}
result.map(|value| (value, false))
}
}
type ReleaseSlot = Arc<Slot<Option<Arc<GitHubRelease>>>>;
#[derive(Default)]
pub(super) struct ReleaseCache {
metadata: Metadata,
index: Slot<Arc<Yaml>>,
releases: Mutex<VecDeque<(Version, ReleaseSlot)>>,
}
impl ReleaseCache {
pub(super) async fn local(
&self,
config: &SystemUpdateConfig,
channel: Channel,
) -> SystemUpdateDetail {
if let Some(reason) = config.update_support_error() {
return base_update_detail(config, channel, Some(reason), None);
}
let Some(index) = self.index.cached(NORMAL).await else {
return base_update_detail(config, channel, None, None);
};
let summary = index.and_then(|index| candidate(config, channel, &index));
match summary {
Ok((mut detail, target)) => {
if detail.has_update {
let slot = self
.releases
.lock()
.await
.iter()
.find(|(v, _)| v == &target)
.map(|(_, s)| Arc::clone(s));
let checked = match slot {
Some(slot) => slot.cached(NORMAL).await,
None => None,
};
match checked {
Some(Ok(Some(_))) => {}
Some(Err(error)) => {
return base_update_detail(config, channel, None, Some(&error));
}
_ => return base_update_detail(config, channel, None, None),
}
}
detail.cached = true;
detail
}
Err(error) => base_update_detail(config, channel, None, Some(&error)),
}
}
pub(super) async fn detail(
&self,
config: &SystemUpdateConfig,
channel: Channel,
check: Check,
diagnostics: &Diagnostics,
) -> Result<SystemUpdateDetail, OperationError> {
if let Some(reason) = config.update_support_error() {
return Ok(base_update_detail(config, channel, Some(reason), None));
}
let age = if check == Check::Refresh {
REFRESH
} else {
NORMAL
};
timeout(BUDGET, async {
let (index, cached) = self
.index
.get(age, self.metadata.index(config), diagnostics)
.await?;
let (mut detail, target) = candidate(config, channel, &index)?;
detail.cached = cached;
if check != Check::Summary || detail.has_update {
let (release, reused) = self.exact(config, &target, age, diagnostics).await?;
if let Some(release) = release
&& check != Check::Summary
{
detail = detail_from_release(config, &release, channel);
}
detail.cached = cached && reused;
}
Ok(detail)
})
.await
.map_err(|_| upstream("检查更新超时"))?
}
async fn exact(
&self,
config: &SystemUpdateConfig,
target: &Version,
age: Duration,
diagnostics: &Diagnostics,
) -> Result<(Option<Arc<GitHubRelease>>, bool), OperationError> {
// 四个普通通道和当前版本最多需要五份详情,相同 tag 跨通道共享
let slot = {
let mut entries = self.releases.lock().await;
let slot = entries
.iter()
.position(|(tag, _)| tag == target)
.and_then(|i| entries.remove(i))
.map(|(_, slot)| slot)
.unwrap_or_default();
if entries.len() == 5 {
entries.pop_front();
}
entries.push_back((target.clone(), Arc::clone(&slot)));
slot
};
slot.get(age, self.metadata.release(config, target), diagnostics)
.await
}
pub(super) async fn install(
&self,
config: &SystemUpdateConfig,
channel: Channel,
confirmed: &str,
diagnostics: &Diagnostics,
) -> Result<Arc<GitHubRelease>, OperationError> {
// 执行时重新核对发布索引与精确 Release,不用浏览时的成功结果替代确认
timeout(BUDGET, async {
let (index, _) = self
.index
.get(Duration::ZERO, self.metadata.index(config), diagnostics)
.await?;
let (detail, target) = candidate(config, channel, &index)?;
if !detail.has_update || detail.latest_version != confirmed {
return Err(conflict("远端最新版本已变化,请重新检查并确认"));
}
self.exact(config, &target, Duration::ZERO, diagnostics)
.await?
.0
.ok_or_else(|| upstream("发布索引指向的 Release 不存在"))
})
.await
.map_err(|_| upstream("安装前版本复核超时"))?
}
}
fn candidate(
config: &SystemUpdateConfig,
channel: Channel,
index: &Yaml,
) -> Result<(SystemUpdateDetail, Version), OperationError> {
let current = Version::parse(config.version.trim_start_matches('v'))
.map_err(|_| conflict("当前版本号无效"))?;
let target = published_version(index, &current, channel)?;
let mut detail = base_update_detail(config, channel, None, None);
if update_target_allowed(&current, &target, channel) {
detail.latest_version = target.to_string();
detail.has_update = true;
Ok((detail, target))
} else {
Ok((detail, current))
}
}
@@ -9,12 +9,11 @@ use futures::StreamExt as _;
use sha2::Digest as _;
use super::events::UpdateEvents;
use super::release::{download_client, validate_download_url};
use super::release::{
MAX_CHECKSUM_SIZE, MAX_DOWNLOAD_SIZE, download_client, validate_download_url,
};
use super::{OperationError, invalid, upstream};
pub(crate) const MAX_DOWNLOAD_SIZE: u64 = 500 * 1024 * 1024;
pub(crate) const MAX_CHECKSUM_SIZE: u64 = 1024 * 1024;
pub(crate) async fn download_file(
url: &str,
destination: &Path,
@@ -81,19 +80,18 @@ pub(crate) async fn download_file(
Ok(())
}
pub(crate) async fn verify_checksum(
file_path: &Path,
pub(crate) async fn fetch_expected_checksum(
file_name: &str,
checksum_url: &str,
checksum_size: u64,
api_base: &str,
) -> Result<(), OperationError> {
) -> Result<String, OperationError> {
validate_download_url(checksum_url, api_base)?;
if checksum_size == 0 || checksum_size > MAX_CHECKSUM_SIZE {
return Err(invalid("校验文件大小无效或超出限制"));
}
let client = download_client(api_base, Duration::from_secs(30))?;
let response = client
let mut response = client
.get(checksum_url)
.header(reqwest::header::ACCEPT_ENCODING, "identity")
.send()
@@ -105,17 +103,27 @@ pub(crate) async fn verify_checksum(
response.status().as_u16()
)));
}
let bytes = response
.bytes()
let size = checksum_size as usize;
let mut bytes = Vec::with_capacity(size);
while let Some(chunk) = response
.chunk()
.await
.map_err(|error| super::error::http("校验文件读取失败", error))?;
if bytes.len() > MAX_CHECKSUM_SIZE as usize || bytes.len() as u64 != checksum_size {
.map_err(|error| super::error::http("校验文件读取失败", error))?
{
if chunk.len() > size - bytes.len() {
return Err(invalid("校验文件大小与发行信息不一致"));
}
bytes.extend_from_slice(&chunk);
}
if bytes.len() != size {
return Err(invalid("校验文件大小与发行信息不一致"));
}
let body =
std::str::from_utf8(&bytes).map_err(|_| invalid("校验文件编码无效,无法校验更新包"))?;
let expected = checksum_for(body, file_name)
.ok_or_else(|| upstream("校验文件中缺少当前更新包的校验值"))?;
checksum_for(body, file_name).ok_or_else(|| upstream("校验文件中缺少当前更新包的校验值"))
}
pub(crate) fn verify_checksum(file_path: &Path, expected: &str) -> Result<(), OperationError> {
let actual = sha256_file(file_path)?;
if !expected.eq_ignore_ascii_case(&actual) {
return Err(upstream("更新包校验失败,请重新下载"));
@@ -0,0 +1,296 @@
//! 从固定发布索引和精确版本入口读取有界元数据
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use chrono::{DateTime, Utc};
use gateway_admin::model::system::SystemUpdateChannel as Channel;
use reqwest::{Client, StatusCode, Url};
use semver::Version;
use serde::Deserialize;
use tokio::sync::{Mutex, OnceCell};
use tokio::time::Instant;
use yaml_rust2::{
Yaml, YamlLoader,
parser::{Event, Parser},
};
use super::release::{GitHubRelease, channel_accepts, channel_for_version, download_client};
use super::{OperationError, SystemUpdateConfig, invalid, upstream};
#[derive(Default)]
pub(super) struct Metadata {
client: OnceCell<Client>,
snapshot: Mutex<Option<(Version, Arc<Yaml>)>>,
asset_until: Mutex<Option<Cooldown>>,
api_until: Mutex<Option<Cooldown>>,
}
#[derive(Clone, Copy)]
struct Cooldown {
until: Instant,
retry_at: DateTime<Utc>,
}
impl Cooldown {
fn warning(self) -> OperationError {
upstream("GitHub 暂时限制了更新检查").with_retry_at(self.retry_at)
}
}
impl Metadata {
pub(super) async fn index(
&self,
config: &SystemUpdateConfig,
) -> Result<Arc<Yaml>, OperationError> {
#[derive(Deserialize)]
struct Pointer {
current: Option<String>,
}
let pointer = self
.by_tag(config, "update-index")
.await?
.filter(|r| r.tag_name == "update-index" && !r.draft && r.prerelease)
.ok_or_else(|| upstream("更新仓库缺少公开的 update-index 指针"))?;
let pointer: Pointer = serde_json::from_str(pointer.body.as_deref().unwrap_or_default())
.map_err(|_| upstream("发行索引指针格式无效"))?;
let version = pointer
.current
.ok_or_else(|| upstream("尚未发布通道索引"))?;
let version = Version::parse(&version).map_err(|_| upstream("发行索引指针版本无效"))?;
let mut snapshot = self.snapshot.lock().await;
if let Some((previous, yaml)) = snapshot.as_ref()
&& previous == &version
{
return Ok(Arc::clone(yaml));
}
// 正文只接受版本号,下载地址始终由已配置的仓库和固定附件名构造
let mut url =
Url::parse(&config.github_api_base).map_err(|_| invalid("发行查询地址无效"))?;
if url.host_str() == Some("api.github.com") {
url = Url::parse("https://github.com").expect("固定的 GitHub 发行源");
}
url.set_path(&format!(
"/{}/releases/download/v{version}/version.yaml",
repository(config)?
));
let client = download_client(&config.github_api_base, Duration::from_secs(10))?;
let bytes = self
.read(client.get(url), &self.asset_until, 128 * 1024)
.await?
.ok_or_else(|| upstream("发行附件缺少 version.yaml"))?;
let source = std::str::from_utf8(&bytes).map_err(|_| upstream("发布索引格式无效"))?;
let bad = || upstream("发布索引格式无效");
// 远程 YAML 不使用别名,限制嵌套后再构造节点,避免小响应放大内存
let mut parser = Parser::new_from_str(source);
let mut depth = 0;
loop {
match parser.next_token().map_err(|_| bad())?.0 {
Event::StreamEnd => break,
Event::Alias(_) => return Err(bad()),
Event::MappingStart(..) | Event::SequenceStart(..) => {
depth += 1;
if depth > 8 {
return Err(bad());
}
}
Event::MappingEnd | Event::SequenceEnd => depth -= 1,
_ => {}
}
}
let mut docs = YamlLoader::load_from_str(source).map_err(|_| bad())?;
if docs.len() != 1 || docs[0]["published"].as_hash().is_none() {
return Err(upstream("发布索引缺少有效的 published 通道表"));
}
let yaml = Arc::new(docs.remove(0));
if yaml["version"].as_str() != Some(version.to_string().as_str()) {
return Err(upstream("发行索引附件与指针版本不符"));
}
*snapshot = Some((version, Arc::clone(&yaml)));
Ok(yaml)
}
pub(super) async fn release(
&self,
config: &SystemUpdateConfig,
version: &Version,
) -> Result<Option<Arc<GitHubRelease>>, OperationError> {
let stable = version.pre.is_empty();
let version = version.to_string();
let tag = format!("v{version}");
let Some(release) = self.by_tag(config, &tag).await? else {
// 当前运行版本可以没有公开说明;索引承诺的升级目标缺失属于短期查询失败
if config.version.trim_start_matches('v') == version {
return Ok(None);
}
return Err(upstream("发布索引指向的 Release 不存在"));
};
if release.tag_name != tag || release.draft || release.prerelease == stable {
return Err(upstream("GitHub 发行身份与发布索引不符"));
}
if config.version.trim_start_matches('v') != version {
release.install_assets(&version, &config.github_api_base)?;
}
Ok(Some(Arc::new(release)))
}
async fn by_tag(
&self,
config: &SystemUpdateConfig,
tag: &str,
) -> Result<Option<GitHubRelease>, OperationError> {
let mut url =
Url::parse(&config.github_api_base).map_err(|_| invalid("发行查询地址无效"))?;
url.path_segments_mut()
.map_err(|()| invalid("发行查询地址无效"))?
.pop_if_empty()
.extend(repository(config)?.split('/'))
.extend(["releases", "tags", tag]);
let client = self
.client
.get_or_try_init(|| async {
Client::builder()
.user_agent("codex-proxy-rs")
.timeout(Duration::from_secs(10))
.connect_timeout(Duration::from_secs(5))
.redirect(reqwest::redirect::Policy::none())
.retry(reqwest::retry::never())
.build()
})
.await
.map_err(|e| super::error::http("无法初始化发行查询连接", e))?;
let bytes = self
.read(
client
.get(url)
.header("Accept", "application/vnd.github+json"),
&self.api_until,
4 * 1024 * 1024,
)
.await?;
bytes
.map(|bytes| {
serde_json::from_slice(&bytes)
.map_err(|error| upstream("GitHub 发行信息格式无效").with_source(error))
})
.transpose()
}
async fn read(
&self,
request: reqwest::RequestBuilder,
cooldown: &Mutex<Option<Cooldown>>,
limit: usize,
) -> Result<Option<Vec<u8>>, OperationError> {
if let Some(active) = cooldown
.lock()
.await
.filter(|active| active.until > Instant::now())
{
return Err(active.warning());
}
let mut response = request
.send()
.await
.map_err(|e| super::error::http("无法查询 GitHub 发行版本", e))?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Ok(None);
}
if !status.is_success() {
let headers = response.headers();
let header = |name: &str| headers.get(name).and_then(|h| h.to_str().ok());
let retry = header("retry-after").and_then(|s| s.parse::<u64>().ok());
let exhausted = header("x-ratelimit-remaining") == Some("0");
if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::FORBIDDEN {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let reset = if exhausted {
header("x-ratelimit-reset")
.and_then(|s| s.parse::<u64>().ok())
.map(|v| v.saturating_sub(now))
} else {
None
};
let seconds = retry
.into_iter()
.chain(reset)
.max()
.unwrap_or(60)
.clamp(1, 86400);
let duration = Duration::from_secs(seconds);
let limit = Cooldown {
until: Instant::now() + duration,
retry_at: Utc::now() + duration,
};
let mut guard = cooldown.lock().await;
let active = guard.filter(|old| old.until > limit.until).unwrap_or(limit);
*guard = Some(active);
let warning = active.warning();
return Err(match response.error_for_status_ref() {
Err(error) => warning.with_source(error.without_url()),
Ok(_) => warning,
});
}
return Err(upstream(format!(
"GitHub 发行查询失败(HTTP {})",
status.as_u16()
)));
}
let too_large = || upstream("GitHub 发行元数据超出大小限制");
if response
.content_length()
.is_some_and(|size| size > limit as u64)
{
return Err(too_large());
}
let mut bytes = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|e| super::error::http("读取 GitHub 发行元数据失败", e))?
{
if chunk.len() > limit - bytes.len() {
return Err(too_large());
}
bytes.extend_from_slice(&chunk);
}
Ok(Some(bytes))
}
}
fn repository(config: &SystemUpdateConfig) -> Result<&str, OperationError> {
config
.update_repository
.as_deref()
.ok_or_else(|| invalid("未配置更新仓库"))
}
pub(super) fn published_version(
index: &Yaml,
current: &Version,
channel: Channel,
) -> Result<Version, OperationError> {
let line = &index["published"][current.major.to_string().as_str()];
let node = if channel == Channel::Experimental {
&line["exp"][format!("{}.{}.{}", current.major, current.minor, current.patch).as_str()]
} else {
&line[channel.as_str()]
};
let raw = node
.as_str()
.ok_or_else(|| upstream("所选更新通道尚未发布"))?;
let target = Version::parse(raw).map_err(|_| upstream("发布索引中的版本号无效"))?;
let allowed = if channel == Channel::Experimental {
channel_for_version(&target) == Some(channel)
&& (current.minor, current.patch) == (target.minor, target.patch)
} else {
channel_for_version(&target).is_some_and(|stage| channel_accepts(channel, stage))
};
if current.major != target.major || !allowed {
return Err(upstream("发布索引中的版本不属于所选通道"));
}
Ok(target)
}
@@ -3,10 +3,12 @@
mod archive;
mod config;
mod diagnostics;
mod discovery;
mod download;
mod error;
mod events;
mod installation;
mod metadata;
mod operations;
mod process;
mod release;
@@ -5,7 +5,7 @@ use std::{fs, path::Path, sync::Arc, time::Duration};
use async_trait::async_trait;
use gateway_admin::model::system::{
SystemOperationAccepted, SystemOperationKind, SystemUpdateChannel as UpdateChannel,
SystemUpdateDetail, SystemUpdateStatus, SystemVersion,
SystemUpdateCheck, SystemUpdateDetail, SystemUpdateStatus, SystemVersion,
};
use gateway_admin::ports::system::{
SystemOperationErrorKind, SystemOperations, SystemRestartPreflight, SystemUpdateCandidate,
@@ -16,22 +16,21 @@ use serde::Deserialize;
use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard, broadcast};
use super::archive::extract_release;
use super::download::{
MAX_CHECKSUM_SIZE, MAX_DOWNLOAD_SIZE, download_file, format_bytes, verify_checksum,
};
use super::discovery::ReleaseCache;
use super::download::{download_file, fetch_expected_checksum, format_bytes, verify_checksum};
use super::events::UpdateEvents;
use super::installation::{OFFICIAL_PLUGIN_MANIFEST, ReleaseFiles};
use super::process::spawn_replacement;
use super::release::{
GitHubRelease, ReleaseCache, base_update_detail, confirmed_target, detail_from_release,
fetch_latest, select_archive, update_policy, validate_update_target, version_channel,
GitHubRelease, base_update_detail, confirmed_target, update_policy, validate_update_target,
version_channel,
};
use super::state::{
OperationFileLock, UpdateOperation, UpdateTempDir, finish, operation_id, read_status,
reconcile_installation, recover_interrupted, set_running,
};
use super::swap::{replace_release_files, rollback_official_plugins_dir, rollback_release};
use super::{OperationError, SystemUpdateConfig, conflict, internal, invalid, upstream};
use super::{OperationError, SystemUpdateConfig, conflict, internal, invalid};
const MAX_OFFICIAL_PLUGIN_MANIFEST_SIZE: u64 = 256 * 1024;
@@ -180,31 +179,15 @@ impl ProcessSystemOperations {
operation_id: &str,
preflight: &dyn SystemUpdatePreflight,
) -> Result<ReleaseFiles, OperationError> {
let repository = self
.config
.update_repository
.as_deref()
.ok_or_else(|| conflict("未配置更新仓库 CPR_UPDATE_REPOSITORY"))?;
self.events.info(
Some(operation_id),
Some("release"),
"正在获取最新 Release 信息",
);
let release = fetch_latest(
&self.config.github_api_base,
repository,
&self.config.version,
channel,
)
.await?
.ok_or_else(|| conflict("当前发行通道没有可用更新"))?;
let detail = detail_from_release(&self.config, &release, channel);
if detail.latest_version != target {
return Err(conflict("远端最新版本已变化,请重新确认"));
}
if !detail.has_update {
return Err(conflict("当前没有可用更新"));
}
let release = self
.release_cache
.install(&self.config, channel, target, &self.diagnostics)
.await?;
self.events.info(
Some(operation_id),
Some("prepare"),
@@ -226,7 +209,7 @@ impl ProcessSystemOperations {
Some("asset"),
"正在选择匹配当前平台的更新包",
);
let archive = select_archive(release, version)?;
let (archive, checksum) = release.install_assets(version, &self.config.github_api_base)?;
self.events.info(
Some(operation_id),
Some("asset"),
@@ -238,25 +221,24 @@ impl ProcessSystemOperations {
);
self.events
.info(Some(operation_id), Some("verify"), "正在校验更新资源");
if archive.size == 0 || archive.size > MAX_DOWNLOAD_SIZE {
return Err(invalid("更新包大小无效或超出限制"));
}
let checksum = release
.assets
.iter()
.find(|asset| asset.name == "checksums.txt")
.ok_or_else(|| upstream("发行版本缺少 checksums.txt 校验文件,无法安全更新"))?;
if checksum.size == 0 || checksum.size > MAX_CHECKSUM_SIZE {
return Err(invalid("校验文件大小无效或超出限制"));
}
self.events
.info(Some(operation_id), Some("prepare"), "正在创建临时更新目录");
self.events.info(
Some(operation_id),
Some("prepare"),
"正在获取更新包的 SHA-256 校验值",
);
let expected_checksum = fetch_expected_checksum(
&archive.name,
&checksum.browser_download_url,
checksum.size,
&self.config.github_api_base,
)
.await?;
fs::create_dir_all(&self.config.update_temp_dir)
.map_err(|error| internal("无法创建更新临时目录").with_source(error))?;
let temp_root = fs::canonicalize(&self.config.update_temp_dir)
.map_err(|error| internal("无法确定更新临时目录位置").with_source(error))?;
let temp = UpdateTempDir::create(&temp_root)?;
let archive_path = temp.path().join(&archive.name);
let archive_path = temp.path().join("release.tar.gz");
self.events
.info(Some(operation_id), Some("download"), "开始下载更新包");
download_file(
@@ -272,14 +254,8 @@ impl ProcessSystemOperations {
.success(Some(operation_id), Some("download"), "更新包下载完成");
self.events
.info(Some(operation_id), Some("checksum"), "正在校验 checksum");
verify_checksum(
&archive_path,
&archive.name,
&checksum.browser_download_url,
checksum.size,
&self.config.github_api_base,
)
.await?;
// 安装包是最后一项远端资源,后续只使用已经固定的目标和校验值
verify_checksum(&archive_path, &expected_checksum)?;
self.events
.success(Some(operation_id), Some("checksum"), "checksum 校验通过");
self.events
@@ -368,7 +344,10 @@ impl ProcessSystemOperations {
#[async_trait]
impl SystemOperations for ProcessSystemOperations {
async fn version(&self) -> Result<SystemVersion, OperationError> {
let detail = self.update_detail(false, None).await?;
let detail = self
.release_cache
.local(&self.config, self.resolve_channel(None)?)
.await;
Ok(SystemVersion {
version: self.config.version.clone(),
git_sha: self.config.git_sha.clone(),
@@ -386,25 +365,22 @@ impl SystemOperations for ProcessSystemOperations {
async fn update_detail(
&self,
refresh: bool,
check: SystemUpdateCheck,
channel: Option<UpdateChannel>,
) -> Result<SystemUpdateDetail, OperationError> {
let channel = self.resolve_channel(channel)?;
match self
.release_cache
.detail(&self.config, refresh, channel)
.detail(&self.config, channel, check, &self.diagnostics)
.await
{
Ok(detail) => Ok(detail),
Err(error) => {
self.diagnostics.record("check_release", None, &error).await;
Ok(base_update_detail(
&self.config,
channel,
self.config.update_support_error(),
Some(error.message().to_owned()),
))
}
Err(error) => Ok(base_update_detail(
&self.config,
channel,
self.config.update_support_error(),
Some(&error),
)),
}
}
@@ -1,19 +1,19 @@
//! GitHub Release 发现、缓存、版本比较与下载信任边界
//! Release 详情、版本比较与下载信任边界
use std::env;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use std::time::Duration;
use gateway_admin::model::system::{
SystemUpdateChannel as UpdateChannel, SystemUpdateDetail, SystemUpdatePolicy,
};
use serde::Deserialize;
use tokio::sync::Mutex;
use super::{OperationError, SystemUpdateConfig, conflict, invalid, upstream};
use super::{OperationError, SystemUpdateConfig, conflict, invalid};
pub(super) const MAX_DOWNLOAD_SIZE: u64 = 500 * 1024 * 1024;
pub(super) const MAX_CHECKSUM_SIZE: u64 = 1024 * 1024;
const APP_BINARY_NAME: &str = "codex-proxy-rs";
const CACHE_TTL: Duration = Duration::from_secs(20 * 60);
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct GitHubRelease {
@@ -22,7 +22,6 @@ pub(crate) struct GitHubRelease {
pub(crate) body: Option<String>,
pub(crate) html_url: Option<String>,
pub(crate) prerelease: bool,
#[serde(default)]
pub(crate) draft: bool,
#[serde(default)]
pub(crate) assets: Vec<GitHubAsset>,
@@ -35,166 +34,42 @@ pub(crate) struct GitHubAsset {
pub(crate) size: u64,
}
struct CachedRelease {
key: String,
detail: SystemUpdateDetail,
cached_at: Instant,
}
#[derive(Default)]
pub(crate) struct ReleaseCache {
entry: Mutex<Option<CachedRelease>>,
request_sequence: AtomicU64,
}
impl ReleaseCache {
pub(crate) async fn detail(
impl GitHubRelease {
pub(super) fn install_assets(
&self,
config: &SystemUpdateConfig,
refresh: bool,
channel: UpdateChannel,
) -> Result<SystemUpdateDetail, OperationError> {
if let Some(reason) = config.update_support_error() {
return Ok(base_update_detail(config, channel, Some(reason), None));
version: &str,
api_base: &str,
) -> Result<(&GitHubAsset, &GitHubAsset), OperationError> {
let os = match env::consts::OS {
"macos" => "darwin",
os => os,
};
let arch = match env::consts::ARCH {
"x86_64" => "amd64",
"aarch64" => "arm64",
arch => arch,
};
let name = format!("{APP_BINARY_NAME}_{version}_{os}_{arch}.tar.gz");
let archive = self
.assets
.iter()
.find(|a| a.name == name)
.ok_or_else(|| conflict(format!("当前发行版本没有适用于 {os}/{arch} 的更新包")))?;
if !(1..=MAX_DOWNLOAD_SIZE).contains(&archive.size) {
return Err(invalid("更新包大小无效或超出限制"));
}
let repository = config
.update_repository
.as_deref()
.ok_or_else(|| conflict("未配置更新仓库 CPR_UPDATE_REPOSITORY"))?;
validate_repository(repository)?;
validate_api_base(&config.github_api_base).map_err(conflict)?;
let key = format!("{}|{}", config.release_cache_key(), channel.as_str());
if !refresh && let Some(detail) = self.cached(&key).await {
return Ok(detail);
}
let sequence = self.request_sequence.fetch_add(1, Ordering::Relaxed) + 1;
match fetch_latest(
&config.github_api_base,
repository,
&config.version,
channel,
)
.await
{
Ok(release) => {
let detail = release.as_ref().map_or_else(
|| base_update_detail(config, channel, config.update_support_error(), None),
|release| detail_from_release(config, release, channel),
);
let mut entry = self.entry.lock().await;
// 较慢的旧请求不能覆盖或清除新检查的结果,包括切走再切回同一通道
if self.request_sequence.load(Ordering::Relaxed) == sequence {
*entry = Some(CachedRelease {
key,
detail: detail.clone(),
cached_at: Instant::now(),
});
}
Ok(detail)
}
Err(error) => {
// 强制检查失败后清掉旧结果,避免后续版本查询再次显示旧的更新标记
let mut entry = self.entry.lock().await;
if self.request_sequence.load(Ordering::Relaxed) == sequence
&& entry.as_ref().is_some_and(|cached| cached.key == key)
{
*entry = None;
}
Err(error)
}
let checksum = self
.assets
.iter()
.find(|a| a.name == "checksums.txt")
.ok_or_else(|| super::upstream("发行版本缺少 checksums.txt 校验文件,无法安全更新"))?;
if !(1..=MAX_CHECKSUM_SIZE).contains(&checksum.size) {
return Err(invalid("校验文件大小无效或超出限制"));
}
validate_download_url(&archive.browser_download_url, api_base)?;
validate_download_url(&checksum.browser_download_url, api_base)?;
Ok((archive, checksum))
}
async fn cached(&self, key: &str) -> Option<SystemUpdateDetail> {
let entry = self.entry.lock().await;
let cached = entry.as_ref()?;
(cached.key == key && cached.cached_at.elapsed() <= CACHE_TTL).then(|| {
let mut detail = cached.detail.clone();
detail.cached = true;
detail
})
}
}
/// 优先返回可升级的最高版本,没有候选时保留当前版本的发布信息
pub(crate) async fn fetch_latest(
api_base: &str,
repository: &str,
current_version: &str,
channel: UpdateChannel,
) -> Result<Option<GitHubRelease>, OperationError> {
validate_api_base(api_base).map_err(conflict)?;
validate_repository(repository)?;
let base = api_base.trim_end_matches('/');
let current = semver::Version::parse(&normalize_version(current_version))
.map_err(|_| invalid("当前版本号无效,无法检查更新"))?;
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| super::error::http("无法初始化发行查询连接", error))?;
// 所有发行线先过滤再排序,避免更高大版本或其他实验遮住仍可安装的更新
let mut latest: Option<(semver::Version, GitHubRelease)> = None;
let mut page = 1;
loop {
let releases: Vec<GitHubRelease> = fetch_release_json(
&client,
&format!("{base}/{repository}/releases?per_page=100&page={page}"),
)
.await?;
let last_page = releases.len() < 100;
for release in releases {
let Some(version) = eligible_release_version(&release, &current, channel) else {
continue;
};
if latest
.as_ref()
.is_none_or(|(previous, _)| version.cmp_precedence(previous).is_gt())
{
latest = Some((version, release));
}
}
if last_page {
return Ok(latest.map(|(_, release)| release));
}
page += 1;
}
}
fn eligible_release_version(
release: &GitHubRelease,
current: &semver::Version,
channel: UpdateChannel,
) -> Option<semver::Version> {
let version = semver::Version::parse(&normalize_version(&release.tag_name)).ok()?;
(!release.draft
&& release.prerelease != version.pre.is_empty()
&& (version == *current || update_target_allowed(current, &version, channel)))
.then_some(version)
}
async fn fetch_release_json<T: serde::de::DeserializeOwned>(
client: &reqwest::Client,
url: &str,
) -> Result<T, OperationError> {
let response = client
.get(url)
.header(reqwest::header::ACCEPT, "application/vnd.github+json")
.header(reqwest::header::USER_AGENT, APP_BINARY_NAME)
.send()
.await
.map_err(|error| super::error::http("无法查询 GitHub 发行版本", error))?;
if !response.status().is_success() {
return Err(upstream(format!(
"GitHub 发行查询失败(HTTP {})",
response.status().as_u16()
)));
}
response
.json::<T>()
.await
.map_err(|error| super::error::http("GitHub 发行信息格式无效", error))
}
pub(crate) fn detail_from_release(
@@ -223,6 +98,7 @@ pub(crate) fn detail_from_release(
update_supported: true,
unsupported_reason: None,
warning: None,
retry_at: None,
}
}
@@ -235,36 +111,6 @@ pub(crate) fn confirmed_target(target: Option<String>) -> Result<String, Operati
Ok(target)
}
pub(crate) fn select_archive<'a>(
release: &'a GitHubRelease,
version: &str,
) -> Result<&'a GitHubAsset, OperationError> {
let os = platform_os_aliases();
let arch = platform_arch_aliases();
let normalized = normalize_version(version);
release
.assets
.iter()
.find(|asset| {
asset.name.contains(APP_BINARY_NAME)
&& asset.name.contains(&normalized)
&& asset_matches_platform(&asset.name, os, arch)
&& !asset.name.ends_with(".txt")
})
.or_else(|| {
release.assets.iter().find(|asset| {
asset_matches_platform(&asset.name, os, arch) && !asset.name.ends_with(".txt")
})
})
.ok_or_else(|| {
conflict(format!(
"当前发行版本没有适用于 {}/{} 的更新包",
env::consts::OS,
env::consts::ARCH
))
})
}
/// 校验 GitHub Release 资产 URL 及其重定向目标是否位于受信任边界内
pub fn validate_download_url(raw: &str, api_base: &str) -> Result<(), OperationError> {
let url = reqwest::Url::parse(raw)
@@ -293,7 +139,7 @@ pub(crate) fn download_client(
reqwest::Client::builder()
.timeout(timeout)
.redirect(reqwest::redirect::Policy::custom(move |attempt| {
if download_url_allowed(attempt.url(), &api_base) {
if attempt.previous().len() < 5 && download_url_allowed(attempt.url(), &api_base) {
attempt.follow()
} else {
attempt.error(std::io::Error::other(
@@ -310,6 +156,8 @@ pub(crate) fn validate_repository(repository: &str) -> Result<(), OperationError
let owner = segments.next().unwrap_or_default();
let name = segments.next().unwrap_or_default();
if owner.is_empty()
|| matches!(owner, "." | "..")
|| matches!(name, "." | "..")
|| name.is_empty()
|| segments.next().is_some()
|| !owner.chars().all(repository_character)
@@ -337,7 +185,7 @@ pub(crate) fn validate_api_base(raw: &str) -> Result<(), String> {
.then_some(())
.ok_or_else(|| "本地发行查询地址必须使用 HTTP 或 HTTPS".to_owned());
}
if url.scheme() != "https" || url.host_str() != Some("api.github.com") {
if url.scheme() != "https" || url.host_str() != Some("api.github.com") || url.port().is_some() {
return Err("发行查询地址必须为 https://api.github.com/repos".to_owned());
}
Ok(())
@@ -347,7 +195,7 @@ fn normalize_version(version: &str) -> String {
version.trim().trim_start_matches('v').to_owned()
}
fn channel_for_version(version: &semver::Version) -> Option<UpdateChannel> {
pub(super) fn channel_for_version(version: &semver::Version) -> Option<UpdateChannel> {
if version.pre.is_empty() {
return Some(UpdateChannel::Stable);
}
@@ -371,12 +219,12 @@ pub(crate) fn version_channel(version: &str) -> Option<UpdateChannel> {
}
/// 检查更新与执行更新共用同一规则;构建元数据不构成更新
fn update_target_allowed(
pub(super) fn update_target_allowed(
current: &semver::Version,
target: &semver::Version,
selected: UpdateChannel,
) -> bool {
use UpdateChannel::{Alpha, Beta, Experimental, Rc, Stable};
use UpdateChannel::Experimental;
if current.major != target.major || !target.cmp_precedence(current).is_gt() {
return false;
}
@@ -392,6 +240,11 @@ fn update_target_allowed(
&& target_channel == Experimental
&& (current.minor, current.patch) == (target.minor, target.patch);
}
channel_accepts(selected, target_channel)
}
pub(super) fn channel_accepts(selected: UpdateChannel, target_channel: UpdateChannel) -> bool {
use UpdateChannel::{Alpha, Beta, Rc, Stable};
matches!(
(selected, target_channel),
(Stable, Stable)
@@ -422,26 +275,6 @@ fn repository_character(character: char) -> bool {
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')
}
fn platform_os_aliases() -> &'static [&'static str] {
match env::consts::OS {
"macos" => &["macos", "darwin"],
"linux" => &["linux"],
_ => &[env::consts::OS],
}
}
fn platform_arch_aliases() -> &'static [&'static str] {
match env::consts::ARCH {
"x86_64" => &["x86_64", "amd64"],
"aarch64" => &["aarch64", "arm64"],
_ => &[env::consts::ARCH],
}
}
fn asset_matches_platform(name: &str, os: &[&str], arch: &[&str]) -> bool {
os.iter().any(|alias| name.contains(alias)) && arch.iter().any(|alias| name.contains(alias))
}
fn github_download_host_allowed(host: &str) -> bool {
host == "github.com"
|| host.ends_with(".github.com")
@@ -504,17 +337,6 @@ impl SystemUpdateConfig {
}
None
}
fn release_cache_key(&self) -> String {
format!(
"{}|{}|{}|{}|{}",
self.update_repository.as_deref().unwrap_or_default(),
self.github_api_base,
self.version,
self.deployment_mode,
self.build_type,
)
}
}
pub(super) fn update_policy(
@@ -536,7 +358,7 @@ pub(super) fn base_update_detail(
config: &SystemUpdateConfig,
channel: UpdateChannel,
unsupported_reason: Option<String>,
warning: Option<String>,
warning: Option<&OperationError>,
) -> SystemUpdateDetail {
SystemUpdateDetail {
policy: update_policy(config, channel),
@@ -550,6 +372,7 @@ pub(super) fn base_update_detail(
cached: false,
update_supported: unsupported_reason.is_none() && warning.is_none(),
unsupported_reason,
warning,
warning: warning.map(|error| error.message().to_owned()),
retry_at: warning.and_then(OperationError::retry_at),
}
}
@@ -7,13 +7,17 @@ use gateway_admin::model::system::SystemUpdateChannel::{Alpha, Beta, Experimenta
async fn temporary_channel_should_filter_future_releases_without_changing_running_channel() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let releases: Vec<_> = ["1.1.0", "1.2.0-rc.1", "1.3.0-beta.1", "1.4.0-alpha.1", "1.9.0-exp.9", "2.0.0"]
.into_iter().map(|version| serde_json::json!({"tag_name": format!("v{version}"), "prerelease": version.contains('-')})).collect();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(releases))
.mount(&server)
.await;
mount_index(&server, serde_json::json!({"published":{"1":{"stable":"1.1.0","rc":"1.2.0-rc.1","beta":"1.3.0-beta.1","alpha":"1.4.0-alpha.1"}}})).await;
for version in ["1.1.0", "1.2.0-rc.1", "1.3.0-beta.1", "1.4.0-alpha.1"] {
Mock::given(method("GET"))
.and(path(format!(
"/repos/owner/repository/releases/tags/v{version}"
)))
.respond_with(release_response(version, available_assets(version)))
.expect(1)
.mount(&server)
.await;
}
let service = fixture.service(&server);
for (channel, target) in [
(Stable, "1.1.0"),
@@ -23,7 +27,7 @@ async fn temporary_channel_should_filter_future_releases_without_changing_runnin
(Stable, "1.1.0"),
] {
let detail = service
.update_detail(false, Some(channel))
.update_detail(SystemUpdateCheck::Detail, Some(channel))
.await
.expect("detail");
assert_eq!(detail.latest_version, target);
@@ -34,7 +38,7 @@ async fn temporary_channel_should_filter_future_releases_without_changing_runnin
);
assert!(detail.has_update);
let cached = service
.update_detail(false, Some(channel))
.update_detail(SystemUpdateCheck::Detail, Some(channel))
.await
.expect("cached");
assert!(cached.cached);
@@ -45,28 +49,19 @@ async fn temporary_channel_should_filter_future_releases_without_changing_runnin
assert_eq!(version.latest_version, "1.1.0");
assert!(!fixture.state().exists(), "检查通道不能写入实例偏好");
let requests = server.received_requests().await.expect("requests");
assert_eq!(requests.len(), 5);
assert!(
requests
.iter()
.all(|request| request.url.path().ends_with("/releases"))
);
assert_eq!(requests.len(), 6);
}
#[tokio::test]
async fn reopening_should_infer_running_channel_and_switching_to_stable_should_not_downgrade() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(release_response("1.0.0", Vec::new()))
.mount(&server)
.await;
fixture.mount_release_once(&server, "1.0.0").await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = "1.1.0-beta.1".into();
let service = ProcessSystemOperations::new(CancellationToken::new(), config.clone());
let temporary = service
.update_detail(true, Some(Stable))
.update_detail(SystemUpdateCheck::Refresh, Some(Stable))
.await
.expect("view stable");
assert_eq!(temporary.policy.channel, Stable);
@@ -74,7 +69,7 @@ async fn reopening_should_infer_running_channel_and_switching_to_stable_should_n
assert_eq!(temporary.latest_version, "1.1.0-beta.1");
assert_eq!(
service
.update_detail(false, None)
.update_detail(SystemUpdateCheck::Detail, None)
.await
.expect("reopened")
.policy
@@ -84,7 +79,7 @@ async fn reopening_should_infer_running_channel_and_switching_to_stable_should_n
let restarted = ProcessSystemOperations::new(CancellationToken::new(), config.clone());
assert_eq!(
restarted
.update_detail(false, None)
.update_detail(SystemUpdateCheck::Detail, None)
.await
.expect("restarted")
.policy
@@ -95,7 +90,7 @@ async fn reopening_should_infer_running_channel_and_switching_to_stable_should_n
let promoted = ProcessSystemOperations::new(CancellationToken::new(), config);
assert_eq!(
promoted
.update_detail(false, None)
.update_detail(SystemUpdateCheck::Detail, None)
.await
.expect("now stable")
.policy
@@ -175,7 +170,7 @@ async fn install_should_use_confirmed_channel_and_infer_new_default_only_after_r
);
assert_eq!(
restarted
.update_detail(false, None)
.update_detail(SystemUpdateCheck::Detail, None)
.await
.expect("new default")
.policy
@@ -191,7 +186,7 @@ async fn temporary_channels_should_keep_experimental_isolation() {
let service = fixture.service(&server);
assert!(
service
.update_detail(true, Some(Experimental))
.update_detail(SystemUpdateCheck::Refresh, Some(Experimental))
.await
.is_err()
);
@@ -202,7 +197,7 @@ async fn temporary_channels_should_keep_experimental_isolation() {
for channel in [Stable, Rc, Beta, Alpha] {
assert!(
experimental
.update_detail(true, Some(channel))
.update_detail(SystemUpdateCheck::Refresh, Some(channel))
.await
.is_err()
);
@@ -216,58 +211,6 @@ async fn temporary_channels_should_keep_experimental_isolation() {
);
}
#[tokio::test]
async fn slower_check_should_not_overwrite_newer_cached_candidate() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(release_response("1.1.0", Vec::new()).set_delay(Duration::from_millis(150)))
.up_to_n_times(1)
.with_priority(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(release_response("1.2.0", Vec::new()))
.mount(&server)
.await;
let service = fixture.service(&server);
let first_service = service.clone();
let first = tokio::spawn(async move { first_service.update_detail(true, None).await });
tokio::time::timeout(Duration::from_secs(1), async {
while server
.received_requests()
.await
.expect("requests")
.is_empty()
{
tokio::task::yield_now().await;
}
})
.await
.expect("first request sent");
assert_eq!(
service
.update_detail(true, None)
.await
.expect("new check")
.latest_version,
"1.2.0"
);
assert_eq!(
first
.await
.expect("join")
.expect("old check")
.latest_version,
"1.1.0"
);
let cached = service.update_detail(false, None).await.expect("cache");
assert!(cached.cached);
assert_eq!(cached.latest_version, "1.2.0");
}
#[tokio::test]
async fn inferred_prerelease_policy_should_follow_future_minor_and_patch_releases() {
for (current, target) in [
@@ -283,7 +226,10 @@ async fn inferred_prerelease_policy_should_follow_future_minor_and_patch_release
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = current.to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("check");
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("check");
assert!(detail.has_update);
assert_eq!(detail.latest_version, target);
assert_eq!(
@@ -0,0 +1,551 @@
//! 用本地 HTTP 源验证请求预算、限流隔离与执行前复核
use super::*;
use gateway_admin::model::system::SystemUpdateChannel;
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn discovery_request_budget_is_constant_under_load() {
for history in [1, 100, 101, 10_000] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = Arc::new(fixture.service(&server));
let target = format!("1.1.{history}");
let releases: Vec<_> = (0..history)
.map(|i| {
let tag = if i == 0 {
format!("v{target}")
} else {
format!("v1.0.{i}")
};
serde_json::json!({"tag_name":tag, "draft":false, "prerelease":false})
})
.collect();
Mock::given(path("/repos/owner/repository/releases"))
.respond_with(move |req: &wiremock::Request| {
let page = req
.url
.query_pairs()
.find(|(k, _)| k == "page")
.and_then(|(_, v)| v.parse::<usize>().ok())
.unwrap_or(1);
ResponseTemplate::new(200).set_body_json(
releases
.iter()
.skip(page.saturating_sub(1) * 100)
.take(100)
.collect::<Vec<_>>(),
)
})
.expect(0)
.mount(&server)
.await;
let mut snapshot = fixture_index(&target);
if history >= 1000 {
// 实际快照同时保留一千条实验基线,列表中的历史发行仍不应被读取
snapshot["published"]["1"]["exp"] = (0..1000)
.map(|i| {
(
format!("1.2.{i}"),
serde_json::Value::from(format!("1.2.{i}-exp.1")),
)
})
.collect::<serde_json::Map<_, _>>()
.into();
}
mount_pointer(&server).await;
Mock::given(path(INDEX_PATH))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(snapshot)
.set_delay(Duration::from_millis(80)),
)
.expect(1)
.mount(&server)
.await;
Mock::given(path(format!(
"/repos/owner/repository/releases/tags/v{target}"
)))
.respond_with(
release_response(&target, available_assets(&target))
.set_delay(Duration::from_millis(30)),
)
.expect(1)
.mount(&server)
.await;
let started = std::time::Instant::now();
assert!(!service.version().await.unwrap().update_cached);
assert!(server.received_requests().await.unwrap().is_empty());
let summary = service
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap();
assert_eq!(summary.latest_version, target);
assert!(summary.notes.is_none());
assert_eq!(server.received_requests().await.unwrap().len(), 3);
let mut callers = tokio::task::JoinSet::new();
for i in 0..512 {
let service = service.clone();
callers.spawn(async move {
let check = if i % 2 == 0 {
SystemUpdateCheck::Refresh
} else {
SystemUpdateCheck::Detail
};
service.update_detail(check, None).await.unwrap()
});
}
while let Some(detail) = callers.join_next().await {
let detail = detail.unwrap();
assert_eq!(detail.latest_version, target);
assert_eq!(detail.notes.as_deref(), Some("notes"));
}
for _ in 0..64 {
assert!(service.version().await.unwrap().has_update);
}
assert_eq!(server.received_requests().await.unwrap().len(), 3);
eprintln!(
"history={history} concurrent_detail=512 local_probes=65 pointer=1 snapshot=1 release=1 lists=0 elapsed={:?}",
started.elapsed()
);
}
}
#[tokio::test]
async fn cold_summary_stampede_and_slow_source_do_not_block_runtime_probes() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = Arc::new(fixture.service(&server));
let (started, mut observed) = tokio::sync::mpsc::unbounded_channel();
mount_pointer(&server).await;
Mock::given(path(INDEX_PATH))
.respond_with(move |_: &wiremock::Request| {
started.send(()).unwrap();
ResponseTemplate::new(200)
.set_body_json(fixture_index(TARGET_VERSION))
.set_delay(Duration::from_millis(500))
})
.expect(1)
.mount(&server)
.await;
Mock::given(path(format!(
"/repos/owner/repository/releases/tags/v{TARGET_VERSION}"
)))
.respond_with(release_response(
TARGET_VERSION,
available_assets(TARGET_VERSION),
))
.expect(1)
.mount(&server)
.await;
let mut callers = tokio::task::JoinSet::new();
for _ in 0..256 {
let service = service.clone();
callers.spawn(async move {
service
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap()
});
}
observed.recv().await.unwrap();
tokio::time::timeout(Duration::from_millis(150), async {
for _ in 0..128 {
let version = service.version().await.unwrap();
assert_eq!(version.version, "1.0.0");
assert!(!version.update_cached);
}
})
.await
.expect("本地版本探测不能等待在途的 GitHub 查询");
while let Some(summary) = callers.join_next().await {
assert!(summary.unwrap().has_update);
}
assert_eq!(server.received_requests().await.unwrap().len(), 3);
}
#[tokio::test]
async fn api_cooldown_is_shared_and_summary_does_not_advertise_an_unverified_release() {
for status in [403, 429] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_index(
&server,
serde_json::json!({"published":{"1":{"stable":"1.1.0","beta":"1.2.0-beta.1"}}}),
)
.await;
let limit = if status == 429 {
ResponseTemplate::new(status).insert_header("retry-after", "300")
} else {
let reset = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
+ 300;
ResponseTemplate::new(status)
.insert_header("x-ratelimit-remaining", "0")
.insert_header("x-ratelimit-reset", reset.to_string())
};
Mock::given(path("/repos/owner/repository/releases/tags/v1.1.0"))
.respond_with(limit)
.expect(1)
.mount(&server)
.await;
let first = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.unwrap();
assert!(first.warning.is_some());
let retry_at = first.retry_at.expect("限流应提供重试时间");
assert!((298..=300).contains(&(retry_at - chrono::Utc::now()).num_seconds()));
for _ in 0..32 {
let second = service
.update_detail(SystemUpdateCheck::Refresh, Some(SystemUpdateChannel::Beta))
.await
.unwrap();
assert_eq!(second.warning, first.warning);
assert_eq!(second.retry_at, Some(retry_at));
assert!(!second.has_update);
}
assert!(
!service
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap()
.has_update
);
assert_eq!(server.received_requests().await.unwrap().len(), 3);
}
}
#[tokio::test]
async fn invalid_or_missing_published_data_does_not_fallback_to_build_version_or_lists() {
for body in [
"version: '1.9.9'".to_owned(),
"version: '9.0.0'\npublished: {'2': {stable: '2.0.0'}}".to_owned(),
format!("version: '{SNAPSHOT}'\npublished: {{'1': {{stable: '2.0.0'}}}}"),
format!("version: '{SNAPSHOT}'\npublished: {{'1': {{stable: '1.2.0-rc.1'}}}}"),
"bad: &loop [1]\npublished: *loop".to_owned(),
"x".repeat(128 * 1024 + 1),
] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_pointer(&server).await;
Mock::given(path(INDEX_PATH))
.respond_with(ResponseTemplate::new(200).set_body_string(body))
.expect(1)
.mount(&server)
.await;
for _ in 0..2 {
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.unwrap();
assert!(detail.warning.is_some());
assert!(!detail.has_update);
}
assert_eq!(server.received_requests().await.unwrap().len(), 2);
}
}
#[tokio::test]
async fn exact_release_requires_public_matching_identity_before_display() {
for (status, tag, draft, prerelease) in [
(404, "", false, false),
(200, "v1.2.0", false, false),
(200, "v1.1.0", true, false),
(200, "v1.1.0", false, true),
] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_index(&server, fixture_index("1.1.0")).await;
Mock::given(path("/repos/owner/repository/releases/tags/v1.1.0"))
.respond_with(ResponseTemplate::new(status).set_body_json(
serde_json::json!({"tag_name":tag,"draft":draft,"prerelease":prerelease}),
))
.mount(&server)
.await;
let detail = service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap();
assert!(!detail.has_update);
assert!(detail.warning.is_some());
}
}
#[tokio::test]
async fn install_rechecks_even_a_warm_candidate_and_never_downloads_a_changed_target() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
let requests = AtomicUsize::new(0);
Mock::given(path(POINTER_PATH))
.respond_with(move |_: &wiremock::Request| {
pointer_response(if requests.fetch_add(1, Ordering::SeqCst) == 0 {
SNAPSHOT
} else {
"4.0.0"
})
})
.expect(2)
.mount(&server)
.await;
for (path_name, latest, target) in [
(INDEX_PATH, SNAPSHOT, "1.1.0"),
(
"/owner/repository/releases/download/v4.0.0/version.yaml",
"4.0.0",
"1.2.0",
),
] {
let mut index = fixture_index(target);
index["version"] = latest.into();
Mock::given(path(path_name))
.respond_with(ResponseTemplate::new(200).set_body_json(index))
.expect(1)
.mount(&server)
.await;
}
Mock::given(path("/repos/owner/repository/releases/tags/v1.1.0"))
.respond_with(release_response("1.1.0", available_assets("1.1.0")))
.expect(1)
.mount(&server)
.await;
let detail = service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap();
assert!(detail.has_update);
let status = complete_update(&service, &detail.latest_version).await;
assert_eq!(status.operation.status, SystemOperationStatus::Failed);
assert!(status.operation.error.unwrap().contains("重新检查并确认"));
assert_eq!(server.received_requests().await.unwrap().len(), 5);
}
#[tokio::test]
async fn channels_pointing_to_one_release_share_its_detail() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_release_json(
&server,
TARGET_VERSION,
available_assets(TARGET_VERSION),
None,
)
.await;
for channel in [
SystemUpdateChannel::Stable,
SystemUpdateChannel::Rc,
SystemUpdateChannel::Beta,
SystemUpdateChannel::Alpha,
] {
assert!(
service
.update_detail(SystemUpdateCheck::Refresh, Some(channel))
.await
.unwrap()
.has_update
);
}
assert_eq!(server.received_requests().await.unwrap().len(), 3);
}
#[tokio::test]
async fn transient_release_absence_expires_and_refresh_interval_is_bounded() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_pointer(&server).await;
Mock::given(path(INDEX_PATH))
.respond_with(ResponseTemplate::new(200).set_body_json(fixture_index(TARGET_VERSION)))
.expect(1)
.mount(&server)
.await;
let calls = AtomicUsize::new(0);
Mock::given(path(format!(
"/repos/owner/repository/releases/tags/v{TARGET_VERSION}"
)))
.respond_with(move |_: &wiremock::Request| {
if calls.fetch_add(1, Ordering::SeqCst) == 0 {
ResponseTemplate::new(404)
} else {
release_response(TARGET_VERSION, available_assets(TARGET_VERSION))
}
})
.expect(3)
.mount(&server)
.await;
assert!(
service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap()
.warning
.is_some()
);
assert!(
service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.unwrap()
.warning
.is_some()
);
// 只推进本进程缓存时钟,实际 HTTP 连接使用正常计时
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
assert!(
service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap()
.has_update
);
tokio::time::pause();
tokio::time::advance(Duration::from_secs(61)).await;
tokio::time::resume();
assert!(
service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap()
.cached
);
assert!(
!service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.unwrap()
.cached
);
assert!(
service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.unwrap()
.cached
);
}
#[tokio::test]
async fn metadata_never_follows_redirects_or_accepts_oversized_release() {
for status in [302, 200] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_index(&server, fixture_index(TARGET_VERSION)).await;
let response = if status == 302 {
ResponseTemplate::new(status)
.insert_header("location", format!("{}/untrusted", server.uri()))
} else {
ResponseTemplate::new(200).set_body_bytes(vec![b' '; 4 * 1024 * 1024 + 1])
};
Mock::given(path(format!(
"/repos/owner/repository/releases/tags/v{TARGET_VERSION}"
)))
.respond_with(response)
.expect(1)
.mount(&server)
.await;
let detail = service
.update_detail(SystemUpdateCheck::Detail, None)
.await
.unwrap();
assert!(!detail.has_update);
assert!(detail.warning.unwrap().contains(if status == 302 {
"HTTP 302"
} else {
"大小限制"
}));
assert_eq!(server.received_requests().await.unwrap().len(), 3);
}
}
#[tokio::test]
async fn slow_metadata_times_out_without_retry_or_blocking_future_version_reads() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = Arc::new(fixture.service(&server));
let (sent, mut received) = tokio::sync::mpsc::unbounded_channel();
mount_pointer(&server).await;
Mock::given(path(INDEX_PATH))
.respond_with(move |_: &wiremock::Request| {
sent.send(()).unwrap();
ResponseTemplate::new(200).set_delay(Duration::from_secs(30))
})
.expect(1)
.mount(&server)
.await;
let worker = Arc::clone(&service);
let pending = tokio::spawn(async move {
worker
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap()
});
received.recv().await.unwrap();
tokio::time::pause();
tokio::time::advance(Duration::from_secs(11)).await;
let detail = pending.await.unwrap();
tokio::time::resume();
assert!(detail.warning.is_some());
assert!(!detail.has_update);
assert_eq!(service.version().await.unwrap().version, "1.0.0");
assert_eq!(server.received_requests().await.unwrap().len(), 2);
}
#[tokio::test]
async fn summary_and_local_probe_require_downloadable_target() {
for assets in [
vec![],
available_assets("1.2.999"),
vec![available_assets(TARGET_VERSION).remove(0)],
{
let mut assets = available_assets(TARGET_VERSION);
assets[1]["size"] = 0.into();
assets
},
] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
mount_release_json(&server, TARGET_VERSION, assets, None).await;
let summary = service
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap();
assert!(summary.warning.is_some());
assert!(!summary.has_update);
assert!(!service.version().await.unwrap().has_update);
}
}
#[tokio::test]
async fn unpublished_or_untrusted_pointer_cannot_redirect_discovery() {
for (draft, current) in [
(true, Some("3.42.0")),
(false, None),
(false, Some("https://example.com/v1")),
] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let service = fixture.service(&server);
Mock::given(path(POINTER_PATH)).respond_with(ResponseTemplate::new(200).set_body_json(
serde_json::json!({"tag_name":"update-index","prerelease":true,"draft":draft,
"body":serde_json::json!({"current":current,"pending":{"version":"3.42.0"}}).to_string()})
)).expect(1).mount(&server).await;
let detail = service
.update_detail(SystemUpdateCheck::Summary, None)
.await
.unwrap();
assert!(!detail.has_update);
assert!(detail.warning.is_some());
assert_eq!(server.received_requests().await.unwrap().len(), 1);
}
}
@@ -12,3 +12,67 @@ fn update_should_trust_github_release_asset_redirect_host() {
.is_ok()
);
}
#[tokio::test]
async fn archive_is_the_last_remote_request_before_activation() {
let server = MockServer::start().await;
let fixture = Fixture::new();
fixture
.mount_release(
&server,
TARGET_VERSION,
ArchiveKind::Safe,
ChecksumKind::Valid,
)
.await;
let service = fixture.service(&server);
let installed = complete_update(&service, TARGET_VERSION).await;
assert_eq!(installed.operation.status, SystemOperationStatus::Succeeded);
assert!(installed.need_restart);
// 从磁盘恢复和探测新版本同样不能在下载之后依赖 GitHub
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = TARGET_VERSION.to_owned();
let activated = ProcessSystemOperations::new(CancellationToken::new(), config);
assert_eq!(activated.version().await.unwrap().version, TARGET_VERSION);
assert!(!activated.update_status().await.unwrap().need_restart);
let requests = server.received_requests().await.unwrap();
let paths: Vec<_> = requests.iter().map(|req| req.url.path()).collect();
assert_eq!(
paths,
[
POINTER_PATH,
INDEX_PATH,
&format!("/repos/owner/repository/releases/tags/v{TARGET_VERSION}"),
"/checksums",
"/archive"
]
);
}
#[tokio::test]
async fn rate_limited_checksums_leave_the_package_undownloaded_and_installation_untouched() {
let server = MockServer::start().await;
let fixture = Fixture::new();
fixture
.mount_custom_release(
&server,
&format!("{}/archive", server.uri()),
Some(&format!("{}/limited-checksums", server.uri())),
)
.await;
Mock::given(path("/limited-checksums"))
.respond_with(ResponseTemplate::new(429))
.expect(1)
.mount(&server)
.await;
Mock::given(path("/archive"))
.respond_with(ResponseTemplate::new(200))
.expect(0)
.mount(&server)
.await;
let status = complete_update(&fixture.service(&server), TARGET_VERSION).await;
assert_eq!(status.operation.status, SystemOperationStatus::Failed);
assert!(status.operation.error.unwrap().contains("HTTP 429"));
assert_eq!(fs::read(fixture.executable()).unwrap(), b"old-binary");
assert!(!status.need_restart);
}
@@ -33,7 +33,7 @@ async fn manual_image_replacement_should_clear_pending_state_and_invalid_backup(
);
assert!(
redeployed
.update_detail(true, None)
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("check")
.has_update
@@ -1,6 +1,7 @@
//! 系统更新测试入口与发行包、前置校验和本地安装夹具
mod channels;
mod discovery;
mod download;
mod events;
mod installation;
@@ -23,7 +24,7 @@ use futures::StreamExt as _;
use gateway_admin::model::{
Revision,
system::{
SystemOperationAccepted, SystemOperationKind, SystemOperationStatus,
SystemOperationAccepted, SystemOperationKind, SystemOperationStatus, SystemUpdateCheck,
SystemUpdateEventLevel, SystemUpdateStatus,
},
};
@@ -37,7 +38,7 @@ use gateway_host::system_update::{
};
use sha2::{Digest as _, Sha256};
use tar::{Builder, EntryType, Header};
use wiremock::matchers::{method, path, query_param};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
const BINARY_NAME: &str = if cfg!(windows) {
@@ -361,13 +362,7 @@ impl Fixture {
}
async fn mount_release_once(&self, server: &MockServer, version: &str) {
let assets = Vec::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(release_response(version, assets))
.up_to_n_times(1)
.mount(server)
.await;
mount_release_json(server, version, available_assets(version), Some(1)).await;
}
async fn mount_custom_release(
@@ -378,23 +373,17 @@ impl Fixture {
) {
let archive = release_archive(ArchiveKind::Safe);
let name = archive_name(TARGET_VERSION);
let mut assets = vec![serde_json::json!({
"name": name,
"browser_download_url": archive_url,
"size": archive.len(),
})];
if let Some(checksum_url) = checksum_url {
assets.push(serde_json::json!({
"name": "checksums.txt",
"browser_download_url": checksum_url,
"size": 80,
}));
} else {
assets.push(serde_json::json!({
"name": "checksums.txt",
"browser_download_url": format!("{}/checksums", server.uri()),
"size": 80,
}));
let checksums = format!("{} {name}\n", hex::encode(Sha256::digest(&archive)));
let local_url = format!("{}/checksums", server.uri());
let assets = vec![
serde_json::json!({"name": name, "browser_download_url": archive_url, "size": archive.len()}),
serde_json::json!({"name": "checksums.txt", "browser_download_url": checksum_url.unwrap_or(&local_url), "size": checksums.len()}),
];
if checksum_url.is_none() {
Mock::given(path("/checksums"))
.respond_with(ResponseTemplate::new(200).set_body_string(checksums))
.mount(server)
.await;
}
mount_release_json(server, TARGET_VERSION, assets, None).await;
}
@@ -495,11 +484,17 @@ fn append_file(
}
fn archive_name(version: &str) -> String {
format!(
"codex-proxy-rs-{version}-{}-{}.tar.gz",
std::env::consts::OS,
std::env::consts::ARCH
)
let os = if cfg!(target_os = "macos") {
"darwin"
} else {
std::env::consts::OS
};
let arch = if cfg!(target_arch = "aarch64") {
"arm64"
} else {
"amd64"
};
format!("codex-proxy-rs_{version}_{os}_{arch}.tar.gz")
}
async fn mount_release_json(
@@ -508,22 +503,64 @@ async fn mount_release_json(
assets: Vec<serde_json::Value>,
times: Option<u64>,
) {
mount_index(server, fixture_index(version)).await;
let mock = Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(path(format!(
"/repos/owner/repository/releases/tags/v{version}"
)))
.respond_with(release_response(version, assets));
match times {
Some(times) => mock.up_to_n_times(times).mount(server).await,
Some(n) => mock.up_to_n_times(n).mount(server).await,
None => mock.mount(server).await,
}
}
const POINTER_PATH: &str = "/repos/owner/repository/releases/tags/update-index";
const SNAPSHOT: &str = "4.0.0-beta.1";
const INDEX_PATH: &str = "/owner/repository/releases/download/v4.0.0-beta.1/version.yaml";
fn pointer_response(version: &str) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_json(serde_json::json!({
"tag_name":"update-index", "draft":false, "prerelease":true,
"body":serde_json::json!({"current":version}).to_string()
}))
}
async fn mount_pointer(server: &MockServer) {
Mock::given(path(POINTER_PATH))
.respond_with(pointer_response(SNAPSHOT))
.mount(server)
.await;
}
fn available_assets(version: &str) -> Vec<serde_json::Value> {
[archive_name(version), "checksums.txt".to_owned()].into_iter()
.map(|name| serde_json::json!({"name":name,"size":128,"browser_download_url":"https://github.com/owner/repository/releases/download/test/file"})).collect()
}
fn fixture_index(version: &str) -> serde_json::Value {
let major = version.split('.').next().unwrap();
let base = version.split('-').next().unwrap();
serde_json::json!({"version":SNAPSHOT, "published": {major: {"stable": version,"beta":version,"rc":version,"alpha":version,"exp":{base: version}}}})
}
async fn mount_index(server: &MockServer, mut index: serde_json::Value) {
mount_pointer(server).await;
index["version"] = SNAPSHOT.into();
Mock::given(method("GET"))
.and(path(INDEX_PATH))
.respond_with(ResponseTemplate::new(200).set_body_json(index))
.mount(server)
.await;
}
fn release_response(version: &str, assets: Vec<serde_json::Value>) -> ResponseTemplate {
let prerelease = !semver::Version::parse(version)
.expect("version")
.pre
.is_empty();
let release = serde_json::json!({
"tag_name": format!("v{version}"),
"draft": false, "tag_name": format!("v{version}"),
"name": format!("Release {version}"),
"body": "notes",
"html_url": "https://github.com/owner/repository/releases/latest",
@@ -531,7 +568,7 @@ fn release_response(version: &str, assets: Vec<serde_json::Value>) -> ResponseTe
"published_at": "2026-07-19T00:00:00Z",
"assets": assets,
});
ResponseTemplate::new(200).set_body_json(serde_json::json!([release]))
ResponseTemplate::new(200).set_body_json(release)
}
struct RejectingRestartPreflight;
@@ -793,7 +793,7 @@ async fn prerelease_updates_should_install_later_stages_and_stable_release() {
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
assert!(
service
.update_detail(true, None)
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("detail")
.has_update
@@ -5,8 +5,11 @@ use super::*;
#[tokio::test]
async fn update_detail_should_record_decode_failure_while_returning_safe_warning() {
let server = MockServer::start().await;
mount_index(&server, fixture_index(TARGET_VERSION)).await;
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(path(format!(
"/repos/owner/repository/releases/tags/v{TARGET_VERSION}"
)))
.respond_with(ResponseTemplate::new(200).set_body_string("not release JSON"))
.mount(&server)
.await;
@@ -16,7 +19,7 @@ async fn update_detail_should_record_decode_failure_while_returning_safe_warning
service.set_diagnostics(diagnostics.clone());
let detail = service
.update_detail(true, None)
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("failure detail");
assert_eq!(detail.warning.as_deref(), Some("GitHub 发行信息格式无效"));
@@ -28,71 +31,6 @@ async fn update_detail_should_record_decode_failure_while_returning_safe_warning
assert!(!details.as_str().contains(&server.uri()));
}
#[tokio::test]
async fn update_detail_should_report_refresh_failure_and_clear_previous_update() {
let server = MockServer::start().await;
let fixture = Fixture::new();
fixture.mount_release_once(&server, TARGET_VERSION).await;
let service = fixture.service(&server);
service
.update_detail(true, None)
.await
.expect("prime cache");
let detail = service
.update_detail(true, None)
.await
.expect("failure detail");
assert!(!detail.has_update);
assert_eq!(
detail.warning.as_deref(),
Some("GitHub 发行查询失败(HTTP 404)")
);
assert!(!detail.cached);
assert_eq!(detail.latest_version, "1.0.0");
assert!(detail.notes.is_none());
let version = service
.version()
.await
.expect("version after failed refresh");
assert!(!version.has_update);
assert!(version.update_warning.is_some());
}
#[tokio::test]
async fn update_detail_should_offer_latest_non_draft_release_in_current_channel() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
{
"tag_name": "v1.1.0-beta.3",
"name": "Draft 1.1.0-beta.3",
"prerelease": true,
"draft": true,
"assets": [],
},
{
"tag_name": "v1.1.0-beta.2",
"name": "Release 1.1.0-beta.2",
"body": "notes",
"html_url": "https://github.com/owner/repository/releases",
"prerelease": true,
"assets": [],
},
])))
.mount(&server)
.await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = "1.1.0-beta.1".to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
assert_eq!(detail.latest_version, "1.1.0-beta.2");
assert!(detail.has_update);
}
#[tokio::test]
async fn update_detail_should_reject_untrusted_github_api_base() {
let fixture = Fixture::new();
@@ -102,7 +40,7 @@ async fn update_detail_should_reject_untrusted_github_api_base() {
);
let detail = service
.update_detail(true, None)
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("safe rejection");
assert!(!detail.update_supported);
@@ -117,7 +55,10 @@ async fn update_detail_should_withhold_updates_for_source_builds() {
config.build_type = "source".to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("detail");
assert_eq!(detail.latest_version, "1.0.0");
assert!(!detail.update_supported);
assert!(detail.unsupported_reason.is_some());
@@ -134,90 +75,6 @@ async fn update_detail_should_withhold_updates_for_source_builds() {
);
}
#[tokio::test]
async fn experimental_build_should_ignore_newer_stable_and_other_experimental_releases() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
{ "tag_name": "v4.0.0", "prerelease": false },
{ "tag_name": "v3.12.0", "prerelease": false },
{ "tag_name": "v3.10.0", "prerelease": false },
{ "tag_name": "v3.11.0-exp.3", "prerelease": true },
{ "tag_name": "v3.10.1-exp.3", "prerelease": true },
{ "tag_name": "v3.11.0-beta.1", "prerelease": true },
{ "tag_name": "v3.10.0-exp.other.3", "prerelease": true },
{ "tag_name": "v3.10.0-exp.3", "prerelease": true, "draft": true },
{ "tag_name": "v3.10.0-exp.4", "prerelease": false },
{ "tag_name": "invalid", "prerelease": true }
])))
.expect(1)
.mount(&server)
.await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = "3.10.0-exp.2".to_owned();
config.build_type = "experimental".to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
assert_eq!(detail.latest_version, "3.10.0-exp.2");
assert!(!detail.has_update);
assert!(detail.update_supported);
assert!(detail.notes.is_none());
assert!(detail.release_url.is_none());
assert!(detail.warning.is_none());
let version = service.version().await.expect("version");
assert!(!version.has_update);
assert_eq!(version.update_channel, "exp");
let cached = service
.update_detail(false, None)
.await
.expect("cached detail");
assert!(cached.cached);
assert!(!cached.has_update);
}
#[tokio::test]
async fn experimental_update_should_find_highest_same_channel_version_across_pages() {
let server = MockServer::start().await;
let fixture = Fixture::new();
let mut first_page = vec![
serde_json::json!({
"tag_name": "v3.12.0", "prerelease": false
});
100
];
first_page[1] = serde_json::json!({ "tag_name": "v3.10.0-exp.3", "prerelease": true });
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(query_param("page", "1"))
.and(query_param("per_page", "100"))
.respond_with(ResponseTemplate::new(200).set_body_json(first_page))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(query_param("page", "2"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
{ "tag_name": "v3.10.0-exp.10", "prerelease": true, "body": "exp.10 notes" },
{ "tag_name": "v3.10.0-exp.2", "prerelease": true }
])))
.mount(&server)
.await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = "3.10.0-exp.2".to_owned();
config.build_type = "experimental".to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
assert_eq!(detail.latest_version, "3.10.0-exp.10");
assert_eq!(detail.notes.as_deref(), Some("exp.10 notes"));
assert!(detail.has_update);
assert!(detail.update_supported);
assert!(service.version().await.expect("version").has_update);
}
#[tokio::test]
async fn update_checks_should_follow_stage_promotion_and_experimental_isolation() {
let cases = [
@@ -245,7 +102,10 @@ async fn update_checks_should_follow_stage_promotion_and_experimental_isolation(
config.version = current.to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("detail");
assert_eq!(detail.has_update, allowed, "{current} -> {target}");
assert_eq!(
detail.latest_version,
@@ -253,7 +113,7 @@ async fn update_checks_should_follow_stage_promotion_and_experimental_isolation(
);
assert_eq!(detail.notes.is_some(), allowed);
assert_eq!(detail.release_url.is_some(), allowed);
assert!(detail.warning.is_none());
assert_eq!(detail.warning.is_none(), allowed);
let version = service.version().await.expect("version");
assert_eq!(version.has_update, allowed);
assert_eq!(version.update_channel, channel);
@@ -274,16 +134,19 @@ async fn update_detail_should_keep_current_release_notes_without_allowing_reinst
let server = MockServer::start().await;
let fixture = Fixture::new();
let release_url = format!("https://github.com/owner/repository/releases/tag/v{current}");
mount_index(&server, fixture_index(current)).await;
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
.and(path(format!(
"/repos/owner/repository/releases/tags/v{current}"
)))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!(
{
"tag_name": format!("v{current}"),
"draft": false, "tag_name": format!("v{current}"),
"prerelease": !semver::Version::parse(current).expect("version").pre.is_empty(),
"body": "## 当前版本\n\n- 修复更新状态",
"html_url": release_url,
}
])))
)))
.expect(1)
.mount(&server)
.await;
@@ -295,7 +158,7 @@ async fn update_detail_should_keep_current_release_notes_without_allowing_reinst
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service
.update_detail(true, None)
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("current release");
assert_eq!(detail.latest_version, current);
@@ -307,7 +170,7 @@ async fn update_detail_should_keep_current_release_notes_without_allowing_reinst
);
assert_eq!(detail.release_url.as_deref(), Some(release_url.as_str()));
let cached = service
.update_detail(false, None)
.update_detail(SystemUpdateCheck::Detail, None)
.await
.expect("cached release");
assert!(cached.cached);
@@ -328,103 +191,6 @@ async fn update_detail_should_keep_current_release_notes_without_allowing_reinst
}
}
#[tokio::test]
async fn update_detail_should_prefer_available_update_notes_over_current_release() {
for current_first in [true, false] {
let server = MockServer::start().await;
let fixture = Fixture::new();
let mut releases = vec![
serde_json::json!({
"tag_name": "v1.0.0", "prerelease": false, "body": "current notes"
}),
serde_json::json!({
"tag_name": "v1.9.9", "prerelease": false, "body": "update notes",
"html_url": "https://github.com/owner/repository/releases/tag/v1.9.9"
}),
];
if !current_first {
releases.reverse();
}
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(releases))
.mount(&server)
.await;
let detail = fixture
.service(&server)
.update_detail(true, None)
.await
.expect("detail");
assert!(detail.has_update);
assert_eq!(detail.latest_version, TARGET_VERSION);
assert_eq!(detail.notes.as_deref(), Some("update notes"));
assert_eq!(
detail.release_url.as_deref(),
Some("https://github.com/owner/repository/releases/tag/v1.9.9")
);
}
}
#[tokio::test]
async fn update_detail_should_find_current_release_notes_across_pages() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(query_param("page", "1"))
.respond_with(ResponseTemplate::new(200).set_body_json(vec![
serde_json::json!({ "tag_name": "v2.0.0", "prerelease": false });
100
]))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.and(query_param("page", "2"))
.respond_with(release_response("1.0.0", Vec::new()))
.expect(1)
.mount(&server)
.await;
let detail = fixture
.service(&server)
.update_detail(true, None)
.await
.expect("detail");
assert!(!detail.has_update);
assert_eq!(detail.latest_version, "1.0.0");
assert_eq!(detail.notes.as_deref(), Some("notes"));
assert!(detail.release_url.is_some());
}
#[tokio::test]
async fn update_detail_should_not_use_draft_or_mislabeled_current_release_notes() {
for (current, prerelease) in [("1.0.0", false), ("1.0.0-beta.1", true)] {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
{ "tag_name": format!("v{current}"), "prerelease": prerelease, "draft": true, "body": "draft notes" },
{ "tag_name": format!("v{current}"), "prerelease": !prerelease, "body": "mislabeled notes" },
{ "tag_name": "v0.9.0", "prerelease": false, "body": "older notes" },
])))
.mount(&server)
.await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = current.to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
assert!(!detail.has_update);
assert_eq!(detail.latest_version, current);
assert!(detail.notes.is_none());
assert!(detail.release_url.is_none());
}
}
#[tokio::test]
async fn update_checks_should_ignore_downgrades_unknown_stages_and_build_metadata() {
for (current, target) in [
@@ -443,7 +209,10 @@ async fn update_checks_should_ignore_downgrades_unknown_stages_and_build_metadat
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = current.to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("detail");
assert!(!detail.has_update, "{current} -> {target}");
assert_eq!(detail.latest_version, current);
assert!(detail.notes.is_none());
@@ -467,7 +236,10 @@ async fn update_checks_should_fail_closed_for_unknown_current_channels() {
config.version = current.to_owned();
config.build_type = build_type.to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
let detail = service
.update_detail(SystemUpdateCheck::Refresh, None)
.await
.expect("detail");
assert!(!detail.has_update);
assert!(!detail.update_supported);
assert!(detail.unsupported_reason.is_some());
@@ -480,62 +252,3 @@ async fn update_checks_should_fail_closed_for_unknown_current_channels() {
);
}
}
#[tokio::test]
async fn stable_update_should_select_allowed_version_below_newer_major_releases() {
let server = MockServer::start().await;
let fixture = Fixture::new();
Mock::given(method("GET"))
.and(path("/repos/owner/repository/releases"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
{ "tag_name": "v4.0.0", "prerelease": false },
{ "tag_name": "v3.13.0-beta.1", "prerelease": true },
{ "tag_name": "v3.12.1", "prerelease": true },
{ "tag_name": "v3.12.0", "prerelease": false, "draft": true },
{ "tag_name": "v3.11.2", "prerelease": false },
{ "tag_name": "v3.11.10", "prerelease": false, "body": "stable notes" }
])))
.mount(&server)
.await;
let mut config = fixture.config(&format!("{}/repos", server.uri()));
config.version = "3.11.0".to_owned();
let service = ProcessSystemOperations::new(CancellationToken::new(), config);
let detail = service.update_detail(true, None).await.expect("detail");
assert!(detail.has_update);
assert_eq!(detail.latest_version, "3.11.10");
assert_eq!(detail.notes.as_deref(), Some("stable notes"));
}
#[tokio::test]
async fn update_detail_should_use_cached_release_when_not_refreshed() {
let server = MockServer::start().await;
let fixture = Fixture::new();
fixture.mount_release_once(&server, TARGET_VERSION).await;
let service = fixture.service(&server);
service
.update_detail(true, None)
.await
.expect("prime cache");
assert!(service.update_detail(false, None).await.is_ok());
}
#[tokio::test]
async fn update_detail_should_withhold_cross_major_release() {
let server = MockServer::start().await;
let fixture = Fixture::new();
fixture
.mount_release_once(&server, CROSS_MAJOR_VERSION)
.await;
let detail = fixture
.service(&server)
.update_detail(true, None)
.await
.expect("detail");
assert_eq!(detail.latest_version, "1.0.0");
assert!(!detail.has_update);
assert!(detail.warning.is_none());
assert!(detail.notes.is_none());
assert!(detail.release_url.is_none());
}
@@ -32,6 +32,7 @@ struct Facts {
fn account() -> AccountRecord {
let at = "2026-01-01T00:00:00Z".parse().unwrap();
AccountRecord {
auto_use_quota_credits: false,
outbound_proxy_id: None,
outbound_proxy_enabled: true,
id: "acct_facts".into(),
@@ -129,18 +129,15 @@ async fn preparing_diagnostics_never_wait_for_the_serialized_prepare_io() {
#[tokio::test]
async fn diagnostics_report_a_published_process_fault_without_plugin_details() {
let (cache, store, runtime) = super::setup().await;
let controls = tempfile::tempdir().unwrap();
let exit = controls.path().join("exit");
let mut snapshot = store.snapshot.lock().unwrap().clone();
snapshot.instances[0].configuration = serde_json::json!({"exit_after_ready_ms":20});
snapshot.instances[0].configuration = serde_json::json!({"exit_after_ready_signals":[exit]});
let active = PluginPreparation::prepare(&runtime, snapshot.clone())
.await
.expect("candidate starts before the controlled exit");
tokio::time::timeout(std::time::Duration::from_secs(2), async {
while active.is_ready() {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("worker exit observed");
std::fs::write(exit, b"exit").unwrap();
wait_until_unready(&active).await;
let diagnostics = gateway_admin::ports::plugins::PluginRuntimeDiagnostics::runtime_diagnostics(
&runtime,
@@ -171,17 +168,20 @@ async fn diagnostics_report_a_published_process_fault_without_plugin_details() {
#[tokio::test]
async fn a_published_process_crash_preserves_the_serving_snapshot_and_other_plugin_status() {
let (cache, store, runtime) = super::setup().await;
let controls = tempfile::tempdir().unwrap();
let exit = controls.path().join("exit");
let snapshot = {
let mut snapshot = store.snapshot.lock().unwrap();
let mut failed = snapshot.instances[0].clone();
failed.id = "crashing-plugin".into();
failed.configuration = serde_json::json!({"exit_after_ready_ms":100});
failed.configuration = serde_json::json!({"exit_after_ready_signals":[exit]});
snapshot.instances.push(failed);
snapshot.clone()
};
let generation = ExtensionPreparationPort::prepare(&runtime, ConfigRevision::new(1).unwrap())
.await
.unwrap();
std::fs::write(exit, b"exit").unwrap();
wait_until_unready(&generation).await;
assert!(generation.can_serve());
assert!(!generation.is_ready());
@@ -94,29 +94,32 @@ async fn registration_with_a_changed_contribution_id_is_rejected() {
async fn restart_circuit_caps_short_lived_crashes_and_a_new_revision_recovers() {
let restart_circuit = gateway_plugin_runtime::PluginRestartCircuitConfig {
maximum_failures: NonZeroUsize::new(3).unwrap(),
stability_window: Duration::from_secs(1),
// 本用例只验证连续崩溃的预算,不让宿主调度延迟触发稳定运行后的重置
stability_window: Duration::MAX,
};
let (cache, store, runtime) = super::setup_with_restart_circuit(restart_circuit).await;
let controls = tempfile::tempdir().unwrap();
let marker = controls.path().join("starts.jsonl");
let exits = [1, 2, 3].map(|start| controls.path().join(format!("exit-{start}")));
let mut snapshot = store.snapshot.lock().unwrap().clone();
snapshot.instances[0].configuration = serde_json::json!({
"startup_marker": marker,
"exit_after_ready_ms": 20,
"exit_after_ready_signals": exits,
});
let mut active = PluginPreparation::prepare(&runtime, snapshot.clone())
.await
.expect("first incarnation");
for expected_start in 2..=3 {
for (index, exit) in exits.iter().enumerate() {
assert_eq!(startup_count(&marker), index + 1);
// Ready 早于注册完成;只有宿主拿到这一代后,才让对应的进程退出
std::fs::write(exit, b"exit").unwrap();
wait_until_unready(&active).await;
let replacement = PluginPreparation::prepare(&runtime, snapshot.clone())
.await
.expect("failure budget still permits a restart");
drop(active);
active = replacement;
assert_eq!(startup_count(&marker), expected_start);
if index + 1 < exits.len() {
active = PluginPreparation::prepare(&runtime, snapshot.clone())
.await
.expect("failure budget still permits a restart");
}
}
wait_until_unready(&active).await;
assert!(
PluginPreparation::prepare(&runtime, snapshot.clone())
.await
@@ -434,7 +437,6 @@ async fn abandoned_candidate_keeps_shared_process_failure_in_restart_budget() {
let mut snapshot = store.snapshot.lock().unwrap().clone();
snapshot.instances[0].configuration = serde_json::json!({
"startup_marker":marker,
"exit_after_ready_delays_ms":[null,null],
"exit_after_ready_signals":[old_exit,null],
});
let mut other = snapshot.instances[0].clone();
@@ -959,7 +959,7 @@ impl SystemOperations for UnusedAdminRuntime {
async fn update_detail(
&self,
_: bool,
_: gateway_admin::model::system::SystemUpdateCheck,
_: Option<gateway_admin::model::system::SystemUpdateChannel>,
) -> Result<SystemUpdateDetail, SystemOperationError> {
Err(unused_system())
@@ -1546,11 +1546,6 @@ async fn main() {
resumptions: Mutex::new(BTreeMap::new()),
streams: Mutex::new(BTreeMap::new()),
});
let exit_after_ready = peer.configuration["exit_after_ready_delays_ms"]
.as_array()
.and_then(|delays| delays.get(startup))
.and_then(Value::as_u64)
.or_else(|| peer.configuration["exit_after_ready_ms"].as_u64());
let exit_after_ready_signal = peer.configuration["exit_after_ready_signals"]
.as_array()
.and_then(|signals| signals.get(startup))
@@ -1571,12 +1566,7 @@ async fn main() {
vec![],
)
.await;
if let Some(delay) = exit_after_ready {
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(delay)).await;
std::process::exit(7);
});
} else if let Some(signal) = exit_after_ready_signal {
if let Some(signal) = exit_after_ready_signal {
tokio::spawn(async move {
while !std::path::Path::new(&signal).exists() {
tokio::time::sleep(Duration::from_millis(5)).await;
@@ -144,7 +144,9 @@ pub struct RuntimeSettings {
pub max_concurrent_per_account: u32,
pub max_waiting_per_key: u32,
pub max_waiting_per_account: u32,
pub concurrency_wait_timeout_seconds: u32,
pub key_queue_timeout_seconds: u32,
pub account_queue_timeout_seconds: u32,
pub switch_account_on_queue_failure: bool,
pub max_retry_after_seconds: u32,
pub openai_guardian_reserved_concurrency: u32,
pub openai_account_affinity: AccountAffinity,
@@ -180,7 +182,9 @@ pub struct ReplaceRuntimeSettings {
pub max_concurrent_per_account: u32,
pub max_waiting_per_key: u32,
pub max_waiting_per_account: u32,
pub concurrency_wait_timeout_seconds: u32,
pub key_queue_timeout_seconds: u32,
pub account_queue_timeout_seconds: u32,
pub switch_account_on_queue_failure: bool,
pub max_retry_after_seconds: u32,
pub openai_guardian_reserved_concurrency: u32,
pub openai_account_affinity: AccountAffinity,
@@ -378,7 +382,9 @@ impl From<RuntimeSettings> for ReplaceRuntimeSettings {
max_concurrent_per_account: settings.max_concurrent_per_account,
max_waiting_per_key: settings.max_waiting_per_key,
max_waiting_per_account: settings.max_waiting_per_account,
concurrency_wait_timeout_seconds: settings.concurrency_wait_timeout_seconds,
key_queue_timeout_seconds: settings.key_queue_timeout_seconds,
account_queue_timeout_seconds: settings.account_queue_timeout_seconds,
switch_account_on_queue_failure: settings.switch_account_on_queue_failure,
max_retry_after_seconds: settings.max_retry_after_seconds,
openai_guardian_reserved_concurrency: settings.openai_guardian_reserved_concurrency,
openai_account_affinity: settings.openai_account_affinity,
@@ -103,7 +103,9 @@ impl SettingsStore for AdminSettingsStoreAdapter {
"max_concurrent_per_account".to_owned(),
"max_waiting_per_key".to_owned(),
"max_waiting_per_account".to_owned(),
"concurrency_wait_timeout_seconds".to_owned(),
"key_queue_timeout_seconds".to_owned(),
"account_queue_timeout_seconds".to_owned(),
"switch_account_on_queue_failure".to_owned(),
"max_retry_after_seconds".to_owned(),
"openai_guardian_reserved_concurrency".to_owned(),
"openai_account_affinity".to_owned(),
@@ -242,7 +242,7 @@ impl AccountGroupStore for PgAccountGroupRepository {
account.upstream_user_id, account.upstream_account_id, account.plan_type,
account.authentication_kind, account.credential_revision, account.outbound_proxy_id, account.outbound_proxy_url,
account.has_refresh_token, account.access_token_expires_at,
account.next_refresh_at, account.enabled, account.concurrency_limit,
account.next_refresh_at, account.enabled, account.auto_use_quota_credits, account.concurrency_limit,
account.quota_observed_at, account.weight, account.model_access_json, account.credential_state, account.quota_access_state,
account.quota_evidence, account.quota_access_observed_at,
account.quota_reset_at, account.last_error_reason,
@@ -664,6 +664,7 @@ impl AccountStore for PgAdminAccountStore {
.accounts
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
account_ids: vec![command.account_id.clone()],
auto_use_quota_credits: None,
name: command.name,
notes: command.notes,
enabled: Some(command.enabled),
@@ -747,6 +748,10 @@ impl AccountStore for PgAdminAccountStore {
};
let mut changed_fields = Vec::new();
for (changed, field) in [
(
command.auto_use_quota_credits.is_some(),
"auto_use_quota_credits",
),
(command.enabled.is_some(), "enabled"),
(command.concurrency_limit.is_some(), "concurrency_limit"),
(command.weight.is_some(), "weight"),
@@ -766,6 +771,7 @@ impl AccountStore for PgAdminAccountStore {
.accounts
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
account_ids: command.account_ids,
auto_use_quota_credits: command.auto_use_quota_credits,
name: None,
notes: None,
enabled: command.enabled,
@@ -106,7 +106,7 @@ pub(crate) async fn load_admin_account_page(
select coalesce(p.enabled, true) as outbound_proxy_enabled, coalesce(p.max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, p.auto_location, p.detected_location_json, p.location_country, p.location_region, p.location_city, p.location_timezone, a.outbound_proxy_id, a.outbound_proxy_url, a.id, a.provider_kind, a.name, a.notes, a.email, a.upstream_user_id,
a.upstream_account_id, a.plan_type, a.authentication_kind,
a.credential_revision, a.has_refresh_token, a.access_token_expires_at,
a.next_refresh_at, a.enabled, a.concurrency_limit, a.weight, a.model_access_json,
a.next_refresh_at, a.enabled, a.auto_use_quota_credits, a.concurrency_limit, a.weight, a.model_access_json,
a.credential_state, a.quota_access_state, a.quota_evidence,
a.quota_access_observed_at, a.quota_reset_at, a.last_error_reason,
a.last_error_message, a.credential_observed_at, a.created_at, a.updated_at,
@@ -88,6 +88,26 @@ impl ProviderAccountStore for PgProviderAccountRepository {
.collect()
}
async fn list_for_provider_accounts(
&self,
provider: &ProviderKind,
account_ids: &[CoreProviderAccountId],
) -> Result<Vec<CoreProviderAccount>, CoreStoreError> {
if account_ids.is_empty() {
return Ok(Vec::new());
}
let account_ids = account_ids
.iter()
.map(CoreProviderAccountId::as_str)
.collect::<Vec<_>>();
self.list_provider_account_summaries(Some(provider.as_str()), false, Some(&account_ids))
.await
.map_err(core_store_error)?
.into_iter()
.map(core_account_from_summary)
.collect()
}
async fn list_refresh_candidates(
&self,
query: CoreProviderRefreshQuery,
@@ -10,6 +10,7 @@ pub(crate) fn admin_account_record(
summary: ProviderAccountSummary,
) -> AdminStoreResult<AccountRecord> {
Ok(AccountRecord {
auto_use_quota_credits: summary.auto_use_quota_credits,
outbound_proxy_id: summary.outbound_proxy_id,
outbound_proxy_enabled: summary.outbound_proxy_enabled,
outbound_proxy: summary.outbound_proxy,
@@ -108,6 +108,45 @@ impl PgProviderAccountRepository {
self
}
pub(super) async fn list_provider_account_summaries(
&self,
provider_kind: Option<&str>,
include_disabled: bool,
account_ids: Option<&[&str]>,
) -> StoreResult<Vec<ProviderAccountSummary>> {
let mut query = sqlx::QueryBuilder::<Postgres>::new(
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
credential_observed_at, quota_access_state, quota_evidence,
quota_access_observed_at, quota_reset_at,
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
from provider_accounts
left join (select id as location_proxy_id, enabled as proxy_enabled, max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone from outbound_proxies) proxy_location
on outbound_proxy_id = location_proxy_id
where true",
);
if let Some(provider_kind) = provider_kind {
query.push(" and provider_kind = ").push_bind(provider_kind);
}
if !include_disabled {
query.push(" and enabled");
}
if let Some(account_ids) = account_ids {
query
.push(" and id = any(")
.push_bind(account_ids)
.push("::text[])");
}
query.push(" order by provider_kind, name, id");
let rows = query
.build()
.fetch_all(&self.pool)
.await
.map_err(|source| postgres_unavailable("list provider accounts", source))?;
rows.into_iter().map(account_summary_from_row).collect()
}
/// PostgreSQL 已提交的凭据结果不能因可丢失运行态失败而回报未提交
pub(super) async fn advance_protection_revision(&self, account: &str, revision: Revision) {
let Some(protection) = &self.protection else {
@@ -159,7 +198,7 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
let rows = sqlx::query(
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
credential_observed_at, quota_access_state, quota_evidence,
quota_access_observed_at, quota_reset_at,
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
@@ -185,25 +224,8 @@ impl ProviderAccountRepository for PgProviderAccountRepository {
provider_kind: Option<&str>,
include_disabled: bool,
) -> StoreResult<Vec<ProviderAccountSummary>> {
let rows = sqlx::query(
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
credential_observed_at, quota_access_state, quota_evidence,
quota_access_observed_at, quota_reset_at,
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
from provider_accounts
left join (select id as location_proxy_id, enabled as proxy_enabled, max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone from outbound_proxies) proxy_location
on outbound_proxy_id = location_proxy_id
where ($1::text is null or provider_kind = $1) and ($2 or enabled)
order by provider_kind, name, id",
)
.bind(provider_kind)
.bind(include_disabled)
.fetch_all(&self.pool)
.await
.map_err(|source| postgres_unavailable("list provider accounts", source))?;
rows.into_iter().map(account_summary_from_row).collect()
self.list_provider_account_summaries(provider_kind, include_disabled, None)
.await
}
async fn insert_provider_account(&self, account: NewProviderAccount) -> StoreResult<()> {
@@ -530,6 +552,11 @@ impl ProviderAccountAdminRepository for PgProviderAccountRepository {
command.outbound_proxy.as_ref(),
)
.await?;
if let Some(enabled) = command.auto_use_quota_credits {
sqlx::query("update provider_accounts set auto_use_quota_credits = $2 where id = any($1) and auto_use_quota_credits <> $2")
.bind(&command.account_ids).bind(enabled).execute(&mut *transaction).await
.map_err(|source| postgres_unavailable("set account quota credits policy", source))?;
}
if let Some(group_ids) = &command.group_ids {
replace_account_group_assignments_in_transaction(
&mut transaction,
@@ -1214,7 +1241,7 @@ impl PgProviderAccountRepository {
let rows = sqlx::query(
"select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, credential_revision, has_refresh_token,
access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
credential_observed_at, quota_access_state, quota_evidence,
quota_access_observed_at, quota_reset_at,
quota_observed_at, last_error_reason, last_error_message, created_at, updated_at
@@ -51,6 +51,7 @@ pub(crate) fn parse_error_reason(value: Option<String>) -> StoreResult<Option<Ac
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderAccountSummary {
pub auto_use_quota_credits: bool,
pub outbound_proxy_id: Option<String>,
pub outbound_proxy_enabled: bool,
pub request_location: Option<gateway_core::account::RequestLocation>,
@@ -249,6 +250,7 @@ impl fmt::Debug for RotateProviderAccount {
#[derive(Debug, Clone)]
pub struct BatchUpdateProviderAccountsAdmin {
pub auto_use_quota_credits: Option<bool>,
pub outbound_proxy: Option<gateway_admin::model::proxies::AccountProxySelection>,
pub account_ids: Vec<String>,
pub name: Option<String>,
@@ -322,7 +324,7 @@ impl ProviderAccountStateUpdate {
pub(crate) const ACCOUNT_SELECT: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
last_error_reason, last_error_message,
credential_observed_at, quota_observed_at, created_at, updated_at
@@ -333,7 +335,7 @@ pub(crate) const ACCOUNT_SELECT: &str = "select coalesce(proxy_enabled, true) as
pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
last_error_reason, last_error_message,
credential_observed_at, quota_observed_at, created_at, updated_at
@@ -345,7 +347,7 @@ pub(crate) const ACCOUNT_SELECT_BY_IDS: &str = "select coalesce(proxy_enabled, t
pub(crate) const REFRESH_CANDIDATES_SELECT: &str = "select coalesce(proxy_enabled, true) as outbound_proxy_enabled, coalesce(max_websocket_connections, 0) as outbound_proxy_max_websocket_connections, auto_location, detected_location_json, location_country, location_region, location_city, location_timezone, outbound_proxy_id, outbound_proxy_url, id, provider_kind, name, notes, email, upstream_user_id,
upstream_account_id, plan_type, authentication_kind, provider_credentials_json, credential_revision,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, concurrency_limit, weight, model_access_json, credential_state,
has_refresh_token, access_token_expires_at, next_refresh_at, enabled, auto_use_quota_credits, concurrency_limit, weight, model_access_json, credential_state,
quota_access_state, quota_evidence, quota_access_observed_at, quota_reset_at,
last_error_reason, last_error_message,
credential_observed_at, quota_observed_at, created_at, updated_at
@@ -415,6 +417,7 @@ pub(crate) fn core_account_from_summary(
)
.with_scheduling(summary.concurrency_limit, summary.weight)
.with_model_access(summary.model_access)
.with_auto_use_quota_credits(summary.auto_use_quota_credits)
.with_outbound_proxy(
summary
.outbound_proxy
@@ -470,6 +473,7 @@ pub(crate) fn account_summary_from_row(
.and_then(AccountWeight::new)
.ok_or_else(|| invalid("invalid weight"))?;
Ok(ProviderAccountSummary {
auto_use_quota_credits: get(&row, "auto_use_quota_credits")?,
outbound_proxy_id: get(&row, "outbound_proxy_id")?,
outbound_proxy_enabled: get(&row, "outbound_proxy_enabled")?,
request_location: super::super::proxies::effective_location_from_row(&row)?,
@@ -155,7 +155,7 @@ pub(crate) async fn load_runtime_settings_from_pool(pool: &PgPool) -> StoreResul
refresh_concurrency, max_concurrent_per_account,
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
audit_retention_days, min_codex_desktop_version,
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
account_error_protection_json, account_error_protection_revision,
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
from runtime_settings where id = 1",
@@ -311,7 +311,7 @@ pub(crate) async fn load_runtime_settings_in_transaction(
refresh_concurrency, max_concurrent_per_account,
rotation_strategy, smart_scheduling_json, model_mappings_json, usage_retention_days, ops_event_retention_days,
audit_retention_days, min_codex_desktop_version,
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
min_codex_cli_version, updated_at, responses_max_decompressed_body_bytes, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours,
account_error_protection_json, account_error_protection_revision,
account_warmup_enabled, account_warmup_schedule_time, account_warmup_model, reset_credit_lead_minutes, reset_credit_max_weekly_remaining_percent
from runtime_settings where id = 1",
@@ -373,7 +373,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
min_codex_cli_version = $10,
max_waiting_per_key = $11,
max_waiting_per_account = $12,
concurrency_wait_timeout_seconds = $13,
key_queue_timeout_seconds = $13,
account_error_protection_revision = case when account_error_protection_json is distinct from $14::jsonb then config_revision + 1 else account_error_protection_revision end,
account_error_protection_json = $14,
request_location_json = $15,
@@ -392,6 +392,8 @@ pub(crate) async fn update_runtime_settings_in_transaction(
reset_credit_lead_minutes = $29,
reset_credit_max_weekly_remaining_percent = $30,
max_retry_after_seconds = $31,
account_queue_timeout_seconds = $32,
switch_account_on_queue_failure = $33,
updated_at = now()
where id = 1
returning config_revision",
@@ -408,7 +410,7 @@ pub(crate) async fn update_runtime_settings_in_transaction(
.bind(update.values.min_codex_cli_version.as_deref())
.bind(i64::from(update.values.max_waiting_per_key))
.bind(i64::from(update.values.max_waiting_per_account))
.bind(i64::from(update.values.concurrency_wait_timeout_seconds))
.bind(i64::from(update.values.key_queue_timeout_seconds))
.bind(sqlx::types::Json(&update.values.account_error_protection))
.bind(sqlx::types::Json(
update.values.request_location
@@ -435,6 +437,8 @@ pub(crate) async fn update_runtime_settings_in_transaction(
.bind(i64::from(update.values.reset_credit_lead_minutes))
.bind(update.values.reset_credit_max_weekly_remaining_percent.map(i64::from))
.bind(i64::from(update.values.max_retry_after_seconds))
.bind(i64::from(update.values.account_queue_timeout_seconds))
.bind(update.values.switch_account_on_queue_failure)
.fetch_optional(&mut **transaction)
.await
.map_err(|source| postgres_unavailable("update runtime settings in transaction", source))?
@@ -521,7 +525,9 @@ struct RuntimeSettingsRow {
updated_at: DateTime<Utc>,
max_waiting_per_key: i64,
max_waiting_per_account: i64,
concurrency_wait_timeout_seconds: i64,
key_queue_timeout_seconds: i64,
account_queue_timeout_seconds: i64,
switch_account_on_queue_failure: bool,
max_retry_after_seconds: i64,
openai_guardian_reserved_concurrency: i64,
openai_account_affinity: String,
@@ -580,7 +586,9 @@ fn runtime_settings_from_row(row: RuntimeSettingsRow) -> StoreResult<RuntimeSett
min_codex_cli_version: row.min_codex_cli_version,
max_waiting_per_key: to_u32(row.max_waiting_per_key)?,
max_waiting_per_account: to_u32(row.max_waiting_per_account)?,
concurrency_wait_timeout_seconds: to_u32(row.concurrency_wait_timeout_seconds)?,
key_queue_timeout_seconds: to_u32(row.key_queue_timeout_seconds)?,
account_queue_timeout_seconds: to_u32(row.account_queue_timeout_seconds)?,
switch_account_on_queue_failure: row.switch_account_on_queue_failure,
max_retry_after_seconds: to_u32(row.max_retry_after_seconds)?,
openai_guardian_reserved_concurrency: to_u32(row.openai_guardian_reserved_concurrency)?,
openai_account_affinity: gateway_core::account::AccountAffinity::parse(
@@ -227,7 +227,9 @@ struct SnapshotSettingsRow {
min_codex_cli_version: Option<String>,
max_waiting_per_key: i64,
max_waiting_per_account: i64,
concurrency_wait_timeout_seconds: i64,
key_queue_timeout_seconds: i64,
account_queue_timeout_seconds: i64,
switch_account_on_queue_failure: bool,
max_retry_after_seconds: i64,
openai_guardian_reserved_concurrency: i64,
openai_account_affinity: String,
@@ -246,7 +248,7 @@ async fn load_settings(
transaction: &mut Transaction<'_, Postgres>,
) -> StoreResult<(Revision, SettingsValues)> {
let row = sqlx::query_as::<_, SnapshotSettingsRow>(
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, concurrency_wait_timeout_seconds, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
"select config_revision, account_error_protection_revision, account_error_protection_json, max_concurrent_per_account, rotation_strategy, smart_scheduling_json, model_mappings_json, min_codex_desktop_version, min_codex_cli_version, max_waiting_per_key, max_waiting_per_account, key_queue_timeout_seconds, account_queue_timeout_seconds, switch_account_on_queue_failure, max_retry_after_seconds, openai_guardian_reserved_concurrency, openai_account_affinity, max_account_rotations, openai_session_affinity_ttl_hours, request_location_json, request_location_enabled, codex_privacy_policy_json, responses_max_decompressed_body_bytes, provider_request_profiles_json, pricing_overrides_json, pricing_synced_json from runtime_settings where id = 1",
)
.fetch_optional(&mut **transaction)
.await
@@ -297,7 +299,9 @@ async fn load_settings(
.with_concurrency_queues(
to_u32(row.max_waiting_per_key)?,
to_u32(row.max_waiting_per_account)?,
to_u32(row.concurrency_wait_timeout_seconds)?,
to_u32(row.key_queue_timeout_seconds)?,
to_u32(row.account_queue_timeout_seconds)?,
row.switch_account_on_queue_failure,
),
))
}
@@ -335,6 +335,38 @@ impl RedisCredentialCooldownRepository {
parse_cooldown(account, &result)
}
pub(super) async fn read_effective_cooldowns(
&self,
account_ids: &[ProviderAccountId],
) -> Vec<StoreResult<Option<CredentialCooldown>>> {
let mut results = Vec::with_capacity(account_ids.len());
for batch in account_ids.chunks(super::credential_cooldown::COOLDOWN_READ_BATCH_SIZE) {
let commands = batch
.iter()
.map(|account_id| {
let account = account_id.as_str();
let mut command = redis::cmd("EVALSHA");
command
.arg(READ_COOLDOWN.get_hash())
.arg(4)
.arg(self.key(account)?)
.arg(self.active_index_key())
.arg(self.protection_key(account)?)
.arg(self.protection_index_key())
.arg(account);
Ok(command)
})
.collect();
let values = self.read_script_batch(&READ_COOLDOWN, commands).await;
results.extend(batch.iter().zip(values).map(|(account, value)| {
let values: Vec<String> = redis::from_redis_value(value?)
.map_err(|source| redis_unavailable("decode account cooldown", source))?;
parse_cooldown(account.as_str(), &values)
}));
}
results
}
pub(super) async fn read_account_runtime(
&self,
account: &str,
@@ -17,6 +17,8 @@ use crate::{Revision, StoreError, StoreResult, redis_unavailable, require_nonemp
use super::{namespace, resource_fingerprint};
pub(super) const COOLDOWN_READ_BATCH_SIZE: usize = 128;
const WRITE_SCRIPT: &str = r#"
local current = tonumber(redis.call('HGET', KEYS[1], 'revision') or '0')
local incoming = tonumber(ARGV[1])
@@ -194,7 +196,7 @@ impl RedisCredentialCooldownRepository {
&self,
key: String,
index_member: Option<&str>,
) -> StoreResult<Option<(Revision, DateTime<Utc>, ProviderCooldownKind, String)>> {
) -> StoreResult<Option<(Revision, DateTime<Utc>)>> {
let mut connection = self.connection.clone();
let result = if let Some(index_member) = index_member {
Script::new(READ_SCRIPT)
@@ -210,26 +212,86 @@ impl RedisCredentialCooldownRepository {
.invoke_async(&mut connection)
.await
};
let (present, revision, until_ms, kind, generation): (i64, String, String, String, String) =
result.map_err(|source| redis_unavailable("read credential cooldown", source))?;
if present == 0 {
return Ok(None);
parse_scoped_cooldown(
result.map_err(|source| redis_unavailable("read credential cooldown", source))?,
)
}
/// 保留每条命令的错误,脚本失效时只补发尚未执行的 NOSCRIPT 项
pub(super) async fn read_script_batch(
&self,
script: &Script,
commands: Vec<StoreResult<redis::Cmd>>,
) -> Vec<StoreResult<redis::Value>> {
let mut pipeline = redis::pipe();
pipeline.ignore_errors();
for command in commands.iter().filter_map(|command| command.as_ref().ok()) {
pipeline.add_command(command.clone());
}
let revision = revision
.parse::<u64>()
.map_err(|source| invalid("cached cooldown revision is invalid").with_source(source))?;
let until_ms = until_ms
.parse::<i64>()
.map_err(|source| invalid("cached cooldown expiry is invalid").with_source(source))?;
let cooldown_until = DateTime::from_timestamp_millis(until_ms)
.ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
let kind = ProviderCooldownKind::parse(&kind).unwrap_or(ProviderCooldownKind::RateLimit);
Ok(Some((
Revision::new(revision)?,
cooldown_until,
kind,
generation,
)))
let mut connection = self.connection.clone();
let values = pipeline
.query_async::<Vec<redis::RedisResult<redis::Value>>>(&mut connection)
.await;
let values = match values {
Ok(values) => {
let mut retry = redis::pipe();
retry.ignore_errors();
let mut retry_indices = Vec::new();
for (index, (command, value)) in pipeline.cmd_iter().zip(&values).enumerate() {
if value.as_ref().is_err_and(|error| {
error.kind() == redis::ErrorKind::Server(redis::ServerErrorKind::NoScript)
}) {
retry.add_command(command.clone());
retry_indices.push(index);
}
}
let mut values = values
.into_iter()
.map(|value| {
value.map_err(|source| redis_unavailable("read cooldown batch", source))
})
.collect::<Vec<_>>();
if !retry_indices.is_empty() {
let retried = async {
script.load_async(&mut connection).await?;
retry
.query_async::<Vec<redis::RedisResult<redis::Value>>>(&mut connection)
.await
}
.await;
match retried {
Ok(retried) => {
for (index, value) in retry_indices.into_iter().zip(retried) {
values[index] = value.map_err(|source| {
redis_unavailable("retry cooldown batch", source)
});
}
}
Err(source) => {
let error = redis_unavailable("retry cooldown batch", source);
for index in retry_indices {
values[index] = Err(error.clone());
}
}
}
}
values
}
Err(source) => {
vec![Err(redis_unavailable("read cooldown batch", source)); pipeline.len()]
}
};
let mut values = values.into_iter();
commands
.into_iter()
.map(|command| {
command.and_then(|_| {
values
.next()
.unwrap_or_else(|| Err(invalid("missing cooldown batch response")))
})
})
.collect()
}
async fn invalidate_at_key(
@@ -579,22 +641,32 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
CredentialCooldownRepository::read_credential_cooldown(self, account_id.as_str())
.await
.map_err(|source| crate::provider_unavailable("read credential cooldown", source))?
.map(|record| {
let account_id = ProviderAccountId::new(record.provider_account_id)
.map_err(|_| provider_invalid("decode credential cooldown"))?;
let revision = CredentialRevision::new(record.credential_revision.get())
.map_err(|_| provider_invalid("decode credential cooldown"))?;
Ok(ProviderCooldown::new_with_kind(
account_id,
revision,
record.cooldown_until.into(),
record.kind,
))
})
.map(provider_cooldown)
.transpose()
})
}
fn read_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> futures::future::BoxFuture<'a, Vec<Result<Option<ProviderCooldown>, ProviderStoreError>>>
{
Box::pin(async move {
self.read_effective_cooldowns(account_ids)
.await
.into_iter()
.map(|result| {
result
.map_err(|source| {
crate::provider_unavailable("read credential cooldown", source)
})?
.map(provider_cooldown)
.transpose()
})
.collect()
})
}
fn clear<'a>(
&'a self,
account_id: &'a ProviderAccountId,
@@ -651,7 +723,7 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
.map_err(|source| {
crate::provider_unavailable("read scoped credential cooldown", source)
})?
.map(|(revision, until, _, _)| {
.map(|(revision, until)| {
Ok(ProviderScopedCooldown::new(
account_id.clone(),
CredentialRevision::new(revision.get())
@@ -664,6 +736,60 @@ impl ProviderCooldownPort for RedisCredentialCooldownRepository {
})
}
fn read_scoped_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
scope: &'a ProviderCooldownScope,
) -> futures::future::BoxFuture<
'a,
Vec<Result<Option<ProviderScopedCooldown>, ProviderStoreError>>,
> {
Box::pin(async move {
let script = Script::new(READ_SCRIPT);
let mut results = Vec::with_capacity(account_ids.len());
for batch in account_ids.chunks(COOLDOWN_READ_BATCH_SIZE) {
let commands = batch
.iter()
.map(|account| {
let mut command = redis::cmd("EVALSHA");
command
.arg(script.get_hash())
.arg(1)
.arg(self.scoped_key(account.as_str(), scope)?)
.arg("");
Ok(command)
})
.collect();
let values = self.read_script_batch(&script, commands).await;
results.extend(batch.iter().zip(values).map(|(account, value)| {
let value = value
.and_then(|value| {
redis::from_redis_value(value).map_err(|source| {
redis_unavailable("decode scoped cooldown", source)
})
})
.and_then(parse_scoped_cooldown)
.map_err(|source| {
crate::provider_unavailable("read scoped credential cooldown", source)
})?;
value
.map(|(revision, until)| {
Ok(ProviderScopedCooldown::new(
account.clone(),
CredentialRevision::new(revision.get()).map_err(|_| {
provider_invalid("decode scoped credential cooldown")
})?,
scope.clone(),
until.into(),
))
})
.transpose()
}));
}
results
})
}
fn clear_scoped<'a>(
&'a self,
account_id: &'a ProviderAccountId,
@@ -740,6 +866,36 @@ pub(super) fn invalid(message: &str) -> StoreError {
}
}
fn parse_scoped_cooldown(
(present, revision, until_ms, _, _): (i64, String, String, String, String),
) -> StoreResult<Option<(Revision, DateTime<Utc>)>> {
if present == 0 {
return Ok(None);
}
let revision = revision
.parse::<u64>()
.map_err(|source| invalid("cached cooldown revision is invalid").with_source(source))?;
let until_ms = until_ms
.parse::<i64>()
.map_err(|source| invalid("cached cooldown expiry is invalid").with_source(source))?;
let cooldown_until = DateTime::from_timestamp_millis(until_ms)
.ok_or_else(|| invalid("cached cooldown expiry is invalid"))?;
Ok(Some((Revision::new(revision)?, cooldown_until)))
}
fn provider_cooldown(record: CredentialCooldown) -> Result<ProviderCooldown, ProviderStoreError> {
let account_id = ProviderAccountId::new(record.provider_account_id)
.map_err(|_| provider_invalid("decode credential cooldown"))?;
let revision = CredentialRevision::new(record.credential_revision.get())
.map_err(|_| provider_invalid("decode credential cooldown"))?;
Ok(ProviderCooldown::new_with_kind(
account_id,
revision,
record.cooldown_until.into(),
record.kind,
))
}
fn provider_invalid(operation: &'static str) -> ProviderStoreError {
ProviderStoreError::new(ProviderStoreErrorKind::InvalidData, operation)
}
@@ -1646,6 +1646,37 @@ impl StaticCooldowns {
}
impl ProviderCooldownPort for StaticCooldowns {
fn read_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
) -> BoxFuture<'a, Vec<Result<Option<ProviderCooldown>, ProviderStoreError>>> {
Box::pin(async move {
let mut results = Vec::with_capacity(account_ids.len());
for batch in account_ids.chunks(128) {
results
.extend(futures::future::join_all(batch.iter().map(|id| self.read(id))).await);
}
results
})
}
fn read_scoped_many<'a>(
&'a self,
account_ids: &'a [ProviderAccountId],
scope: &'a ProviderCooldownScope,
) -> BoxFuture<'a, Vec<Result<Option<ProviderScopedCooldown>, ProviderStoreError>>> {
Box::pin(async move {
let mut results = Vec::with_capacity(account_ids.len());
for batch in account_ids.chunks(128) {
results.extend(
futures::future::join_all(batch.iter().map(|id| self.read_scoped(id, scope)))
.await,
);
}
results
})
}
fn put_if_later(
&self,
_cooldown: ProviderCooldown,
@@ -0,0 +1,105 @@
//! Core 账号列表的冻结范围与实时启停过滤
use super::*;
#[tokio::test]
async fn scoped_account_list_empty_scope_does_not_use_database() {
let pool = sqlx::postgres::PgPoolOptions::new()
.connect_lazy("postgres://test:test@localhost/test")
.expect("lazy database pool");
pool.close().await;
let repository = PgProviderAccountRepository::new(pool);
let provider = ProviderKind::new("openai").expect("provider kind");
assert!(
repository
.list_for_provider_accounts(&provider, &[])
.await
.expect("empty scope without a database connection")
.is_empty()
);
}
#[tokio::test]
async fn scoped_account_list_filters_ids_provider_and_live_enabled_state() {
let Some(database) = TestDatabase::create("scoped_account_list").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let first = account("acct_scope_a", "scope-a");
let last = account("acct_scope_z", "scope-z");
let outside = account("acct_scope_outside", "scope-outside");
let mut disabled = account("acct_scope_disabled", "scope-disabled");
disabled.enabled = false;
let mut other_provider = account("acct_scope_other_provider", "scope-other-provider");
other_provider.provider_kind = "xai".to_owned();
for fixture in [first, last, outside, disabled, other_provider] {
repository
.insert_provider_account(fixture)
.await
.expect("seed scoped account list");
}
let provider = ProviderKind::new("openai").expect("provider kind");
let account_ids = [
"acct_scope_z",
"acct_scope_disabled",
"acct_scope_other_provider",
"acct_scope_z",
"acct_scope_missing",
"acct_scope_a",
]
.map(|id| ProviderAccountId::new(id).expect("account id"));
let scoped = repository
.list_for_provider_accounts(&provider, &account_ids)
.await
.expect("load scoped account list");
assert_eq!(
scoped
.iter()
.map(|account| account.id().as_str())
.collect::<Vec<_>>(),
["acct_scope_a", "acct_scope_z"]
);
let full_pool = repository
.list_for_provider(&provider)
.await
.expect("load full provider pool");
assert_eq!(
full_pool
.iter()
.map(|account| account.id().as_str())
.collect::<Vec<_>>(),
["acct_scope_a", "acct_scope_outside", "acct_scope_z"]
);
let all = repository
.list_provider_accounts(None, true)
.await
.expect("load administrative account list");
assert_eq!(all.len(), 5);
assert!(all.iter().any(|account| !account.enabled));
assert!(all.iter().any(|account| account.provider_kind == "xai"));
let disabled_id = ProviderAccountId::new("acct_scope_disabled").expect("disabled account id");
let enabled_id = ProviderAccountId::new("acct_scope_a").expect("enabled account id");
repository
.set_enabled(&enabled_id, false)
.await
.expect("disable an account between scoped reads");
repository
.set_enabled(&disabled_id, true)
.await
.expect("enable an account between scoped reads");
let changed = repository
.list_for_provider_accounts(&provider, &account_ids)
.await
.expect("reload scoped account list after enabled state changes");
assert_eq!(
changed
.iter()
.map(|account| account.id().as_str())
.collect::<Vec<_>>(),
["acct_scope_disabled", "acct_scope_z"]
);
database.close().await;
}
@@ -8,6 +8,7 @@ use std::{
mod admin_adapter;
mod authorization;
mod core_adapter;
mod deletion;
mod protection_revision;
mod quota;
@@ -1539,6 +1540,7 @@ async fn account_notes_round_trip_and_survive_import_and_scheduling_updates() {
store
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
account_ids: vec!["acct_notes".to_owned()],
enabled: Some(false),
concurrency_limit: Some(None),
@@ -1777,6 +1779,7 @@ async fn account_enable_preserves_facts_and_explicit_recovery_clears_them() {
let enabled = store
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
account_ids: vec!["acct_recovery".to_owned()],
enabled: Some(true),
concurrency_limit: None,
@@ -1901,6 +1904,7 @@ async fn terminal_batch_update_replaces_state_and_groups_once_or_rolls_back_ever
let result = store
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
model_access: Default::default(),
outbound_proxy: None,
account_ids: account_ids.clone(),
@@ -1938,6 +1942,7 @@ async fn terminal_batch_update_replaces_state_and_groups_once_or_rolls_back_ever
store
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
model_access: Default::default(),
outbound_proxy: None,
account_ids: account_ids.clone(),
@@ -2677,6 +2682,7 @@ async fn provider_account_admin_mutations_are_scoped_audited_and_atomic() {
let revision = repository
.batch_update_provider_accounts_admin(BatchUpdateProviderAccountsAdmin {
auto_use_quota_credits: None,
name: None,
notes: None,
model_access: Default::default(),
@@ -3692,6 +3698,7 @@ async fn model_access_only_batch_update_preserves_other_settings_and_survives_re
.expect("policy");
let store = admin_account_store(&database.pool);
let command = BatchUpdateAccounts {
auto_use_quota_credits: None,
account_ids: vec![input.id.clone()],
enabled: None,
concurrency_limit: None,
@@ -3812,3 +3819,83 @@ async fn connection_configuration_update_preserves_credential_health() {
assert_eq!(after["credential_revision"], 2);
database.close().await;
}
#[tokio::test]
async fn disabling_quota_credits_revokes_credit_only_access_even_with_a_late_observation() {
let Some(database) = TestDatabase::create("quota_credit_policy").await else {
return;
};
let repository = PgProviderAccountRepository::new(database.pool.clone());
let input = account("acct_credit_policy", "user-credit-policy");
repository
.insert_provider_account(input.clone())
.await
.expect("insert");
let id = ProviderAccountId::new(input.id.clone()).expect("id");
let before = repository.get_account(&id).await.unwrap().unwrap();
assert!(!before.auto_use_quota_credits());
let admin = admin_account_store(&database.pool);
let context = MutationContext {
actor: MutationActor::System,
request_id: "quota_credits".into(),
};
let set_policy = async |enabled| {
let command = BatchUpdateAccounts {
account_ids: vec![input.id.clone()],
auto_use_quota_credits: Some(enabled),
enabled: None,
concurrency_limit: None,
weight: None,
model_access: None,
group_ids: None,
outbound_proxy: None,
};
admin
.batch_update_accounts(command, &context)
.await
.unwrap();
};
let observe = async |state: QuotaState| {
let observation = QuotaObservation {
expected_connection: None,
plan_type: None,
account_id: id.clone(),
expected_revision: before.revision(),
quota: OpaqueProviderData::new(Default::default()),
observed_at: state.observed_at().unwrap(),
state,
};
assert_eq!(
repository
.compare_and_swap_quota(observation)
.await
.unwrap(),
QuotaWriteOutcome::Updated
);
};
let access = async || {
repository
.get_account(&id)
.await
.unwrap()
.unwrap()
.quota()
.access()
};
set_policy(true).await;
observe(QuotaState::credits_only(SystemTime::now(), None)).await;
assert_eq!(access().await, QuotaAccessState::Allowed);
set_policy(false).await;
assert_eq!(access().await, QuotaAccessState::Exhausted);
observe(QuotaState::credits_only(SystemTime::now(), None)).await;
assert_eq!(access().await, QuotaAccessState::Exhausted);
set_policy(true).await;
assert_eq!(access().await, QuotaAccessState::Exhausted);
observe(QuotaState::allowed(SystemTime::now())).await;
set_policy(false).await;
assert_eq!(access().await, QuotaAccessState::Allowed);
database.close().await;
}
@@ -355,6 +355,7 @@ async fn disabled_proxy_projects_direct_while_preserving_bindings_and_credential
let error = admin
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
account_ids: vec![
"acct_enabled_openai".to_owned(),
"acct_enabled_xai".to_owned(),
@@ -1180,6 +1181,7 @@ async fn legacy_urls_join_one_catalog_entry_and_invalid_batch_rolls_back() {
admin
.batch_update_accounts(
BatchUpdateAccounts {
auto_use_quota_credits: None,
model_access: Default::default(),
account_ids: vec!["acct_one".to_owned(), "acct_missing".to_owned()],
enabled: Some(false),
@@ -30,7 +30,9 @@ pub(super) fn settings_with_margin(refresh_margin_seconds: u64) -> RuntimeSettin
max_concurrent_per_account: 3,
max_waiting_per_key: 0,
max_waiting_per_account: 0,
concurrency_wait_timeout_seconds: 30,
key_queue_timeout_seconds: 30,
account_queue_timeout_seconds: 30,
switch_account_on_queue_failure: false,
max_retry_after_seconds: 240,
openai_guardian_reserved_concurrency: 0,
openai_account_affinity: gateway_core::account::AccountAffinity::Relaxed,
@@ -568,17 +570,21 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
before.values.max_concurrent_per_account,
before.values.max_waiting_per_key,
before.values.max_waiting_per_account,
before.values.concurrency_wait_timeout_seconds,
before.values.key_queue_timeout_seconds,
before.values.account_queue_timeout_seconds,
before.values.switch_account_on_queue_failure,
before.values.openai_guardian_reserved_concurrency,
),
(3, 10, 10, 30, 5)
(3, 10, 10, 30, 30, false, 5)
);
let mut update = settings_with_margin(1234);
update.values.max_concurrent_per_account = 7;
update.values.refresh_concurrency = 12;
update.values.max_waiting_per_key = 5;
update.values.max_waiting_per_account = 7;
update.values.concurrency_wait_timeout_seconds = 12;
update.values.key_queue_timeout_seconds = 12;
update.values.account_queue_timeout_seconds = 5;
update.values.switch_account_on_queue_failure = true;
update.values.openai_guardian_reserved_concurrency = 1;
repository.update_runtime_settings(update).await.unwrap();
let settings = repository.load_runtime_settings().await.unwrap();
@@ -589,10 +595,12 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
settings.values.refresh_margin_seconds,
settings.values.max_waiting_per_key,
settings.values.max_waiting_per_account,
settings.values.concurrency_wait_timeout_seconds,
settings.values.key_queue_timeout_seconds,
settings.values.account_queue_timeout_seconds,
settings.values.switch_account_on_queue_failure,
settings.values.openai_guardian_reserved_concurrency,
),
(7, 12, 1234, 5, 7, 12, 1)
(7, 12, 1234, 5, 7, 12, 5, true, 1)
);
let snapshot = PgRuntimeSnapshotRepository::new(database.pool.clone())
.load_runtime_snapshot()
@@ -606,14 +614,20 @@ async fn concurrency_queue_settings_round_trip_into_the_runtime_snapshot() {
.unwrap(),
snapshot_values["max_waiting_per_key"].as_u64().unwrap(),
snapshot_values["max_waiting_per_account"].as_u64().unwrap(),
snapshot_values["concurrency_wait_timeout_seconds"]
snapshot_values["key_queue_timeout_seconds"]
.as_u64()
.unwrap(),
snapshot_values["switch_account_on_queue_failure"]
.as_bool()
.unwrap(),
snapshot_values["account_queue_timeout_seconds"]
.as_u64()
.unwrap(),
snapshot_values["openai_guardian_reserved_concurrency"]
.as_u64()
.unwrap(),
),
(7, 5, 7, 12, 1)
(7, 5, 7, 12, true, 5, 1)
);
assert!(snapshot.config_revision > before.config_revision);
database.close().await;
@@ -327,6 +327,158 @@ fn cooldown(account_id: &str, revision: u64, seconds: i64) -> CredentialCooldown
}
}
#[tokio::test]
async fn cooldown_batch_preserves_order_partial_errors_and_due_probe_freezes() {
let Some((repository, mut connection, namespace)) = repository().await else {
return;
};
let mut accounts = (0..131)
.map(|index| ProviderAccountId::new(format!("acct_batch_{index}")).unwrap())
.collect::<Vec<_>>();
accounts.push(accounts[130].clone());
repository
.cache_credential_cooldown(&cooldown(accounts[1].as_str(), 1, 60))
.await
.unwrap();
let bad_type_key = namespace_keys(&mut connection, &namespace)
.await
.into_iter()
.find(|key| key.ends_with(":cooldown"))
.unwrap();
redis::cmd("SET")
.arg(&bad_type_key)
.arg("invalid")
.query_async::<()>(&mut connection)
.await
.unwrap();
repository
.cache_credential_cooldown(&cooldown(accounts[2].as_str(), 2, 60))
.await
.unwrap();
let bad_value_key = namespace_keys(&mut connection, &namespace)
.await
.into_iter()
.find(|key| key.ends_with(":cooldown") && key != &bad_type_key)
.unwrap();
redis::cmd("HSET")
.arg(bad_value_key)
.arg("kind")
.arg("unknown")
.query_async::<i64>(&mut connection)
.await
.unwrap();
repository
.cache_credential_cooldown(&CredentialCooldown {
kind: ProviderCooldownKind::CapacityFreezeProbe,
..cooldown(accounts[3].as_str(), 3, -60)
})
.await
.unwrap();
repository
.cache_credential_cooldown(&cooldown(accounts[130].as_str(), 4, 60))
.await
.unwrap();
assert!(repository.read_many(&[]).await.is_empty());
let results = repository.read_many(&accounts).await;
assert_eq!(results.len(), accounts.len());
assert!(
results[3]
.as_ref()
.unwrap()
.as_ref()
.unwrap()
.kind()
.requires_probe()
);
assert!(results[130].as_ref().unwrap().is_some());
for (index, (account, result)) in accounts.iter().zip(results).enumerate() {
if matches!(index, 1 | 2) {
assert!(result.is_err(), "{index}");
} else {
assert_eq!(
result.unwrap(),
repository.read(account).await.unwrap(),
"{index}"
);
}
}
}
#[tokio::test]
async fn scoped_cooldown_batch_preserves_model_revision_and_partial_errors() {
let Some((repository, mut connection, namespace)) = repository().await else {
return;
};
let mut accounts = (0..130)
.map(|index| ProviderAccountId::new(format!("acct_scoped_batch_{index}")).unwrap())
.collect::<Vec<_>>();
accounts.push(accounts[129].clone());
let scope = ProviderCooldownScope::upstream_model(
gateway_core::routing::UpstreamModelId::new("gpt-test").unwrap(),
);
let other_scope = ProviderCooldownScope::upstream_model(
gateway_core::routing::UpstreamModelId::new("gpt-other").unwrap(),
);
let revision = CredentialRevision::new(3).unwrap();
let until = millisecond_precision(Utc::now() + Duration::seconds(60)).into();
repository
.put_scoped_if_later(ProviderScopedCooldown::new(
accounts[1].clone(),
revision,
scope.clone(),
until,
))
.await
.unwrap();
let bad_key = namespace_keys(&mut connection, &namespace)
.await
.pop()
.unwrap();
redis::cmd("SET")
.arg(bad_key)
.arg("invalid")
.query_async::<()>(&mut connection)
.await
.unwrap();
for (account, model) in [(&accounts[2], &other_scope), (&accounts[129], &scope)] {
repository
.put_scoped_if_later(ProviderScopedCooldown::new(
account.clone(),
revision,
model.clone(),
until,
))
.await
.unwrap();
}
assert!(repository.read_scoped_many(&[], &scope).await.is_empty());
let results = repository.read_scoped_many(&accounts, &scope).await;
assert_eq!(results.len(), accounts.len());
assert!(results[2].as_ref().unwrap().is_none());
assert_eq!(
results[129]
.as_ref()
.unwrap()
.as_ref()
.unwrap()
.credential_revision(),
revision
);
for (index, (account, result)) in accounts.iter().zip(results).enumerate() {
if index == 1 {
assert!(result.is_err());
} else {
assert_eq!(
result.unwrap(),
repository.read_scoped(account, &scope).await.unwrap(),
"{index}"
);
}
}
}
fn millisecond_precision(value: DateTime<Utc>) -> DateTime<Utc> {
DateTime::from_timestamp_millis(value.timestamp_millis()).expect("valid timestamp")
}
@@ -205,6 +205,7 @@ impl ProviderAdmin for OpenAiAdminProvider {
) -> gateway_admin::model::accounts::ProviderAccountCapabilities {
let oauth = authentication_kind == crate::credential::CODEX_AUTHENTICATION_KIND_OAUTH;
gateway_admin::model::accounts::ProviderAccountCapabilities {
quota_credits: oauth,
quota: oauth,
quota_refresh: oauth,
profile: oauth,
@@ -233,11 +234,11 @@ impl ProviderAdmin for OpenAiAdminProvider {
) -> Result<gateway_core::account::OpaqueProviderData, ProviderAdminError> {
if configuration
.expose_to_provider()
.contains_key("installationId")
.contains_key("installationSeed")
|| gateway_core::settings::request_profiles::inherited_fields(configuration).is_some()
{
return Err(provider_admin_error(ProviderAdminErrorKind::Invalid)
.with_public_message("安装 ID 只能在 API 密钥中设置"));
.with_public_message("安装身份种子只能在 API 密钥中设置"));
}
self.profile
.preview_selection(configuration)
@@ -1151,6 +1152,7 @@ fn account_from_record(account: &AccountRecord) -> Result<ProviderAccount, Provi
revision,
account.access_token_expires_at.map(SystemTime::from),
)
.with_auto_use_quota_credits(account.auto_use_quota_credits)
.with_outbound_proxy(account.outbound_proxy.clone())
.with_notes(account.notes.clone())
.with_profile(
@@ -2123,7 +2125,7 @@ fn map_client_profile_error(
use crate::transport::profile::selection::ClientProfileError;
let message = match error {
ClientProfileError::Invalid => "客户端身份字段或版本组合不合法",
ClientProfileError::InvalidInstallationId => "安装 ID 必须是带连字符的 UUID",
ClientProfileError::InvalidInstallationSeed => "安装身份种子必须是带连字符的 UUID",
ClientProfileError::InvalidUserAgent => {
"User-Agent 必须是 1 至 4096 字节的单行 ASCII 文本,且首尾不能含空白"
}
@@ -5,6 +5,7 @@ use serde_json::{Map, Value, json};
/// 非官方客户端及跨客户端历史回填的兼容入口;调用方必须已选定 Codex/OAuth 上游
/// 只处理已验证被拒绝的字段,不根据客户端品牌推断整份历史是否合法
pub(super) fn normalize_non_codex_request_body(body: &mut Map<String, Value>) {
normalize_codex_oauth_options(body);
let Some(input) = body.get_mut("input").and_then(Value::as_array_mut) else {
return;
};
@@ -33,6 +34,55 @@ pub(super) fn normalize_non_codex_request_body(body: &mut Map<String, Value>) {
}
}
fn normalize_codex_oauth_options(body: &mut Map<String, Value>) {
// Codex OAuth 拒绝这些客户端选项,即使采样参数取默认值
for field in [
"top_p",
"frequency_penalty",
"presence_penalty",
"chat_template_kwargs",
"user",
"metadata",
"safety_identifier",
] {
body.shift_remove(field);
}
// Responses 自带用量;保留官方流式推理摘要等其他选项
if let Some(options) = body
.get_mut("stream_options")
.and_then(Value::as_object_mut)
{
options.shift_remove("include_usage");
}
// 只省略没有实际效果的配置,不静默取消客户端要求的截断、停止条件或缓存策略
if body.get("truncation").and_then(Value::as_str) == Some("disabled") {
body.shift_remove("truncation");
}
if body
.get("stop_sequences")
.and_then(Value::as_array)
.is_some_and(Vec::is_empty)
{
body.shift_remove("stop_sequences");
}
if body
.get("prompt_cache_options")
.and_then(Value::as_object)
.is_some_and(Map::is_empty)
{
body.shift_remove("prompt_cache_options");
}
// 原生上限优先,Chat 别名只补缺,非法值仍交给上游校验
for alias in ["max_completion_tokens", "max_tokens"] {
if let Some(limit) = body.shift_remove(alias)
&& !limit.is_null()
&& body.get("max_output_tokens").is_none_or(Value::is_null)
{
body.insert("max_output_tokens".to_owned(), limit);
}
}
}
/// 补齐 Codex 请求缺省字段并适配已确认不兼容的请求形状,不递归清洗业务正文
///
/// 兼容基准是 Codex Core/Desktop 的模型请求,不是公开 OpenAI Responses API
@@ -8,8 +8,10 @@ use sha2::{Digest, Sha256};
use crate::request_identity::{account_session_with_headers, follows_session_with_headers};
use crate::transport::protocol::responses::CodexResponsesRequest;
use crate::transport::request::derive_conversation_anchor;
use crate::transport::request::{derive_conversation_anchor, message_text_parts};
const CLIENT_CACHE_PREFERENCE: &str = "client-cache-preference";
const CONTENT_PREFERENCE: &str = "content-preference";
const AFFINITY_KEY_HASH_LENGTH: usize = 12;
/// 一次请求派生出的账号亲和键及其结构化日志上下文
@@ -18,7 +20,6 @@ pub(crate) struct CodexSessionAffinity {
turn_alias: Option<(ProviderSessionAffinityKey, ProviderSessionAlias)>,
key_hash: String,
anchor_source: &'static str,
anchor: String,
session_id: Option<String>,
follow_only: bool,
}
@@ -35,7 +36,6 @@ impl CodexSessionAffinity {
key_hash: short_key_hash(&key),
key,
anchor_source: "turn-session",
anchor: String::new(),
session_id: None,
follow_only: root.is_some() || alias.follow_only,
turn_alias: Some((turn, alias)),
@@ -88,9 +88,20 @@ impl CodexSessionAffinity {
self.anchor_source
}
#[must_use]
pub(crate) fn is_cache_preference(&self) -> bool {
matches!(
self.anchor_source,
CLIENT_CACHE_PREFERENCE | CONTENT_PREFERENCE
)
}
#[must_use]
pub(crate) fn anchor(&self) -> &str {
&self.anchor
if self.is_cache_preference() {
return self.key_hash();
}
self.session_id.as_deref().unwrap_or_default()
}
#[must_use]
@@ -130,13 +141,60 @@ pub(crate) fn derive_codex_session_affinity(
let session_id = non_empty(request.client_account_session_id.as_deref())?;
session_affinity(
"root-session",
session_id.to_owned(),
Some(session_id.to_owned()),
session_id,
Some(session_id),
client_api_key_id,
)
.map(|affinity| affinity.with_follow_only(request.client_account_follow_only))
}
/// 仅供选号使用。缓存键与输入正文均不作为会话、轮次或传输身份。
pub(crate) fn derive_cache_preference(
request: &CodexResponsesRequest,
client_key: &ClientApiKeyId,
) -> Option<CodexSessionAffinity> {
if request.client_account_follow_only || request.previous_response_id().is_some() {
return None;
}
if request.explicit_prompt_cache_key
&& let Some(key) = non_empty(request.prompt_cache_key())
{
return session_affinity(CLIENT_CACHE_PREFERENCE, key, None, client_key);
}
let mut hash = Sha256::new();
hash_text_field(&mut hash, std::iter::once(request.model()));
hash_text_field(&mut hash, std::iter::once(request.instructions()));
let mut items = request.input().iter().peekable();
while let Some(item) =
items.next_if(|item| matches!(item["role"].as_str(), Some("system" | "developer")))
{
hash_text_field(&mut hash, message_text_parts(item));
}
let first_user = items.next().filter(|item| item["role"] == "user")?;
if !hash_text_field(&mut hash, message_text_parts(first_user)) {
return None;
}
session_affinity(
CONTENT_PREFERENCE,
&hex::encode(hash.finalize()),
None,
client_key,
)
}
/// 长度前缀区分消息边界;同一消息的文本块按顺序流式计算,避免复制长提示词。
fn hash_text_field<'a>(hash: &mut Sha256, parts: impl Iterator<Item = &'a str> + Clone) -> bool {
if !parts.clone().any(|part| !part.trim().is_empty()) {
return false;
}
let length: usize = parts.clone().map(str::len).sum();
hash.update((length as u64).to_be_bytes());
for part in parts {
hash.update(part.as_bytes());
}
true
}
/// 原生 JSON 端点与 Responses 共用绑定命名空间,Search id 是逻辑会话身份
pub(crate) fn derive_codex_endpoint_session_affinity(
payload: &RawJsonPayload,
@@ -159,8 +217,8 @@ pub(crate) fn derive_endpoint_affinity_with_headers(
let session_id = account_session_with_headers(&body, payload.context(), headers)?;
session_affinity(
"root-session",
session_id.clone(),
Some(session_id),
&session_id,
Some(&session_id),
client_api_key_id,
)
.map(|affinity| {
@@ -195,8 +253,8 @@ pub(crate) fn derive_live_session_affinity(
let session_id = account_session_with_headers(&Map::new(), &Map::new(), &effective)?;
session_affinity(
"root-session",
session_id.clone(),
Some(session_id),
&session_id,
Some(&session_id),
client_api_key_id,
)
.map(|affinity| {
@@ -216,16 +274,16 @@ pub(crate) fn derive_codex_transport_key(
let (source, anchor) = non_empty(request.client_thread_id.as_deref())
.map(|thread| ("thread-transport", thread.to_owned()))
.or_else(|| derive_conversation_anchor(request))?;
session_affinity(source, anchor, None, client_api_key_id).map(CodexSessionAffinity::into_key)
session_affinity(source, &anchor, None, client_api_key_id).map(CodexSessionAffinity::into_key)
}
fn session_affinity(
anchor_source: &'static str,
anchor: String,
session_id: Option<String>,
anchor: &str,
session_id: Option<&str>,
client_api_key_id: &ClientApiKeyId,
) -> Option<CodexSessionAffinity> {
let session_key = opaque_affinity_key(anchor_source, &anchor)?;
let session_key = opaque_affinity_key(anchor_source, anchor)?;
let key = opaque_affinity_key(
"client-session",
&format!(
@@ -240,8 +298,7 @@ fn session_affinity(
turn_alias: None,
key_hash,
anchor_source,
anchor,
session_id,
session_id: session_id.map(str::to_owned),
follow_only: false,
})
}
@@ -10,6 +10,7 @@ use futures::{StreamExt, stream};
use gateway_core::account::{
CredentialRevision, CredentialState, OpaqueProviderData, ProviderAccount, ProviderAccountId,
};
use gateway_core::policy::ClientApiKeyId;
use gateway_core::provider_ports::{
ProviderCatalogCacheKey, ProviderCatalogCachePort, ProviderCatalogScope,
};
@@ -206,6 +207,7 @@ struct CatalogCacheState {
#[derive(PartialEq, Eq)]
struct ClientCatalogKey {
client_key_id: Option<ClientApiKeyId>,
account_id: ProviderAccountId,
revision: CredentialRevision,
plan: Option<String>,
@@ -311,6 +313,7 @@ impl CodexCredentialCatalogService {
/// 同一账号/凭据版本/客户端版本的并发读取合并;原生正文仅保留在有界进程缓存
pub async fn client_model_catalog(
&self,
client_key_id: &ClientApiKeyId,
scope: &FrozenAccountScope,
client_version: &str,
) -> ClientCatalogResult {
@@ -339,7 +342,12 @@ impl CodexCredentialCatalogService {
let profile = request_profile.clone();
async move {
let result = self
.cached_client_account_catalog(&account, client_version, profile)
.cached_client_account_catalog(
client_key_id,
&account,
client_version,
profile,
)
.await;
(account, result)
}
@@ -374,7 +382,12 @@ impl CodexCredentialCatalogService {
let mut last_error = CodexCredentialCatalogError::NoEligibleCredential;
for account in accounts.iter().take(MAX_CATALOG_FETCH_ATTEMPTS) {
match self
.cached_client_account_catalog(account, client_version, request_profile.clone())
.cached_client_account_catalog(
client_key_id,
account,
client_version,
request_profile.clone(),
)
.await
{
Ok(models) => {
@@ -403,11 +416,17 @@ impl CodexCredentialCatalogService {
async fn cached_client_account_catalog(
&self,
client_key_id: &ClientApiKeyId,
account: &ProviderAccount,
client_version: &str,
profile: CodexWireProfile,
) -> ClientCatalogResult {
let key = ClientCatalogKey {
// 无种子时沿用账号级目录缓存,启用后按 Key 隔离出站身份
client_key_id: profile
.installation_seed
.as_ref()
.map(|_| client_key_id.clone()),
account_id: account.id().clone(),
revision: account.revision(),
plan: account.plan_type().map(str::to_owned),
@@ -450,7 +469,7 @@ impl CodexCredentialCatalogService {
CodexWireProfileState::new(profile),
);
let result = self
.fetch_account_models(&client, account, Some(client_version))
.fetch_account_models(&client, account, Some((client_key_id, client_version)))
.await
.map(|fetched| {
fetched
@@ -805,7 +824,7 @@ impl CodexCredentialCatalogService {
&self,
client: &CodexBackendClient,
account: &ProviderAccount,
client_version: Option<&str>,
client_identity: Option<(&ClientApiKeyId, &str)>,
) -> Result<FetchedAccountModels, CodexCredentialCatalogError> {
tokio::time::timeout(MODEL_CATALOG_TIMEOUT, async {
let credential = self
@@ -813,6 +832,14 @@ impl CodexCredentialCatalogService {
.load_runtime_credential(account)
.await
.map_err(|_| CodexCredentialCatalogError::InvalidCredentialData)?;
let installation_id = client_identity.map(|(key_id, _)| {
client.profile_state().effective_installation_id(
key_id,
account.id(),
&credential.installation_id,
)
});
let client_version = client_identity.map(|(_, version)| version);
let request_id = format!("catalog_{}", Uuid::now_v7().simple());
let authorization = credential
.authentication
@@ -829,7 +856,7 @@ impl CodexCredentialCatalogService {
authorization.expose_secret(),
account.upstream_account_id(),
&request_id,
client_version.map(|_| credential.installation_id.as_str()),
installation_id.as_deref(),
),
client_version,
)
@@ -13,12 +13,13 @@ mod quota;
mod recovery_log;
mod refresh;
mod repository;
mod scheduling_projection;
mod selector;
pub mod token_client;
mod types;
pub(crate) use affinity::{
CodexSessionAffinity, derive_codex_cyber_policy_session_key,
CodexSessionAffinity, derive_cache_preference, derive_codex_cyber_policy_session_key,
derive_codex_endpoint_session_affinity, derive_codex_session_affinity,
derive_codex_transport_key, derive_endpoint_affinity_with_headers,
derive_live_session_affinity, derive_previous_response_id_hash, derive_turn_alias,
@@ -57,27 +57,11 @@ impl CodexCredentialQuotaService {
})
.next();
let plan_type = observed_account_plan(account.plan_type(), observed_plan);
// 套餐、credits 等元数据可以更新,但没有额度窗口事实时必须保留旧观察时刻,
// 也不能借旧快照重新推导 quota state
if !has_quota_facts {
let Some(state) = existing_state else {
return Ok(false);
};
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
expected_connection: None,
plan_type,
account_id: account.id().clone(),
expected_revision: account.revision(),
quota: OpaqueProviderData::new(merge_passive_quota(existing, rate_limits)),
observed_at: SystemTime::now(),
state,
})
.await?;
return Ok(outcome != QuotaWriteOutcome::Conflict);
if !has_quota_facts && existing_state.is_none() {
return Ok(false);
}
let quota = merge_passive_quota(existing, rate_limits);
let prior = existing_state.unwrap_or_else(|| account.quota());
let (quota, updated_default) = merge_passive_quota(existing, rate_limits);
let observed_at = SystemTime::now();
let snapshot = parse_account_quota_snapshot(
account.id().clone(),
@@ -85,8 +69,33 @@ impl CodexCredentialQuotaService {
observed_at,
&Value::Object(quota.clone()),
)?;
// 这些 headers 来自一次成功推理,访问结论优先于可能滞后的百分比
let state = QuotaState::allowed(observed_at);
let credit_access = prior.evidence() == Some(QuotaEvidence::CreditsAvailable);
let fresh_credits = rate_limits.iter().any(|limits| limits.credits.is_some());
// 具名桶和元数据不改变套餐结论;点数不能再用时,仍须撤销原先仅凭点数的放行
let subscription = if !updated_default
|| (snapshot.quota().access() == QuotaAccessState::Unknown
&& (credit_access || prior.is_exhausted()))
{
if credit_access {
QuotaState::exhausted(
QuotaEvidence::AccountLimitReached,
observed_at,
prior.reset_at(),
)
} else {
prior
}
} else if snapshot.quota().is_exhausted() {
snapshot.quota()
} else {
QuotaState::allowed(observed_at)
};
// 套餐刚耗尽时不能拿合并快照里的旧余额作为首次放行依据
let state = snapshot.access_with_credits(
subscription,
account.auto_use_quota_credits()
&& (credit_access || (updated_default && fresh_credits)),
);
let outcome = self
.store
.compare_and_swap_quota(QuotaObservation {
@@ -102,7 +111,9 @@ impl CodexCredentialQuotaService {
if outcome == QuotaWriteOutcome::Conflict {
return Ok(false);
}
self.scheduling.observe(&snapshot);
if has_quota_facts {
self.scheduling.observe(&snapshot);
}
Ok(true)
}
}
@@ -110,7 +121,8 @@ impl CodexCredentialQuotaService {
fn merge_passive_quota(
mut quota: Map<String, Value>,
observations: &[ParsedRateLimits],
) -> Map<String, Value> {
) -> (Map<String, Value>, bool) {
let mut updated_default = false;
let mut snapshots = RateLimitSnapshotsByLimitId::take_from_document(&mut quota);
for rate_limits in observations {
let default_is_named_alias = default_limit_is_named_alias(rate_limits);
@@ -127,6 +139,7 @@ fn merge_passive_quota(
let Some(rate_limit) = passive_rate_limit_snapshot(details) else {
continue;
};
updated_default |= limit_id == DEFAULT_CODEX_LIMIT_ID;
snapshots.upsert(&limit_id, details, rate_limit);
resolved_limit_ids.insert(wire_limit_id.as_str(), limit_id);
}
@@ -147,7 +160,7 @@ fn merge_passive_quota(
merge_passive_metadata(&mut quota, rate_limits);
}
snapshots.write_to_document(&mut quota);
quota
(quota, updated_default)
}
fn merge_passive_metadata(quota: &mut Map<String, Value>, rate_limits: &ParsedRateLimits) {

Some files were not shown because too many files have changed in this diff Show More